fix: namespace cache delete actions (#5887)

* namespace cache operations on cluster deletion

* wip

* fixing nil value of k8s service

* k8s informer factory nil handling

* common function for cluster secret

Author: iamayushm

cmd/external-app/wire_gen.go

@@ -56,7 +56,6 @@ const (
 	CLUSTER_MODIFY_EVENT_SECRET_TYPE = "cluster.request/modify"
 	CLUSTER_ACTION_ADD               = "add"
 	CLUSTER_ACTION_UPDATE            = "update"
-	SECRET_NAME                      = "cluster-event"
 	SECRET_FIELD_CLUSTER_ID          = "cluster_id"
 	SECRET_FIELD_UPDATED_ON          = "updated_on"
 	SECRET_FIELD_ACTION              = "action"
@@ -167,7 +166,7 @@ type ClusterService interface {
 	FindAllExceptVirtual() ([]*ClusterBean, error)
 	FindAllWithoutConfig() ([]*ClusterBean, error)
 	FindAllActive() ([]ClusterBean, error)
-	DeleteFromDb(bean *ClusterBean, userId int32) error
+	DeleteFromDb(bean *ClusterBean, userId int32) (string, error)
 
 	FindById(id int) (*ClusterBean, error)
 	FindByIdWithoutConfig(id int) (*ClusterBean, error)
@@ -331,7 +330,7 @@ func (impl *ClusterServiceImpl) Save(parent context.Context, bean *ClusterBean,
 		return bean, nil
 	}
 	//creating cluster secret, this secret will be read informer in kubelink to know that a new cluster has been added
-	secretName := fmt.Sprintf("%s-%v", SECRET_NAME, bean.Id)
+	secretName := ParseSecretNameForKubelinkInformer(bean.Id)
 
 	data := make(map[string][]byte)
 	data[SECRET_FIELD_CLUSTER_ID] = []byte(fmt.Sprintf("%v", bean.Id))
@@ -572,7 +571,7 @@ func (impl *ClusterServiceImpl) Update(ctx context.Context, bean *ClusterBean, u
 	}
 	// below secret will act as an event for informer running on secret object in kubelink
 	if bean.HasConfigOrUrlChanged {
-		secretName := fmt.Sprintf("%s-%v", SECRET_NAME, bean.Id)
+		secretName := ParseSecretNameForKubelinkInformer(bean.Id)
 		secret, err := impl.K8sUtil.GetSecret(DEFAULT_NAMESPACE, secretName, k8sClient)
 		statusError, _ := err.(*errors.StatusError)
 		if err != nil && statusError.Status().Code != http.StatusNotFound {
@@ -675,29 +674,29 @@ func (impl *ClusterServiceImpl) buildInformer() {
 	impl.K8sInformerFactory.BuildInformer(clusterInfo)
 }
 
-func (impl *ClusterServiceImpl) DeleteFromDb(bean *ClusterBean, userId int32) error {
+func (impl *ClusterServiceImpl) DeleteFromDb(bean *ClusterBean, userId int32) (string, error) {
 	existingCluster, err := impl.clusterRepository.FindById(bean.Id)
 	if err != nil {
 		impl.logger.Errorw("No matching entry found for delete.", "id", bean.Id)
-		return err
+		return "", err
 	}
 	deleteReq := existingCluster
 	deleteReq.UpdatedOn = time.Now()
 	deleteReq.UpdatedBy = userId
 	err = impl.clusterRepository.MarkClusterDeleted(deleteReq)
 	if err != nil {
 		impl.logger.Errorw("error in deleting cluster", "id", bean.Id, "err", err)
-		return err
+		return "", err
 	}
 	k8sClient, err := impl.K8sUtil.GetCoreV1ClientInCluster()
 	if err != nil {
 		impl.logger.Errorw("error in getting in cluster k8s client", "err", err, "clusterName", bean.ClusterName)
-		return nil
+		return "", nil
 	}
-	secretName := fmt.Sprintf("%s-%v", SECRET_NAME, bean.Id)
+	secretName := ParseSecretNameForKubelinkInformer(bean.Id)
 	err = impl.K8sUtil.DeleteSecret(DEFAULT_NAMESPACE, secretName, k8sClient)
 	impl.logger.Errorw("error in deleting secret", "error", err)
-	return nil
+	return existingCluster.ClusterName, nil
 }
 
 func (impl *ClusterServiceImpl) CheckIfConfigIsValid(cluster *ClusterBean) error {

pkg/cluster/ClusterService.go

@@ -381,26 +381,19 @@ func (impl *ClusterServiceImplExtended) Save(ctx context.Context, bean *ClusterB
 	return clusterBean, nil
 }
 
-func (impl ClusterServiceImplExtended) DeleteFromDb(bean *ClusterBean, userId int32) error {
+func (impl ClusterServiceImplExtended) DeleteFromDb(bean *ClusterBean, userId int32) (string, error) {
 	existingCluster, err := impl.clusterRepository.FindById(bean.Id)
 	if err != nil {
 		impl.logger.Errorw("No matching entry found for delete.", "id", bean.Id)
-		return err
+		return "", err
 	}
 	deleteReq := existingCluster
 	deleteReq.UpdatedOn = time.Now()
 	deleteReq.UpdatedBy = userId
 	err = impl.clusterRepository.MarkClusterDeleted(deleteReq)
 	if err != nil {
 		impl.logger.Errorw("error in deleting cluster", "id", bean.Id, "err", err)
-		return err
+		return "", err
 	}
-	k8sClient, err := impl.ClusterServiceImpl.K8sUtil.GetCoreV1ClientInCluster()
-	if err != nil {
-		impl.logger.Errorw("error in creating k8s client set", "err", err, "clusterName", bean.ClusterName)
-	}
-	secretName := fmt.Sprintf("%s-%v", "cluster-event", bean.Id)
-	err = impl.K8sUtil.DeleteSecret("default", secretName, k8sClient)
-	impl.logger.Errorw("error in deleting secret", "error", err)
-	return nil
+	return existingCluster.ClusterName, nil
 }

pkg/cluster/ClusterServiceExtended.go

@@ -0,0 +1,11 @@
+package cluster
+
+import "fmt"
+
+const (
+	SECRET_NAME = "cluster-event"
+)
+
+func ParseSecretNameForKubelinkInformer(clusterId int) string {
+	return fmt.Sprintf("%s-%d", SECRET_NAME, clusterId)
+}

pkg/cluster/clusterUtil.go

@@ -18,11 +18,13 @@ package delete
 
 import (
 	"fmt"
+	"github.com/devtron-labs/common-lib/utils/k8s"
 	dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry"
 	"github.com/devtron-labs/devtron/pkg/appStore/installedApp/repository"
 	"github.com/devtron-labs/devtron/pkg/chartRepo"
 	"github.com/devtron-labs/devtron/pkg/cluster"
 	"github.com/devtron-labs/devtron/pkg/cluster/repository/bean"
+	"github.com/devtron-labs/devtron/pkg/k8s/informer"
 	"github.com/devtron-labs/devtron/pkg/pipeline"
 	"github.com/devtron-labs/devtron/pkg/pipeline/types"
 	"github.com/devtron-labs/devtron/pkg/team"
@@ -37,6 +39,7 @@ type DeleteService interface {
 	DeleteChartRepo(deleteRequest *chartRepo.ChartRepoDto) error
 	DeleteDockerRegistryConfig(deleteRequest *types.DockerArtifactStoreBean) error
 	CanDeleteChartRegistryPullConfig(storeId string) bool
+	DeleteClusterSecret(deleteRequest *cluster.ClusterBean, err error) error
 }
 
 type DeleteServiceImpl struct {
@@ -48,6 +51,8 @@ type DeleteServiceImpl struct {
 	installedAppRepository   repository.InstalledAppRepository
 	dockerRegistryConfig     pipeline.DockerRegistryConfig
 	dockerRegistryRepository dockerRegistryRepository.DockerArtifactStoreRepository
+	K8sUtil                  k8s.K8sService
+	k8sInformerFactory       informer.K8sInformerFactory
 }
 
 func NewDeleteServiceImpl(logger *zap.SugaredLogger,
@@ -58,6 +63,8 @@ func NewDeleteServiceImpl(logger *zap.SugaredLogger,
 	installedAppRepository repository.InstalledAppRepository,
 	dockerRegistryConfig pipeline.DockerRegistryConfig,
 	dockerRegistryRepository dockerRegistryRepository.DockerArtifactStoreRepository,
+	k8sInformerFactory informer.K8sInformerFactory,
+	K8sUtil k8s.K8sService,
 ) *DeleteServiceImpl {
 	return &DeleteServiceImpl{
 		logger:                   logger,
@@ -68,18 +75,38 @@ func NewDeleteServiceImpl(logger *zap.SugaredLogger,
 		installedAppRepository:   installedAppRepository,
 		dockerRegistryConfig:     dockerRegistryConfig,
 		dockerRegistryRepository: dockerRegistryRepository,
+		K8sUtil:                  K8sUtil,
+		k8sInformerFactory:       k8sInformerFactory,
 	}
 }
 
 func (impl DeleteServiceImpl) DeleteCluster(deleteRequest *cluster.ClusterBean, userId int32) error {
-	err := impl.clusterService.DeleteFromDb(deleteRequest, userId)
+	clusterName, err := impl.clusterService.DeleteFromDb(deleteRequest, userId)
 	if err != nil {
 		impl.logger.Errorw("error im deleting cluster", "err", err, "deleteRequest", deleteRequest)
 		return err
 	}
+	err = impl.DeleteClusterSecret(deleteRequest, err)
+	if err != nil {
+		impl.logger.Errorw("error in deleting cluster secret", "clusterId", deleteRequest.Id, "error", err)
+		return err
+	}
+	impl.k8sInformerFactory.DeleteClusterFromCache(clusterName)
 	return nil
 }
 
+func (impl DeleteServiceImpl) DeleteClusterSecret(deleteRequest *cluster.ClusterBean, err error) error {
+	// kubelink informers are listening this secret, deleting this secret will inform kubelink that this cluster is deleted
+	k8sClient, err := impl.K8sUtil.GetCoreV1ClientInCluster()
+	if err != nil {
+		impl.logger.Errorw("error in getting in cluster k8s client", "err", err, "clusterName", deleteRequest.ClusterName)
+		return nil
+	}
+	secretName := cluster.ParseSecretNameForKubelinkInformer(deleteRequest.Id)
+	err = impl.K8sUtil.DeleteSecret(cluster.DEFAULT_NAMESPACE, secretName, k8sClient)
+	return err
+}
+
 func (impl DeleteServiceImpl) DeleteEnvironment(deleteRequest *bean.EnvironmentBean, userId int32) error {
 	err := impl.environmentService.Delete(deleteRequest, userId)
 	if err != nil {

pkg/delete/DeleteService.go

@@ -18,6 +18,7 @@ package delete
 
 import (
 	"fmt"
+	k8sUtil "github.com/devtron-labs/common-lib/utils/k8s"
 	"github.com/devtron-labs/devtron/internal/sql/repository/app"
 	dockerRegistryRepository "github.com/devtron-labs/devtron/internal/sql/repository/dockerRegistry"
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
@@ -28,6 +29,7 @@ import (
 	"github.com/devtron-labs/devtron/pkg/cluster"
 	"github.com/devtron-labs/devtron/pkg/cluster/repository"
 	"github.com/devtron-labs/devtron/pkg/cluster/repository/bean"
+	"github.com/devtron-labs/devtron/pkg/k8s/informer"
 	"github.com/devtron-labs/devtron/pkg/pipeline"
 	"github.com/devtron-labs/devtron/pkg/team"
 	"github.com/go-pg/pg"
@@ -53,6 +55,8 @@ func NewDeleteServiceExtendedImpl(logger *zap.SugaredLogger,
 	installedAppRepository repository2.InstalledAppRepository,
 	dockerRegistryConfig pipeline.DockerRegistryConfig,
 	dockerRegistryRepository dockerRegistryRepository.DockerArtifactStoreRepository,
+	K8sService k8sUtil.K8sService,
+	factory informer.K8sInformerFactory,
 ) *DeleteServiceExtendedImpl {
 	return &DeleteServiceExtendedImpl{
 		appRepository:         appRepository,
@@ -67,6 +71,8 @@ func NewDeleteServiceExtendedImpl(logger *zap.SugaredLogger,
 			installedAppRepository:   installedAppRepository,
 			dockerRegistryConfig:     dockerRegistryConfig,
 			dockerRegistryRepository: dockerRegistryRepository,
+			K8sUtil:                  K8sService,
+			k8sInformerFactory:       factory,
 		},
 	}
 }
@@ -82,11 +88,17 @@ func (impl DeleteServiceExtendedImpl) DeleteCluster(deleteRequest *cluster.Clust
 		impl.logger.Errorw("err in deleting cluster, found env in this cluster", "clusterName", deleteRequest.ClusterName, "err", err)
 		return &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: " Please delete all related environments before deleting this cluster"}
 	}
-	err = impl.clusterService.DeleteFromDb(deleteRequest, userId)
+	clusterName, err := impl.clusterService.DeleteFromDb(deleteRequest, userId)
 	if err != nil {
 		impl.logger.Errorw("error im deleting cluster", "err", err, "deleteRequest", deleteRequest)
 		return err
 	}
+	err = impl.DeleteClusterSecret(deleteRequest, err)
+	if err != nil {
+		impl.logger.Errorw("error in deleting cluster secret", "clusterId", deleteRequest.Id, "error", err)
+		return err
+	}
+	impl.k8sInformerFactory.DeleteClusterFromCache(clusterName)
 	return nil
 }
 

pkg/delete/DeleteServiceExtended.go

@@ -43,6 +43,7 @@ type K8sInformerFactory interface {
 	GetLatestNamespaceListGroupByCLuster() map[string]map[string]bool
 	BuildInformer(clusterInfo []*bean.ClusterInfo)
 	CleanNamespaceInformer(clusterName string)
+	DeleteClusterFromCache(clusterName string)
 }
 
 func NewK8sInformerFactoryImpl(logger *zap.SugaredLogger, globalMapClusterNamespace sync.Map, k8sUtil *k8s.K8sServiceImpl) *K8sInformerFactoryImpl {
@@ -127,3 +128,9 @@ func (impl *K8sInformerFactoryImpl) CleanNamespaceInformer(clusterName string) {
 	}
 	return
 }
+
+func (impl *K8sInformerFactoryImpl) DeleteClusterFromCache(clusterName string) {
+	impl.CleanNamespaceInformer(clusterName)
+	impl.globalMapClusterNamespace.Delete(clusterName)
+	return
+}