Merge pull request openshift#1927 from 2uasimojo/HIVE-2064/pause-cd

Pause ClusterDeployment reconciliation

Author: openshift-merge-robot

pkg/constants/constants.go

@@ -134,6 +134,24 @@ const (
 	// the cluster, causing its machines to remain in their current state (unless acted on externally) regardless of CD.Spec.PowerState.
 	PowerStatePauseAnnotation = "hive.openshift.io/powerstate-pause"
 
+	// ReconcilePauseAnnotation is an annotation used by ClusterDeployment. If "true", (most) controllers will (mostly) ignore the CD until
+	// the annotation is cleared/non-truthy. Exceptions:
+	// - clusterclaim: We'll still mark the CD for deletion by the clusterpool controller when the claim is deleted.
+	// - clusterpool:
+	//   - We'll still delete CDs* for various reasons, including
+	//     - Associated claim is deleted (per above)
+	//     - To satisfy pool capacity constraints
+	//     - To flush out broken or stale clusters
+	//   - We'll still update CD PowerState when it's time to hibernate or resume a pool cluster. But the hibernation controller won't
+	//   effect that change while the annotation is still set.
+	//   - We'll still create new ClusterDeployments to satisfy pool capacities. If you include this annotation in
+	//   ClusterPool.Spec.Annotations, those ClusterDeployments will be created with it set, so we won't do anything (like provision the
+	//   cluster) until it's unset.
+	// - clusterpoolnamespace: We'll still delete CDs*.
+	// *Note that this only entails setting the deletionTimestamp; the other controllers won't do anything about it (like deprovisioning
+	// the cluster) while the annotation is still set.
+	ReconcilePauseAnnotation = "hive.openshift.io/reconcile-pause"
+
 	// HiveManagedLabel is a label added to any resources we sync to the remote cluster to help identify that they are
 	// managed by Hive, and any manual changes may be undone the next time the resource is reconciled.
 	HiveManagedLabel = "hive.openshift.io/managed"

pkg/controller/argocdregister/argocdregister_controller.go

@@ -26,6 +26,7 @@ import (
 	"net/url"
 	"os"
 	"reflect"
+	"strconv"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
@@ -153,6 +154,11 @@ func (r *ArgoCDRegisterController) Reconcile(ctx context.Context, request reconc
 	}
 	cdLog = controllerutils.AddLogFields(controllerutils.MetaObjectLogTagger{Object: cd}, cdLog)
 
+	if paused, err := strconv.ParseBool(cd.Annotations[constants.ReconcilePauseAnnotation]); err == nil && paused {
+		cdLog.Info("skipping reconcile due to ClusterDeployment pause annotation")
+		return reconcile.Result{}, nil
+	}
+
 	if !cd.Spec.Installed {
 		cdLog.Info("cluster installation is not complete")
 		return reconcile.Result{}, nil

pkg/controller/awsprivatelink/awsprivatelink_controller.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -173,6 +174,11 @@ func (r *ReconcileAWSPrivateLink) Reconcile(ctx context.Context, request reconci
 	}
 	logger = controllerutils.AddLogFields(controllerutils.MetaObjectLogTagger{Object: cd}, logger)
 
+	if paused, err := strconv.ParseBool(cd.Annotations[constants.ReconcilePauseAnnotation]); err == nil && paused {
+		logger.Info("skipping reconcile due to ClusterDeployment pause annotation")
+		return reconcile.Result{}, nil
+	}
+
 	// Initialize cluster deployment conditions if not present
 	newConditions, changed := controllerutils.InitializeClusterDeploymentConditions(cd.Status.Conditions, clusterDeploymentAWSPrivateLinkConditions)
 	if changed {

pkg/controller/clusterdeployment/clusterdeployment_controller.go

@@ -299,6 +299,11 @@ func (r *ReconcileClusterDeployment) Reconcile(ctx context.Context, request reco
 	}
 	cdLog = controllerutils.AddLogFields(controllerutils.MetaObjectLogTagger{Object: cd}, cdLog)
 
+	if paused, err := strconv.ParseBool(cd.Annotations[constants.ReconcilePauseAnnotation]); err == nil && paused {
+		cdLog.Info("skipping reconcile due to ClusterDeployment pause annotation")
+		return reconcile.Result{}, nil
+	}
+
 	// Ensure owner references are correctly set
 	err = controllerutils.ReconcileOwnerReferences(cd, generateOwnershipUniqueKeys(cd), r, r.scheme, r.logger)
 	if err != nil {

pkg/controller/clusterrelocate/clusterrelocate_controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -27,6 +28,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	hivev1 "github.com/openshift/hive/apis/hive/v1"
+	"github.com/openshift/hive/pkg/constants"
 	hivemetrics "github.com/openshift/hive/pkg/controller/metrics"
 	controllerutils "github.com/openshift/hive/pkg/controller/utils"
 	"github.com/openshift/hive/pkg/remoteclient"
@@ -174,6 +176,11 @@ func (r *ReconcileClusterRelocate) Reconcile(ctx context.Context, request reconc
 	}
 	logger = controllerutils.AddLogFields(controllerutils.MetaObjectLogTagger{Object: cd}, logger)
 
+	if paused, err := strconv.ParseBool(cd.Annotations[constants.ReconcilePauseAnnotation]); err == nil && paused {
+		logger.Info("skipping reconcile due to ClusterDeployment pause annotation")
+		return reconcile.Result{}, nil
+	}
+
 	// Initialize cluster deployment conditions if not present
 	newConditions, changed := controllerutils.InitializeClusterDeploymentConditions(cd.Status.Conditions,
 		clusterDeploymentClusterRelocateConditions)

pkg/controller/clusterstate/clusterstate_controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strconv"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -121,6 +122,10 @@ func (r *ReconcileClusterState) Reconcile(ctx context.Context, request reconcile
 		return reconcile.Result{}, err
 	}
 	logger = controllerutils.AddLogFields(controllerutils.MetaObjectLogTagger{Object: cd}, logger)
+	if paused, err := strconv.ParseBool(cd.Annotations[constants.ReconcilePauseAnnotation]); err == nil && paused {
+		logger.Info("skipping reconcile due to ClusterDeployment pause annotation")
+		return reconcile.Result{}, nil
+	}
 
 	// Ensure owner references are correctly set
 	err = controllerutils.ReconcileOwnerReferences(cd, generateOwnershipUniqueKeys(cd), r, r.scheme, logger)

pkg/controller/clusterversion/clusterversion_controller.go

@@ -3,6 +3,7 @@ package clusterversion
 import (
 	"context"
 	"fmt"
+	"strconv"
 
 	"github.com/blang/semver/v4"
 	log "github.com/sirupsen/logrus"
@@ -111,6 +112,12 @@ func (r *ReconcileClusterVersion) Reconcile(ctx context.Context, request reconci
 		return reconcile.Result{}, err
 	}
 	cdLog = controllerutils.AddLogFields(controllerutils.MetaObjectLogTagger{Object: cd}, cdLog)
+
+	if paused, err := strconv.ParseBool(cd.Annotations[constants.ReconcilePauseAnnotation]); err == nil && paused {
+		cdLog.Info("skipping reconcile due to ClusterDeployment pause annotation")
+		return reconcile.Result{}, nil
+	}
+
 	// If the clusterdeployment is deleted, do not reconcile.
 	if cd.DeletionTimestamp != nil {
 		return reconcile.Result{}, nil

pkg/controller/controlplanecerts/controlplanecerts_controller.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/url"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -144,6 +145,11 @@ func (r *ReconcileControlPlaneCerts) Reconcile(ctx context.Context, request reco
 	}
 	cdLog = controllerutils.AddLogFields(controllerutils.MetaObjectLogTagger{Object: cd}, cdLog)
 
+	if paused, err := strconv.ParseBool(cd.Annotations[constants.ReconcilePauseAnnotation]); err == nil && paused {
+		cdLog.Info("skipping reconcile due to ClusterDeployment pause annotation")
+		return reconcile.Result{}, nil
+	}
+
 	// Initialize cluster deployment conditions if not present
 	newConditions, changed := controllerutils.InitializeClusterDeploymentConditions(cd.Status.Conditions, clusterDeploymentControlPlaneCertsConditions)
 	if changed {

pkg/controller/hibernation/hibernation_controller.go

@@ -171,6 +171,11 @@ func (r *hibernationReconciler) Reconcile(ctx context.Context, request reconcile
 	}
 	cdLog = controllerutils.AddLogFields(controllerutils.MetaObjectLogTagger{Object: cd}, cdLog)
 
+	if paused, err := strconv.ParseBool(cd.Annotations[constants.ReconcilePauseAnnotation]); err == nil && paused {
+		cdLog.Info("skipping reconcile due to ClusterDeployment pause annotation")
+		return reconcile.Result{}, nil
+	}
+
 	// If cluster is already deleted, skip any processing
 	if !cd.DeletionTimestamp.IsZero() {
 		return reconcile.Result{}, nil

pkg/controller/remoteingress/remoteingress_controller.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"reflect"
 	"sort"
+	"strconv"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -156,6 +157,11 @@ func (r *ReconcileRemoteClusterIngress) Reconcile(ctx context.Context, request r
 		return reconcile.Result{}, err
 	}
 	cdLog = controllerutils.AddLogFields(controllerutils.MetaObjectLogTagger{Object: cd}, cdLog)
+	if paused, err := strconv.ParseBool(cd.Annotations[constants.ReconcilePauseAnnotation]); err == nil && paused {
+		cdLog.Info("skipping reconcile due to ClusterDeployment pause annotation")
+		return reconcile.Result{}, nil
+	}
+
 	rContext.clusterDeployment = cd
 
 	// Initialize cluster deployment conditions if not present