diff --git a/go.mod b/go.mod index 8d5256098..45650696f 100644 --- a/go.mod +++ b/go.mod @@ -133,3 +133,5 @@ require ( ) replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 + +replace github.com/openshift/library-go => github.com/tjungblu/library-go v0.0.0-20260611114644-374bd672722f diff --git a/go.sum b/go.sum index b49a42562..e5cf865b8 100644 --- a/go.sum +++ b/go.sum @@ -152,8 +152,6 @@ github.com/openshift/build-machinery-go v0.0.0-20251023084048-5d77c1a5e5af h1:Ui github.com/openshift/build-machinery-go v0.0.0-20251023084048-5d77c1a5e5af/go.mod h1:8jcm8UPtg2mCAsxfqKil1xrmRMI3a+XU2TZ9fF8A7TE= github.com/openshift/client-go v0.0.0-20260512113608-deb4dc54551a h1:EKx2XhOKehd1C5ptY7IrLl4WV35E8kP0pRPnG5BUZXk= github.com/openshift/client-go v0.0.0-20260512113608-deb4dc54551a/go.mod h1:V933kvY/cb/Un7UCEOhXHUySNX327u7Epe8g9KNqg2Q= -github.com/openshift/library-go v0.0.0-20260611070125-7fd5f333df33 h1:k7zdFrfi8699bfWL+ykAn3GF7r2hzIFmMBM8onHrLY0= -github.com/openshift/library-go v0.0.0-20260611070125-7fd5f333df33/go.mod h1:/HBhy6jm/igWI3Y1vYFwFG3ZCcXmnNsKUT6VBpPyM9A= github.com/openshift/multi-operator-manager v0.0.0-20241205181422-20aa3906b99d h1:Rzx23P63JFNNz5D23ubhC0FCN5rK8CeJhKcq5QKcdyU= github.com/openshift/multi-operator-manager v0.0.0-20241205181422-20aa3906b99d/go.mod h1:iVi9Bopa5cLhjG5ie9DoZVVqkH8BGb1FQVTtecOLn4I= github.com/openshift/oauth-apiserver v0.0.0-20260430140618-160ac7fb4ea6 h1:WvXToDt/IVTXb4NxbqEjY0cuPpVadTK6ATu75mlVM/s= @@ -208,6 +206,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tjungblu/library-go v0.0.0-20260611114644-374bd672722f h1:SzokGHqJd/T+U7h2bCJhCeWrMw8qascHjk6sCuL+8aQ= +github.com/tjungblu/library-go v0.0.0-20260611114644-374bd672722f/go.mod h1:/HBhy6jm/igWI3Y1vYFwFG3ZCcXmnNsKUT6VBpPyM9A= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go index 38c8357b3..6bb04d7a7 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go @@ -19,6 +19,7 @@ import ( operatorv1helpers "github.com/openshift/library-go/pkg/operator/v1helpers" "github.com/openshift/library-go/pkg/operator/encryption/controllers" + kmshealth "github.com/openshift/library-go/pkg/operator/encryption/kms/health" "github.com/openshift/library-go/pkg/operator/encryption/secrets" "github.com/openshift/library-go/pkg/operator/encryption/statemachine" ) @@ -49,6 +50,11 @@ func NewControllers( return nil, err } + convergedKEKReporter := kmshealth.NewMOCK_ConfigMapConvergedKEKReporter( + kubeInformersForNamespaces.ConfigMapLister(), + "", + ) + // for testing resourceSyncer might be nil if resourceSyncer != nil { if err := resourceSyncer.SyncSecretConditionally( @@ -125,6 +131,19 @@ func NewControllers( encryptionSecretSelector, eventRecorder, ), + controllers.NewKMSRotationController( + component, + provider, + deployer, + encryptionEnabledChecker.PreconditionFulfilled, + convergedKEKReporter, + operatorClient, + apiServerInformer, + kubeInformersForNamespaces, + secretsClient, + encryptionSecretSelector, + eventRecorder, + ), }, nil } diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go index af38a8d59..7482e6e75 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go @@ -391,6 +391,11 @@ func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, extern return 0, "", false } + // KMS KEK rotation migration must complete before minting a new key + if latestKey.NeedsKekMigration() { + return 0, "", false + } + // if the most recent secret was encrypted in a mode different than the current mode, we need to generate a new key if latestKey.Mode != currentMode { return latestKeyID, "encryption-mode-changed", true diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/kms_rotation_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/kms_rotation_controller.go new file mode 100644 index 000000000..e1c5c1d1c --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/kms_rotation_controller.go @@ -0,0 +1,278 @@ +package controllers + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + + configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" + "github.com/openshift/library-go/pkg/controller/factory" + "github.com/openshift/library-go/pkg/operator/encryption/secrets" + "github.com/openshift/library-go/pkg/operator/encryption/state" + "github.com/openshift/library-go/pkg/operator/encryption/statemachine" + "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/library-go/pkg/operator/resource/resourcehelper" + operatorv1helpers "github.com/openshift/library-go/pkg/operator/v1helpers" +) + +// ConvergedKEKReporter provides the cluster-converged KMS KEK identity from health aggregation. +type ConvergedKEKReporter interface { + ConvergedKekID() (kekID string, converged bool) +} + +type kmsRotationController struct { + instanceName string + controllerInstanceName string + operatorClient operatorv1helpers.OperatorClient + secretClient corev1client.SecretsGetter + encryptionSecretSelector metav1.ListOptions + deployer statemachine.Deployer + provider Provider + preconditionsFulfilledFn preconditionsFulfilled + convergedKEKReporter ConvergedKEKReporter + now func() time.Time +} + +func NewKMSRotationController( + instanceName string, + provider Provider, + deployer statemachine.Deployer, + preconditionsFulfilledFn preconditionsFulfilled, + convergedKEKReporter ConvergedKEKReporter, + operatorClient operatorv1helpers.OperatorClient, + apiServerConfigInformer configv1informers.APIServerInformer, + kubeInformersForNamespaces operatorv1helpers.KubeInformersForNamespaces, + secretClient corev1client.SecretsGetter, + encryptionSecretSelector metav1.ListOptions, + eventRecorder events.Recorder, +) factory.Controller { + c := &kmsRotationController{ + instanceName: instanceName, + controllerInstanceName: factory.ControllerInstanceName(instanceName, "EncryptionKMSRotation"), + operatorClient: operatorClient, + encryptionSecretSelector: encryptionSecretSelector, + secretClient: secretClient, + deployer: deployer, + provider: provider, + preconditionsFulfilledFn: preconditionsFulfilledFn, + convergedKEKReporter: convergedKEKReporter, + now: time.Now, + } + + return factory.New().ResyncEvery(time.Minute).WithSync(c.sync).WithControllerInstanceName(c.controllerInstanceName).WithInformers( + operatorClient.Informer(), + kubeInformersForNamespaces.InformersFor("openshift-config-managed").Core().V1().Secrets().Informer(), + kubeInformersForNamespaces.InformersFor("openshift-config").Core().V1().ConfigMaps().Informer(), + apiServerConfigInformer.Informer(), + deployer, + ).ToController( + c.controllerInstanceName, + eventRecorder.WithComponentSuffix("encryption-kms-rotation-controller"), + ) +} + +func (c *kmsRotationController) sync(ctx context.Context, syncCtx factory.SyncContext) error { + if ready, err := shouldRunEncryptionController(c.operatorClient, c.preconditionsFulfilledFn, c.provider.ShouldRunEncryptionControllers); err != nil || !ready { + return err + } + + _, _, encryptionSecrets, isTransitionalReason, err := statemachine.GetEncryptionConfigAndState( + ctx, c.deployer, c.secretClient, c.encryptionSecretSelector, c.provider.EncryptedGRs(), + ) + if err != nil { + return err + } + if len(isTransitionalReason) > 0 { + return nil + } + + writeKeySecret, writeKeyState, ok := latestKMSWriteKeySecret(encryptionSecrets) + if !ok { + return nil + } + + convergedKekID, converged := c.convergedKEKReporter.ConvergedKekID() + if !converged || convergedKekID == "" { + return c.updateWriteKeySecret(ctx, syncCtx, writeKeySecret, clearKekConvergenceAnnotations) + } + + return c.reconcileKekAnnotations(ctx, syncCtx, writeKeySecret, writeKeyState, convergedKekID, c.provider.EncryptedGRs()) +} + +func latestKMSWriteKeySecret(encryptionSecrets []*corev1.Secret) (*corev1.Secret, state.KeyState, bool) { + var latestSecret *corev1.Secret + var latestKey state.KeyState + var latestKeyID uint64 + for _, s := range encryptionSecrets { + ks, err := secrets.ToKeyState(s) + if err != nil || ks.Mode != state.KMS { + continue + } + keyID, valid := state.NameToKeyID(s.Name) + if !valid { + continue + } + if latestSecret == nil || keyID > latestKeyID { + latestSecret = s + latestKey = ks + latestKeyID = keyID + } + } + return latestSecret, latestKey, latestSecret != nil +} + +func (c *kmsRotationController) reconcileKekAnnotations( + ctx context.Context, + syncCtx factory.SyncContext, + writeKeySecret *corev1.Secret, + writeKeyState state.KeyState, + convergedKekID string, + encryptedGRs []schema.GroupResource, +) error { + kekMigration := secrets.KekMigrationFromSecret(writeKeySecret) + + // Bootstrap: initial migration complete, no kekId annotations yet. + if kekMigration.TargetKekID == "" && kekMigration.MigratedKekID == "" { + allMigrated, _, _ := state.MigratedFor(encryptedGRs, writeKeyState) + if !allMigrated { + return nil + } + return c.updateWriteKeySecret(ctx, syncCtx, writeKeySecret, func(s *corev1.Secret) (bool, error) { + return setKekBootstrapAnnotations(s, convergedKekID) + }) + } + + // Steady state or migration in flight: converged kekId matches current target. + if convergedKekID == kekMigration.TargetKekID { + if kekMigration.KekConvergedID != "" || !kekMigration.KekConvergedAt.IsZero() { + return c.updateWriteKeySecret(ctx, syncCtx, writeKeySecret, clearKekConvergenceAnnotations) + } + return nil + } + + // Candidate kekId differs from target: start or maintain the convergence clock. + if convergedKekID != kekMigration.KekConvergedID { + return c.updateWriteKeySecret(ctx, syncCtx, writeKeySecret, func(s *corev1.Secret) (bool, error) { + return setKekConvergenceClock(s, convergedKekID, c.now()) + }) + } + + if kekMigration.KekConvergedAt.IsZero() { + return c.updateWriteKeySecret(ctx, syncCtx, writeKeySecret, func(s *corev1.Secret) (bool, error) { + return setKekConvergenceClock(s, convergedKekID, c.now()) + }) + } + + if c.now().Sub(kekMigration.KekConvergedAt) >= secrets.KekConvergenceDelay { + return c.updateWriteKeySecret(ctx, syncCtx, writeKeySecret, func(s *corev1.Secret) (bool, error) { + return promoteConvergedKekToTarget(s, convergedKekID) + }) + } + + return nil +} + +type secretAnnotationMutator func(s *corev1.Secret) (changed bool, err error) + +func (c *kmsRotationController) updateWriteKeySecret(ctx context.Context, syncCtx factory.SyncContext, writeKeySecret *corev1.Secret, mutate secretAnnotationMutator) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + s, err := c.secretClient.Secrets(writeKeySecret.Namespace).Get(ctx, writeKeySecret.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get key secret %s/%s: %w", writeKeySecret.Namespace, writeKeySecret.Name, err) + } + + changed, err := mutate(s) + if err != nil { + return err + } + if !changed { + return nil + } + + _, updateErr := c.secretClient.Secrets(s.Namespace).Update(ctx, s, metav1.UpdateOptions{}) + resourcehelper.ReportUpdateEvent(syncCtx.Recorder(), s, updateErr) + return updateErr + }) +} + +func setKekBootstrapAnnotations(s *corev1.Secret, kekID string) (bool, error) { + if kekID == "" { + return false, nil + } + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + if s.Annotations[secrets.EncryptionSecretTargetKekID] == kekID && + s.Annotations[secrets.EncryptionSecretMigratedKekID] == kekID { + return false, nil + } + s.Annotations[secrets.EncryptionSecretTargetKekID] = kekID + s.Annotations[secrets.EncryptionSecretMigratedKekID] = kekID + delete(s.Annotations, secrets.EncryptionSecretKekConvergedAt) + delete(s.Annotations, secrets.EncryptionSecretKekConvergedID) + klog.V(2).Infof("bootstrapped KMS kekId annotations on secret %s/%s to %q", s.Namespace, s.Name, kekID) + return true, nil +} + +func setKekConvergenceClock(s *corev1.Secret, kekID string, now time.Time) (bool, error) { + if kekID == "" { + return false, nil + } + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + changed := false + if s.Annotations[secrets.EncryptionSecretKekConvergedID] != kekID { + s.Annotations[secrets.EncryptionSecretKekConvergedID] = kekID + s.Annotations[secrets.EncryptionSecretKekConvergedAt] = now.Format(time.RFC3339) + changed = true + } else if s.Annotations[secrets.EncryptionSecretKekConvergedAt] == "" { + s.Annotations[secrets.EncryptionSecretKekConvergedAt] = now.Format(time.RFC3339) + changed = true + } + if changed { + klog.V(2).Infof("started KMS kekId convergence clock on secret %s/%s for candidate %q", s.Namespace, s.Name, kekID) + } + return changed, nil +} + +func promoteConvergedKekToTarget(s *corev1.Secret, kekID string) (bool, error) { + if kekID == "" { + return false, nil + } + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + if s.Annotations[secrets.EncryptionSecretTargetKekID] == kekID && + s.Annotations[secrets.EncryptionSecretKekConvergedID] == "" && + s.Annotations[secrets.EncryptionSecretKekConvergedAt] == "" { + return false, nil + } + s.Annotations[secrets.EncryptionSecretTargetKekID] = kekID + delete(s.Annotations, secrets.EncryptionSecretKekConvergedAt) + delete(s.Annotations, secrets.EncryptionSecretKekConvergedID) + klog.V(2).Infof("updated target-kek-id on secret %s/%s to %q after convergence delay", s.Namespace, s.Name, kekID) + return true, nil +} + +func clearKekConvergenceAnnotations(s *corev1.Secret) (bool, error) { + if s.Annotations == nil { + return false, nil + } + _, hasID := s.Annotations[secrets.EncryptionSecretKekConvergedID] + _, hasAt := s.Annotations[secrets.EncryptionSecretKekConvergedAt] + if !hasID && !hasAt { + return false, nil + } + delete(s.Annotations, secrets.EncryptionSecretKekConvergedID) + delete(s.Annotations, secrets.EncryptionSecretKekConvergedAt) + klog.V(4).Infof("cleared KMS kekId convergence clock on secret %s/%s", s.Namespace, s.Name) + return true, nil +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/migration_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/migration_controller.go index 7fc649f02..682fb8c57 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/migration_controller.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/migration_controller.go @@ -215,39 +215,57 @@ func (c *migrationController) migrateKeysIfNeededAndRevisionStable(ctx context.C // we never want to migrate during an intermediate state because that could lead to one API server // using a write key that another API server has not observed // this could lead to etcd storing data that not all API servers can decrypt + writeKeySecret, err := writeKeySecretForState(encryptionSecrets, currentState) + if err != nil { + return nil, err + } + needsKekMigration := secrets.NeedsKekMigration(writeKeySecret) + var errs []error + kekMigrationComplete := needsKekMigration for _, gr := range grs { grActualKeys := currentState[gr] if !grActualKeys.HasWriteKey() { + kekMigrationComplete = false continue // no write key to migrate to } - if alreadyMigrated, _, _ := state.MigratedFor([]schema.GroupResource{gr}, grActualKeys.WriteKey); alreadyMigrated { - continue + if !needsKekMigration { + if alreadyMigrated, _, _ := state.MigratedFor([]schema.GroupResource{gr}, grActualKeys.WriteKey); alreadyMigrated { + continue + } + } + + writeKeyForGR := grActualKeys.WriteKey.Key.Name + if needsKekMigration { + writeKeyForGR = secrets.MigrationWriteKey(grActualKeys.WriteKey.Key.Name, writeKeySecret) } // idem-potent migration start - finished, result, when, err := c.migrator.EnsureMigration(gr, grActualKeys.WriteKey.Key.Name) + finished, result, when, err := c.migrator.EnsureMigration(gr, writeKeyForGR) if err == nil && finished && result != nil && time.Since(when) > migrationRetryDuration { // last migration error is far enough ago. Prune and retry. if err := c.migrator.PruneMigration(gr); err != nil { errs = append(errs, err) continue } - finished, result, when, err = c.migrator.EnsureMigration(gr, grActualKeys.WriteKey.Key.Name) + finished, result, when, err = c.migrator.EnsureMigration(gr, writeKeyForGR) } if err != nil { errs = append(errs, err) + kekMigrationComplete = false continue } if finished && result != nil { errs = append(errs, result) + kekMigrationComplete = false continue } if !finished { migratingResources = append(migratingResources, gr) + kekMigrationComplete = false continue } @@ -279,13 +297,70 @@ func (c *migrationController) migrateKeysIfNeededAndRevisionStable(ctx context.C return updateErr }); err != nil { errs = append(errs, err) + kekMigrationComplete = false continue } } + if kekMigrationComplete && writeKeySecret != nil { + if err := c.setMigratedKekID(ctx, syncContext, writeKeySecret); err != nil { + errs = append(errs, err) + } + } + return migratingResources, errors.NewAggregate(errs) } +func writeKeySecretForState(encryptionSecrets []*corev1.Secret, currentState map[schema.GroupResource]state.GroupResourceState) (*corev1.Secret, error) { + var writeKey state.KeyState + for _, grState := range currentState { + if grState.HasWriteKey() { + writeKey = grState.WriteKey + break + } + } + if len(writeKey.Key.Name) == 0 { + return nil, nil + } + + for _, s := range encryptionSecrets { + keyID, valid := state.NameToKeyID(s.Name) + if !valid { + continue + } + writeKeyID, ok := state.NameToKeyID(writeKey.Key.Name) + if !ok || keyID != writeKeyID { + continue + } + return s, nil + } + return nil, fmt.Errorf("write key secret for key ID %s not found", writeKey.Key.Name) +} + +func (c *migrationController) setMigratedKekID(ctx context.Context, syncContext factory.SyncContext, writeKeySecret *corev1.Secret) error { + targetKekID := writeKeySecret.Annotations[secrets.EncryptionSecretTargetKekID] + if targetKekID == "" { + return nil + } + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + s, err := c.secretClient.Secrets(writeKeySecret.Namespace).Get(ctx, writeKeySecret.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get key secret %s/%s: %v", writeKeySecret.Namespace, writeKeySecret.Name, err) + } + if s.Annotations[secrets.EncryptionSecretMigratedKekID] == targetKekID { + return nil + } + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + s.Annotations[secrets.EncryptionSecretMigratedKekID] = targetKekID + + _, updateErr := c.secretClient.Secrets(s.Namespace).Update(ctx, s, metav1.UpdateOptions{}) + resourcehelper.ReportUpdateEvent(syncContext.Recorder(), s, updateErr) + return updateErr + }) +} + func setResourceMigrated(gr schema.GroupResource, s *corev1.Secret) (bool, error) { migratedGRs := secrets.MigratedGroupResources{} if existing, found := s.Annotations[secrets.EncryptionSecretMigratedResources]; found { diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/cmd.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/cmd.go new file mode 100644 index 000000000..d6733848c --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/cmd.go @@ -0,0 +1,110 @@ +package health + +import ( + "context" + "fmt" + "regexp" + "time" + + "github.com/openshift/library-go/pkg/operator/v1helpers" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" +) + +// kmsSocketPattern matches the socket path each co-located KMSv2 plugin is +// mounted at, e.g. unix:///var/run/kmsplugin/kms-1.sock. +var kmsSocketPattern = regexp.MustCompile(`^unix:///var/run/kmsplugin/kms-\d+\.sock$`) + +// options' flag-bound fields are exported so the struct can be logged as a +// whole via klog.InfoS, which JSON-marshals its values. +type options struct { + KMSSockets []string + Interval time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + NodeName string + Kubeconfig string + + newOperatorClient func(*rest.Config) (v1helpers.OperatorClient, error) +} + +func NewCommand(ctx context.Context, newOperatorClient func(*rest.Config) (v1helpers.OperatorClient, error)) *cobra.Command { + o := &options{ + newOperatorClient: newOperatorClient, + } + + cmd := &cobra.Command{ + Use: "kms-health-reporter", + Short: "Observes co-located KMSv2 plugins and publishes status as an OperatorCondition.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.validate(); err != nil { + return err + } + return o.run() + }, + } + o.addFlags(cmd.Flags()) + return cmd +} + +func (o *options) addFlags(fs *pflag.FlagSet) { + fs.StringSliceVar(&o.KMSSockets, "kms-sockets", nil, "KMS plugin endpoints in unix:// URI format (e.g. unix:///var/run/kmsplugin/kms-1.sock)") + fs.DurationVar(&o.Interval, "interval", 30*time.Second, "cadence between probe+emit cycles") + fs.DurationVar(&o.ReadTimeout, "read-timeout", 5*time.Second, "deadline for each Status RPC") + fs.DurationVar(&o.WriteTimeout, "write-timeout", 10*time.Second, "deadline for each condition update") + fs.StringVar(&o.NodeName, "node-name", "", "node name recorded in the condition used to help to identify the origin") + fs.StringVar(&o.Kubeconfig, "kubeconfig", "", "path to a kubeconfig; empty uses in-cluster config") +} + +func (o *options) validate() error { + if len(o.KMSSockets) == 0 { + return fmt.Errorf("--kms-sockets is required, at least one") + } + for _, s := range o.KMSSockets { + if !kmsSocketPattern.MatchString(s) { + return fmt.Errorf("--kms-sockets entry %q must match %s", s, kmsSocketPattern) + } + } + + if o.Interval <= 0 { + return fmt.Errorf("--interval must be positive") + } + if o.ReadTimeout <= 0 { + return fmt.Errorf("--read-timeout must be positive") + } + if o.WriteTimeout <= 0 { + return fmt.Errorf("--write-timeout must be positive") + } + if o.NodeName == "" { + return fmt.Errorf("--node-name is required") + } + + return nil +} + +func (o *options) run() error { + cfg, err := buildRESTConfig(o.Kubeconfig) + if err != nil { + return fmt.Errorf("build rest config: %w", err) + } + + if _, err := o.newOperatorClient(cfg); err != nil { + return fmt.Errorf("build operator client: %w", err) + } + + klog.InfoS("kms-health-reporter starting", "config", o) + + return nil +} + +func buildRESTConfig(kubeconfig string) (*rest.Config, error) { + if kubeconfig != "" { + return clientcmd.BuildConfigFromFlags("", kubeconfig) + } + return rest.InClusterConfig() +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/configmap_reporter.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/configmap_reporter.go new file mode 100644 index 000000000..6549b61e6 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/configmap_reporter.go @@ -0,0 +1,70 @@ +package health + +import ( + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/klog/v2" +) + +const ( + // ConvergedKekConfigMapNamespace is where the mock converged-kek ConfigMap lives. + ConvergedKekConfigMapNamespace = "openshift-config" + // DefaultConvergedKekConfigMapName is the default ConfigMap name for mock KEK health input. + DefaultConvergedKekConfigMapName = "encryption-kms-converged-kek" + // ConvergedKekConfigMapDataKeyKekID holds the cluster-converged KMS kekId. + ConvergedKekConfigMapDataKeyKekID = "converged-kek-id" + // ConvergedKekConfigMapDataKeyConverged is an optional "true"/"false" override. + // When omitted, a non-empty converged-kek-id is treated as converged. + ConvergedKekConfigMapDataKeyConverged = "converged" +) + +// MOCK_ConfigMapConvergedKEKReporter reads cluster-converged kekId from a ConfigMap in openshift-config. +// It is intended for development and testing until kms-health-reporter publishes real health input. +type MOCK_ConfigMapConvergedKEKReporter struct { + lister corev1listers.ConfigMapLister + namespace string + name string +} + +// NewMOCK_ConfigMapConvergedKEKReporter returns a mock reporter backed by the named ConfigMap. +// An empty name uses DefaultConvergedKekConfigMapName. +func NewMOCK_ConfigMapConvergedKEKReporter(lister corev1listers.ConfigMapLister, name string) *MOCK_ConfigMapConvergedKEKReporter { + if name == "" { + name = DefaultConvergedKekConfigMapName + } + return &MOCK_ConfigMapConvergedKEKReporter{ + lister: lister, + namespace: ConvergedKekConfigMapNamespace, + name: name, + } +} + +func (r *MOCK_ConfigMapConvergedKEKReporter) ConvergedKekID() (string, bool) { + cm, err := r.lister.ConfigMaps(r.namespace).Get(r.name) + if err != nil { + klog.V(4).InfoS("converged kek configmap not available", "namespace", r.namespace, "name", r.name, "err", err) + return "", false + } + return ConvergedKekFromConfigMap(cm) +} + +// ConvergedKekFromConfigMap parses mock health input from a ConfigMap. +func ConvergedKekFromConfigMap(cm *corev1.ConfigMap) (kekID string, converged bool) { + if cm == nil || cm.Data == nil { + return "", false + } + kekID = strings.TrimSpace(cm.Data[ConvergedKekConfigMapDataKeyKekID]) + if kekID == "" { + return "", false + } + if v, ok := cm.Data[ConvergedKekConfigMapDataKeyConverged]; ok { + parsed, err := strconv.ParseBool(strings.TrimSpace(v)) + if err != nil || !parsed { + return "", false + } + } + return kekID, true +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/kek.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/kek.go new file mode 100644 index 000000000..2e2a8a729 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/kek.go @@ -0,0 +1,58 @@ +package secrets + +import ( + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" +) + +// KekMigrationState holds KMS KEK rotation annotations from a write-key secret. +type KekMigrationState struct { + TargetKekID string + MigratedKekID string + KekConvergedAt time.Time + KekConvergedID string +} + +// KekMigrationFromSecret parses KMS KEK rotation annotations from a secret. +func KekMigrationFromSecret(s *corev1.Secret) KekMigrationState { + if s == nil || s.Annotations == nil { + return KekMigrationState{} + } + state := KekMigrationState{ + TargetKekID: s.Annotations[EncryptionSecretTargetKekID], + MigratedKekID: s.Annotations[EncryptionSecretMigratedKekID], + KekConvergedID: s.Annotations[EncryptionSecretKekConvergedID], + } + if v, ok := s.Annotations[EncryptionSecretKekConvergedAt]; ok && len(v) > 0 { + if ts, err := time.Parse(time.RFC3339, v); err == nil { + state.KekConvergedAt = ts + } + } + return state +} + +// NeedsKekMigration reports whether target-kek-id is set and differs from migrated-kek-id. +func NeedsKekMigration(s *corev1.Secret) bool { + if s == nil || s.Annotations == nil { + return false + } + target := s.Annotations[EncryptionSecretTargetKekID] + if target == "" { + return false + } + return target != s.Annotations[EncryptionSecretMigratedKekID] +} + +// MigrationWriteKey returns the migrator write-key identity for the given key name. +// When target-kek-id is set, the format is {keyName}-{kekId}; otherwise {keyName}. +func MigrationWriteKey(keyName string, s *corev1.Secret) string { + if s == nil || s.Annotations == nil { + return keyName + } + if target := s.Annotations[EncryptionSecretTargetKekID]; target != "" { + return fmt.Sprintf("%s-%s", keyName, target) + } + return keyName +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go index 716764f08..7db4404ae 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go @@ -60,6 +60,16 @@ func ToKeyState(s *corev1.Secret) (state.KeyState, error) { key.ExternalReason = v } + kekMigration := KekMigrationFromSecret(s) + if kekMigration.TargetKekID != "" || kekMigration.MigratedKekID != "" || kekMigration.KekConvergedID != "" || !kekMigration.KekConvergedAt.IsZero() { + key.KekMigration = &state.KekMigrationState{ + TargetKekID: kekMigration.TargetKekID, + MigratedKekID: kekMigration.MigratedKekID, + KekConvergedAt: kekMigration.KekConvergedAt, + KekConvergedID: kekMigration.KekConvergedID, + } + } + keyMode := state.Mode(s.Annotations[encryptionSecretMode]) switch keyMode { case state.AESCBC, state.AESGCM, state.SecretBox, state.Identity: diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go index 5a49727cf..ff9f7eee2 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go @@ -1,6 +1,8 @@ package secrets import ( + "time" + "k8s.io/apimachinery/pkg/runtime/schema" ) @@ -67,8 +69,21 @@ const ( // values fetched from the referenced configmap in openshift-config. The full data key is // constructed as prefix + configMapName + separator + dataKey. encryptionSecretKMSConfigMapDataPrefix = "encryption.apiserver.operator.openshift.io-kms-plugin-configmap-" + + // EncryptionSecretTargetKekID is the target KMS KEK identity to migrate toward. + EncryptionSecretTargetKekID = "encryption.apiserver.operator.openshift.io/target-kek-id" + // EncryptionSecretMigratedKekID is the last fully migrated KMS KEK identity. + EncryptionSecretMigratedKekID = "encryption.apiserver.operator.openshift.io/migrated-kek-id" + // EncryptionSecretKekConvergedAt records when a candidate kekId first achieved cluster convergence (RFC3339). + EncryptionSecretKekConvergedAt = "encryption.apiserver.operator.openshift.io/kek-converged-at" + // EncryptionSecretKekConvergedID is the candidate kekId the kek-converged-at timestamp belongs to. + EncryptionSecretKekConvergedID = "encryption.apiserver.operator.openshift.io/kek-converged-id" ) +// KekConvergenceDelay is the minimum duration a candidate kekId must remain cluster-converged +// before target-kek-id is updated to a new value (KEP-3299). +const KekConvergenceDelay = 5 * time.Minute + // MigratedGroupResources is the data structured stored in the // encryption.apiserver.operator.openshift.io/migrated-resources // of a key secret. diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go index fd1af2ebe..0226dca5f 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go @@ -50,6 +50,27 @@ type KeyState struct { ExternalReason string // stores all the KMS encryption mode related configurations KMS *KMSState + // KekMigration stores KMS KEK rotation annotations from the write-key secret. + KekMigration *KekMigrationState +} + +// KekMigrationState holds KMS KEK rotation annotations parsed from a key secret. +type KekMigrationState struct { + TargetKekID string + MigratedKekID string + KekConvergedAt time.Time + KekConvergedID string +} + +// NeedsKekMigration reports whether target-kek-id is set and differs from migrated-kek-id. +func (k KeyState) NeedsKekMigration() bool { + if k.Mode != KMS || k.KekMigration == nil { + return false + } + if k.KekMigration.TargetKekID == "" { + return false + } + return k.KekMigration.TargetKekID != k.KekMigration.MigratedKekID } func (k *KeyState) HasKMSEncryption() bool { diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/statemachine/transition.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/statemachine/transition.go index 12eb5b8b9..51e9ba8be 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/statemachine/transition.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/statemachine/transition.go @@ -193,6 +193,10 @@ func getDesiredEncryptionState(oldSecretData *encryptiondata.Config, encryptionS // STEP 4: with consistent read-keys and write-keys, remove every read-key other than the write-key and one last read key. // // Note: because read-keys are consistent, currentlyEncryptedGRs equals toBeEncryptedGRs + if writeKey.NeedsKekMigration() { + klog.V(4).Infof("KMS kek migration in progress for key ID %s", writeKey.Key.Name) + return desiredEncryptionState + } allMigrated, _, reason := state.MigratedFor(currentlyEncryptedGRs, writeKey) if !allMigrated { klog.V(4).Infof("%s", reason) diff --git a/vendor/modules.txt b/vendor/modules.txt index 8b9f5681d..002eb2cf4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -383,7 +383,7 @@ github.com/openshift/client-go/user/applyconfigurations/internal github.com/openshift/client-go/user/applyconfigurations/user/v1 github.com/openshift/client-go/user/clientset/versioned/scheme github.com/openshift/client-go/user/clientset/versioned/typed/user/v1 -# github.com/openshift/library-go v0.0.0-20260611070125-7fd5f333df33 +# github.com/openshift/library-go v0.0.0-20260611070125-7fd5f333df33 => github.com/tjungblu/library-go v0.0.0-20260611114644-374bd672722f ## explicit; go 1.25.0 github.com/openshift/library-go/pkg/apiserver/jsonpatch github.com/openshift/library-go/pkg/apps/deployment @@ -428,6 +428,7 @@ github.com/openshift/library-go/pkg/operator/encryption/crypto github.com/openshift/library-go/pkg/operator/encryption/deployer github.com/openshift/library-go/pkg/operator/encryption/encoding github.com/openshift/library-go/pkg/operator/encryption/encryptiondata +github.com/openshift/library-go/pkg/operator/encryption/kms/health github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle github.com/openshift/library-go/pkg/operator/encryption/observer github.com/openshift/library-go/pkg/operator/encryption/secrets @@ -1651,3 +1652,4 @@ sigs.k8s.io/structured-merge-diff/v6/value ## explicit; go 1.22 sigs.k8s.io/yaml # github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 +# github.com/openshift/library-go => github.com/tjungblu/library-go v0.0.0-20260611114644-374bd672722f