From 0c80fb20fa9b782b061e041a9fd9bf210ca48bc7 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Mon, 1 Jun 2026 13:37:15 +0200 Subject: [PATCH 1/2] NO-JIRA: kms rotation controller --- go.mod | 2 + go.sum | 2 + .../pkg/operator/encryption/controllers.go | 13 + .../encryption_rotation_controller.go | 293 ++++++++++++++++++ .../encryption/encryptiondata/config.go | 19 ++ .../encryptionstatus/convergence.go | 50 +++ .../encryption/encryptionstatus/migration.go | 47 +++ .../encryption/encryptionstatus/operator.go | 142 +++++++++ .../encryption/encryptionstatus/rotation.go | 101 ++++++ .../encryption/encryptionstatus/types.go | 44 +++ .../operator/encryption/secrets/secrets.go | 11 + vendor/modules.txt | 2 + 12 files changed, 726 insertions(+) create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/encryption_rotation_controller.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/convergence.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/migration.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/operator.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/rotation.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/types.go diff --git a/go.mod b/go.mod index 8d5256098..f0dc1b217 100644 --- a/go.mod +++ b/go.mod @@ -133,3 +133,5 @@ require ( ) replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 + +replace github.com/openshift/library-go => github.com/tjungblu/library-go v0.0.0-20260608092146-53a048269fb0 diff --git a/go.sum b/go.sum index b49a42562..ca8652a99 100644 --- a/go.sum +++ b/go.sum @@ -208,6 +208,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tjungblu/library-go v0.0.0-20260608092146-53a048269fb0 h1:bXxmi7xEzASxdfffvAN1F8KV84ElCE1A4hXsOxhAbl0= +github.com/tjungblu/library-go v0.0.0-20260608092146-53a048269fb0/go.mod h1:/HBhy6jm/igWI3Y1vYFwFG3ZCcXmnNsKUT6VBpPyM9A= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go index 38c8357b3..3ccd1078b 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go @@ -125,6 +125,19 @@ func NewControllers( encryptionSecretSelector, eventRecorder, ), + controllers.NewEncryptionRotationController( + component, + provider, + deployer, + encryptionEnabledChecker.PreconditionFulfilled, + migrator, + operatorClient, + apiServerInformer, + kubeInformersForNamespaces, + secretsClient, + encryptionSecretSelector, + eventRecorder, + ), }, nil } diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/encryption_rotation_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/encryption_rotation_controller.go new file mode 100644 index 000000000..8ccd89079 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/encryption_rotation_controller.go @@ -0,0 +1,293 @@ +package controllers + +import ( + "context" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + + configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" + + "github.com/openshift/library-go/pkg/controller/factory" + "github.com/openshift/library-go/pkg/operator/encryption/controllers/migrators" + "github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus" + "github.com/openshift/library-go/pkg/operator/encryption/secrets" + "github.com/openshift/library-go/pkg/operator/encryption/statemachine" + "github.com/openshift/library-go/pkg/operator/events" + operatorv1helpers "github.com/openshift/library-go/pkg/operator/v1helpers" +) + +const ( + encryptionRotationConvergenceDelay = 5 * time.Minute +) + +// encryptionRotationController orchestrates KMS KEK rotation by resetting migration state on the +// encryption key secret and recording rotation progress on operator status. +type encryptionRotationController struct { + instanceName string + controllerInstanceName string + operatorClient operatorv1helpers.OperatorClient + encryptionSecretSelector metav1.ListOptions + secretClient corev1client.SecretsGetter + deployer statemachine.Deployer + migrator migrators.Migrator + provider Provider + preconditionsFulfilledFn preconditionsFulfilled +} + +func NewEncryptionRotationController( + instanceName string, + provider Provider, + deployer statemachine.Deployer, + preconditionsFulfilledFn preconditionsFulfilled, + migrator migrators.Migrator, + operatorClient operatorv1helpers.OperatorClient, + apiServerConfigInformer configv1informers.APIServerInformer, + kubeInformersForNamespaces operatorv1helpers.KubeInformersForNamespaces, + secretClient corev1client.SecretsGetter, + encryptionSecretSelector metav1.ListOptions, + eventRecorder events.Recorder, +) factory.Controller { + c := &encryptionRotationController{ + instanceName: instanceName, + controllerInstanceName: factory.ControllerInstanceName(instanceName, "EncryptionRotation"), + operatorClient: operatorClient, + encryptionSecretSelector: encryptionSecretSelector, + secretClient: secretClient, + deployer: deployer, + migrator: migrator, + provider: provider, + preconditionsFulfilledFn: preconditionsFulfilledFn, + } + + return factory.New().ResyncEvery(time.Minute).WithSync(c.sync).WithControllerInstanceName(c.controllerInstanceName).WithInformers( + operatorClient.Informer(), + kubeInformersForNamespaces.InformersFor("openshift-config-managed").Core().V1().Secrets().Informer(), + apiServerConfigInformer.Informer(), + deployer, + ).ToController( + c.controllerInstanceName, + eventRecorder.WithComponentSuffix("encryption-rotation-controller"), + ) +} + +func (c *encryptionRotationController) sync(ctx context.Context, syncCtx factory.SyncContext) error { + if ready, err := shouldRunEncryptionController(c.operatorClient, c.preconditionsFulfilledFn, c.provider.ShouldRunEncryptionControllers); err != nil || !ready { + return err + } + return c.reconcile(ctx) +} + +func (c *encryptionRotationController) reconcile(ctx context.Context) error { + currentConfig, _, encryptionSecrets, _, err := statemachine.GetEncryptionConfigAndState( + ctx, c.deployer, c.secretClient, c.encryptionSecretSelector, c.provider.EncryptedGRs(), + ) + if err != nil { + return err + } + + if currentConfig == nil || !currentConfig.UsesKMS() { + return nil + } + + _, operatorStatus, _, err := c.operatorClient.GetOperatorState() + if err != nil { + return err + } + + healthReports := encryptionstatus.HealthReportsFromOperatorStatus(operatorStatus) + rotations, err := encryptionstatus.KeyRotationStatusFromOperatorStatus(operatorStatus) + if err != nil { + return err + } + + encryptedGRs := currentConfig.EncryptedGroupResources() + klog.Infof("%s: reconciling KMS key rotation (%d plugin(s), %d health report(s), %d rotation entr(ies))", + c.instanceName, len(currentConfig.KMSPlugins), len(healthReports), len(rotations)) + for keyID := range currentConfig.KMSPlugins { + writeKeySecret := secrets.FindKeySecret(encryptionSecrets, c.instanceName, keyID) + if writeKeySecret == nil { + klog.Infof("%s: no write key secret for KMS plugin keyID %q, skipping", c.instanceName, keyID) + continue + } + + rotations, err = c.reconcileKMSPlugin( + ctx, keyID, encryptedGRs, writeKeySecret, healthReports, rotations, + ) + if err != nil { + return err + } + } + + if err := c.persistRotations(ctx, rotations); err != nil { + return err + } + + return nil +} + +func (c *encryptionRotationController) reconcileKMSPlugin( + ctx context.Context, + keyID string, + encryptedGRs []schema.GroupResource, + writeKeySecret *corev1.Secret, + healthReports []encryptionstatus.KMSPluginHealthReport, + rotations []encryptionstatus.KMSPluginRotationStatus, +) ([]encryptionstatus.KMSPluginRotationStatus, error) { + // Check KEK convergence across all nodes for this keyID. + convergedKEKID, converged := encryptionstatus.ConvergedKEKForKeyID(healthReports, keyID) + if !converged { + klog.Infof("%s: KEK not yet converged for keyID %q, waiting for all nodes to report the same kekID", c.instanceName, keyID) + return rotations, nil + } + klog.Infof("%s: KEK converged for keyID %q: kekID=%q", c.instanceName, keyID, convergedKEKID) + + // If the converged kekID matches the last completed rotation, we are in steady state. + lastCompleted, hasCompleted := encryptionstatus.LatestCompletedRotationForKeyID(rotations, keyID) + if hasCompleted && convergedKEKID == lastCompleted.KEKID { + klog.V(4).Infof("%s: converged kekID %q matches last completed rotation for keyID %q, nothing to do", c.instanceName, convergedKEKID, keyID) + return rotations, nil + } + + // Ensure an open rotation entry exists with discoveryTime for the converged kekID. + now := metav1.Now() + rotations, openIdx := encryptionstatus.GetOrCreateOpenRotation(rotations, keyID, convergedKEKID, now) + klog.Infof("%s: tracking open rotation for keyID=%q kekID=%q (discoveryTime=%s)", + c.instanceName, keyID, convergedKEKID, rotations[openIdx].DiscoveryTime.Format(time.RFC3339)) + + // Mirror migration finish time from the write key secret annotations when all + // encrypted group resources have been migrated by the migration controller. + migrated, err := encryptionstatus.AllEncryptedGRsMigrated(writeKeySecret, encryptedGRs) + if err != nil { + return rotations, err + } + rotations = mirrorMigrationFinish(c.instanceName, rotations, openIdx, migrated, writeKeySecret) + + // Bootstrap: if no prior completed rotation exists, this is the initial provider + // migration driven by the migration controller. We only track convergence and mirror + // the finish time — never prune migrations or clear annotations. + if !hasCompleted { + klog.Infof("%s: bootstrap for keyID %q — no prior completed rotation, waiting for initial migration to complete", c.instanceName, keyID) + return rotations, nil + } + + // From here on a KEK change was detected (convergedKEKID != lastCompleted.KEKID). + // Check guards before starting a storage re-migration. + entry := rotations[openIdx] + + if entry.MigrationStartTime != nil { + klog.Infof("%s: rotation for kekID %q already started at %s, waiting for migration to complete", + c.instanceName, convergedKEKID, entry.MigrationStartTime.Format(time.RFC3339)) + return rotations, nil + } + + if entry.DiscoveryTime != nil && time.Since(entry.DiscoveryTime.Time) < encryptionRotationConvergenceDelay { + remaining := encryptionRotationConvergenceDelay - time.Since(entry.DiscoveryTime.Time) + klog.Infof("%s: rotation for kekID %q waiting for convergence delay (%s remaining)", + c.instanceName, convergedKEKID, remaining.Round(time.Second)) + return rotations, nil + } + + // All guards passed — start the rotation by pruning existing storage version + // migrations and clearing migration annotations on the write key secret so the + // migration controller picks up the work again. + klog.Infof("%s: starting storage re-migration for keyID=%q: kekID changed from %q to %q", + c.instanceName, keyID, lastCompleted.KEKID, convergedKEKID) + if err := c.startRotation(ctx, encryptedGRs, writeKeySecret); err != nil { + return rotations, err + } + + rotations = encryptionstatus.SetMigrationStartTime(rotations, openIdx, now) + klog.Infof("%s: rotation migration started for kekID %q at %s", c.instanceName, convergedKEKID, now.Format(time.RFC3339)) + return rotations, nil +} + +func (c *encryptionRotationController) startRotation(ctx context.Context, encryptedGRs []schema.GroupResource, secret *corev1.Secret) error { + for _, gr := range encryptedGRs { + klog.Infof("%s: pruning storage migration for %s", c.instanceName, gr) + if err := c.migrator.PruneMigration(gr); err != nil { + return err + } + } + klog.Infof("%s: clearing migration annotations on secret %s/%s", c.instanceName, secret.Namespace, secret.Name) + return c.clearMigrationAnnotations(ctx, secret) +} + +func mirrorMigrationFinish( + instanceName string, + rotations []encryptionstatus.KMSPluginRotationStatus, + idx int, + migrated bool, + secret *corev1.Secret, +) []encryptionstatus.KMSPluginRotationStatus { + if !migrated || idx < 0 || idx >= len(rotations) || rotations[idx].MigrationFinishTime != nil { + return rotations + } + finish, ok := migrationFinishTimeFromSecret(secret) + if !ok { + return rotations + } + entry := rotations[idx] + klog.Infof("%s: mirroring migrationFinishTime for keyID %q kekID %q from secret %s/%s at %s", + instanceName, entry.KeyID, entry.KEKID, secret.Namespace, secret.Name, finish.Format(time.RFC3339)) + return encryptionstatus.SetMigrationFinishTime(rotations, idx, finish) +} + +func migrationFinishTimeFromSecret(secret *corev1.Secret) (metav1.Time, bool) { + if secret == nil || secret.Annotations == nil { + return metav1.Time{}, false + } + raw, ok := secret.Annotations[secrets.EncryptionSecretMigratedTimestamp] + if !ok || raw == "" { + return metav1.Time{}, false + } + ts, err := time.Parse(time.RFC3339, raw) + if err != nil { + klog.Warningf("ignoring invalid %s annotation on secret %s/%s: %v", + secrets.EncryptionSecretMigratedTimestamp, secret.Namespace, secret.Name, err) + return metav1.Time{}, false + } + return metav1.NewTime(ts), true +} + +func (c *encryptionRotationController) clearMigrationAnnotations(ctx context.Context, secret *corev1.Secret) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + current, err := c.secretClient.Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{}) + if err != nil { + return err + } + if current.Annotations == nil { + return nil + } + changed := false + if _, ok := current.Annotations[secrets.EncryptionSecretMigratedTimestamp]; ok { + delete(current.Annotations, secrets.EncryptionSecretMigratedTimestamp) + changed = true + } + if _, ok := current.Annotations[secrets.EncryptionSecretMigratedResources]; ok { + delete(current.Annotations, secrets.EncryptionSecretMigratedResources) + changed = true + } + if !changed { + return nil + } + _, err = c.secretClient.Secrets(current.Namespace).Update(ctx, current, metav1.UpdateOptions{}) + return err + }) +} + +func (c *encryptionRotationController) persistRotations(ctx context.Context, rotations []encryptionstatus.KMSPluginRotationStatus) error { + _, updated, err := operatorv1helpers.UpdateStatus(ctx, c.operatorClient, encryptionstatus.SetKeyRotationStatusCondition(rotations)) + if err != nil { + return err + } + if updated { + klog.Infof("%s: updated key rotation status (%d entries)", c.instanceName, len(rotations)) + } + return nil +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/config.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/config.go index 02c94c23f..8bc6ba0a2 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/config.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/config.go @@ -92,6 +92,25 @@ func (c *Config) HasEncryptionConfiguration() bool { return c != nil && c.Encryption != nil } +// UsesKMS returns whether the deployed encryption configuration includes KMS plugins. +func (c *Config) UsesKMS() bool { + return c != nil && len(c.KMSPlugins) > 0 +} + +// EncryptedGroupResources returns group resources from the deployed encryption configuration. +func (c *Config) EncryptedGroupResources() []schema.GroupResource { + if !c.HasEncryptionConfiguration() { + return nil + } + grs := make([]schema.GroupResource, 0, len(c.Encryption.Resources)) + for _, resourceConfig := range c.Encryption.Resources { + for _, resource := range resourceConfig.Resources { + grs = append(grs, schema.ParseGroupResource(resource)) + } + } + return grs +} + // FromEncryptionState converts encryption state to Config. func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupResourceState) (*Config, error) { resourceConfigs := make([]apiserverconfigv1.ResourceConfiguration, 0, len(encryptionState)) diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/convergence.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/convergence.go new file mode 100644 index 000000000..d87545553 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/convergence.go @@ -0,0 +1,50 @@ +package encryptionstatus + +import ( + "strings" +) + +// KEKByKeyID groups observed kekIds per plugin keyId across health reports. +// Only healthy reports with non-empty keyId and kekId are included. +func KEKByKeyID(reports []KMSPluginHealthReport) map[string][]string { + result := map[string][]string{} + for _, report := range reports { + if !isHealthyReport(report) { + continue + } + result[report.KeyID] = append(result[report.KeyID], report.KEKID) + } + return result +} + +// ConvergedKEKForKeyID returns the unanimous kekId for keyID when every healthy report for that keyId agrees. +func ConvergedKEKForKeyID(reports []KMSPluginHealthReport, keyID string) (kekID string, ok bool) { + if keyID == "" { + return "", false + } + + byKeyID := KEKByKeyID(reports) + kekIDs, found := byKeyID[keyID] + if !found || len(kekIDs) == 0 { + return "", false + } + + uniq := map[string]struct{}{} + for _, id := range kekIDs { + if id == "" { + return "", false + } + uniq[id] = struct{}{} + } + if len(uniq) != 1 { + return "", false + } + for id := range uniq { + return id, true + } + return "", false +} + +func isHealthyReport(report KMSPluginHealthReport) bool { + return strings.EqualFold(report.Status, "healthy") +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/migration.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/migration.go new file mode 100644 index 000000000..1fe6e57c3 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/migration.go @@ -0,0 +1,47 @@ +package encryptionstatus + +import ( + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/openshift/library-go/pkg/operator/encryption/secrets" +) + +func parseMigratedResources(secret *corev1.Secret) ([]schema.GroupResource, error) { + if secret == nil || secret.Annotations == nil { + return nil, nil + } + raw, ok := secret.Annotations[secrets.EncryptionSecretMigratedResources] + if !ok || len(raw) == 0 { + return nil, nil + } + migrated := secrets.MigratedGroupResources{} + if err := json.Unmarshal([]byte(raw), &migrated); err != nil { + return nil, fmt.Errorf("invalid %s annotation: %w", secrets.EncryptionSecretMigratedResources, err) + } + return migrated.Resources, nil +} + +// AllEncryptedGRsMigrated returns true when every encrypted GR is listed in migrated-resources. +func AllEncryptedGRsMigrated(secret *corev1.Secret, encryptedGRs []schema.GroupResource) (bool, error) { + migrated, err := parseMigratedResources(secret) + if err != nil { + return false, err + } + for _, gr := range encryptedGRs { + found := false + for _, migratedGR := range migrated { + if migratedGR == gr { + found = true + break + } + } + if !found { + return false, nil + } + } + return len(encryptedGRs) > 0, nil +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/operator.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/operator.go new file mode 100644 index 000000000..2ec91de0f --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/operator.go @@ -0,0 +1,142 @@ +package encryptionstatus + +import ( + "encoding/json" + "fmt" + "strings" + + operatorv1 "github.com/openshift/api/operator/v1" +) + +// interimHealthReport is the JSON shape published in KMSHealthReporter_* condition messages. +type interimHealthReport struct { + KEKID string `json:"kekID"` + KeyID string `json:"keyID"` + Status string `json:"status"` + LastChecked string `json:"lastChecked"` +} + +// encryptionStatusFromOperator reads structured encryption status from operator status. +// Interim conditions remain the source until operator API types expose EncryptionStatus. +func encryptionStatusFromOperator(status *operatorv1.OperatorStatus) APIServerEncryptionStatus { + _ = status + return APIServerEncryptionStatus{} +} + +// HealthReportsFromOperatorStatus returns health reports from structured status when present, +// otherwise parses interim KMSHealthReporter_* operator conditions. +func HealthReportsFromOperatorStatus(status *operatorv1.OperatorStatus) []KMSPluginHealthReport { + encryptionStatus := encryptionStatusFromOperator(status) + if len(encryptionStatus.HealthReports) > 0 { + return encryptionStatus.HealthReports + } + return healthReportsFromConditions(status) +} + +func healthReportsFromConditions(status *operatorv1.OperatorStatus) []KMSPluginHealthReport { + if status == nil { + return nil + } + + var reports []KMSPluginHealthReport + for _, condition := range status.Conditions { + if !strings.HasPrefix(condition.Type, KMSHealthReporterConditionPrefix) { + continue + } + nodeName := strings.TrimPrefix(condition.Type, KMSHealthReporterConditionPrefix) + parsed, err := parseInterimHealthMessage(condition.Message) + if err != nil { + continue + } + for _, entry := range parsed { + reports = append(reports, KMSPluginHealthReport{ + KeyID: entry.KeyID, + NodeName: nodeName, + KEKID: entry.KEKID, + Status: entry.Status, + // LastChecked left zero when only interim JSON timestamp is available without parsing. + }) + } + } + return reports +} + +func parseInterimHealthMessage(message string) ([]interimHealthReport, error) { + if len(message) == 0 { + return nil, nil + } + var parsed []interimHealthReport + if err := json.Unmarshal([]byte(message), &parsed); err != nil { + return nil, err + } + return parsed, nil +} + +// KeyRotationStatusFromOperatorStatus reads rotation history from structured status when present, +// otherwise from the interim EncryptionKeyRotationStatus condition. +func KeyRotationStatusFromOperatorStatus(status *operatorv1.OperatorStatus) ([]KMSPluginRotationStatus, error) { + encryptionStatus := encryptionStatusFromOperator(status) + if len(encryptionStatus.KeyRotationStatus) > 0 { + return encryptionStatus.KeyRotationStatus, nil + } + return keyRotationStatusFromCondition(status) +} + +func keyRotationStatusFromCondition(status *operatorv1.OperatorStatus) ([]KMSPluginRotationStatus, error) { + if status == nil { + return nil, nil + } + for _, condition := range status.Conditions { + if condition.Type != KeyRotationStatusConditionType { + continue + } + if len(condition.Message) == 0 { + return nil, nil + } + var rotations []KMSPluginRotationStatus + if err := json.Unmarshal([]byte(condition.Message), &rotations); err != nil { + return nil, fmt.Errorf("failed to parse %s condition: %w", KeyRotationStatusConditionType, err) + } + return rotations, nil + } + return nil, nil +} + +// SetKeyRotationStatusCondition returns an update func that stores keyRotationStatus in operator conditions +// until status.encryptionStatus is available on the operator API type. +func SetKeyRotationStatusCondition(rotations []KMSPluginRotationStatus) func(*operatorv1.OperatorStatus) error { + return func(status *operatorv1.OperatorStatus) error { + if status == nil { + return fmt.Errorf("operator status is nil") + } + message := "" + if len(rotations) > 0 { + bs, err := json.Marshal(rotations) + if err != nil { + return err + } + message = string(bs) + } + condition := operatorv1.OperatorCondition{ + Type: KeyRotationStatusConditionType, + Status: operatorv1.ConditionTrue, + Reason: KeyRotationStatusConditionReason, + Message: message, + } + setOperatorCondition(&status.Conditions, condition) + return nil + } +} + +func setOperatorCondition(conditions *[]operatorv1.OperatorCondition, newCondition operatorv1.OperatorCondition) { + if conditions == nil { + return + } + for i, existing := range *conditions { + if existing.Type == newCondition.Type { + (*conditions)[i] = newCondition + return + } + } + *conditions = append(*conditions, newCondition) +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/rotation.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/rotation.go new file mode 100644 index 000000000..07cefae72 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/rotation.go @@ -0,0 +1,101 @@ +package encryptionstatus + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// LatestCompletedRotationForKeyID returns the most recent completed entry for keyID. +// Rotations are stored newest-first. +func LatestCompletedRotationForKeyID(rotations []KMSPluginRotationStatus, keyID string) (KMSPluginRotationStatus, bool) { + for _, rotation := range rotations { + if rotation.KeyID != keyID { + continue + } + if rotation.MigrationFinishTime != nil { + return rotation, true + } + } + return KMSPluginRotationStatus{}, false +} + +// OpenRotation returns the in-progress rotation for keyID and kekID (no migrationFinishTime). +func OpenRotation(rotations []KMSPluginRotationStatus, keyID, kekID string) (KMSPluginRotationStatus, int, bool) { + for i, rotation := range rotations { + if rotation.KeyID != keyID || rotation.KEKID != kekID { + continue + } + if rotation.MigrationFinishTime == nil { + return rotation, i, true + } + } + return KMSPluginRotationStatus{}, -1, false +} + +// GetOrCreateOpenRotation ensures an open rotation entry exists for keyID and kekID with discoveryTime set. +func GetOrCreateOpenRotation(rotations []KMSPluginRotationStatus, keyID, kekID string, now metav1.Time) ([]KMSPluginRotationStatus, int) { + if idx := indexRotation(rotations, keyID, kekID); idx >= 0 { + if rotations[idx].MigrationFinishTime != nil { + rotations = prependRotation(rotations, newOpenRotation(keyID, kekID, now)) + return rotations, 0 + } + return SetDiscoveryTime(rotations, idx, now), idx + } + rotations = prependRotation(rotations, newOpenRotation(keyID, kekID, now)) + return rotations, 0 +} + +func newOpenRotation(keyID, kekID string, now metav1.Time) KMSPluginRotationStatus { + discoveryTime := now.DeepCopy() + return KMSPluginRotationStatus{ + KeyID: keyID, + KEKID: kekID, + DiscoveryTime: discoveryTime, + } +} + +// SetDiscoveryTime sets discoveryTime on the rotation at index when unset. +func SetDiscoveryTime(rotations []KMSPluginRotationStatus, index int, discoveryTime metav1.Time) []KMSPluginRotationStatus { + if index < 0 || index >= len(rotations) { + return rotations + } + if rotations[index].DiscoveryTime != nil { + return rotations + } + rotations[index].DiscoveryTime = discoveryTime.DeepCopy() + return rotations +} + +// SetMigrationStartTime sets migrationStartTime on the rotation at index. +func SetMigrationStartTime(rotations []KMSPluginRotationStatus, index int, startTime metav1.Time) []KMSPluginRotationStatus { + if index < 0 || index >= len(rotations) { + return rotations + } + rotations[index].MigrationStartTime = startTime.DeepCopy() + return rotations +} + +// SetMigrationFinishTime sets migrationFinishTime on the rotation at index. +func SetMigrationFinishTime(rotations []KMSPluginRotationStatus, index int, finishTime metav1.Time) []KMSPluginRotationStatus { + if index < 0 || index >= len(rotations) { + return rotations + } + rotations[index].MigrationFinishTime = finishTime.DeepCopy() + return rotations +} + +func indexRotation(rotations []KMSPluginRotationStatus, keyID, kekID string) int { + for i, rotation := range rotations { + if rotation.KeyID == keyID && rotation.KEKID == kekID { + return i + } + } + return -1 +} + +func prependRotation(rotations []KMSPluginRotationStatus, rotation KMSPluginRotationStatus) []KMSPluginRotationStatus { + rotations = append([]KMSPluginRotationStatus{rotation}, rotations...) + if len(rotations) > MaxKeyRotationStatusEntries { + rotations = rotations[:MaxKeyRotationStatusEntries] + } + return rotations +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/types.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/types.go new file mode 100644 index 000000000..96af1ed5f --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/types.go @@ -0,0 +1,44 @@ +package encryptionstatus + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// APIServerEncryptionStatus mirrors the future operator/config status field for KMS encryption. +type APIServerEncryptionStatus struct { + HealthReports []KMSPluginHealthReport `json:"healthReports,omitempty"` + KeyRotationStatus []KMSPluginRotationStatus `json:"keyRotationStatus,omitempty"` +} + +// KMSPluginHealthReport describes per-node KMS plugin health (filled by the health controller). +type KMSPluginHealthReport struct { + KeyID string `json:"keyId"` + NodeName string `json:"nodeName"` + KEKID string `json:"kekId,omitempty"` + Status string `json:"status"` + LastChecked metav1.Time `json:"lastChecked"` + Detail string `json:"detail,omitempty"` +} + +// KMSPluginRotationStatus tracks one KEK rotation episode on the operand operator. +type KMSPluginRotationStatus struct { + KeyID string `json:"keyId"` + KEKID string `json:"kekId"` + DiscoveryTime *metav1.Time `json:"discoveryTime,omitempty"` + MigrationStartTime *metav1.Time `json:"migrationStartTime,omitempty"` + MigrationFinishTime *metav1.Time `json:"migrationFinishTime,omitempty"` +} + +const ( + // MaxKeyRotationStatusEntries is the maximum number of rotation history entries kept in status. + MaxKeyRotationStatusEntries = 10 + + // KMSHealthReporterConditionPrefix is the interim condition type prefix used by the health controller. + KMSHealthReporterConditionPrefix = "KMSHealthReporter_" + + // KeyRotationStatusConditionType stores keyRotationStatus JSON until status.encryptionStatus is available on the operator API. + KeyRotationStatusConditionType = "EncryptionKeyRotationStatus" + + // KeyRotationStatusConditionReason is set when the condition carries the current rotation status snapshot. + KeyRotationStatusConditionReason = "AsExpected" +) diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go index 716764f08..271ffc9c4 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go @@ -203,6 +203,17 @@ func (m *MigratedGroupResources) HasResource(resource schema.GroupResource) bool return false } +// FindKeySecret returns the encryption key secret for keyID from a ListKeySecrets result. +func FindKeySecret(encryptionSecrets []*corev1.Secret, component, keyID string) *corev1.Secret { + name := fmt.Sprintf("encryption-key-%s-%s", component, keyID) + for _, secret := range encryptionSecrets { + if secret.Namespace == "openshift-config-managed" && secret.Name == name { + return secret + } + } + return nil +} + // ListKeySecrets returns the current key secrets from openshift-config-managed. func ListKeySecrets(ctx context.Context, secretClient corev1client.SecretsGetter, encryptionSecretSelector metav1.ListOptions) ([]*corev1.Secret, error) { encryptionSecretList, err := secretClient.Secrets("openshift-config-managed").List(ctx, encryptionSecretSelector) diff --git a/vendor/modules.txt b/vendor/modules.txt index 8b9f5681d..1ca73e664 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -428,6 +428,7 @@ github.com/openshift/library-go/pkg/operator/encryption/crypto github.com/openshift/library-go/pkg/operator/encryption/deployer github.com/openshift/library-go/pkg/operator/encryption/encoding github.com/openshift/library-go/pkg/operator/encryption/encryptiondata +github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle github.com/openshift/library-go/pkg/operator/encryption/observer github.com/openshift/library-go/pkg/operator/encryption/secrets @@ -1651,3 +1652,4 @@ sigs.k8s.io/structured-merge-diff/v6/value ## explicit; go 1.22 sigs.k8s.io/yaml # github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 +# github.com/openshift/library-go => github.com/tjungblu/library-go v0.0.0-20260608092146-53a048269fb0 From 907ed99084edc9ea3956c1ff327ddddd83fe0c2c Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Thu, 11 Jun 2026 14:40:19 +0200 Subject: [PATCH 2/2] NO-JIRA: bump(library-go): kms_rotation_annotation branch Co-Authored-By: Claude Opus 4.6 Signed-off-by: Thomas Jungblut --- go.mod | 2 +- go.sum | 6 +- .../pkg/operator/encryption/controllers.go | 10 +- .../encryption_rotation_controller.go | 293 ------------------ .../encryption/controllers/key_controller.go | 5 + .../controllers/kms_rotation_controller.go | 278 +++++++++++++++++ .../controllers/migration_controller.go | 83 ++++- .../encryption/encryptiondata/config.go | 19 -- .../encryptionstatus/convergence.go | 50 --- .../encryption/encryptionstatus/migration.go | 47 --- .../encryption/encryptionstatus/operator.go | 142 --------- .../encryption/encryptionstatus/rotation.go | 101 ------ .../encryption/encryptionstatus/types.go | 44 --- .../pkg/operator/encryption/kms/health/cmd.go | 110 +++++++ .../kms/health/configmap_reporter.go | 70 +++++ .../pkg/operator/encryption/secrets/kek.go | 58 ++++ .../operator/encryption/secrets/secrets.go | 21 +- .../pkg/operator/encryption/secrets/types.go | 15 + .../pkg/operator/encryption/state/types.go | 21 ++ .../encryption/statemachine/transition.go | 4 + vendor/modules.txt | 6 +- 21 files changed, 664 insertions(+), 721 deletions(-) delete mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/encryption_rotation_controller.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/kms_rotation_controller.go delete mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/convergence.go delete mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/migration.go delete mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/operator.go delete mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/rotation.go delete mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/types.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/cmd.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/configmap_reporter.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/kek.go diff --git a/go.mod b/go.mod index f0dc1b217..45650696f 100644 --- a/go.mod +++ b/go.mod @@ -134,4 +134,4 @@ require ( replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 -replace github.com/openshift/library-go => github.com/tjungblu/library-go v0.0.0-20260608092146-53a048269fb0 +replace github.com/openshift/library-go => github.com/tjungblu/library-go v0.0.0-20260611114644-374bd672722f diff --git a/go.sum b/go.sum index ca8652a99..e5cf865b8 100644 --- a/go.sum +++ b/go.sum @@ -152,8 +152,6 @@ github.com/openshift/build-machinery-go v0.0.0-20251023084048-5d77c1a5e5af h1:Ui github.com/openshift/build-machinery-go v0.0.0-20251023084048-5d77c1a5e5af/go.mod h1:8jcm8UPtg2mCAsxfqKil1xrmRMI3a+XU2TZ9fF8A7TE= github.com/openshift/client-go v0.0.0-20260512113608-deb4dc54551a h1:EKx2XhOKehd1C5ptY7IrLl4WV35E8kP0pRPnG5BUZXk= github.com/openshift/client-go v0.0.0-20260512113608-deb4dc54551a/go.mod h1:V933kvY/cb/Un7UCEOhXHUySNX327u7Epe8g9KNqg2Q= -github.com/openshift/library-go v0.0.0-20260611070125-7fd5f333df33 h1:k7zdFrfi8699bfWL+ykAn3GF7r2hzIFmMBM8onHrLY0= -github.com/openshift/library-go v0.0.0-20260611070125-7fd5f333df33/go.mod h1:/HBhy6jm/igWI3Y1vYFwFG3ZCcXmnNsKUT6VBpPyM9A= github.com/openshift/multi-operator-manager v0.0.0-20241205181422-20aa3906b99d h1:Rzx23P63JFNNz5D23ubhC0FCN5rK8CeJhKcq5QKcdyU= github.com/openshift/multi-operator-manager v0.0.0-20241205181422-20aa3906b99d/go.mod h1:iVi9Bopa5cLhjG5ie9DoZVVqkH8BGb1FQVTtecOLn4I= github.com/openshift/oauth-apiserver v0.0.0-20260430140618-160ac7fb4ea6 h1:WvXToDt/IVTXb4NxbqEjY0cuPpVadTK6ATu75mlVM/s= @@ -208,8 +206,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/tjungblu/library-go v0.0.0-20260608092146-53a048269fb0 h1:bXxmi7xEzASxdfffvAN1F8KV84ElCE1A4hXsOxhAbl0= -github.com/tjungblu/library-go v0.0.0-20260608092146-53a048269fb0/go.mod h1:/HBhy6jm/igWI3Y1vYFwFG3ZCcXmnNsKUT6VBpPyM9A= +github.com/tjungblu/library-go v0.0.0-20260611114644-374bd672722f h1:SzokGHqJd/T+U7h2bCJhCeWrMw8qascHjk6sCuL+8aQ= +github.com/tjungblu/library-go v0.0.0-20260611114644-374bd672722f/go.mod h1:/HBhy6jm/igWI3Y1vYFwFG3ZCcXmnNsKUT6VBpPyM9A= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go index 3ccd1078b..6bb04d7a7 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers.go @@ -19,6 +19,7 @@ import ( operatorv1helpers "github.com/openshift/library-go/pkg/operator/v1helpers" "github.com/openshift/library-go/pkg/operator/encryption/controllers" + kmshealth "github.com/openshift/library-go/pkg/operator/encryption/kms/health" "github.com/openshift/library-go/pkg/operator/encryption/secrets" "github.com/openshift/library-go/pkg/operator/encryption/statemachine" ) @@ -49,6 +50,11 @@ func NewControllers( return nil, err } + convergedKEKReporter := kmshealth.NewMOCK_ConfigMapConvergedKEKReporter( + kubeInformersForNamespaces.ConfigMapLister(), + "", + ) + // for testing resourceSyncer might be nil if resourceSyncer != nil { if err := resourceSyncer.SyncSecretConditionally( @@ -125,12 +131,12 @@ func NewControllers( encryptionSecretSelector, eventRecorder, ), - controllers.NewEncryptionRotationController( + controllers.NewKMSRotationController( component, provider, deployer, encryptionEnabledChecker.PreconditionFulfilled, - migrator, + convergedKEKReporter, operatorClient, apiServerInformer, kubeInformersForNamespaces, diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/encryption_rotation_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/encryption_rotation_controller.go deleted file mode 100644 index 8ccd89079..000000000 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/encryption_rotation_controller.go +++ /dev/null @@ -1,293 +0,0 @@ -package controllers - -import ( - "context" - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - corev1client "k8s.io/client-go/kubernetes/typed/core/v1" - "k8s.io/client-go/util/retry" - "k8s.io/klog/v2" - - configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" - - "github.com/openshift/library-go/pkg/controller/factory" - "github.com/openshift/library-go/pkg/operator/encryption/controllers/migrators" - "github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus" - "github.com/openshift/library-go/pkg/operator/encryption/secrets" - "github.com/openshift/library-go/pkg/operator/encryption/statemachine" - "github.com/openshift/library-go/pkg/operator/events" - operatorv1helpers "github.com/openshift/library-go/pkg/operator/v1helpers" -) - -const ( - encryptionRotationConvergenceDelay = 5 * time.Minute -) - -// encryptionRotationController orchestrates KMS KEK rotation by resetting migration state on the -// encryption key secret and recording rotation progress on operator status. -type encryptionRotationController struct { - instanceName string - controllerInstanceName string - operatorClient operatorv1helpers.OperatorClient - encryptionSecretSelector metav1.ListOptions - secretClient corev1client.SecretsGetter - deployer statemachine.Deployer - migrator migrators.Migrator - provider Provider - preconditionsFulfilledFn preconditionsFulfilled -} - -func NewEncryptionRotationController( - instanceName string, - provider Provider, - deployer statemachine.Deployer, - preconditionsFulfilledFn preconditionsFulfilled, - migrator migrators.Migrator, - operatorClient operatorv1helpers.OperatorClient, - apiServerConfigInformer configv1informers.APIServerInformer, - kubeInformersForNamespaces operatorv1helpers.KubeInformersForNamespaces, - secretClient corev1client.SecretsGetter, - encryptionSecretSelector metav1.ListOptions, - eventRecorder events.Recorder, -) factory.Controller { - c := &encryptionRotationController{ - instanceName: instanceName, - controllerInstanceName: factory.ControllerInstanceName(instanceName, "EncryptionRotation"), - operatorClient: operatorClient, - encryptionSecretSelector: encryptionSecretSelector, - secretClient: secretClient, - deployer: deployer, - migrator: migrator, - provider: provider, - preconditionsFulfilledFn: preconditionsFulfilledFn, - } - - return factory.New().ResyncEvery(time.Minute).WithSync(c.sync).WithControllerInstanceName(c.controllerInstanceName).WithInformers( - operatorClient.Informer(), - kubeInformersForNamespaces.InformersFor("openshift-config-managed").Core().V1().Secrets().Informer(), - apiServerConfigInformer.Informer(), - deployer, - ).ToController( - c.controllerInstanceName, - eventRecorder.WithComponentSuffix("encryption-rotation-controller"), - ) -} - -func (c *encryptionRotationController) sync(ctx context.Context, syncCtx factory.SyncContext) error { - if ready, err := shouldRunEncryptionController(c.operatorClient, c.preconditionsFulfilledFn, c.provider.ShouldRunEncryptionControllers); err != nil || !ready { - return err - } - return c.reconcile(ctx) -} - -func (c *encryptionRotationController) reconcile(ctx context.Context) error { - currentConfig, _, encryptionSecrets, _, err := statemachine.GetEncryptionConfigAndState( - ctx, c.deployer, c.secretClient, c.encryptionSecretSelector, c.provider.EncryptedGRs(), - ) - if err != nil { - return err - } - - if currentConfig == nil || !currentConfig.UsesKMS() { - return nil - } - - _, operatorStatus, _, err := c.operatorClient.GetOperatorState() - if err != nil { - return err - } - - healthReports := encryptionstatus.HealthReportsFromOperatorStatus(operatorStatus) - rotations, err := encryptionstatus.KeyRotationStatusFromOperatorStatus(operatorStatus) - if err != nil { - return err - } - - encryptedGRs := currentConfig.EncryptedGroupResources() - klog.Infof("%s: reconciling KMS key rotation (%d plugin(s), %d health report(s), %d rotation entr(ies))", - c.instanceName, len(currentConfig.KMSPlugins), len(healthReports), len(rotations)) - for keyID := range currentConfig.KMSPlugins { - writeKeySecret := secrets.FindKeySecret(encryptionSecrets, c.instanceName, keyID) - if writeKeySecret == nil { - klog.Infof("%s: no write key secret for KMS plugin keyID %q, skipping", c.instanceName, keyID) - continue - } - - rotations, err = c.reconcileKMSPlugin( - ctx, keyID, encryptedGRs, writeKeySecret, healthReports, rotations, - ) - if err != nil { - return err - } - } - - if err := c.persistRotations(ctx, rotations); err != nil { - return err - } - - return nil -} - -func (c *encryptionRotationController) reconcileKMSPlugin( - ctx context.Context, - keyID string, - encryptedGRs []schema.GroupResource, - writeKeySecret *corev1.Secret, - healthReports []encryptionstatus.KMSPluginHealthReport, - rotations []encryptionstatus.KMSPluginRotationStatus, -) ([]encryptionstatus.KMSPluginRotationStatus, error) { - // Check KEK convergence across all nodes for this keyID. - convergedKEKID, converged := encryptionstatus.ConvergedKEKForKeyID(healthReports, keyID) - if !converged { - klog.Infof("%s: KEK not yet converged for keyID %q, waiting for all nodes to report the same kekID", c.instanceName, keyID) - return rotations, nil - } - klog.Infof("%s: KEK converged for keyID %q: kekID=%q", c.instanceName, keyID, convergedKEKID) - - // If the converged kekID matches the last completed rotation, we are in steady state. - lastCompleted, hasCompleted := encryptionstatus.LatestCompletedRotationForKeyID(rotations, keyID) - if hasCompleted && convergedKEKID == lastCompleted.KEKID { - klog.V(4).Infof("%s: converged kekID %q matches last completed rotation for keyID %q, nothing to do", c.instanceName, convergedKEKID, keyID) - return rotations, nil - } - - // Ensure an open rotation entry exists with discoveryTime for the converged kekID. - now := metav1.Now() - rotations, openIdx := encryptionstatus.GetOrCreateOpenRotation(rotations, keyID, convergedKEKID, now) - klog.Infof("%s: tracking open rotation for keyID=%q kekID=%q (discoveryTime=%s)", - c.instanceName, keyID, convergedKEKID, rotations[openIdx].DiscoveryTime.Format(time.RFC3339)) - - // Mirror migration finish time from the write key secret annotations when all - // encrypted group resources have been migrated by the migration controller. - migrated, err := encryptionstatus.AllEncryptedGRsMigrated(writeKeySecret, encryptedGRs) - if err != nil { - return rotations, err - } - rotations = mirrorMigrationFinish(c.instanceName, rotations, openIdx, migrated, writeKeySecret) - - // Bootstrap: if no prior completed rotation exists, this is the initial provider - // migration driven by the migration controller. We only track convergence and mirror - // the finish time — never prune migrations or clear annotations. - if !hasCompleted { - klog.Infof("%s: bootstrap for keyID %q — no prior completed rotation, waiting for initial migration to complete", c.instanceName, keyID) - return rotations, nil - } - - // From here on a KEK change was detected (convergedKEKID != lastCompleted.KEKID). - // Check guards before starting a storage re-migration. - entry := rotations[openIdx] - - if entry.MigrationStartTime != nil { - klog.Infof("%s: rotation for kekID %q already started at %s, waiting for migration to complete", - c.instanceName, convergedKEKID, entry.MigrationStartTime.Format(time.RFC3339)) - return rotations, nil - } - - if entry.DiscoveryTime != nil && time.Since(entry.DiscoveryTime.Time) < encryptionRotationConvergenceDelay { - remaining := encryptionRotationConvergenceDelay - time.Since(entry.DiscoveryTime.Time) - klog.Infof("%s: rotation for kekID %q waiting for convergence delay (%s remaining)", - c.instanceName, convergedKEKID, remaining.Round(time.Second)) - return rotations, nil - } - - // All guards passed — start the rotation by pruning existing storage version - // migrations and clearing migration annotations on the write key secret so the - // migration controller picks up the work again. - klog.Infof("%s: starting storage re-migration for keyID=%q: kekID changed from %q to %q", - c.instanceName, keyID, lastCompleted.KEKID, convergedKEKID) - if err := c.startRotation(ctx, encryptedGRs, writeKeySecret); err != nil { - return rotations, err - } - - rotations = encryptionstatus.SetMigrationStartTime(rotations, openIdx, now) - klog.Infof("%s: rotation migration started for kekID %q at %s", c.instanceName, convergedKEKID, now.Format(time.RFC3339)) - return rotations, nil -} - -func (c *encryptionRotationController) startRotation(ctx context.Context, encryptedGRs []schema.GroupResource, secret *corev1.Secret) error { - for _, gr := range encryptedGRs { - klog.Infof("%s: pruning storage migration for %s", c.instanceName, gr) - if err := c.migrator.PruneMigration(gr); err != nil { - return err - } - } - klog.Infof("%s: clearing migration annotations on secret %s/%s", c.instanceName, secret.Namespace, secret.Name) - return c.clearMigrationAnnotations(ctx, secret) -} - -func mirrorMigrationFinish( - instanceName string, - rotations []encryptionstatus.KMSPluginRotationStatus, - idx int, - migrated bool, - secret *corev1.Secret, -) []encryptionstatus.KMSPluginRotationStatus { - if !migrated || idx < 0 || idx >= len(rotations) || rotations[idx].MigrationFinishTime != nil { - return rotations - } - finish, ok := migrationFinishTimeFromSecret(secret) - if !ok { - return rotations - } - entry := rotations[idx] - klog.Infof("%s: mirroring migrationFinishTime for keyID %q kekID %q from secret %s/%s at %s", - instanceName, entry.KeyID, entry.KEKID, secret.Namespace, secret.Name, finish.Format(time.RFC3339)) - return encryptionstatus.SetMigrationFinishTime(rotations, idx, finish) -} - -func migrationFinishTimeFromSecret(secret *corev1.Secret) (metav1.Time, bool) { - if secret == nil || secret.Annotations == nil { - return metav1.Time{}, false - } - raw, ok := secret.Annotations[secrets.EncryptionSecretMigratedTimestamp] - if !ok || raw == "" { - return metav1.Time{}, false - } - ts, err := time.Parse(time.RFC3339, raw) - if err != nil { - klog.Warningf("ignoring invalid %s annotation on secret %s/%s: %v", - secrets.EncryptionSecretMigratedTimestamp, secret.Namespace, secret.Name, err) - return metav1.Time{}, false - } - return metav1.NewTime(ts), true -} - -func (c *encryptionRotationController) clearMigrationAnnotations(ctx context.Context, secret *corev1.Secret) error { - return retry.RetryOnConflict(retry.DefaultBackoff, func() error { - current, err := c.secretClient.Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{}) - if err != nil { - return err - } - if current.Annotations == nil { - return nil - } - changed := false - if _, ok := current.Annotations[secrets.EncryptionSecretMigratedTimestamp]; ok { - delete(current.Annotations, secrets.EncryptionSecretMigratedTimestamp) - changed = true - } - if _, ok := current.Annotations[secrets.EncryptionSecretMigratedResources]; ok { - delete(current.Annotations, secrets.EncryptionSecretMigratedResources) - changed = true - } - if !changed { - return nil - } - _, err = c.secretClient.Secrets(current.Namespace).Update(ctx, current, metav1.UpdateOptions{}) - return err - }) -} - -func (c *encryptionRotationController) persistRotations(ctx context.Context, rotations []encryptionstatus.KMSPluginRotationStatus) error { - _, updated, err := operatorv1helpers.UpdateStatus(ctx, c.operatorClient, encryptionstatus.SetKeyRotationStatusCondition(rotations)) - if err != nil { - return err - } - if updated { - klog.Infof("%s: updated key rotation status (%d entries)", c.instanceName, len(rotations)) - } - return nil -} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go index af38a8d59..7482e6e75 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/key_controller.go @@ -391,6 +391,11 @@ func needsNewKey(grKeys state.GroupResourceState, currentMode state.Mode, extern return 0, "", false } + // KMS KEK rotation migration must complete before minting a new key + if latestKey.NeedsKekMigration() { + return 0, "", false + } + // if the most recent secret was encrypted in a mode different than the current mode, we need to generate a new key if latestKey.Mode != currentMode { return latestKeyID, "encryption-mode-changed", true diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/kms_rotation_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/kms_rotation_controller.go new file mode 100644 index 000000000..e1c5c1d1c --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/kms_rotation_controller.go @@ -0,0 +1,278 @@ +package controllers + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + + configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1" + "github.com/openshift/library-go/pkg/controller/factory" + "github.com/openshift/library-go/pkg/operator/encryption/secrets" + "github.com/openshift/library-go/pkg/operator/encryption/state" + "github.com/openshift/library-go/pkg/operator/encryption/statemachine" + "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/library-go/pkg/operator/resource/resourcehelper" + operatorv1helpers "github.com/openshift/library-go/pkg/operator/v1helpers" +) + +// ConvergedKEKReporter provides the cluster-converged KMS KEK identity from health aggregation. +type ConvergedKEKReporter interface { + ConvergedKekID() (kekID string, converged bool) +} + +type kmsRotationController struct { + instanceName string + controllerInstanceName string + operatorClient operatorv1helpers.OperatorClient + secretClient corev1client.SecretsGetter + encryptionSecretSelector metav1.ListOptions + deployer statemachine.Deployer + provider Provider + preconditionsFulfilledFn preconditionsFulfilled + convergedKEKReporter ConvergedKEKReporter + now func() time.Time +} + +func NewKMSRotationController( + instanceName string, + provider Provider, + deployer statemachine.Deployer, + preconditionsFulfilledFn preconditionsFulfilled, + convergedKEKReporter ConvergedKEKReporter, + operatorClient operatorv1helpers.OperatorClient, + apiServerConfigInformer configv1informers.APIServerInformer, + kubeInformersForNamespaces operatorv1helpers.KubeInformersForNamespaces, + secretClient corev1client.SecretsGetter, + encryptionSecretSelector metav1.ListOptions, + eventRecorder events.Recorder, +) factory.Controller { + c := &kmsRotationController{ + instanceName: instanceName, + controllerInstanceName: factory.ControllerInstanceName(instanceName, "EncryptionKMSRotation"), + operatorClient: operatorClient, + encryptionSecretSelector: encryptionSecretSelector, + secretClient: secretClient, + deployer: deployer, + provider: provider, + preconditionsFulfilledFn: preconditionsFulfilledFn, + convergedKEKReporter: convergedKEKReporter, + now: time.Now, + } + + return factory.New().ResyncEvery(time.Minute).WithSync(c.sync).WithControllerInstanceName(c.controllerInstanceName).WithInformers( + operatorClient.Informer(), + kubeInformersForNamespaces.InformersFor("openshift-config-managed").Core().V1().Secrets().Informer(), + kubeInformersForNamespaces.InformersFor("openshift-config").Core().V1().ConfigMaps().Informer(), + apiServerConfigInformer.Informer(), + deployer, + ).ToController( + c.controllerInstanceName, + eventRecorder.WithComponentSuffix("encryption-kms-rotation-controller"), + ) +} + +func (c *kmsRotationController) sync(ctx context.Context, syncCtx factory.SyncContext) error { + if ready, err := shouldRunEncryptionController(c.operatorClient, c.preconditionsFulfilledFn, c.provider.ShouldRunEncryptionControllers); err != nil || !ready { + return err + } + + _, _, encryptionSecrets, isTransitionalReason, err := statemachine.GetEncryptionConfigAndState( + ctx, c.deployer, c.secretClient, c.encryptionSecretSelector, c.provider.EncryptedGRs(), + ) + if err != nil { + return err + } + if len(isTransitionalReason) > 0 { + return nil + } + + writeKeySecret, writeKeyState, ok := latestKMSWriteKeySecret(encryptionSecrets) + if !ok { + return nil + } + + convergedKekID, converged := c.convergedKEKReporter.ConvergedKekID() + if !converged || convergedKekID == "" { + return c.updateWriteKeySecret(ctx, syncCtx, writeKeySecret, clearKekConvergenceAnnotations) + } + + return c.reconcileKekAnnotations(ctx, syncCtx, writeKeySecret, writeKeyState, convergedKekID, c.provider.EncryptedGRs()) +} + +func latestKMSWriteKeySecret(encryptionSecrets []*corev1.Secret) (*corev1.Secret, state.KeyState, bool) { + var latestSecret *corev1.Secret + var latestKey state.KeyState + var latestKeyID uint64 + for _, s := range encryptionSecrets { + ks, err := secrets.ToKeyState(s) + if err != nil || ks.Mode != state.KMS { + continue + } + keyID, valid := state.NameToKeyID(s.Name) + if !valid { + continue + } + if latestSecret == nil || keyID > latestKeyID { + latestSecret = s + latestKey = ks + latestKeyID = keyID + } + } + return latestSecret, latestKey, latestSecret != nil +} + +func (c *kmsRotationController) reconcileKekAnnotations( + ctx context.Context, + syncCtx factory.SyncContext, + writeKeySecret *corev1.Secret, + writeKeyState state.KeyState, + convergedKekID string, + encryptedGRs []schema.GroupResource, +) error { + kekMigration := secrets.KekMigrationFromSecret(writeKeySecret) + + // Bootstrap: initial migration complete, no kekId annotations yet. + if kekMigration.TargetKekID == "" && kekMigration.MigratedKekID == "" { + allMigrated, _, _ := state.MigratedFor(encryptedGRs, writeKeyState) + if !allMigrated { + return nil + } + return c.updateWriteKeySecret(ctx, syncCtx, writeKeySecret, func(s *corev1.Secret) (bool, error) { + return setKekBootstrapAnnotations(s, convergedKekID) + }) + } + + // Steady state or migration in flight: converged kekId matches current target. + if convergedKekID == kekMigration.TargetKekID { + if kekMigration.KekConvergedID != "" || !kekMigration.KekConvergedAt.IsZero() { + return c.updateWriteKeySecret(ctx, syncCtx, writeKeySecret, clearKekConvergenceAnnotations) + } + return nil + } + + // Candidate kekId differs from target: start or maintain the convergence clock. + if convergedKekID != kekMigration.KekConvergedID { + return c.updateWriteKeySecret(ctx, syncCtx, writeKeySecret, func(s *corev1.Secret) (bool, error) { + return setKekConvergenceClock(s, convergedKekID, c.now()) + }) + } + + if kekMigration.KekConvergedAt.IsZero() { + return c.updateWriteKeySecret(ctx, syncCtx, writeKeySecret, func(s *corev1.Secret) (bool, error) { + return setKekConvergenceClock(s, convergedKekID, c.now()) + }) + } + + if c.now().Sub(kekMigration.KekConvergedAt) >= secrets.KekConvergenceDelay { + return c.updateWriteKeySecret(ctx, syncCtx, writeKeySecret, func(s *corev1.Secret) (bool, error) { + return promoteConvergedKekToTarget(s, convergedKekID) + }) + } + + return nil +} + +type secretAnnotationMutator func(s *corev1.Secret) (changed bool, err error) + +func (c *kmsRotationController) updateWriteKeySecret(ctx context.Context, syncCtx factory.SyncContext, writeKeySecret *corev1.Secret, mutate secretAnnotationMutator) error { + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + s, err := c.secretClient.Secrets(writeKeySecret.Namespace).Get(ctx, writeKeySecret.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get key secret %s/%s: %w", writeKeySecret.Namespace, writeKeySecret.Name, err) + } + + changed, err := mutate(s) + if err != nil { + return err + } + if !changed { + return nil + } + + _, updateErr := c.secretClient.Secrets(s.Namespace).Update(ctx, s, metav1.UpdateOptions{}) + resourcehelper.ReportUpdateEvent(syncCtx.Recorder(), s, updateErr) + return updateErr + }) +} + +func setKekBootstrapAnnotations(s *corev1.Secret, kekID string) (bool, error) { + if kekID == "" { + return false, nil + } + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + if s.Annotations[secrets.EncryptionSecretTargetKekID] == kekID && + s.Annotations[secrets.EncryptionSecretMigratedKekID] == kekID { + return false, nil + } + s.Annotations[secrets.EncryptionSecretTargetKekID] = kekID + s.Annotations[secrets.EncryptionSecretMigratedKekID] = kekID + delete(s.Annotations, secrets.EncryptionSecretKekConvergedAt) + delete(s.Annotations, secrets.EncryptionSecretKekConvergedID) + klog.V(2).Infof("bootstrapped KMS kekId annotations on secret %s/%s to %q", s.Namespace, s.Name, kekID) + return true, nil +} + +func setKekConvergenceClock(s *corev1.Secret, kekID string, now time.Time) (bool, error) { + if kekID == "" { + return false, nil + } + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + changed := false + if s.Annotations[secrets.EncryptionSecretKekConvergedID] != kekID { + s.Annotations[secrets.EncryptionSecretKekConvergedID] = kekID + s.Annotations[secrets.EncryptionSecretKekConvergedAt] = now.Format(time.RFC3339) + changed = true + } else if s.Annotations[secrets.EncryptionSecretKekConvergedAt] == "" { + s.Annotations[secrets.EncryptionSecretKekConvergedAt] = now.Format(time.RFC3339) + changed = true + } + if changed { + klog.V(2).Infof("started KMS kekId convergence clock on secret %s/%s for candidate %q", s.Namespace, s.Name, kekID) + } + return changed, nil +} + +func promoteConvergedKekToTarget(s *corev1.Secret, kekID string) (bool, error) { + if kekID == "" { + return false, nil + } + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + if s.Annotations[secrets.EncryptionSecretTargetKekID] == kekID && + s.Annotations[secrets.EncryptionSecretKekConvergedID] == "" && + s.Annotations[secrets.EncryptionSecretKekConvergedAt] == "" { + return false, nil + } + s.Annotations[secrets.EncryptionSecretTargetKekID] = kekID + delete(s.Annotations, secrets.EncryptionSecretKekConvergedAt) + delete(s.Annotations, secrets.EncryptionSecretKekConvergedID) + klog.V(2).Infof("updated target-kek-id on secret %s/%s to %q after convergence delay", s.Namespace, s.Name, kekID) + return true, nil +} + +func clearKekConvergenceAnnotations(s *corev1.Secret) (bool, error) { + if s.Annotations == nil { + return false, nil + } + _, hasID := s.Annotations[secrets.EncryptionSecretKekConvergedID] + _, hasAt := s.Annotations[secrets.EncryptionSecretKekConvergedAt] + if !hasID && !hasAt { + return false, nil + } + delete(s.Annotations, secrets.EncryptionSecretKekConvergedID) + delete(s.Annotations, secrets.EncryptionSecretKekConvergedAt) + klog.V(4).Infof("cleared KMS kekId convergence clock on secret %s/%s", s.Namespace, s.Name) + return true, nil +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/migration_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/migration_controller.go index 7fc649f02..682fb8c57 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/migration_controller.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/controllers/migration_controller.go @@ -215,39 +215,57 @@ func (c *migrationController) migrateKeysIfNeededAndRevisionStable(ctx context.C // we never want to migrate during an intermediate state because that could lead to one API server // using a write key that another API server has not observed // this could lead to etcd storing data that not all API servers can decrypt + writeKeySecret, err := writeKeySecretForState(encryptionSecrets, currentState) + if err != nil { + return nil, err + } + needsKekMigration := secrets.NeedsKekMigration(writeKeySecret) + var errs []error + kekMigrationComplete := needsKekMigration for _, gr := range grs { grActualKeys := currentState[gr] if !grActualKeys.HasWriteKey() { + kekMigrationComplete = false continue // no write key to migrate to } - if alreadyMigrated, _, _ := state.MigratedFor([]schema.GroupResource{gr}, grActualKeys.WriteKey); alreadyMigrated { - continue + if !needsKekMigration { + if alreadyMigrated, _, _ := state.MigratedFor([]schema.GroupResource{gr}, grActualKeys.WriteKey); alreadyMigrated { + continue + } + } + + writeKeyForGR := grActualKeys.WriteKey.Key.Name + if needsKekMigration { + writeKeyForGR = secrets.MigrationWriteKey(grActualKeys.WriteKey.Key.Name, writeKeySecret) } // idem-potent migration start - finished, result, when, err := c.migrator.EnsureMigration(gr, grActualKeys.WriteKey.Key.Name) + finished, result, when, err := c.migrator.EnsureMigration(gr, writeKeyForGR) if err == nil && finished && result != nil && time.Since(when) > migrationRetryDuration { // last migration error is far enough ago. Prune and retry. if err := c.migrator.PruneMigration(gr); err != nil { errs = append(errs, err) continue } - finished, result, when, err = c.migrator.EnsureMigration(gr, grActualKeys.WriteKey.Key.Name) + finished, result, when, err = c.migrator.EnsureMigration(gr, writeKeyForGR) } if err != nil { errs = append(errs, err) + kekMigrationComplete = false continue } if finished && result != nil { errs = append(errs, result) + kekMigrationComplete = false continue } if !finished { migratingResources = append(migratingResources, gr) + kekMigrationComplete = false continue } @@ -279,13 +297,70 @@ func (c *migrationController) migrateKeysIfNeededAndRevisionStable(ctx context.C return updateErr }); err != nil { errs = append(errs, err) + kekMigrationComplete = false continue } } + if kekMigrationComplete && writeKeySecret != nil { + if err := c.setMigratedKekID(ctx, syncContext, writeKeySecret); err != nil { + errs = append(errs, err) + } + } + return migratingResources, errors.NewAggregate(errs) } +func writeKeySecretForState(encryptionSecrets []*corev1.Secret, currentState map[schema.GroupResource]state.GroupResourceState) (*corev1.Secret, error) { + var writeKey state.KeyState + for _, grState := range currentState { + if grState.HasWriteKey() { + writeKey = grState.WriteKey + break + } + } + if len(writeKey.Key.Name) == 0 { + return nil, nil + } + + for _, s := range encryptionSecrets { + keyID, valid := state.NameToKeyID(s.Name) + if !valid { + continue + } + writeKeyID, ok := state.NameToKeyID(writeKey.Key.Name) + if !ok || keyID != writeKeyID { + continue + } + return s, nil + } + return nil, fmt.Errorf("write key secret for key ID %s not found", writeKey.Key.Name) +} + +func (c *migrationController) setMigratedKekID(ctx context.Context, syncContext factory.SyncContext, writeKeySecret *corev1.Secret) error { + targetKekID := writeKeySecret.Annotations[secrets.EncryptionSecretTargetKekID] + if targetKekID == "" { + return nil + } + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + s, err := c.secretClient.Secrets(writeKeySecret.Namespace).Get(ctx, writeKeySecret.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get key secret %s/%s: %v", writeKeySecret.Namespace, writeKeySecret.Name, err) + } + if s.Annotations[secrets.EncryptionSecretMigratedKekID] == targetKekID { + return nil + } + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + s.Annotations[secrets.EncryptionSecretMigratedKekID] = targetKekID + + _, updateErr := c.secretClient.Secrets(s.Namespace).Update(ctx, s, metav1.UpdateOptions{}) + resourcehelper.ReportUpdateEvent(syncContext.Recorder(), s, updateErr) + return updateErr + }) +} + func setResourceMigrated(gr schema.GroupResource, s *corev1.Secret) (bool, error) { migratedGRs := secrets.MigratedGroupResources{} if existing, found := s.Annotations[secrets.EncryptionSecretMigratedResources]; found { diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/config.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/config.go index 8bc6ba0a2..02c94c23f 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/config.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptiondata/config.go @@ -92,25 +92,6 @@ func (c *Config) HasEncryptionConfiguration() bool { return c != nil && c.Encryption != nil } -// UsesKMS returns whether the deployed encryption configuration includes KMS plugins. -func (c *Config) UsesKMS() bool { - return c != nil && len(c.KMSPlugins) > 0 -} - -// EncryptedGroupResources returns group resources from the deployed encryption configuration. -func (c *Config) EncryptedGroupResources() []schema.GroupResource { - if !c.HasEncryptionConfiguration() { - return nil - } - grs := make([]schema.GroupResource, 0, len(c.Encryption.Resources)) - for _, resourceConfig := range c.Encryption.Resources { - for _, resource := range resourceConfig.Resources { - grs = append(grs, schema.ParseGroupResource(resource)) - } - } - return grs -} - // FromEncryptionState converts encryption state to Config. func FromEncryptionState(encryptionState map[schema.GroupResource]state.GroupResourceState) (*Config, error) { resourceConfigs := make([]apiserverconfigv1.ResourceConfiguration, 0, len(encryptionState)) diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/convergence.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/convergence.go deleted file mode 100644 index d87545553..000000000 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/convergence.go +++ /dev/null @@ -1,50 +0,0 @@ -package encryptionstatus - -import ( - "strings" -) - -// KEKByKeyID groups observed kekIds per plugin keyId across health reports. -// Only healthy reports with non-empty keyId and kekId are included. -func KEKByKeyID(reports []KMSPluginHealthReport) map[string][]string { - result := map[string][]string{} - for _, report := range reports { - if !isHealthyReport(report) { - continue - } - result[report.KeyID] = append(result[report.KeyID], report.KEKID) - } - return result -} - -// ConvergedKEKForKeyID returns the unanimous kekId for keyID when every healthy report for that keyId agrees. -func ConvergedKEKForKeyID(reports []KMSPluginHealthReport, keyID string) (kekID string, ok bool) { - if keyID == "" { - return "", false - } - - byKeyID := KEKByKeyID(reports) - kekIDs, found := byKeyID[keyID] - if !found || len(kekIDs) == 0 { - return "", false - } - - uniq := map[string]struct{}{} - for _, id := range kekIDs { - if id == "" { - return "", false - } - uniq[id] = struct{}{} - } - if len(uniq) != 1 { - return "", false - } - for id := range uniq { - return id, true - } - return "", false -} - -func isHealthyReport(report KMSPluginHealthReport) bool { - return strings.EqualFold(report.Status, "healthy") -} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/migration.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/migration.go deleted file mode 100644 index 1fe6e57c3..000000000 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/migration.go +++ /dev/null @@ -1,47 +0,0 @@ -package encryptionstatus - -import ( - "encoding/json" - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - - "github.com/openshift/library-go/pkg/operator/encryption/secrets" -) - -func parseMigratedResources(secret *corev1.Secret) ([]schema.GroupResource, error) { - if secret == nil || secret.Annotations == nil { - return nil, nil - } - raw, ok := secret.Annotations[secrets.EncryptionSecretMigratedResources] - if !ok || len(raw) == 0 { - return nil, nil - } - migrated := secrets.MigratedGroupResources{} - if err := json.Unmarshal([]byte(raw), &migrated); err != nil { - return nil, fmt.Errorf("invalid %s annotation: %w", secrets.EncryptionSecretMigratedResources, err) - } - return migrated.Resources, nil -} - -// AllEncryptedGRsMigrated returns true when every encrypted GR is listed in migrated-resources. -func AllEncryptedGRsMigrated(secret *corev1.Secret, encryptedGRs []schema.GroupResource) (bool, error) { - migrated, err := parseMigratedResources(secret) - if err != nil { - return false, err - } - for _, gr := range encryptedGRs { - found := false - for _, migratedGR := range migrated { - if migratedGR == gr { - found = true - break - } - } - if !found { - return false, nil - } - } - return len(encryptedGRs) > 0, nil -} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/operator.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/operator.go deleted file mode 100644 index 2ec91de0f..000000000 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/operator.go +++ /dev/null @@ -1,142 +0,0 @@ -package encryptionstatus - -import ( - "encoding/json" - "fmt" - "strings" - - operatorv1 "github.com/openshift/api/operator/v1" -) - -// interimHealthReport is the JSON shape published in KMSHealthReporter_* condition messages. -type interimHealthReport struct { - KEKID string `json:"kekID"` - KeyID string `json:"keyID"` - Status string `json:"status"` - LastChecked string `json:"lastChecked"` -} - -// encryptionStatusFromOperator reads structured encryption status from operator status. -// Interim conditions remain the source until operator API types expose EncryptionStatus. -func encryptionStatusFromOperator(status *operatorv1.OperatorStatus) APIServerEncryptionStatus { - _ = status - return APIServerEncryptionStatus{} -} - -// HealthReportsFromOperatorStatus returns health reports from structured status when present, -// otherwise parses interim KMSHealthReporter_* operator conditions. -func HealthReportsFromOperatorStatus(status *operatorv1.OperatorStatus) []KMSPluginHealthReport { - encryptionStatus := encryptionStatusFromOperator(status) - if len(encryptionStatus.HealthReports) > 0 { - return encryptionStatus.HealthReports - } - return healthReportsFromConditions(status) -} - -func healthReportsFromConditions(status *operatorv1.OperatorStatus) []KMSPluginHealthReport { - if status == nil { - return nil - } - - var reports []KMSPluginHealthReport - for _, condition := range status.Conditions { - if !strings.HasPrefix(condition.Type, KMSHealthReporterConditionPrefix) { - continue - } - nodeName := strings.TrimPrefix(condition.Type, KMSHealthReporterConditionPrefix) - parsed, err := parseInterimHealthMessage(condition.Message) - if err != nil { - continue - } - for _, entry := range parsed { - reports = append(reports, KMSPluginHealthReport{ - KeyID: entry.KeyID, - NodeName: nodeName, - KEKID: entry.KEKID, - Status: entry.Status, - // LastChecked left zero when only interim JSON timestamp is available without parsing. - }) - } - } - return reports -} - -func parseInterimHealthMessage(message string) ([]interimHealthReport, error) { - if len(message) == 0 { - return nil, nil - } - var parsed []interimHealthReport - if err := json.Unmarshal([]byte(message), &parsed); err != nil { - return nil, err - } - return parsed, nil -} - -// KeyRotationStatusFromOperatorStatus reads rotation history from structured status when present, -// otherwise from the interim EncryptionKeyRotationStatus condition. -func KeyRotationStatusFromOperatorStatus(status *operatorv1.OperatorStatus) ([]KMSPluginRotationStatus, error) { - encryptionStatus := encryptionStatusFromOperator(status) - if len(encryptionStatus.KeyRotationStatus) > 0 { - return encryptionStatus.KeyRotationStatus, nil - } - return keyRotationStatusFromCondition(status) -} - -func keyRotationStatusFromCondition(status *operatorv1.OperatorStatus) ([]KMSPluginRotationStatus, error) { - if status == nil { - return nil, nil - } - for _, condition := range status.Conditions { - if condition.Type != KeyRotationStatusConditionType { - continue - } - if len(condition.Message) == 0 { - return nil, nil - } - var rotations []KMSPluginRotationStatus - if err := json.Unmarshal([]byte(condition.Message), &rotations); err != nil { - return nil, fmt.Errorf("failed to parse %s condition: %w", KeyRotationStatusConditionType, err) - } - return rotations, nil - } - return nil, nil -} - -// SetKeyRotationStatusCondition returns an update func that stores keyRotationStatus in operator conditions -// until status.encryptionStatus is available on the operator API type. -func SetKeyRotationStatusCondition(rotations []KMSPluginRotationStatus) func(*operatorv1.OperatorStatus) error { - return func(status *operatorv1.OperatorStatus) error { - if status == nil { - return fmt.Errorf("operator status is nil") - } - message := "" - if len(rotations) > 0 { - bs, err := json.Marshal(rotations) - if err != nil { - return err - } - message = string(bs) - } - condition := operatorv1.OperatorCondition{ - Type: KeyRotationStatusConditionType, - Status: operatorv1.ConditionTrue, - Reason: KeyRotationStatusConditionReason, - Message: message, - } - setOperatorCondition(&status.Conditions, condition) - return nil - } -} - -func setOperatorCondition(conditions *[]operatorv1.OperatorCondition, newCondition operatorv1.OperatorCondition) { - if conditions == nil { - return - } - for i, existing := range *conditions { - if existing.Type == newCondition.Type { - (*conditions)[i] = newCondition - return - } - } - *conditions = append(*conditions, newCondition) -} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/rotation.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/rotation.go deleted file mode 100644 index 07cefae72..000000000 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/rotation.go +++ /dev/null @@ -1,101 +0,0 @@ -package encryptionstatus - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// LatestCompletedRotationForKeyID returns the most recent completed entry for keyID. -// Rotations are stored newest-first. -func LatestCompletedRotationForKeyID(rotations []KMSPluginRotationStatus, keyID string) (KMSPluginRotationStatus, bool) { - for _, rotation := range rotations { - if rotation.KeyID != keyID { - continue - } - if rotation.MigrationFinishTime != nil { - return rotation, true - } - } - return KMSPluginRotationStatus{}, false -} - -// OpenRotation returns the in-progress rotation for keyID and kekID (no migrationFinishTime). -func OpenRotation(rotations []KMSPluginRotationStatus, keyID, kekID string) (KMSPluginRotationStatus, int, bool) { - for i, rotation := range rotations { - if rotation.KeyID != keyID || rotation.KEKID != kekID { - continue - } - if rotation.MigrationFinishTime == nil { - return rotation, i, true - } - } - return KMSPluginRotationStatus{}, -1, false -} - -// GetOrCreateOpenRotation ensures an open rotation entry exists for keyID and kekID with discoveryTime set. -func GetOrCreateOpenRotation(rotations []KMSPluginRotationStatus, keyID, kekID string, now metav1.Time) ([]KMSPluginRotationStatus, int) { - if idx := indexRotation(rotations, keyID, kekID); idx >= 0 { - if rotations[idx].MigrationFinishTime != nil { - rotations = prependRotation(rotations, newOpenRotation(keyID, kekID, now)) - return rotations, 0 - } - return SetDiscoveryTime(rotations, idx, now), idx - } - rotations = prependRotation(rotations, newOpenRotation(keyID, kekID, now)) - return rotations, 0 -} - -func newOpenRotation(keyID, kekID string, now metav1.Time) KMSPluginRotationStatus { - discoveryTime := now.DeepCopy() - return KMSPluginRotationStatus{ - KeyID: keyID, - KEKID: kekID, - DiscoveryTime: discoveryTime, - } -} - -// SetDiscoveryTime sets discoveryTime on the rotation at index when unset. -func SetDiscoveryTime(rotations []KMSPluginRotationStatus, index int, discoveryTime metav1.Time) []KMSPluginRotationStatus { - if index < 0 || index >= len(rotations) { - return rotations - } - if rotations[index].DiscoveryTime != nil { - return rotations - } - rotations[index].DiscoveryTime = discoveryTime.DeepCopy() - return rotations -} - -// SetMigrationStartTime sets migrationStartTime on the rotation at index. -func SetMigrationStartTime(rotations []KMSPluginRotationStatus, index int, startTime metav1.Time) []KMSPluginRotationStatus { - if index < 0 || index >= len(rotations) { - return rotations - } - rotations[index].MigrationStartTime = startTime.DeepCopy() - return rotations -} - -// SetMigrationFinishTime sets migrationFinishTime on the rotation at index. -func SetMigrationFinishTime(rotations []KMSPluginRotationStatus, index int, finishTime metav1.Time) []KMSPluginRotationStatus { - if index < 0 || index >= len(rotations) { - return rotations - } - rotations[index].MigrationFinishTime = finishTime.DeepCopy() - return rotations -} - -func indexRotation(rotations []KMSPluginRotationStatus, keyID, kekID string) int { - for i, rotation := range rotations { - if rotation.KeyID == keyID && rotation.KEKID == kekID { - return i - } - } - return -1 -} - -func prependRotation(rotations []KMSPluginRotationStatus, rotation KMSPluginRotationStatus) []KMSPluginRotationStatus { - rotations = append([]KMSPluginRotationStatus{rotation}, rotations...) - if len(rotations) > MaxKeyRotationStatusEntries { - rotations = rotations[:MaxKeyRotationStatusEntries] - } - return rotations -} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/types.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/types.go deleted file mode 100644 index 96af1ed5f..000000000 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus/types.go +++ /dev/null @@ -1,44 +0,0 @@ -package encryptionstatus - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// APIServerEncryptionStatus mirrors the future operator/config status field for KMS encryption. -type APIServerEncryptionStatus struct { - HealthReports []KMSPluginHealthReport `json:"healthReports,omitempty"` - KeyRotationStatus []KMSPluginRotationStatus `json:"keyRotationStatus,omitempty"` -} - -// KMSPluginHealthReport describes per-node KMS plugin health (filled by the health controller). -type KMSPluginHealthReport struct { - KeyID string `json:"keyId"` - NodeName string `json:"nodeName"` - KEKID string `json:"kekId,omitempty"` - Status string `json:"status"` - LastChecked metav1.Time `json:"lastChecked"` - Detail string `json:"detail,omitempty"` -} - -// KMSPluginRotationStatus tracks one KEK rotation episode on the operand operator. -type KMSPluginRotationStatus struct { - KeyID string `json:"keyId"` - KEKID string `json:"kekId"` - DiscoveryTime *metav1.Time `json:"discoveryTime,omitempty"` - MigrationStartTime *metav1.Time `json:"migrationStartTime,omitempty"` - MigrationFinishTime *metav1.Time `json:"migrationFinishTime,omitempty"` -} - -const ( - // MaxKeyRotationStatusEntries is the maximum number of rotation history entries kept in status. - MaxKeyRotationStatusEntries = 10 - - // KMSHealthReporterConditionPrefix is the interim condition type prefix used by the health controller. - KMSHealthReporterConditionPrefix = "KMSHealthReporter_" - - // KeyRotationStatusConditionType stores keyRotationStatus JSON until status.encryptionStatus is available on the operator API. - KeyRotationStatusConditionType = "EncryptionKeyRotationStatus" - - // KeyRotationStatusConditionReason is set when the condition carries the current rotation status snapshot. - KeyRotationStatusConditionReason = "AsExpected" -) diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/cmd.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/cmd.go new file mode 100644 index 000000000..d6733848c --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/cmd.go @@ -0,0 +1,110 @@ +package health + +import ( + "context" + "fmt" + "regexp" + "time" + + "github.com/openshift/library-go/pkg/operator/v1helpers" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" +) + +// kmsSocketPattern matches the socket path each co-located KMSv2 plugin is +// mounted at, e.g. unix:///var/run/kmsplugin/kms-1.sock. +var kmsSocketPattern = regexp.MustCompile(`^unix:///var/run/kmsplugin/kms-\d+\.sock$`) + +// options' flag-bound fields are exported so the struct can be logged as a +// whole via klog.InfoS, which JSON-marshals its values. +type options struct { + KMSSockets []string + Interval time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + NodeName string + Kubeconfig string + + newOperatorClient func(*rest.Config) (v1helpers.OperatorClient, error) +} + +func NewCommand(ctx context.Context, newOperatorClient func(*rest.Config) (v1helpers.OperatorClient, error)) *cobra.Command { + o := &options{ + newOperatorClient: newOperatorClient, + } + + cmd := &cobra.Command{ + Use: "kms-health-reporter", + Short: "Observes co-located KMSv2 plugins and publishes status as an OperatorCondition.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.validate(); err != nil { + return err + } + return o.run() + }, + } + o.addFlags(cmd.Flags()) + return cmd +} + +func (o *options) addFlags(fs *pflag.FlagSet) { + fs.StringSliceVar(&o.KMSSockets, "kms-sockets", nil, "KMS plugin endpoints in unix:// URI format (e.g. unix:///var/run/kmsplugin/kms-1.sock)") + fs.DurationVar(&o.Interval, "interval", 30*time.Second, "cadence between probe+emit cycles") + fs.DurationVar(&o.ReadTimeout, "read-timeout", 5*time.Second, "deadline for each Status RPC") + fs.DurationVar(&o.WriteTimeout, "write-timeout", 10*time.Second, "deadline for each condition update") + fs.StringVar(&o.NodeName, "node-name", "", "node name recorded in the condition used to help to identify the origin") + fs.StringVar(&o.Kubeconfig, "kubeconfig", "", "path to a kubeconfig; empty uses in-cluster config") +} + +func (o *options) validate() error { + if len(o.KMSSockets) == 0 { + return fmt.Errorf("--kms-sockets is required, at least one") + } + for _, s := range o.KMSSockets { + if !kmsSocketPattern.MatchString(s) { + return fmt.Errorf("--kms-sockets entry %q must match %s", s, kmsSocketPattern) + } + } + + if o.Interval <= 0 { + return fmt.Errorf("--interval must be positive") + } + if o.ReadTimeout <= 0 { + return fmt.Errorf("--read-timeout must be positive") + } + if o.WriteTimeout <= 0 { + return fmt.Errorf("--write-timeout must be positive") + } + if o.NodeName == "" { + return fmt.Errorf("--node-name is required") + } + + return nil +} + +func (o *options) run() error { + cfg, err := buildRESTConfig(o.Kubeconfig) + if err != nil { + return fmt.Errorf("build rest config: %w", err) + } + + if _, err := o.newOperatorClient(cfg); err != nil { + return fmt.Errorf("build operator client: %w", err) + } + + klog.InfoS("kms-health-reporter starting", "config", o) + + return nil +} + +func buildRESTConfig(kubeconfig string) (*rest.Config, error) { + if kubeconfig != "" { + return clientcmd.BuildConfigFromFlags("", kubeconfig) + } + return rest.InClusterConfig() +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/configmap_reporter.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/configmap_reporter.go new file mode 100644 index 000000000..6549b61e6 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/health/configmap_reporter.go @@ -0,0 +1,70 @@ +package health + +import ( + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/klog/v2" +) + +const ( + // ConvergedKekConfigMapNamespace is where the mock converged-kek ConfigMap lives. + ConvergedKekConfigMapNamespace = "openshift-config" + // DefaultConvergedKekConfigMapName is the default ConfigMap name for mock KEK health input. + DefaultConvergedKekConfigMapName = "encryption-kms-converged-kek" + // ConvergedKekConfigMapDataKeyKekID holds the cluster-converged KMS kekId. + ConvergedKekConfigMapDataKeyKekID = "converged-kek-id" + // ConvergedKekConfigMapDataKeyConverged is an optional "true"/"false" override. + // When omitted, a non-empty converged-kek-id is treated as converged. + ConvergedKekConfigMapDataKeyConverged = "converged" +) + +// MOCK_ConfigMapConvergedKEKReporter reads cluster-converged kekId from a ConfigMap in openshift-config. +// It is intended for development and testing until kms-health-reporter publishes real health input. +type MOCK_ConfigMapConvergedKEKReporter struct { + lister corev1listers.ConfigMapLister + namespace string + name string +} + +// NewMOCK_ConfigMapConvergedKEKReporter returns a mock reporter backed by the named ConfigMap. +// An empty name uses DefaultConvergedKekConfigMapName. +func NewMOCK_ConfigMapConvergedKEKReporter(lister corev1listers.ConfigMapLister, name string) *MOCK_ConfigMapConvergedKEKReporter { + if name == "" { + name = DefaultConvergedKekConfigMapName + } + return &MOCK_ConfigMapConvergedKEKReporter{ + lister: lister, + namespace: ConvergedKekConfigMapNamespace, + name: name, + } +} + +func (r *MOCK_ConfigMapConvergedKEKReporter) ConvergedKekID() (string, bool) { + cm, err := r.lister.ConfigMaps(r.namespace).Get(r.name) + if err != nil { + klog.V(4).InfoS("converged kek configmap not available", "namespace", r.namespace, "name", r.name, "err", err) + return "", false + } + return ConvergedKekFromConfigMap(cm) +} + +// ConvergedKekFromConfigMap parses mock health input from a ConfigMap. +func ConvergedKekFromConfigMap(cm *corev1.ConfigMap) (kekID string, converged bool) { + if cm == nil || cm.Data == nil { + return "", false + } + kekID = strings.TrimSpace(cm.Data[ConvergedKekConfigMapDataKeyKekID]) + if kekID == "" { + return "", false + } + if v, ok := cm.Data[ConvergedKekConfigMapDataKeyConverged]; ok { + parsed, err := strconv.ParseBool(strings.TrimSpace(v)) + if err != nil || !parsed { + return "", false + } + } + return kekID, true +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/kek.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/kek.go new file mode 100644 index 000000000..2e2a8a729 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/kek.go @@ -0,0 +1,58 @@ +package secrets + +import ( + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" +) + +// KekMigrationState holds KMS KEK rotation annotations from a write-key secret. +type KekMigrationState struct { + TargetKekID string + MigratedKekID string + KekConvergedAt time.Time + KekConvergedID string +} + +// KekMigrationFromSecret parses KMS KEK rotation annotations from a secret. +func KekMigrationFromSecret(s *corev1.Secret) KekMigrationState { + if s == nil || s.Annotations == nil { + return KekMigrationState{} + } + state := KekMigrationState{ + TargetKekID: s.Annotations[EncryptionSecretTargetKekID], + MigratedKekID: s.Annotations[EncryptionSecretMigratedKekID], + KekConvergedID: s.Annotations[EncryptionSecretKekConvergedID], + } + if v, ok := s.Annotations[EncryptionSecretKekConvergedAt]; ok && len(v) > 0 { + if ts, err := time.Parse(time.RFC3339, v); err == nil { + state.KekConvergedAt = ts + } + } + return state +} + +// NeedsKekMigration reports whether target-kek-id is set and differs from migrated-kek-id. +func NeedsKekMigration(s *corev1.Secret) bool { + if s == nil || s.Annotations == nil { + return false + } + target := s.Annotations[EncryptionSecretTargetKekID] + if target == "" { + return false + } + return target != s.Annotations[EncryptionSecretMigratedKekID] +} + +// MigrationWriteKey returns the migrator write-key identity for the given key name. +// When target-kek-id is set, the format is {keyName}-{kekId}; otherwise {keyName}. +func MigrationWriteKey(keyName string, s *corev1.Secret) string { + if s == nil || s.Annotations == nil { + return keyName + } + if target := s.Annotations[EncryptionSecretTargetKekID]; target != "" { + return fmt.Sprintf("%s-%s", keyName, target) + } + return keyName +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go index 271ffc9c4..7db4404ae 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/secrets.go @@ -60,6 +60,16 @@ func ToKeyState(s *corev1.Secret) (state.KeyState, error) { key.ExternalReason = v } + kekMigration := KekMigrationFromSecret(s) + if kekMigration.TargetKekID != "" || kekMigration.MigratedKekID != "" || kekMigration.KekConvergedID != "" || !kekMigration.KekConvergedAt.IsZero() { + key.KekMigration = &state.KekMigrationState{ + TargetKekID: kekMigration.TargetKekID, + MigratedKekID: kekMigration.MigratedKekID, + KekConvergedAt: kekMigration.KekConvergedAt, + KekConvergedID: kekMigration.KekConvergedID, + } + } + keyMode := state.Mode(s.Annotations[encryptionSecretMode]) switch keyMode { case state.AESCBC, state.AESGCM, state.SecretBox, state.Identity: @@ -203,17 +213,6 @@ func (m *MigratedGroupResources) HasResource(resource schema.GroupResource) bool return false } -// FindKeySecret returns the encryption key secret for keyID from a ListKeySecrets result. -func FindKeySecret(encryptionSecrets []*corev1.Secret, component, keyID string) *corev1.Secret { - name := fmt.Sprintf("encryption-key-%s-%s", component, keyID) - for _, secret := range encryptionSecrets { - if secret.Namespace == "openshift-config-managed" && secret.Name == name { - return secret - } - } - return nil -} - // ListKeySecrets returns the current key secrets from openshift-config-managed. func ListKeySecrets(ctx context.Context, secretClient corev1client.SecretsGetter, encryptionSecretSelector metav1.ListOptions) ([]*corev1.Secret, error) { encryptionSecretList, err := secretClient.Secrets("openshift-config-managed").List(ctx, encryptionSecretSelector) diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go index 5a49727cf..ff9f7eee2 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/secrets/types.go @@ -1,6 +1,8 @@ package secrets import ( + "time" + "k8s.io/apimachinery/pkg/runtime/schema" ) @@ -67,8 +69,21 @@ const ( // values fetched from the referenced configmap in openshift-config. The full data key is // constructed as prefix + configMapName + separator + dataKey. encryptionSecretKMSConfigMapDataPrefix = "encryption.apiserver.operator.openshift.io-kms-plugin-configmap-" + + // EncryptionSecretTargetKekID is the target KMS KEK identity to migrate toward. + EncryptionSecretTargetKekID = "encryption.apiserver.operator.openshift.io/target-kek-id" + // EncryptionSecretMigratedKekID is the last fully migrated KMS KEK identity. + EncryptionSecretMigratedKekID = "encryption.apiserver.operator.openshift.io/migrated-kek-id" + // EncryptionSecretKekConvergedAt records when a candidate kekId first achieved cluster convergence (RFC3339). + EncryptionSecretKekConvergedAt = "encryption.apiserver.operator.openshift.io/kek-converged-at" + // EncryptionSecretKekConvergedID is the candidate kekId the kek-converged-at timestamp belongs to. + EncryptionSecretKekConvergedID = "encryption.apiserver.operator.openshift.io/kek-converged-id" ) +// KekConvergenceDelay is the minimum duration a candidate kekId must remain cluster-converged +// before target-kek-id is updated to a new value (KEP-3299). +const KekConvergenceDelay = 5 * time.Minute + // MigratedGroupResources is the data structured stored in the // encryption.apiserver.operator.openshift.io/migrated-resources // of a key secret. diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go index fd1af2ebe..0226dca5f 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/state/types.go @@ -50,6 +50,27 @@ type KeyState struct { ExternalReason string // stores all the KMS encryption mode related configurations KMS *KMSState + // KekMigration stores KMS KEK rotation annotations from the write-key secret. + KekMigration *KekMigrationState +} + +// KekMigrationState holds KMS KEK rotation annotations parsed from a key secret. +type KekMigrationState struct { + TargetKekID string + MigratedKekID string + KekConvergedAt time.Time + KekConvergedID string +} + +// NeedsKekMigration reports whether target-kek-id is set and differs from migrated-kek-id. +func (k KeyState) NeedsKekMigration() bool { + if k.Mode != KMS || k.KekMigration == nil { + return false + } + if k.KekMigration.TargetKekID == "" { + return false + } + return k.KekMigration.TargetKekID != k.KekMigration.MigratedKekID } func (k *KeyState) HasKMSEncryption() bool { diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/statemachine/transition.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/statemachine/transition.go index 12eb5b8b9..51e9ba8be 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/encryption/statemachine/transition.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/statemachine/transition.go @@ -193,6 +193,10 @@ func getDesiredEncryptionState(oldSecretData *encryptiondata.Config, encryptionS // STEP 4: with consistent read-keys and write-keys, remove every read-key other than the write-key and one last read key. // // Note: because read-keys are consistent, currentlyEncryptedGRs equals toBeEncryptedGRs + if writeKey.NeedsKekMigration() { + klog.V(4).Infof("KMS kek migration in progress for key ID %s", writeKey.Key.Name) + return desiredEncryptionState + } allMigrated, _, reason := state.MigratedFor(currentlyEncryptedGRs, writeKey) if !allMigrated { klog.V(4).Infof("%s", reason) diff --git a/vendor/modules.txt b/vendor/modules.txt index 1ca73e664..002eb2cf4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -383,7 +383,7 @@ github.com/openshift/client-go/user/applyconfigurations/internal github.com/openshift/client-go/user/applyconfigurations/user/v1 github.com/openshift/client-go/user/clientset/versioned/scheme github.com/openshift/client-go/user/clientset/versioned/typed/user/v1 -# github.com/openshift/library-go v0.0.0-20260611070125-7fd5f333df33 +# github.com/openshift/library-go v0.0.0-20260611070125-7fd5f333df33 => github.com/tjungblu/library-go v0.0.0-20260611114644-374bd672722f ## explicit; go 1.25.0 github.com/openshift/library-go/pkg/apiserver/jsonpatch github.com/openshift/library-go/pkg/apps/deployment @@ -428,7 +428,7 @@ github.com/openshift/library-go/pkg/operator/encryption/crypto github.com/openshift/library-go/pkg/operator/encryption/deployer github.com/openshift/library-go/pkg/operator/encryption/encoding github.com/openshift/library-go/pkg/operator/encryption/encryptiondata -github.com/openshift/library-go/pkg/operator/encryption/encryptionstatus +github.com/openshift/library-go/pkg/operator/encryption/kms/health github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle github.com/openshift/library-go/pkg/operator/encryption/observer github.com/openshift/library-go/pkg/operator/encryption/secrets @@ -1652,4 +1652,4 @@ sigs.k8s.io/structured-merge-diff/v6/value ## explicit; go 1.22 sigs.k8s.io/yaml # github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20251001123353-fd5b1fb35db1 -# github.com/openshift/library-go => github.com/tjungblu/library-go v0.0.0-20260608092146-53a048269fb0 +# github.com/openshift/library-go => github.com/tjungblu/library-go v0.0.0-20260611114644-374bd672722f