diff --git a/cmd/cluster-authentication-operator-tests-ext/main.go b/cmd/cluster-authentication-operator-tests-ext/main.go index fdd363a2ce..5121f35d7b 100644 --- a/cmd/cluster-authentication-operator-tests-ext/main.go +++ b/cmd/cluster-authentication-operator-tests-ext/main.go @@ -13,7 +13,11 @@ import ( "github.com/openshift/cluster-authentication-operator/pkg/version" _ "github.com/openshift/cluster-authentication-operator/test/e2e" + _ "github.com/openshift/cluster-authentication-operator/test/e2e-encryption" _ "github.com/openshift/cluster-authentication-operator/test/e2e-encryption-kms" + _ "github.com/openshift/cluster-authentication-operator/test/e2e-encryption-perf" + _ "github.com/openshift/cluster-authentication-operator/test/e2e-encryption-rotation" + _ "github.com/openshift/cluster-authentication-operator/test/e2e-oidc" "k8s.io/klog/v2" ) @@ -74,11 +78,12 @@ func prepareOperatorTestsRegistry() (*oteextension.Registry, error) { // The following suite runs tests that must execute serially (one at a time) // because they modify cluster-wide resources like OAuth configuration. // Tests tagged with [Serial] and any of [Operator], [OIDC], [Templates], [Tokens] are included in this suite. + // Disruptive tests are excluded as they run in the disruptive suite instead. extension.AddSuite(oteextension.Suite{ Name: "openshift/cluster-authentication-operator/operator/serial", Parallelism: 1, Qualifiers: []string{ - `name.contains("[Serial]") && (name.contains("[Operator]") || name.contains("[OIDC]") || name.contains("[Templates]") || name.contains("[Tokens]"))`, + `name.contains("[Serial]") && !name.contains("[Disruptive]") && (name.contains("[Operator]") || name.contains("[OIDC]") || name.contains("[Templates]") || name.contains("[Tokens]"))`, }, }) @@ -92,6 +97,36 @@ func prepareOperatorTestsRegistry() (*oteextension.Registry, error) { ClusterStability: oteextension.ClusterStabilityDisruptive, }) + // ClusterStability set to Disruptive: encryption tests trigger API server rollouts. + extension.AddSuite(oteextension.Suite{ + Name: "openshift/cluster-authentication-operator/operator-encryption/serial", + Parallelism: 1, + ClusterStability: oteextension.ClusterStabilityDisruptive, + Qualifiers: []string{ + `name.contains("[Encryption]") && name.contains("[Serial]") && !name.contains("Rotation") && !name.contains("Perf") && !name.contains("KMS")`, + }, + }) + + // ClusterStability set to Disruptive: encryption rotation triggers API server rollouts. + extension.AddSuite(oteextension.Suite{ + Name: "openshift/cluster-authentication-operator/operator-encryption-rotation/serial", + Parallelism: 1, + ClusterStability: oteextension.ClusterStabilityDisruptive, + Qualifiers: []string{ + `name.contains("[Encryption]") && name.contains("[Serial]") && name.contains("Rotation")`, + }, + }) + + // ClusterStability set to Disruptive: encryption perf tests trigger API server rollouts. + extension.AddSuite(oteextension.Suite{ + Name: "openshift/cluster-authentication-operator/operator-encryption-perf/serial", + Parallelism: 1, + ClusterStability: oteextension.ClusterStabilityDisruptive, + Qualifiers: []string{ + `name.contains("[Encryption]") && name.contains("[Serial]") && name.contains("Perf")`, + }, + }) + // The following suite runs KMS encryption tests. extension.AddSuite(oteextension.Suite{ Name: "openshift/cluster-authentication-operator/encryption-kms", @@ -101,6 +136,18 @@ func prepareOperatorTestsRegistry() (*oteextension.Registry, error) { }, }) + // Register external images used by OIDC tests + extension.RegisterImage(oteextension.Image{ + Registry: "quay.io", + Name: "keycloak/keycloak", + Version: "25.0", + }) + extension.RegisterImage(oteextension.Image{ + Registry: "docker.io", + Name: "gitlab/gitlab-ce", + Version: "13.8.4-ce.0", + }) + specs, err := oteginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite() if err != nil { return nil, fmt.Errorf("couldn't build extension test specs from ginkgo: %w", err) diff --git a/test/e2e-encryption-perf/encryption_perf.go b/test/e2e-encryption-perf/encryption_perf.go new file mode 100644 index 0000000000..8a3a2db471 --- /dev/null +++ b/test/e2e-encryption-perf/encryption_perf.go @@ -0,0 +1,117 @@ +package e2e_encryption_perf + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + g "github.com/onsi/ginkgo/v2" + "github.com/stretchr/testify/require" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + configv1 "github.com/openshift/api/config/v1" + oauthapiv1 "github.com/openshift/api/oauth/v1" + operatorv1 "github.com/openshift/api/operator/v1" + oauthclient "github.com/openshift/client-go/oauth/clientset/versioned/typed/oauth/v1" + operatorlibrary "github.com/openshift/cluster-authentication-operator/test/library" + operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption" + library "github.com/openshift/library-go/test/library/encryption" +) + +const ( + tokenStatsKey = "created oauthaccesstokens" +) + +var _ = g.Describe("[sig-auth] authentication operator", func() { + g.It("[Encryption][Serial] TestPerfEncryptionTypeAESCBC", func() { + testPerfEncryptionTypeAESCBC(g.GinkgoTB()) + }) +}) + +func testPerfEncryptionTypeAESCBC(tt testing.TB) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + tt.Cleanup(cancel) + clientSet := getPerfClients(tt) + operatorlibrary.TestPerfEncryption(tt, library.PerfScenario{ + BasicScenario: library.BasicScenario{ + Namespace: "openshift-config-managed", + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", + EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", "openshift-oauth-apiserver"), + EncryptionConfigSecretNamespace: "openshift-config-managed", + OperatorNamespace: "openshift-authentication-operator", + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertTokens, + }, + GetOperatorConditionsFunc: func(t testing.TB) ([]operatorv1.OperatorCondition, error) { + apiServerOperator, err := clientSet.OperatorClient.Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return nil, err + } + return apiServerOperator.Status.Conditions, nil + }, + AssertDBPopulatedFunc: func(t testing.TB, errorStore map[string]int, statStore map[string]int) { + tokenCount, ok := statStore[tokenStatsKey] + if !ok { + err := errors.New("missing oauth access tokens count stats, can't continue the test") + require.NoError(t, err) + } + if tokenCount < 14000 { + err := fmt.Errorf("expected to create at least 14000 tokens but %d were created", tokenCount) + require.NoError(t, err) + } + t.Logf("Created %d access tokens", tokenCount) + }, + AssertMigrationTime: func(t testing.TB, migrationTime time.Duration) { + t.Logf("migration took %v", migrationTime) + expectedMigrationTime := 10 * time.Minute + if migrationTime > expectedMigrationTime { + t.Errorf("migration took too long (%v), expected it to take no more than %v", migrationTime, expectedMigrationTime) + } + }, + DBLoaderWorkers: 3, + DBLoaderFunc: library.DBLoaderRepeat(1, false, + library.DBLoaderRepeatParallel(5010, 50, false, createAccessTokenWrapper(ctx, clientSet.TokenClient), reportSecret)), + EncryptionProvider: library.EncryptionProvider{ + APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionTypeAESCBC}, + }, + }) +} + +func createAccessTokenWrapper(ctx context.Context, tokenClient oauthclient.OAuthAccessTokensGetter) library.DBLoaderFuncType { + return func(_ kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string)) error { + _, tokenNameHash := operatorlibrary.GenerateOAuthTokenPair() + token := &oauthapiv1.OAuthAccessToken{ + ObjectMeta: metav1.ObjectMeta{ + Name: tokenNameHash, + }, + RefreshToken: "I have no special talents. I am only passionately curious", + UserName: "kube:admin", + Scopes: []string{"user:full"}, + RedirectURI: "redirect.me.to.token.of.life", + ClientName: "console", + UserUID: "non-existing-user-id", + } + _, err := tokenClient.OAuthAccessTokens().Create(ctx, token, metav1.CreateOptions{}) + return err + } +} + +func reportSecret(_ kubernetes.Interface, _ string, _ func(error), statsCollector func(string)) error { + statsCollector(tokenStatsKey) + return nil +} + +func getPerfClients(t testing.TB) operatorencryption.ClientSet { + t.Helper() + + kubeConfig := operatorlibrary.NewClientConfigForTest(t) + + kubeConfig.QPS = 300 + kubeConfig.Burst = 600 + + return operatorencryption.GetClientsFor(t, kubeConfig) +} diff --git a/test/e2e-encryption-perf/encryption_perf_test.go b/test/e2e-encryption-perf/encryption_perf_test.go index 491959931c..2a8f051f77 100644 --- a/test/e2e-encryption-perf/encryption_perf_test.go +++ b/test/e2e-encryption-perf/encryption_perf_test.go @@ -1,109 +1,14 @@ package e2e_encryption_perf import ( - "context" - "errors" - "fmt" "testing" - "time" - - "github.com/stretchr/testify/require" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - - configv1 "github.com/openshift/api/config/v1" - oauthapiv1 "github.com/openshift/api/oauth/v1" - operatorv1 "github.com/openshift/api/operator/v1" - oauthclient "github.com/openshift/client-go/oauth/clientset/versioned/typed/oauth/v1" - operatorlibrary "github.com/openshift/cluster-authentication-operator/test/library" - operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption" - library "github.com/openshift/library-go/test/library/encryption" -) - -const ( - tokenStatsKey = "created oauthaccesstokens" ) +// This test calls the shared test function which +// can be called from both standard Go tests and Ginkgo tests. +// +// This situation is temporary until we verify the new e2e-aws-operator-encryption-perf-serial-ote job. +// Eventually all tests will be run only as part of the OTE framework. func TestPerfEncryptionTypeAESCBC(tt *testing.T) { - ctx := context.TODO() - clientSet := getPerfClients(tt) - library.TestPerfEncryption(tt.Context(), tt, library.PerfScenario{ - BasicScenario: library.BasicScenario{ - Namespace: "openshift-config-managed", - LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", - EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", "openshift-oauth-apiserver"), - EncryptionConfigSecretNamespace: "openshift-config-managed", - OperatorNamespace: "openshift-authentication-operator", - TargetGRs: operatorencryption.DefaultTargetGRs, - AssertFunc: operatorencryption.AssertTokens, - }, - GetOperatorConditionsFunc: func(t testing.TB) ([]operatorv1.OperatorCondition, error) { - apiServerOperator, err := clientSet.OperatorClient.Get(ctx, "cluster", metav1.GetOptions{}) - if err != nil { - return nil, err - } - return apiServerOperator.Status.Conditions, nil - }, - AssertDBPopulatedFunc: func(t testing.TB, errorStore map[string]int, statStore map[string]int) { - tokenCount, ok := statStore[tokenStatsKey] - if !ok { - err := errors.New("missing oauth access tokens count stats, can't continue the test") - require.NoError(t, err) - } - if tokenCount < 14000 { - err := fmt.Errorf("expected to create at least 14000 tokens but %d were created", tokenCount) - require.NoError(t, err) - } - t.Logf("Created %d access tokens", tokenCount) - }, - AssertMigrationTime: func(t testing.TB, migrationTime time.Duration) { - t.Logf("migration took %v", migrationTime) - expectedMigrationTime := 10 * time.Minute - if migrationTime > expectedMigrationTime { - t.Errorf("migration took too long (%v), expected it to take no more than %v", migrationTime, expectedMigrationTime) - } - }, - DBLoaderWorkers: 3, - DBLoaderFunc: library.DBLoaderRepeat(1, false, - library.DBLoaderRepeatParallel(5010, 50, false, createAccessTokenWrapper(ctx, clientSet.TokenClient), reportSecret)), - EncryptionProvider: library.EncryptionProvider{ - APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionType("aescbc")}, - }, - }) -} - -func createAccessTokenWrapper(ctx context.Context, tokenClient oauthclient.OAuthAccessTokensGetter) library.DBLoaderFuncType { - return func(_ kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string)) error { - _, tokenNameHash := operatorlibrary.GenerateOAuthTokenPair() - token := &oauthapiv1.OAuthAccessToken{ - ObjectMeta: metav1.ObjectMeta{ - Name: tokenNameHash, - }, - RefreshToken: "I have no special talents. I am only passionately curious", - UserName: "kube:admin", - Scopes: []string{"user:full"}, - RedirectURI: "redirect.me.to.token.of.life", - ClientName: "console", - UserUID: "non-existing-user-id", - } - _, err := tokenClient.OAuthAccessTokens().Create(ctx, token, metav1.CreateOptions{}) - return err - } -} - -func reportSecret(_ kubernetes.Interface, _ string, _ func(error), statsCollector func(string)) error { - statsCollector(tokenStatsKey) - return nil -} - -func getPerfClients(t *testing.T) operatorencryption.ClientSet { - t.Helper() - - kubeConfig := operatorlibrary.NewClientConfigForTest(t) - - kubeConfig.QPS = 300 - kubeConfig.Burst = 600 - - return operatorencryption.GetClientsFor(t, kubeConfig) + testPerfEncryptionTypeAESCBC(tt) } diff --git a/test/e2e-encryption-rotation/e2e-encryption-rotation_test.go b/test/e2e-encryption-rotation/e2e-encryption-rotation_test.go index c79b2c78ec..4615f412fe 100644 --- a/test/e2e-encryption-rotation/e2e-encryption-rotation_test.go +++ b/test/e2e-encryption-rotation/e2e-encryption-rotation_test.go @@ -1,73 +1,14 @@ package e2e_encryption_rotation import ( - "context" - "encoding/json" - "fmt" "testing" - - configv1 "github.com/openshift/api/config/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - - oauthapiconfigobservercontroller "github.com/openshift/cluster-authentication-operator/pkg/operator/configobservation/configobservercontroller" - operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption" - library "github.com/openshift/library-go/test/library/encryption" ) -// TestEncryptionRotation first encrypts data with aescbc key -// then it forces a key rotation by setting the "encyrption.Reason" in the operator's configuration file +// This test calls the shared test function which +// can be called from both standard Go tests and Ginkgo tests. +// +// This situation is temporary until we verify the new e2e-aws-operator-encryption-rotation-serial-ote job. +// Eventually all tests will be run only as part of the OTE framework. func TestEncryptionRotation(t *testing.T) { - ctx := context.TODO() - library.TestEncryptionRotation(ctx, t, library.RotationScenario{ - BasicScenario: library.BasicScenario{ - Namespace: "openshift-config-managed", - LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", - EncryptionConfigSecretName: fmt.Sprintf("encryption-config-openshift-oauth-apiserver"), - EncryptionConfigSecretNamespace: "openshift-config-managed", - OperatorNamespace: "openshift-authentication-operator", - TargetGRs: operatorencryption.DefaultTargetGRs, - AssertFunc: operatorencryption.AssertTokens, - }, - CreateResourceFunc: func(t testing.TB, _ library.ClientSet, _ string) runtime.Object { - return operatorencryption.CreateAndStoreTokenOfLife(ctx, t, operatorencryption.GetClients(t)) - }, - GetRawResourceFunc: func(t testing.TB, clientSet library.ClientSet, _ string) string { - return operatorencryption.GetRawTokenOfLife(t, clientSet) - }, - EncryptionProvider: library.EncryptionProvider{ - APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionType("aescbc")}, - }, - ForceRotationFunc: library.StaticEncryptionForceRotation(func(rawUnsupportedEncryptionCfg []byte) error { - cs := operatorencryption.GetClients(t) - authOperator, err := cs.OperatorClient.Get(ctx, "cluster", metav1.GetOptions{}) - if err != nil { - return err - } - - unsupportedConfigAsMap := map[string]interface{}{} - if len(authOperator.Spec.UnsupportedConfigOverrides.Raw) > 0 { - if err := json.Unmarshal(authOperator.Spec.UnsupportedConfigOverrides.Raw, &unsupportedConfigAsMap); err != nil { - return err - } - } - unsupportedEncryptionConfigAsMap := map[string]interface{}{} - if err := json.Unmarshal(rawUnsupportedEncryptionCfg, &unsupportedEncryptionConfigAsMap); err != nil { - return err - } - if err := unstructured.SetNestedMap(unsupportedConfigAsMap, unsupportedEncryptionConfigAsMap, oauthapiconfigobservercontroller.OAuthAPIServerConfigPrefix); err != nil { - return err - } - rawUnsupportedCfg, err := json.Marshal(unsupportedConfigAsMap) - if err != nil { - return err - } - authOperator.Spec.UnsupportedConfigOverrides.Raw = rawUnsupportedCfg - - _, err = cs.OperatorClient.Update(ctx, authOperator, metav1.UpdateOptions{}) - return err - }), - WaitForRotationCompleteFunc: library.WaitForNextEncryptionKeyRotation(), - }) + testEncryptionRotation(t) } diff --git a/test/e2e-encryption-rotation/encryption_rotation.go b/test/e2e-encryption-rotation/encryption_rotation.go new file mode 100644 index 0000000000..cc6a619204 --- /dev/null +++ b/test/e2e-encryption-rotation/encryption_rotation.go @@ -0,0 +1,85 @@ +package e2e_encryption_rotation + +import ( + "context" + "encoding/json" + "testing" + "time" + + g "github.com/onsi/ginkgo/v2" + + configv1 "github.com/openshift/api/config/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/util/retry" + + oauthapiconfigobservercontroller "github.com/openshift/cluster-authentication-operator/pkg/operator/configobservation/configobservercontroller" + operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption" + library "github.com/openshift/library-go/test/library/encryption" +) + +var _ = g.Describe("[sig-auth] authentication operator", func() { + g.It("[Encryption][Serial] TestEncryptionRotation [Timeout:3h]", func() { + testEncryptionRotation(g.GinkgoTB()) + }) +}) + +// testEncryptionRotation first encrypts data with aescbc key +// then it forces a key rotation by setting the "encryption.Reason" in the operator's configuration file +func testEncryptionRotation(t testing.TB) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Hour) + t.Cleanup(cancel) + library.TestEncryptionRotation(ctx, t, library.RotationScenario{ + BasicScenario: library.BasicScenario{ + Namespace: "openshift-config-managed", + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", + EncryptionConfigSecretName: "encryption-config-openshift-oauth-apiserver", + EncryptionConfigSecretNamespace: "openshift-config-managed", + OperatorNamespace: "openshift-authentication-operator", + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertTokens, + }, + CreateResourceFunc: func(t testing.TB, _ library.ClientSet, _ string) runtime.Object { + return operatorencryption.CreateAndStoreTokenOfLife(ctx, t, operatorencryption.GetClients(t)) + }, + GetRawResourceFunc: func(t testing.TB, clientSet library.ClientSet, _ string) string { + return operatorencryption.GetRawTokenOfLife(t, clientSet) + }, + EncryptionProvider: library.EncryptionProvider{ + APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionTypeAESCBC}, + }, + ForceRotationFunc: library.StaticEncryptionForceRotation(func(rawUnsupportedEncryptionCfg []byte) error { + cs := operatorencryption.GetClients(t) + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + authOperator, err := cs.OperatorClient.Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + + unsupportedConfigAsMap := map[string]interface{}{} + if len(authOperator.Spec.UnsupportedConfigOverrides.Raw) > 0 { + if err := json.Unmarshal(authOperator.Spec.UnsupportedConfigOverrides.Raw, &unsupportedConfigAsMap); err != nil { + return err + } + } + unsupportedEncryptionConfigAsMap := map[string]interface{}{} + if err := json.Unmarshal(rawUnsupportedEncryptionCfg, &unsupportedEncryptionConfigAsMap); err != nil { + return err + } + if err := unstructured.SetNestedMap(unsupportedConfigAsMap, unsupportedEncryptionConfigAsMap, oauthapiconfigobservercontroller.OAuthAPIServerConfigPrefix); err != nil { + return err + } + rawUnsupportedCfg, err := json.Marshal(unsupportedConfigAsMap) + if err != nil { + return err + } + authOperator.Spec.UnsupportedConfigOverrides.Raw = rawUnsupportedCfg + + _, err = cs.OperatorClient.Update(ctx, authOperator, metav1.UpdateOptions{}) + return err + }) + }), + WaitForRotationCompleteFunc: library.WaitForNextEncryptionKeyRotation(), + }) +} diff --git a/test/e2e-encryption/encryption.go b/test/e2e-encryption/encryption.go new file mode 100644 index 0000000000..e55341002d --- /dev/null +++ b/test/e2e-encryption/encryption.go @@ -0,0 +1,79 @@ +package e2e_encryption + +import ( + "context" + "testing" + "time" + + g "github.com/onsi/ginkgo/v2" + "k8s.io/apimachinery/pkg/runtime" + + configv1 "github.com/openshift/api/config/v1" + testlibrary "github.com/openshift/cluster-authentication-operator/test/library" + operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption" + library "github.com/openshift/library-go/test/library/encryption" +) + +var _ = g.Describe("[sig-auth] authentication operator", func() { + g.It("[Encryption][Serial] TestEncryptionTypeIdentity", func() { + testEncryptionTypeIdentity(g.GinkgoTB()) + }) + + g.It("[Encryption][Serial] TestEncryptionTypeUnset", func() { + testEncryptionTypeUnset(g.GinkgoTB()) + }) + + g.It("[Encryption][Serial] TestEncryptionTurnOnAndOff [Timeout:3h]", func() { + testEncryptionTurnOnAndOff(g.GinkgoTB()) + }) +}) + +func testEncryptionTypeIdentity(t testing.TB) { + testlibrary.TestEncryptionTypeIdentity(t, library.BasicScenario{ + Namespace: "openshift-config-managed", + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", + EncryptionConfigSecretName: "encryption-config-openshift-oauth-apiserver", + EncryptionConfigSecretNamespace: "openshift-config-managed", + OperatorNamespace: "openshift-authentication-operator", + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertTokens, + }) +} + +func testEncryptionTypeUnset(t testing.TB) { + testlibrary.TestEncryptionTypeUnset(t, library.BasicScenario{ + Namespace: "openshift-config-managed", + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", + EncryptionConfigSecretName: "encryption-config-openshift-oauth-apiserver", + EncryptionConfigSecretNamespace: "openshift-config-managed", + OperatorNamespace: "openshift-authentication-operator", + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertTokens, + }) +} + +func testEncryptionTurnOnAndOff(t testing.TB) { + testlibrary.TestEncryptionTurnOnAndOff(t, library.OnOffScenario{ + BasicScenario: library.BasicScenario{ + Namespace: "openshift-config-managed", + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", + EncryptionConfigSecretName: "encryption-config-openshift-oauth-apiserver", + EncryptionConfigSecretNamespace: "openshift-config-managed", + OperatorNamespace: "openshift-authentication-operator", + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertTokens, + }, + CreateResourceFunc: func(t testing.TB, _ library.ClientSet, namespace string) runtime.Object { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + t.Cleanup(cancel) + return operatorencryption.CreateAndStoreTokenOfLife(ctx, t, operatorencryption.GetClients(t)) + }, + AssertResourceEncryptedFunc: operatorencryption.AssertTokenOfLifeEncrypted, + AssertResourceNotEncryptedFunc: operatorencryption.AssertTokenOfLifeNotEncrypted, + ResourceFunc: func(t testing.TB, _ string) runtime.Object { return operatorencryption.TokenOfLife(t) }, + ResourceName: "TokenOfLife", + EncryptionProvider: library.EncryptionProvider{ + APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionTypeAESCBC}, + }, + }) +} diff --git a/test/e2e-encryption/encryption_test.go b/test/e2e-encryption/encryption_test.go index 8dbf1ed6dd..1d3d6a6ce6 100644 --- a/test/e2e-encryption/encryption_test.go +++ b/test/e2e-encryption/encryption_test.go @@ -1,61 +1,32 @@ -package e2eencryption +package e2e_encryption import ( - "context" - "fmt" "testing" - - "k8s.io/apimachinery/pkg/runtime" - - configv1 "github.com/openshift/api/config/v1" - operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption" - library "github.com/openshift/library-go/test/library/encryption" ) +// This test calls the shared test function which +// can be called from both standard Go tests and Ginkgo tests. +// +// This situation is temporary until we verify the new e2e-aws-operator-encryption-serial-ote job. +// Eventually all tests will be run only as part of the OTE framework. func TestEncryptionTypeIdentity(t *testing.T) { - library.TestEncryptionTypeIdentity(t.Context(), t, library.BasicScenario{ - Namespace: "openshift-config-managed", - LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", - EncryptionConfigSecretName: fmt.Sprintf("encryption-config-openshift-oauth-apiserver"), - EncryptionConfigSecretNamespace: "openshift-config-managed", - OperatorNamespace: "openshift-authentication-operator", - TargetGRs: operatorencryption.DefaultTargetGRs, - AssertFunc: operatorencryption.AssertTokens, - }) + testEncryptionTypeIdentity(t) } +// This test calls the shared test function which +// can be called from both standard Go tests and Ginkgo tests. +// +// This situation is temporary until we verify the new e2e-aws-operator-encryption-serial-ote job. +// Eventually all tests will be run only as part of the OTE framework. func TestEncryptionTypeUnset(t *testing.T) { - library.TestEncryptionTypeUnset(t.Context(), t, library.BasicScenario{ - Namespace: "openshift-config-managed", - LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", - EncryptionConfigSecretName: fmt.Sprintf("encryption-config-openshift-oauth-apiserver"), - EncryptionConfigSecretNamespace: "openshift-config-managed", - OperatorNamespace: "openshift-authentication-operator", - TargetGRs: operatorencryption.DefaultTargetGRs, - AssertFunc: operatorencryption.AssertTokens, - }) + testEncryptionTypeUnset(t) } +// This test calls the shared test function which +// can be called from both standard Go tests and Ginkgo tests. +// +// This situation is temporary until we verify the new e2e-aws-operator-encryption-serial-ote job. +// Eventually all tests will be run only as part of the OTE framework. func TestEncryptionTurnOnAndOff(t *testing.T) { - library.TestEncryptionTurnOnAndOff(t.Context(), t, library.OnOffScenario{ - BasicScenario: library.BasicScenario{ - Namespace: "openshift-config-managed", - LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", - EncryptionConfigSecretName: fmt.Sprintf("encryption-config-openshift-oauth-apiserver"), - EncryptionConfigSecretNamespace: "openshift-config-managed", - OperatorNamespace: "openshift-authentication-operator", - TargetGRs: operatorencryption.DefaultTargetGRs, - AssertFunc: operatorencryption.AssertTokens, - }, - CreateResourceFunc: func(t testing.TB, _ library.ClientSet, namespace string) runtime.Object { - return operatorencryption.CreateAndStoreTokenOfLife(context.TODO(), t, operatorencryption.GetClients(t)) - }, - AssertResourceEncryptedFunc: operatorencryption.AssertTokenOfLifeEncrypted, - AssertResourceNotEncryptedFunc: operatorencryption.AssertTokenOfLifeNotEncrypted, - ResourceFunc: func(t testing.TB, _ string) runtime.Object { return operatorencryption.TokenOfLife(t) }, - ResourceName: "TokenOfLife", - EncryptionProvider: library.EncryptionProvider{ - APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionType("aescbc")}, - }, - }) + testEncryptionTurnOnAndOff(t) } diff --git a/test/e2e-encryption/main_test.go b/test/e2e-encryption/main_test.go index b9a4df642b..4b83293c1e 100644 --- a/test/e2e-encryption/main_test.go +++ b/test/e2e-encryption/main_test.go @@ -1,4 +1,4 @@ -package e2eencryption +package e2e_encryption import ( "math/rand" diff --git a/test/e2e-oidc/external_oidc.go b/test/e2e-oidc/external_oidc.go new file mode 100644 index 0000000000..01e4cd9656 --- /dev/null +++ b/test/e2e-oidc/external_oidc.go @@ -0,0 +1,1133 @@ +package e2e_oidc + +import ( + "context" + "crypto/rsa" + "crypto/tls" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "math/big" + "net/http" + "strings" + "testing" + "time" + + g "github.com/onsi/ginkgo/v2" + "github.com/stretchr/testify/require" + + authenticationv1 "k8s.io/api/authentication/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/storage/names" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + "k8s.io/utils/clock" + "k8s.io/utils/ptr" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/api/features" + operatorv1 "github.com/openshift/api/operator/v1" + routev1 "github.com/openshift/api/route/v1" + configclient "github.com/openshift/client-go/config/clientset/versioned" + oauthclient "github.com/openshift/client-go/oauth/clientset/versioned" + operatorversionedclient "github.com/openshift/client-go/operator/clientset/versioned" + routeclient "github.com/openshift/client-go/route/clientset/versioned" + "github.com/openshift/cluster-authentication-operator/pkg/operator" + test "github.com/openshift/cluster-authentication-operator/test/library" + "github.com/openshift/library-go/pkg/operator/genericoperatorclient" + "github.com/openshift/library-go/pkg/operator/v1helpers" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + + "github.com/golang-jwt/jwt/v5" +) + +const ( + oidcClientID = "admin-cli" + oidcGroupsClaim = "groups" + oidcGroupsPrefix = "" + + managedNS = "openshift-config-managed" + authCM = "auth-config" +) + +var _ = g.Describe("[sig-auth] authentication operator", func() { + g.It("[OIDC][Serial][Disruptive] TestExternalOIDCWithKeycloak [Timeout:3h]", func(ctx context.Context) { + testExternalOIDCWithKeycloak(ctx, g.GinkgoTB()) + }) +}) + +func testExternalOIDCWithKeycloak(testCtx context.Context, t testing.TB) { + testClient, err := newTestClient(testCtx, t) + require.NoError(t, err) + + checkFeatureGatesOrSkip(testCtx, t, testClient.configClient, features.FeatureGateExternalOIDC, features.FeatureGateExternalOIDCWithAdditionalClaimMappings) + + newExternalOIDCArchitectureEnabled := featureGateEnabled(testCtx, testClient.configClient, features.FeatureGateExternalOIDCExternalClaimsSourcing) + + // post-test cluster cleanup + var cleanups []func() + defer test.IDPCleanupWrapper(func() { + t.Logf("cleaning up after test") + ts := time.Now() + for _, c := range cleanups { + c() + } + t.Logf("cleanup completed after %s", time.Since(ts)) + + // IMPORTANT: Wait for cluster to recover to healthy state before exiting. + // This ensures that even if the test times out or fails, subsequent tests + // will run on a stable cluster instead of inheriting degraded state. + t.Logf("waiting for cluster to recover to healthy state before test exit") + recoveryStart := time.Now() + + // Wait for authentication operator to be available (not requiring 10-minute stability) + if err := test.WaitForClusterOperatorAvailableNotProgressingNotDegraded(t, testClient.configClient.ConfigV1(), "authentication"); err != nil { + t.Logf("WARNING: authentication operator did not become healthy after cleanup: %v", err) + } else { + t.Logf("authentication operator is healthy") + } + + // Wait for kube-apiserver to be available (not requiring 10-minute stability) + if err := test.WaitForClusterOperatorAvailableNotProgressingNotDegraded(t, testClient.configClient.ConfigV1(), "kube-apiserver"); err != nil { + t.Logf("WARNING: kube-apiserver operator did not become healthy after cleanup: %v", err) + } else { + t.Logf("kube-apiserver operator is healthy") + } + + t.Logf("cluster recovery completed after %s (total cleanup time: %s)", time.Since(recoveryStart), time.Since(ts)) + })() + + origAuthSpec := (*testClient.getAuth(testCtx, t)).Spec.DeepCopy() + cleanups = append(cleanups, func() { + kasOriginalRevision := testClient.kasLatestAvailableRevision(testCtx, t) + + err := testClient.authResourceRollback(testCtx, origAuthSpec) + require.NoError(t, err, "failed to rollback auth resource during cleanup") + + // KAS should not need to perform a rollout when the new architecture is enabled. + // The oauth-apiserver will, but that should be fairly quick and handled by cluster operator + // stability checks prior to running tests. + if !newExternalOIDCArchitectureEnabled { + err = test.WaitForNewKASRollout(testCtx, t, testClient.operatorConfigClient.OperatorV1().KubeAPIServers(), kasOriginalRevision) + require.NoError(t, err, "failed to wait for KAS rollout during cleanup") + } + + testClient.validateOAuthState(testCtx, t, false, newExternalOIDCArchitectureEnabled) + }) + + // keycloak setup + var idpName string + var kcClient *test.KeycloakClient + kcClient, idpName, c := test.AddKeycloakIDP(t, testClient.kubeConfig, true) + cleanups = append(cleanups, c...) + t.Logf("keycloak Admin URL: %s", kcClient.AdminURL()) + + // default-ingress-cert is copied to openshift-config and used as the CA for the IdP + // see test/library/idpdeployment.go:332 + caBundleName := idpName + "-ca" + idpURL := kcClient.IssuerURL() + + // run tests + + testSpec := authSpecForOIDCProvider(idpName, idpURL, caBundleName, oidcGroupsClaim, oidcClientID) + + typeOAuth := ptr.To(configv1.AuthenticationTypeIntegratedOAuth) + typeOIDC := ptr.To(configv1.AuthenticationTypeOIDC) + operatorAvailable := []configv1.ClusterOperatorStatusCondition{ + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorDegraded, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorUpgradeable, Status: configv1.ConditionTrue}, + } + + // Test: auth-config cm must not exist and gets deleted by the CAO if manually created when type not OIDC + t.Logf("auth-config cm must not exist and gets deleted by the CAO if manually created when type not OIDC") + testClient.checkPreconditions(testCtx, t, typeOAuth, operatorAvailable, nil) + + _, err = testClient.kubeClient.CoreV1().ConfigMaps(managedNS).Get(testCtx, authCM, metav1.GetOptions{}) + require.True(t, errors.IsNotFound(err), "openshift-config-managed/auth-config configmap must be missing") + + // create cm + cm := v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: authCM, + Namespace: managedNS, + }, + Data: map[string]string{ + "test": "value", + }, + } + newCM, err := testClient.kubeClient.CoreV1().ConfigMaps(managedNS).Create(testCtx, &cm, metav1.CreateOptions{}) + require.NoError(t, err) + require.Equal(t, cm.Data, newCM.Data) + + // wait for CAO to delete it + var cmErr error + waitErr := wait.PollUntilContextTimeout(testCtx, 2*time.Second, 1*time.Minute, false, func(ctx context.Context) (bool, error) { + cmErr = nil + _, err := testClient.kubeClient.CoreV1().ConfigMaps(managedNS).Get(testCtx, authCM, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return true, nil + } + cmErr = err + return false, nil + }) + require.NoError(t, cmErr, "failed to get auth configmap: %v", cmErr) + require.NoError(t, waitErr, "failed to wait for auth configmap to get deleted: %v", err) + + // Test: invalid CEL expression rejects auth CR admission + t.Logf("invalid CEL expression rejects auth CR admission") + for _, tt := range []struct { + name string + specUpdate func(*configv1.AuthenticationSpec) + requireFeatureGates []configv1.FeatureGateName + }{ + { + name: "uncompilable CEL expression for uid claim mapping", + specUpdate: func(s *configv1.AuthenticationSpec) { + s.OIDCProviders[0].ClaimMappings.UID = &configv1.TokenClaimOrExpressionMapping{ + Expression: "^&*!@#^*(", + } + }, + requireFeatureGates: []configv1.FeatureGateName{features.FeatureGateExternalOIDCWithAdditionalClaimMappings}, + }, + { + name: "uncompilable CEL expression for extras claim mapping", + specUpdate: func(s *configv1.AuthenticationSpec) { + s.OIDCProviders[0].ClaimMappings.Extra = []configv1.ExtraMapping{ + { + Key: "testing/key", + ValueExpression: "^&*!@#^*(", + }, + } + }, + requireFeatureGates: []configv1.FeatureGateName{features.FeatureGateExternalOIDCWithAdditionalClaimMappings}, + }, + } { + t.Logf("testing: %s", tt.name) + skip := false + for _, fg := range tt.requireFeatureGates { + if !featureGateEnabled(testCtx, testClient.configClient, fg) { + t.Logf(" skipping as required feature gate %q is not enabled", fg) + skip = true + break + } + } + if !skip { + _, err := testClient.updateAuthResource(testCtx, t, testSpec, tt.specUpdate) + require.Error(t, err, "uncompilable CEL expression should return in admission error") + } + } + + // Test: invalid OIDC config degrades auth operator + t.Logf("invalid OIDC config degrades auth operator") + for _, tt := range []struct { + name string + specUpdate func(*configv1.AuthenticationSpec) + requireFeatureGates []configv1.FeatureGateName + }{ + { + name: "invalid issuer CA bundle", + specUpdate: func(s *configv1.AuthenticationSpec) { + s.OIDCProviders[0].Issuer.CertificateAuthority.Name = "invalid-ca-bundle" + }, + requireFeatureGates: []configv1.FeatureGateName{}, + }, + { + name: "invalid issuer URL", + specUpdate: func(s *configv1.AuthenticationSpec) { + s.OIDCProviders[0].Issuer.URL = "https://invalid-idp.testing" + }, + requireFeatureGates: []configv1.FeatureGateName{}, + }, + } { + t.Logf("testing: %s", tt.name) + skip := false + for _, fg := range tt.requireFeatureGates { + if !featureGateEnabled(testCtx, testClient.configClient, fg) { + t.Logf(" skipping as required feature gate %q is not enabled", fg) + skip = true + break + } + } + if !skip { + err := testClient.authResourceRollback(testCtx, origAuthSpec) + require.NoError(t, err, "failed to roll back auth resource") + + testClient.checkPreconditions(testCtx, t, typeOAuth, operatorAvailable, nil) + + _, err = testClient.updateAuthResource(testCtx, t, testSpec, tt.specUpdate) + require.NoError(t, err, "failed to update authentication/cluster") + + require.NoError(t, test.WaitForClusterOperatorDegraded(t, testClient.configClient.ConfigV1(), "authentication")) + + testClient.validateOAuthState(testCtx, t, false, newExternalOIDCArchitectureEnabled) + } + } + + // Test: OIDC config rolls out successfully + t.Logf("OIDC config rolls out successfully") + err = testClient.authResourceRollback(testCtx, origAuthSpec) + require.NoError(t, err, "failed to roll back auth resource") + + for _, tt := range []struct { + claim string + prefixPolicy configv1.UsernamePrefixPolicy + prefix *configv1.UsernamePrefix + expectedPrefix string + }{ + {"email", configv1.Prefix, &configv1.UsernamePrefix{PrefixString: "oidc-test:"}, "oidc-test:"}, + {"email", configv1.NoPrefix, nil, ""}, + {"sub", configv1.NoOpinion, nil, idpURL + "#"}, + {"email", configv1.NoOpinion, nil, ""}, + } { + policyStr := "NoOpinion" + if len(tt.prefixPolicy) > 0 { + policyStr = string(tt.prefixPolicy) + } + testName := fmt.Sprintf("username claim %s prefix policy %s", tt.claim, policyStr) + t.Logf("testing: %s", testName) + + testClient.checkPreconditions(testCtx, t, nil, operatorAvailable, operatorAvailable) + + kasOriginalRevision := testClient.kasLatestAvailableRevision(testCtx, t) + auth, err := testClient.updateAuthResource(testCtx, t, testSpec, func(baseSpec *configv1.AuthenticationSpec) { + baseSpec.OIDCProviders[0].ClaimMappings.Username = configv1.UsernameClaimMapping{ + Claim: tt.claim, + PrefixPolicy: tt.prefixPolicy, + Prefix: tt.prefix, + } + }) + require.NoError(t, err, "failed to update authentication/cluster") + + require.NoError(t, test.WaitForClusterOperatorStatusAlwaysAvailable(testCtx, t, testClient.configClient.ConfigV1(), "authentication")) + require.NoError(t, test.WaitForClusterOperatorStatusAlwaysAvailable(testCtx, t, testClient.configClient.ConfigV1(), "kube-apiserver")) + + testClient.requireKASRolloutSuccessful(testCtx, t, &auth.Spec, kasOriginalRevision, tt.expectedPrefix) + + testClient.validateOAuthState(testCtx, t, true, newExternalOIDCArchitectureEnabled) + + testClient.testOIDCAuthentication(testCtx, t, kcClient, tt.claim, tt.expectedPrefix, true) + } + + // Test: auth-config cm must exist and gets overwritten by the CAO if manually modified when type OIDC + t.Logf("auth-config cm must exist and gets overwritten by the CAO if manually modified when type OIDC") + testClient.checkPreconditions(testCtx, t, typeOIDC, operatorAvailable, nil) + + cmPtr, err := testClient.kubeClient.CoreV1().ConfigMaps(managedNS).Get(testCtx, authCM, metav1.GetOptions{}) + require.NoError(t, err) + require.NotNil(t, cmPtr) + + orig := cmPtr.DeepCopy() + cmPtr.Data["auth-config.json"] = "manually overwritten" + cmPtr, err = testClient.kubeClient.CoreV1().ConfigMaps(managedNS).Update(testCtx, cmPtr, metav1.UpdateOptions{}) + require.NoError(t, err) + require.NotEqual(t, cmPtr.Data, orig.Data) + + // wait for CAO to overwrite it + cmErr = nil + waitErr = wait.PollUntilContextTimeout(testCtx, 2*time.Second, 1*time.Minute, false, func(ctx context.Context) (bool, error) { + cmPtr, cmErr = testClient.kubeClient.CoreV1().ConfigMaps(managedNS).Get(testCtx, authCM, metav1.GetOptions{}) + if cmErr != nil { + return false, nil + } + + return equality.Semantic.DeepEqual(cmPtr.Data, orig.Data), nil + }) + require.NoError(t, cmErr, "failed to get auth configmap: %v", cmErr) + require.NoError(t, waitErr, "failed to wait for auth configmap to get overwritten: %v", waitErr) + + // Test: OIDC config rolls out successfully but breaks authentication when username claim is unknown + t.Logf("OIDC config rolls out successfully but breaks authentication when username claim is unknown") + testClient.checkPreconditions(testCtx, t, nil, operatorAvailable, operatorAvailable) + + kasOriginalRevision := testClient.kasLatestAvailableRevision(testCtx, t) + auth, err := testClient.updateAuthResource(testCtx, t, testSpec, func(baseSpec *configv1.AuthenticationSpec) { + baseSpec.OIDCProviders[0].ClaimMappings.Username = configv1.UsernameClaimMapping{ + Claim: "unknown", + PrefixPolicy: configv1.NoPrefix, + Prefix: nil, + } + }) + require.NoError(t, err, "failed to update authentication/cluster") + + require.NoError(t, test.WaitForClusterOperatorStatusAlwaysAvailable(testCtx, t, testClient.configClient.ConfigV1(), "authentication")) + require.NoError(t, test.WaitForClusterOperatorStatusAlwaysAvailable(testCtx, t, testClient.configClient.ConfigV1(), "kube-apiserver")) + + testClient.requireKASRolloutSuccessful(testCtx, t, &auth.Spec, kasOriginalRevision, "") + + testClient.validateOAuthState(testCtx, t, true, newExternalOIDCArchitectureEnabled) + + testClient.testOIDCAuthentication(testCtx, t, kcClient, "", "", false) +} + +type expectedClaims struct { + jwt.RegisteredClaims + Email string `json:"email"` + Type string `json:"typ"` + Name string `json:"name"` + GivenName string `json:"given_name"` + FamilyName string `json:"family_name"` + PreferredUsername string `json:"preferred_username"` +} + +type jwks struct { + Keys []struct { + KID string `json:"kid"` + Use string `json:"use"` + KTY string `json:"kty"` + Alg string `json:"alg"` + N string `json:"n"` + E string `json:"e"` + } `json:"keys"` +} + +func fetchIssuerJWKS(ctx context.Context, issuerURL string) (*jwks, error) { + client := &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS12, + }, + }, + } + + // grab openid-configuration JSON which contains the URL of the provider's JWKS + req, err := http.NewRequestWithContext(ctx, "GET", issuerURL+"/.well-known/openid-configuration", nil) + if err != nil { + return nil, fmt.Errorf("could not create request for OpenID configuration: %v", err) + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("could not get issuer OpenID well-known configuration: %v", err) + } + defer resp.Body.Close() + + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("could not parse well-known response body: %v", err) + } + + var oidcConfig struct { + JwksURL string `json:"jwks_uri"` + } + + if err := json.Unmarshal(respBytes, &oidcConfig); err != nil { + return nil, fmt.Errorf("could not unmarshal OpenID config: %v", err) + } + + // grab the provider's JWKS which contains the pubkey to verify token signatures + req, err = http.NewRequestWithContext(ctx, "GET", oidcConfig.JwksURL, nil) + if err != nil { + return nil, fmt.Errorf("could not create request for JWKS: %v", err) + } + resp, err = client.Do(req) + if err != nil { + return nil, fmt.Errorf("could not get issuer OpenID well-known JWKS configuration: %v", err) + } + defer resp.Body.Close() + + respBytes, err = io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("could not parse well-known JWKS response body: %v", err) + } + + var issuerJWKS jwks + if err := json.Unmarshal(respBytes, &issuerJWKS); err != nil { + return nil, fmt.Errorf("could not unmarshal JWKS: %v", err) + } + + return &issuerJWKS, nil +} + +func extractRSAPubKeyFunc(issuerJWKS *jwks) func(*jwt.Token) (any, error) { + return func(token *jwt.Token) (any, error) { + for _, key := range issuerJWKS.Keys { + if key.KID == token.Header["kid"] { + switch key.Alg { + case "RS256": + n, err := base64.RawURLEncoding.DecodeString(key.N) + if err != nil { + return nil, fmt.Errorf("could not decode N: %v", err) + } + e, err := base64.RawURLEncoding.DecodeString(key.E) + if err != nil { + return nil, fmt.Errorf("could not decode E: %v", err) + } + + pubkey := &rsa.PublicKey{ + N: new(big.Int).SetBytes(n), + E: int(new(big.Int).SetBytes(e).Int64()), + } + + return pubkey, nil + } + + return nil, fmt.Errorf("unexpected signing algorithm for key '%s': %s", key.KID, key.Alg) + } + } + + return nil, fmt.Errorf("could not find an RSA key for signing use in the provided JWKS") + } +} + +func checkFeatureGatesOrSkip(ctx context.Context, t testing.TB, configClient *configclient.Clientset, features ...configv1.FeatureGateName) { + featureGates, err := configClient.ConfigV1().FeatureGates().Get(ctx, "cluster", metav1.GetOptions{}) + require.NoError(t, err) + + if len(featureGates.Status.FeatureGates) != 1 { + // fail test if there are multiple feature gate versions (i.e. ongoing upgrade) + t.Fatalf("multiple feature gate versions detected") + } + + atLeastOneFeatureEnabled := false + for _, feature := range features { + for _, gate := range featureGates.Status.FeatureGates[0].Enabled { + if gate.Name == feature { + atLeastOneFeatureEnabled = true + break + } + } + + if atLeastOneFeatureEnabled { + break + } + } + + if !atLeastOneFeatureEnabled { + t.Skipf("skipping as none of the feature gates in %v are enabled", features) + } +} + +func authSpecForOIDCProvider(idpName, idpURL, caBundleName, groupsClaim string, oidcClientID configv1.TokenAudience) *configv1.AuthenticationSpec { + spec := configv1.AuthenticationSpec{ + Type: configv1.AuthenticationTypeOIDC, + WebhookTokenAuthenticator: nil, + OIDCProviders: []configv1.OIDCProvider{ + { + Name: idpName, + Issuer: configv1.TokenIssuer{ + URL: idpURL, + Audiences: []configv1.TokenAudience{oidcClientID}, + CertificateAuthority: configv1.ConfigMapNameReference{ + Name: caBundleName, + }, + }, + ClaimMappings: configv1.TokenClaimMappings{ + Username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.Prefix, + Prefix: &configv1.UsernamePrefix{ + PrefixString: "oidc-test:", + }, + }, + Groups: configv1.PrefixedClaimMapping{ + TokenClaimMapping: configv1.TokenClaimMapping{ + Claim: groupsClaim, + }, + }, + }, + }, + }, + } + + return &spec +} + +type testClient struct { + kubeConfig *rest.Config + kubeClient *kubernetes.Clientset + configClient *configclient.Clientset + operatorClient v1helpers.OperatorClient + operatorConfigClient *operatorversionedclient.Clientset + oauthClient oauthclient.Interface + routeClient routeclient.Interface + apiregistrationClient apiregistrationclient.Interface +} + +func newTestClient(ctx context.Context, t testing.TB) (*testClient, error) { + tc := &testClient{ + kubeConfig: test.NewClientConfigForTest(t), + } + + var err error + tc.kubeClient, err = kubernetes.NewForConfig(tc.kubeConfig) + if err != nil { + return nil, err + } + + tc.configClient, err = configclient.NewForConfig(tc.kubeConfig) + if err != nil { + return nil, err + } + + tc.operatorConfigClient, err = operatorversionedclient.NewForConfig(tc.kubeConfig) + if err != nil { + return nil, err + } + + tc.oauthClient, err = oauthclient.NewForConfig(tc.kubeConfig) + if err != nil { + return nil, err + } + + tc.routeClient, err = routeclient.NewForConfig(tc.kubeConfig) + if err != nil { + return nil, err + } + + tc.apiregistrationClient, err = apiregistrationclient.NewForConfig(tc.kubeConfig) + if err != nil { + return nil, err + } + + var dynamicInformers dynamicinformer.DynamicSharedInformerFactory + tc.operatorClient, dynamicInformers, err = genericoperatorclient.NewClusterScopedOperatorClient( + clock.RealClock{}, + tc.kubeConfig, + operatorv1.GroupVersion.WithResource("authentications"), + operatorv1.GroupVersion.WithKind("Authentication"), + operator.ExtractOperatorSpec, + operator.ExtractOperatorStatus, + ) + if err != nil { + return nil, err + } + + dynamicInformers.Start(ctx.Done()) + dynamicInformers.WaitForCacheSync(ctx.Done()) + + return tc, nil +} + +func (tc *testClient) getAuth(ctx context.Context, t testing.TB) *configv1.Authentication { + auth, err := tc.configClient.ConfigV1().Authentications().Get(ctx, "cluster", metav1.GetOptions{}) + require.NoError(t, err, "failed to get authentication/cluster") + require.NotNil(t, auth) + + return auth +} + +// updateAuthResource deep-copies the baseSpec, applies updates to the copy and persists them in the auth resource +func (tc *testClient) updateAuthResource(ctx context.Context, t testing.TB, baseSpec *configv1.AuthenticationSpec, updateAuthSpec func(baseSpec *configv1.AuthenticationSpec)) (*configv1.Authentication, error) { + auth := tc.getAuth(ctx, t) + if updateAuthSpec == nil { + return auth, nil + } + + spec := baseSpec.DeepCopy() + updateAuthSpec(spec) + + auth.Spec = *spec + auth, err := tc.configClient.ConfigV1().Authentications().Update(ctx, auth, metav1.UpdateOptions{}) + if err != nil { + return nil, err + } + + require.True(t, equality.Semantic.DeepEqual(auth.Spec, *spec)) + + return auth, nil +} + +func (tc *testClient) checkPreconditions(ctx context.Context, t testing.TB, authType *configv1.AuthenticationType, caoStatus []configv1.ClusterOperatorStatusCondition, kasoStatus []configv1.ClusterOperatorStatusCondition) { + var preconditionErr error + waitErr := wait.PollUntilContextTimeout(ctx, 30*time.Second, 20*time.Minute, false, func(ctx context.Context) (bool, error) { + preconditionErr = nil + if authType != nil { + expected := *authType + if len(expected) == 0 { + expected = configv1.AuthenticationTypeIntegratedOAuth + } + + auth := tc.getAuth(ctx, t) + actual := auth.Spec.Type + if len(actual) == 0 { + actual = configv1.AuthenticationTypeIntegratedOAuth + } + + if expected != actual { + preconditionErr = fmt.Errorf("unexpected auth type; test requires '%s', but got '%s'", expected, actual) + return false, nil + } + } + + if len(caoStatus) > 0 { + ok, conditions, err := test.CheckClusterOperatorStatus(ctx, t, tc.configClient.ConfigV1(), "authentication", caoStatus...) + if err != nil { + preconditionErr = fmt.Errorf("could not determine authentication operator status: %v", err) + return false, nil + } else if !ok { + preconditionErr = fmt.Errorf("unexpected authentication operator status: %v", conditions) + return false, nil + } + } + + if len(kasoStatus) > 0 { + ok, conditions, err := test.CheckClusterOperatorStatus(ctx, t, tc.configClient.ConfigV1(), "kube-apiserver", kasoStatus...) + if err != nil { + preconditionErr = fmt.Errorf("could not determine kube-apiserver operator status: %v", err) + return false, nil + } else if !ok { + preconditionErr = fmt.Errorf("unexpected kube-apiserver operator status: %v", conditions) + return false, nil + } + } + + return true, nil + }) + + require.NoError(t, preconditionErr, "failed to assert preconditions: %v", preconditionErr) + require.NoError(t, waitErr, "failed to wait for test preconditions: %v", waitErr) +} + +func (tc *testClient) kasLatestAvailableRevision(ctx context.Context, t testing.TB) int32 { + kas, err := tc.operatorConfigClient.OperatorV1().KubeAPIServers().Get(ctx, "cluster", metav1.GetOptions{}) + require.NoError(t, err, "failed to get kubeapiserver/cluster") + return kas.Status.LatestAvailableRevision +} + +func (tc *testClient) validateKASConfig(ctx context.Context, t testing.TB) int32 { + kas, err := tc.operatorConfigClient.OperatorV1().KubeAPIServers().Get(ctx, "cluster", metav1.GetOptions{}) + require.NoError(t, err) + + var observedConfig map[string]any + err = json.Unmarshal(kas.Spec.ObservedConfig.Raw, &observedConfig) + require.NoError(t, err) + + apiServerArguments := observedConfig["apiServerArguments"].(map[string]any) + + require.Nil(t, apiServerArguments["authentication-token-webhook-config-file"]) + require.Nil(t, apiServerArguments["authentication-token-webhook-version"]) + require.Nil(t, observedConfig["authConfig"]) + + authConfigArg := apiServerArguments["authentication-config"].([]any) + require.NotEmpty(t, authConfigArg) + require.Equal(t, authConfigArg[0].(string), "/etc/kubernetes/static-pod-resources/configmaps/auth-config/auth-config.json") + + return kas.Status.LatestAvailableRevision +} + +func (tc *testClient) validateAuthConfigJSON(ctx context.Context, t testing.TB, authSpec *configv1.AuthenticationSpec, usernamePrefix, groupsClaim, groupsPrefix string, kasRevision int32) { + idpURL := authSpec.OIDCProviders[0].Issuer.URL + caBundleName := authSpec.OIDCProviders[0].Issuer.CertificateAuthority.Name + certData := "" + if len(caBundleName) > 0 { + cm, err := tc.kubeClient.CoreV1().ConfigMaps("openshift-config").Get(ctx, caBundleName, metav1.GetOptions{}) + require.NoError(t, err) + certData = cm.Data["ca-bundle.crt"] + } + + authConfigJSONTemplate := `{"kind":"AuthenticationConfiguration","apiVersion":"apiserver.config.k8s.io/v1beta1","jwt":[{"issuer":{"url":"%s","certificateAuthority":"%s","audiences":[%s],"audienceMatchPolicy":"MatchAny"},"claimMappings":{"username":{"claim":"%s","prefix":"%s"},"groups":{"claim":"%s","prefix":"%s"},"uid":{}}}]}` + // If the ExternalOIDCWithUIDAndExtraClaimMappings feature gate is enabled, default the uid claim to "sub" + if featureGateEnabled(ctx, tc.configClient, features.FeatureGateExternalOIDCWithAdditionalClaimMappings) { + authConfigJSONTemplate = `{"kind":"AuthenticationConfiguration","apiVersion":"apiserver.config.k8s.io/v1beta1","jwt":[{"issuer":{"url":"%s","certificateAuthority":"%s","audiences":[%s],"audienceMatchPolicy":"MatchAny"},"claimMappings":{"username":{"claim":"%s","prefix":"%s"},"groups":{"claim":"%s","prefix":"%s"},"uid":{"claim":"sub"}}}]}` + } + + expectedAuthConfigJSON := fmt.Sprintf(authConfigJSONTemplate, + idpURL, + strings.ReplaceAll(certData, "\n", "\\n"), + strings.Join([]string{fmt.Sprintf(`"%s"`, oidcClientID)}, ","), + authSpec.OIDCProviders[0].ClaimMappings.Username.Claim, + usernamePrefix, + groupsClaim, + groupsPrefix, + ) + + for _, cm := range []struct { + ns string + name string + }{ + {"openshift-config-managed", "auth-config"}, + {"openshift-kube-apiserver", "auth-config"}, + {"openshift-kube-apiserver", fmt.Sprintf("auth-config-%d", kasRevision)}, + } { + actualCM, err := tc.kubeClient.CoreV1().ConfigMaps(cm.ns).Get(ctx, cm.name, metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, expectedAuthConfigJSON, actualCM.Data["auth-config.json"], "unexpected auth-config.json contents in %s/%s", actualCM.Namespace, actualCM.Name) + } +} + +func (tc *testClient) validateOAuthState(ctx context.Context, t testing.TB, requireMissing, newExternalOIDCArchitectureEnabled bool) { + dynamicClient, err := dynamic.NewForConfig(tc.kubeConfig) + require.NoError(t, err, "unexpected error while creating dynamic client") + + var validationErrs []error + waitErr := wait.PollUntilContextTimeout(ctx, 30*time.Second, 5*time.Minute, false, func(_ context.Context) (bool, error) { + validationErrs = make([]error, 0) + validationErrs = append(validationErrs, validateOAuthResources(ctx, dynamicClient, requireMissing)...) + validationErrs = append(validationErrs, validateOAuthRoutes(ctx, tc.routeClient, tc.configClient, requireMissing)...) + validationErrs = append(validationErrs, validateOAuthControllerConditions(tc.operatorClient, requireMissing)...) + validationErrs = append(validationErrs, validateOperandVersions(ctx, tc.configClient, requireMissing)...) + validationErrs = append(validationErrs, validateOAuthRelatedObjects(ctx, tc.configClient, requireMissing)...) + return len(validationErrs) == 0, nil + }) + + require.NoError(t, utilerrors.NewAggregate(validationErrs), "failed to validate OAuth state") + require.NoError(t, waitErr, "failed to wait for OAuth state validation") +} + +func validateOAuthResources(ctx context.Context, dynamicClient *dynamic.DynamicClient, requireMissing bool) []error { + errs := make([]error, 0) + for _, obj := range []struct { + gvr schema.GroupVersionResource + namespace string + name string + }{ + {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}, "openshift-authentication", "v4-0-config-system-cliconfig"}, + {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}, "openshift-authentication", "v4-0-config-system-metadata"}, + {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}, "openshift-authentication", "v4-0-config-system-service-ca"}, + {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}, "openshift-authentication", "v4-0-config-system-trusted-ca-bundle"}, + {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}, "openshift-config-managed", "oauth-serving-cert"}, + {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"}, "openshift-authentication", "v4-0-config-system-ocp-branding-template"}, + {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"}, "openshift-authentication", "v4-0-config-system-session"}, + {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"}, "openshift-config", "webhook-authentication-integrated-oauth"}, + {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "serviceaccounts"}, "openshift-authentication", "oauth-openshift"}, + {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "serviceaccounts"}, "openshift-oauth-apiserver", "oauth-apiserver-sa"}, + {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services"}, "openshift-authentication", "oauth-openshift"}, + {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services"}, "openshift-oauth-apiserver", "api"}, + {schema.GroupVersionResource{Group: "apiregistration.k8s.io", Version: "v1", Resource: "apiservices"}, "", "v1.oauth.openshift.io"}, + {schema.GroupVersionResource{Group: "apiregistration.k8s.io", Version: "v1", Resource: "apiservices"}, "", "v1.user.openshift.io"}, + {schema.GroupVersionResource{Group: "oauth.openshift.io", Version: "v1", Resource: "oauthclients"}, "", "openshift-browser-client"}, + {schema.GroupVersionResource{Group: "oauth.openshift.io", Version: "v1", Resource: "oauthclients"}, "", "openshift-challenging-client"}, + {schema.GroupVersionResource{Group: "oauth.openshift.io", Version: "v1", Resource: "oauthclients"}, "", "openshift-cli-client"}, + {schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"}, "", "system:openshift:oauth-apiserver"}, + {schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"}, "", "system:openshift:openshift-authentication"}, + {schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"}, "", "system:openshift:useroauthaccesstoken-manager"}, + {schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterroles"}, "", "system:openshift:useroauthaccesstoken-manager"}, + {schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "rolebindings"}, "openshift-config-managed", "system:openshift:oauth-servercert-trust"}, + {schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "roles"}, "openshift-config-managed", "system:openshift:oauth-servercert-trust"}, + } { + _, err := dynamicClient.Resource(obj.gvr).Namespace(obj.namespace).Get(ctx, obj.name, metav1.GetOptions{}) + if err != nil && !errors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("unexpected error while getting resource %s/%s: %v", obj.namespace, obj.name, err)) + } else if requireMissing != errors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("resource %s '%s/%s' wanted missing: %v; got: %v (error: %v)", obj.gvr.String(), obj.namespace, obj.name, requireMissing, errors.IsNotFound(err), err)) + } + } + + return errs +} + +func validateOAuthRoutes(ctx context.Context, routeClient routeclient.Interface, configClient *configclient.Clientset, requireMissing bool) []error { + errs := make([]error, 0) + for _, obj := range []struct{ namespace, name string }{ + {"openshift-authentication", "oauth-openshift"}, + } { + _, err := routeClient.RouteV1().Routes(obj.namespace).Get(ctx, obj.name, metav1.GetOptions{}) + if err != nil && !errors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("unexpected error while getting route %s/%s: %v", obj.namespace, obj.name, err)) + } else if requireMissing != errors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("route %s/%s wanted missing: %v; got: %v", obj.namespace, obj.name, requireMissing, !errors.IsNotFound((err)))) + } + + // ingress status + ingress, err := configClient.ConfigV1().Ingresses().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return append(errs, err) + } + + found := false + for _, route := range ingress.Status.ComponentRoutes { + if route.Name == obj.name && route.Namespace == obj.namespace { + found = true + break + } + } + + if !requireMissing && !found { + errs = append(errs, fmt.Errorf("route %s required but was not found", obj)) + } else if requireMissing && found { + errs = append(errs, fmt.Errorf("route %s required to be missing but was found", obj)) + } + } + + return errs +} + +func validateOAuthControllerConditions(operatorClient v1helpers.OperatorClient, requireMissing bool) []error { + errs := make([]error, 0) + controllerConditionTypes := sets.New[string]( + // endpointAccessibleController + "OAuthServerRouteEndpointAccessibleControllerAvailable", + "OAuthServerServiceEndpointAccessibleControllerAvailable", + "OAuthServerServiceEndpointsEndpointAccessibleControllerAvailable", + // payloadConfigController + "OAuthConfigDegraded", + "OAuthSessionSecretDegraded", + "OAuthConfigRouteDegraded", + "OAuthConfigIngressDegraded", + "OAuthConfigServiceDegraded", + // ingressNodesAvailableController + "ReadyIngressNodesAvailable", + // ingressStateController + "IngressStateEndpointsDegraded", + "IngressStatePodsDegraded", + // metadataController + "IngressConfigDegraded", + "AuthConfigDegraded", + "OAuthSystemMetadataDegraded", + // routerCertsDomainValidationController + "RouterCertsDegraded", + // serviceCAController + "OAuthServiceDegraded", + "SystemServiceCAConfigDegraded", + // webhookAuthenticatorController + "AuthenticatorCertKeyProgressing", + // wellKnownReadyController + "WellKnownAvailable", + "WellKnownReadyControllerDegradationObserved", + ) + + _, operatorStatus, _, err := operatorClient.GetOperatorState() + if err != nil { + return append(errs, err) + } + + allConditions := sets.New[string]() + for _, condition := range operatorStatus.Conditions { + allConditions.Insert(condition.Type) + } + + if requireMissing { + // no controller conditions must exist in operator status + if intersection := controllerConditionTypes.Intersection(allConditions); intersection.Len() > 0 { + return append(errs, fmt.Errorf("expected conditions to be missing but were found: %v", intersection.UnsortedList())) + } + return nil + } + + if diff := controllerConditionTypes.Difference(allConditions); diff.Len() > 0 { + // all controller conditions must exist in operator status + return append(errs, fmt.Errorf("expected conditions to exist, but were not found: %v", diff.UnsortedList())) + } + + return nil +} + +func validateOperandVersions(ctx context.Context, cfgClient *configclient.Clientset, requireMissing bool) []error { + operands := sets.New("oauth-apiserver", "oauth-openshift") + + authnClusterOperator, err := cfgClient.ConfigV1().ClusterOperators().Get(ctx, "authentication", metav1.GetOptions{}) + if err != nil { + return []error{fmt.Errorf("fetching authentication ClusterOperator: %w", err)} + } + + foundOperands := []string{} + for _, version := range authnClusterOperator.Status.Versions { + if operands.Has(version.Name) { + foundOperands = append(foundOperands, version.Name) + } + } + + if requireMissing && len(foundOperands) > 0 { + return []error{fmt.Errorf("authentication ClusterOperator status has operands %v in versions when they should be unset", foundOperands)} + } + + foundSet := sets.New(foundOperands...) + if !requireMissing && !foundSet.Equal(operands) { + return []error{fmt.Errorf("authentication ClusterOperator status expected to have operands %v in versions but got %v", operands.UnsortedList(), foundOperands)} + } + + return nil +} + +func validateOAuthRelatedObjects(ctx context.Context, configClient *configclient.Clientset, requireMissing bool) []error { + co, err := configClient.ConfigV1().ClusterOperators().Get(ctx, "authentication", metav1.GetOptions{}) + if err != nil { + return []error{err} + } + + oauthRelatedObjects := []configv1.ObjectReference{ + {Group: routev1.GroupName, Resource: "routes", Name: "oauth-openshift", Namespace: "openshift-authentication"}, + {Resource: "services", Name: "oauth-openshift", Namespace: "openshift-authentication"}, + } + + errs := make([]error, 0) + for _, oauthObj := range oauthRelatedObjects { + found := false + for _, existingObj := range co.Status.RelatedObjects { + if oauthObj.Group == existingObj.Group && + oauthObj.Resource == existingObj.Resource && + oauthObj.Name == existingObj.Name && + oauthObj.Namespace == existingObj.Namespace { + found = true + break + } + } + + if requireMissing && found { + errs = append(errs, fmt.Errorf("oauth related object %s/%s %s/%s should be missing but was found in RelatedObjects", + oauthObj.Group, oauthObj.Resource, oauthObj.Namespace, oauthObj.Name)) + } else if !requireMissing && !found { + errs = append(errs, fmt.Errorf("oauth related object %s/%s %s/%s should be present but was not found in RelatedObjects", + oauthObj.Group, oauthObj.Resource, oauthObj.Namespace, oauthObj.Name)) + } + } + + return errs +} + +func (tc *testClient) testOIDCAuthentication(ctx context.Context, t testing.TB, kcClient *test.KeycloakClient, usernameClaim, usernamePrefix string, expectAuthSuccess bool) { + // re-authenticate to ensure we always have a fresh token + var err error + waitErr := wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) { + err = kcClient.AuthenticatePassword(ctx, oidcClientID, "", "admin", "password") + return err == nil, nil + }) + require.NoError(t, err, "failed to authenticate to keycloak: %v", err) + require.NoError(t, waitErr, "failed to wait for keycloak authentication: %v", waitErr) + + group := names.SimpleNameGenerator.GenerateName("e2e-keycloak-group-") + err = kcClient.CreateGroup(ctx, group) + require.NoError(t, err) + + user := names.SimpleNameGenerator.GenerateName("e2e-keycloak-user-") + email := fmt.Sprintf("%s@test.dev", user) + password := "password" + firstName := "Homer" + lastName := "Simpson" + err = kcClient.CreateUser(ctx, + user, + email, + password, + []string{group}, + map[string]string{ + "firstName": firstName, + "lastName": lastName, + }, + ) + require.NoError(t, err) + + // use a keycloak client for the user created above to fetch its tokens + transport, err := rest.TransportFor(tc.kubeConfig) + require.NoError(t, err) + userClient := test.KeycloakClientFor(t, transport, kcClient.IssuerURL(), "master") + err = userClient.AuthenticatePassword(ctx, oidcClientID, "", user, password) + require.NoError(t, err) + accessTokenStr, idTokenStr := userClient.Tokens() + require.NotEmpty(t, accessTokenStr, "access token must not be empty") + require.NotEmpty(t, idTokenStr, "id token must not be empty") + + // fetch issuer's JWKS and use it to parse JWT tokens + issuerJWKS, err := fetchIssuerJWKS(ctx, kcClient.IssuerURL()) + require.NoError(t, err) + require.NotNil(t, issuerJWKS) + keyfunc := extractRSAPubKeyFunc(issuerJWKS) + + accessToken, err := jwt.ParseWithClaims(accessTokenStr, &expectedClaims{}, keyfunc) + require.NoError(t, err) + require.NotNil(t, accessToken) + + idToken, err := jwt.ParseWithClaims(idTokenStr, &expectedClaims{}, keyfunc) + require.NoError(t, err) + require.NotNil(t, idToken) + + // validate the contents of the OIDC tokens + actualAccessTokenClaims := accessToken.Claims.(*expectedClaims) + require.True(t, accessToken.Valid) + require.Equal(t, userClient.IssuerURL(), actualAccessTokenClaims.Issuer) + require.Equal(t, user, actualAccessTokenClaims.PreferredUsername) + require.Equal(t, email, actualAccessTokenClaims.Email) + require.Equal(t, "Bearer", actualAccessTokenClaims.Type) + require.Equal(t, firstName, actualAccessTokenClaims.GivenName) + require.Equal(t, lastName, actualAccessTokenClaims.FamilyName) + require.Equal(t, fmt.Sprintf("%s %s", firstName, lastName), actualAccessTokenClaims.Name) + require.NotEmpty(t, actualAccessTokenClaims.Subject) + + actualIDTokenClaims := idToken.Claims.(*expectedClaims) + require.True(t, idToken.Valid) + require.Equal(t, userClient.IssuerURL(), actualIDTokenClaims.Issuer) + require.Equal(t, user, actualIDTokenClaims.PreferredUsername) + require.Equal(t, email, actualIDTokenClaims.Email) + require.Equal(t, "ID", actualIDTokenClaims.Type) + require.Equal(t, jwt.ClaimStrings{oidcClientID}, actualIDTokenClaims.Audience) + require.Equal(t, firstName, actualIDTokenClaims.GivenName) + require.Equal(t, lastName, actualIDTokenClaims.FamilyName) + require.Equal(t, fmt.Sprintf("%s %s", firstName, lastName), actualIDTokenClaims.Name) + require.NotEmpty(t, actualIDTokenClaims.Subject) + + // test authentication via the kube-apiserver + // create a new kube client that uses the OIDC id_token as a bearer token + kubeConfig := rest.AnonymousClientConfig(tc.kubeConfig) + kubeConfig.BearerToken = idTokenStr + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + require.NoError(t, err) + + ssr, err := kubeClient.AuthenticationV1().SelfSubjectReviews().Create(ctx, &authenticationv1.SelfSubjectReview{}, metav1.CreateOptions{}) + if expectAuthSuccess { + // test authentication with the OIDC token using a self subject review + expectedUsername := "" + switch usernameClaim { + case "email": + expectedUsername = usernamePrefix + email + case "sub": + expectedUsername = usernamePrefix + actualIDTokenClaims.Subject + default: + t.Fatalf("unexpected username claim: %s", usernameClaim) + } + + require.NoError(t, err) + require.NotNil(t, ssr) + require.Contains(t, ssr.Status.UserInfo.Groups, "system:authenticated") + require.Equal(t, expectedUsername, ssr.Status.UserInfo.Username) + } else { + require.Error(t, err) + require.True(t, errors.IsUnauthorized(err)) + } +} + +func (tc *testClient) requireKASRolloutSuccessful(testCtx context.Context, t testing.TB, authSpec *configv1.AuthenticationSpec, kasOriginalRevision int32, expectedUsernamePrefix string) { + // wait for KAS rollout + err := test.WaitForNewKASRollout(testCtx, t, tc.operatorConfigClient.OperatorV1().KubeAPIServers(), kasOriginalRevision) + require.NoError(t, err, "failed to wait for KAS rollout") + + kasRevision := tc.validateKASConfig(testCtx, t) + tc.validateAuthConfigJSON(testCtx, t, authSpec, expectedUsernamePrefix, oidcGroupsClaim, oidcGroupsPrefix, kasRevision) +} + +func (tc *testClient) authResourceRollback(ctx context.Context, origAuthSpec *configv1.AuthenticationSpec) error { + const authName = "cluster" + auth, err := tc.configClient.ConfigV1().Authentications().Get(ctx, authName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("rollback failed for authentication '%s' while retrieving fresh object: %v", authName, err) + } + + if !equality.Semantic.DeepEqual(auth.Spec, *origAuthSpec) { + auth.Spec = *origAuthSpec + if _, err := tc.configClient.ConfigV1().Authentications().Update(ctx, auth, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("rollback failed for authentication '%s' while updating object: %v", authName, err) + } + } + + return nil +} + +func featureGateEnabled(ctx context.Context, configClient *configclient.Clientset, feature configv1.FeatureGateName) bool { + featureGates, err := configClient.ConfigV1().FeatureGates().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return false + } + + if len(featureGates.Status.FeatureGates) == 0 { + return false + } + + for _, enabled := range featureGates.Status.FeatureGates[0].Enabled { + if enabled.Name == feature { + return true + } + } + + return false +} diff --git a/test/e2e-oidc/external_oidc_test.go b/test/e2e-oidc/external_oidc_test.go index 298b12b94a..fc3bacaea2 100644 --- a/test/e2e-oidc/external_oidc_test.go +++ b/test/e2e-oidc/external_oidc_test.go @@ -1,1141 +1,17 @@ -package e2e +package e2e_oidc import ( "context" - "crypto/rsa" - "crypto/tls" - "encoding/base64" - "encoding/json" - "fmt" - "io" - "math/big" - "net/http" - "strings" "testing" - "time" - - configv1 "github.com/openshift/api/config/v1" - "github.com/openshift/api/features" - operatorv1 "github.com/openshift/api/operator/v1" - routev1 "github.com/openshift/api/route/v1" - configclient "github.com/openshift/client-go/config/clientset/versioned" - oauthclient "github.com/openshift/client-go/oauth/clientset/versioned" - operatorversionedclient "github.com/openshift/client-go/operator/clientset/versioned" - routeclient "github.com/openshift/client-go/route/clientset/versioned" - "github.com/openshift/cluster-authentication-operator/pkg/operator" - test "github.com/openshift/cluster-authentication-operator/test/library" - "github.com/openshift/library-go/pkg/operator/genericoperatorclient" - "github.com/openshift/library-go/pkg/operator/v1helpers" - "github.com/stretchr/testify/require" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" - "k8s.io/utils/clock" - "k8s.io/utils/ptr" - - authenticationv1 "k8s.io/api/authentication/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/storage/names" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/dynamic/dynamicinformer" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - - "github.com/golang-jwt/jwt/v5" -) - -const ( - oidcClientId = "admin-cli" - oidcGroupsClaim = "groups" - oidcGroupsPrefix = "" - - managedNS = "openshift-config-managed" - authCM = "auth-config" ) +// This test function allows the e2e-oidc test to be run via standard `go test` command. +// It calls the shared test implementation which is also used by the Ginkgo/OTE framework. +// +// This situation is temporary until we verify the new e2e-oidc-ote CI job. +// Eventually all tests will be run only as part of the OTE framework. func TestExternalOIDCWithKeycloak(t *testing.T) { - testCtx := t.Context() - testClient, err := newTestClient(t, testCtx) - require.NoError(t, err) - - checkFeatureGatesOrSkip(t, testCtx, testClient.configClient, features.FeatureGateExternalOIDC, features.FeatureGateExternalOIDCWithAdditionalClaimMappings) - - newExternalOIDCArchitectureEnabled := featureGateEnabled(testCtx, testClient.configClient, features.FeatureGateExternalOIDCExternalClaimsSourcing) - - // post-test cluster cleanup - var cleanups []func() - defer test.IDPCleanupWrapper(func() { - t.Logf("cleaning up after test") - ts := time.Now() - for _, c := range cleanups { - c() - } - t.Logf("cleanup completed after %s", time.Since(ts)) - })() - - origAuthSpec := (*testClient.getAuth(t, testCtx)).Spec.DeepCopy() - cleanups = append(cleanups, func() { - kasOriginalRevision := testClient.kasLatestAvailableRevision(t, testCtx) - - err := testClient.authResourceRollback(testCtx, origAuthSpec) - require.NoError(t, err, "failed to rollback auth resource during cleanup") - - // KAS should not need to perform a rollout when the new architecture is enabled. - // The oauth-apiserver will, but that should be fairly quick and handled by cluster operator - // stability checks prior to running tests. - if !newExternalOIDCArchitectureEnabled { - err = test.WaitForNewKASRollout(t, testCtx, testClient.operatorConfigClient.OperatorV1().KubeAPIServers(), kasOriginalRevision) - require.NoError(t, err, "failed to wait for KAS rollout during cleanup") - } - - testClient.validateOAuthState(t, testCtx, false, newExternalOIDCArchitectureEnabled) - }) - - // keycloak setup - var idpName string - var kcClient *test.KeycloakClient - kcClient, idpName, c := test.AddKeycloakIDP(t, testClient.kubeConfig, true) - cleanups = append(cleanups, c...) - t.Logf("keycloak Admin URL: %s", kcClient.AdminURL()) - - // default-ingress-cert is copied to openshift-config and used as the CA for the IdP - // see test/library/idpdeployment.go:332 - caBundleName := idpName + "-ca" - idpURL := kcClient.IssuerURL() - - // run tests - - testSpec := authSpecForOIDCProvider(idpName, idpURL, caBundleName, oidcGroupsClaim, oidcClientId) - - typeOAuth := ptr.To(configv1.AuthenticationTypeIntegratedOAuth) - typeOIDC := ptr.To(configv1.AuthenticationTypeOIDC) - operatorAvailable := []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorDegraded, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorUpgradeable, Status: configv1.ConditionTrue}, - } - - t.Run("auth-config cm must not exist and gets deleted by the CAO if manually created when type not OIDC", func(t *testing.T) { - testClient.checkPreconditions(t, testCtx, typeOAuth, operatorAvailable, nil) - - _, err := testClient.kubeClient.CoreV1().ConfigMaps(managedNS).Get(testCtx, authCM, metav1.GetOptions{}) - require.True(t, errors.IsNotFound(err), "openshift-config-managed/auth-config configmap must be missing") - - // create cm - cm := v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: authCM, - Namespace: managedNS, - }, - Data: map[string]string{ - "test": "value", - }, - } - newCM, err := testClient.kubeClient.CoreV1().ConfigMaps(managedNS).Create(testCtx, &cm, metav1.CreateOptions{}) - require.NoError(t, err) - require.Equal(t, cm.Data, newCM.Data) - - // wait for CAO to delete it - var cmErr error - waitErr := wait.PollUntilContextTimeout(testCtx, 2*time.Second, 1*time.Minute, false, func(ctx context.Context) (bool, error) { - cmErr = nil - _, err := testClient.kubeClient.CoreV1().ConfigMaps(managedNS).Get(testCtx, authCM, metav1.GetOptions{}) - if errors.IsNotFound(err) { - return true, nil - } - cmErr = err - return false, nil - }) - require.NoError(t, cmErr, "failed to get auth configmap: %v", cmErr) - require.NoError(t, waitErr, "failed to wait for auth configmap to get deleted: %v", err) - }) - - t.Run("invalid CEL expression rejects auth CR admission", func(t *testing.T) { - for _, tt := range []struct { - name string - specUpdate func(*configv1.AuthenticationSpec) - requireFeatureGates []configv1.FeatureGateName - }{ - { - name: "uncompilable CEL expression for uid claim mapping", - specUpdate: func(s *configv1.AuthenticationSpec) { - s.OIDCProviders[0].ClaimMappings.UID = &configv1.TokenClaimOrExpressionMapping{ - Expression: "^&*!@#^*(", - } - }, - requireFeatureGates: []configv1.FeatureGateName{features.FeatureGateExternalOIDCWithAdditionalClaimMappings}, - }, - { - name: "uncompilable CEL expression for extras claim mapping", - specUpdate: func(s *configv1.AuthenticationSpec) { - s.OIDCProviders[0].ClaimMappings.Extra = []configv1.ExtraMapping{ - { - Key: "testing/key", - ValueExpression: "^&*!@#^*(", - }, - } - }, - requireFeatureGates: []configv1.FeatureGateName{features.FeatureGateExternalOIDCWithAdditionalClaimMappings}, - }, - } { - t.Run(tt.name, func(t *testing.T) { - for _, fg := range tt.requireFeatureGates { - if !featureGateEnabled(testCtx, testClient.configClient, fg) { - t.Skipf("skipping as required feature gate %q is not enabled", fg) - } - } - _, err := testClient.updateAuthResource(t, testCtx, testSpec, tt.specUpdate) - require.Error(t, err, "uncompilable CEL expression should return in admission error") - }) - } - }) - - t.Run("invalid OIDC config degrades auth operator", func(t *testing.T) { - for _, tt := range []struct { - name string - specUpdate func(*configv1.AuthenticationSpec) - requireFeatureGates []configv1.FeatureGateName - }{ - { - name: "invalid issuer CA bundle", - specUpdate: func(s *configv1.AuthenticationSpec) { - s.OIDCProviders[0].Issuer.CertificateAuthority.Name = "invalid-ca-bundle" - }, - requireFeatureGates: []configv1.FeatureGateName{}, - }, - { - name: "invalid issuer URL", - specUpdate: func(s *configv1.AuthenticationSpec) { - s.OIDCProviders[0].Issuer.URL = "https://invalid-idp.testing" - }, - requireFeatureGates: []configv1.FeatureGateName{}, - }, - } { - t.Run(tt.name, func(t *testing.T) { - for _, fg := range tt.requireFeatureGates { - if !featureGateEnabled(testCtx, testClient.configClient, fg) { - t.Skipf("skipping as required feature gate %q is not enabled", fg) - } - } - - err := testClient.authResourceRollback(testCtx, origAuthSpec) - require.NoError(t, err, "failed to roll back auth resource") - - testClient.checkPreconditions(t, testCtx, typeOAuth, operatorAvailable, nil) - - _, err = testClient.updateAuthResource(t, testCtx, testSpec, tt.specUpdate) - require.NoError(t, err, "failed to update authentication/cluster") - - require.NoError(t, test.WaitForClusterOperatorDegraded(t, testClient.configClient.ConfigV1(), "authentication")) - - testClient.validateOAuthState(t, testCtx, false, newExternalOIDCArchitectureEnabled) - }) - } - }) - - t.Run("OIDC config rolls out successfully", func(t *testing.T) { - err := testClient.authResourceRollback(testCtx, origAuthSpec) - require.NoError(t, err, "failed to roll back auth resource") - - for _, tt := range []struct { - claim string - prefixPolicy configv1.UsernamePrefixPolicy - prefix *configv1.UsernamePrefix - expectedPrefix string - }{ - {"email", configv1.Prefix, &configv1.UsernamePrefix{PrefixString: "oidc-test:"}, "oidc-test:"}, - {"email", configv1.NoPrefix, nil, ""}, - {"sub", configv1.NoOpinion, nil, idpURL + "#"}, - {"email", configv1.NoOpinion, nil, ""}, - } { - policyStr := "NoOpinion" - if len(tt.prefixPolicy) > 0 { - policyStr = string(tt.prefixPolicy) - } - testName := fmt.Sprintf("username claim %s prefix policy %s", tt.claim, policyStr) - t.Run(testName, func(t *testing.T) { - testClient.checkPreconditions(t, testCtx, nil, operatorAvailable, operatorAvailable) - - kasOriginalRevision := testClient.kasLatestAvailableRevision(t, testCtx) - auth, err := testClient.updateAuthResource(t, testCtx, testSpec, func(baseSpec *configv1.AuthenticationSpec) { - baseSpec.OIDCProviders[0].ClaimMappings.Username = configv1.UsernameClaimMapping{ - Claim: tt.claim, - PrefixPolicy: tt.prefixPolicy, - Prefix: tt.prefix, - } - }) - require.NoError(t, err, "failed to update authentication/cluster") - - require.NoError(t, test.WaitForClusterOperatorStatusAlwaysAvailable(t, testCtx, testClient.configClient.ConfigV1(), "authentication")) - - if !newExternalOIDCArchitectureEnabled { - require.NoError(t, test.WaitForClusterOperatorStatusAlwaysAvailable(t, testCtx, testClient.configClient.ConfigV1(), "kube-apiserver")) - testClient.requireKASRolloutSuccessful(t, testCtx, &auth.Spec, kasOriginalRevision, tt.expectedPrefix) - } - - testClient.validateOAuthState(t, testCtx, true, newExternalOIDCArchitectureEnabled) - - testClient.testOIDCAuthentication(t, testCtx, kcClient, tt.claim, tt.expectedPrefix, true) - }) - } - }) - - t.Run("auth-config cm must exist and gets overwritten by the CAO if manually modified when type OIDC", func(t *testing.T) { - testClient.checkPreconditions(t, testCtx, typeOIDC, operatorAvailable, nil) - - cm, err := testClient.kubeClient.CoreV1().ConfigMaps(managedNS).Get(testCtx, authCM, metav1.GetOptions{}) - require.NoError(t, err) - require.NotNil(t, cm) - - orig := cm.DeepCopy() - cm.Data["auth-config.json"] = "manually overwritten" - cm, err = testClient.kubeClient.CoreV1().ConfigMaps(managedNS).Update(testCtx, cm, metav1.UpdateOptions{}) - require.NoError(t, err) - require.NotEqual(t, cm.Data, orig.Data) - - // wait for CAO to overwrite it - var cmErr error - waitErr := wait.PollUntilContextTimeout(testCtx, 2*time.Second, 1*time.Minute, false, func(ctx context.Context) (bool, error) { - cm, cmErr = testClient.kubeClient.CoreV1().ConfigMaps(managedNS).Get(testCtx, authCM, metav1.GetOptions{}) - if err != nil { - return false, nil - } - - return equality.Semantic.DeepEqual(cm.Data, orig.Data), nil - }) - require.NoError(t, cmErr, "failed to get auth configmap: %v", err) - require.NoError(t, waitErr, "failed to wait for auth configmap to get overwritten: %v", err) - }) - - t.Run("OIDC config rolls out successfully but breaks authentication when username claim is unknown", func(t *testing.T) { - testClient.checkPreconditions(t, testCtx, nil, operatorAvailable, operatorAvailable) - - kasOriginalRevision := testClient.kasLatestAvailableRevision(t, testCtx) - auth, err := testClient.updateAuthResource(t, testCtx, testSpec, func(baseSpec *configv1.AuthenticationSpec) { - baseSpec.OIDCProviders[0].ClaimMappings.Username = configv1.UsernameClaimMapping{ - Claim: "unknown", - PrefixPolicy: configv1.NoPrefix, - Prefix: nil, - } - }) - require.NoError(t, err, "failed to update authentication/cluster") - - require.NoError(t, test.WaitForClusterOperatorStatusAlwaysAvailable(t, testCtx, testClient.configClient.ConfigV1(), "authentication")) - - if !newExternalOIDCArchitectureEnabled { - require.NoError(t, test.WaitForClusterOperatorStatusAlwaysAvailable(t, testCtx, testClient.configClient.ConfigV1(), "kube-apiserver")) - testClient.requireKASRolloutSuccessful(t, testCtx, &auth.Spec, kasOriginalRevision, "") - } - - testClient.validateOAuthState(t, testCtx, true, newExternalOIDCArchitectureEnabled) - - testClient.testOIDCAuthentication(t, testCtx, kcClient, "", "", false) - }) -} - -type oidcAuthResponse struct { - AccessToken string `json:"access_token"` - ExpiresIn int `json:"expires_in"` - RefreshToken string `json:"refresh_token"` - RefreshExpiresIn int `json:"refresh_expires_in"` - TokenType string `json:"token_type"` - IdToken string `json:"id_token"` - NotBeforePolicy int `json:"not_before_policy"` - SessionState string `json:"session_state"` - Scope string `json:"scope"` -} - -type expectedClaims struct { - jwt.RegisteredClaims - Email string `json:"email"` - Type string `json:"typ"` - Name string `json:"name"` - GivenName string `json:"given_name"` - FamilyName string `json:"family_name"` - PreferredUsername string `json:"preferred_username"` -} - -type jwks struct { - Keys []struct { - KID string `json:"kid"` - Use string `json:"use"` - KTY string `json:"kty"` - Alg string `json:"alg"` - N string `json:"n"` - E string `json:"e"` - } `json:"keys"` -} - -func fetchIssuerJWKS(issuerURL string) (*jwks, error) { - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - } - - // grab openid-configuration JSON which contains the URL of the provider's JWKS - resp, err := client.Get(issuerURL + "/.well-known/openid-configuration") - if err != nil { - return nil, fmt.Errorf("could not get issuer OpenID well-known configuration: %v", err) - } - defer resp.Body.Close() - - respBytes, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("could not parse well-known response body: %v", err) - } - - var oidcConfig struct { - JwksURL string `json:"jwks_uri"` - } - - if err := json.Unmarshal(respBytes, &oidcConfig); err != nil { - return nil, fmt.Errorf("could not unmarshal OpenID config: %v", err) - } - - // grab the provider's JWKS which contains the pubkey to verify token signatures - resp, err = client.Get(oidcConfig.JwksURL) - if err != nil { - return nil, fmt.Errorf("could not get issuer OpenID well-known JWKS configuration: %v", err) - } - defer resp.Body.Close() - - respBytes, err = io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("could not parse well-known JWKS response body: %v", err) - } - - var issuerJWKS jwks - if err := json.Unmarshal(respBytes, &issuerJWKS); err != nil { - return nil, fmt.Errorf("could not unmarshal JWKS: %v", err) - } - - return &issuerJWKS, nil -} - -func extractRSAPubKeyFunc(issuerJWKS *jwks) func(*jwt.Token) (any, error) { - return func(token *jwt.Token) (any, error) { - for _, key := range issuerJWKS.Keys { - if key.KID == token.Header["kid"] { - switch key.Alg { - case "RS256": - n, err := base64.RawURLEncoding.DecodeString(key.N) - if err != nil { - return nil, fmt.Errorf("could not decode N: %v", err) - } - e, err := base64.RawURLEncoding.DecodeString(key.E) - if err != nil { - return nil, fmt.Errorf("could not decode E: %v", err) - } - - pubkey := &rsa.PublicKey{ - N: new(big.Int).SetBytes(n), - E: int(new(big.Int).SetBytes(e).Int64()), - } - - return pubkey, nil - } - - return nil, fmt.Errorf("unexpected signing algorithm for key '%s': %s", key.KID, key.Alg) - } - } - - return nil, fmt.Errorf("could not find an RSA key for signing use in the provided JWKS") - } -} - -func checkFeatureGatesOrSkip(t *testing.T, ctx context.Context, configClient *configclient.Clientset, features ...configv1.FeatureGateName) { - featureGates, err := configClient.ConfigV1().FeatureGates().Get(ctx, "cluster", metav1.GetOptions{}) - require.NoError(t, err) - - if len(featureGates.Status.FeatureGates) != 1 { - // fail test if there are multiple feature gate versions (i.e. ongoing upgrade) - t.Fatalf("multiple feature gate versions detected") - return - } - - atLeastOneFeatureEnabled := false - for _, feature := range features { - for _, gate := range featureGates.Status.FeatureGates[0].Enabled { - if gate.Name == feature { - atLeastOneFeatureEnabled = true - break - } - } - - if atLeastOneFeatureEnabled { - break - } - } - - if !atLeastOneFeatureEnabled { - t.Skipf("skipping as none of the feature gates in %v are enabled", features) - } -} - -func authSpecForOIDCProvider(idpName, idpURL, caBundleName, groupsClaim string, oidcClientID configv1.TokenAudience) *configv1.AuthenticationSpec { - spec := configv1.AuthenticationSpec{ - Type: configv1.AuthenticationTypeOIDC, - WebhookTokenAuthenticator: nil, - OIDCProviders: []configv1.OIDCProvider{ - { - Name: idpName, - Issuer: configv1.TokenIssuer{ - URL: idpURL, - Audiences: []configv1.TokenAudience{oidcClientID}, - CertificateAuthority: configv1.ConfigMapNameReference{ - Name: caBundleName, - }, - }, - ClaimMappings: configv1.TokenClaimMappings{ - Username: configv1.UsernameClaimMapping{ - Claim: "email", - PrefixPolicy: configv1.Prefix, - Prefix: &configv1.UsernamePrefix{ - PrefixString: "oidc-test:", - }, - }, - Groups: configv1.PrefixedClaimMapping{ - TokenClaimMapping: configv1.TokenClaimMapping{ - Claim: groupsClaim, - }, - }, - }, - }, - }, - } - - return &spec -} - -type testClient struct { - kubeConfig *rest.Config - kubeClient *kubernetes.Clientset - configClient *configclient.Clientset - operatorClient v1helpers.OperatorClient - operatorConfigClient *operatorversionedclient.Clientset - oauthClient oauthclient.Interface - routeClient routeclient.Interface - apiregistrationClient apiregistrationclient.Interface -} - -func newTestClient(t *testing.T, ctx context.Context) (*testClient, error) { - tc := &testClient{ - kubeConfig: test.NewClientConfigForTest(t), - } - - var err error - tc.kubeClient, err = kubernetes.NewForConfig(tc.kubeConfig) - if err != nil { - return nil, err - } - - tc.configClient, err = configclient.NewForConfig(tc.kubeConfig) - if err != nil { - return nil, err - } - - tc.operatorConfigClient, err = operatorversionedclient.NewForConfig(tc.kubeConfig) - if err != nil { - return nil, err - } - - tc.oauthClient, err = oauthclient.NewForConfig(tc.kubeConfig) - if err != nil { - return nil, err - } - - tc.routeClient, err = routeclient.NewForConfig(tc.kubeConfig) - if err != nil { - return nil, err - } - - tc.apiregistrationClient, err = apiregistrationclient.NewForConfig(tc.kubeConfig) - if err != nil { - return nil, err - } - - var dynamicInformers dynamicinformer.DynamicSharedInformerFactory - tc.operatorClient, dynamicInformers, err = genericoperatorclient.NewClusterScopedOperatorClient( - clock.RealClock{}, - tc.kubeConfig, - operatorv1.GroupVersion.WithResource("authentications"), - operatorv1.GroupVersion.WithKind("Authentication"), - operator.ExtractOperatorSpec, - operator.ExtractOperatorStatus, - ) - if err != nil { - return nil, err - } - - dynamicInformers.Start(ctx.Done()) - dynamicInformers.WaitForCacheSync(ctx.Done()) - - return tc, nil -} - -func (tc *testClient) getAuth(t *testing.T, ctx context.Context) *configv1.Authentication { - auth, err := tc.configClient.ConfigV1().Authentications().Get(ctx, "cluster", metav1.GetOptions{}) - require.NoError(t, err, "failed to get authentication/cluster") - require.NotNil(t, auth) - - return auth -} - -// updateAuthResource deep-copies the baseSpec, applies updates to the copy and persists them in the auth resource -func (tc *testClient) updateAuthResource(t *testing.T, ctx context.Context, baseSpec *configv1.AuthenticationSpec, updateAuthSpec func(baseSpec *configv1.AuthenticationSpec)) (*configv1.Authentication, error) { - auth := tc.getAuth(t, ctx) - if updateAuthSpec == nil { - return auth, nil - } - - spec := baseSpec.DeepCopy() - updateAuthSpec(spec) - - auth.Spec = *spec - auth, err := tc.configClient.ConfigV1().Authentications().Update(ctx, auth, metav1.UpdateOptions{}) - if err != nil { - return nil, err - } - - require.True(t, equality.Semantic.DeepEqual(auth.Spec, *spec)) - - return auth, nil -} - -func (tc *testClient) checkPreconditions(t *testing.T, ctx context.Context, authType *configv1.AuthenticationType, caoStatus []configv1.ClusterOperatorStatusCondition, kasoStatus []configv1.ClusterOperatorStatusCondition) { - var preconditionErr error - waitErr := wait.PollUntilContextTimeout(ctx, 30*time.Second, 20*time.Minute, false, func(ctx context.Context) (bool, error) { - preconditionErr = nil - if authType != nil { - expected := *authType - if len(expected) == 0 { - expected = configv1.AuthenticationTypeIntegratedOAuth - } - - auth := tc.getAuth(t, ctx) - actual := auth.Spec.Type - if len(actual) == 0 { - actual = configv1.AuthenticationTypeIntegratedOAuth - } - - if expected != actual { - preconditionErr = fmt.Errorf("unexpected auth type; test requires '%s', but got '%s'", expected, actual) - return false, nil - } - } - - if len(caoStatus) > 0 { - ok, conditions, err := test.CheckClusterOperatorStatus(t, ctx, tc.configClient.ConfigV1(), "authentication", caoStatus...) - if err != nil { - preconditionErr = fmt.Errorf("could not determine authentication operator status: %v", err) - return false, nil - } else if !ok { - preconditionErr = fmt.Errorf("unexpected authentication operator status: %v", conditions) - return false, nil - } - } - - if len(kasoStatus) > 0 { - ok, conditions, err := test.CheckClusterOperatorStatus(t, ctx, tc.configClient.ConfigV1(), "kube-apiserver", kasoStatus...) - if err != nil { - preconditionErr = fmt.Errorf("could not determine kube-apiserver operator status: %v", err) - return false, nil - } else if !ok { - preconditionErr = fmt.Errorf("unexpected kube-apiserver operator status: %v", conditions) - return false, nil - } - } - - return true, nil - }) - - require.NoError(t, preconditionErr, "failed to assert preconditions: %v", preconditionErr) - require.NoError(t, waitErr, "failed to wait for test preconditions: %v", waitErr) -} - -func (tc *testClient) kasLatestAvailableRevision(t *testing.T, ctx context.Context) int32 { - kas, err := tc.operatorConfigClient.OperatorV1().KubeAPIServers().Get(ctx, "cluster", metav1.GetOptions{}) - require.NoError(t, err, "failed to get kubeapiserver/cluster") - return kas.Status.LatestAvailableRevision -} - -func (tc *testClient) validateKASConfig(t *testing.T, ctx context.Context) int32 { - kas, err := tc.operatorConfigClient.OperatorV1().KubeAPIServers().Get(ctx, "cluster", metav1.GetOptions{}) - require.NoError(t, err) - - var observedConfig map[string]any - err = json.Unmarshal(kas.Spec.ObservedConfig.Raw, &observedConfig) - require.NoError(t, err) - - apiServerArguments := observedConfig["apiServerArguments"].(map[string]any) - - require.Nil(t, apiServerArguments["authentication-token-webhook-config-file"]) - require.Nil(t, apiServerArguments["authentication-token-webhook-version"]) - require.Nil(t, observedConfig["authConfig"]) - - authConfigArg := apiServerArguments["authentication-config"].([]any) - require.NotEmpty(t, authConfigArg) - require.Equal(t, authConfigArg[0].(string), "/etc/kubernetes/static-pod-resources/configmaps/auth-config/auth-config.json") - - return kas.Status.LatestAvailableRevision -} - -func (tc *testClient) validateAuthConfigJSON(t *testing.T, ctx context.Context, authSpec *configv1.AuthenticationSpec, usernamePrefix, groupsClaim, groupsPrefix string, kasRevision int32) { - idpURL := authSpec.OIDCProviders[0].Issuer.URL - caBundleName := authSpec.OIDCProviders[0].Issuer.CertificateAuthority.Name - certData := "" - if len(caBundleName) > 0 { - cm, err := tc.kubeClient.CoreV1().ConfigMaps("openshift-config").Get(ctx, caBundleName, metav1.GetOptions{}) - require.NoError(t, err) - certData = cm.Data["ca-bundle.crt"] - } - - authConfigJSONTemplate := `{"kind":"AuthenticationConfiguration","apiVersion":"apiserver.config.k8s.io/v1beta1","jwt":[{"issuer":{"url":"%s","certificateAuthority":"%s","audiences":[%s],"audienceMatchPolicy":"MatchAny"},"claimMappings":{"username":{"claim":"%s","prefix":"%s"},"groups":{"claim":"%s","prefix":"%s"},"uid":{}}}]}` - // If the ExternalOIDCWithUIDAndExtraClaimMappings feature gate is enabled, default the uid claim to "sub" - if featureGateEnabled(ctx, tc.configClient, features.FeatureGateExternalOIDCWithAdditionalClaimMappings) { - authConfigJSONTemplate = `{"kind":"AuthenticationConfiguration","apiVersion":"apiserver.config.k8s.io/v1beta1","jwt":[{"issuer":{"url":"%s","certificateAuthority":"%s","audiences":[%s],"audienceMatchPolicy":"MatchAny"},"claimMappings":{"username":{"claim":"%s","prefix":"%s"},"groups":{"claim":"%s","prefix":"%s"},"uid":{"claim":"sub"}}}]}` - } - - expectedAuthConfigJSON := fmt.Sprintf(authConfigJSONTemplate, - idpURL, - strings.ReplaceAll(certData, "\n", "\\n"), - strings.Join([]string{fmt.Sprintf(`"%s"`, oidcClientId)}, ","), - authSpec.OIDCProviders[0].ClaimMappings.Username.Claim, - usernamePrefix, - groupsClaim, - groupsPrefix, - ) - - for _, cm := range []struct { - ns string - name string - }{ - {"openshift-config-managed", "auth-config"}, - {"openshift-kube-apiserver", "auth-config"}, - {"openshift-kube-apiserver", fmt.Sprintf("auth-config-%d", kasRevision)}, - } { - actualCM, err := tc.kubeClient.CoreV1().ConfigMaps(cm.ns).Get(ctx, cm.name, metav1.GetOptions{}) - require.NoError(t, err) - require.Equal(t, expectedAuthConfigJSON, actualCM.Data["auth-config.json"], "unexpected auth-config.json contents in %s/%s", actualCM.Namespace, actualCM.Name) - } -} - -func (tc *testClient) validateOAuthState(t *testing.T, ctx context.Context, requireMissing, newExternalOIDCArchitectureEnabled bool) { - dynamicClient, err := dynamic.NewForConfig(tc.kubeConfig) - require.NoError(t, err, "unexpected error while creating dynamic client") - - var validationErrs []error - waitErr := wait.PollUntilContextTimeout(ctx, 30*time.Second, 5*time.Minute, false, func(_ context.Context) (bool, error) { - validationErrs = make([]error, 0) - validationErrs = append(validationErrs, validateOAuthResources(ctx, dynamicClient, requireMissing, newExternalOIDCArchitectureEnabled)...) - validationErrs = append(validationErrs, validateOAuthRoutes(ctx, tc.routeClient, tc.configClient, requireMissing)...) - validationErrs = append(validationErrs, validateOAuthControllerConditions(tc.operatorClient, requireMissing, newExternalOIDCArchitectureEnabled)...) - validationErrs = append(validationErrs, validateOperandVersions(ctx, tc.configClient, requireMissing, newExternalOIDCArchitectureEnabled)...) - validationErrs = append(validationErrs, validateOAuthRelatedObjects(ctx, tc.configClient, requireMissing)...) - return len(validationErrs) == 0, nil - }) - - require.NoError(t, utilerrors.NewAggregate(validationErrs), "failed to validate OAuth state") - require.NoError(t, waitErr, "failed to wait for OAuth state validation") -} - -func validateOAuthResources(ctx context.Context, dynamicClient *dynamic.DynamicClient, requireMissing, newExternalOIDCArchitectureEnabled bool) []error { - errs := make([]error, 0) - - type resourceReference struct { - gvr schema.GroupVersionResource - namespace string - name string - } - - resourceReferences := []resourceReference{ - {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}, "openshift-authentication", "v4-0-config-system-cliconfig"}, - {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}, "openshift-authentication", "v4-0-config-system-metadata"}, - {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}, "openshift-authentication", "v4-0-config-system-service-ca"}, - {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}, "openshift-authentication", "v4-0-config-system-trusted-ca-bundle"}, - {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}, "openshift-config-managed", "oauth-serving-cert"}, - {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"}, "openshift-authentication", "v4-0-config-system-ocp-branding-template"}, - {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"}, "openshift-authentication", "v4-0-config-system-session"}, - {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "serviceaccounts"}, "openshift-authentication", "oauth-openshift"}, - {schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services"}, "openshift-authentication", "oauth-openshift"}, - {schema.GroupVersionResource{Group: "apiregistration.k8s.io", Version: "v1", Resource: "apiservices"}, "", "v1.oauth.openshift.io"}, - {schema.GroupVersionResource{Group: "apiregistration.k8s.io", Version: "v1", Resource: "apiservices"}, "", "v1.user.openshift.io"}, - {schema.GroupVersionResource{Group: "oauth.openshift.io", Version: "v1", Resource: "oauthclients"}, "", "openshift-browser-client"}, - {schema.GroupVersionResource{Group: "oauth.openshift.io", Version: "v1", Resource: "oauthclients"}, "", "openshift-challenging-client"}, - {schema.GroupVersionResource{Group: "oauth.openshift.io", Version: "v1", Resource: "oauthclients"}, "", "openshift-cli-client"}, - {schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"}, "", "system:openshift:openshift-authentication"}, - {schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"}, "", "system:openshift:useroauthaccesstoken-manager"}, - {schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterroles"}, "", "system:openshift:useroauthaccesstoken-manager"}, - {schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "rolebindings"}, "openshift-config-managed", "system:openshift:oauth-servercert-trust"}, - {schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "roles"}, "openshift-config-managed", "system:openshift:oauth-servercert-trust"}, - } - - // when running under the new architecture, the following resources should - // always be present and shouldn't be included in our checks - if !newExternalOIDCArchitectureEnabled { - resourceReferences = append(resourceReferences, - resourceReference{schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"}, "openshift-config", "webhook-authentication-integrated-oauth"}, - resourceReference{schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services"}, "openshift-oauth-apiserver", "api"}, - resourceReference{schema.GroupVersionResource{Group: "", Version: "v1", Resource: "serviceaccounts"}, "openshift-oauth-apiserver", "oauth-apiserver-sa"}, - resourceReference{schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"}, "", "system:openshift:oauth-apiserver"}, - ) - } - - for _, obj := range resourceReferences { - _, err := dynamicClient.Resource(obj.gvr).Namespace(obj.namespace).Get(ctx, obj.name, metav1.GetOptions{}) - if err != nil && !errors.IsNotFound(err) { - errs = append(errs, fmt.Errorf("unexpected error while getting resource %s/%s: %v", obj.namespace, obj.name, err)) - } else if requireMissing != errors.IsNotFound(err) { - errs = append(errs, fmt.Errorf("resource %s '%s/%s' wanted missing: %v; got: %v (error: %v)", obj.gvr.String(), obj.namespace, obj.name, requireMissing, errors.IsNotFound(err), err)) - } - } - - return errs -} - -func validateOAuthRoutes(ctx context.Context, routeClient routeclient.Interface, configClient *configclient.Clientset, requireMissing bool) []error { - errs := make([]error, 0) - for _, obj := range []struct{ namespace, name string }{ - {"openshift-authentication", "oauth-openshift"}, - } { - _, err := routeClient.RouteV1().Routes(obj.namespace).Get(ctx, obj.name, metav1.GetOptions{}) - if err != nil && !errors.IsNotFound(err) { - errs = append(errs, fmt.Errorf("unexpected error while getting route %s/%s: %v", obj.namespace, obj.name, err)) - } else if requireMissing != errors.IsNotFound(err) { - errs = append(errs, fmt.Errorf("route %s/%s wanted missing: %v; got: %v", obj.namespace, obj.name, requireMissing, !errors.IsNotFound((err)))) - } - - // ingress status - ingress, err := configClient.ConfigV1().Ingresses().Get(ctx, "cluster", metav1.GetOptions{}) - if err != nil { - return append(errs, err) - } - - found := false - for _, route := range ingress.Status.ComponentRoutes { - if route.Name == obj.name && route.Namespace == obj.namespace { - found = true - break - } - } - - if !requireMissing && !found { - errs = append(errs, fmt.Errorf("route %s required but was not found", obj)) - } else if requireMissing && found { - errs = append(errs, fmt.Errorf("route %s required to be missing but was found", obj)) - } - } - - return errs -} - -func validateOAuthControllerConditions(operatorClient v1helpers.OperatorClient, requireMissing, newExternalOIDCArchitectureEnabled bool) []error { - errs := make([]error, 0) - controllerConditionTypes := sets.New[string]( - // endpointAccessibleController - "OAuthServerRouteEndpointAccessibleControllerAvailable", - "OAuthServerServiceEndpointAccessibleControllerAvailable", - "OAuthServerServiceEndpointsEndpointAccessibleControllerAvailable", - // payloadConfigController - "OAuthConfigDegraded", - "OAuthSessionSecretDegraded", - "OAuthConfigRouteDegraded", - "OAuthConfigIngressDegraded", - "OAuthConfigServiceDegraded", - // ingressNodesAvailableController - "ReadyIngressNodesAvailable", - // ingressStateController - "IngressStateEndpointsDegraded", - "IngressStatePodsDegraded", - // metadataController - "IngressConfigDegraded", - "AuthConfigDegraded", - "OAuthSystemMetadataDegraded", - // routerCertsDomainValidationController - "RouterCertsDegraded", - // serviceCAController - "OAuthServiceDegraded", - "SystemServiceCAConfigDegraded", - // wellKnownReadyController - "WellKnownAvailable", - "WellKnownReadyControllerDegradationObserved", - ) - - if !newExternalOIDCArchitectureEnabled { - // webhookAuthenticatorController - controllerConditionTypes.Insert("AuthenticatorCertKeyProgressing") - } - - _, operatorStatus, _, err := operatorClient.GetOperatorState() - if err != nil { - return append(errs, err) - } - - allConditions := sets.New[string]() - for _, condition := range operatorStatus.Conditions { - allConditions.Insert(condition.Type) - } - - if requireMissing { - // no controller conditions must exist in operator status - if intersection := controllerConditionTypes.Intersection(allConditions); intersection.Len() > 0 { - return append(errs, fmt.Errorf("expected conditions to be missing but were found: %v", intersection.UnsortedList())) - } - return nil - } - - if diff := controllerConditionTypes.Difference(allConditions); diff.Len() > 0 { - // all controller conditions must exist in operator status - return append(errs, fmt.Errorf("expected conditions to exist, but were not found: %v", diff.UnsortedList())) - } - - return nil -} - -func validateOperandVersions(ctx context.Context, cfgClient *configclient.Clientset, requireMissing, newExternalOIDCArchitectureEnabled bool) []error { - operands := sets.New("oauth-openshift") - - // If the new architecture for external OIDC is not enabled, - // test this as we always did and ensure that the oauth-apiserver - // operand version gets removed appropriately. - if !newExternalOIDCArchitectureEnabled { - operands.Insert("oauth-apiserver") - } - - authnClusterOperator, err := cfgClient.ConfigV1().ClusterOperators().Get(ctx, "authentication", metav1.GetOptions{}) - if err != nil { - return []error{fmt.Errorf("fetching authentication ClusterOperator: %w", err)} - } - - foundOperands := []string{} - for _, version := range authnClusterOperator.Status.Versions { - if operands.Has(version.Name) { - foundOperands = append(foundOperands, version.Name) - } - } - - if requireMissing && len(foundOperands) > 0 { - return []error{fmt.Errorf("authentication ClusterOperator status has operands %v in versions when they should be unset", foundOperands)} - } - - foundSet := sets.New(foundOperands...) - if !requireMissing && !foundSet.Equal(operands) { - return []error{fmt.Errorf("authentication ClusterOperator status expected to have operands %v in versions but got %v", operands.UnsortedList(), foundOperands)} - } - - // If the new architecture for external OIDC is enabled, - // ensure the oauth-apiserver operand is always present. - if newExternalOIDCArchitectureEnabled { - found := false - for _, version := range authnClusterOperator.Status.Versions { - if version.Name == "oauth-apiserver" { - found = true - break - } - } - - if !found { - return []error{fmt.Errorf("authentication ClusterOperator status expected to have operand \"oauth-apiserver\" in versions but it was missing")} - } - } - - return nil -} - -func validateOAuthRelatedObjects(ctx context.Context, configClient *configclient.Clientset, requireMissing bool) []error { - co, err := configClient.ConfigV1().ClusterOperators().Get(ctx, "authentication", metav1.GetOptions{}) - if err != nil { - return []error{err} - } - - oauthRelatedObjects := []configv1.ObjectReference{ - {Group: routev1.GroupName, Resource: "routes", Name: "oauth-openshift", Namespace: "openshift-authentication"}, - {Resource: "services", Name: "oauth-openshift", Namespace: "openshift-authentication"}, - } - - errs := make([]error, 0) - for _, oauthObj := range oauthRelatedObjects { - found := false - for _, existingObj := range co.Status.RelatedObjects { - if oauthObj.Group == existingObj.Group && - oauthObj.Resource == existingObj.Resource && - oauthObj.Name == existingObj.Name && - oauthObj.Namespace == existingObj.Namespace { - found = true - break - } - } - - if requireMissing && found { - errs = append(errs, fmt.Errorf("oauth related object %s/%s %s/%s should be missing but was found in RelatedObjects", - oauthObj.Group, oauthObj.Resource, oauthObj.Namespace, oauthObj.Name)) - } else if !requireMissing && !found { - errs = append(errs, fmt.Errorf("oauth related object %s/%s %s/%s should be present but was not found in RelatedObjects", - oauthObj.Group, oauthObj.Resource, oauthObj.Namespace, oauthObj.Name)) - } - } - - return errs -} - -func (tc *testClient) testOIDCAuthentication(t *testing.T, ctx context.Context, kcClient *test.KeycloakClient, usernameClaim, usernamePrefix string, expectAuthSuccess bool) { - // re-authenticate to ensure we always have a fresh token - var err error - waitErr := wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) { - err = kcClient.AuthenticatePassword(oidcClientId, "", "admin", "password") - return err == nil, nil - }) - require.NoError(t, err, "failed to authenticate to keycloak: %v", err) - require.NoError(t, waitErr, "failed to wait for keycloak authentication: %v", waitErr) - - group := names.SimpleNameGenerator.GenerateName("e2e-keycloak-group-") - err = kcClient.CreateGroup(group) - require.NoError(t, err) - - user := names.SimpleNameGenerator.GenerateName("e2e-keycloak-user-") - email := fmt.Sprintf("%s@test.dev", user) - password := "password" - firstName := "Homer" - lastName := "Simpson" - err = kcClient.CreateUser( - user, - email, - password, - []string{group}, - map[string]string{ - "firstName": firstName, - "lastName": lastName, - }, - ) - require.NoError(t, err) - - // use a keycloak client for the user created above to fetch its tokens - transport, err := rest.TransportFor(tc.kubeConfig) - require.NoError(t, err) - userClient := test.KeycloakClientFor(t, transport, kcClient.IssuerURL(), "master") - err = userClient.AuthenticatePassword(oidcClientId, "", user, password) - require.NoError(t, err) - accessTokenStr, idTokenStr := userClient.Tokens() - require.NotEmpty(t, accessTokenStr, "access token must not be empty") - require.NotEmpty(t, idTokenStr, "id token must not be empty") - - // fetch issuer's JWKS and use it to parse JWT tokens - issuerJWKS, err := fetchIssuerJWKS(kcClient.IssuerURL()) - require.NoError(t, err) - require.NotNil(t, issuerJWKS) - keyfunc := extractRSAPubKeyFunc(issuerJWKS) - - accessToken, err := jwt.ParseWithClaims(accessTokenStr, &expectedClaims{}, keyfunc) - require.NoError(t, err) - require.NotNil(t, accessToken) - - idToken, err := jwt.ParseWithClaims(idTokenStr, &expectedClaims{}, keyfunc) - require.NoError(t, err) - require.NotNil(t, idToken) - - // validate the contents of the OIDC tokens - actualAccessTokenClaims := accessToken.Claims.(*expectedClaims) - require.True(t, accessToken.Valid) - require.Equal(t, userClient.IssuerURL(), actualAccessTokenClaims.Issuer) - require.Equal(t, user, actualAccessTokenClaims.PreferredUsername) - require.Equal(t, email, actualAccessTokenClaims.Email) - require.Equal(t, "Bearer", actualAccessTokenClaims.Type) - require.Equal(t, firstName, actualAccessTokenClaims.GivenName) - require.Equal(t, lastName, actualAccessTokenClaims.FamilyName) - require.Equal(t, fmt.Sprintf("%s %s", firstName, lastName), actualAccessTokenClaims.Name) - require.NotEmpty(t, actualAccessTokenClaims.Subject) - - actualIDTokenClaims := idToken.Claims.(*expectedClaims) - require.True(t, idToken.Valid) - require.Equal(t, userClient.IssuerURL(), actualIDTokenClaims.Issuer) - require.Equal(t, user, actualIDTokenClaims.PreferredUsername) - require.Equal(t, email, actualIDTokenClaims.Email) - require.Equal(t, "ID", actualIDTokenClaims.Type) - require.Equal(t, jwt.ClaimStrings{oidcClientId}, actualIDTokenClaims.Audience) - require.Equal(t, firstName, actualIDTokenClaims.GivenName) - require.Equal(t, lastName, actualIDTokenClaims.FamilyName) - require.Equal(t, fmt.Sprintf("%s %s", firstName, lastName), actualIDTokenClaims.Name) - require.NotEmpty(t, actualIDTokenClaims.Subject) - - // test authentication via the kube-apiserver - // create a new kube client that uses the OIDC id_token as a bearer token - kubeConfig := rest.AnonymousClientConfig(tc.kubeConfig) - kubeConfig.BearerToken = idTokenStr - kubeClient, err := kubernetes.NewForConfig(kubeConfig) - require.NoError(t, err) - - ssr, err := kubeClient.AuthenticationV1().SelfSubjectReviews().Create(ctx, &authenticationv1.SelfSubjectReview{}, metav1.CreateOptions{}) - if expectAuthSuccess { - // test authentication with the OIDC token using a self subject review - expectedUsername := "" - switch usernameClaim { - case "email": - expectedUsername = usernamePrefix + email - case "sub": - expectedUsername = usernamePrefix + actualIDTokenClaims.Subject - default: - t.Fatalf("unexpected username claim: %s", usernameClaim) - } - - require.NoError(t, err) - require.NotNil(t, ssr) - require.Contains(t, ssr.Status.UserInfo.Groups, "system:authenticated") - require.Equal(t, expectedUsername, ssr.Status.UserInfo.Username) - } else { - require.Error(t, err) - require.True(t, errors.IsUnauthorized(err)) - } -} - -func (tc *testClient) requireKASRolloutSuccessful(t *testing.T, testCtx context.Context, authSpec *configv1.AuthenticationSpec, kasOriginalRevision int32, expectedUsernamePrefix string) { - // wait for KAS rollout - err := test.WaitForNewKASRollout(t, testCtx, tc.operatorConfigClient.OperatorV1().KubeAPIServers(), kasOriginalRevision) - require.NoError(t, err, "failed to wait for KAS rollout") - - kasRevision := tc.validateKASConfig(t, testCtx) - tc.validateAuthConfigJSON(t, testCtx, authSpec, expectedUsernamePrefix, oidcGroupsClaim, oidcGroupsPrefix, kasRevision) -} - -func (tc *testClient) authResourceRollback(ctx context.Context, origAuthSpec *configv1.AuthenticationSpec) error { - auth, err := tc.configClient.ConfigV1().Authentications().Get(ctx, "cluster", metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("rollback failed for authentication '%s' while retrieving fresh object: %v", auth.Name, err) - } - - if !equality.Semantic.DeepEqual(auth.Spec, *origAuthSpec) { - auth.Spec = *origAuthSpec - if _, err := tc.configClient.ConfigV1().Authentications().Update(ctx, auth, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("rollback failed for authentication '%s' while updating object: %v", auth.Name, err) - } - } - - return nil -} - -func featureGateEnabled(ctx context.Context, configClient *configclient.Clientset, feature configv1.FeatureGateName) bool { - featureGates, err := configClient.ConfigV1().FeatureGates().Get(ctx, "cluster", metav1.GetOptions{}) - if err != nil { - return false - } - - if len(featureGates.Status.FeatureGates) == 0 { - return false - } - - for _, enabled := range featureGates.Status.FeatureGates[0].Enabled { - if enabled.Name == feature { - return true - } - } - - return false + testContext, cancel := context.WithCancel(context.Background()) + defer cancel() + testExternalOIDCWithKeycloak(testContext, t) } diff --git a/test/e2e/keycloak.go b/test/e2e/keycloak.go index 792254d28e..441c3b03f0 100644 --- a/test/e2e/keycloak.go +++ b/test/e2e/keycloak.go @@ -104,13 +104,13 @@ func testKeycloakAsOIDCPasswordGrantCheckAndGroupSync(t testing.TB) { require.NoError(t, err) kcClient := test.KeycloakClientFor(t, transport, configIDP.OpenID.Issuer, "master") - err = kcClient.AuthenticatePassword("admin-cli", "", "admin", "password") + err = kcClient.AuthenticatePassword(testContext, "admin-cli", "", "admin", "password") require.NoError(t, err) - client, err := kcClient.GetClientByClientID(configIDP.OpenID.ClientID) + client, err := kcClient.GetClientByClientID(testContext, configIDP.OpenID.ClientID) require.NoError(t, err) - err = kcClient.UpdateClientDirectAccessGrantsEnabled(client["id"].(string), true) + err = kcClient.UpdateClientDirectAccessGrantsEnabled(testContext, client["id"].(string), true) require.NoError(t, err) // =================================================================== @@ -124,7 +124,7 @@ func testKeycloakAsOIDCPasswordGrantCheckAndGroupSync(t testing.TB) { err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) { // Re-authenticate to verify admin API is still responsive - err := kcClient.AuthenticatePassword("admin-cli", "", "admin", "password") + err := kcClient.AuthenticatePassword(testContext, "admin-cli", "", "admin", "password") if err != nil { t.Logf("Keycloak authentication not ready: %v", err) return false, nil @@ -216,12 +216,13 @@ func testKeycloakAsOIDCPasswordGrantCheckAndGroupSync(t testing.TB) { // Test groups are synced from OIDC // ================================ groups := []string{"group1", "group2", "group3"} - require.NoError(t, kcClient.CreateGroup("group1")) - require.NoError(t, kcClient.CreateGroup("group2")) - require.NoError(t, kcClient.CreateGroup("group3")) + require.NoError(t, kcClient.CreateGroup(testContext, "group1")) + require.NoError(t, kcClient.CreateGroup(testContext, "group2")) + require.NoError(t, kcClient.CreateGroup(testContext, "group3")) username := "douglasnoeladams" require.NoError(t, kcClient.CreateUser( + testContext, username, "", "password42", groups, nil, @@ -260,18 +261,18 @@ func testKeycloakAsOIDCPasswordGrantCheckAndGroupSync(t testing.TB) { // ================================================================================== // Test groups get removed if the user is the last and they were synced from the OIDC // ================================================================================== - users, err := kcClient.ListUsers() + users, err := kcClient.ListUsers(testContext) require.NoError(t, err) - var userId string + var userID string for _, u := range users { if u["username"] == username { - userId = u["id"].(string) + userID = u["id"].(string) break } } - require.NotEmpty(t, userId, "failed to find user id for %q", username) + require.NotEmpty(t, userID, "failed to find user id for %q", username) - userGroups, err := kcClient.ListUserGroups(userId) + userGroups, err := kcClient.ListUserGroups(testContext, userID) require.NoError(t, err) userGroupsIDMap := make(map[string]string, len(userGroups)) @@ -279,7 +280,7 @@ func testKeycloakAsOIDCPasswordGrantCheckAndGroupSync(t testing.TB) { userGroupsIDMap[g["name"].(string)] = g["id"].(string) } - require.NoError(t, kcClient.DeleteUserFromGroups(userId, userGroupsIDMap["group2"], userGroupsIDMap["group3"])) + require.NoError(t, kcClient.DeleteUserFromGroups(testContext, userID, userGroupsIDMap["group2"], userGroupsIDMap["group3"])) removedGroups := sets.New[string]("group2", "group3") _, err = tokenrequest.RequestTokenWithChallengeHandlers(kubeConfig, createChallengeHandler(username, "password42")) diff --git a/test/library/client.go b/test/library/client.go index 2397cb2313..9f5583b539 100644 --- a/test/library/client.go +++ b/test/library/client.go @@ -35,11 +35,8 @@ func NewClientConfigForTest(t testing.TB) *rest.Config { loader := clientcmd.NewDefaultClientConfigLoadingRules() clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loader, &clientcmd.ConfigOverrides{ClusterInfo: api.Cluster{InsecureSkipTLSVerify: true}}) config, err := clientConfig.ClientConfig() - if err == nil { - fmt.Printf("Found configuration for host %v.\n", config.Host) - } - require.NoError(t, err) + t.Logf("Found configuration for API server") return config } @@ -119,12 +116,14 @@ func GenerateOAuthTokenPair() (privToken, pubToken string) { return sha256Prefix + string(randomToken), sha256Prefix + base64.RawURLEncoding.EncodeToString(hashed[:]) } -type testNamespaceBuilder struct { +// TestNamespaceBuilder provides a fluent interface for building test namespaces with Pod Security Admission configuration. +type TestNamespaceBuilder struct { ns *corev1.Namespace } -func NewTestNamespaceBuilder(namePrefix string) *testNamespaceBuilder { - return &testNamespaceBuilder{ +// NewTestNamespaceBuilder creates a new builder for test namespaces with the given name prefix. +func NewTestNamespaceBuilder(namePrefix string) *TestNamespaceBuilder { + return &TestNamespaceBuilder{ ns: &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ GenerateName: namePrefix, @@ -137,31 +136,41 @@ func NewTestNamespaceBuilder(namePrefix string) *testNamespaceBuilder { } } -func (b *testNamespaceBuilder) WithLabels(labels map[string]string) *testNamespaceBuilder { +// WithLabels adds the specified labels to the namespace. +func (b *TestNamespaceBuilder) WithLabels(labels map[string]string) *TestNamespaceBuilder { for k, v := range labels { b.ns.Labels[k] = v } return b } -func (b *testNamespaceBuilder) WithPSaEnforcement(level psapi.Level) *testNamespaceBuilder { +// WithPSaEnforcement sets the Pod Security Admission enforcement level for the namespace. +// This sets all three PSA labels (enforce, audit, warn) to ensure consistent behavior +// and prevent audit log violations when using privileged mode. +func (b *TestNamespaceBuilder) WithPSaEnforcement(level psapi.Level) *TestNamespaceBuilder { b.ns.Labels[psapi.EnforceLevelLabel] = string(level) + b.ns.Labels[psapi.AuditLevelLabel] = string(level) + b.ns.Labels[psapi.WarnLevelLabel] = string(level) return b } -func (b *testNamespaceBuilder) WithRestrictedPSaEnforcement() *testNamespaceBuilder { +// WithRestrictedPSaEnforcement sets the namespace to use restricted Pod Security Admission enforcement. +func (b *TestNamespaceBuilder) WithRestrictedPSaEnforcement() *TestNamespaceBuilder { return b.WithPSaEnforcement(psapi.LevelRestricted) } -func (b *testNamespaceBuilder) WithBaselinePSaEnforcement() *testNamespaceBuilder { +// WithBaselinePSaEnforcement sets the namespace to use baseline Pod Security Admission enforcement. +func (b *TestNamespaceBuilder) WithBaselinePSaEnforcement() *TestNamespaceBuilder { return b.WithPSaEnforcement(psapi.LevelBaseline) } -func (b *testNamespaceBuilder) WithPrivilegedPSaEnforcement() *testNamespaceBuilder { +// WithPrivilegedPSaEnforcement sets the namespace to use privileged Pod Security Admission enforcement. +func (b *TestNamespaceBuilder) WithPrivilegedPSaEnforcement() *TestNamespaceBuilder { return b.WithPSaEnforcement(psapi.LevelPrivileged) } -func (b *testNamespaceBuilder) Create(t testing.TB, kubeClient corev1client.NamespaceInterface) string { +// Create creates the namespace in the cluster and returns its name. +func (b *TestNamespaceBuilder) Create(t testing.TB, kubeClient corev1client.NamespaceInterface) string { ns, err := kubeClient.Create(context.Background(), b.ns, metav1.CreateOptions{}) require.NoError(t, err) diff --git a/test/library/encryption/helpers.go b/test/library/encryption/helpers.go index a9e29cd202..da603799fc 100644 --- a/test/library/encryption/helpers.go +++ b/test/library/encryption/helpers.go @@ -22,11 +22,13 @@ import ( library "github.com/openshift/library-go/test/library/encryption" ) +// ClientSet holds the clients needed for encryption tests. type ClientSet struct { OperatorClient operatorv1client.AuthenticationInterface TokenClient oauthclient.OAuthAccessTokensGetter } +// GetClientsFor returns a ClientSet configured with the given kubeConfig for encryption tests. func GetClientsFor(t testing.TB, kubeConfig *rest.Config) ClientSet { t.Helper() @@ -39,6 +41,7 @@ func GetClientsFor(t testing.TB, kubeConfig *rest.Config) ClientSet { return ClientSet{OperatorClient: operatorClient.Authentications(), TokenClient: oc} } +// GetClients returns a ClientSet configured with the default test kubeConfig. func GetClients(t testing.TB) ClientSet { t.Helper() @@ -52,13 +55,12 @@ func NewClientConfigForTest(t testing.TB) *rest.Config { loader := clientcmd.NewDefaultClientConfigLoadingRules() clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loader, &clientcmd.ConfigOverrides{ClusterInfo: kubecmdapi.Cluster{InsecureSkipTLSVerify: true}}) config, err := clientConfig.ClientConfig() - if err == nil { - fmt.Printf("Found configuration for host %v.\n", config.Host) - } require.NoError(t, err) + t.Logf("Found configuration for API server") return config } +// CreateAndStoreTokenOfLife creates a test OAuth access token and stores it in the cluster. func CreateAndStoreTokenOfLife(ctx context.Context, t testing.TB, cs ClientSet) runtime.Object { t.Helper() { @@ -81,6 +83,7 @@ func CreateAndStoreTokenOfLife(ctx context.Context, t testing.TB, cs ClientSet) return tokenOfLife } +// GetRawTokenOfLife retrieves the raw token value from etcd for the test OAuth access token. func GetRawTokenOfLife(t testing.TB, clientSet library.ClientSet) string { t.Helper() timeout, cancel := context.WithTimeout(context.Background(), 5*time.Minute) @@ -97,6 +100,7 @@ func GetRawTokenOfLife(t testing.TB, clientSet library.ClientSet) string { return string(resp.Kvs[0].Value) } +// TokenOfLife returns a test OAuth access token object for encryption testing. func TokenOfLife(t testing.TB) runtime.Object { t.Helper() return &oauthapiv1.OAuthAccessToken{ diff --git a/test/library/encryption/perf_helpers.go b/test/library/encryption/perf_helpers.go new file mode 100644 index 0000000000..7d25e5cc09 --- /dev/null +++ b/test/library/encryption/perf_helpers.go @@ -0,0 +1,186 @@ +package encryption + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + + operatorv1 "github.com/openshift/api/operator/v1" + library "github.com/openshift/library-go/test/library/encryption" +) + +const ( + waitPollInterval = 15 * time.Second + waitPollTimeout = 69*time.Minute + 10*time.Minute +) + +// watchForMigrationControllerProgressingConditionAsync starts watching for the migration +// controller progressing condition in a background goroutine. +func watchForMigrationControllerProgressingConditionAsync(ctx context.Context, t testing.TB, getOperatorCondFn library.GetOperatorConditionsFuncType, migrationStartedCh chan time.Time, testStartTime time.Time) { + t.Helper() + go watchForMigrationControllerProgressingCondition(ctx, t, getOperatorCondFn, migrationStartedCh, testStartTime) +} + +// watchForMigrationControllerProgressingCondition waits for the EncryptionMigrationControllerProgressing +// condition to be set to true with reason "Migrating" and a fresh transition timestamp. +// It validates that the condition transition occurred after the test start time to avoid accepting stale conditions. +func watchForMigrationControllerProgressingCondition(ctx context.Context, t testing.TB, getOperatorConditionsFn library.GetOperatorConditionsFuncType, migrationStartedCh chan time.Time, testStartTime time.Time) { + t.Helper() + + t.Logf("Waiting up to %s for the condition %q with the reason %q to be set to true (after %v)", waitPollTimeout.String(), "EncryptionMigrationControllerProgressing", "Migrating", testStartTime) + err := wait.PollUntilContextTimeout(ctx, waitPollInterval, waitPollTimeout, true, func(ctx context.Context) (bool, error) { + conditions, err := getOperatorConditionsFn(t) + if err != nil { + return false, err + } + for _, cond := range conditions { + if cond.Type == "EncryptionMigrationControllerProgressing" && + cond.Status == operatorv1.ConditionTrue && + cond.Reason == "Migrating" && + cond.LastTransitionTime.Time.After(testStartTime) { + t.Logf("EncryptionMigrationControllerProgressing condition observed at %v with reason %q", cond.LastTransitionTime, cond.Reason) + migrationStartedCh <- cond.LastTransitionTime.Time + return true, nil + } + } + return false, nil + }) + if err != nil { + t.Logf("failed waiting for the condition %q with the reason %q to be set to true, err was %v", "EncryptionMigrationControllerProgressing", "Migrating", err) + close(migrationStartedCh) + } +} + +// populateDatabase populates the database using the provided loader function with multiple workers. +func populateDatabase(t testing.TB, workers int, dbLoaderFun library.DBLoaderFuncType, assertDBPopulatedFunc func(t testing.TB, errorStore map[string]int, statStore map[string]int)) { + t.Helper() + start := time.Now() + defer func() { + end := time.Now() + t.Logf("Populating etcd took %v", end.Sub(start)) + }() + + r := newRunner() + + // run executes loaderFunc for each worker + r.run(t, workers, dbLoaderFun) + + assertDBPopulatedFunc(t, r.errorStore, r.statsStore) +} + +// runner manages parallel execution of database loader functions. +type runner struct { + errorStore map[string]int + lock *sync.Mutex + + statsStore map[string]int + lockStats *sync.Mutex + wg *sync.WaitGroup +} + +// newRunner creates a new runner for executing database load functions. +func newRunner() *runner { + r := &runner{} + + r.errorStore = map[string]int{} + r.lock = &sync.Mutex{} + r.statsStore = map[string]int{} + r.lockStats = &sync.Mutex{} + + r.wg = &sync.WaitGroup{} + + return r +} + +// run executes the provided work functions using multiple workers. +func (r *runner) run(t testing.TB, workers int, workFunc ...library.DBLoaderFuncType) { + t.Logf("Executing provided load function for %d workers", workers) + for i := 0; i < workers; i++ { + wrapper := func(wg *sync.WaitGroup) { + defer wg.Done() + kubeClient, err := newKubeClient(t, 300, 600) + if err != nil { + t.Errorf("Unable to create a kube client for a worker due to %v", err) + r.collectError(err) + return + } + if err := runWorkFunctions(kubeClient, "", r.collectError, r.collectStat, workFunc...); err != nil { + t.Logf("worker finished with loader error: %v", err) + } + } + r.wg.Add(1) + go wrapper(r.wg) + } + r.wg.Wait() + if len(r.errorStore) > 0 { + t.Logf("Workers completed with %d distinct error type(s)", len(r.errorStore)) + } else { + t.Log("All workers completed successfully") + } +} + +// collectError collects and counts errors from workers. +func (r *runner) collectError(err error) { + r.lock.Lock() + defer r.lock.Unlock() + errCount, ok := r.errorStore[err.Error()] + if !ok { + r.errorStore[err.Error()] = 1 + return + } + errCount += 1 + r.errorStore[err.Error()] = errCount +} + +// collectStat collects and counts statistics from workers. +func (r *runner) collectStat(stat string) { + r.lockStats.Lock() + defer r.lockStats.Unlock() + statCount, ok := r.statsStore[stat] + if !ok { + r.statsStore[stat] = 1 + return + } + statCount += 1 + r.statsStore[stat] = statCount +} + +// runWorkFunctions executes a series of database loader functions. +func runWorkFunctions(kubeClient kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string), workFunc ...library.DBLoaderFuncType) error { + if len(namespace) == 0 { + namespace = createNamespaceName() + } + for _, work := range workFunc { + err := work(kubeClient, namespace, errorCollector, statsCollector) + if err != nil { + errorCollector(err) + return err + } + } + return nil +} + +// createNamespaceName generates a unique namespace name for testing. +func createNamespaceName() string { + return fmt.Sprintf("encryption-%s", rand.String(10)) +} + +// newKubeClient creates a Kubernetes client with specified QPS and burst settings. +func newKubeClient(t testing.TB, qps float32, burst int) (kubernetes.Interface, error) { + kubeConfig := NewClientConfigForTest(t) + + kubeConfig.QPS = qps + kubeConfig.Burst = burst + + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + return nil, err + } + return kubeClient, nil +} diff --git a/test/library/encryption/scenarios.go b/test/library/encryption/scenarios.go new file mode 100644 index 0000000000..d70917d2b2 --- /dev/null +++ b/test/library/encryption/scenarios.go @@ -0,0 +1,247 @@ +package encryption + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" + + configv1 "github.com/openshift/api/config/v1" + configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" + library "github.com/openshift/library-go/test/library/encryption" +) + +// TestPerfEncryption tests encryption performance. +// This is a local implementation that accepts testing.TB instead of *testing.T. +// It populates the database with test data, enables encryption, and measures migration time. +func TestPerfEncryption(tb testing.TB, scenario library.PerfScenario) { + tb.Logf("Starting encryption performance test for %q provider", scenario.EncryptionProvider.Type) + + migrationStartedCh := make(chan time.Time, 1) + + // Create a cancelable context for the watcher goroutine to ensure it stops when the test finishes + watcherCtx, cancel := context.WithCancel(context.Background()) + tb.Cleanup(cancel) + + // Step 1: Populate the database with test data + tb.Logf("Step 1/3: Populating database with test data using %d workers", scenario.DBLoaderWorkers) + populateDatabase(tb, scenario.DBLoaderWorkers, scenario.DBLoaderFunc, scenario.AssertDBPopulatedFunc) + + // Step 2: Start watching for migration controller progressing condition asynchronously + // Capture test start time to validate fresh condition transitions + testStartTime := time.Now() + tb.Logf("Step 2/3: Starting migration progress monitor (test start time: %v)", testStartTime) + watchForMigrationControllerProgressingConditionAsync(watcherCtx, tb, scenario.GetOperatorConditionsFunc, migrationStartedCh, testStartTime) + + // Step 3: Run encryption test and measure time + tb.Logf("Step 3/3: Enabling encryption and measuring migration time") + endTimeStamp := runTestEncryptionPerf(tb, scenario) + + // Calculate and assert migration time + select { + case migrationStarted := <-migrationStartedCh: + if migrationStarted.IsZero() { + tb.Error("unable to calculate the migration time, migration watcher encountered an error") + } else { + migrationTime := endTimeStamp.Sub(migrationStarted) + tb.Logf("Migration completed in %v", migrationTime) + scenario.AssertMigrationTime(tb, migrationTime) + } + case <-time.After(30 * time.Second): + tb.Error("unable to calculate the migration time, failed to observe when the migration has started") + } + + tb.Logf("Encryption performance test completed") +} + +func runTestEncryptionPerf(tb testing.TB, scenario library.PerfScenario) time.Time { + var ts time.Time + TestEncryptionType(tb, library.BasicScenario{ + Namespace: scenario.Namespace, + LabelSelector: scenario.LabelSelector, + EncryptionConfigSecretName: scenario.EncryptionConfigSecretName, + EncryptionConfigSecretNamespace: scenario.EncryptionConfigSecretNamespace, + OperatorNamespace: scenario.OperatorNamespace, + TargetGRs: scenario.TargetGRs, + AssertFunc: func(t testing.TB, clientSet library.ClientSet, expectedMode configv1.EncryptionType, namespace, labelSelector string) { + // Note that AssertFunc is executed after an encryption secret has been annotated + ts = time.Now() + scenario.AssertFunc(t, clientSet, expectedMode, scenario.Namespace, scenario.LabelSelector) + tb.Logf("AssertFunc for TestEncryption scenario with %q provider took %v", scenario.EncryptionProvider.Type, time.Since(ts)) + }, + }, scenario.EncryptionProvider) + return ts +} + +// TestEncryptionType is a helper that dispatches to the appropriate encryption type test. +// This is a local implementation that accepts testing.TB instead of *testing.T. +func TestEncryptionType(tb testing.TB, scenario library.BasicScenario, provider library.EncryptionProvider) { + switch provider.Type { + case configv1.EncryptionTypeAESCBC: + TestEncryptionTypeAESCBC(tb, scenario) + case configv1.EncryptionTypeAESGCM: + TestEncryptionTypeAESGCM(tb, scenario) + case configv1.EncryptionTypeKMS: + TestEncryptionTypeKMS(tb, scenario) + case configv1.EncryptionTypeIdentity: + TestEncryptionTypeIdentity(tb, scenario) + case "": + TestEncryptionTypeUnset(tb, scenario) + default: + tb.Fatalf("Unknown encryption type: %s", provider.Type) + } +} + +// TestEncryptionTypeIdentity tests encryption with identity mode (no encryption). +// This is a local implementation that accepts testing.TB instead of *testing.T. +func TestEncryptionTypeIdentity(tb testing.TB, scenario library.BasicScenario) { + testEncryptionTypeBase(tb, scenario, configv1.EncryptionTypeIdentity, configv1.EncryptionTypeIdentity) +} + +// TestEncryptionTypeUnset tests encryption with unset mode (defaults to identity). +// This is a local implementation that accepts testing.TB instead of *testing.T. +func TestEncryptionTypeUnset(tb testing.TB, scenario library.BasicScenario) { + testEncryptionTypeBase(tb, scenario, "", configv1.EncryptionTypeIdentity) +} + +// TestEncryptionTypeAESCBC tests encryption with AESCBC mode. +// This is a local implementation that accepts testing.TB instead of *testing.T. +func TestEncryptionTypeAESCBC(tb testing.TB, scenario library.BasicScenario) { + testEncryptionTypeBase(tb, scenario, configv1.EncryptionTypeAESCBC, configv1.EncryptionTypeAESCBC) +} + +// TestEncryptionTypeAESGCM tests encryption with AESGCM mode. +// This is a local implementation that accepts testing.TB instead of *testing.T. +func TestEncryptionTypeAESGCM(tb testing.TB, scenario library.BasicScenario) { + testEncryptionTypeBase(tb, scenario, configv1.EncryptionTypeAESGCM, configv1.EncryptionTypeAESGCM) +} + +// TestEncryptionTypeKMS tests KMS encryption. +// This is a local implementation that accepts testing.TB instead of *testing.T. +func TestEncryptionTypeKMS(tb testing.TB, scenario library.BasicScenario) { + testEncryptionTypeBase(tb, scenario, configv1.EncryptionTypeKMS, configv1.EncryptionTypeKMS) +} + +// testEncryptionTypeBase is the base implementation for all encryption type tests. +func testEncryptionTypeBase(tb testing.TB, scenario library.BasicScenario, encryptionType configv1.EncryptionType, expectedType configv1.EncryptionType) { + if encryptionType == "" { + tb.Logf("Starting encryption e2e test for unset mode (defaults to identity)") + } else { + tb.Logf("Starting encryption e2e test for %q mode", encryptionType) + } + + clientSet := SetAndWaitForEncryptionType(tb, encryptionType, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) + + scenario.AssertFunc(tb, clientSet, expectedType, scenario.Namespace, scenario.LabelSelector) + + // For actual encryption types (not identity/unset), also assert encryption config + if encryptionType != "" && encryptionType != configv1.EncryptionTypeIdentity { + library.AssertEncryptionConfig(tb, clientSet, scenario.EncryptionConfigSecretName, scenario.EncryptionConfigSecretNamespace, scenario.TargetGRs) + } +} + +// createLibraryClientSet creates a library.ClientSet from kubeconfig. +// This helper consolidates the duplicated clientset creation logic. +func createLibraryClientSet(tb testing.TB) library.ClientSet { + kubeConfig := NewClientConfigForTest(tb) + libClientSet := library.ClientSet{} + libClientSet.Kube = kubernetes.NewForConfigOrDie(kubeConfig) + libClientSet.Etcd = library.NewEtcdClient(libClientSet.Kube) + configClient := configv1client.NewForConfigOrDie(kubeConfig) + libClientSet.ApiServerConfig = configClient.APIServers() + return libClientSet +} + +// TestEncryptionTurnOnAndOff tests turning encryption on and off. +// This is a local implementation that accepts testing.TB instead of *testing.T. +// It runs through a complete cycle twice to ensure repeatability: +// 1. Create resource -> Enable encryption -> Verify encrypted -> Disable -> Verify not encrypted +// 2. Repeat the cycle to ensure it works multiple times +func TestEncryptionTurnOnAndOff(tb testing.TB, scenario library.OnOffScenario) { + tb.Logf("Starting encryption turn-on-and-off test for resource %q", scenario.ResourceName) + + // Helper to get library clientset - uses shared helper function + getLibClientSet := func() library.ClientSet { + return createLibraryClientSet(tb) + } + + // Step 1: Create and store the resource + tb.Logf("Step 1/9: Creating and storing %s", scenario.ResourceName) + scenario.CreateResourceFunc(tb, getLibClientSet(), scenario.Namespace) + + // Step 2: Turn on encryption with the specified provider + tb.Logf("Step 2/9: Enabling %s encryption", scenario.EncryptionProvider.Type) + TestEncryptionType(tb, scenario.BasicScenario, scenario.EncryptionProvider) + + // Step 3: Assert the resource is encrypted + tb.Logf("Step 3/9: Verifying %s is encrypted", scenario.ResourceName) + scenario.AssertResourceEncryptedFunc(tb, getLibClientSet(), scenario.ResourceFunc(tb, scenario.Namespace)) + + // Step 4: Turn off encryption (switch to identity mode) + tb.Logf("Step 4/9: Disabling encryption (switching to identity mode)") + TestEncryptionTypeIdentity(tb, scenario.BasicScenario) + + // Step 5: Assert the resource is not encrypted + tb.Logf("Step 5/9: Verifying %s is not encrypted", scenario.ResourceName) + scenario.AssertResourceNotEncryptedFunc(tb, getLibClientSet(), scenario.ResourceFunc(tb, scenario.Namespace)) + + // Step 6: Turn on encryption again (second cycle to test repeatability) + tb.Logf("Step 6/9: Enabling %s encryption (second cycle)", scenario.EncryptionProvider.Type) + TestEncryptionType(tb, scenario.BasicScenario, scenario.EncryptionProvider) + + // Step 7: Assert the resource is encrypted again + tb.Logf("Step 7/9: Verifying %s is encrypted (second cycle)", scenario.ResourceName) + scenario.AssertResourceEncryptedFunc(tb, getLibClientSet(), scenario.ResourceFunc(tb, scenario.Namespace)) + + // Step 8: Turn off encryption again (second cycle) + tb.Logf("Step 8/9: Disabling encryption (identity mode, second cycle)") + TestEncryptionTypeIdentity(tb, scenario.BasicScenario) + + // Step 9: Assert the resource is not encrypted again + tb.Logf("Step 9/9: Verifying %s is not encrypted (second cycle)", scenario.ResourceName) + scenario.AssertResourceNotEncryptedFunc(tb, getLibClientSet(), scenario.ResourceFunc(tb, scenario.Namespace)) + + tb.Logf("Encryption turn-on-and-off test completed successfully") +} + +// SetAndWaitForEncryptionType sets the encryption type and waits for encryption to complete. +// This is a local implementation that accepts testing.TB instead of *testing.T. +func SetAndWaitForEncryptionType(tb testing.TB, encryptionType configv1.EncryptionType, defaultTargetGRs []schema.GroupResource, namespace, labelSelector string) library.ClientSet { + // Create library clientset using shared helper + libClientSet := createLibraryClientSet(tb) + + lastMigratedKeyMeta, err := library.GetLastKeyMeta(tb, libClientSet.Kube, namespace, labelSelector) + require.NoError(tb, err) + + // Update encryption type with retry on conflict + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + reqCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Get current API server config + apiServer, err := libClientSet.ApiServerConfig.Get(reqCtx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + + // Update encryption type if needed + if apiServer.Spec.Encryption.Type != encryptionType { + tb.Logf("Updating encryption type in the config file for APIServer to %q", encryptionType) + apiServer.Spec.Encryption.Type = encryptionType + _, err = libClientSet.ApiServerConfig.Update(reqCtx, apiServer, metav1.UpdateOptions{}) + return err + } + tb.Logf("APIServer is already configured to use %q mode", encryptionType) + return nil + }) + require.NoError(tb, err) + + library.WaitForEncryptionKeyBasedOn(tb, libClientSet.Kube, lastMigratedKeyMeta, encryptionType, defaultTargetGRs, namespace, labelSelector) + + return libClientSet +} diff --git a/test/library/encryption_wrappers.go b/test/library/encryption_wrappers.go new file mode 100644 index 0000000000..8387c876c8 --- /dev/null +++ b/test/library/encryption_wrappers.go @@ -0,0 +1,39 @@ +package library + +import ( + "testing" + + localEncryption "github.com/openshift/cluster-authentication-operator/test/library/encryption" + library "github.com/openshift/library-go/test/library/encryption" +) + +// These wrapper functions provide compatibility between Ginkgo v2's testing.TB +// and library-go's test functions that expect *testing.T. +// +// Instead of using unsafe pointer conversions (which cause concurrent map access +// panics when t.Helper() is called), we use local implementations that properly +// handle testing.TB. + +// TestEncryptionTypeIdentity tests encryption with identity mode. +// This calls the local implementation instead of library-go to avoid unsafe conversions. +func TestEncryptionTypeIdentity(tb testing.TB, scenario library.BasicScenario) { + localEncryption.TestEncryptionTypeIdentity(tb, scenario) +} + +// TestEncryptionTypeUnset tests encryption with unset mode. +// This calls the local implementation instead of library-go to avoid unsafe conversions. +func TestEncryptionTypeUnset(tb testing.TB, scenario library.BasicScenario) { + localEncryption.TestEncryptionTypeUnset(tb, scenario) +} + +// TestEncryptionTurnOnAndOff tests turning encryption on and off. +// This calls the local implementation instead of library-go to avoid unsafe conversions. +func TestEncryptionTurnOnAndOff(tb testing.TB, scenario library.OnOffScenario) { + localEncryption.TestEncryptionTurnOnAndOff(tb, scenario) +} + +// TestPerfEncryption tests encryption performance. +// This calls the local implementation instead of library-go to avoid unsafe conversions. +func TestPerfEncryption(tb testing.TB, scenario library.PerfScenario) { + localEncryption.TestPerfEncryption(tb, scenario) +} diff --git a/test/library/featuregates.go b/test/library/featuregates.go new file mode 100644 index 0000000000..726734045e --- /dev/null +++ b/test/library/featuregates.go @@ -0,0 +1,42 @@ +package library + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configv1 "github.com/openshift/api/config/v1" + configclient "github.com/openshift/client-go/config/clientset/versioned" +) + +// CheckFeatureGatesOrSkip checks if any of the required feature gates are enabled. +// If none are enabled, the test is skipped. +func CheckFeatureGatesOrSkip(ctx context.Context, t testing.TB, configClient *configclient.Clientset, features ...configv1.FeatureGateName) { + featureGates, err := configClient.ConfigV1().FeatureGates().Get(ctx, "cluster", metav1.GetOptions{}) + require.NoError(t, err) + + if len(featureGates.Status.FeatureGates) != 1 { + // fail test if there are multiple feature gate versions (i.e. ongoing upgrade) + t.Fatalf("multiple feature gate versions detected") + } + + atLeastOneFeatureEnabled := false + for _, feature := range features { + for _, gate := range featureGates.Status.FeatureGates[0].Enabled { + if gate.Name == feature { + atLeastOneFeatureEnabled = true + break + } + } + + if atLeastOneFeatureEnabled { + break + } + } + + if !atLeastOneFeatureEnabled { + t.Skipf("skipping as none of the feature gates in %v are enabled", features) + } +} diff --git a/test/library/gitlabidp.go b/test/library/gitlabidp.go index 91ba7b5e31..56779ddcdc 100644 --- a/test/library/gitlabidp.go +++ b/test/library/gitlabidp.go @@ -37,6 +37,7 @@ func AddGitlabIDP( // TODO: possibly make this be a wrapper to a function to sim configclients, err := configv1client.NewForConfig(kubeconfig) require.NoError(t, err) + // GitLab requires privileged mode to manage system services (PostgreSQL, Redis, nginx, Sidekiq). nsName, gitlabHost, cleanup := deployPod(t, kubeClients, routeClient, "gitlab", "docker.io/gitlab/gitlab-ce:13.8.4-ce.0", @@ -85,6 +86,7 @@ func AddGitlabIDP( // TODO: possibly make this be a wrapper to a function to sim nil, nil, false, + true, // GitLab requires privileged mode ) cleanups = []func(){cleanup} diff --git a/test/library/idpdeployment.go b/test/library/idpdeployment.go index 7c6e082b4d..ae1e6fedb5 100644 --- a/test/library/idpdeployment.go +++ b/test/library/idpdeployment.go @@ -11,6 +11,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/intstr" @@ -22,7 +23,6 @@ import ( routev1 "github.com/openshift/api/route/v1" configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" routev1client "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1" - v1 "k8s.io/api/rbac/v1" "k8s.io/client-go/tools/cache" watchtools "k8s.io/client-go/tools/watch" ) @@ -33,6 +33,26 @@ func boolptr(b bool) *bool { return &b } +func createContainerSecurityContext(usePrivileged bool) *corev1.SecurityContext { + if usePrivileged { + return &corev1.SecurityContext{ + Privileged: boolptr(true), + } + } + + // Restricted security context compliant with PodSecurity restricted:latest policy + return &corev1.SecurityContext{ + AllowPrivilegeEscalation: boolptr(false), + RunAsNonRoot: boolptr(true), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + } +} + func deployPod( t testing.TB, clients *kubernetes.Clientset, @@ -46,6 +66,7 @@ func deployPod( readinessProbe *corev1.Probe, livenessProbe *corev1.Probe, useTLS bool, + usePrivilegedSecurity bool, command ...string, ) (namespace, host string, cleanup func()) { testContext := context.TODO() @@ -53,10 +74,17 @@ func deployPod( var err error cleanup = func() {} - namespace = NewTestNamespaceBuilder("e2e-test-authentication-operator-"). - WithPrivilegedPSaEnforcement(). - WithLabels(CAOE2ETestLabels()). - Create(t, clients.CoreV1().Namespaces()) + // Configure PSA enforcement: privileged for GitLab (requires root), restricted for Keycloak. + nsBuilder := NewTestNamespaceBuilder("e2e-test-authentication-operator-"). + WithLabels(CAOE2ETestLabels()) + + if usePrivilegedSecurity { + nsBuilder = nsBuilder.WithPrivilegedPSaEnforcement() + } else { + nsBuilder = nsBuilder.WithRestrictedPSaEnforcement() + } + + namespace = nsBuilder.Create(t, clients.CoreV1().Namespaces()) cleanup = func() { // remove the NS, it will take away all the resources create here along with it @@ -82,7 +110,7 @@ func deployPod( ) saName := name - pod := podTemplate(name, image, httpPort, httpsPort, command...) + pod := podTemplate(name, image, httpPort, httpsPort, usePrivilegedSecurity, command...) pod.Spec.Volumes = volumes pod.Spec.Containers[0].VolumeMounts = volumeMounts pod.Spec.Containers[0].Env = env @@ -95,6 +123,29 @@ func deployPod( } pod.Spec.ServiceAccountName = saName + // Grant privileged SCC when required (e.g., GitLab needs root access). + if usePrivilegedSecurity { + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "privileged-scc-to-default-sa", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: "system:openshift:scc:privileged", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: saName, + }, + }, + } + + _, err = clients.RbacV1().RoleBindings(namespace).Create(testContext, roleBinding, metav1.CreateOptions{}) + require.NoError(t, err) + } + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -110,26 +161,6 @@ func deployPod( }, } - roleBinding := &v1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "privileged-scc-to-default-sa", - }, - RoleRef: v1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: "system:openshift:scc:privileged", - }, - Subjects: []v1.Subject{ - { - Kind: "ServiceAccount", - Name: saName, - }, - }, - } - - _, err = clients.RbacV1().RoleBindings(namespace).Create(testContext, roleBinding, metav1.CreateOptions{}) - require.NoError(t, err) - _, err = clients.AppsV1().Deployments(namespace).Create(testContext, deployment, metav1.CreateOptions{}) require.NoError(t, err) @@ -159,8 +190,8 @@ func deployPod( return } -func podTemplate(name, image string, httpPort, httpsPort int32, command ...string) *corev1.Pod { - return &corev1.Pod{ +func podTemplate(name, image string, httpPort, httpsPort int32, usePrivilegedSecurity bool, command ...string) *corev1.Pod { + pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, Labels: map[string]string{ @@ -170,11 +201,9 @@ func podTemplate(name, image string, httpPort, httpsPort int32, command ...strin Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "payload", - Image: image, - SecurityContext: &corev1.SecurityContext{ - Privileged: boolptr(true), - }, + Name: "payload", + Image: image, + SecurityContext: createContainerSecurityContext(usePrivilegedSecurity), Ports: []corev1.ContainerPort{ { ContainerPort: httpsPort, @@ -188,6 +217,8 @@ func podTemplate(name, image string, httpPort, httpsPort int32, command ...strin }, }, } + + return pod } func svcTemplate(httpPort, httpsPort int32) *corev1.Service { @@ -247,6 +278,7 @@ func routeTemplate(useTLS bool) *routev1.Route { return r } +// CleanIDPConfigByName removes the identity provider with the given name from the OAuth cluster configuration. func CleanIDPConfigByName(t testing.TB, configClient configv1client.OAuthInterface, idpName string) { config, err := configClient.Get(context.TODO(), "cluster", metav1.GetOptions{}) if err != nil { @@ -279,6 +311,7 @@ func CleanIDPConfigByName(t testing.TB, configClient configv1client.OAuthInterfa } } +// IDPCleanupWrapper wraps a cleanup function and skips cleanup if OPENSHIFT_KEEP_IDP environment variable is set. func IDPCleanupWrapper(cleanup func()) func() { return func() { // allow keeping the IdP for manual testing @@ -290,8 +323,7 @@ func IDPCleanupWrapper(cleanup func()) func() { } } -// labels for listing/deleting stuff by hand, e.g. NS or simple openshift-config -// NS CMs and Secrets cleanup +// CAOE2ETestLabels returns labels used for listing/deleting test resources such as namespaces, ConfigMaps, and Secrets. func CAOE2ETestLabels() map[string]string { return map[string]string{ "e2e-test": "openshift-authentication-operator", diff --git a/test/library/keycloakidp.go b/test/library/keycloakidp.go index 8d3b254879..8f2606f416 100644 --- a/test/library/keycloakidp.go +++ b/test/library/keycloakidp.go @@ -27,6 +27,7 @@ import ( routev1client "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1" ) +// AddKeycloakIDP deploys a Keycloak identity provider for testing and returns the client, IDP name, and cleanup functions. func AddKeycloakIDP( t testing.TB, kubeconfig *rest.Config, @@ -103,6 +104,7 @@ func AddKeycloakIDP( &readinessProbe, &livenessProbe, true, + false, // Keycloak works with restricted mode "/opt/keycloak/bin/kc.sh", "start-dev", ) cleanups = []func(){cleanup} @@ -131,7 +133,7 @@ func AddKeycloakIDP( // In resource-constrained CI environments with parallel test execution, Keycloak can take // 40-60+ seconds to fully initialize its admin API even after passing readiness probes err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { - err := kcClient.AuthenticatePassword("admin-cli", "", "admin", "password") + err := kcClient.AuthenticatePassword(ctx, "admin-cli", "", "admin", "password") if err != nil { t.Logf("failed to authenticate to Keycloak: %v", err) return false, nil @@ -140,20 +142,20 @@ func AddKeycloakIDP( }) require.NoError(t, err) - clientList, err := kcClient.ListClients() + clientList, err := kcClient.ListClients(context.Background()) require.NoError(t, err) - var adminClientId, passwdClientId, passwdClientClientId string + var adminClientID, passwdClientID, passwdClientClientID string for _, c := range clientList { if clientID := c["clientId"].(string); clientID == "admin-cli" { - adminClientId = c["id"].(string) + adminClientID = c["id"].(string) } else if len(c["redirectUris"].([]interface{})) > 0 { // just reuse one other client that's already there - passwdClientId = c["id"].(string) - passwdClientClientId = clientID + passwdClientID = c["id"].(string) + passwdClientClientID = clientID } - if len(passwdClientId) > 0 && len(adminClientId) > 0 { + if len(passwdClientID) > 0 && len(adminClientID) > 0 { break } } @@ -161,11 +163,11 @@ func AddKeycloakIDP( // change the client's access token timeout just in case we need it for the test // Wrap in retry logic as Keycloak may still be unstable after initial authentication err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { - err := kcClient.UpdateClientAccessTokenTimeout(adminClientId, 60*30) + err := kcClient.UpdateClientAccessTokenTimeout(ctx, adminClientID, 60*30) if err != nil { t.Logf("failed to update client access token timeout: %v, retrying", err) // Re-authenticate in case the connection was dropped - if authErr := kcClient.AuthenticatePassword("admin-cli", "", "admin", "password"); authErr != nil { + if authErr := kcClient.AuthenticatePassword(ctx, "admin-cli", "", "admin", "password"); authErr != nil { t.Logf("failed to re-authenticate: %v", authErr) } return false, nil @@ -175,18 +177,18 @@ func AddKeycloakIDP( require.NoError(t, err) // reauthenticate for a new, longer-lived token - err = kcClient.AuthenticatePassword("admin-cli", "", "admin", "password") + err = kcClient.AuthenticatePassword(context.Background(), "admin-cli", "", "admin", "password") require.NoError(t, err) // Regenerate client secret with retry logic for Keycloak stability var clientSecret string err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { var err error - clientSecret, err = kcClient.RegenerateClientSecret(passwdClientId) + clientSecret, err = kcClient.RegenerateClientSecret(ctx, passwdClientID) if err != nil { t.Logf("failed to regenerate client secret: %v, retrying", err) // Re-authenticate in case the connection was dropped - if authErr := kcClient.AuthenticatePassword("admin-cli", "", "admin", "password"); authErr != nil { + if authErr := kcClient.AuthenticatePassword(ctx, "admin-cli", "", "admin", "password"); authErr != nil { t.Logf("failed to re-authenticate: %v", authErr) } return false, nil @@ -198,11 +200,11 @@ func AddKeycloakIDP( // Create client group mapper with retry logic const groupsClaimName = "groups" err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { - err := kcClient.CreateClientGroupMapper(passwdClientId, "test-groups-mapper", groupsClaimName) + err := kcClient.CreateClientGroupMapper(ctx, passwdClientID, "test-groups-mapper", groupsClaimName) if err != nil { t.Logf("failed to create client group mapper: %v, retrying", err) // Re-authenticate in case the connection was dropped - if authErr := kcClient.AuthenticatePassword("admin-cli", "", "admin", "password"); authErr != nil { + if authErr := kcClient.AuthenticatePassword(ctx, "admin-cli", "", "admin", "password"); authErr != nil { t.Logf("failed to re-authenticate: %v", authErr) } return false, nil @@ -214,7 +216,7 @@ func AddKeycloakIDP( idpCleans, err := addOIDCIDentityProvider(t, kubeClients, configClient, - passwdClientClientId, clientSecret, + passwdClientClientID, clientSecret, openshiftIDPName, keycloakURL, configv1.OpenIDClaims{ @@ -229,6 +231,7 @@ func AddKeycloakIDP( return kcClient, openshiftIDPName, cleanups } +// KeycloakClient provides methods for interacting with a Keycloak server for testing. type KeycloakClient struct { keycloakAdminURL *url.URL realm string @@ -248,6 +251,7 @@ func KeycloakClientFor(t testing.TB, transport http.RoundTripper, keycloakURL, k u.Path = "/admin/realms/" + keycloakRealm client := &http.Client{ + Timeout: 30 * time.Second, Transport: transport, } @@ -261,7 +265,8 @@ func KeycloakClientFor(t testing.TB, transport http.RoundTripper, keycloakURL, k return c } -func (kc *KeycloakClient) AuthenticatePassword(clientID, clientSecret, name, password string) error { +// AuthenticatePassword authenticates a user with the given username and password against the Keycloak server. +func (kc *KeycloakClient) AuthenticatePassword(ctx context.Context, clientID, clientSecret, name, password string) error { data := url.Values{ "username": []string{name}, "password": []string{password}, @@ -273,7 +278,7 @@ func (kc *KeycloakClient) AuthenticatePassword(clientID, clientSecret, name, pas data.Add("client_secret", clientSecret) } - authReq, err := http.NewRequest(http.MethodPost, kc.TokenURL(), bytes.NewBufferString(data.Encode())) + authReq, err := http.NewRequestWithContext(ctx, http.MethodPost, kc.TokenURL(), bytes.NewBufferString(data.Encode())) if err != nil { return err } @@ -321,13 +326,16 @@ func (kc *KeycloakClient) AuthenticatePassword(clientID, clientSecret, name, pas return nil } +// Tokens returns the current access token and ID token. func (kc *KeycloakClient) Tokens() (accessToken, idToken string) { return kc.accessToken, kc.idToken + // CreateClientGroupMapper creates a group membership mapper for the specified client. } -func (kc *KeycloakClient) CreateClientGroupMapper(clientId, mapperName, groupsClaimName string) error { +// CreateClientGroupMapper creates a group membership mapper for the specified client. +func (kc *KeycloakClient) CreateClientGroupMapper(ctx context.Context, clientID, mapperName, groupsClaimName string) error { mappersURL := *kc.keycloakAdminURL - mappersURL.Path += "/clients/" + clientId + "/protocol-mappers/models" + mappersURL.Path += "/clients/" + clientID + "/protocol-mappers/models" mapper := map[string]interface{}{ "name": mapperName, @@ -348,7 +356,7 @@ func (kc *KeycloakClient) CreateClientGroupMapper(clientId, mapperName, groupsCl } // Keycloak does not return the object on successful create so there's no need to attempt to retrieve it from the response - resp, err := kc.do(http.MethodPost, mappersURL.String(), bytes.NewBuffer(mapperBytes)) + resp, err := kc.do(ctx, http.MethodPost, mappersURL.String(), bytes.NewBuffer(mapperBytes)) if err != nil { return err } @@ -359,10 +367,12 @@ func (kc *KeycloakClient) CreateClientGroupMapper(clientId, mapperName, groupsCl return fmt.Errorf("failed creating mapper %q: %s %s", mapperName, resp.Status, respBytes) } + // CreateGroup creates a new group with the given name. return nil } -func (kc *KeycloakClient) CreateGroup(groupName string) error { +// CreateGroup creates a new group with the given name. +func (kc *KeycloakClient) CreateGroup(ctx context.Context, groupName string) error { groupsURL := *kc.keycloakAdminURL groupsURL.Path += "/groups" @@ -376,7 +386,7 @@ func (kc *KeycloakClient) CreateGroup(groupName string) error { } // Keycloak does not return the object on successful create so there's no need to attempt to retrieve it from the response - resp, err := kc.do(http.MethodPost, groupsURL.String(), bytes.NewBuffer(groupBytes)) + resp, err := kc.do(ctx, http.MethodPost, groupsURL.String(), bytes.NewBuffer(groupBytes)) if err != nil { return err } @@ -386,11 +396,98 @@ func (kc *KeycloakClient) CreateGroup(groupName string) error { respBytes, _ := io.ReadAll(resp.Body) return fmt.Errorf("failed creating group %q: %s %s", groupName, resp.Status, respBytes) } + // ListGroups returns all groups in the realm. + return nil +} + +// ListGroups returns all groups in the realm. +func (kc *KeycloakClient) ListGroups(ctx context.Context) ([]map[string]interface{}, error) { + allGroups := []map[string]interface{}{} + first := 0 + max := 100 // Keycloak default page size + + for { + groupsURL := *kc.keycloakAdminURL + groupsURL.Path += "/groups" + q := groupsURL.Query() + q.Set("first", fmt.Sprintf("%d", first)) + q.Set("max", fmt.Sprintf("%d", max)) + groupsURL.RawQuery = q.Encode() + + resp, err := kc.do(ctx, http.MethodGet, groupsURL.String(), nil) + if err != nil { + return nil, err + } + + respBytes, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("listing groups failed: %s: %s", resp.Status, respBytes) + } + + groups := []map[string]interface{}{} + if err := json.Unmarshal(respBytes, &groups); err != nil { + return nil, err + } + + if len(groups) == 0 { + break + } + + allGroups = append(allGroups, groups...) + first += len(groups) + // DeleteGroup deletes the group with the given name. + } + + return allGroups, nil +} + +// DeleteGroup deletes the group with the given name. +func (kc *KeycloakClient) DeleteGroup(ctx context.Context, groupName string) error { + // First, find the group by name to get its ID + groups, err := kc.ListGroups(ctx) + if err != nil { + return fmt.Errorf("failed to list groups: %w", err) + } + + var groupID string + for _, group := range groups { + if name, ok := group["name"].(string); ok && name == groupName { + if id, ok := group["id"].(string); ok { + groupID = id + break + } + } + } + + if groupID == "" { + // Group not found - not an error, it may have already been deleted + return nil + } + + groupsURL := *kc.keycloakAdminURL + groupsURL.Path += "/groups/" + groupID + + resp, err := kc.do(ctx, http.MethodDelete, groupsURL.String(), nil) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusNotFound { + respBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed deleting group %q (ID: %s): %s %s", groupName, groupID, resp.Status, respBytes) + } return nil } -func (kc *KeycloakClient) CreateUser(username, email, password string, groups []string, extraFields map[string]string) error { +// CreateUser creates a new user with the given details. +func (kc *KeycloakClient) CreateUser(ctx context.Context, username, email, password string, groups []string, extraFields map[string]string) error { usersURL := *kc.keycloakAdminURL usersURL.Path += "/users" @@ -423,7 +520,7 @@ func (kc *KeycloakClient) CreateUser(username, email, password string, groups [] } // Keycloak does not return the object on successful create so there's no need to attempt to retrieve it from the response - resp, err := kc.do(http.MethodPost, usersURL.String(), bytes.NewBuffer(userBytes)) + resp, err := kc.do(ctx, http.MethodPost, usersURL.String(), bytes.NewBuffer(userBytes)) if err != nil { return err } @@ -437,35 +534,94 @@ func (kc *KeycloakClient) CreateUser(username, email, password string, groups [] return nil } -func (kc *KeycloakClient) ListUsers() ([]map[string]interface{}, error) { +// DeleteUser deletes the user with the given username. +func (kc *KeycloakClient) DeleteUser(ctx context.Context, username string) error { + // First, find the user by username to get its ID + users, err := kc.ListUsers(ctx) + if err != nil { + return fmt.Errorf("failed to list users: %w", err) + } + + var userID string + for _, user := range users { + if name, ok := user["username"].(string); ok && name == username { + if id, ok := user["id"].(string); ok { + userID = id + break + } + } + } + + if userID == "" { + // User not found - not an error, it may have already been deleted + return nil + } + usersURL := *kc.keycloakAdminURL - usersURL.Path += "/users" + usersURL.Path += "/users/" + userID - resp, err := kc.do(http.MethodGet, usersURL.String(), nil) + resp, err := kc.do(ctx, http.MethodDelete, usersURL.String(), nil) if err != nil { - return nil, err + return err } defer resp.Body.Close() - respBytes, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusNotFound { + respBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed deleting user %q (ID: %s): %s %s", username, userID, resp.Status, respBytes) } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("listing users failed: %s: %s", resp.Status, respBytes) - } + return nil +} - users := []map[string]interface{}{} - if err := json.Unmarshal(respBytes, &users); err != nil { - return nil, err +// ListUsers returns all users in the realm. +func (kc *KeycloakClient) ListUsers(ctx context.Context) ([]map[string]interface{}, error) { + allUsers := []map[string]interface{}{} + first := 0 + max := 100 // Keycloak default page size + + for { + usersURL := *kc.keycloakAdminURL + usersURL.Path += "/users" + q := usersURL.Query() + q.Set("first", fmt.Sprintf("%d", first)) + q.Set("max", fmt.Sprintf("%d", max)) + usersURL.RawQuery = q.Encode() + + resp, err := kc.do(ctx, http.MethodGet, usersURL.String(), nil) + if err != nil { + return nil, err + } + + respBytes, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("listing users failed: %s: %s", resp.Status, respBytes) + } + + users := []map[string]interface{}{} + if err := json.Unmarshal(respBytes, &users); err != nil { + return nil, err + } + + if len(users) == 0 { + break + } + + allUsers = append(allUsers, users...) + first += len(users) } - return users, nil + return allUsers, nil } -func (kc *KeycloakClient) UpdateUser(id string, changes map[string]interface{}) error { - user, err := kc.GetUser(id) +// UpdateUser updates the user with the given ID using the specified changes. +func (kc *KeycloakClient) UpdateUser(ctx context.Context, id string, changes map[string]interface{}) error { + user, err := kc.GetUser(ctx, id) if err != nil { return err } @@ -482,7 +638,7 @@ func (kc *KeycloakClient) UpdateUser(id string, changes map[string]interface{}) usersURL := *kc.keycloakAdminURL usersURL.Path += "/users/" + id - resp, err := kc.do(http.MethodPut, usersURL.String(), bytes.NewBuffer(userBytes)) + resp, err := kc.do(ctx, http.MethodPut, usersURL.String(), bytes.NewBuffer(userBytes)) if err != nil { return err } @@ -496,11 +652,12 @@ func (kc *KeycloakClient) UpdateUser(id string, changes map[string]interface{}) return nil } -func (kc *KeycloakClient) GetUser(id string) (map[string]interface{}, error) { +// GetUser returns the user with the given ID. +func (kc *KeycloakClient) GetUser(ctx context.Context, id string) (map[string]interface{}, error) { usersURL := *kc.keycloakAdminURL usersURL.Path += "/users/" + id - resp, err := kc.do(http.MethodGet, usersURL.String(), nil) + resp, err := kc.do(ctx, http.MethodGet, usersURL.String(), nil) if err != nil { return nil, err } @@ -519,11 +676,12 @@ func (kc *KeycloakClient) GetUser(id string) (map[string]interface{}, error) { return user, nil } -func (kc *KeycloakClient) ListUserGroups(id string) ([]map[string]interface{}, error) { +// ListUserGroups returns all groups that the user with the given ID belongs to. +func (kc *KeycloakClient) ListUserGroups(ctx context.Context, id string) ([]map[string]interface{}, error) { userGroupsURL := *kc.keycloakAdminURL userGroupsURL.Path += "/users/" + id + "/groups" - resp, err := kc.do(http.MethodGet, userGroupsURL.String(), nil) + resp, err := kc.do(ctx, http.MethodGet, userGroupsURL.String(), nil) if err != nil { return nil, err } @@ -542,20 +700,23 @@ func (kc *KeycloakClient) ListUserGroups(id string) ([]map[string]interface{}, e return userGroups, nil } -func (kc *KeycloakClient) DeleteUserFromGroups(userId string, groupIds ...string) error { +// DeleteUser deletes the user with the given username. +// DeleteUserFromGroups removes the user from the specified groups. +func (kc *KeycloakClient) DeleteUserFromGroups(ctx context.Context, userID string, groupIds ...string) error { userGroupsURL := *kc.keycloakAdminURL - userGroupsURL.Path += "/users/" + userId + "/groups/" + userGroupsURL.Path += "/users/" + userID + "/groups/" for _, gid := range groupIds { - resp, err := kc.do(http.MethodDelete, userGroupsURL.String()+gid, nil) + resp, err := kc.do(ctx, http.MethodDelete, userGroupsURL.String()+gid, nil) if err != nil { return err } - defer resp.Body.Close() if resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("failed removing group %q from user %q: the server returned %s", gid, userId, resp.Status) + resp.Body.Close() + return fmt.Errorf("failed removing group %q from user %q: the server returned %s", gid, userID, resp.Status) } + resp.Body.Close() } return nil @@ -563,29 +724,30 @@ func (kc *KeycloakClient) DeleteUserFromGroups(userId string, groupIds ...string // UpdateClientAccessTokenTimeout updates the timeout for a client of the given id // timeout is a timeout in seconds -func (kc *KeycloakClient) UpdateClientAccessTokenTimeout(id string, timeout int32) error { +func (kc *KeycloakClient) UpdateClientAccessTokenTimeout(ctx context.Context, id string, timeout int32) error { changes := map[string]interface{}{ "attributes": map[string]interface{}{ "access.token.lifespan": timeout, }, } - return kc.UpdateClient(id, changes) + return kc.UpdateClient(ctx, id, changes) } // UpdateClientDirectAccessGrantsEnabled updates the `directAccessGrantsEnabled` // attribute of the client which influences whether the password grant is allowed // via the client or not -func (kc *KeycloakClient) UpdateClientDirectAccessGrantsEnabled(id string, allow bool) error { +func (kc *KeycloakClient) UpdateClientDirectAccessGrantsEnabled(ctx context.Context, id string, allow bool) error { changes := map[string]interface{}{ "directAccessGrantsEnabled": allow, } - return kc.UpdateClient(id, changes) + return kc.UpdateClient(ctx, id, changes) } -func (kc *KeycloakClient) UpdateClient(id string, changedFields map[string]interface{}) error { - client, err := kc.GetClient(id) +// UpdateClient updates the client with the given ID using the specified fields. +func (kc *KeycloakClient) UpdateClient(ctx context.Context, id string, changedFields map[string]interface{}) error { + client, err := kc.GetClient(ctx, id) if err != nil { return err } @@ -601,7 +763,7 @@ func (kc *KeycloakClient) UpdateClient(id string, changedFields map[string]inter clientsURL := *kc.keycloakAdminURL clientsURL.Path += "/clients/" + id - resp, err := kc.do(http.MethodPut, clientsURL.String(), bytes.NewBuffer(clientBytes)) + resp, err := kc.do(ctx, http.MethodPut, clientsURL.String(), bytes.NewBuffer(clientBytes)) if err != nil { return err } @@ -620,16 +782,17 @@ func (kc *KeycloakClient) UpdateClient(id string, changedFields map[string]inter } // GetClient retrieves a client based on its id (NOTE: id != clientID) -func (kc *KeycloakClient) GetClient(id string) (map[string]interface{}, error) { +func (kc *KeycloakClient) GetClient(ctx context.Context, id string) (map[string]interface{}, error) { clientsURL := *kc.keycloakAdminURL clientsURL.Path += "/clients/" + id - resp, err := kc.do(http.MethodGet, clientsURL.String(), nil) + resp, err := kc.do(ctx, http.MethodGet, clientsURL.String(), nil) if err != nil { return nil, err } defer resp.Body.Close() + // GetClientByClientID returns the client with the given client ID. respBytes, err := io.ReadAll(resp.Body) if err != nil { return nil, err @@ -644,8 +807,10 @@ func (kc *KeycloakClient) GetClient(id string) (map[string]interface{}, error) { return client, err } -func (kc *KeycloakClient) GetClientByClientID(clientID string) (map[string]interface{}, error) { - clients, err := kc.ListClients() +// GetClientByClientID returns the client with the given client ID. +func (kc *KeycloakClient) GetClientByClientID(ctx context.Context, clientID string) (map[string]interface{}, error) { + // ListClients returns all clients in the realm. + clients, err := kc.ListClients(ctx) if err != nil { return nil, err } @@ -660,14 +825,15 @@ func (kc *KeycloakClient) GetClientByClientID(clientID string) (map[string]inter } // GetClient retrieves a client based on its id (NOTE: id != name) -func (kc *KeycloakClient) ListClients() ([]map[string]interface{}, error) { +func (kc *KeycloakClient) ListClients(ctx context.Context) ([]map[string]interface{}, error) { clientsURL := *kc.keycloakAdminURL clientsURL.Path += "/clients" - resp, err := kc.do(http.MethodGet, clientsURL.String(), nil) + resp, err := kc.do(ctx, http.MethodGet, clientsURL.String(), nil) if err != nil { return nil, err } + // RegenerateClientSecret regenerates and returns the client secret for the client with the given ID. defer resp.Body.Close() respBytes, err := io.ReadAll(resp.Body) @@ -684,11 +850,12 @@ func (kc *KeycloakClient) ListClients() ([]map[string]interface{}, error) { return clients, err } -func (kc *KeycloakClient) RegenerateClientSecret(id string) (string, error) { +// RegenerateClientSecret regenerates and returns the client secret for the client with the given ID. +func (kc *KeycloakClient) RegenerateClientSecret(ctx context.Context, id string) (string, error) { clientRegenURL := *kc.keycloakAdminURL clientRegenURL.Path += "/clients/" + id + "/client-secret" - resp, err := kc.do(http.MethodPost, clientRegenURL.String(), nil) + resp, err := kc.do(ctx, http.MethodPost, clientRegenURL.String(), nil) if err != nil { return "", err } @@ -715,12 +882,12 @@ func (kc *KeycloakClient) RegenerateClientSecret(id string) (string, error) { return secretVal, nil } -func (kc *KeycloakClient) do(method, url string, body io.Reader) (*http.Response, error) { +func (kc *KeycloakClient) do(ctx context.Context, method, url string, body io.Reader) (*http.Response, error) { if len(kc.accessToken) == 0 { return nil, fmt.Errorf("authenticate first") } - req, err := http.NewRequest(method, url, body) + req, err := http.NewRequestWithContext(ctx, method, url, body) if err != nil { return nil, err } @@ -732,10 +899,12 @@ func (kc *KeycloakClient) do(method, url string, body io.Reader) (*http.Response return kc.client.Do(req) } +// AdminURL returns the Keycloak admin URL. func (kc *KeycloakClient) AdminURL() string { return kc.keycloakAdminURL.String() } +// TokenURL returns the Keycloak token endpoint URL. func (kc *KeycloakClient) TokenURL() string { authURL := *kc.keycloakAdminURL authURL.Path = "/realms/" + kc.realm + "/protocol/openid-connect/token" @@ -743,6 +912,7 @@ func (kc *KeycloakClient) TokenURL() string { return authURL.String() } +// IssuerURL returns the Keycloak OIDC issuer URL. func (kc *KeycloakClient) IssuerURL() string { issuerURL := *kc.keycloakAdminURL issuerURL.Path = "/realms/" + kc.realm diff --git a/test/library/waits.go b/test/library/waits.go index 692896b411..7a924a3878 100644 --- a/test/library/waits.go +++ b/test/library/waits.go @@ -22,6 +22,7 @@ import ( "github.com/openshift/library-go/pkg/route/routeapihelpers" ) +// WaitForOperatorToPickUpChanges waits for the operator to pick up changes by checking if it becomes progressing and then available. func WaitForOperatorToPickUpChanges(t testing.TB, configClient configv1client.ConfigV1Interface, name string) error { if err := WaitForClusterOperatorProgressing(t, configClient, name); err != nil { return fmt.Errorf("authentication operator never became progressing: %v", err) @@ -34,6 +35,7 @@ func WaitForOperatorToPickUpChanges(t testing.TB, configClient configv1client.Co return nil } +// WaitForClusterOperatorAvailableNotProgressingNotDegraded waits for the cluster operator to be available, not progressing, and not degraded. func WaitForClusterOperatorAvailableNotProgressingNotDegraded(t testing.TB, client configv1client.ConfigV1Interface, name string) error { return WaitForClusterOperatorStatus(t, client, name, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue}, @@ -42,18 +44,21 @@ func WaitForClusterOperatorAvailableNotProgressingNotDegraded(t testing.TB, clie ) } +// WaitForClusterOperatorDegraded waits for the cluster operator to become degraded. func WaitForClusterOperatorDegraded(t testing.TB, client configv1client.ConfigV1Interface, name string) error { return WaitForClusterOperatorStatus(t, client, name, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorDegraded, Status: configv1.ConditionTrue}, ) } +// WaitForClusterOperatorProgressing waits for the cluster operator to become progressing. func WaitForClusterOperatorProgressing(t testing.TB, client configv1client.ConfigV1Interface, name string) error { return WaitForClusterOperatorStatus(t, client, name, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue}, ) } +// WaitForClusterOperatorStatus waits for the cluster operator to reach the specified status conditions. func WaitForClusterOperatorStatus(t testing.TB, client configv1client.ConfigV1Interface, name string, requiredConditions ...configv1.ClusterOperatorStatusCondition) error { var done bool var conditions []configv1.ClusterOperatorStatusCondition @@ -62,7 +67,7 @@ func WaitForClusterOperatorStatus(t testing.TB, client configv1client.ConfigV1In t.Logf("will wait up to 10m for clusteroperators.config.openshift.io/%s status to be: %v", name, conditionsStatusString(requiredConditions)) ts := time.Now() err := wait.PollUntilContextTimeout(context.TODO(), 10*time.Second, 10*time.Minute, true, func(ctx context.Context) (bool, error) { - done, conditions, checkErr = CheckClusterOperatorStatus(t, ctx, client, name, requiredConditions...) + done, conditions, checkErr = CheckClusterOperatorStatus(ctx, t, client, name, requiredConditions...) return done, checkErr }) @@ -78,13 +83,13 @@ func WaitForClusterOperatorStatus(t testing.TB, client configv1client.ConfigV1In // WaitForClusterOperatorStatusStable checks that the specified cluster operator's status does not diverge // from the conditions specified for 10 minutes. It returns nil if all conditions were matching expectations for that // period, and an error otherwise. -func WaitForClusterOperatorStatusStable(t *testing.T, ctx context.Context, client configv1client.ConfigV1Interface, name string, requiredConditions ...configv1.ClusterOperatorStatusCondition) error { +func WaitForClusterOperatorStatusStable(ctx context.Context, t testing.TB, client configv1client.ConfigV1Interface, name string, requiredConditions ...configv1.ClusterOperatorStatusCondition) error { t.Logf("will wait up to 10m for clusteroperators.config.openshift.io/%s status to be stable: %v", name, conditionsStatusString(requiredConditions)) var endConditions []configv1.ClusterOperatorStatusCondition ts := time.Now() err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 10*time.Minute, true, func(_ context.Context) (bool, error) { - done, conditions, checkErr := CheckClusterOperatorStatus(t, ctx, client, name, requiredConditions...) + done, conditions, checkErr := CheckClusterOperatorStatus(ctx, t, client, name, requiredConditions...) if len(conditions) > 0 { endConditions = conditions } @@ -113,6 +118,7 @@ func conditionsStatusString(conditions []configv1.ClusterOperatorStatusCondition return strings.Join(conditionStrings, "/") } +// WaitForRouteAdmitted waits for the route to be admitted and returns its URL. func WaitForRouteAdmitted(t testing.TB, client routev1client.RouteV1Interface, name, ns string) (string, error) { var admittedURL string @@ -123,18 +129,19 @@ func WaitForRouteAdmitted(t testing.TB, client routev1client.RouteV1Interface, n t.Logf("route.Get(%s/%s) error: %v", ns, name, err) return false, nil } - if _, ingress, err := routeapihelpers.IngressURI(route, ""); err != nil { + _, ingress, err := routeapihelpers.IngressURI(route, "") + if err != nil { t.Log(err) return false, nil - } else { - admittedURL = ingress.Host } + admittedURL = ingress.Host return true, nil }) return admittedURL, err } +// WaitForHTTPStatus waits for the target URL to return the expected HTTP status code. func WaitForHTTPStatus(t testing.TB, waitDuration time.Duration, client *http.Client, targetURL string, expectedStatus int) error { t.Logf("waiting for HEAD at %q to report %d", targetURL, expectedStatus) @@ -157,7 +164,8 @@ func WaitForHTTPStatus(t testing.TB, waitDuration time.Duration, client *http.Cl }) } -func WaitForNewKASRollout(t *testing.T, ctx context.Context, kasClient operatorv1client.KubeAPIServerInterface, origRevision int32) error { +// WaitForNewKASRollout waits for the Kube API Server to complete a rollout to a new revision. +func WaitForNewKASRollout(ctx context.Context, t testing.TB, kasClient operatorv1client.KubeAPIServerInterface, origRevision int32) error { t.Logf("will wait for KAS rollout; latest available revision: %d", origRevision) var latestRevision int32 err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 30*time.Minute, true, func(ctx context.Context) (bool, error) { @@ -167,11 +175,14 @@ func WaitForNewKASRollout(t *testing.T, ctx context.Context, kasClient operatorv return false, nil } - for _, nodeStatus := range kas.Status.NodeStatuses { - if kas.Status.LatestAvailableRevision == origRevision { - return false, nil - } + if kas.Status.LatestAvailableRevision == origRevision { + return false, nil + } + if len(kas.Status.NodeStatuses) == 0 { + return false, nil + } + for _, nodeStatus := range kas.Status.NodeStatuses { if nodeStatus.CurrentRevision != kas.Status.LatestAvailableRevision { return false, nil } @@ -189,14 +200,16 @@ func WaitForNewKASRollout(t *testing.T, ctx context.Context, kasClient operatorv return nil } -func WaitForClusterOperatorStatusAlwaysAvailable(t *testing.T, ctx context.Context, client configv1client.ConfigV1Interface, name string) error { - return WaitForClusterOperatorStatusStable(t, ctx, client, name, +// WaitForClusterOperatorStatusAlwaysAvailable waits for the cluster operator to remain available and not degraded for the entire duration. +func WaitForClusterOperatorStatusAlwaysAvailable(ctx context.Context, t testing.TB, client configv1client.ConfigV1Interface, name string) error { + return WaitForClusterOperatorStatusStable(ctx, t, client, name, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue}, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorDegraded, Status: configv1.ConditionFalse}, ) } -func CheckClusterOperatorStatus(t testing.TB, ctx context.Context, client configv1client.ConfigV1Interface, name string, requiredConditions ...configv1.ClusterOperatorStatusCondition) (bool, []configv1.ClusterOperatorStatusCondition, error) { +// CheckClusterOperatorStatus checks if the cluster operator status matches the required conditions. +func CheckClusterOperatorStatus(ctx context.Context, t testing.TB, client configv1client.ConfigV1Interface, name string, requiredConditions ...configv1.ClusterOperatorStatusCondition) (bool, []configv1.ClusterOperatorStatusCondition, error) { clusterOperator, err := client.ClusterOperators().Get(ctx, name, metav1.GetOptions{}) if kerrors.IsNotFound(err) || retry.IsHTTPClientError(err) { t.Logf("error while getting clusteroperators.config.openshift.io/%v, will retry: %v", name, err)