From 1e3de6b709bfe34e93ed79c26ba4d7d2c96b47e0 Mon Sep 17 00:00:00 2001 From: Benji Date: Fri, 12 Jun 2026 18:01:26 +0000 Subject: [PATCH 1/3] feat(kas): enable topology aware routing on kube-apiserver ClusterIP service --- .../controllers/hostedcontrolplane/kas/service.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/service.go b/control-plane-operator/controllers/hostedcontrolplane/kas/service.go index ffd30fb0c040..7b142cb8fdfb 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/service.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/service.go @@ -95,6 +95,11 @@ func ReconcileService(svc *corev1.Service, strategy *hyperv1.ServicePublishingSt } } else { svc.Spec.Type = corev1.ServiceTypeClusterIP + // Enable topology aware routing so that callers (KCM, scheduler, operators, etc.) + // are routed to a KAS pod in their own AZ, avoiding cross-AZ data transfer charges. + // KAS pods are spread across zones via requiredDuringScheduling anti-affinity, + // satisfying the >=1 endpoint per zone precondition for TAR to activate. + svc.Annotations["service.kubernetes.io/topology-mode"] = "Auto" } case hyperv1.NodePort: svc.Spec.Type = corev1.ServiceTypeNodePort @@ -140,6 +145,9 @@ func ReconcileServiceClusterIP(svc *corev1.Service, owner *metav1.OwnerReference if svc.Annotations == nil { svc.Annotations = map[string]string{} } + // Enable topology aware routing so that callers are routed to a KAS pod in their + // own AZ, avoiding cross-AZ data transfer charges. + svc.Annotations["service.kubernetes.io/topology-mode"] = "Auto" svc.Spec.Type = corev1.ServiceTypeClusterIP svc.Spec.Ports[0] = portSpec return nil From 01a55f36796ab5071f658a7483ff7fbe55ea4da7 Mon Sep 17 00:00:00 2001 From: Benji Date: Fri, 12 Jun 2026 18:01:35 +0000 Subject: [PATCH 2/3] test(kas): add tests for topology aware routing annotation --- .../hostedcontrolplane/kas/service_test.go | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/service_test.go b/control-plane-operator/controllers/hostedcontrolplane/kas/service_test.go index 67dfe1e8d4bb..4f6ee07f78a3 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/service_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/service_test.go @@ -125,6 +125,81 @@ func TestReconcileService(t *testing.T) { } } +func TestReconcileServiceTopologyAwareRouting(t *testing.T) { + const topologyModeAnnotation = "service.kubernetes.io/topology-mode" + + testCases := []struct { + name string + hcp *hyperv1.HostedControlPlane + strategy hyperv1.ServicePublishingStrategy + expectTARAnnotation bool + }{ + { + name: "When Route strategy it should set topology-mode annotation on ClusterIP service", + hcp: &hyperv1.HostedControlPlane{ + Spec: hyperv1.HostedControlPlaneSpec{ + Platform: hyperv1.PlatformSpec{Type: hyperv1.AWSPlatform}, + }, + }, + strategy: hyperv1.ServicePublishingStrategy{Type: hyperv1.Route}, + expectTARAnnotation: true, + }, + { + name: "When private-only AWS LoadBalancer strategy it should set topology-mode annotation on ClusterIP service", + hcp: &hyperv1.HostedControlPlane{ + Spec: hyperv1.HostedControlPlaneSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AWSPlatform, + AWS: &hyperv1.AWSPlatformSpec{ + EndpointAccess: hyperv1.Private, + }, + }, + }, + }, + strategy: hyperv1.ServicePublishingStrategy{Type: hyperv1.LoadBalancer}, + expectTARAnnotation: true, + }, + { + name: "When public AWS LoadBalancer strategy it should not set topology-mode annotation on LB service", + hcp: &hyperv1.HostedControlPlane{ + Spec: hyperv1.HostedControlPlaneSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AWSPlatform, + AWS: &hyperv1.AWSPlatformSpec{ + EndpointAccess: hyperv1.Public, + }, + }, + }, + }, + strategy: hyperv1.ServicePublishingStrategy{Type: hyperv1.LoadBalancer}, + expectTARAnnotation: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + svc := &corev1.Service{} + err := ReconcileService(svc, &tc.strategy, &v1.OwnerReference{}, 6443, []string{}, tc.hcp) + g.Expect(err).To(BeNil()) + if tc.expectTARAnnotation { + g.Expect(svc.Annotations).To(HaveKeyWithValue(topologyModeAnnotation, "Auto")) + } else { + g.Expect(svc.Annotations).ToNot(HaveKey(topologyModeAnnotation)) + } + }) + } +} + +func TestReconcileServiceClusterIPTopologyAwareRouting(t *testing.T) { + g := NewWithT(t) + svc := &corev1.Service{} + err := ReconcileServiceClusterIP(svc, &v1.OwnerReference{}) + g.Expect(err).To(BeNil()) + g.Expect(svc.Annotations).To(HaveKeyWithValue("service.kubernetes.io/topology-mode", "Auto")) + g.Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) +} + func TestReconcileServiceAzureInternalLB(t *testing.T) { // The main KAS service (kube-apiserver-azure-lb) should never have the internal LB // annotation. For Azure Private, this service becomes ClusterIP because isPublic=false. From d5d9a11ccd801b48ed56828afbb6987a009cdd34 Mon Sep 17 00:00:00 2001 From: Benji Date: Fri, 12 Jun 2026 18:40:27 +0000 Subject: [PATCH 3/3] fix(kas): add topology-mode annotation to Route strategy ClusterIP path --- .../controllers/hostedcontrolplane/kas/service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/service.go b/control-plane-operator/controllers/hostedcontrolplane/kas/service.go index 7b142cb8fdfb..327b93049d96 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/service.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/service.go @@ -109,6 +109,7 @@ func ReconcileService(svc *corev1.Service, strategy *hyperv1.ServicePublishingSt case hyperv1.Route: if hcp.Spec.Platform.Type != hyperv1.IBMCloudPlatform || svc.Spec.Type != corev1.ServiceTypeNodePort { svc.Spec.Type = corev1.ServiceTypeClusterIP + svc.Annotations["service.kubernetes.io/topology-mode"] = "Auto" } default: return fmt.Errorf("invalid publishing strategy for Kube API server service: %s", strategy.Type)