Merge pull request #568 from chengshifan/disable-legacy-auth

chengshifan · web-flow · commit edf54c04bef8 · 2023-09-01T15:23:02.000+08:00
chore: disable legacy-authorization when creating clusters
diff --git a/deploy/infrabox/templates/api/deployment.yaml b/deploy/infrabox/templates/api/deployment.yaml
@@ -48,7 +48,7 @@ spec:
                         memory: "1200Mi"
                     limits:
                         cpu: "1000m"
-                        memory: "2000Mi"
+                        memory: "4Gi"
                 volumeMounts:
                 {{ include "mounts_rsa_public" . | indent 16 }}
                 {{ include "mounts_rsa_private" . | indent 16 }}
diff --git a/deploy/infrabox/templates/scheduler/deployment.yaml b/deploy/infrabox/templates/scheduler/deployment.yaml
@@ -29,7 +29,7 @@ spec:
                         memory: "400Mi"
                     limits:
                         cpu: "500m"
-                        memory: "800Mi"
+                        memory: "2Gi"
                 env:
                 {{ include "env_database" . | indent 16 }}
                 {{ include "env_general" . | indent 16 }}
diff --git a/src/services/gcp/Dockerfile b/src/services/gcp/Dockerfile
@@ -25,6 +25,8 @@ RUN apk --no-cache add \
         libc6-compat \
         openssh-client \
         git \
+        python3 \
+        py3-pip \
     && curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
     tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
     rm google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \
@@ -34,10 +36,12 @@ RUN apk --no-cache add \
     gcloud config set metrics/environment github_docker_image && \
     gcloud components install kubectl && \
     gcloud --version && \
-    gcloud components install beta
+    gcloud components install beta && \
+    pip3 install update pip && pip3 install ipaddress
 
 WORKDIR /app
 COPY --from=build-env /go/src/github.com/sap/infrabox/src/services/gcp/tmp/_output/bin/gcp /app/gcp
+COPY --from=build-env /go/src/github.com/sap/infrabox/src/services/gcp/scripts/subnet_fetcher.py /app/subnet_fetcher.py
 
 RUN addgroup -S infrabox && adduser -S -G infrabox infrabox
 USER infrabox
diff --git a/src/services/gcp/infrabox-service-gcp/templates/deployment.yaml b/src/services/gcp/infrabox-service-gcp/templates/deployment.yaml
@@ -36,7 +36,7 @@ spec:
                 memory: "1200Mi"
             limits:
                 cpu: "1000m"
-                memory: "2400Mi"
+                memory: "4Gi"
           volumeMounts:
           - name: service-account
             mountPath: "/var/run/infrabox.net/gcp"
diff --git a/src/services/gcp/pkg/stub/handler.go b/src/services/gcp/pkg/stub/handler.go
@@ -1,57 +1,57 @@
 package stub
 
 import (
-    "bytes"
-    "crypto/tls"
-    "crypto/x509"
-    b64 "encoding/base64"
-    "encoding/json"
-    "fmt"
-    "io/ioutil"
-    "mime/multipart"
-    "net/http"
-    "os"
-    "os/exec"
-    "path"
-    "strconv"
-    "strings"
-    "time"
-
-    uuid "github.com/satori/go.uuid"
-
-    "github.com/sap/infrabox/src/services/gcp/pkg/apis/gcp/v1alpha1"
-    "github.com/sap/infrabox/src/services/gcp/pkg/stub/cleaner"
-
-    goerrors "errors"
-
-    "k8s.io/client-go/discovery"
-    "k8s.io/client-go/discovery/cached"
-    "k8s.io/client-go/dynamic"
-    "k8s.io/client-go/kubernetes"
-    _ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
-    "k8s.io/client-go/rest"
-    "k8s.io/client-go/tools/clientcmd"
-    clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
-
-    "github.com/operator-framework/operator-sdk/pkg/sdk/action"
-    "github.com/operator-framework/operator-sdk/pkg/sdk/handler"
-    "github.com/operator-framework/operator-sdk/pkg/sdk/types"
-    "github.com/operator-framework/operator-sdk/pkg/util/k8sutil"
-
-    "github.com/sirupsen/logrus"
-
-    appsv1 "k8s.io/api/apps/v1"
-    v1 "k8s.io/api/core/v1"
-    rbacv1 "k8s.io/api/rbac/v1"
-    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-    "k8s.io/apimachinery/pkg/api/errors"
-    "k8s.io/apimachinery/pkg/api/meta"
-    "k8s.io/apimachinery/pkg/api/resource"
-    "k8s.io/apimachinery/pkg/runtime/schema"
-    "k8s.io/apimachinery/pkg/util/intstr"
-
-    "github.com/mholt/archiver"
+	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	b64 "encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	uuid "github.com/satori/go.uuid"
+
+	"github.com/sap/infrabox/src/services/gcp/pkg/apis/gcp/v1alpha1"
+	"github.com/sap/infrabox/src/services/gcp/pkg/stub/cleaner"
+
+	goerrors "errors"
+
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/discovery/cached"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+
+	"github.com/operator-framework/operator-sdk/pkg/sdk/action"
+	"github.com/operator-framework/operator-sdk/pkg/sdk/handler"
+	"github.com/operator-framework/operator-sdk/pkg/sdk/types"
+	"github.com/operator-framework/operator-sdk/pkg/util/k8sutil"
+
+	"github.com/sirupsen/logrus"
+
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/intstr"
+
+	"github.com/mholt/archiver"
 )
 
 const adminSAName = "admin"
@@ -66,11 +66,12 @@ type MasterAuth struct {
 }
 
 type RemoteCluster struct {
-    Name       string
-    Status     string
-    Zone       string
-    Endpoint   string
-    MasterAuth MasterAuth
+	Name                 string
+	Status               string
+	Zone                 string
+	Endpoint             string
+	PrivateClusterConfig map[string]interface{}
+	MasterAuth           MasterAuth
 }
 
 func NewHandler() handler.Handler {
@@ -95,15 +96,13 @@ func setClusterName(cr *v1alpha1.GKECluster, log *logrus.Entry) error {
 func createCluster(cr *v1alpha1.GKECluster, log *logrus.Entry) (*v1alpha1.GKEClusterStatus, error) {
     limit := os.Getenv("MAX_NUM_CLUSTERS")
     status := cr.Status
-
-    if limit != "" {
         gkeclusters, err := getRemoteClusters(log)
         if err != nil && !errors.IsNotFound(err) {
             err = fmt.Errorf("could not get GKE Clusters: %v", err)
             log.Error(err)
             return nil, err
         }
-
+	if limit != "" {
         l, err := strconv.Atoi(limit)
 
         if err != nil {
@@ -118,6 +117,24 @@ func createCluster(cr *v1alpha1.GKECluster, log *logrus.Entry) (*v1alpha1.GKEClu
             return &status, nil
         }
     }
+	masterIPv4CIRDs := getExistingMasterIPv4CIRDs(gkeclusters)
+	finalCIDR := ""
+    log.Debugf("existng ipv4 cidr: %s", masterIPv4CIRDs)
+	for {
+		cidr, err := getRandomIPv4CIRD()
+        log.Infof("generate master ipv4 cidr: %s", cidr)
+		if err != nil {
+			err = fmt.Errorf("err while getting CIDR for Cluster %s: %s", cr.Status.ClusterName, err)
+			log.Error(err)
+			return nil, err
+		} else {
+            log.Debugf("check if %s in %s", cidr, masterIPv4CIRDs)
+			if !contains(masterIPv4CIRDs, cidr) {
+				finalCIDR = cidr
+				break
+			}
+		}
+	}
 
     log.Infof("Create GKE cluster %s", cr.Status.ClusterName)
     args := []string{"container", "clusters",
@@ -202,7 +219,11 @@ func createCluster(cr *v1alpha1.GKECluster, log *logrus.Entry) (*v1alpha1.GKEClu
         args = append(args, "--services-ipv4-cidr", cr.Spec.ServiceCidr)
     }
 
-
+	args = append(args, "--no-enable-legacy-authorization")
+    args = append(args, "--enable-private-nodes")
+	args = append(args, "--master-ipv4-cidr", finalCIDR)
+	args = append(args, "--enable-master-authorized-networks")
+    args = append(args, "--master-authorized-networks", "0.0.0.0/0")
     cmd := exec.Command("gcloud" , args...)
     out, err := cmd.CombinedOutput()
 
@@ -847,6 +868,36 @@ func getOutdatedClusters(maxAge string, log *logrus.Entry) ([]RemoteCluster, err
     return gkeclusters, nil
 }
 
+func getExistingMasterIPv4CIRDs(gkeclusters []RemoteCluster) []string {
+	var allMasterIPv4CIRSs = make([]string, 0)
+	for _, gkecluster := range gkeclusters {
+        masterIpv4CidrBlock, ok := gkecluster.PrivateClusterConfig["masterIpv4CidrBlock"].(string)
+		if ok && masterIpv4CidrBlock != "" {
+			allMasterIPv4CIRSs = append(allMasterIPv4CIRSs, masterIpv4CidrBlock)
+		}
+	}
+	return allMasterIPv4CIRSs
+}
+
+func getRandomIPv4CIRD() (string, error) {
+    cmd := exec.Command("python3" , "/app/subnet_fetcher.py")
+    out, err := cmd.CombinedOutput()
+    if err != nil {
+        return "", err
+    }
+    result := strings.ReplaceAll(string(out), "\n", "")
+    return result, nil
+}
+
+func contains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}
+
 func generateKubeconfig(c *RemoteCluster) []byte {
     caCrt, _ := b64.StdEncoding.DecodeString(c.MasterAuth.ClusterCaCertificate)
     clusters := make(map[string]*clientcmdapi.Cluster)
@@ -1488,4 +1539,4 @@ func newCollectorDaemonSet() *appsv1.DaemonSet {
             },
         },
     }
-}
+}
diff --git a/src/services/gcp/scripts/subnet_fetcher.py b/src/services/gcp/scripts/subnet_fetcher.py
@@ -0,0 +1,5 @@
+import ipaddress
+import random
+network="192.168.0.0/16"
+blocks = list(ipaddress.IPv4Network(network).subnets(new_prefix=28))
+print(str(random.choice(blocks)))
