Skip to content

Use the modern API to import clusters #184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
moio wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Use the modern API to import clusters #184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 52 additions & 15 deletions cmd/dartboard/subcommands/deploy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package subcommands

import (
"crypto/tls"
"encoding/json"
"fmt"
"io"
"log"
Expand Down Expand Up @@ -618,19 +619,19 @@ func importDownstreamClusterDo(r *dart.Dart, rancherImageTag string, tf *tofu.To
return
}

if err := kubectl.WaitForReadyCondition(clusters["upstream"].Kubeconfig,
"clusters.management.cattle.io", clusterID, "", "ready", 10); err != nil {
if err := kubectl.WaitFor(clusters["upstream"].Kubeconfig,
"clusters.management.cattle.io", clusterID, "", "condition=ready=true", 10); err != nil {
errCh <- fmt.Errorf("%s import failed: %w", clusterName, err)
return
}

if err := kubectl.WaitForReadyCondition(clusters["upstream"].Kubeconfig,
"cluster.fleet.cattle.io", clusterName, "fleet-default", "ready", 10); err != nil {
if err := kubectl.WaitFor(clusters["upstream"].Kubeconfig,
"cluster.fleet.cattle.io", clusterID, "fleet-default", "condition=ready=true", 10); err != nil {
errCh <- fmt.Errorf("%s import failed: %w", clusterName, err)
return
}

err = kubectl.WaitForReadyCondition(downstream.Kubeconfig, "deployment", "rancher-webhook", "cattle-system", "available", 15)
err = kubectl.WaitFor(downstream.Kubeconfig, "deployment", "rancher-webhook", "cattle-system", "condition=available=true", 15)
if err != nil {
errCh <- fmt.Errorf("%s waiting for rancher-webhook failed: %w", clusterName, err)
return
Expand Down Expand Up @@ -663,19 +664,50 @@ func importDownstreamClustersRancherSetup(r *dart.Dart, clusters map[string]tofu
}
}

importedClusterNames := strings.Join(downstreamClusters, ",")

envVars := map[string]string{
"BASE_URL": upstreamAdd.Public.HTTPSURL,
"BOOTSTRAP_PASSWORD": "admin",
"PASSWORD": r.ChartVariables.AdminPassword,
"IMPORTED_CLUSTER_NAMES": importedClusterNames,
"BASE_URL": upstreamAdd.Public.HTTPSURL,
"BOOTSTRAP_PASSWORD": "admin",
"PASSWORD": r.ChartVariables.AdminPassword,
}

if err = kubectl.K6run(tester.Kubeconfig, "rancher/rancher_setup.js", envVars, nil, true, upstreamAdd.Local.HTTPSURL, false); err != nil {
return err
}

yamlFile, err := os.CreateTemp("", "dartboard-imported-clusters-*.yaml")
if err != nil {
return fmt.Errorf("creating temp file for imported clusters: %w", err)
}

encoder := json.NewEncoder(yamlFile)

for _, clusterName := range downstreamClusters {
managementCluster := map[string]any{
"apiVersion": "management.cattle.io/v3",
"kind": "Cluster",
"metadata": map[string]any{
"generateName": "c-",
},
"spec": map[string]any{
"displayName": clusterName,
},
}

err := encoder.Encode(managementCluster)
if err != nil {
return fmt.Errorf("encoding management cluster for %s: %w", clusterName, err)
}
}

err = yamlFile.Sync()
if err != nil {
return fmt.Errorf("syncing temp file for imported clusters: %w", err)
}

if err := kubectl.Create(upstream.Kubeconfig, yamlFile.Name()); err != nil {
return fmt.Errorf("applying imported clusters YAML: %e", err)
}

return nil
}

Expand All @@ -689,13 +721,18 @@ func importClustersDownstreamGetYAML(clusters map[string]tofu.Cluster, name stri
return
}

if status, err = kubectl.GetStatus(upstream.Kubeconfig, "clusters.provisioning.cattle.io", name, "fleet-default"); err != nil {
jsonPath := fmt.Sprintf(".items[?(@.spec.displayName==\"%s\")].metadata.name", name)

out := new(string)
if err = kubectl.Get(upstream.Kubeconfig, "clusters.management.cattle.io", "", "", jsonPath, out); err != nil {
return
}

clusterID, ok := status["clusterName"].(string)
if !ok {
err = fmt.Errorf("error accessing fleet-default/%s clusters: no valid 'clusterName' in 'Status'", name)
clusterID = strings.Split(*out, " ")[0]

err = kubectl.WaitFor(upstream.Kubeconfig, "clusterregistrationtokens.management.cattle.io", "default-token", clusterID, "create", 1)
if err != nil {
err = fmt.Errorf("%s waiting for clusterregistrationtoken failed: %w", clusterID, err)
return
}

Expand Down
21 changes: 16 additions & 5 deletions internal/kubectl/kubectl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,23 +248,27 @@ func Apply(kubePath, filePath string) error {
return Exec(kubePath, log.Writer(), "apply", "-f", filePath)
}

func Create(kubePath string, filePath string) interface{} {
return Exec(kubePath, log.Writer(), "create", "-f", filePath)
}

func WaitRancher(kubePath string) error {
err := WaitForReadyCondition(kubePath, "deployment", "rancher", "cattle-system", "available", 20)
err := WaitFor(kubePath, "deployment", "rancher", "cattle-system", "condition=available=true", 20)
if err != nil {
return err
}

err = WaitForReadyCondition(kubePath, "deployment", "rancher-webhook", "cattle-system", "available", 3)
err = WaitFor(kubePath, "deployment", "rancher-webhook", "cattle-system", "condition=available=true", 3)
if err != nil {
return err
}

err = WaitForReadyCondition(kubePath, "deployment", "fleet-controller", "cattle-fleet-system", "available", 5)
err = WaitFor(kubePath, "deployment", "fleet-controller", "cattle-fleet-system", "condition=available=true", 5)

return err
}

func WaitForReadyCondition(kubePath, resource, name, namespace string, condition string, minutes int) error {
func WaitFor(kubePath, resource, name, namespace string, condition string, minutes int) error {
var err error

args := []string{"wait", resource, name}
Expand All @@ -273,7 +277,7 @@ func WaitForReadyCondition(kubePath, resource, name, namespace string, condition
args = append(args, "--namespace", namespace)
}

args = append(args, "--for", fmt.Sprintf("condition=%s=true", condition), fmt.Sprintf("--timeout=%dm", minutes))
args = append(args, "--for", condition, fmt.Sprintf("--timeout=%dm", minutes))

maxRetries := minutes * 30
for i := 1; i < maxRetries; i++ {
Expand Down Expand Up @@ -335,6 +339,13 @@ func Get(kubePath string, kind string, name string, namespace string, jsonpath s
return fmt.Errorf("failed to kubectl get %v: %w", name, err)
}

// if out is a string pointer, just set the value directly without json unmarshaling
if strPtr, ok := out.(*string); ok {
*strPtr = output.String()
return nil
}

// otherwise, unmarshal the output as JSON
if err := json.Unmarshal(output.Bytes(), out); err != nil {
return fmt.Errorf("cannot unmarshal kubectl data for %v: %w\n%s", name, err, output.String())
}
Expand Down
7 changes: 1 addition & 6 deletions k6/rancher/rancher_setup.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {getCookies, createImportedCluster, firstLogin, logout} from "./rancher_utils.js";
import {getCookies, firstLogin, logout} from "./rancher_utils.js";

export const options = {
insecureSkipTLSVerify: true,
Expand All @@ -8,15 +8,10 @@ export default function main() {
const baseUrl = __ENV.BASE_URL
const bootstrapPassword = __ENV.BOOTSTRAP_PASSWORD
const password = __ENV.PASSWORD
const importedClusterNames = __ENV.IMPORTED_CLUSTER_NAMES === "" ? [] : __ENV.IMPORTED_CLUSTER_NAMES.split(",")

const cookies = getCookies(baseUrl)

firstLogin(baseUrl, cookies, bootstrapPassword, password)

for (const i in importedClusterNames) {
createImportedCluster(baseUrl, cookies, importedClusterNames[i])
}

logout(baseUrl, cookies)
}
180 changes: 0 additions & 180 deletions k6/rancher/rancher_utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,186 +197,6 @@ export function addMinutes(date, minutes) {
return new Date(date.getTime() + (minutes*60000));
}

export function createImportedCluster(baseUrl, cookies, name) {
let response

const userId = getCurrentUserId(baseUrl, cookies)
const userPreferences = getUserPreferences(baseUrl, cookies);

userPreferences["last-visited"] = "{\"name\":\"c-cluster-product\",\"params\":{\"cluster\":\"_\",\"product\":\"manager\"}}"
userPreferences["locale"] = "en-us"
userPreferences["seen-whatsnew"] = "\"v2.7.1\""
userPreferences["seen-cluster"] = "_"
setUserPreferences(baseUrl, cookies, userId, userPreferences)

response = http.get(`${baseUrl}/v1/catalog.cattle.io.clusterrepos`, {
headers: {
accept: 'application/json',
},
cookies: cookies,
})
check(response, {
'querying clusterrepos works': (r) => r.status === 200,
})

response = http.get(`${baseUrl}/v1/management.cattle.io.kontainerdrivers`, {
headers: {
accept: 'application/json',
cookies: cookies,
},
})
check(response, {
'querying kontainerdrivers works': (r) => r.status === 200,
})

response = http.get(
`${baseUrl}/v1/catalog.cattle.io.clusterrepos/rancher-charts?link=index`,
{
headers: {
accept: 'application/json',
},
cookies: cookies,
}
)
check(response, {
'querying rancher-charts works': (r) => r.status === 200,
})

response = http.get(
`${baseUrl}/v1/catalog.cattle.io.clusterrepos/rancher-partner-charts?link=index`,
{
headers: {
accept: 'application/json',
},
cookies: cookies,
}
)
check(response, {
'querying rancher-partners-charts works': (r) => r.status === 200,
})

response = http.get(
`${baseUrl}/v1/catalog.cattle.io.clusterrepos/rancher-rke2-charts?link=index`,
{
headers: {
accept: 'application/json',
},
cookies: cookies,
}
)
check(response, {
'querying rancher-rke2-charts works': (r) => r.status === 200,
})

response = http.get(`${baseUrl}/v3/clusterroletemplatebindings`, {
headers: {
accept: 'application/json',
},
cookies: cookies,
})
check(response, {
'querying clusterroletemplatebindings works': (r) => r.status === 200,
})

response = http.get(`${baseUrl}/v1/management.cattle.io.roletemplates`, {
headers: {
accept: 'application/json',
},
cookies: cookies,
})
check(response, {
'querying roletemplates works': (r) => r.status === 200,
})

response = http.post(
`${baseUrl}/v1/provisioning.cattle.io.clusters`,
JSON.stringify({ "type": "provisioning.cattle.io.cluster", "metadata": { "namespace": "fleet-default", "name": name }, "spec": {} }),
{
headers: {
accept: 'application/json',
'content-type': 'application/json',
},
cookies: cookies,
}
)

check(response, {
'creating an imported cluster works': (r) => r.status === 201 || r.status === 409,
})
if (response.status === 409) {
console.warn(`cluster ${name} already exists`)
}

response = http.get(
`${baseUrl}/v1/provisioning.cattle.io.clusters/fleet-default/${name}`,
{
headers: {
accept: 'application/json',
},
cookies: cookies,
}
)
check(response, {
'querying clusters works': (r) => r.status === 200,
})
if (!response.status === 200) {
fail(`cluster ${name} not found`)
}

response = http.get(`${baseUrl}/v1/cluster.x-k8s.io.machinedeployments`, {
headers: {
accept: 'application/json',
},
cookies: cookies,
})
check(response, {
'querying machinedeployments works': (r) => r.status === 200,
})

response = http.get(`${baseUrl}/v1/rke.cattle.io.etcdsnapshots`, {
headers: {
accept: 'application/json',
},
cookies: cookies,
})
check(response, {
'querying etcdsnapshots works': (r) => r.status === 200,
})

response = http.get(`${baseUrl}/v1/management.cattle.io.nodetemplates`, {
headers: {
accept: 'application/json',
},
cookies: cookies,
})
check(response, {
'querying nodetemplates works': (r) => r.status === 200,
})

response = http.get(`${baseUrl}/v1/management.cattle.io.clustertemplates`, {
headers: {
accept: 'application/json',
},
cookies: cookies,
})
check(response, {
'querying clustertemplates works': (r) => r.status === 200,
})

response = http.get(
`${baseUrl}/v1/management.cattle.io.clustertemplaterevisions`,
{
headers: {
accept: 'application/json',
},
cookies: cookies,
}
)
check(response, {
'querying clustertemplaterevisions works': (r) => r.status === 200,
})
}

export function getProvisioningClusters(baseUrl, cookies) {
const response = http.get(`${baseUrl}/v1/provisioning.cattle.io.clusters?pagesize=100000&exclude=metadata.managedFields`, {
headers: {
Expand Down
Loading