Merge pull request #1217 from gtardif/kube_e2e

First kube e2e. Adapted context create kubernetes command to allow non interactive mode.

Author: gtardif

.github/workflows/kube-tests.yml

@@ -0,0 +1,63 @@
+name: Kube integration tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  check-optional-tests:
+    name: Check if needs to run Kube tests
+    runs-on: ubuntu-latest
+    outputs:
+      trigger-kube: ${{steps.runkubetest.outputs.triggered}}
+    steps:
+      - uses: khan/pull-request-comment-trigger@master
+        name: Check if test Kube
+        if: github.event_name == 'pull_request'
+        id: runkubetest
+        with:
+          trigger: '/test-kube'
+
+
+  kube-tests:
+    name: Kube e2e tests
+    runs-on: ubuntu-latest
+    env:
+      GO111MODULE: "on"
+    needs: check-optional-tests
+    if: github.ref == 'refs/heads/main' || needs.check-optional-tests.outputs.trigger-kube == 'true'
+    steps:
+      - name: Set up Go 1.15
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.15
+        id: go
+
+      - name: Setup docker CLI
+        run: |
+          curl https://download.docker.com/linux/static/stable/x86_64/docker-20.10.2.tgz | tar xz
+          sudo cp ./docker/docker /usr/bin/ && rm -rf docker && docker version
+
+      - name: Setup Kube tools
+        run: |
+          sudo apt-get install jq && jq --version
+          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.10.0/kind-linux-amd64 && chmod +x ./kind && sudo mv ./kind /usr/bin/ && kind version
+          curl -LO "https://dl.k8s.io/release/v1.20.2/bin/linux/amd64/kubectl" && sudo mv kubectl /usr/bin/ && kubectl version --client
+
+      - name: Checkout code into the Go module directory
+        uses: actions/checkout@v2
+
+      - uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: go-${{ hashFiles('**/go.sum') }}
+
+      - name: Build for Kube e2e tests
+        env:
+          BUILD_TAGS: kube
+        run: make -f builder.Makefile cli
+
+      - name: Kube e2e Test
+        run: make e2e-kube

Makefile

@@ -49,6 +49,9 @@ e2e-local: ## Run End to end local tests. Set E2E_TEST=TestName to run a single
 e2e-win-ci: ## Run end to end local tests on Windows CI, no Docker for Linux containers available ATM. Set E2E_TEST=TestName to run a single test
 	go test -count=1 -v $(TEST_FLAGS) ./local/e2e/cli-only
 
+e2e-kube: ## Run End to end Kube tests. Set E2E_TEST=TestName to run a single test
+	go test -timeout 10m -count=1 -v $(TEST_FLAGS) ./kube/e2e
+
 e2e-aci: ## Run End to end ACI tests. Set E2E_TEST=TestName to run a single test
 	go test -timeout 15m -count=1 -v $(TEST_FLAGS) ./aci/e2e
 

cli/cmd/context/create_kube.go

@@ -50,7 +50,8 @@ func createKubeCommand() *cobra.Command {
 	}
 
 	addDescriptionFlag(cmd, &opts.Description)
-	cmd.Flags().StringVar(&opts.KubeconfigPath, "kubeconfig", "", "The endpoint of the Kubernetes manager")
+	cmd.Flags().StringVar(&opts.KubeConfigPath, "kubeconfig", "", "The endpoint of the Kubernetes manager")
+	cmd.Flags().StringVar(&opts.KubeContextName, "kubecontext", "", "The name of the context to use in kubeconfig")
 	cmd.Flags().BoolVar(&opts.FromEnvironment, "from-env", false, "Get endpoint and creds from env vars")
 	return cmd
 }

kube/context.go

@@ -29,9 +29,9 @@ import (
 
 // ContextParams options for creating a Kubernetes context
 type ContextParams struct {
-	ContextName     string
+	KubeContextName string
 	Description     string
-	KubeconfigPath  string
+	KubeConfigPath  string
 	FromEnvironment bool
 }
 
@@ -45,7 +45,7 @@ func (cp ContextParams) CreateContextData() (interface{}, string, error) {
 	}
 	user := prompt.User{}
 	selectContext := func() error {
-		contexts, err := kubernetes.ListAvailableKubeConfigContexts(cp.KubeconfigPath)
+		contexts, err := kubernetes.ListAvailableKubeConfigContexts(cp.KubeConfigPath)
 		if err != nil {
 			return err
 		}
@@ -57,11 +57,18 @@ func (cp ContextParams) CreateContextData() (interface{}, string, error) {
 			}
 			return err
 		}
-		cp.ContextName = contexts[selected]
+		cp.KubeContextName = contexts[selected]
 		return nil
 	}
 
-	if cp.KubeconfigPath != "" {
+	if cp.KubeConfigPath != "" {
+		if cp.KubeContextName != "" {
+			return store.KubeContext{
+				ContextName:     cp.KubeContextName,
+				KubeconfigPath:  cp.KubeConfigPath,
+				FromEnvironment: cp.FromEnvironment,
+			}, cp.Description, nil
+		}
 		err := selectContext()
 		if err != nil {
 			return nil, "", err
@@ -95,8 +102,8 @@ func (cp ContextParams) CreateContextData() (interface{}, string, error) {
 		}
 	}
 	return store.KubeContext{
-		ContextName:     cp.ContextName,
-		KubeconfigPath:  cp.KubeconfigPath,
+		ContextName:     cp.KubeContextName,
+		KubeconfigPath:  cp.KubeConfigPath,
 		FromEnvironment: cp.FromEnvironment,
 	}, cp.Description, nil
 }

kube/e2e/compose_test.go

@@ -18,8 +18,8 @@ package e2e
 
 import (
 	"fmt"
-	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -48,31 +48,50 @@ func TestComposeUp(t *testing.T) {
 	c := NewParallelE2eCLI(t, binDir)
 
 	const projectName = "compose-kube-demo"
+	kubeconfig := filepath.Join(c.ConfigDir, "kubeconfig")
+	kindClusterName := "e2e"
+	kubeContextName := "kind-" + kindClusterName
+	dockerContextName := "kube-e2e-ctx"
+
+	t.Run("create kube cluster", func(t *testing.T) {
+		c.RunCmd("kind", "create", "cluster", "--name", kindClusterName, "--kubeconfig", kubeconfig, "--wait", "180s")
+	})
+	defer func() {
+		c.RunDockerCmd("context", "use", "default")
+		c.RunCmd("kind", "delete", "cluster", "--name", kindClusterName, "--kubeconfig", kubeconfig)
+	}()
 
 	t.Run("create kube context", func(t *testing.T) {
-		res := c.RunDockerCmd("context", "create", "kubernetes", "--kubeconfig", "/Users/gtardif/.kube/config", "--kubecontext", "docker-desktop", "kube-e2e")
-		res.Assert(t, icmd.Expected{Out: `Successfully created kube context "kube-e2e"`})
-		c.RunDockerCmd("context", "use", "kube-e2e")
+		res := c.RunDockerCmd("context", "create", "kubernetes", "--kubeconfig", kubeconfig, "--kubecontext", kubeContextName, dockerContextName)
+		res.Assert(t, icmd.Expected{Out: fmt.Sprintf("Successfully created kube context %q", dockerContextName)})
+		c.RunDockerCmd("context", "use", dockerContextName)
 	})
 
 	t.Run("up", func(t *testing.T) {
 		c.RunDockerCmd("compose", "-f", "./kube-simple-demo/demo_sentences.yaml", "--project-name", projectName, "up", "-d")
 	})
 
+	t.Run("compose ls", func(t *testing.T) {
+		res := c.RunDockerCmd("compose", "ls", "--format", "json")
+		res.Assert(t, icmd.Expected{Out: `[{"Name":"compose-kube-demo","Status":"deployed"}]`})
+	})
+
 	t.Run("check running project", func(t *testing.T) {
-		res := c.RunDockerCmd("compose", "-p", projectName, "ps")
-		res.Assert(t, icmd.Expected{Out: `web`})
+		// Docker Desktop kube cluster automatically exposes ports on the host, this is not the case with kind on Desktop,
+		//we need to connect to the clusterIP, from the kind container
+		res := c.RunCmd("sh", "-c", "kubectl --kubeconfig "+kubeconfig+" get service/web -o json | jq -r '.spec.clusterIP'")
+		clusterIP := strings.ReplaceAll(strings.TrimSpace(res.Stdout()), `"`, "")
 
-		endpoint := "http://localhost:95"
-		output := HTTPGetWithRetry(t, endpoint+"/words/noun", http.StatusOK, 2*time.Second, 20*time.Second)
-		assert.Assert(t, strings.Contains(output, `"word":`))
+		endpoint := fmt.Sprintf("http://%s:80/words/noun", clusterIP)
+		c.WaitForCmdResult(icmd.Command("docker", "--context", "default", "exec", "e2e-control-plane", "curl", endpoint), StdoutContains(`"word":`), 3*time.Minute, 3*time.Second)
 	})
+
 	t.Run("down", func(t *testing.T) {
 		_ = c.RunDockerCmd("compose", "--project-name", projectName, "down")
 	})
 
-	t.Run("check containers after down", func(t *testing.T) {
-		res := c.RunDockerCmd("ps", "--all")
+	t.Run("check stack after down", func(t *testing.T) {
+		res := c.RunDockerCmd("compose", "ls")
 		assert.Assert(t, !strings.Contains(res.Combined(), projectName), res.Combined())
 	})
 }

utils/e2e/framework.go

@@ -203,13 +203,44 @@ func (c *E2eCLI) RunDockerOrExitError(args ...string) *icmd.Result {
 	return icmd.RunCmd(c.NewDockerCmd(args...))
 }
 
+// RunCmd runs a command, expects no error and returns a result
+func (c *E2eCLI) RunCmd(args ...string) *icmd.Result {
+	fmt.Printf("	[%s] %s\n", c.test.Name(), strings.Join(args, " "))
+	assert.Assert(c.test, len(args) >= 1, "require at least one command in parameters")
+	res := icmd.RunCmd(c.NewCmd(args[0], args[1:]...))
+	res.Assert(c.test, icmd.Success)
+	return res
+}
+
 // RunDockerCmd runs a docker command, expects no error and returns a result
 func (c *E2eCLI) RunDockerCmd(args ...string) *icmd.Result {
 	res := c.RunDockerOrExitError(args...)
 	res.Assert(c.test, icmd.Success)
 	return res
 }
 
+// StdoutContains returns a predicate on command result expecting a string in stdout
+func StdoutContains(expected string) func(*icmd.Result) bool {
+	return func(res *icmd.Result) bool {
+		return strings.Contains(res.Stdout(), expected)
+	}
+}
+
+// WaitForCmdResult try to execute a cmd until resulting output matches given predicate
+func (c *E2eCLI) WaitForCmdResult(command icmd.Cmd, predicate func(*icmd.Result) bool, timeout time.Duration, delay time.Duration) {
+	assert.Assert(c.test, timeout.Nanoseconds() > delay.Nanoseconds(), "timeout must be greater than delay")
+	var res *icmd.Result
+	checkStopped := func(logt poll.LogT) poll.Result {
+		fmt.Printf("	[%s] %s\n", c.test.Name(), strings.Join(command.Command, " "))
+		res = icmd.RunCmd(command)
+		if !predicate(res) {
+			return poll.Continue("Cmd output did not match requirement: %q", res.Combined())
+		}
+		return poll.Success()
+	}
+	poll.WaitOn(c.test, checkStopped, poll.WithDelay(delay), poll.WithTimeout(timeout))
+}
+
 // PathEnvVar returns path (os sensitive) for running test
 func (c *E2eCLI) PathEnvVar() string {
 	path := c.BinDir + ":" + os.Getenv("PATH")