Merge pull request #97 from nokia/e2e-test-2

Prepare adding end-to-end tests to the controller go module

.dockerignore

@@ -2,3 +2,4 @@
 .cache/
 default.etcd/
 apiserver.local.config/
+__debug_bin*

.github/workflows/porch-e2e.yaml

@@ -56,8 +56,8 @@ jobs:
           kubectl_version: v1.30.0
       - name: Build Images and Deploy porch kpt pkg
         run: IMAGE_REPO=porch-kind IMAGE_TAG=${GITHUB_SHA:0:8} KIND_CONTEXT_NAME=kind make run-in-kind
-      - name: Sleep for 15 seconds
-        run: sleep 15s
+      - name: Sleep for 30 seconds
+        run: sleep 30s
         shell: bash
       - name: e2e test
         run: E2E=1 go test -v -timeout 20m ${GITHUB_WORKSPACE}/test/e2e

.github/workflows/porchctl-cli-e2e.yaml

@@ -49,9 +49,7 @@ jobs:
       - name: Build and install porchctl
         run: |
           make porchctl
-          mv ${GITHUB_WORKSPACE}/.build/porchctl /usr/local/bin/porchctl
-          sudo chmod 755 /usr/local/bin/porchctl
-          porchctl version
+          .build/porchctl version
       - name: Create k8s Kind Cluster
         uses: helm/kind-action@v1
         with:

Makefile

@@ -293,35 +293,37 @@ load-images-to-kind: ## Build porch images and load them into a kind cluster
   # only build test-git-server & function-runner if they are not already loaded into kind
 	@if ! docker exec "${KIND_CONTEXT_NAME}-control-plane" crictl images | grep -q "$(IMAGE_REPO)/$(TEST_GIT_SERVER_IMAGE)  *${IMAGE_TAG} " ; then \
 		echo "Building $(IMAGE_REPO)/$(TEST_GIT_SERVER_IMAGE):${IMAGE_TAG}" ; \
-		IMAGE_NAME="$(TEST_GIT_SERVER_IMAGE)" make -C test/ build-image ; \
+		IMAGE_NAME="$(TEST_GIT_SERVER_IMAGE)" make -C test/ build-image && \
 		kind load docker-image $(IMAGE_REPO)/$(TEST_GIT_SERVER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} ; \
 	else \
 		echo "Skipping building $(IMAGE_REPO)/$(TEST_GIT_SERVER_IMAGE):${IMAGE_TAG} as it is already loaded into kind" ; \
 	fi
 	@if ! docker exec "${KIND_CONTEXT_NAME}-control-plane" crictl images | grep -q "$(IMAGE_REPO)/$(PORCH_FUNCTION_RUNNER_IMAGE)  *${IMAGE_TAG} " ; then \
 		echo "Building $(IMAGE_REPO)/$(PORCH_FUNCTION_RUNNER_IMAGE):${IMAGE_TAG}" ; \
-		IMAGE_NAME="$(PORCH_FUNCTION_RUNNER_IMAGE)" WRAPPER_SERVER_IMAGE_NAME="$(PORCH_WRAPPER_SERVER_IMAGE)" make -C func/ build-image ; \
-		kind load docker-image $(IMAGE_REPO)/$(PORCH_FUNCTION_RUNNER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} ; \
+		IMAGE_NAME="$(PORCH_FUNCTION_RUNNER_IMAGE)" WRAPPER_SERVER_IMAGE_NAME="$(PORCH_WRAPPER_SERVER_IMAGE)" make -C func/ build-image && \
+		kind load docker-image $(IMAGE_REPO)/$(PORCH_FUNCTION_RUNNER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} && \
 		kind load docker-image $(IMAGE_REPO)/$(PORCH_WRAPPER_SERVER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} ; \
 	else \
 		echo "Skipping building $(IMAGE_REPO)/$(PORCH_FUNCTION_RUNNER_IMAGE):${IMAGE_TAG} as it is already loaded into kind" ; \
 	fi
     # NOTE: SKIP_PORCHSERVER_BUILD must be evaluated at runtime, hence the shell conditional (if) here
 	@if [ "$(SKIP_PORCHSERVER_BUILD)" = "false"	]; then \
 		echo "Building $(IMAGE_REPO)/$(PORCH_SERVER_IMAGE):${IMAGE_TAG}" ; \
-		docker buildx build --load --tag $(IMAGE_REPO)/$(PORCH_SERVER_IMAGE):$(IMAGE_TAG) -f ./build/Dockerfile "$(PORCHDIR)" ; \
-		kind load docker-image $(IMAGE_REPO)/$(PORCH_SERVER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} ; \
+		docker buildx build --load --tag $(IMAGE_REPO)/$(PORCH_SERVER_IMAGE):$(IMAGE_TAG) -f ./build/Dockerfile "$(PORCHDIR)" && \
+		kind load docker-image $(IMAGE_REPO)/$(PORCH_SERVER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} && \
+		kubectl delete deployment -n porch-system --ignore-not-found=true porch-server ; \
 	fi
 	@if [ "$(SKIP_CONTROLLER_BUILD)" = "false"	]; then \
 		echo "Building $(IMAGE_REPO)/$(PORCH_CONTROLLERS_IMAGE):${IMAGE_TAG}" ; \
-		IMAGE_NAME="$(PORCH_CONTROLLERS_IMAGE)" make -C controllers/ build-image ; \
-		kind load docker-image $(IMAGE_REPO)/$(PORCH_CONTROLLERS_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} ; \
+		IMAGE_NAME="$(PORCH_CONTROLLERS_IMAGE)" make -C controllers/ build-image && \
+		kind load docker-image $(IMAGE_REPO)/$(PORCH_CONTROLLERS_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} && \
+		kubectl delete deployment -n porch-system --ignore-not-found=true porch-controllers ; \
 	fi
 
   else
-	kind load docker-image $(IMAGE_REPO)/$(TEST_GIT_SERVER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} ; \
-	kind load docker-image $(IMAGE_REPO)/$(PORCH_FUNCTION_RUNNER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} ; \
-	kind load docker-image $(IMAGE_REPO)/$(PORCH_WRAPPER_SERVER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} ; \
+	kind load docker-image $(IMAGE_REPO)/$(TEST_GIT_SERVER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} 
+	kind load docker-image $(IMAGE_REPO)/$(PORCH_FUNCTION_RUNNER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME}
+	kind load docker-image $(IMAGE_REPO)/$(PORCH_WRAPPER_SERVER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME} 
 	kind load docker-image $(IMAGE_REPO)/$(PORCH_SERVER_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME}
 	kind load docker-image $(IMAGE_REPO)/$(PORCH_CONTROLLERS_IMAGE):${IMAGE_TAG} -n ${KIND_CONTEXT_NAME}
   endif
@@ -360,5 +362,5 @@ test-e2e: ## Run end-to-end tests
 	E2E=1 go test -v -failfast ./test/e2e/cli
 
 .PHONY: test-e2e-clean
-test-e2e-clean: ## Run end-to-end tests aginst a newly deployed porch in a newly created kind cluster
+test-e2e-clean: porchctl ## Run end-to-end tests aginst a newly deployed porch in a newly created kind cluster
 	./scripts/clean-e2e-test.sh

go.mod

@@ -4,6 +4,8 @@ go 1.22.0
 
 retract v1.3.0
 
+replace github.com/GoogleContainerTools/kpt-functions-sdk/go/fn => github.com/GoogleContainerTools/kpt-functions-sdk/go/fn v0.0.0-20220506190241-f85503febd54
+
 require (
 	cloud.google.com/go/iam v1.1.12
 	github.com/GoogleContainerTools/kpt v1.0.0-beta.48

test/e2e/cli/cli_e2e_test.go

@@ -0,0 +1,31 @@
+// Copyright 2024 The kpt and Nephio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestPorch(t *testing.T) {
+	e2e := os.Getenv("E2E")
+	if e2e == "" {
+		t.Skip("set E2E to run this test")
+	}
+
+	suite := NewCliTestSuite(t, filepath.Join(".", "testdata"))
+	suite.RunTests(t)
+}

test/e2e/cli/cluster.go

@@ -20,10 +20,8 @@ import (
 	"strings"
 	"testing"
 	"time"
-)
 
-const (
-	TestGitServerImage = "test-git-server"
+	e2etest "github.com/nephio-project/porch/test/e2e"
 )
 
 func IsPorchServerRunningInCluster(t *testing.T) bool {
@@ -67,17 +65,7 @@ func GetGitServerImageName(t *testing.T) string {
 	if image == "" {
 		t.Fatalf("Cannot determine Porch server image: output was %q", out)
 	}
-	return InferGitServerImage(image)
-}
-
-func InferGitServerImage(porchImage string) string {
-	slash := strings.LastIndex(porchImage, "/")
-	repo := porchImage[:slash+1]
-	image := porchImage[slash+1:]
-	colon := strings.LastIndex(image, ":")
-	tag := image[colon+1:]
-
-	return repo + TestGitServerImage + ":" + tag
+	return e2etest.InferGitServerImage(image)
 }
 
 func KubectlApply(t *testing.T, config string) {
@@ -141,7 +129,7 @@ func KubectlWaitForLoadBalancerIp(t *testing.T, namespace, name string) string {
 
 		err := cmd.Run()
 		ip := stdout.String()
-		if err == nil && len(ip) > 0 { // Loadbalancer assigned an external IP
+		if err == nil && len(ip) > 0 { // LoadBalancer assigned an external IP
 			t.Logf("LoadBalancer external IP: %s", ip)
 			return ip
 		}
@@ -158,47 +146,6 @@ func KubectlWaitForLoadBalancerIp(t *testing.T, namespace, name string) string {
 	}
 }
 
-// Kubernetes DNS needs time to propagate the updated address
-// Wait until we can register the repository and list its contents.
-func KubectlWaitForGitDNS(t *testing.T, gitServerURL string) {
-	const name = "test-git-dns-resolve"
-
-	KubectlCreateNamespace(t, name)
-	defer KubectlDeleteNamespace(t, name)
-
-	// We expect repos to automatically be created (albeit empty)
-	repoURL := gitServerURL + "/" + name
-
-	cmd := exec.Command("porchctl", "repo", "register", "--namespace", name, "--name", name, repoURL)
-	t.Logf("running command %v", strings.Join(cmd.Args, " "))
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("Failed to register probe repository: %v\n%s", err, string(out))
-	}
-
-	// Based on experience, DNS seems to get updated inside the cluster within
-	// few seconds. We will wait about a minute.
-	// If this turns out to be an issue, we will sidestep DNS and use the Endpoints
-	// IP address directly.
-	giveUp := time.Now().Add(1 * time.Minute)
-	for {
-		cmd := exec.Command("porchctl", "rpkg", "get", "--namespace", name)
-		t.Logf("running command %v", strings.Join(cmd.Args, " "))
-		out, err := cmd.CombinedOutput()
-		t.Logf("output: %v", string(out))
-
-		if err == nil {
-			break
-		}
-
-		if time.Now().After(giveUp) {
-			t.Fatalf("Git service DNS resolution failed: %v", err)
-		}
-
-		time.Sleep(5 * time.Second)
-	}
-}
-
 func KubectlCreateNamespace(t *testing.T, name string) {
 	cmd := exec.Command("kubectl", "create", "namespace", name)
 	t.Logf("running command %v", strings.Join(cmd.Args, " "))
@@ -232,12 +179,12 @@ func RemovePackagerevFinalizers(t *testing.T, namespace string) {
 		t.Fatalf("Error when getting packagerevs from namespace: %v: %s", err, stderr.String())
 	}
 
-	packagerevs := realySplit(stdout.String(), " ")
+	packagerevs := reallySplit(stdout.String(), " ")
 	if len(packagerevs) == 0 {
 		t.Log("kubectl get packagerevs didn't return any objects - continue")
 		return
 	}
-	t.Logf("Removing Finalizers from PackagRevs: %v", packagerevs)
+	t.Logf("Removing Finalizers from PackageRevs: %v", packagerevs)
 
 	for _, pkgrev := range packagerevs {
 		cmd := exec.Command("kubectl", "patch", "packagerev", pkgrev, "--type", "json", "--patch=[{\"op\": \"remove\", \"path\": \"/metadata/finalizers\"}]", "--namespace", namespace)
@@ -248,19 +195,9 @@ func RemovePackagerevFinalizers(t *testing.T, namespace string) {
 	}
 }
 
-func realySplit(s, sep string) []string {
+func reallySplit(s, sep string) []string {
 	if len(s) == 0 {
 		return []string{}
 	}
 	return strings.Split(s, sep)
 }
-
-func RegisterRepository(t *testing.T, repoURL, namespace, name string) {
-	cmd := exec.Command("porchctl", "repo", "register", "--namespace", namespace, "--name", name, repoURL)
-	t.Logf("running command %v", strings.Join(cmd.Args, " "))
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("Failed to register repository %q: %v\n%s", repoURL, err, string(out))
-	}
-	t.Logf("output: %v", string(out))
-}

test/e2e/cli/config.go

@@ -36,7 +36,7 @@ type Command struct {
 	ExitCode int `yaml:"exitCode,omitempty"`
 	// Yaml indicates that stdout is yaml and the test will reformat it for stable ordering
 	Yaml bool `yaml:"yaml,omitempty"`
-	// IgnoreWhitespace inidicates that whitespace differences should be ignored in the output
+	// IgnoreWhitespace indicates that whitespace differences should be ignored in the output
 	IgnoreWhitespace bool `yaml:"ignoreWhitespace,omitempty"`
 }