E2E CI Test for Operator Bundle

This change augments the e2e test suite to simulate the
operator's deployment using OLM. The setup consists of the
following components:

- A docker/distribution container registry, running in
  docker outside of any Kubernetes cluster.
- A KinD cluster that is configured to resolve image refs
  which use "localhost" as the image registry domain.
- Installing OLM on the KinD cluster.

Once set up, the operator and its associated OLM bundle are
built and pushed to the local container registry. Next,
an OLM catalog is built based on the catalog published in
operatorhub.io. The catalog is what allows OLM to find the
Tekton operator that Shipwright depends on, and is likewise
pushed to the local container registry.

Building the operator, bundle, and catalog with a fully on-cluster
registry is problematic for several reasons:

- Not all tools can push to the on-cluster registry in this fashion
- Mainfests need to be rewritten to reference the on-cluster DNS name
for the registry
- The catalog source needs to be pullable within the cluster.

The test runs as follows:

- Create a namespace to run the operator under test
- Create a CatalogSource using the catalog containing the
  operator under test.
- Create an OperatorGroup which allows AllNamespace operators
  to be installed in the given namespace.
- Create a Subscription to install the Shipwright operator
  and its associated Tekton operator.
- Verify that the shipwright operator deploys successfully.

See also:
- https://kind.sigs.k8s.io/docs/user/local-registry/
- https://olm.operatorframework.io/docs/tasks/creating-a-catalog/
- https://olm.operatorframework.io/docs/tasks/make-catalog-available-on-cluster/
- https://olm.operatorframework.io/docs/tasks/install-operator-with-olm/
- https://olm.operatorframework.io/docs/advanced-tasks/operator-scoping-with-operatorgroups/

.github/workflows/ci.yml

@@ -50,7 +50,7 @@ jobs:
             go-version: [1.15.x]
             os: [ubuntu-latest]
             kubernetes:
-              - v1.19.7
+              - v1.20.7
           max-parallel: 2
         runs-on: ${{ matrix.os }}
         steps:
@@ -60,5 +60,52 @@ jobs:
             go-version: ${{ matrix.go-version }}
         - name: Check out code
           uses: actions/checkout@v2
-        - name: Build Image
-          run: make ko-publish IMAGE_REPO=ko.local
+        - name: Install kubectl
+          uses: azure/setup-kubectl@v1
+          with:
+            version: ${{ matrix.kubernetes }}
+        - name: Deploy KinD Local Container Registry
+          run: make deploy-kind-registry
+        - name: Create KinD cluster
+          uses: helm/[email protected]
+          with:
+            version: v0.11.1
+            node_image: kindest/node:${{ matrix.kubernetes }}
+            cluster_name: kind
+            config: test/kind/config.yaml
+            wait: 120s
+        - name: Verify KinD cluster
+          run: |
+            echo "# Using KinD context..."
+            kubectl config use-context "kind-kind"
+            echo "# KinD nodes:"
+            kubectl get nodes
+
+            NODE_STATUS=$(kubectl get node kind-control-plane -o json | jq -r .'status.conditions[] | select(.type == "Ready") | .status')
+            if [ "${NODE_STATUS}" != "True" ]; then
+              echo "# Node is not ready:"
+              kubectl describe node kind-control-plane
+
+              echo "# Pods:"
+              kubectl get pod -A
+              echo "# Events:"
+              kubectl get events -A
+
+              exit 1
+            fi
+        - name: Install KinD post-actions
+          run: make deploy-kind-registry-post
+        - name: Install OLM
+          run: make install-olm
+        # Builds the operator and makes the image readable in the KinD cluster
+        - name: Build Operator Image
+          run: |
+            make ko-publish IMAGE_REPO=localhost:5000
+        - name: Build Operator Bundle
+          run: |
+            make bundle-push IMAGE_REPO=localhost:5000
+        - name: Build Catalog Source
+          run: |
+            make catalog-push IMAGE_REPO=localhost:5000
+        - name: Run Operator with Catalog
+          run: make catalog-run IMAGE_REPO=localhost:5000

Makefile

@@ -39,20 +39,24 @@ IMAGE_REPO ?= quay.io/shipwright
 TAG ?= $(VERSION)
 IMAGE_PUSH ?= true
 
-BUNDLE_IMG_NAME ?= operator-bundle
-OPERATOR_IMG_NAME ?= operator
+IMAGE_TAG_BASE ?= $(IMAGE_REPO)/operator
 
 # Image URL to use all building/pushing image targets
-IMG ?= $(IMAGE_REPO)/$(OPERATOR_IMG_NAME):$(TAG)
+IMG ?= $(IMAGE_TAG_BASE):$(TAG)
 
 # BUNDLE_IMG defines the image:tag used for the bundle. 
 # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=<some-registry>/<project-name-bundle>:<tag>)
-BUNDLE_IMG ?= $(IMAGE_REPO)/$(BUNDLE_IMG_NAME):$(TAG)
+BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(TAG)
+
+# SKIP_TLS indicates if TLS should be skipped for a given make command
+SKIP_TLS ?= false
 
 # operating-system type and architecture based on golang
 OS ?= $(shell go env GOOS)
 ARCH ?= $(shell go env GOARCH)
 
+KUBECTL_BIN ?= kubectl
+
 all: operator
 
 build: operator
@@ -76,20 +80,20 @@ run: generate fmt vet manifests
 
 # Install CRDs into a cluster
 install: manifests kustomize
-	$(KUSTOMIZE) build config/crd | kubectl apply -f -
+	$(KUSTOMIZE) build config/crd | $(KUBECTL_BIN) apply -f -
 
 # Uninstall CRDs from a cluster
 uninstall: manifests kustomize
-	$(KUSTOMIZE) build config/crd | kubectl delete -f -
+	$(KUSTOMIZE) build config/crd | $(KUBECTL_BIN) delete -f -
 
 # Deploy controller in the configured Kubernetes cluster in ~/.kube/config
 deploy: manifests kustomize
 	cd config/manager && $(KUSTOMIZE) edit set image controller="$(IMG)"
-	$(KUSTOMIZE) build config/default | kubectl apply -f -
+	$(KUSTOMIZE) build config/default | $(KUBECTL_BIN) apply -f -
 
 # UnDeploy controller from the configured Kubernetes cluster in ~/.kube/config
 undeploy:
-	$(KUSTOMIZE) build config/default | kubectl delete -f -
+	$(KUSTOMIZE) build config/default | $(KUBECTL_BIN) delete -f -
 
 # Generate manifests e.g. CRD, RBAC etc.
 SED_BIN ?= sed
@@ -185,6 +189,16 @@ bundle-build: bundle
 bundle-push: bundle-build
 	$(CONTAINER_ENGINE) push $(BUNDLE_IMG)
 
+# Install OLM on the current cluster
+.PHONY: install-olm
+install-olm: operator-sdk
+	$(OPERATOR_SDK) olm install
+
+# Run the published operator bundle
+.PHONY: bundle-run
+bundle-run: operator-sdk
+	BUNDLE_IMG=$(BUNDLE_IMG) SKIP_TLS=$(SKIP_TLS) hack/run-bundle.sh
+
 .PHONY: opm
 OPM = ./bin/opm
 opm:
@@ -202,12 +216,35 @@ endif
 endif
 
 BUNDLE_IMGS ?= $(BUNDLE_IMG) 
-CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) ifneq ($(origin CATALOG_BASE_IMG), undefined) FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) endif 
+CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:$(VERSION)
+
+#
+# ifneq ($(origin CATALOG_BASE_IMG), undefined) 
+# FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG)
+# endif
+# $(OPM) index add --container-tool $(CONTAINER_ENGINE) --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT)
+
+CATALOG_INDEX_IMG ?= quay.io/operatorhubio/catalog:latest
 
 .PHONY: catalog-build
 catalog-build: opm
-	$(OPM) index add --container-tool $(CONTAINER_ENGINE) --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT)
+	$(OPM) index add --container-tool $(CONTAINER_ENGINE) --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) --from-index=$(CATALOG_INDEX_IMG)
+
 
 .PHONY: catalog-push
-catalog-push: ## Push the catalog image.
+catalog-push: catalog-build
 	$(CONTAINER_ENGINE) push $(CATALOG_IMG)
+
+
+CATALOG_NAMESPACE ?= shipwright-operator
+.PHONY: catalog-run
+catalog-run:
+	CATALOG_IMG=$(CATALOG_IMG) CSV_VERSION=$(VERSION) KUBECTL_BIN=$(KUBECTL_BIN) NAMESPACE=$(CATALOG_NAMESPACE) hack/run-bundle.sh
+
+.PHONY: deploy-kind-registry
+deploy-kind-registry:
+	CONTAINER_ENGINE=$(CONTAINER_ENGINE) KUBECTL_BIN=$(KUBECTL_BIN) test/kind/deploy-registry.sh
+
+.PHONY: deploy-kind-registry-post
+deploy-kind-registry-post:
+	CONTAINER_ENGINE=$(CONTAINER_ENGINE) KUBECTL_BIN=$(KUBECTL_BIN) test/kind/deploy-registry-post.sh

config/catalog/catalog_source.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    app: shipwright-operator
+  name: system
+---
+apiVersion: operators.coreos.com/v1alpha1
+kind: CatalogSource
+metadata:
+  name: operator
+  namespace: system
+spec:
+  sourceType: grpc
+  image: catalog-source:latest
+  displayName: Shipwright Operator Catalog
+  publisher: The Shipwright Contributors
+  updateStrategy:
+    registryPoll:
+      interval: 10m

config/catalog/kustomization.yaml

@@ -0,0 +1,7 @@
+namespace: shipwright-operator
+namePrefix: shipwright-
+
+resources:
+- catalog_source.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization

config/subscription/kustomization.yaml

@@ -0,0 +1,8 @@
+namespace: shipwright-operator
+namePrefix: shipwright-
+
+resources:
+- subscription.yaml
+- operator_group.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization

config/subscription/operator_group.yaml

@@ -0,0 +1,6 @@
+apiVersion: operators.coreos.com/v1alpha2
+kind: OperatorGroup
+metadata:
+  name: operator
+  namespace: system
+spec: {}

config/subscription/subscription.yaml

@@ -0,0 +1,12 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: operator
+  namespace: system
+spec:
+  channel: alpha
+  name: shipwright-operator
+  source: shipwright-operator
+  sourceNamespace: shipwright-operator
+  installPlanApproval: Automatic
+  startingCSV: shipwright-operator.v0.0.0

hack/run-bundle.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+# Run the catalog image
+# CATALOG_IMG - catalog image to deploy
+# CSV_VERSION - version tag of the cluster service version
+
+set -e
+
+kustomize="${KUSTOMIZE_BIN:-${PWD}/bin/kustomize}"
+k8s="${KUBECTL_BIN:-kubectl}"
+catalogNamespace="${CATALOG_NAMESPACE:-shipwright-operator}"
+subNamespace="${SUBSCRIPTION_NAMESPACE:-shipwright-operator}"
+namePrefix="${NAME_PREFIX:-shipwright-}"
+
+if [[ -z ${CATALOG_IMG} ]]; then
+    echo "CATALOG_IMG environment variable must be set"
+    exit 1
+fi
+
+if [[ -z ${CSV_VERSION} ]]; then
+    echo "CSV_VERSION environment variable must be set"
+    exit 1
+fi
+
+echo "Adding replacements not supported by kustomize"
+sed -i -E "s|image: (.+)$|image: ${CATALOG_IMG}|g" config/catalog/catalog_source.yaml
+sed -i -E "s|startingCSV: (.+)$|startingCSV: shipwright-operator.v${CSV_VERSION}|g" config/subscription/subscription.yaml
+sed -i -E "s|sourceNamespace: (.+)$|sourceNamespace: ${catalogNamespace}|g" config/subscription/subscription.yaml
+sed -i -E "s|source: (.+)$|source: ${namePrefix}operator|g" config/subscription/subscription.yaml
+
+echo "Applying catalog source and subscription from kustomize"
+
+pushd config/catalog
+${kustomize} edit set namespace "${catalogNamespace}"
+${kustomize} edit set nameprefix "${namePrefix}"
+popd
+
+pushd config/subscription
+${kustomize} edit set namespace "${subNamespace}"
+${kustomize} edit set nameprefix "${namePrefix}"
+popd
+
+echo "Deploying catalog source"
+${kustomize} build config/catalog | ${k8s} apply -f -
+
+echo "Waiting for catalog source to be ready"
+attempts=1
+while [[ ${attempts} -le 5 ]]; do
+    echo "Checking the status of the catalog source - attempt ${attempts}"
+    if ${k8s} wait --for=condition=Ready pod -l "olm.catalogSource=${namePrefix}operator" -n "${catalogNamespace}"; then
+        attempts=100
+    fi
+    attempts=$(( attempts + 1 ))
+    sleep 10
+done
+
+if [[ ${attempts} -le 100 ]]; then
+    echo "Timed out waiting for the catalog source to be ready"
+    exit 1
+fi
+
+echo "Deploying subscription"
+${kustomize} build config/subscription | ${k8s} apply -f -
+
+echo "Waiting for operator deployment"
+
+attempts=1
+while [[ ${attempts} -le 10 ]]; do
+    echo "Checking the status of the operator rollout - attempt ${attempts}"
+    if ${k8s} rollout status deployment "${namePrefix}operator" -n "${subNamespace}"; then
+        echo "Operator successfully deployed"
+        exit 0
+    fi
+    attempts=$(( attempts + 1 ))
+    sleep 10
+done
+
+echo "Failed to deploy, dumping operator state"
+echo "Dumping pods"
+${k8s} get pods -n "${subNamespace}" -o yaml
+if [[ ${catalogNamespace} -ne ${subNamespace} ]]; then
+    ${k8s} get pods -n "${catalogNamespace}" -o yaml
+fi
+echo "Dumping OLM catalog sources"
+${k8s} get catalogsources -n "${catalogNamespace}" -o yaml
+echo "Dumping OLM subscriptions"
+${k8s} get subscriptions -n "${subNamespace}" -o yaml
+echo "Dumping OLM installplans"
+${k8s} get installplans -n "${subNamespace}" -o yaml
+echo "Dumping OLM CSVs"
+${k8s} get clusterserviceversions -n "${subNamespace}" -o yaml
+exit 1

test/kind/config.yaml

@@ -0,0 +1,14 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
+    endpoint = ["http://kind-registry:5000"]
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: ClusterConfiguration
+    apiServer:
+      extraArgs:
+        enable-admission-plugins: CertificateApproval,CertificateSigning,CertificateSubjectRestriction,DefaultIngressClass,DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,MutatingAdmissionWebhook,NamespaceLifecycle,NodeRestriction,OwnerReferencesPermissionEnforcement,PersistentVolumeClaimResize,PersistentVolumeLabel,PodNodeSelector,PodTolerationRestriction,Priority,ResourceQuota,RuntimeClass,ServiceAccount,StorageObjectInUseProtection,TaintNodesByCondition,ValidatingAdmissionWebhook

test/kind/deploy-registry-post.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+
+docker="${CONTAINER_ENGINE:-docker}"
+k8s="${KUBECTL_BIN:-kubectl}"
+
+echo "Running KinD registry post-install actions"
+
+reg_name='kind-registry'
+
+echo "Connecting registry ${reg_name} to kind network"
+# connect the registry to the cluster network
+# (the network may already be connected)
+${docker} network connect "kind" "${reg_name}" || true
+
+echo "Registry connected to kind network"
+
+echo "Publishing local container registry on the KinD cluster"
+
+${k8s} apply -f test/kind/local-registry-cm.yaml
+
+echo "Done"

test/kind/deploy-registry.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -e
+
+docker="${CONTAINER_ENGINE:-docker}"
+
+# create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5000'
+echo "Deploying container registry ${reg_name}:${reg_port}"
+running="$(${docker} inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)"
+if [ "${running}" != 'true' ]; then
+  ${docker} run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
+    registry:2
+fi
+
+echo "Container registry ${reg_name} deployed"

test/kind/local-registry-cm.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:5000"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"