diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4b755b45..4a1090396 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: go-version: [1.15.x] os: [ubuntu-latest] kubernetes: - - v1.19.7 + - v1.20.7 max-parallel: 2 runs-on: ${{ matrix.os }} steps: @@ -60,5 +60,35 @@ jobs: go-version: ${{ matrix.go-version }} - name: Check out code uses: actions/checkout@v2 - - name: Build Image - run: make ko-publish IMAGE_REPO=ko.local + - name: Install kubectl + uses: azure/setup-kubectl@v1 + with: + version: ${{ matrix.kubernetes }} + - name: Deploy KinD Local Container Registry + run: make deploy-kind-registry + - name: Create KinD cluster + uses: helm/kind-action@v1.2.0 + with: + version: v0.11.1 + node_image: kindest/node:${{ matrix.kubernetes }} + cluster_name: kind + config: test/kind/config.yaml + wait: 120s + - name: Verify KinD cluster + run: make verify-kind + - name: Install KinD post-actions + run: make deploy-kind-registry-post + - name: Install OLM + run: make install-olm + # Builds the operator and makes the image readable in the KinD cluster + - name: Build Operator Image + run: | + make ko-publish IMAGE_REPO=localhost:5000 + - name: Build Operator Bundle + run: | + make bundle-push IMAGE_REPO=localhost:5000 + - name: Build Catalog Source + run: | + make catalog-push IMAGE_REPO=localhost:5000 + - name: Run Operator with Catalog + run: make catalog-run IMAGE_REPO=localhost:5000 diff --git a/Makefile b/Makefile index 4c1b7f41a..4707128fa 100644 --- a/Makefile +++ b/Makefile @@ -39,20 +39,22 @@ IMAGE_REPO ?= quay.io/shipwright TAG ?= $(VERSION) IMAGE_PUSH ?= true -BUNDLE_IMG_NAME ?= operator-bundle -OPERATOR_IMG_NAME ?= operator +IMAGE_TAG_BASE ?= $(IMAGE_REPO)/operator # Image URL to use all building/pushing image targets -IMG ?= $(IMAGE_REPO)/$(OPERATOR_IMG_NAME):$(TAG) +IMG ?= $(IMAGE_TAG_BASE):$(TAG) # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) -BUNDLE_IMG ?= $(IMAGE_REPO)/$(BUNDLE_IMG_NAME):$(TAG) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(TAG) # operating-system type and architecture based on golang OS ?= $(shell go env GOOS) ARCH ?= $(shell go env GOARCH) +KUBECTL_BIN ?= kubectl +SED_BIN ?= sed + all: operator build: operator @@ -76,23 +78,22 @@ run: generate fmt vet manifests # Install CRDs into a cluster install: manifests kustomize - $(KUSTOMIZE) build config/crd | kubectl apply -f - + $(KUSTOMIZE) build config/crd | $(KUBECTL_BIN) apply -f - # Uninstall CRDs from a cluster uninstall: manifests kustomize - $(KUSTOMIZE) build config/crd | kubectl delete -f - + $(KUSTOMIZE) build config/crd | $(KUBECTL_BIN) delete -f - # Deploy controller in the configured Kubernetes cluster in ~/.kube/config deploy: manifests kustomize cd config/manager && $(KUSTOMIZE) edit set image controller="$(IMG)" - $(KUSTOMIZE) build config/default | kubectl apply -f - + $(KUSTOMIZE) build config/default | $(KUBECTL_BIN) apply -f - # UnDeploy controller from the configured Kubernetes cluster in ~/.kube/config undeploy: - $(KUSTOMIZE) build config/default | kubectl delete -f - + $(KUSTOMIZE) build config/default | $(KUBECTL_BIN) delete -f - # Generate manifests e.g. CRD, RBAC etc. -SED_BIN ?= sed manifests: controller-gen $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases @@ -185,6 +186,11 @@ bundle-build: bundle bundle-push: bundle-build $(CONTAINER_ENGINE) push $(BUNDLE_IMG) +# Install OLM on the current cluster +.PHONY: install-olm +install-olm: operator-sdk + $(OPERATOR_SDK) olm install + .PHONY: opm OPM = ./bin/opm opm: @@ -202,12 +208,42 @@ endif endif BUNDLE_IMGS ?= $(BUNDLE_IMG) -CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) ifneq ($(origin CATALOG_BASE_IMG), undefined) FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) endif +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:$(VERSION) +# +# ifneq ($(origin CATALOG_BASE_IMG), undefined) +# FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) +# endif +# $(OPM) index add --container-tool $(CONTAINER_ENGINE) --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) + +CATALOG_INDEX_IMG ?= quay.io/operatorhubio/catalog:latest + +# Build a catalog image with the operator bundle included .PHONY: catalog-build catalog-build: opm - $(OPM) index add --container-tool $(CONTAINER_ENGINE) --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) - + $(OPM) index add --container-tool $(CONTAINER_ENGINE) --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) --from-index=$(CATALOG_INDEX_IMG) + +# Build and push a catalog image with the operator bundle to a container registry .PHONY: catalog-push -catalog-push: ## Push the catalog image. +catalog-push: catalog-build $(CONTAINER_ENGINE) push $(CATALOG_IMG) + + +CATALOG_NAMESPACE ?= shipwright-operator + +# Run the operator from a catalog image, using an OLM subscription +.PHONY: catalog-run +catalog-run: + CATALOG_IMG=$(CATALOG_IMG) CSV_VERSION=$(VERSION) KUBECTL_BIN=$(KUBECTL_BIN) NAMESPACE=$(CATALOG_NAMESPACE) SED_BIN=$(SED_BIN) hack/run-operator-catalog.sh + +.PHONY: verify-kind +verify-kind: + KUBECTL_BIN=$(KUBECTL_BIN) test/kind/verify-kind.sh + +.PHONY: deploy-kind-registry +deploy-kind-registry: + CONTAINER_ENGINE=$(CONTAINER_ENGINE) KUBECTL_BIN=$(KUBECTL_BIN) test/kind/deploy-registry.sh + +.PHONY: deploy-kind-registry-post +deploy-kind-registry-post: + CONTAINER_ENGINE=$(CONTAINER_ENGINE) KUBECTL_BIN=$(KUBECTL_BIN) test/kind/deploy-registry-post.sh \ No newline at end of file diff --git a/config/catalog/catalog_source.yaml b/config/catalog/catalog_source.yaml new file mode 100644 index 000000000..a8bff0024 --- /dev/null +++ b/config/catalog/catalog_source.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + app: shipwright-operator + name: system +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: operator + namespace: system +spec: + sourceType: grpc + image: catalog-source:latest + displayName: Shipwright Operator Catalog + publisher: The Shipwright Contributors + updateStrategy: + registryPoll: + interval: 10m \ No newline at end of file diff --git a/config/catalog/kustomization.yaml b/config/catalog/kustomization.yaml new file mode 100644 index 000000000..e16deb160 --- /dev/null +++ b/config/catalog/kustomization.yaml @@ -0,0 +1,7 @@ +namespace: shipwright-operator +namePrefix: shipwright- + +resources: +- catalog_source.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization diff --git a/config/subscription/kustomization.yaml b/config/subscription/kustomization.yaml new file mode 100644 index 000000000..4bf0724fc --- /dev/null +++ b/config/subscription/kustomization.yaml @@ -0,0 +1,8 @@ +namespace: shipwright-operator +namePrefix: shipwright- + +resources: +- subscription.yaml +- operator_group.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization diff --git a/config/subscription/operator_group.yaml b/config/subscription/operator_group.yaml new file mode 100644 index 000000000..ee327b68c --- /dev/null +++ b/config/subscription/operator_group.yaml @@ -0,0 +1,6 @@ +apiVersion: operators.coreos.com/v1alpha2 +kind: OperatorGroup +metadata: + name: operator + namespace: system +spec: {} diff --git a/config/subscription/subscription.yaml b/config/subscription/subscription.yaml new file mode 100644 index 000000000..8bc5c8407 --- /dev/null +++ b/config/subscription/subscription.yaml @@ -0,0 +1,12 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: operator + namespace: system +spec: + channel: alpha + name: shipwright-operator + source: shipwright-operator + sourceNamespace: shipwright-operator + installPlanApproval: Automatic + startingCSV: shipwright-operator.v0.0.0 diff --git a/docs/development/local-development.md b/docs/development/local-development.md index 3058fc0d5..3d9aa3d7d 100644 --- a/docs/development/local-development.md +++ b/docs/development/local-development.md @@ -34,8 +34,8 @@ Refer to the [ko documentation](https://github.com/google/ko#local-publishing-op To test the operator on a Kubernetes cluster, you first must have the following: -* Access to a Kubernetes cluster v1.19 or higher, with cluster admin permissions. -* Install Tekton v0.21 on the cluster. +* Access to a Kubernetes cluster v1.20 or higher, with cluster admin permissions. +* Install [Tekton operator](https://github.com/tektoncd/operator) v0.49 or higher on the cluster. ```bash $ export KUBECONFIG=/path/to/kubeconfig @@ -61,6 +61,8 @@ Finally, use the `make deploy` command with appropriate `IMAGE_REPO` and `TAG` a $ make deploy IMAGE_REPO="/" TAG="" ``` +_Note:_ + Scripts in `hack` folder may require `sed` (GNU), therefore in platforms other than Linux you may have it with a different name. For instance, on macOS it's usually named `gsed`, in this case provide the `SED_BIN` make variable with the alternative name. ```bash diff --git a/docs/development/olm-development.md b/docs/development/olm-development.md new file mode 100644 index 000000000..9bef624b4 --- /dev/null +++ b/docs/development/olm-development.md @@ -0,0 +1,61 @@ +# OLM Development + +The Shipwright operator is meant to be deployed on a cluster using the +[Operator Lifecycle Manager](https://olm.operatorframework.io/) (OLM). +OLM provides mechanisms to support over the air upgrades and automatically deploy related operators +that are packaged in operator [catalogs](https://olm.operatorframework.io/). +Additional steps need to be taken to ensure the operator can be deployed with OLM. + +## Prerequisites + +* Ensure you have access to a Kubernetes cluster via `kubectl` with cluster admin permissions. +* Install Go version 1.15 or higher. +* Install OLM on your cluster. This can be done using the `make install-olm` command. +* Ability to push to a container registry that is accessible inside your Kubernetes cluster. + +## Step 1: Push the operator image to a registry + +Run `make ko-publish IMAGE_REPO=`, pushing to a container registry that is accessible inside your Kubernetes cluster. +Using `ko.local` or `kind.local` is not recommended, as this will not push the resulting image to a container registry. + +If you are using [KinD](https://kind.sigs.k8s.io/), follow the instructions on how to configure a +[local registry](https://kind.sigs.k8s.io/docs/user/local-registry/). +This will let you use `localhost:` as your container registry. + +## Step 2: Build and push the operator bundle + +Next, run `make bundle-push IMAGE_REPO=`. +This will push the [operator bundle](https://olm.operatorframework.io/docs/tasks/creating-operator-bundle/) +to the container registry. +An operator bundle is an OCI aritfact that tells OLM how to deploy your operator. +Be sure to use the same container registry for the `IMAGE_REPO` argument. + +## Step 3: Build and push an operator catalog + +Next, run `make catalog-push IMAGE_REPO=`. +This will build and push an [operator catalog](https://olm.operatorframework.io/docs/tasks/creating-a-catalog/), +which packages your test operator bundle with the other operators available on [operatorhub.io](https://operatorhub.io). +As in step 2, be sure to use the same container registry for the `IMAGE_REPO` argument. + +## Step 4: Deploy the operator using the catalog image + +Finally, deploy the operator using `make catalog-run IMAGE_REPO=`, using the same +value for `IMAGE_REPO` as in the previous steps. +This will run a script that does the following: + +1. Creates a custom [CatalogSource](https://olm.operatorframework.io/docs/tasks/make-catalog-available-on-cluster/) + and [OperatorGroup](https://olm.operatorframework.io/docs/advanced-tasks/operator-scoping-with-operatorgroups/), + which allows the operators in step 3's catalog to be installed anywhere on the clsuter. +2. Creates a [Subscription](https://olm.operatorframework.io/docs/tasks/install-operator-with-olm/), + which instructs OLM to install the operator and any dependent operators. +3. Checks that the operator has successfully been installed and rolled out. + +Once the script completes, the Shipwright and Tekton operators will be installed on the cluster. + +_Note:_ + +Scripts in `hack` folder may require `sed` (GNU), therefore in platforms other than Linux you may have it with a different name. For instance, on macOS it's usually named `gsed`, in this case provide the `SED_BIN` make variable with the alternative name. + +```bash +$ make catalog-run SED_BIN=gsed ... +``` diff --git a/hack/run-operator-catalog.sh b/hack/run-operator-catalog.sh new file mode 100755 index 000000000..ecf7c514d --- /dev/null +++ b/hack/run-operator-catalog.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +# hack/run-operator-catalog.sh +# +# Run the operator from a catalog image. +# Required environment variables: +# +# - CATALOG_IMG: catalog image to deploy +# - CSV_VERSION: version tag of the cluster service version +# +# Optional environment variables +# +# - KUSTOMIZE_BIN: path to kustomize +# - KUBECTL_BIN: path to kubectl (or equivalent command line) +# - SED_BIN: path to GNU sed +# - CATALOG_NAMESPACE: Namespace to deploy the catalog. Defaults to shipwright-operator. +# - SUBSCRIPTION_NAMESPACE: Namespace to install the operator via an OLM subscription. Defaults to +# shipwright-operator. +# - NAME_PREFIX: prefix to use for all resource names. Defaults to "shipwright-" + +set -e + +kustomize="${KUSTOMIZE_BIN:-${PWD}/bin/kustomize}" +k8s="${KUBECTL_BIN:-kubectl}" +sed="${SED_BIN:-sed}" +catalogNamespace="${CATALOG_NAMESPACE:-shipwright-operator}" +subNamespace="${SUBSCRIPTION_NAMESPACE:-shipwright-operator}" +namePrefix="${NAME_PREFIX:-shipwright-}" + +if [[ -z ${CATALOG_IMG} ]]; then + echo "CATALOG_IMG environment variable must be set" + exit 1 +fi + +if [[ -z ${CSV_VERSION} ]]; then + echo "CSV_VERSION environment variable must be set" + exit 1 +fi + +echo "Adding replacements not supported by kustomize" +${sed} -i -E "s|image: (.+)$|image: ${CATALOG_IMG}|g" config/catalog/catalog_source.yaml +${sed} -i -E "s|startingCSV: (.+)$|startingCSV: shipwright-operator.v${CSV_VERSION}|g" config/subscription/subscription.yaml +${sed} -i -E "s|sourceNamespace: (.+)$|sourceNamespace: ${catalogNamespace}|g" config/subscription/subscription.yaml +${sed} -i -E "s|source: (.+)$|source: ${namePrefix}operator|g" config/subscription/subscription.yaml + +echo "Applying catalog source and subscription from kustomize" + +pushd config/catalog +${kustomize} edit set namespace "${catalogNamespace}" +${kustomize} edit set nameprefix "${namePrefix}" +popd + +pushd config/subscription +${kustomize} edit set namespace "${subNamespace}" +${kustomize} edit set nameprefix "${namePrefix}" +popd + +echo "Deploying catalog source" +${kustomize} build config/catalog | ${k8s} apply -f - + +echo "Waiting for catalog source to be ready" +attempts=1 +while [[ ${attempts} -le 5 ]]; do + echo "Checking the status of the catalog source - attempt ${attempts}" + if ${k8s} wait --for=condition=Ready pod -l "olm.catalogSource=${namePrefix}operator" -n "${catalogNamespace}"; then + attempts=100 + fi + attempts=$(( attempts + 1 )) + sleep 10 +done + +if [[ ${attempts} -le 100 ]]; then + echo "Timed out waiting for the catalog source to be ready" + exit 1 +fi + +echo "Deploying subscription" +${kustomize} build config/subscription | ${k8s} apply -f - + +echo "Waiting for operator deployment" + +attempts=1 +while [[ ${attempts} -le 10 ]]; do + echo "Checking the status of the operator rollout - attempt ${attempts}" + if ${k8s} rollout status deployment "${namePrefix}operator" -n "${subNamespace}"; then + echo "Operator successfully deployed" + exit 0 + fi + attempts=$(( attempts + 1 )) + sleep 10 +done + +echo "Failed to deploy, dumping operator state" +echo "Dumping pods" +${k8s} get pods -n "${subNamespace}" -o yaml +if [[ ${catalogNamespace} -ne ${subNamespace} ]]; then + ${k8s} get pods -n "${catalogNamespace}" -o yaml +fi +echo "Dumping OLM catalog sources" +${k8s} get catalogsources -n "${catalogNamespace}" -o yaml +echo "Dumping OLM subscriptions" +${k8s} get subscriptions -n "${subNamespace}" -o yaml +echo "Dumping OLM installplans" +${k8s} get installplans -n "${subNamespace}" -o yaml +echo "Dumping OLM CSVs" +${k8s} get clusterserviceversions -n "${subNamespace}" -o yaml +exit 1 diff --git a/test/kind/config.yaml b/test/kind/config.yaml new file mode 100644 index 000000000..b6b9c8703 --- /dev/null +++ b/test/kind/config.yaml @@ -0,0 +1,14 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +containerdConfigPatches: +- |- + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"] + endpoint = ["http://kind-registry:5000"] +nodes: +- role: control-plane + kubeadmConfigPatches: + - | + kind: ClusterConfiguration + apiServer: + extraArgs: + enable-admission-plugins: CertificateApproval,CertificateSigning,CertificateSubjectRestriction,DefaultIngressClass,DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,MutatingAdmissionWebhook,NamespaceLifecycle,NodeRestriction,OwnerReferencesPermissionEnforcement,PersistentVolumeClaimResize,PersistentVolumeLabel,PodNodeSelector,PodTolerationRestriction,Priority,ResourceQuota,RuntimeClass,ServiceAccount,StorageObjectInUseProtection,TaintNodesByCondition,ValidatingAdmissionWebhook diff --git a/test/kind/deploy-registry-post.sh b/test/kind/deploy-registry-post.sh new file mode 100755 index 000000000..d9440f2bf --- /dev/null +++ b/test/kind/deploy-registry-post.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +set -e + +docker="${CONTAINER_ENGINE:-docker}" +k8s="${KUBECTL_BIN:-kubectl}" + +echo "Running KinD registry post-install actions" + +reg_name='kind-registry' + +echo "Connecting registry ${reg_name} to kind network" +# connect the registry to the cluster network +# (the network may already be connected) +${docker} network connect "kind" "${reg_name}" || true + +echo "Registry connected to kind network" + +echo "Publishing local container registry on the KinD cluster" + +${k8s} apply -f test/kind/local-registry-cm.yaml + +echo "Done" diff --git a/test/kind/deploy-registry.sh b/test/kind/deploy-registry.sh new file mode 100755 index 000000000..a0bd18def --- /dev/null +++ b/test/kind/deploy-registry.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +set -e + +docker="${CONTAINER_ENGINE:-docker}" + +# create registry container unless it already exists +reg_name='kind-registry' +reg_port='5000' +echo "Deploying container registry ${reg_name}:${reg_port}" +running="$(${docker} inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" +if [ "${running}" != 'true' ]; then + ${docker} run \ + -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \ + registry:2 +fi + +echo "Container registry ${reg_name} deployed" diff --git a/test/kind/local-registry-cm.yaml b/test/kind/local-registry-cm.yaml new file mode 100644 index 000000000..4fbd78ab8 --- /dev/null +++ b/test/kind/local-registry-cm.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: local-registry-hosting + namespace: kube-public +data: + localRegistryHosting.v1: | + host: "localhost:5000" + help: "https://kind.sigs.k8s.io/docs/user/local-registry/" diff --git a/test/kind/verify-kind.sh b/test/kind/verify-kind.sh new file mode 100755 index 000000000..00cc23d32 --- /dev/null +++ b/test/kind/verify-kind.sh @@ -0,0 +1,20 @@ +#! /bin/sh + +k8s=${KUBECTL_BIN:-kubectl} + +echo "# Using KinD context..." +$k8s config use-context "kind-kind" +cho "# KinD nodes:" +$k8s get nodes + +nodeStatus=$(${k8s} get node kind-control-plane -o json | jq -r .'status.conditions[] | select(.type == "Ready") | .status') +if [ "${nodeStatus}" != "True" ]; then + echo "# Node is not ready:" + $k8s describe node kind-control-plane + echo "# Pods:" + $k8s get pod -A + echo "# Events:" + $k8s get events -A + + exit 1 +fi \ No newline at end of file