webhosting-operator: drop legacy sharder implementation, use shard li…

…brary (#228) * Introduce `go.work` file * Adapt to controller-runtime v0.17 * Drop internal sharder implementation, use shard library Instead of relying on the sharding implementation in the controller-runtime fork, webhosting-operator now uses the shard library for complying with the external sharder implementation (similar to the example shard). The internal sharder implementation is dropped accordingly. * Merge Makefile and tooling * Merge skaffold configs * Merge CI workflows * Adapt to new cluster setup Create a new cluster at STACKIT replacing the dedicated Gardener environment. * Update docs * Split CI workflows
timebertt · Apr 24, 2024 · 41797ff · 41797ff
1 parent b7a2c96
commit 41797ff
Show file tree

Hide file tree

Showing 38 changed files with 703 additions and 960 deletions.
diff --git a/.github/renovate.json5 b/.github/renovate.json5
@@ -126,23 +126,6 @@
       matchPackagePrefixes: ["gomodules.xyz/jsonpatch"],
       enabled: false
     },
-    {
-      // bump webhosting-operator dependencies and tools in separate PRs
-      matchFileNames: [
-        "webhosting-operator/**",
-        ".github/workflows/webhosting-operator.yaml"
-      ],
-      additionalBranchPrefix: "webhosting-operator-",
-      commitMessagePrefix: "webhosting-operator:"
-    },
-    {
-      // k8s and controller-runtime dependencies in webhosting-operator can only be upgraded after rebasing the sharding
-      // implementation in the controller-runtime fork, disable bumps
-      matchDatasources: ["go"],
-      matchFileNames: ["webhosting-operator/go.mod"],
-      matchPackagePrefixes: ["k8s.io/", "sigs.k8s.io/controller-runtime", "github.com/timebertt/controller-runtime"],
-      enabled: false
-    },
     {
       // kind k8s version should be updated together with shoot k8s version
       matchPackageNames: ["kindest/node"],

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -0,0 +1,30 @@
+name: e2e
+
+on:
+  push:
+    branches:
+    - main
+    tags:
+    - v*
+    paths-ignore:
+    - "**.md"
+  pull_request:
+
+jobs:
+  e2e-kind:
+    runs-on: ubuntu-latest
+    env:
+      ARTIFACTS: artifacts
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+    - run: make ci-e2e-kind
+    - uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: e2e-artifacts
+        path: artifacts
+        if-no-files-found: error
diff --git a/.github/workflows/controller-sharding.yaml → .github/workflows/images.yaml b/.github/workflows/controller-sharding.yaml → .github/workflows/images.yaml
@@ -1,49 +1,14 @@
-name: controller-sharding
+name: images
 
 on:
   push:
     branches:
     - main
     tags:
     - v*
-    paths-ignore:
-    - "webhosting-operator/**"
-    - ".github/workflows/webhosting-operator.yaml"
-    - "**.md"
   pull_request:
-    paths-ignore:
-    - "webhosting-operator/**"
-    - ".github/workflows/webhosting-operator.yaml"
 
 jobs:
-  verify:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-go@v5
-      with:
-        go-version-file: go.mod
-    - run: make verify
-
-  e2e-kind:
-    runs-on: ubuntu-latest
-    env:
-      ARTIFACTS: artifacts
-
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-go@v5
-      with:
-        go-version-file: go.mod
-    - run: make ci-e2e-kind
-    - uses: actions/upload-artifact@v4
-      if: always()
-      with:
-        name: e2e-artifacts
-        path: artifacts
-        if-no-files-found: error
-
   images:
     runs-on: ubuntu-latest
     env:
@@ -72,6 +37,8 @@ jobs:
           ./cmd/sharder
           ./cmd/shard
           ./hack/cmd/janitor
+          ./webhosting-operator/cmd/experiment
+          ./webhosting-operator/cmd/webhosting-operator
         )
         
         echo builds: > .ko.yaml

diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml
@@ -0,0 +1,20 @@
+name: verify
+
+on:
+  push:
+    branches:
+    - main
+    tags:
+    - v*
+  pull_request:
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+    - run: make verify
diff --git a/.github/workflows/webhosting-operator.yaml b/.github/workflows/webhosting-operator.yaml
diff --git a/.golangci.yaml b/.golangci.yaml
@@ -24,6 +24,8 @@ linters-settings:
     alias:
     - pkg: github.com/timebertt/kubernetes-controller-sharding/apis/(\w+)/(v[\w\d]+)
       alias: $1$2
+    - pkg: github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/apis/(\w+)/(v[\w\d]+)
+      alias: $1$2
     - pkg: k8s.io/api/(\w+)/(v[\w\d]+)
       alias: $1$2
     - pkg: k8s.io/apimachinery/pkg/apis/(\w+)/(v[\w\d]+)

diff --git a/...g-operator/.run/experiment (kind).run.xml → .run/experiment (kind).run.xml b/...g-operator/.run/experiment (kind).run.xml → .run/experiment (kind).run.xml
diff --git a/...r/.run/webhosting-operator (kind).run.xml → .run/webhosting-operator (kind).run.xml b/...r/.run/webhosting-operator (kind).run.xml → .run/webhosting-operator (kind).run.xml
@@ -3,10 +3,7 @@
     <module name="kubernetes-controller-sharding" />
     <working_directory value="$PROJECT_DIR$/webhosting-operator" />
     <envs>
-      <env name="LEADER_ELECT" value="false" />
-      <env name="SHARD_ID" value="webhosting-operator-0" />
-      <env name="SHARD_MODE" value="both" />
-      <env name="SHARDING_ENABLED" value="true" />
+      <env name="ENABLE_SHARDING" value="true" />
       <env name="KUBECONFIG" value="$PROJECT_DIR$/hack/kind_kubeconfig.yaml" />
     </envs>
     <kind value="PACKAGE" />

diff --git a/Makefile b/Makefile
@@ -6,6 +6,8 @@ GHCR_REPO ?= ghcr.io/timebertt/kubernetes-controller-sharding
 SHARDER_IMG ?= $(GHCR_REPO)/sharder:$(TAG)
 SHARD_IMG ?= $(GHCR_REPO)/shard:$(TAG)
 JANITOR_IMG ?= $(GHCR_REPO)/janitor:$(TAG)
+WEBHOSTING_OPERATOR_IMG ?= $(GHCR_REPO)/webhosting-operator:$(TAG)
+EXPERIMENT_IMG ?= $(GHCR_REPO)/experiment:$(TAG)
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.27
@@ -40,22 +42,31 @@ help: ## Display this help.
 include hack/tools.mk
 
 .PHONY: clean-tools-bin
-clean-tools-bin: ## Empty the tools binary directory
+clean-tools-bin: ## Empty the tools binary directory.
 	rm -rf $(TOOLS_BIN_DIR)/*
 
 ##@ Development
 
 .PHONY: modules
 modules: ## Runs go mod to ensure modules are up to date.
 	go mod tidy
+	cd webhosting-operator && go mod tidy
+	@# regenerate go.work.sum
+	rm -f go.work.sum
+	go mod download
 
 .PHONY: generate-fast
-generate-fast: $(CONTROLLER_GEN) modules ## Run all fast code generators
+generate-fast: $(CONTROLLER_GEN) modules ## Run all fast code generators for the main module.
 	$(CONTROLLER_GEN) rbac:roleName=sharder crd paths="./pkg/..." output:rbac:artifacts:config=config/rbac output:crd:artifacts:config=config/crds
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./pkg/..."
 
+.PHONY: generate-fast-webhosting
+generate-fast-webhosting: $(CONTROLLER_GEN) modules ## Run all fast code generators for the webhosting-operator module.
+	$(CONTROLLER_GEN) rbac:roleName=operator crd paths="./webhosting-operator/..." output:rbac:artifacts:config=webhosting-operator/config/manager/rbac output:crd:artifacts:config=webhosting-operator/config/manager/crds
+	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./webhosting-operator/..."
+
 .PHONY: generate
-generate: $(VGOPATH) generate-fast modules ## Run all code generators
+generate: $(VGOPATH) generate-fast generate-fast-webhosting modules ## Run all code generators.
 	hack/update-codegen.sh
 
 .PHONY: fmt
@@ -64,7 +75,7 @@ fmt: ## Run go fmt against code.
 
 .PHONY: test
 test: $(SETUP_ENVTEST) ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(SETUP_ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -race ./cmd/... ./pkg/...
+	KUBEBUILDER_ASSETS="$(shell $(SETUP_ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -race ./cmd/... ./pkg/... ./webhosting-operator/pkg/...
 
 .PHONY: test-kyverno
 test-kyverno: $(KYVERNO) ## Run kyverno policy tests.
@@ -139,7 +150,7 @@ images: export KO_DOCKER_REPO = $(GHCR_REPO)
 .PHONY: images
 images: $(KO) ## Build and push container images using ko.
 	$(KO) build --push=$(PUSH) --sbom none --base-import-paths -t $(TAG) --platform linux/amd64,linux/arm64 \
-		./cmd/sharder ./cmd/shard ./hack/cmd/janitor
+		./cmd/sharder ./cmd/shard ./hack/cmd/janitor ./webhosting-operator/cmd/webhosting-operator
 
 ##@ Deployment
 
@@ -166,7 +177,7 @@ up dev: export SKAFFOLD_TAIL ?= true
 
 .PHONY: deploy
 deploy: $(SKAFFOLD) $(KUBECTL) $(YQ) ## Build all images and deploy everything to K8s cluster specified in $KUBECONFIG.
-	$(SKAFFOLD) deploy -i $(SHARDER_IMG) -i $(SHARD_IMG) -i $(JANITOR_IMG)
+	$(SKAFFOLD) deploy -i $(SHARDER_IMG) -i $(SHARD_IMG) -i $(JANITOR_IMG) -i $(WEBHOSTING_OPERATOR_IMG) -i $(EXPERIMENT_IMG)
 
 .PHONY: up
 up: $(SKAFFOLD) $(KUBECTL) $(YQ) ## Build all images, deploy everything to K8s cluster specified in $KUBECONFIG, start port-forward and tail logs.

diff --git a/docs/development.md b/docs/development.md
@@ -26,6 +26,8 @@ Note that you might need to push images to a remote registry though.
 The development setup reuses the deployment manifests of the main sharding components developed in this repository, located in [`config`](../config).
 See [Install the Sharding Components](installation.md).
 
+It also includes the [example shard](../pkg/shard) (see [Implement Sharding in Your Controller](implement-sharding.md)) and the [webhosting-operator](../webhosting-operator/README.md) (see [Evaluating the Sharding Mechanism](evaluation.md)).
+
 Apart from this, the development setup also includes some external components, located in [`hack/config`](../hack/config).
 This includes [cert-manager](https://cert-manager.io/), [ingress-nginx](https://kubernetes.github.io/ingress-nginx/), [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus), [kyverno](https://kyverno.io/), and [parca](https://parca.dev/).
 These components are installed for a seamless development and testing experience but also for this project's [Evaluation](evaluation.md) on a remote cluster in the cloud.

diff --git a/docs/evaluation.md b/docs/evaluation.md
@@ -9,14 +9,14 @@ The evaluation setup builds upon the [Development and Testing Setup](development
 To demonstrate and evaluate the implemented sharding mechanisms using a fully functioning controller, a dedicated example operator was developed: the [webhosting-operator](../webhosting-operator/README.md).
 While the webhosting-operator is developed in the same repository, it only serves as an example.
 
-After deploying the sharding components using `make deploy` or `make up`, the webhosting-operator can be deployed in a similar manner along with the other evaluation components.
-Assuming you're in the repository's root directory:
+When deploying the sharding components using `make deploy` or `make up`, the webhosting-operator is automatically deployed along with the other evaluation components.
+Assuming you're in the repository's root directory, you can deploy the webhosting-operator using:
 
 ```bash
 # deploy the webhosting-operator using pre-built images
-make -C webhosting-operator deploy TAG=latest
+make deploy SKAFFOLD_MODULE=webhosting-operator TAG=latest
 # alternatively, build and deploy fresh images
-make -C webhosting-operator up
+make up SKAFFOLD_MODULE=webhosting-operator
 ```
 
 To perform a quick test of the webhosting-operator, create some example `Website` objects:
@@ -57,8 +57,7 @@ In addition to creating the preconfigured websites, you can also generate some m
 
 ```bash
 # create a random number of websites per project namespace (up to 50 each)
-$ cd webhosting-operator
-$ go run ./cmd/samples-generator
+$ go run ./webhosting-operator/cmd/samples-generator
 created 32 Websites in project "project-foo"
 ```
 
@@ -67,8 +66,7 @@ created 32 Websites in project "project-foo"
 The [experiment](./cmd/experiment) tool allows executing different scenarios for load testing the webhosting-operator, which are used for evaluating the sharding mechanism:
 
 ```text
-$ cd webhosting-operator
-$ go run ./cmd/experiment -h
+$ go run ./webhosting-operator/cmd/experiment -h
 Usage:
   experiment [command]
 
@@ -85,10 +83,10 @@ A load test scenario can be executed using one of these commands:
 go run ./cmd/experiment basic
 
 # build the experiment image and run the basic scenario as a Job on the cluster
-make -C webhosting-operator up SKAFFOLD_MODULE=experiment EXPERIMENT_SCENARIO=basic
+make up SKAFFOLD_MODULE=experiment EXPERIMENT_SCENARIO=basic
 
 # use a pre-built experiment image to run the basic scenario as a Job on the cluster
-make -C webhosting-operator deploy SKAFFOLD_MODULE=experiment EXPERIMENT_SCENARIO=basic TAG=latest
+make deploy SKAFFOLD_MODULE=experiment EXPERIMENT_SCENARIO=basic TAG=latest
 ```
 
 All scenarios put load on webhosting-operator by creating and mutating a large amount of `Website` objects.

diff --git a/go.mod b/go.mod
@@ -62,9 +62,10 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/client_model v0.6.0 // indirect
+	github.com/prometheus/common v0.53.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/spf13/afero v1.10.0 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
@@ -79,12 +80,12 @@ require (
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	golang.org/x/mod v0.16.0 // indirect
 	golang.org/x/net v0.23.0 // indirect
-	golang.org/x/oauth2 v0.16.0 // indirect
+	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.19.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 // indirect