Skip to content

Commit

Permalink
add clusterclass controller image building and tilt deployment
Browse files Browse the repository at this point in the history 
Signed-off-by: Carlos Salas <[email protected]>
  • Loading branch information
@salasberryfin
salasberryfin committed Dec 20, 2024
1 parent 79dcd2c commit 6ee4727
Show file tree
Hide file tree
Showing 4 changed files with 183 additions and 62 deletions.
79 changes: 79 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ TEST_DIR := test
TOOLS_DIR := hack/tools
TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR))
EXP_ETCDRESTORE_DIR := exp/etcdrestore
EXP_CLUSTERCLASS_DIR := exp/clusterclass

$(TOOLS_BIN_DIR):
mkdir -p $@
Expand Down Expand Up @@ -184,6 +185,14 @@ CONTROLLER_IMG ?= $(REGISTRY)/$(ORG)/$(CONTROLLER_IMAGE_NAME)
CONTROLLER_IMAGE_VERSION ?= $(shell git describe --abbrev=0 2>/dev/null)
IID_FILE ?= $(shell mktemp)

# etcdrestore
ETCDRESTORE_IMAGE_NAME ?= turtles-etcd-snapshot-restore
ETCDRESTORE_IMG ?= $(REGISTRY)/$(ORG)/$(ETCDRESTORE_IMAGE_NAME)

# clusterclass
CLUSTERCLASS_IMAGE_NAME ?= turtles-clusterclass-operations
CLUSTERCLASS_IMG ?= $(REGISTRY)/$(ORG)/$(CLUSTERCLASS_IMAGE_NAME)

# Release
# Exclude tags with the prefix 'test/'
RELEASE_TAG ?= $(shell git describe --abbrev=0 --exclude 'test/*' 2>/dev/null)
Expand Down Expand Up @@ -261,11 +270,19 @@ generate-exp-etcdrestore-manifests-api: controller-gen ## Generate ClusterRole a
output:webhook:dir=./exp/etcdrestore/config/webhook \
webhook

.PHONY: generate-exp-clusterclass-manifests-api
generate-exp-clusterclass-manifests-api: controller-gen ## Generate ClusterRole and CustomResourceDefinition objects for experimental API.
$(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./exp/etcdrestore/api/v1alpha1/..." \
paths=./exp/clusterclass/internal/controller/... \
output:crd:artifacts:config=./exp/clusterclass/config/crd/bases \
output:rbac:dir=./exp/clusterclass/config/rbac

.PHONY: generate-modules
generate-modules: ## Run go mod tidy to ensure modules are up to date
go mod tidy
cd $(TEST_DIR); go mod tidy
cd $(EXP_ETCDRESTORE_DIR); go mod tidy
cd $(EXP_CLUSTERCLASS_DIR); go mod tidy

.PHONY: generate-go-deepcopy
generate-go-deepcopy: ## Run deepcopy generation
Expand Down Expand Up @@ -332,6 +349,10 @@ test: $(SETUP_ENVTEST) manifests test-exp-etcdrestore ## Run all generators and
test-exp-etcdrestore: $(SETUP_ENVTEST) ## Run tests for experimental etcdrestore API.
cd $(EXP_ETCDRESTORE_DIR); KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... $(TEST_ARGS)

.PHONY: test-exp-clusterclass
test-exp-clusterclass: $(SETUP_ENVTEST) ## Run tests for experimental clusterclass API.
cd $(EXP_CLUSTERCLASS_DIR); KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... $(TEST_ARGS)

##@ Build

.PHONY: build
Expand Down Expand Up @@ -384,6 +405,64 @@ docker-build-and-push: buildx-machine docker-pull-prerequisites ## Run docker-bu
--build-arg package=. \
--build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG):$(TAG)

## --------------------------------------
## Docker - etcdrestore
## --------------------------------------

.PHONY: docker-build-etcdrestore ## Build the docker image for etcdrestore
docker-build-etcdrestore: buildx-machine docker-pull-prerequisites ## Build docker image for a specific architecture
## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368)
# buildx does not support using local registry for multi-architecture images
cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \
--platform $(ARCH) \
--load \
--build-arg builder_image=$(GO_CONTAINER_IMAGE) \
--build-arg goproxy=$(GOPROXY) \
--build-arg package=./exp/etcdrestore \
--build-arg ldflags="$(LDFLAGS)" . -t $(ETCDRESTORE_IMG):$(TAG) --file - --progress=plain

.PHONY: docker-build-and-push-etcdrestore
docker-build-and-push-etcdrestore: buildx-machine docker-pull-prerequisites ## Run docker-build-and-push-etcdrestore targets for all architectures
cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \
--platform $(TARGET_PLATFORMS) \
--push \
--sbom=true \
--attest type=provenance,mode=max \
--iidfile=$(IID_FILE) \
--build-arg builder_image=$(GO_CONTAINER_IMAGE) \
--build-arg goproxy=$(GOPROXY) \
--build-arg package=./exp/etcdrestore \
--build-arg ldflags="$(LDFLAGS)" . -t $(ETCDRESTORE_IMG):$(TAG) --file - --progress=plain

## --------------------------------------
## Docker - clusterclass
## --------------------------------------

.PHONY: docker-build-clusterclass ## Build the docker image for clusterclass
docker-build-clusterclass: buildx-machine docker-pull-prerequisites ## Build docker image for a specific architecture
## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368)
# buildx does not support using local registry for multi-architecture images
cat $(EXP_CLUSTERCLASS_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \
--platform $(ARCH) \
--load \
--build-arg builder_image=$(GO_CONTAINER_IMAGE) \
--build-arg goproxy=$(GOPROXY) \
--build-arg package=./exp/clusterclass \
--build-arg ldflags="$(LDFLAGS)" . -t $(CLUSTERCLASS_IMG):$(TAG) --file - --progress=plain

.PHONY: docker-build-and-push-clusterclass
docker-build-and-push-clusterclass: buildx-machine docker-pull-prerequisites ## Run docker-build-and-push-clusterclass targets for all architectures
cat $(EXP_CLUSTERCLASS_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \
--platform $(TARGET_PLATFORMS) \
--push \
--sbom=true \
--attest type=provenance,mode=max \
--iidfile=$(IID_FILE) \
--build-arg builder_image=$(GO_CONTAINER_IMAGE) \
--build-arg goproxy=$(GOPROXY) \
--build-arg package=./exp/clusterclass \
--build-arg ldflags="$(LDFLAGS)" . -t $(CLUSTERCLASS_IMG):$(TAG) --file - --progress=plain

docker-list-all:
@echo $(CONTROLLER_IMG):${TAG}

Expand Down
16 changes: 14 additions & 2 deletions Tiltfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ if settings.get("trigger_mode") == "manual":
if settings.get("default_registry") != "":
default_registry(settings.get("default_registry"))

always_enable_projects = ["turtles", "turtles-capiproviders", "turtles-etcdsnapshotrestore"]
always_enable_projects = ["turtles", "turtles-capiproviders", "turtles-etcdsnapshotrestore", "turtles-clusterclass-operations"]

projects = {
"turtles": {
Expand Down Expand Up @@ -74,6 +74,18 @@ projects = {
"kustomize_dir": "config/capiproviders",
"label": "turtles-capiproviders",
"op": "apply"
},
"turtles-clusterclass-operations": {
"context": "exp/clusterclass",
"image": "ghcr.io/rancher/turtles-clusterclass-operations:dev",
"live_reload_deps": [
"main.go",
"go.mod",
"go.sum",
"internal/",
],
"kustomize_dir": "config/default",
"label": "turtles-clusterclass-operations"
}
}

Expand Down Expand Up @@ -101,4 +113,4 @@ def get_projects():

include_user_tilt_files()

enable_projects()
enable_projects()
47 changes: 28 additions & 19 deletions exp/clusterclass/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,37 +1,46 @@
# syntax=docker/dockerfile:1.4

# Copyright 2024 SUSE LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Build the manager binary
ARG builder_image

# Build architecture
ARG ARCH

FROM ${builder_image} as builder
WORKDIR /workspace

# Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy
ARG goproxy=https://proxy.golang.org
# Run this with docker build --build-arg package=./exp/clusterclass
ENV GOPROXY=$goproxy

COPY go.mod go.mod
COPY go.sum go.sum

RUN --mount=type=cache,target=/go/pkg/mod \
go mod download

# Copy the go source
COPY ./ ./

ARG package=.
ARG ARCH
# Build
ARG ldflags

RUN --mount=type=cache,target=/go/pkg/mod \
CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \
go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \
-o manager ${package}
# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder
RUN --mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg/mod \
CGO_ENABLED=0 GOOS=linux \
sh -c "cd exp/clusterclass && ls && go build -trimpath -ldflags \"${ldflags} -extldflags '-static'\" -o manager ${package}"

FROM gcr.io/distroless/static:nonroot-${ARCH}
LABEL org.opencontainers.image.source=https://github.com/rancher/turtles/exp/clusterclass
# Use distroless as minimal base image to package the manager binary
# Refer to https://github.com/GoogleContainerTools/distroless for more details
FROM gcr.io/distroless/static:nonroot
WORKDIR /
COPY --from=builder /workspace/manager .
COPY --from=builder /workspace/exp/clusterclass/manager .
# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies
USER 65532
ENTRYPOINT ["/manager"]
103 changes: 62 additions & 41 deletions scripts/turtles-dev.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,18 @@

RANCHER_HOSTNAME=$1
if [ -z "$RANCHER_HOSTNAME" ]; then
echo "You must pass a rancher host name"
exit 1
echo "You must pass a rancher host name"
exit 1
fi

RANCHER_VERSION=${RANCHER_VERSION:-v2.10.0}
CAPI_VERSION=${CAPI_VERSION:-v1.8.5}
CAPRKE2_VERSION=${CAPRKE2_VERSION:-v0.9.0}
CLUSTER_NAME=${CLUSTER_NAME:-capi-test}
ETCD_CONTROLLER_IMAGE=${ETCD_CONTROLLER_IMAGE:-ghcr.io/rancher/turtles}
ETCD_CONTROLLER_IMAGE_TAG=${ETCD_CONTROLLER_IMAGE_TAG:-dev}
CC_CONTROLLER_IMAGE=${CC_CONTROLLER_IMAGE:-ghcr.io/rancher/turtles-clusterclass-operations}
CC_CONTROLLER_IMAGE_TAG=${CC_CONTROLLER_IMAGE_TAG:-dev}
USE_TILT_DEV=${USE_TILT_DEV:-true}

BASEDIR=$(dirname "$0")
Expand All @@ -39,61 +43,78 @@ helm repo add ngrok https://charts.ngrok.com
helm repo update

helm install cert-manager jetstack/cert-manager \
--namespace cert-manager \
--create-namespace \
--set crds.enabled=true
--namespace cert-manager \
--create-namespace \
--set crds.enabled=true

export EXP_CLUSTER_RESOURCE_SET=true
export CLUSTER_TOPOLOGY=true

helm install capi-operator capi-operator/cluster-api-operator \
--create-namespace -n capi-operator-system \
--set infrastructure=docker:$CAPI_VERSION \
--set core=cluster-api:$CAPI_VERSION \
--set controlPlane=rke2:$CAPRKE2_VERSION \
--set bootstrap=rke2:$CAPRKE2_VERSION \
--timeout 90s --wait

kubectl rollout status deployment capi-operator-cluster-api-operator -n capi-operator-system --timeout=180s

helm upgrade ngrok ngrok/kubernetes-ingress-controller \
--install \
--wait \
--timeout 5m \
--set credentials.apiKey=$NGROK_API_KEY \
--set credentials.authtoken=$NGROK_AUTHTOKEN
--install \
--wait \
--timeout 5m \
--set credentials.apiKey=$NGROK_API_KEY \
--set credentials.authtoken=$NGROK_AUTHTOKEN

kubectl apply -f test/e2e/data/rancher/ingress-class-patch.yaml

helm install rancher rancher-latest/rancher \
--namespace cattle-system \
--create-namespace \
--set bootstrapPassword=rancheradmin \
--set replicas=1 \
--set hostname="$RANCHER_HOSTNAME" \
--version="$RANCHER_VERSION" \
--wait
--namespace cattle-system \
--create-namespace \
--set bootstrapPassword=rancheradmin \
--set replicas=1 \
--set hostname="$RANCHER_HOSTNAME" \
--version="$RANCHER_VERSION" \
--wait

kubectl rollout status deployment rancher -n cattle-system --timeout=180s

kubectl apply -f test/e2e/data/rancher/rancher-service-patch.yaml
envsubst < test/e2e/data/rancher/rancher-setting-patch.yaml | kubectl apply -f -
envsubst <test/e2e/data/rancher/rancher-setting-patch.yaml | kubectl apply -f -

# Install the locally build chart of Rancher Turtles
install_local_rancher_turtles_chart() {
# Remove the previous chart directory
rm -rf out
# Build the chart locally
make build-chart
# Build the etcdrestore controller image
make docker-build
# Load the etcdrestore controller image into the kind cluster
kind load docker-image $ETCD_CONTROLLER_IMAGE:$ETCD_CONTROLLER_IMAGE_TAG --name $CLUSTER_NAME
# Install the Rancher Turtles using a local chart with 'etcd-snapshot-restore' feature flag enabled
# to run etcdrestore controller
helm upgrade --install rancher-turtles out/charts/rancher-turtles \
-n rancher-turtles-system \
--set cluster-api-operator.enabled=true \
--set cluster-api-operator.cluster-api.enabled=false \
--set rancherTurtles.features.etcd-snapshot-restore.enabled=true \
--set rancherTurtles.features.etcd-snapshot-restore.imageVersion=dev \
--set rancherTurtles.imageVersion=dev \
--dependency-update \
--create-namespace --wait \
--timeout 180s
# Remove the previous chart directory
rm -rf out
# Build the chart locally
make build-chart
# Build the etcdrestore controller image
make docker-build
# Load the etcdrestore controller image into the kind cluster
kind load docker-image $ETCD_CONTROLLER_IMAGE:$ETCD_CONTROLLER_IMAGE_TAG --name $CLUSTER_NAME
# Build the clusterclass controller image
make docker-build-clusterclass
# Load the clusterclass controller image into the kind cluster
kind load docker-image $CC_CONTROLLER_IMAGE:$CC_CONTROLLER_IMAGE_TAG --name $CLUSTER_NAME
# Install the Rancher Turtles using a local chart with 'etcd-snapshot-restore' feature flag enabled
# to run etcdrestore controller
helm upgrade --install rancher-turtles out/charts/rancher-turtles \
-n rancher-turtles-system \
--set cluster-api-operator.enabled=true \
--set cluster-api-operator.cluster-api.enabled=false \
--set rancherTurtles.features.etcd-snapshot-restore.enabled=true \
--set rancherTurtles.features.etcd-snapshot-restore.imageVersion=dev \
--set rancherTurtles.imageVersion=dev \
--dependency-update \
--create-namespace --wait \
--timeout 180s
}

echo "Installing local Rancher Turtles chart for development..."
install_local_rancher_turtles_chart

if [ "$USE_TILT_DEV" == "true" ]; then
echo "Using Tilt for development..."
tilt up
fi
echo "Using Tilt for development..."
tilt up
fi

0 comments on commit 6ee4727

Please sign in to comment.