Merge branch 'main' into sharnoff/cleanup-git-info

neondatabase · Dec 12, 2024 · a38da75 · a38da75
2 parents 5701586 + 69163e9
commit a38da75
Show file tree

Hide file tree

Showing 10 changed files with 695 additions and 317 deletions.
diff --git a/.github/workflows/build-test-vm.yaml b/.github/workflows/build-test-vm.yaml
@@ -31,6 +31,7 @@ env:
   IMG_POSTGRES_16_BULLSEYE: "neondatabase/vm-postgres-16-bullseye"
   # using image built in the same workflow
   IMG_DAEMON: "neondatabase/neonvm-daemon"
+  TARGET_ARCH: "amd64"
 
 defaults:
   run:
@@ -87,7 +88,7 @@ jobs:
 
       - name: build ${{ needs.tags.outputs.vm-postgres-16-bullseye }}
         run: |
-          ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ needs.tags.outputs.daemon }}
+          ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ needs.tags.outputs.daemon }} -target-arch linux/${TARGET_ARCH}
       - name: docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }}
         run: |
           docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }}
diff --git a/.github/workflows/vm-example.yaml b/.github/workflows/vm-example.yaml
@@ -8,6 +8,9 @@ on:
       - ".github/workflows/vm-example.yaml"
   workflow_dispatch: # adds ability to run this manually
 
+env:
+  TARGET_ARCH: amd64
+
 jobs:
   vm-example:
     runs-on: ubuntu-latest
@@ -37,26 +40,26 @@ jobs:
           password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
 
       - name: build vm-alpine:3.16
-        run:  bin/vm-builder -src alpine:3.16 -dst neondatabase/vm-alpine:3.16
+        run:  bin/vm-builder -src alpine:3.16 -dst neondatabase/vm-alpine:3.16 -target-arch linux/${TARGET_ARCH}
       - name: push vm-alpine:3.16
         run:  docker push -q neondatabase/vm-alpine:3.16
 
       - name: build vm-ubuntu:22.04
-        run:  bin/vm-builder -src ubuntu:22.04 -dst neondatabase/vm-ubuntu:22.04
+        run:  bin/vm-builder -src ubuntu:22.04 -dst neondatabase/vm-ubuntu:22.04 -target-arch linux/${TARGET_ARCH}
       - name: push  vm-ubuntu:22.04
         run:  docker push -q neondatabase/vm-ubuntu:22.04
 
       - name: build vm-debian:11
-        run:  bin/vm-builder -src debian:11 -dst neondatabase/vm-debian:11
+        run:  bin/vm-builder -src debian:11 -dst neondatabase/vm-debian:11 -target-arch linux/${TARGET_ARCH}
       - name: push  vm-debian:11
         run:  docker push -q neondatabase/vm-debian:11
 
       - name: build vm-postgres:14-alpine
-        run:  bin/vm-builder -src postgres:14-alpine -dst neondatabase/vm-postgres:14-alpine
+        run:  bin/vm-builder -src postgres:14-alpine -dst neondatabase/vm-postgres:14-alpine -target-arch linux/${TARGET_ARCH}
       - name: push vm-postgres:14-alpine
         run:  docker push -q neondatabase/vm-postgres:14-alpine
 
       - name: build vm-postgres:15-alpine
-        run:  bin/vm-builder -src postgres:15-alpine -dst neondatabase/vm-postgres:15-alpine
+        run:  bin/vm-builder -src postgres:15-alpine -dst neondatabase/vm-postgres:15-alpine -target-arch linux/${TARGET_ARCH}
       - name: push vm-postgres:15-alpine
         run:  docker push -q neondatabase/vm-postgres:15-alpine
diff --git a/Makefile b/Makefile
@@ -20,7 +20,7 @@ GOOS ?= $(shell go env GOOS)
 # The target architecture for linux kernel. Possible values: amd64 or arm64.
 # Any other supported by linux kernel architecture could be added by introducing new build step into neonvm/hack/kernel/Dockerfile.kernel-builder
 KERNEL_TARGET_ARCH ?= amd64
-
+TARGET_ARCH ?= amd64
 # Get the currently used golang base path
 GOPATH=$(shell go env GOPATH)
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
@@ -196,6 +196,7 @@ docker-build-runner: docker-build-go-base ## Build docker image for NeonVM runne
 docker-build-daemon: docker-build-go-base ## Build docker image for NeonVM daemon.
 	docker build \
 		--tag $(IMG_DAEMON) \
+		--build-arg TARGET_ARCH=$(TARGET_ARCH) \
 		--file neonvm-daemon/Dockerfile \
 		.
 
@@ -228,11 +229,11 @@ docker-build-scheduler: docker-build-go-base ## Build docker image for (autoscal
 
 .PHONY: docker-build-examples
 docker-build-examples: bin/vm-builder ## Build docker images for testing VMs
-	./bin/vm-builder -src postgres:15-bullseye -dst $(E2E_TESTS_VM_IMG) -spec tests/e2e/image-spec.yaml
+	./bin/vm-builder -src postgres:15-bullseye -dst $(E2E_TESTS_VM_IMG) -spec tests/e2e/image-spec.yaml -target-arch linux/$(TARGET_ARCH)
 
 .PHONY: docker-build-pg16-disk-test
 docker-build-pg16-disk-test: bin/vm-builder ## Build a VM image for testing
-	./bin/vm-builder -src alpine:3.19 -dst $(PG16_DISK_TEST_IMG) -spec vm-examples/pg16-disk-test/image-spec.yaml
+	./bin/vm-builder -src alpine:3.19 -dst $(PG16_DISK_TEST_IMG) -spec vm-examples/pg16-disk-test/image-spec.yaml -target-arch linux/$(TARGET_ARCH)
 
 #.PHONY: docker-push
 #docker-push: ## Push docker image with the controller.
@@ -394,6 +395,10 @@ load-example-vms: check-local-context kubectl kind k3d ## Load the testing VM im
 .PHONY: example-vms
 example-vms: docker-build-examples load-example-vms ## Build and push the testing VM images to the kind/k3d cluster.
 
+.PHONY: example-vms-arm64
+example-vms-arm64: TARGET_ARCH=arm64
+example-vms-arm64: example-vms 
+
 .PHONY: load-pg16-disk-test
 load-pg16-disk-test: check-local-context kubectl kind k3d ## Load the pg16-disk-test VM image to the kind/k3d cluster.
 	@if [ $$($(KUBECTL) config current-context) = k3d-$(CLUSTER_NAME) ]; then $(K3D) image import $(PG16_DISK_TEST_IMG) --cluster $(CLUSTER_NAME) --mode direct; fi
@@ -493,7 +498,13 @@ CODE_GENERATOR_VERSION ?= v0.28.12
 KUTTL ?= $(LOCALBIN)/kuttl
 # k8s deps @ 1.28.3
 KUTTL_VERSION ?= v0.16.0
-
+ifeq ($(GOARCH), arm64)
+    KUTTL_ARCH = arm64
+else ifeq ($(GOARCH), amd64)
+    KUTTL_ARCH = x86_64
+else
+    $(error Unsupported architecture: $(GOARCH))
+endif
 KUBECTL ?= $(LOCALBIN)/kubectl
 KUBECTL_VERSION ?= v1.29.10
 
@@ -535,7 +546,7 @@ $(KUBECTL): $(LOCALBIN)
 .PHONY: kuttl
 kuttl: $(KUTTL) ## Download kuttl locally if necessary.
 $(KUTTL): $(LOCALBIN)
-	@test -s $(LOCALBIN)/kuttl || { curl -sfSLo $(KUTTL) https://github.com/kudobuilder/kuttl/releases/download/$(KUTTL_VERSION)/kubectl-kuttl_$(subst v,,$(KUTTL_VERSION))_$(GOOS)_$(shell uname -m) && chmod +x $(KUTTL); }
+	test -s $(LOCALBIN)/kuttl || { curl -sfSLo $(KUTTL) https://github.com/kudobuilder/kuttl/releases/download/$(KUTTL_VERSION)/kubectl-kuttl_$(subst v,,$(KUTTL_VERSION))_$(GOOS)_$(KUTTL_ARCH) && chmod +x $(KUTTL); }
 
 .PHONY: k3d
 k3d: $(K3D) ## Download k3d locally if necessary.

diff --git a/neonvm-kernel/README.md b/neonvm-kernel/README.md
@@ -29,17 +29,17 @@ with the following sequence of actions:
 2. Then, inside the container, run:
    ```sh
    cd linux-$NEW_VERSION
-   cp /host/linux-config-amd64-6.1.92 .config # Copy current config in
+   cp /host/linux-config-amd64-6.6.64 .config # Copy current config in
    make menuconfig ARCH=x86_64
    # do nothing; just save and exit, overwriting .config
    cp .config /host/linux-config-amd64-$NEW_VERSION # NOTE: Different from existing!
    ```
 3. Back on the host, finish with:
    ```sh
    # compare the two versions
-   diff linux-config-amd64-6.1.92 linux-config-amd64-$NEW_VERSION
+   diff linux-config-amd64-6.6.64 linux-config-amd64-$NEW_VERSION
    # If all looks good, delete the old version. This is required so auto-selection works.
-   rm linux-config-amd64-6.1.92
+   rm linux-config-amd64-6.6.64
    ```
 
 ### On arm64 (aarch64 ARM)
@@ -53,17 +53,17 @@ with the following sequence of actions:
 2. Then, inside the container, run:
    ```sh
    cd linux-$NEW_VERSION
-   cp /host/linux-config-aarch64-6.1.92 .config # Copy current config in
+   cp /host/linux-config-aarch64-6.6.64 .config # Copy current config in
    make menuconfig ARCH=arm64
    # do nothing; just save and exit, overwriting .config
    cp .config /host/linux-config-aarch64-$NEW_VERSION # NOTE: Different from existing!
    ```
 3. Back on the host, finish with:
    ```sh
    # compare the two versions
-   diff linux-config-aarch64-6.1.92 linux-config-aarch64-$NEW_VERSION
+   diff linux-config-aarch64-6.6.64 linux-config-aarch64-$NEW_VERSION
    # If all looks good, delete the old version. This is required so auto-selection works.
-   rm linux-config-aarch64-6.1.92
+   rm linux-config-aarch64-6.6.64
    ```
 
 Afterwards, it's probably also good to do a search-and-replace repo-wide to update all places that
@@ -74,7 +74,7 @@ mention the old kernel version.
 To adjust the kernel config, try the following from this directory:
 
 ```sh
-docker build --build-arg KERNEL_VERSION=6.1.92 --platform linux/x86_64 --target build-deps -t kernel-build-deps -f Dockerfile.kernel-builder .
+docker build --build-arg KERNEL_VERSION=6.6.64 --platform linux/x86_64 --target build-deps -t kernel-build-deps -f Dockerfile.kernel-builder .
 docker run --rm -v $PWD:/host --name kernel-build -it kernel-build-deps bash
 # inside that bash shell, do the menuconfig, then copy-out the config to /host
 ```