neonvm-builder: arm images (#1116)

Modify neonvm-builder to support building images with both amd64 or arm64 architectures. Change busybox loader to use public image busybox:1.35.0-musl instead of downloading busybox from busybox.net Parametrize vector url to be architecture specific. Has no cross-build support, produced image architecture is the same as host machine architecture. #1081 --------- Signed-off-by: Misha Sakhnov <[email protected]>
neondatabase · Dec 10, 2024 · 5b1fb31 · 5b1fb31
1 parent 9299d53
commit 5b1fb31
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 22 deletions.
diff --git a/.github/workflows/build-test-vm.yaml b/.github/workflows/build-test-vm.yaml
@@ -31,6 +31,7 @@ env:
   IMG_POSTGRES_16_BULLSEYE: "neondatabase/vm-postgres-16-bullseye"
   # using image built in the same workflow
   IMG_DAEMON: "neondatabase/neonvm-daemon"
+  TARGET_ARCH: "amd64"
 
 defaults:
   run:
@@ -87,7 +88,7 @@ jobs:
 
       - name: build ${{ needs.tags.outputs.vm-postgres-16-bullseye }}
         run: |
-          ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ needs.tags.outputs.daemon }}
+          ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ needs.tags.outputs.daemon }} -target-arch linux/${TARGET_ARCH}
       - name: docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }}
         run: |
           docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }}
diff --git a/.github/workflows/vm-example.yaml b/.github/workflows/vm-example.yaml
@@ -8,6 +8,9 @@ on:
       - ".github/workflows/vm-example.yaml"
   workflow_dispatch: # adds ability to run this manually
 
+env:
+  TARGET_ARCH: amd64
+
 jobs:
   vm-example:
     runs-on: ubuntu-latest
@@ -37,26 +40,26 @@ jobs:
           password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
 
       - name: build vm-alpine:3.16
-        run:  bin/vm-builder -src alpine:3.16 -dst neondatabase/vm-alpine:3.16
+        run:  bin/vm-builder -src alpine:3.16 -dst neondatabase/vm-alpine:3.16 -target-arch linux/${TARGET_ARCH}
       - name: push vm-alpine:3.16
         run:  docker push -q neondatabase/vm-alpine:3.16
 
       - name: build vm-ubuntu:22.04
-        run:  bin/vm-builder -src ubuntu:22.04 -dst neondatabase/vm-ubuntu:22.04
+        run:  bin/vm-builder -src ubuntu:22.04 -dst neondatabase/vm-ubuntu:22.04 -target-arch linux/${TARGET_ARCH}
       - name: push  vm-ubuntu:22.04
         run:  docker push -q neondatabase/vm-ubuntu:22.04
 
       - name: build vm-debian:11
-        run:  bin/vm-builder -src debian:11 -dst neondatabase/vm-debian:11
+        run:  bin/vm-builder -src debian:11 -dst neondatabase/vm-debian:11 -target-arch linux/${TARGET_ARCH}
       - name: push  vm-debian:11
         run:  docker push -q neondatabase/vm-debian:11
 
       - name: build vm-postgres:14-alpine
-        run:  bin/vm-builder -src postgres:14-alpine -dst neondatabase/vm-postgres:14-alpine
+        run:  bin/vm-builder -src postgres:14-alpine -dst neondatabase/vm-postgres:14-alpine -target-arch linux/${TARGET_ARCH}
       - name: push vm-postgres:14-alpine
         run:  docker push -q neondatabase/vm-postgres:14-alpine
 
       - name: build vm-postgres:15-alpine
-        run:  bin/vm-builder -src postgres:15-alpine -dst neondatabase/vm-postgres:15-alpine
+        run:  bin/vm-builder -src postgres:15-alpine -dst neondatabase/vm-postgres:15-alpine -target-arch linux/${TARGET_ARCH}
       - name: push vm-postgres:15-alpine
         run:  docker push -q neondatabase/vm-postgres:15-alpine
diff --git a/Makefile b/Makefile
@@ -20,7 +20,7 @@ GOOS ?= $(shell go env GOOS)
 # The target architecture for linux kernel. Possible values: amd64 or arm64.
 # Any other supported by linux kernel architecture could be added by introducing new build step into neonvm/hack/kernel/Dockerfile.kernel-builder
 KERNEL_TARGET_ARCH ?= amd64
-
+TARGET_ARCH ?= amd64
 # Get the currently used golang base path
 GOPATH=$(shell go env GOPATH)
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
@@ -196,6 +196,7 @@ docker-build-runner: docker-build-go-base ## Build docker image for NeonVM runne
 docker-build-daemon: docker-build-go-base ## Build docker image for NeonVM daemon.
 	docker build \
 		--tag $(IMG_DAEMON) \
+		--build-arg TARGET_ARCH=$(TARGET_ARCH) \
 		--file neonvm-daemon/Dockerfile \
 		.
 
@@ -228,11 +229,11 @@ docker-build-scheduler: docker-build-go-base ## Build docker image for (autoscal
 
 .PHONY: docker-build-examples
 docker-build-examples: bin/vm-builder ## Build docker images for testing VMs
-	./bin/vm-builder -src postgres:15-bullseye -dst $(E2E_TESTS_VM_IMG) -spec tests/e2e/image-spec.yaml
+	./bin/vm-builder -src postgres:15-bullseye -dst $(E2E_TESTS_VM_IMG) -spec tests/e2e/image-spec.yaml -target-arch linux/$(TARGET_ARCH)
 
 .PHONY: docker-build-pg16-disk-test
 docker-build-pg16-disk-test: bin/vm-builder ## Build a VM image for testing
-	./bin/vm-builder -src alpine:3.19 -dst $(PG16_DISK_TEST_IMG) -spec vm-examples/pg16-disk-test/image-spec.yaml
+	./bin/vm-builder -src alpine:3.19 -dst $(PG16_DISK_TEST_IMG) -spec vm-examples/pg16-disk-test/image-spec.yaml -target-arch linux/$(TARGET_ARCH)
 
 #.PHONY: docker-push
 #docker-push: ## Push docker image with the controller.
@@ -394,6 +395,10 @@ load-example-vms: check-local-context kubectl kind k3d ## Load the testing VM im
 .PHONY: example-vms
 example-vms: docker-build-examples load-example-vms ## Build and push the testing VM images to the kind/k3d cluster.
 
+.PHONY: example-vms-arm64
+example-vms-arm64: TARGET_ARCH=arm64
+example-vms-arm64: example-vms 
+
 .PHONY: load-pg16-disk-test
 load-pg16-disk-test: check-local-context kubectl kind k3d ## Load the pg16-disk-test VM image to the kind/k3d cluster.
 	@if [ $$($(KUBECTL) config current-context) = k3d-$(CLUSTER_NAME) ]; then $(K3D) image import $(PG16_DISK_TEST_IMG) --cluster $(CLUSTER_NAME) --mode direct; fi
@@ -493,7 +498,13 @@ CODE_GENERATOR_VERSION ?= v0.28.12
 KUTTL ?= $(LOCALBIN)/kuttl
 # k8s deps @ 1.28.3
 KUTTL_VERSION ?= v0.16.0
-
+ifeq ($(GOARCH), arm64)
+    KUTTL_ARCH = arm64
+else ifeq ($(GOARCH), amd64)
+    KUTTL_ARCH = x86_64
+else
+    $(error Unsupported architecture: $(GOARCH))
+endif
 KUBECTL ?= $(LOCALBIN)/kubectl
 KUBECTL_VERSION ?= v1.29.10
 
@@ -535,7 +546,7 @@ $(KUBECTL): $(LOCALBIN)
 .PHONY: kuttl
 kuttl: $(KUTTL) ## Download kuttl locally if necessary.
 $(KUTTL): $(LOCALBIN)
-	@test -s $(LOCALBIN)/kuttl || { curl -sfSLo $(KUTTL) https://github.com/kudobuilder/kuttl/releases/download/$(KUTTL_VERSION)/kubectl-kuttl_$(subst v,,$(KUTTL_VERSION))_$(GOOS)_$(shell uname -m) && chmod +x $(KUTTL); }
+	test -s $(LOCALBIN)/kuttl || { curl -sfSLo $(KUTTL) https://github.com/kudobuilder/kuttl/releases/download/$(KUTTL_VERSION)/kubectl-kuttl_$(subst v,,$(KUTTL_VERSION))_$(GOOS)_$(KUTTL_ARCH) && chmod +x $(KUTTL); }
 
 .PHONY: k3d
 k3d: $(K3D) ## Download k3d locally if necessary.

diff --git a/vm-builder/files/Dockerfile.img b/vm-builder/files/Dockerfile.img
@@ -9,14 +9,17 @@ USER root
 
 FROM {{.NeonvmDaemonImage}} AS neonvm-daemon-loader
 
+FROM busybox:1.35.0-musl AS busybox-loader
+
 FROM alpine:3.19 AS vm-runtime
+ARG TARGET_ARCH
+RUN set -e && mkdir -p /neonvm/bin /neonvm/runtime /neonvm/config 
 # add busybox
-ENV BUSYBOX_VERSION 1.35.0
+COPY --from=busybox-loader /bin/busybox /neonvm/bin/busybox
+
 RUN set -e \
-	&& mkdir -p /neonvm/bin /neonvm/runtime /neonvm/config \
-	&& wget -q https://busybox.net/downloads/binaries/${BUSYBOX_VERSION}-x86_64-linux-musl/busybox -O /neonvm/bin/busybox \
-	&& chmod +x /neonvm/bin/busybox \
-	&& /neonvm/bin/busybox --install -s /neonvm/bin
+    chmod +x /neonvm/bin/busybox \
+    && /neonvm/bin/busybox --install -s /neonvm/bin 
 
 COPY helper.move-bins.sh /helper.move-bins.sh
 
@@ -49,8 +52,10 @@ RUN set -e \
 
 # Install vector.dev binary
 RUN set -e \
-    && wget https://packages.timber.io/vector/0.26.0/vector-0.26.0-x86_64-unknown-linux-musl.tar.gz -O - \
-    | tar xzvf - --strip-components 3 -C /neonvm/bin/ ./vector-x86_64-unknown-linux-musl/bin/vector
+    && ARCH=$( [ "$TARGET_ARCH" = "linux/arm64" ] && echo "aarch64" || echo "x86_64") \
+    && wget https://packages.timber.io/vector/0.26.0/vector-${ARCH}-unknown-linux-musl.tar.gz -O - \
+    | tar xzvf - --strip-components 3 -C /neonvm/bin/ ./vector-${ARCH}-unknown-linux-musl/bin/vector
+
 
 # chrony
 RUN set -e \

diff --git a/vm-builder/files/inittab b/vm-builder/files/inittab
@@ -12,5 +12,5 @@
 {{ range .InittabCommands }}
 ::{{.SysvInitAction}}:su -p {{.CommandUser}} -c {{.ShellEscapedCommand}}
 {{ end }}
-ttyS0::respawn:/neonvm/bin/agetty --8bits --local-line --noissue --noclear --noreset --host console --login-program /neonvm/bin/login --login-pause --autologin root 115200 ttyS0 linux
+{{ .AgettyTTY }}::respawn:/neonvm/bin/agetty --8bits --local-line --noissue --noclear --noreset --host console --login-program /neonvm/bin/login --login-pause --autologin root 115200 {{ .AgettyTTY }} linux
 ::shutdown:/neonvm/bin/vmshutdown
diff --git a/vm-builder/main.go b/vm-builder/main.go
@@ -58,6 +58,11 @@ var (
 	configSshd string
 )
 
+const (
+	targetArchLinuxAmd64 = "linux/amd64"
+	targetArchLinuxArm64 = "linux/arm64"
+)
+
 var (
 	Version           string
 	NeonvmDaemonImage string
@@ -72,6 +77,7 @@ var (
 	version   = flag.Bool("version", false, `Print vm-builder version`)
 
 	daemonImageFlag = flag.String("daemon-image", "", `Specify the neonvm-daemon image: --daemon-image=neonvm-daemon:dev`)
+	targetArch      = flag.String("target-arch", "", fmt.Sprintf("Target architecture: --arch %s | %s", targetArchLinuxAmd64, targetArchLinuxArm64))
 )
 
 func AddTemplatedFileToTar(tw *tar.Writer, tmplArgs any, filename string, tmplString string) error {
@@ -84,7 +90,6 @@ func AddTemplatedFileToTar(tw *tar.Writer, tmplArgs any, filename string, tmplSt
 	if err = tmpl.Execute(&buf, tmplArgs); err != nil {
 		return fmt.Errorf("failed to execute template for %q: %w", filename, err)
 	}
-
 	return addFileToTar(tw, filename, buf.Bytes())
 }
 
@@ -117,6 +122,7 @@ type TemplatesContext struct {
 	SpecBuild       string
 	SpecMerge       string
 	InittabCommands []inittabCommand
+	AgettyTTY       string
 	ShutdownHook    string
 }
 
@@ -129,18 +135,28 @@ type inittabCommand struct {
 func main() {
 	flag.Parse()
 	var dstIm string
-
 	if *version {
 		fmt.Println(Version)
 		os.Exit(0)
 	}
-
 	if len(*daemonImageFlag) == 0 && len(NeonvmDaemonImage) == 0 {
 		log.Println("neonvm-daemon image not set, needs to be explicitly passed in, or compiled with -ldflags '-X main.NeonvmDaemonImage=...'")
 		flag.PrintDefaults()
 		os.Exit(1)
 	}
 
+	if targetArch == nil || *targetArch == "" {
+		log.Println("Target architecture not set, see usage info:")
+		flag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	if *targetArch != targetArchLinuxAmd64 && *targetArch != targetArchLinuxArm64 {
+		log.Fatalf("Unsupported target architecture: %q", *targetArch)
+		flag.PrintDefaults()
+		return
+	}
+
 	neonvmDaemonImage := NeonvmDaemonImage
 	if len(*daemonImageFlag) != 0 {
 		neonvmDaemonImage = *daemonImageFlag
@@ -292,6 +308,7 @@ func main() {
 		SpecMerge:       "",  // overridden below if spec != nil
 		InittabCommands: nil, // overridden below if spec != nil
 		ShutdownHook:    "",  // overridden below if spec != nil
+		AgettyTTY:       getAgettyTTY(*targetArch),
 	}
 
 	if len(imageSpec.Config.User) != 0 {
@@ -366,6 +383,7 @@ func main() {
 
 	buildArgs := make(map[string]*string)
 	buildArgs["DISK_SIZE"] = size
+	buildArgs["TARGET_ARCH"] = targetArch
 	opt := types.ImageBuildOptions{
 		AuthConfigs:    authConfigs,
 		Tags:           []string{dstIm},
@@ -376,6 +394,7 @@ func main() {
 		Dockerfile:     "Dockerfile",
 		Remove:         true,
 		ForceRemove:    true,
+		Platform:       *targetArch,
 	}
 	buildResp, err := cli.ImageBuild(ctx, tarBuffer, opt)
 	if err != nil {
@@ -541,3 +560,16 @@ func (f file) validate() []error {
 
 	return errs
 }
+
+// getAgettyTTY returns the tty device name for agetty based on the target architecture.
+func getAgettyTTY(targetArch string) string {
+	switch targetArch {
+	case targetArchLinuxAmd64:
+		return "ttyS0"
+	case targetArchLinuxArm64:
+		return "ttyAMA0"
+	default:
+		log.Fatalf("Unsupported target architecture: %q", targetArch)
+		return ""
+	}
+}