diff --git a/.github/workflows/build-test-vm.yaml b/.github/workflows/build-test-vm.yaml index c63c5c4eb..d6d0eec87 100644 --- a/.github/workflows/build-test-vm.yaml +++ b/.github/workflows/build-test-vm.yaml @@ -31,6 +31,7 @@ env: IMG_POSTGRES_16_BULLSEYE: "neondatabase/vm-postgres-16-bullseye" # using image built in the same workflow IMG_DAEMON: "neondatabase/neonvm-daemon" + TARGET_ARCH: "amd64" defaults: run: @@ -87,7 +88,7 @@ jobs: - name: build ${{ needs.tags.outputs.vm-postgres-16-bullseye }} run: | - ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ needs.tags.outputs.daemon }} + ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ needs.tags.outputs.daemon }} -target-arch linux/${TARGET_ARCH} - name: docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }} run: | docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }} diff --git a/.github/workflows/vm-example.yaml b/.github/workflows/vm-example.yaml index 0b2387a22..3125c5587 100644 --- a/.github/workflows/vm-example.yaml +++ b/.github/workflows/vm-example.yaml @@ -8,6 +8,9 @@ on: - ".github/workflows/vm-example.yaml" workflow_dispatch: # adds ability to run this manually +env: + TARGET_ARCH: amd64 + jobs: vm-example: runs-on: ubuntu-latest @@ -37,26 +40,26 @@ jobs: password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - name: build vm-alpine:3.16 - run: bin/vm-builder -src alpine:3.16 -dst neondatabase/vm-alpine:3.16 + run: bin/vm-builder -src alpine:3.16 -dst neondatabase/vm-alpine:3.16 -target-arch linux/${TARGET_ARCH} - name: push vm-alpine:3.16 run: docker push -q neondatabase/vm-alpine:3.16 - name: build vm-ubuntu:22.04 - run: bin/vm-builder -src ubuntu:22.04 -dst neondatabase/vm-ubuntu:22.04 + run: bin/vm-builder -src ubuntu:22.04 -dst neondatabase/vm-ubuntu:22.04 -target-arch linux/${TARGET_ARCH} - name: push vm-ubuntu:22.04 run: docker push -q neondatabase/vm-ubuntu:22.04 - name: build vm-debian:11 - run: bin/vm-builder -src debian:11 -dst neondatabase/vm-debian:11 + run: bin/vm-builder -src debian:11 -dst neondatabase/vm-debian:11 -target-arch linux/${TARGET_ARCH} - name: push vm-debian:11 run: docker push -q neondatabase/vm-debian:11 - name: build vm-postgres:14-alpine - run: bin/vm-builder -src postgres:14-alpine -dst neondatabase/vm-postgres:14-alpine + run: bin/vm-builder -src postgres:14-alpine -dst neondatabase/vm-postgres:14-alpine -target-arch linux/${TARGET_ARCH} - name: push vm-postgres:14-alpine run: docker push -q neondatabase/vm-postgres:14-alpine - name: build vm-postgres:15-alpine - run: bin/vm-builder -src postgres:15-alpine -dst neondatabase/vm-postgres:15-alpine + run: bin/vm-builder -src postgres:15-alpine -dst neondatabase/vm-postgres:15-alpine -target-arch linux/${TARGET_ARCH} - name: push vm-postgres:15-alpine run: docker push -q neondatabase/vm-postgres:15-alpine diff --git a/Makefile b/Makefile index f420d01f6..2d5ff8577 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ GOOS ?= $(shell go env GOOS) # The target architecture for linux kernel. Possible values: amd64 or arm64. # Any other supported by linux kernel architecture could be added by introducing new build step into neonvm/hack/kernel/Dockerfile.kernel-builder KERNEL_TARGET_ARCH ?= amd64 - +TARGET_ARCH ?= amd64 # Get the currently used golang base path GOPATH=$(shell go env GOPATH) # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) @@ -196,6 +196,7 @@ docker-build-runner: docker-build-go-base ## Build docker image for NeonVM runne docker-build-daemon: docker-build-go-base ## Build docker image for NeonVM daemon. docker build \ --tag $(IMG_DAEMON) \ + --build-arg TARGET_ARCH=$(TARGET_ARCH) \ --file neonvm-daemon/Dockerfile \ . @@ -228,11 +229,11 @@ docker-build-scheduler: docker-build-go-base ## Build docker image for (autoscal .PHONY: docker-build-examples docker-build-examples: bin/vm-builder ## Build docker images for testing VMs - ./bin/vm-builder -src postgres:15-bullseye -dst $(E2E_TESTS_VM_IMG) -spec tests/e2e/image-spec.yaml + ./bin/vm-builder -src postgres:15-bullseye -dst $(E2E_TESTS_VM_IMG) -spec tests/e2e/image-spec.yaml -target-arch linux/$(TARGET_ARCH) .PHONY: docker-build-pg16-disk-test docker-build-pg16-disk-test: bin/vm-builder ## Build a VM image for testing - ./bin/vm-builder -src alpine:3.19 -dst $(PG16_DISK_TEST_IMG) -spec vm-examples/pg16-disk-test/image-spec.yaml + ./bin/vm-builder -src alpine:3.19 -dst $(PG16_DISK_TEST_IMG) -spec vm-examples/pg16-disk-test/image-spec.yaml -target-arch linux/$(TARGET_ARCH) #.PHONY: docker-push #docker-push: ## Push docker image with the controller. @@ -394,6 +395,10 @@ load-example-vms: check-local-context kubectl kind k3d ## Load the testing VM im .PHONY: example-vms example-vms: docker-build-examples load-example-vms ## Build and push the testing VM images to the kind/k3d cluster. +.PHONY: example-vms-arm64 +example-vms-arm64: TARGET_ARCH=arm64 +example-vms-arm64: example-vms + .PHONY: load-pg16-disk-test load-pg16-disk-test: check-local-context kubectl kind k3d ## Load the pg16-disk-test VM image to the kind/k3d cluster. @if [ $$($(KUBECTL) config current-context) = k3d-$(CLUSTER_NAME) ]; then $(K3D) image import $(PG16_DISK_TEST_IMG) --cluster $(CLUSTER_NAME) --mode direct; fi @@ -493,7 +498,13 @@ CODE_GENERATOR_VERSION ?= v0.28.12 KUTTL ?= $(LOCALBIN)/kuttl # k8s deps @ 1.28.3 KUTTL_VERSION ?= v0.16.0 - +ifeq ($(GOARCH), arm64) + KUTTL_ARCH = arm64 +else ifeq ($(GOARCH), amd64) + KUTTL_ARCH = x86_64 +else + $(error Unsupported architecture: $(GOARCH)) +endif KUBECTL ?= $(LOCALBIN)/kubectl KUBECTL_VERSION ?= v1.29.10 @@ -535,7 +546,7 @@ $(KUBECTL): $(LOCALBIN) .PHONY: kuttl kuttl: $(KUTTL) ## Download kuttl locally if necessary. $(KUTTL): $(LOCALBIN) - @test -s $(LOCALBIN)/kuttl || { curl -sfSLo $(KUTTL) https://github.com/kudobuilder/kuttl/releases/download/$(KUTTL_VERSION)/kubectl-kuttl_$(subst v,,$(KUTTL_VERSION))_$(GOOS)_$(shell uname -m) && chmod +x $(KUTTL); } + test -s $(LOCALBIN)/kuttl || { curl -sfSLo $(KUTTL) https://github.com/kudobuilder/kuttl/releases/download/$(KUTTL_VERSION)/kubectl-kuttl_$(subst v,,$(KUTTL_VERSION))_$(GOOS)_$(KUTTL_ARCH) && chmod +x $(KUTTL); } .PHONY: k3d k3d: $(K3D) ## Download k3d locally if necessary. diff --git a/vm-builder/files/Dockerfile.img b/vm-builder/files/Dockerfile.img index b72027975..0abed8262 100644 --- a/vm-builder/files/Dockerfile.img +++ b/vm-builder/files/Dockerfile.img @@ -9,14 +9,17 @@ USER root FROM {{.NeonvmDaemonImage}} AS neonvm-daemon-loader +FROM busybox:1.35.0-musl AS busybox-loader + FROM alpine:3.19 AS vm-runtime +ARG TARGET_ARCH +RUN set -e && mkdir -p /neonvm/bin /neonvm/runtime /neonvm/config # add busybox -ENV BUSYBOX_VERSION 1.35.0 +COPY --from=busybox-loader /bin/busybox /neonvm/bin/busybox + RUN set -e \ - && mkdir -p /neonvm/bin /neonvm/runtime /neonvm/config \ - && wget -q https://busybox.net/downloads/binaries/${BUSYBOX_VERSION}-x86_64-linux-musl/busybox -O /neonvm/bin/busybox \ - && chmod +x /neonvm/bin/busybox \ - && /neonvm/bin/busybox --install -s /neonvm/bin + chmod +x /neonvm/bin/busybox \ + && /neonvm/bin/busybox --install -s /neonvm/bin COPY helper.move-bins.sh /helper.move-bins.sh @@ -49,8 +52,10 @@ RUN set -e \ # Install vector.dev binary RUN set -e \ - && wget https://packages.timber.io/vector/0.26.0/vector-0.26.0-x86_64-unknown-linux-musl.tar.gz -O - \ - | tar xzvf - --strip-components 3 -C /neonvm/bin/ ./vector-x86_64-unknown-linux-musl/bin/vector + && ARCH=$( [ "$TARGET_ARCH" = "linux/arm64" ] && echo "aarch64" || echo "x86_64") \ + && wget https://packages.timber.io/vector/0.26.0/vector-${ARCH}-unknown-linux-musl.tar.gz -O - \ + | tar xzvf - --strip-components 3 -C /neonvm/bin/ ./vector-${ARCH}-unknown-linux-musl/bin/vector + # chrony RUN set -e \ diff --git a/vm-builder/files/inittab b/vm-builder/files/inittab index bc59e8404..92ffeb43d 100644 --- a/vm-builder/files/inittab +++ b/vm-builder/files/inittab @@ -12,5 +12,5 @@ {{ range .InittabCommands }} ::{{.SysvInitAction}}:su -p {{.CommandUser}} -c {{.ShellEscapedCommand}} {{ end }} -ttyS0::respawn:/neonvm/bin/agetty --8bits --local-line --noissue --noclear --noreset --host console --login-program /neonvm/bin/login --login-pause --autologin root 115200 ttyS0 linux +{{ .AgettyTTY }}::respawn:/neonvm/bin/agetty --8bits --local-line --noissue --noclear --noreset --host console --login-program /neonvm/bin/login --login-pause --autologin root 115200 {{ .AgettyTTY }} linux ::shutdown:/neonvm/bin/vmshutdown diff --git a/vm-builder/main.go b/vm-builder/main.go index 3c379aa4c..7cdd17623 100644 --- a/vm-builder/main.go +++ b/vm-builder/main.go @@ -58,6 +58,11 @@ var ( configSshd string ) +const ( + targetArchLinuxAmd64 = "linux/amd64" + targetArchLinuxArm64 = "linux/arm64" +) + var ( Version string NeonvmDaemonImage string @@ -72,6 +77,7 @@ var ( version = flag.Bool("version", false, `Print vm-builder version`) daemonImageFlag = flag.String("daemon-image", "", `Specify the neonvm-daemon image: --daemon-image=neonvm-daemon:dev`) + targetArch = flag.String("target-arch", "", fmt.Sprintf("Target architecture: --arch %s | %s", targetArchLinuxAmd64, targetArchLinuxArm64)) ) func AddTemplatedFileToTar(tw *tar.Writer, tmplArgs any, filename string, tmplString string) error { @@ -84,7 +90,6 @@ func AddTemplatedFileToTar(tw *tar.Writer, tmplArgs any, filename string, tmplSt if err = tmpl.Execute(&buf, tmplArgs); err != nil { return fmt.Errorf("failed to execute template for %q: %w", filename, err) } - return addFileToTar(tw, filename, buf.Bytes()) } @@ -117,6 +122,7 @@ type TemplatesContext struct { SpecBuild string SpecMerge string InittabCommands []inittabCommand + AgettyTTY string ShutdownHook string } @@ -129,18 +135,28 @@ type inittabCommand struct { func main() { flag.Parse() var dstIm string - if *version { fmt.Println(Version) os.Exit(0) } - if len(*daemonImageFlag) == 0 && len(NeonvmDaemonImage) == 0 { log.Println("neonvm-daemon image not set, needs to be explicitly passed in, or compiled with -ldflags '-X main.NeonvmDaemonImage=...'") flag.PrintDefaults() os.Exit(1) } + if targetArch == nil || *targetArch == "" { + log.Println("Target architecture not set, see usage info:") + flag.PrintDefaults() + os.Exit(1) + } + + if *targetArch != targetArchLinuxAmd64 && *targetArch != targetArchLinuxArm64 { + log.Fatalf("Unsupported target architecture: %q", *targetArch) + flag.PrintDefaults() + return + } + neonvmDaemonImage := NeonvmDaemonImage if len(*daemonImageFlag) != 0 { neonvmDaemonImage = *daemonImageFlag @@ -292,6 +308,7 @@ func main() { SpecMerge: "", // overridden below if spec != nil InittabCommands: nil, // overridden below if spec != nil ShutdownHook: "", // overridden below if spec != nil + AgettyTTY: getAgettyTTY(*targetArch), } if len(imageSpec.Config.User) != 0 { @@ -366,6 +383,7 @@ func main() { buildArgs := make(map[string]*string) buildArgs["DISK_SIZE"] = size + buildArgs["TARGET_ARCH"] = targetArch opt := types.ImageBuildOptions{ AuthConfigs: authConfigs, Tags: []string{dstIm}, @@ -376,6 +394,7 @@ func main() { Dockerfile: "Dockerfile", Remove: true, ForceRemove: true, + Platform: *targetArch, } buildResp, err := cli.ImageBuild(ctx, tarBuffer, opt) if err != nil { @@ -541,3 +560,16 @@ func (f file) validate() []error { return errs } + +// getAgettyTTY returns the tty device name for agetty based on the target architecture. +func getAgettyTTY(targetArch string) string { + switch targetArch { + case targetArchLinuxAmd64: + return "ttyS0" + case targetArchLinuxArm64: + return "ttyAMA0" + default: + log.Fatalf("Unsupported target architecture: %q", targetArch) + return "" + } +}