From c4de93154a1cdc202a8b60488eba6f9b8b0b8c9f Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Thu, 26 Sep 2024 10:02:52 -0700 Subject: [PATCH 1/2] neonvm: Add new skeleton neonvm-daemon binary Extracted the minimal version from a PR, because it's clear that this daemon is required in multiple places, and we don't know which PR will merge first. --- .github/workflows/build-images.yaml | 38 +++++------ .github/workflows/e2e-test.yaml | 1 + Makefile | 11 +++- neonvm/daemon/Dockerfile | 11 ++++ neonvm/daemon/main.go | 67 ++++++++++++++++++++ neonvm/tools/vm-builder/files/Dockerfile.img | 16 ++++- neonvm/tools/vm-builder/files/inittab | 1 + neonvm/tools/vm-builder/main.go | 26 +++++++- 8 files changed, 145 insertions(+), 26 deletions(-) create mode 100644 neonvm/daemon/Dockerfile create mode 100644 neonvm/daemon/main.go diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 35e37de44..f95f21516 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -51,6 +51,7 @@ env: IMG_CONTROLLER: "neondatabase/neonvm-controller" IMG_VXLAN_CONTROLLER: "neondatabase/neonvm-vxlan-controller" IMG_RUNNER: "neondatabase/neonvm-runner" + IMG_DAEMON: "neondatabase/neonvm-daemon" IMG_KERNEL: "neondatabase/vm-kernel" IMG_SCHEDULER: "neondatabase/autoscale-scheduler" IMG_AUTOSCALER_AGENT: "neondatabase/autoscaler-agent" @@ -85,6 +86,7 @@ jobs: echo "controller=${{ env.IMG_CONTROLLER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT echo "vxlan-controller=${{ env.IMG_VXLAN_CONTROLLER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT echo "runner=${{ env.IMG_RUNNER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "daemon=${{ env.IMG_DAEMON }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT echo "scheduler=${{ env.IMG_SCHEDULER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT echo "autoscaler-agent=${{ env.IMG_AUTOSCALER_AGENT }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT echo "cluster-autoscaler=${{ env.IMG_CLUSTER_AUTOSCALER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT @@ -102,10 +104,7 @@ jobs: # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'. if: ${{ format('{0}', inputs.skip) != 'true' }} needs: [ tags, vm-kernel ] - runs-on: [ self-hosted, large ] - permissions: - contents: read # This is required for actions/checkout - id-token: write # This is required for aws-actions/configure-aws-credentials + runs-on: [ self-hosted, gen3, large ] services: registry: @@ -149,7 +148,6 @@ jobs: with: driver-opts: network=host - - name: Login to Dockerhub uses: docker/login-action@v3 with: @@ -163,33 +161,21 @@ jobs: username: ${{ secrets.NEON_CI_DOCKERCACHE_USERNAME }} password: ${{ secrets.NEON_CI_DOCKERCACHE_PASSWORD }} - - name: Configure dev AWS credentials - if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }} - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: eu-central-1 - mask-aws-account-id: true - role-to-assume: ${{ secrets.DEV_GHA_OIDC_ECR_ROLE }} - - name: Login to dev ECR if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }} uses: docker/login-action@v3 with: registry: ${{ env.ECR_DEV }} - - - name: Configure prod AWS credentials - if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }} - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: eu-central-1 - mask-aws-account-id: true - role-to-assume: ${{ secrets.PROD_GHA_OIDC_ECR_ROLE }} + username: ${{ secrets.DEV_GHA_RUNNER_LIMITED_AWS_ACCESS_KEY_ID }} + password: ${{ secrets.DEV_GHA_RUNNER_LIMITED_AWS_SECRET_ACCESS_KEY }} - name: Login to prod ECR if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }} uses: docker/login-action@v3 with: registry: ${{ env.ECR_PROD }} + username: ${{ secrets.PROD_GHA_RUNNER_LIMITED_AWS_ACCESS_KEY_ID }} + password: ${{ secrets.PROD_GHA_RUNNER_LIMITED_AWS_SECRET_ACCESS_KEY }} - name: Check dependencies run: | @@ -230,6 +216,15 @@ jobs: build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} + - name: Build and push neonvm-daemon image + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64 + push: true + file: neonvm/daemon/Dockerfile + tags: ${{ needs.tags.outputs.daemon }} + - name: Generate neonvm-controller build tags id: controller-build-tags env: @@ -317,6 +312,7 @@ jobs: neonvm-controller \ neonvm-vxlan-controller \ neonvm-runner \ + neonvm-daemon \ vm-kernel \ autoscale-scheduler \ autoscaler-agent \ diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index 1a238e6a5..91bbdb470 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -112,6 +112,7 @@ jobs: IMG_CONTROLLER: ${{ needs.build-images.outputs.controller }} IMG_VXLAN_CONTROLLER: ${{ needs.build-images.outputs.vxlan-controller }} IMG_RUNNER: ${{ needs.build-images.outputs.runner }} + IMG_DAEMON: ${{ needs.build-images.outputs.daemon }} IMG_SCHEDULER: ${{ needs.build-images.outputs.scheduler }} IMG_AUTOSCALER_AGENT: ${{ needs.build-images.outputs.autoscaler-agent }} diff --git a/Makefile b/Makefile index 8adb85c1f..5b13a8ff6 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ IMG_CONTROLLER ?= controller:dev IMG_VXLAN_CONTROLLER ?= vxlan-controller:dev IMG_RUNNER ?= runner:dev +IMG_DAEMON ?= daemon:dev IMG_SCHEDULER ?= autoscale-scheduler:dev IMG_AUTOSCALER_AGENT ?= autoscaler-agent:dev @@ -132,8 +133,8 @@ build: fmt vet bin/vm-builder ## Build all neonvm binaries. GOOS=linux go build -o bin/runner neonvm/runner/*.go .PHONY: bin/vm-builder -bin/vm-builder: ## Build vm-builder binary. - GOOS=linux CGO_ENABLED=0 go build -o bin/vm-builder -ldflags "-X main.Version=${GIT_INFO}" neonvm/tools/vm-builder/main.go +bin/vm-builder: docker-build-daemon ## Build vm-builder binary. + GOOS=linux CGO_ENABLED=0 go build -o bin/vm-builder -ldflags "-X main.Version=${GIT_INFO} -X main.NeonvmDaemonImage=${IMG_DAEMON}" neonvm/tools/vm-builder/main.go .PHONY: run run: fmt vet ## Run a controller from your host. @@ -147,7 +148,7 @@ lint: ## Run golangci-lint against code. # (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build -docker-build: docker-build-controller docker-build-runner docker-build-vxlan-controller docker-build-autoscaler-agent docker-build-scheduler ## Build docker images for NeonVM controllers, NeonVM runner, autoscaler-agent, scheduler +docker-build: docker-build-controller docker-build-runner docker-build-daemon docker-build-vxlan-controller docker-build-autoscaler-agent docker-build-scheduler ## Build docker images for NeonVM controllers, NeonVM runner, autoscaler-agent, scheduler .PHONY: docker-push docker-push: docker-build ## Push docker images to docker registry @@ -182,6 +183,10 @@ docker-build-runner: docker-build-go-base ## Build docker image for NeonVM runne --file neonvm/runner/Dockerfile \ . +.PHONY: docker-build-daemon +docker-build-daemon: ## Build docker image for NeonVM daemon. + docker build -t $(IMG_DAEMON) -f neonvm/daemon/Dockerfile . + .PHONY: docker-build-vxlan-controller docker-build-vxlan-controller: docker-build-go-base ## Build docker image for NeonVM vxlan controller docker build \ diff --git a/neonvm/daemon/Dockerfile b/neonvm/daemon/Dockerfile new file mode 100644 index 000000000..527bdd04e --- /dev/null +++ b/neonvm/daemon/Dockerfile @@ -0,0 +1,11 @@ +ARG GO_BASE_IMG=autoscaling-go-base:dev +FROM $GO_BASE_IMG AS builder + +# Build the Go binary +COPY . . + +# Build +RUN CGO_ENABLED=0 go build -a -o /neonvmd neonvm/daemon/main.go + +FROM scratch +COPY --from=builder /neonvmd /neonvmd diff --git a/neonvm/daemon/main.go b/neonvm/daemon/main.go new file mode 100644 index 000000000..415c729ca --- /dev/null +++ b/neonvm/daemon/main.go @@ -0,0 +1,67 @@ +package main + +import ( + "flag" + "fmt" + "net/http" + "os" + "time" + + "go.uber.org/zap" +) + +func main() { + addr := flag.String("addr", "", `address to bind for HTTP requests`) + flag.Parse() + + if *addr == "" { + fmt.Println("neonvm-daemon missing -addr flag") + os.Exit(1) + } + + logConfig := zap.NewProductionConfig() + logConfig.Sampling = nil // Disable sampling, which the production config enables by default. + logConfig.Level.SetLevel(zap.InfoLevel) // Only "info" level and above (i.e. not debug logs) + logger := zap.Must(logConfig.Build()).Named("neonvm-daemon") + defer logger.Sync() //nolint:errcheck // what are we gonna do, log something about it? + + logger.Info("Starting neonvm-daemon", zap.String("addr", *addr)) + + srv := cpuServer{} + srv.run(logger, *addr) +} + +type cpuServer struct{} + +func (s *cpuServer) run(logger *zap.Logger, addr string) { + logger = logger.Named("cpu-srv") + + mux := http.NewServeMux() + mux.HandleFunc("/cpu", func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + logger.Error("unimplemented!") + w.WriteHeader(http.StatusInternalServerError) + } else if r.Method == http.MethodPut { + logger.Error("unimplemented!") + w.WriteHeader(http.StatusInternalServerError) + } else { + // unknown method + w.WriteHeader(http.StatusNotFound) + } + }) + + timeout := 5 * time.Second + server := http.Server{ + Addr: addr, + Handler: mux, + ReadTimeout: timeout, + ReadHeaderTimeout: timeout, + WriteTimeout: timeout, + } + + err := server.ListenAndServe() + if err != nil { + logger.Fatal("CPU server exited with error", zap.Error(err)) + } + logger.Info("CPU server exited without error") +} diff --git a/neonvm/tools/vm-builder/files/Dockerfile.img b/neonvm/tools/vm-builder/files/Dockerfile.img index 0905d7993..4ddf63566 100644 --- a/neonvm/tools/vm-builder/files/Dockerfile.img +++ b/neonvm/tools/vm-builder/files/Dockerfile.img @@ -7,6 +7,8 @@ FROM {{.RootDiskImage}} AS rootdisk USER root {{.SpecMerge}} +FROM {{.NeonvmDaemonImage}} AS vm-daemon-loader + FROM alpine:3.19 AS vm-runtime # add busybox ENV BUSYBOX_VERSION 1.35.0 @@ -19,12 +21,17 @@ RUN set -e \ COPY helper.move-bins.sh /helper.move-bins.sh # add udevd and agetty (with shared libs) +# +# We need unshare and nsenter from util-linux-misc because buxybox's implementations don't have +# support for cgroup namespaces (at least, master as of 2024-08-11). RUN set -e \ && apk add --no-cache --no-progress --quiet \ acpid \ udev \ agetty \ su-exec \ + util-linux-misc \ + cgroup-tools \ e2fsprogs-extra \ blkid \ flock \ @@ -35,6 +42,8 @@ RUN set -e \ udevadm \ agetty \ su-exec \ + unshare nsenter \ + cgexec \ resize2fs \ blkid \ flock \ @@ -65,6 +74,8 @@ RUN set -e \ quota-tools \ && /helper.move-bins.sh quota edquota quotacheck quotaoff quotaon quotastats setquota repquota tune2fs +COPY --from=vm-daemon-loader /neonvmd /neonvm/bin/neonvmd + # init scripts & configs COPY inittab /neonvm/bin/inittab COPY vminit /neonvm/bin/vminit @@ -77,6 +88,9 @@ COPY sshd_config /neonvm/config/sshd_config RUN chmod +rx /neonvm/bin/vminit /neonvm/bin/vmstart /neonvm/bin/vmshutdown COPY udev-init.sh /neonvm/bin/udev-init.sh RUN chmod +rx /neonvm/bin/udev-init.sh +COPY cg-setup.sh /neonvm/bin/cg-setup.sh +COPY cg-run.sh /neonvm/bin/cg-run.sh +RUN chmod +rx /neonvm/bin/cg-setup.sh /neonvm/bin/cg-run.sh COPY resize-swap.sh /neonvm/bin/resize-swap RUN chmod +rx /neonvm/bin/resize-swap COPY set-disk-quota.sh /neonvm/bin/set-disk-quota @@ -97,7 +111,7 @@ RUN set -e \ && /neonvm/bin/id -g sshd > /dev/null 2>&1 || /neonvm/bin/addgroup sshd \ && /neonvm/bin/id -u sshd > /dev/null 2>&1 || /neonvm/bin/adduser -D -H -G sshd -g 'sshd privsep' -s /neonvm/bin/nologin sshd -FROM alpine:3.19 AS builder +FROM vm-runtime AS builder ARG DISK_SIZE COPY --from=rootdisk-mod / /rootdisk diff --git a/neonvm/tools/vm-builder/files/inittab b/neonvm/tools/vm-builder/files/inittab index 5d4ea10a4..bc59e8404 100644 --- a/neonvm/tools/vm-builder/files/inittab +++ b/neonvm/tools/vm-builder/files/inittab @@ -1,5 +1,6 @@ ::sysinit:/neonvm/bin/vminit ::once:/neonvm/bin/touch /neonvm/vmstart.allowed +::respawn:/neonvm/bin/neonvmd --addr=0.0.0.0:25183 ::respawn:/neonvm/bin/udhcpc -t 1 -T 1 -A 1 -f -i eth0 -O 121 -O 119 -s /neonvm/bin/udhcpc.script ::respawn:/neonvm/bin/udevd ::wait:/neonvm/bin/udev-init.sh diff --git a/neonvm/tools/vm-builder/main.go b/neonvm/tools/vm-builder/main.go index 462d145e0..466feb376 100644 --- a/neonvm/tools/vm-builder/main.go +++ b/neonvm/tools/vm-builder/main.go @@ -46,6 +46,10 @@ var ( scriptVmInit string //go:embed files/udev-init.sh scriptUdevInit string + //go:embed files/cg-setup.sh + scriptCgSetup string + //go:embed files/cg-run.sh + scriptCgRun string //go:embed files/resize-swap.sh scriptResizeSwap string //go:embed files/set-disk-quota.sh @@ -59,7 +63,8 @@ var ( ) var ( - Version string + Version string + NeonvmDaemonImage string srcImage = flag.String("src", "", `Docker image used as source for virtual machine disk image: --src=alpine:3.19`) dstImage = flag.String("dst", "", `Docker image with resulting disk image: --dst=vm-alpine:3.19`) @@ -69,6 +74,8 @@ var ( quiet = flag.Bool("quiet", false, `Show less output from the docker build process`) forcePull = flag.Bool("pull", false, `Pull src image even if already present locally`) version = flag.Bool("version", false, `Print vm-builder version`) + + daemonImageFlag = flag.String("daemon-image", "", `Specify the neonvm-daemon image: --daemon-image=neonvm-daemon:dev`) ) func AddTemplatedFileToTar(tw *tar.Writer, tmplArgs any, filename string, tmplString string) error { @@ -109,6 +116,8 @@ type TemplatesContext struct { Env []string RootDiskImage string + NeonvmDaemonImage string + SpecBuild string SpecMerge string InittabCommands []inittabCommand @@ -130,6 +139,17 @@ func main() { os.Exit(0) } + if len(*daemonImageFlag) == 0 && len(NeonvmDaemonImage) == 0 { + log.Println("neonvm-daemon image not set, needs to be explicitly passed in, or compiled with -ldflags '-X main.NeonvmDaemonImage=...'") + flag.PrintDefaults() + os.Exit(1) + } + + neonvmDaemonImage := NeonvmDaemonImage + if len(*daemonImageFlag) != 0 { + neonvmDaemonImage = *daemonImageFlag + } + if len(*srcImage) == 0 { log.Println("-src not set, see usage info:") flag.PrintDefaults() @@ -270,6 +290,8 @@ func main() { Env: imageSpec.Config.Env, RootDiskImage: *srcImage, + NeonvmDaemonImage: neonvmDaemonImage, + SpecBuild: "", // overridden below if spec != nil SpecMerge: "", // overridden below if spec != nil InittabCommands: nil, // overridden below if spec != nil @@ -336,6 +358,8 @@ func main() { {"chrony.conf", configChrony}, {"sshd_config", configSshd}, {"udev-init.sh", scriptUdevInit}, + {"cg-setup.sh", scriptCgSetup}, + {"cg-run.sh", scriptCgRun}, {"resize-swap.sh", scriptResizeSwap}, {"set-disk-quota.sh", scriptSetDiskQuota}, } From 2b703ce5355b9ec267fe47b0709b72372e7b2b36 Mon Sep 17 00:00:00 2001 From: Misha Sakhnov Date: Mon, 30 Sep 2024 11:24:19 +0300 Subject: [PATCH 2/2] neonvm: apply ci fixes for extracted control daemon Signed-off-by: Misha Sakhnov --- .github/workflows/build-images.yaml | 6 ++++++ .github/workflows/build-test-vm.yaml | 5 ++++- Makefile | 8 ++++++-- neonvm/tools/vm-builder/main.go | 6 ++---- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index f95f21516..a4a9141fe 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -46,6 +46,9 @@ on: autoscaler-agent: description: 'autoscaler-agent image' value: ${{ jobs.tags.outputs.autoscaler-agent }} + daemon: + description: 'neonvm-daemon image' + value: ${{ jobs.tags.outputs.daemon }} env: IMG_CONTROLLER: "neondatabase/neonvm-controller" @@ -79,6 +82,7 @@ jobs: scheduler: ${{ steps.show-tags.outputs.scheduler }} autoscaler-agent: ${{ steps.show-tags.outputs.autoscaler-agent }} cluster-autoscaler: ${{ steps.show-tags.outputs.cluster-autoscaler }} + daemon: ${{ steps.show-tags.outputs.daemon}} runs-on: ubuntu-latest steps: - id: show-tags @@ -224,6 +228,8 @@ jobs: push: true file: neonvm/daemon/Dockerfile tags: ${{ needs.tags.outputs.daemon }} + build-args: | + GO_BASE_IMG=${{ env.GO_BASE_IMG }} - name: Generate neonvm-controller build tags id: controller-build-tags diff --git a/.github/workflows/build-test-vm.yaml b/.github/workflows/build-test-vm.yaml index 964dd8ab2..c172c8e3e 100644 --- a/.github/workflows/build-test-vm.yaml +++ b/.github/workflows/build-test-vm.yaml @@ -29,6 +29,7 @@ on: env: IMG_POSTGRES_16_BULLSEYE: "neondatabase/vm-postgres-16-bullseye" + IMG_DAEMON: "neondatabase/neonvm-daemon" defaults: run: @@ -38,11 +39,13 @@ jobs: tags: outputs: vm-postgres-16-bullseye: ${{ steps.show-tags.outputs.vm-postgres-16-bullseye }} + daemon: ${{ steps.show-tags.outputs.daemon }} runs-on: ubuntu-latest steps: - id: show-tags run: | echo "vm-postgres-16-bullseye=${{ env.IMG_POSTGRES_16_BULLSEYE }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "daemon=${{ env.IMG_DAEMON }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT build: # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'. @@ -81,7 +84,7 @@ jobs: - name: build ${{ needs.tags.outputs.vm-postgres-16-bullseye }} run: | - ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }} + ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ needs.tags.outputs.daemon }} - name: docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }} run: | docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }} diff --git a/Makefile b/Makefile index 5b13a8ff6..d8954d388 100644 --- a/Makefile +++ b/Makefile @@ -133,7 +133,7 @@ build: fmt vet bin/vm-builder ## Build all neonvm binaries. GOOS=linux go build -o bin/runner neonvm/runner/*.go .PHONY: bin/vm-builder -bin/vm-builder: docker-build-daemon ## Build vm-builder binary. +bin/vm-builder: ## Build vm-builder binary. GOOS=linux CGO_ENABLED=0 go build -o bin/vm-builder -ldflags "-X main.Version=${GIT_INFO} -X main.NeonvmDaemonImage=${IMG_DAEMON}" neonvm/tools/vm-builder/main.go .PHONY: run @@ -185,7 +185,11 @@ docker-build-runner: docker-build-go-base ## Build docker image for NeonVM runne .PHONY: docker-build-daemon docker-build-daemon: ## Build docker image for NeonVM daemon. - docker build -t $(IMG_DAEMON) -f neonvm/daemon/Dockerfile . + docker build \ + --tag $(IMG_DAEMON) \ + --build-arg GO_BASE_IMG=$(GO_BASE_IMG) \ + --file neonvm/daemon/Dockerfile \ + . .PHONY: docker-build-vxlan-controller docker-build-vxlan-controller: docker-build-go-base ## Build docker image for NeonVM vxlan controller diff --git a/neonvm/tools/vm-builder/main.go b/neonvm/tools/vm-builder/main.go index 466feb376..8b1fd1c77 100644 --- a/neonvm/tools/vm-builder/main.go +++ b/neonvm/tools/vm-builder/main.go @@ -46,10 +46,8 @@ var ( scriptVmInit string //go:embed files/udev-init.sh scriptUdevInit string - //go:embed files/cg-setup.sh - scriptCgSetup string - //go:embed files/cg-run.sh - scriptCgRun string + scriptCgSetup string + scriptCgRun string //go:embed files/resize-swap.sh scriptResizeSwap string //go:embed files/set-disk-quota.sh