Misha/neonvm daemon cleanup

.github/workflows/build-images.yaml

@@ -46,11 +46,15 @@ on:
       autoscaler-agent:
         description: 'autoscaler-agent image'
         value: ${{ jobs.tags.outputs.autoscaler-agent }}
+      daemon:
+        description: 'neonvm-daemon image'
+        value: ${{ jobs.tags.outputs.daemon }}
 
 env:
   IMG_CONTROLLER:         "neondatabase/neonvm-controller"
   IMG_VXLAN_CONTROLLER:   "neondatabase/neonvm-vxlan-controller"
   IMG_RUNNER:             "neondatabase/neonvm-runner"
+  IMG_DAEMON:             "neondatabase/neonvm-daemon"
   IMG_KERNEL:             "neondatabase/vm-kernel"
   IMG_SCHEDULER:          "neondatabase/autoscale-scheduler"
   IMG_AUTOSCALER_AGENT:   "neondatabase/autoscaler-agent"
@@ -78,13 +82,15 @@ jobs:
       scheduler:          ${{ steps.show-tags.outputs.scheduler }}
       autoscaler-agent:   ${{ steps.show-tags.outputs.autoscaler-agent }}
       cluster-autoscaler: ${{ steps.show-tags.outputs.cluster-autoscaler }}
+      daemon:             ${{ steps.show-tags.outputs.daemon}}
     runs-on: ubuntu-latest
     steps:
       - id: show-tags
         run: |
           echo "controller=${{ env.IMG_CONTROLLER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
           echo "vxlan-controller=${{ env.IMG_VXLAN_CONTROLLER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
           echo "runner=${{ env.IMG_RUNNER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
+          echo "daemon=${{ env.IMG_DAEMON }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
           echo "scheduler=${{ env.IMG_SCHEDULER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
           echo "autoscaler-agent=${{ env.IMG_AUTOSCALER_AGENT }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
           echo "cluster-autoscaler=${{ env.IMG_CLUSTER_AUTOSCALER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
@@ -102,10 +108,7 @@ jobs:
     # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'.
     if: ${{ format('{0}', inputs.skip) != 'true' }}
     needs: [ tags, vm-kernel ]
-    runs-on: [ self-hosted, large ]
-    permissions:
-      contents: read  # This is required for actions/checkout
-      id-token: write # This is required for aws-actions/configure-aws-credentials
+    runs-on: [ self-hosted, gen3, large ]
 
     services:
       registry:
@@ -149,7 +152,6 @@ jobs:
         with:
           driver-opts: network=host
 
-
       - name: Login to Dockerhub
         uses: docker/login-action@v3
         with:
@@ -163,33 +165,21 @@ jobs:
           username: ${{ secrets.NEON_CI_DOCKERCACHE_USERNAME }}
           password: ${{ secrets.NEON_CI_DOCKERCACHE_PASSWORD }}
 
-      - name: Configure dev AWS credentials
-        if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }}
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-region: eu-central-1
-          mask-aws-account-id: true
-          role-to-assume: ${{ secrets.DEV_GHA_OIDC_ECR_ROLE }}
-
       - name: Login to dev ECR
         if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }}
         uses: docker/login-action@v3
         with:
           registry: ${{ env.ECR_DEV }}
-
-      - name: Configure prod AWS credentials
-        if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }}
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-region: eu-central-1
-          mask-aws-account-id: true
-          role-to-assume: ${{ secrets.PROD_GHA_OIDC_ECR_ROLE }}
+          username: ${{ secrets.DEV_GHA_RUNNER_LIMITED_AWS_ACCESS_KEY_ID }}
+          password: ${{ secrets.DEV_GHA_RUNNER_LIMITED_AWS_SECRET_ACCESS_KEY }}
 
       - name: Login to prod ECR
         if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }}
         uses: docker/login-action@v3
         with:
           registry: ${{ env.ECR_PROD }}
+          username: ${{ secrets.PROD_GHA_RUNNER_LIMITED_AWS_ACCESS_KEY_ID }}
+          password: ${{ secrets.PROD_GHA_RUNNER_LIMITED_AWS_SECRET_ACCESS_KEY }}
 
       - name: Check dependencies
         run: |
@@ -230,6 +220,17 @@ jobs:
           build-args: |
             GO_BASE_IMG=${{ env.GO_BASE_IMG }}
 
+      - name: Build and push neonvm-daemon image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          file: neonvm/daemon/Dockerfile
+          tags: ${{ needs.tags.outputs.daemon }}
+          build-args: |
+            GO_BASE_IMG=${{ env.GO_BASE_IMG }}
+
       - name: Generate neonvm-controller build tags
         id: controller-build-tags
         env:
@@ -317,6 +318,7 @@ jobs:
             neonvm-controller \
             neonvm-vxlan-controller \
             neonvm-runner \
+            neonvm-daemon \
             vm-kernel \
             autoscale-scheduler \
             autoscaler-agent \

.github/workflows/build-test-vm.yaml

@@ -29,6 +29,7 @@ on:
 
 env:
   IMG_POSTGRES_16_BULLSEYE: "neondatabase/vm-postgres-16-bullseye"
+  IMG_DAEMON: "neondatabase/neonvm-daemon"
 
 defaults:
   run:
@@ -38,11 +39,13 @@ jobs:
   tags:
     outputs:
       vm-postgres-16-bullseye: ${{ steps.show-tags.outputs.vm-postgres-16-bullseye }}
+      daemon: ${{ steps.show-tags.outputs.daemon }}
     runs-on: ubuntu-latest
     steps:
       - id: show-tags
         run: |
           echo "vm-postgres-16-bullseye=${{ env.IMG_POSTGRES_16_BULLSEYE }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
+          echo "daemon=${{ env.IMG_DAEMON }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
 
   build:
     # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'.
@@ -81,7 +84,7 @@ jobs:
 
       - name: build ${{ needs.tags.outputs.vm-postgres-16-bullseye }}
         run: |
-          ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }}
+          ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ needs.tags.outputs.daemon }}
       - name: docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }}
         run: |
           docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }}

.github/workflows/e2e-test.yaml

@@ -112,6 +112,7 @@ jobs:
           IMG_CONTROLLER:       ${{ needs.build-images.outputs.controller }}
           IMG_VXLAN_CONTROLLER: ${{ needs.build-images.outputs.vxlan-controller }}
           IMG_RUNNER:           ${{ needs.build-images.outputs.runner }}
+          IMG_DAEMON:           ${{ needs.build-images.outputs.daemon }}
           IMG_SCHEDULER:        ${{ needs.build-images.outputs.scheduler }}
           IMG_AUTOSCALER_AGENT: ${{ needs.build-images.outputs.autoscaler-agent }}
 

Makefile

@@ -2,6 +2,7 @@
 IMG_CONTROLLER ?= controller:dev
 IMG_VXLAN_CONTROLLER ?= vxlan-controller:dev
 IMG_RUNNER ?= runner:dev
+IMG_DAEMON ?= daemon:dev
 IMG_SCHEDULER ?= autoscale-scheduler:dev
 IMG_AUTOSCALER_AGENT ?= autoscaler-agent:dev
 
@@ -133,7 +134,7 @@ build: fmt vet bin/vm-builder ## Build all neonvm binaries.
 
 .PHONY: bin/vm-builder
 bin/vm-builder: ## Build vm-builder binary.
-	GOOS=linux CGO_ENABLED=0 go build -o bin/vm-builder -ldflags "-X main.Version=${GIT_INFO}" neonvm/tools/vm-builder/main.go
+	GOOS=linux CGO_ENABLED=0 go build -o bin/vm-builder -ldflags "-X main.Version=${GIT_INFO} -X main.NeonvmDaemonImage=${IMG_DAEMON}" neonvm/tools/vm-builder/main.go
 
 .PHONY: run
 run: fmt vet ## Run a controller from your host.
@@ -147,7 +148,7 @@ lint: ## Run golangci-lint against code.
 # (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: docker-build
-docker-build: docker-build-controller docker-build-runner docker-build-vxlan-controller docker-build-autoscaler-agent docker-build-scheduler ## Build docker images for NeonVM controllers, NeonVM runner, autoscaler-agent, scheduler
+docker-build: docker-build-controller docker-build-runner docker-build-daemon docker-build-vxlan-controller docker-build-autoscaler-agent docker-build-scheduler ## Build docker images for NeonVM controllers, NeonVM runner, autoscaler-agent, scheduler
 
 .PHONY: docker-push
 docker-push: docker-build ## Push docker images to docker registry
@@ -182,6 +183,14 @@ docker-build-runner: docker-build-go-base ## Build docker image for NeonVM runne
 		--file neonvm/runner/Dockerfile \
 		.
 
+.PHONY: docker-build-daemon
+docker-build-daemon: ## Build docker image for NeonVM daemon.
+	docker build \
+		--tag $(IMG_DAEMON) \
+		--build-arg GO_BASE_IMG=$(GO_BASE_IMG) \
+		--file neonvm/daemon/Dockerfile \
+		.
+
 .PHONY: docker-build-vxlan-controller
 docker-build-vxlan-controller: docker-build-go-base ## Build docker image for NeonVM vxlan controller
 	docker build \

neonvm/daemon/Dockerfile

@@ -0,0 +1,11 @@
+ARG GO_BASE_IMG=autoscaling-go-base:dev
+FROM $GO_BASE_IMG AS builder
+
+# Build the Go binary
+COPY . .
+
+# Build
+RUN CGO_ENABLED=0 go build -a -o /neonvmd neonvm/daemon/main.go
+
+FROM scratch
+COPY --from=builder /neonvmd /neonvmd

neonvm/daemon/main.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+func main() {
+	addr := flag.String("addr", "", `address to bind for HTTP requests`)
+	flag.Parse()
+
+	if *addr == "" {
+		fmt.Println("neonvm-daemon missing -addr flag")
+		os.Exit(1)
+	}
+
+	logConfig := zap.NewProductionConfig()
+	logConfig.Sampling = nil                // Disable sampling, which the production config enables by default.
+	logConfig.Level.SetLevel(zap.InfoLevel) // Only "info" level and above (i.e. not debug logs)
+	logger := zap.Must(logConfig.Build()).Named("neonvm-daemon")
+	defer logger.Sync() //nolint:errcheck // what are we gonna do, log something about it?
+
+	logger.Info("Starting neonvm-daemon", zap.String("addr", *addr))
+
+	srv := cpuServer{}
+	srv.run(logger, *addr)
+}
+
+type cpuServer struct{}
+
+func (s *cpuServer) run(logger *zap.Logger, addr string) {
+	logger = logger.Named("cpu-srv")
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/cpu", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodGet {
+			logger.Error("unimplemented!")
+			w.WriteHeader(http.StatusInternalServerError)
+		} else if r.Method == http.MethodPut {
+			logger.Error("unimplemented!")
+			w.WriteHeader(http.StatusInternalServerError)
+		} else {
+			// unknown method
+			w.WriteHeader(http.StatusNotFound)
+		}
+	})
+
+	timeout := 5 * time.Second
+	server := http.Server{
+		Addr:              addr,
+		Handler:           mux,
+		ReadTimeout:       timeout,
+		ReadHeaderTimeout: timeout,
+		WriteTimeout:      timeout,
+	}
+
+	err := server.ListenAndServe()
+	if err != nil {
+		logger.Fatal("CPU server exited with error", zap.Error(err))
+	}
+	logger.Info("CPU server exited without error")
+}

neonvm/tools/vm-builder/files/Dockerfile.img

@@ -7,6 +7,8 @@ FROM {{.RootDiskImage}} AS rootdisk
 USER root
 {{.SpecMerge}}
 
+FROM {{.NeonvmDaemonImage}} AS vm-daemon-loader
+
 FROM alpine:3.19 AS vm-runtime
 # add busybox
 ENV BUSYBOX_VERSION 1.35.0
@@ -19,12 +21,17 @@ RUN set -e \
 COPY helper.move-bins.sh /helper.move-bins.sh
 
 # add udevd and agetty (with shared libs)
+#
+# We need unshare and nsenter from util-linux-misc because buxybox's implementations don't have
+# support for cgroup namespaces (at least, master as of 2024-08-11).
 RUN set -e \
 	&& apk add --no-cache --no-progress --quiet \
 		acpid \
 		udev \
 		agetty \
 		su-exec \
+        util-linux-misc \
+        cgroup-tools \
 		e2fsprogs-extra \
 		blkid \
 		flock \
@@ -35,6 +42,8 @@ RUN set -e \
 		udevadm \
 		agetty \
 		su-exec \
+        unshare nsenter \
+        cgexec \
 		resize2fs \
 		blkid \
 		flock \
@@ -65,6 +74,8 @@ RUN set -e \
 		quota-tools \
 	&& /helper.move-bins.sh quota edquota quotacheck quotaoff quotaon quotastats setquota repquota tune2fs
 
+COPY --from=vm-daemon-loader /neonvmd /neonvm/bin/neonvmd
+
 # init scripts & configs
 COPY inittab     /neonvm/bin/inittab
 COPY vminit      /neonvm/bin/vminit
@@ -77,6 +88,9 @@ COPY sshd_config /neonvm/config/sshd_config
 RUN chmod +rx /neonvm/bin/vminit /neonvm/bin/vmstart /neonvm/bin/vmshutdown
 COPY udev-init.sh /neonvm/bin/udev-init.sh
 RUN chmod +rx /neonvm/bin/udev-init.sh
+COPY cg-setup.sh /neonvm/bin/cg-setup.sh
+COPY cg-run.sh /neonvm/bin/cg-run.sh
+RUN chmod +rx /neonvm/bin/cg-setup.sh /neonvm/bin/cg-run.sh
 COPY resize-swap.sh /neonvm/bin/resize-swap
 RUN chmod +rx /neonvm/bin/resize-swap
 COPY set-disk-quota.sh /neonvm/bin/set-disk-quota
@@ -97,7 +111,7 @@ RUN set -e \
     && /neonvm/bin/id -g sshd > /dev/null 2>&1 || /neonvm/bin/addgroup sshd \
     && /neonvm/bin/id -u sshd > /dev/null 2>&1 || /neonvm/bin/adduser -D -H -G sshd -g 'sshd privsep' -s /neonvm/bin/nologin sshd
 
-FROM alpine:3.19 AS builder
+FROM vm-runtime AS builder
 ARG DISK_SIZE
 COPY --from=rootdisk-mod / /rootdisk
 

neonvm/tools/vm-builder/files/inittab

@@ -1,5 +1,6 @@
 ::sysinit:/neonvm/bin/vminit
 ::once:/neonvm/bin/touch /neonvm/vmstart.allowed
+::respawn:/neonvm/bin/neonvmd --addr=0.0.0.0:25183
 ::respawn:/neonvm/bin/udhcpc -t 1 -T 1 -A 1 -f -i eth0 -O 121 -O 119 -s /neonvm/bin/udhcpc.script
 ::respawn:/neonvm/bin/udevd
 ::wait:/neonvm/bin/udev-init.sh