Skip to content

Commit

Permalink
neonvm: Add new skeleton neonvm-daemon binary
Browse files Browse the repository at this point in the history
Extracted the minimal version from a PR, because it's clear that this
daemon is required in multiple places, and we don't know which PR will
merge first.
  • Loading branch information
sharnoff committed Sep 26, 2024
1 parent 0caa474 commit c4de931
Show file tree
Hide file tree
Showing 8 changed files with 145 additions and 26 deletions.
38 changes: 17 additions & 21 deletions .github/workflows/build-images.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ env:
IMG_CONTROLLER: "neondatabase/neonvm-controller"
IMG_VXLAN_CONTROLLER: "neondatabase/neonvm-vxlan-controller"
IMG_RUNNER: "neondatabase/neonvm-runner"
IMG_DAEMON: "neondatabase/neonvm-daemon"
IMG_KERNEL: "neondatabase/vm-kernel"
IMG_SCHEDULER: "neondatabase/autoscale-scheduler"
IMG_AUTOSCALER_AGENT: "neondatabase/autoscaler-agent"
Expand Down Expand Up @@ -85,6 +86,7 @@ jobs:
echo "controller=${{ env.IMG_CONTROLLER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
echo "vxlan-controller=${{ env.IMG_VXLAN_CONTROLLER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
echo "runner=${{ env.IMG_RUNNER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
echo "daemon=${{ env.IMG_DAEMON }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
echo "scheduler=${{ env.IMG_SCHEDULER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
echo "autoscaler-agent=${{ env.IMG_AUTOSCALER_AGENT }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
echo "cluster-autoscaler=${{ env.IMG_CLUSTER_AUTOSCALER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT
Expand All @@ -102,10 +104,7 @@ jobs:
# nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'.
if: ${{ format('{0}', inputs.skip) != 'true' }}
needs: [ tags, vm-kernel ]
runs-on: [ self-hosted, large ]
permissions:
contents: read # This is required for actions/checkout
id-token: write # This is required for aws-actions/configure-aws-credentials
runs-on: [ self-hosted, gen3, large ]

services:
registry:
Expand Down Expand Up @@ -149,7 +148,6 @@ jobs:
with:
driver-opts: network=host


- name: Login to Dockerhub
uses: docker/login-action@v3
with:
Expand All @@ -163,33 +161,21 @@ jobs:
username: ${{ secrets.NEON_CI_DOCKERCACHE_USERNAME }}
password: ${{ secrets.NEON_CI_DOCKERCACHE_PASSWORD }}

- name: Configure dev AWS credentials
if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }}
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: eu-central-1
mask-aws-account-id: true
role-to-assume: ${{ secrets.DEV_GHA_OIDC_ECR_ROLE }}

- name: Login to dev ECR
if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }}
uses: docker/login-action@v3
with:
registry: ${{ env.ECR_DEV }}

- name: Configure prod AWS credentials
if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }}
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: eu-central-1
mask-aws-account-id: true
role-to-assume: ${{ secrets.PROD_GHA_OIDC_ECR_ROLE }}
username: ${{ secrets.DEV_GHA_RUNNER_LIMITED_AWS_ACCESS_KEY_ID }}
password: ${{ secrets.DEV_GHA_RUNNER_LIMITED_AWS_SECRET_ACCESS_KEY }}

- name: Login to prod ECR
if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }}
uses: docker/login-action@v3
with:
registry: ${{ env.ECR_PROD }}
username: ${{ secrets.PROD_GHA_RUNNER_LIMITED_AWS_ACCESS_KEY_ID }}
password: ${{ secrets.PROD_GHA_RUNNER_LIMITED_AWS_SECRET_ACCESS_KEY }}

- name: Check dependencies
run: |
Expand Down Expand Up @@ -230,6 +216,15 @@ jobs:
build-args: |
GO_BASE_IMG=${{ env.GO_BASE_IMG }}
- name: Build and push neonvm-daemon image
uses: docker/build-push-action@v3
with:
context: .
platforms: linux/amd64
push: true
file: neonvm/daemon/Dockerfile
tags: ${{ needs.tags.outputs.daemon }}

Check failure on line 226 in .github/workflows/build-images.yaml

View workflow job for this annotation

GitHub Actions / actionlint

[actionlint] .github/workflows/build-images.yaml#L226

property "daemon" is not defined in object type {autoscaler-agent: string; cluster-autoscaler: string; controller: string; runner: string; scheduler: string; vxlan-controller: string} [expression]
Raw output
.github/workflows/build-images.yaml:226:21: property "daemon" is not defined in object type {autoscaler-agent: string; cluster-autoscaler: string; controller: string; runner: string; scheduler: string; vxlan-controller: string} [expression]

- name: Generate neonvm-controller build tags
id: controller-build-tags
env:
Expand Down Expand Up @@ -317,6 +312,7 @@ jobs:
neonvm-controller \
neonvm-vxlan-controller \
neonvm-runner \
neonvm-daemon \
vm-kernel \
autoscale-scheduler \
autoscaler-agent \
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/e2e-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ jobs:
IMG_CONTROLLER: ${{ needs.build-images.outputs.controller }}
IMG_VXLAN_CONTROLLER: ${{ needs.build-images.outputs.vxlan-controller }}
IMG_RUNNER: ${{ needs.build-images.outputs.runner }}
IMG_DAEMON: ${{ needs.build-images.outputs.daemon }}

Check failure on line 115 in .github/workflows/e2e-test.yaml

View workflow job for this annotation

GitHub Actions / actionlint

[actionlint] .github/workflows/e2e-test.yaml#L115

property "daemon" is not defined in object type {autoscaler-agent: string; controller: string; runner: string; scheduler: string; vxlan-controller: string} [expression]
Raw output
.github/workflows/e2e-test.yaml:115:37: property "daemon" is not defined in object type {autoscaler-agent: string; controller: string; runner: string; scheduler: string; vxlan-controller: string} [expression]
IMG_SCHEDULER: ${{ needs.build-images.outputs.scheduler }}
IMG_AUTOSCALER_AGENT: ${{ needs.build-images.outputs.autoscaler-agent }}

Expand Down
11 changes: 8 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
IMG_CONTROLLER ?= controller:dev
IMG_VXLAN_CONTROLLER ?= vxlan-controller:dev
IMG_RUNNER ?= runner:dev
IMG_DAEMON ?= daemon:dev
IMG_SCHEDULER ?= autoscale-scheduler:dev
IMG_AUTOSCALER_AGENT ?= autoscaler-agent:dev

Expand Down Expand Up @@ -132,8 +133,8 @@ build: fmt vet bin/vm-builder ## Build all neonvm binaries.
GOOS=linux go build -o bin/runner neonvm/runner/*.go

.PHONY: bin/vm-builder
bin/vm-builder: ## Build vm-builder binary.
GOOS=linux CGO_ENABLED=0 go build -o bin/vm-builder -ldflags "-X main.Version=${GIT_INFO}" neonvm/tools/vm-builder/main.go
bin/vm-builder: docker-build-daemon ## Build vm-builder binary.
GOOS=linux CGO_ENABLED=0 go build -o bin/vm-builder -ldflags "-X main.Version=${GIT_INFO} -X main.NeonvmDaemonImage=${IMG_DAEMON}" neonvm/tools/vm-builder/main.go

.PHONY: run
run: fmt vet ## Run a controller from your host.
Expand All @@ -147,7 +148,7 @@ lint: ## Run golangci-lint against code.
# (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it.
# More info: https://docs.docker.com/develop/develop-images/build_enhancements/
.PHONY: docker-build
docker-build: docker-build-controller docker-build-runner docker-build-vxlan-controller docker-build-autoscaler-agent docker-build-scheduler ## Build docker images for NeonVM controllers, NeonVM runner, autoscaler-agent, scheduler
docker-build: docker-build-controller docker-build-runner docker-build-daemon docker-build-vxlan-controller docker-build-autoscaler-agent docker-build-scheduler ## Build docker images for NeonVM controllers, NeonVM runner, autoscaler-agent, scheduler

.PHONY: docker-push
docker-push: docker-build ## Push docker images to docker registry
Expand Down Expand Up @@ -182,6 +183,10 @@ docker-build-runner: docker-build-go-base ## Build docker image for NeonVM runne
--file neonvm/runner/Dockerfile \
.

.PHONY: docker-build-daemon
docker-build-daemon: ## Build docker image for NeonVM daemon.
docker build -t $(IMG_DAEMON) -f neonvm/daemon/Dockerfile .

.PHONY: docker-build-vxlan-controller
docker-build-vxlan-controller: docker-build-go-base ## Build docker image for NeonVM vxlan controller
docker build \
Expand Down
11 changes: 11 additions & 0 deletions neonvm/daemon/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
ARG GO_BASE_IMG=autoscaling-go-base:dev
FROM $GO_BASE_IMG AS builder

# Build the Go binary
COPY . .

# Build
RUN CGO_ENABLED=0 go build -a -o /neonvmd neonvm/daemon/main.go

FROM scratch
COPY --from=builder /neonvmd /neonvmd
67 changes: 67 additions & 0 deletions neonvm/daemon/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package main

import (
"flag"
"fmt"
"net/http"
"os"
"time"

"go.uber.org/zap"
)

func main() {
addr := flag.String("addr", "", `address to bind for HTTP requests`)
flag.Parse()

if *addr == "" {
fmt.Println("neonvm-daemon missing -addr flag")
os.Exit(1)
}

logConfig := zap.NewProductionConfig()
logConfig.Sampling = nil // Disable sampling, which the production config enables by default.
logConfig.Level.SetLevel(zap.InfoLevel) // Only "info" level and above (i.e. not debug logs)
logger := zap.Must(logConfig.Build()).Named("neonvm-daemon")
defer logger.Sync() //nolint:errcheck // what are we gonna do, log something about it?

logger.Info("Starting neonvm-daemon", zap.String("addr", *addr))

srv := cpuServer{}
srv.run(logger, *addr)
}

type cpuServer struct{}

func (s *cpuServer) run(logger *zap.Logger, addr string) {
logger = logger.Named("cpu-srv")

mux := http.NewServeMux()
mux.HandleFunc("/cpu", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
logger.Error("unimplemented!")
w.WriteHeader(http.StatusInternalServerError)
} else if r.Method == http.MethodPut {
logger.Error("unimplemented!")
w.WriteHeader(http.StatusInternalServerError)
} else {
// unknown method
w.WriteHeader(http.StatusNotFound)
}
})

timeout := 5 * time.Second
server := http.Server{
Addr: addr,
Handler: mux,
ReadTimeout: timeout,
ReadHeaderTimeout: timeout,
WriteTimeout: timeout,
}

err := server.ListenAndServe()
if err != nil {
logger.Fatal("CPU server exited with error", zap.Error(err))
}
logger.Info("CPU server exited without error")
}
16 changes: 15 additions & 1 deletion neonvm/tools/vm-builder/files/Dockerfile.img
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ FROM {{.RootDiskImage}} AS rootdisk
USER root
{{.SpecMerge}}

FROM {{.NeonvmDaemonImage}} AS vm-daemon-loader

FROM alpine:3.19 AS vm-runtime
# add busybox
ENV BUSYBOX_VERSION 1.35.0
Expand All @@ -19,12 +21,17 @@ RUN set -e \
COPY helper.move-bins.sh /helper.move-bins.sh

# add udevd and agetty (with shared libs)
#
# We need unshare and nsenter from util-linux-misc because buxybox's implementations don't have
# support for cgroup namespaces (at least, master as of 2024-08-11).
RUN set -e \
&& apk add --no-cache --no-progress --quiet \
acpid \
udev \
agetty \
su-exec \
util-linux-misc \
cgroup-tools \
e2fsprogs-extra \
blkid \
flock \
Expand All @@ -35,6 +42,8 @@ RUN set -e \
udevadm \
agetty \
su-exec \
unshare nsenter \
cgexec \
resize2fs \
blkid \
flock \
Expand Down Expand Up @@ -65,6 +74,8 @@ RUN set -e \
quota-tools \
&& /helper.move-bins.sh quota edquota quotacheck quotaoff quotaon quotastats setquota repquota tune2fs

COPY --from=vm-daemon-loader /neonvmd /neonvm/bin/neonvmd

# init scripts & configs
COPY inittab /neonvm/bin/inittab
COPY vminit /neonvm/bin/vminit
Expand All @@ -77,6 +88,9 @@ COPY sshd_config /neonvm/config/sshd_config
RUN chmod +rx /neonvm/bin/vminit /neonvm/bin/vmstart /neonvm/bin/vmshutdown
COPY udev-init.sh /neonvm/bin/udev-init.sh
RUN chmod +rx /neonvm/bin/udev-init.sh
COPY cg-setup.sh /neonvm/bin/cg-setup.sh
COPY cg-run.sh /neonvm/bin/cg-run.sh
RUN chmod +rx /neonvm/bin/cg-setup.sh /neonvm/bin/cg-run.sh
COPY resize-swap.sh /neonvm/bin/resize-swap
RUN chmod +rx /neonvm/bin/resize-swap
COPY set-disk-quota.sh /neonvm/bin/set-disk-quota
Expand All @@ -97,7 +111,7 @@ RUN set -e \
&& /neonvm/bin/id -g sshd > /dev/null 2>&1 || /neonvm/bin/addgroup sshd \
&& /neonvm/bin/id -u sshd > /dev/null 2>&1 || /neonvm/bin/adduser -D -H -G sshd -g 'sshd privsep' -s /neonvm/bin/nologin sshd

FROM alpine:3.19 AS builder
FROM vm-runtime AS builder
ARG DISK_SIZE
COPY --from=rootdisk-mod / /rootdisk

Expand Down
1 change: 1 addition & 0 deletions neonvm/tools/vm-builder/files/inittab
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
::sysinit:/neonvm/bin/vminit
::once:/neonvm/bin/touch /neonvm/vmstart.allowed
::respawn:/neonvm/bin/neonvmd --addr=0.0.0.0:25183
::respawn:/neonvm/bin/udhcpc -t 1 -T 1 -A 1 -f -i eth0 -O 121 -O 119 -s /neonvm/bin/udhcpc.script
::respawn:/neonvm/bin/udevd
::wait:/neonvm/bin/udev-init.sh
Expand Down
26 changes: 25 additions & 1 deletion neonvm/tools/vm-builder/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ var (
scriptVmInit string
//go:embed files/udev-init.sh
scriptUdevInit string
//go:embed files/cg-setup.sh
scriptCgSetup string
//go:embed files/cg-run.sh

Check failure on line 51 in neonvm/tools/vm-builder/main.go

View workflow job for this annotation

GitHub Actions / tests

pattern files/cg-run.sh: no matching files found

Check failure on line 51 in neonvm/tools/vm-builder/main.go

View workflow job for this annotation

GitHub Actions / golangci-lint

pattern files/cg-run.sh: no matching files found (typecheck)
scriptCgRun string
//go:embed files/resize-swap.sh
scriptResizeSwap string
//go:embed files/set-disk-quota.sh
Expand All @@ -59,7 +63,8 @@ var (
)

var (
Version string
Version string
NeonvmDaemonImage string

srcImage = flag.String("src", "", `Docker image used as source for virtual machine disk image: --src=alpine:3.19`)
dstImage = flag.String("dst", "", `Docker image with resulting disk image: --dst=vm-alpine:3.19`)
Expand All @@ -69,6 +74,8 @@ var (
quiet = flag.Bool("quiet", false, `Show less output from the docker build process`)
forcePull = flag.Bool("pull", false, `Pull src image even if already present locally`)
version = flag.Bool("version", false, `Print vm-builder version`)

daemonImageFlag = flag.String("daemon-image", "", `Specify the neonvm-daemon image: --daemon-image=neonvm-daemon:dev`)
)

func AddTemplatedFileToTar(tw *tar.Writer, tmplArgs any, filename string, tmplString string) error {
Expand Down Expand Up @@ -109,6 +116,8 @@ type TemplatesContext struct {
Env []string
RootDiskImage string

NeonvmDaemonImage string

SpecBuild string
SpecMerge string
InittabCommands []inittabCommand
Expand All @@ -130,6 +139,17 @@ func main() {
os.Exit(0)
}

if len(*daemonImageFlag) == 0 && len(NeonvmDaemonImage) == 0 {
log.Println("neonvm-daemon image not set, needs to be explicitly passed in, or compiled with -ldflags '-X main.NeonvmDaemonImage=...'")
flag.PrintDefaults()
os.Exit(1)
}

neonvmDaemonImage := NeonvmDaemonImage
if len(*daemonImageFlag) != 0 {
neonvmDaemonImage = *daemonImageFlag
}

if len(*srcImage) == 0 {
log.Println("-src not set, see usage info:")
flag.PrintDefaults()
Expand Down Expand Up @@ -270,6 +290,8 @@ func main() {
Env: imageSpec.Config.Env,
RootDiskImage: *srcImage,

NeonvmDaemonImage: neonvmDaemonImage,

SpecBuild: "", // overridden below if spec != nil
SpecMerge: "", // overridden below if spec != nil
InittabCommands: nil, // overridden below if spec != nil
Expand Down Expand Up @@ -336,6 +358,8 @@ func main() {
{"chrony.conf", configChrony},
{"sshd_config", configSshd},
{"udev-init.sh", scriptUdevInit},
{"cg-setup.sh", scriptCgSetup},
{"cg-run.sh", scriptCgRun},
{"resize-swap.sh", scriptResizeSwap},
{"set-disk-quota.sh", scriptSetDiskQuota},
}
Expand Down

0 comments on commit c4de931

Please sign in to comment.