From 1a77f56a82ca62b8a3f8a689547d231c6619a452 Mon Sep 17 00:00:00 2001 From: Em Sharnoff Date: Wed, 12 Jun 2024 21:03:18 -0700 Subject: [PATCH] Flatten & unify component structure This basically encompasses two logical changes: 1. More fully integrating neonvm into the rest of the repository; and 2. Using a consistent directory structure for all components The new structure is: - All components get their own top-level directory (e.g. 'autoscaler-agent', 'neonvm-controller', 'vm-builder') - Inside the directory, there is: - cmd/ - directory containing main.go for the component - Dockerfile - for building the component - *.yaml - various YAML files for deploying, if applicable Additionally, all dependencies outside the 'cmd' package are moved into the 'pkg/' subdirectory. In particular, there is now 'pkg/neonvm/'. Also of note: 'neonvm/hack/kernel' has been relocated to the new top-level directory 'neonvm-kernel/'. Under this structure, the user-visible changes are: 1. Introduction of new YAML: neonvm-controller.yaml 2. Introduction of new YAML: neonvm-vxlan-controller.yaml 3. neonvm.yaml now is *mostly* just the CRD. The primary reasons for this change are: 1. Because it's been long enough since we moved the neonvm repository here that we might as well make it more well-integrated. 2. Because as we look to add components in other languages, we need a directory structure that supports it. ... and now that Autoscaling is GA, we can make the bigger changes we were holding off on :) --- .github/workflows/build-images.yaml | 12 +-- .github/workflows/e2e-test.yaml | 6 +- .github/workflows/release.yaml | 2 + .github/workflows/vm-kernel.yaml | 10 +-- .gitignore | 1 + ARCHITECTURE.md | 90 ++++++++++--------- Makefile | 84 +++++++++-------- neonvm/README.md => README-NeonVM.md | 30 +------ README.md | 2 +- .../Dockerfile | 4 +- .../cmd}/main.go | 0 .../config_map.yaml | 0 .../deployment.yaml | 0 .../kustomization.yaml | 0 .../role_binding.yaml | 0 .../service_account.yaml | 0 .../Dockerfile | 4 +- .../cmd}/main.go | 0 .../config_map.yaml | 0 .../agent => autoscaler-agent}/daemonset.yaml | 0 .../kustomization.yaml | 0 .../role_binding.yaml | 0 .../service_account.yaml | 0 deploy/.gitignore | 4 - {neonvm => neonvm-controller}/Dockerfile | 16 ++-- {neonvm => neonvm-controller/cmd}/main.go | 2 +- .../deployment.yaml | 0 neonvm-controller/kustomization.yaml | 66 ++++++++++++++ .../leader_election_role.yaml | 0 .../leader_election_role_binding.yaml | 0 .../rbac => neonvm-controller}/role.yaml | 0 .../role_binding.yaml | 0 .../service_account.yaml | 0 neonvm-kernel/.gitignore | 1 + .../Dockerfile.kernel-builder | 0 neonvm-kernel/README.md | 54 +++++++++++ .../linux-config-6.1.92 | 0 {neonvm/runner => neonvm-runner}/Dockerfile | 22 +++-- .../cmd}/container-mgr/crictl.go | 0 .../cmd}/container-mgr/main.go | 0 {neonvm/runner => neonvm-runner/cmd}/main.go | 0 .../image-loader}/bases/daemonset.yaml | 0 .../image-loader}/bases/kustomization.yaml | 0 neonvm-runner/image-loader/daemonset.yaml | 31 +++++++ .../image-loader}/kustomization.yaml | 3 +- {neonvm/runner => neonvm-runner}/ssh_config | 0 .../Dockerfile | 4 +- .../cmd}/main.go | 0 .../daemonset.yaml | 0 .../kustomization.yaml | 5 ++ .../rbac.yaml | 0 neonvm/config/controller/kustomization.yaml | 9 -- neonvm/config/kustomization.yaml | 44 --------- neonvm/config/rbac/kustomization.yaml | 5 -- .../controllers/buildtag/nodelete_false.go | 0 .../controllers/buildtag/nodelete_true.go | 0 .../neonvm}/controllers/buildtag/tagnames.go | 0 .../neonvm}/controllers/catch_panic.go | 0 {neonvm => pkg/neonvm}/controllers/config.go | 0 .../neonvm}/controllers/failurelag/tracker.go | 0 .../controllers/failurelag/tracker_test.go | 2 +- .../controllers/functests/suite_test.go | 2 +- .../functests/vm_controller_test.go | 2 +- {neonvm => pkg/neonvm}/controllers/metrics.go | 2 +- .../neonvm}/controllers/vm_controller.go | 4 +- .../controllers/vm_controller_unit_test.go | 0 .../neonvm}/controllers/vm_qmp_queries.go | 0 .../neonvm}/controllers/vm_qmp_test.go | 2 +- .../controllers/vmmigration_controller.go | 2 +- {neonvm => pkg/neonvm}/controllers/webhook.go | 0 {neonvm/pkg => pkg/neonvm/ipam}/README.md | 6 +- {neonvm/pkg => pkg/neonvm}/ipam/allocate.go | 0 {neonvm/pkg => pkg/neonvm}/ipam/client.go | 0 .../neonvm/ipam/demo/ipam-nad.yaml | 0 .../neonvm/ipam/demo/ipam.go | 2 +- {neonvm/pkg => pkg/neonvm}/ipam/ipam.go | 0 {neonvm/pkg => pkg/neonvm}/ipam/types.go | 0 .../files/Dockerfile.img | 0 .../files/chrony.conf | 0 .../files/helper.move-bins.sh | 0 .../vm-builder => vm-builder}/files/inittab | 0 .../files/resize-swap.sh | 0 .../files/sshd_config | 0 .../files/udev-init.sh | 0 .../files/vector.yaml | 0 .../vm-builder => vm-builder}/files/vmacpi | 0 .../vm-builder => vm-builder}/files/vminit | 0 .../files/vmshutdown | 0 .../vm-builder => vm-builder}/files/vmstart | 0 .../tools/vm-builder => vm-builder}/main.go | 0 deploy/vmscrape.yaml => vmscrape.yaml | 0 91 files changed, 309 insertions(+), 226 deletions(-) rename neonvm/README.md => README-NeonVM.md (87%) rename {build/autoscale-scheduler => autoscale-scheduler}/Dockerfile (90%) rename {cmd/autoscale-scheduler => autoscale-scheduler/cmd}/main.go (100%) rename {deploy/scheduler => autoscale-scheduler}/config_map.yaml (100%) rename {deploy/scheduler => autoscale-scheduler}/deployment.yaml (100%) rename {deploy/scheduler => autoscale-scheduler}/kustomization.yaml (100%) rename {deploy/scheduler => autoscale-scheduler}/role_binding.yaml (100%) rename {deploy/scheduler => autoscale-scheduler}/service_account.yaml (100%) rename {build/autoscaler-agent => autoscaler-agent}/Dockerfile (91%) rename {cmd/autoscaler-agent => autoscaler-agent/cmd}/main.go (100%) rename {deploy/agent => autoscaler-agent}/config_map.yaml (100%) rename {deploy/agent => autoscaler-agent}/daemonset.yaml (100%) rename {deploy/agent => autoscaler-agent}/kustomization.yaml (100%) rename {deploy/agent => autoscaler-agent}/role_binding.yaml (100%) rename {deploy/agent => autoscaler-agent}/service_account.yaml (100%) delete mode 100644 deploy/.gitignore rename {neonvm => neonvm-controller}/Dockerfile (77%) rename {neonvm => neonvm-controller/cmd}/main.go (99%) rename {neonvm/config/controller => neonvm-controller}/deployment.yaml (100%) create mode 100644 neonvm-controller/kustomization.yaml rename {neonvm/config/rbac => neonvm-controller}/leader_election_role.yaml (100%) rename {neonvm/config/rbac => neonvm-controller}/leader_election_role_binding.yaml (100%) rename {neonvm/config/rbac => neonvm-controller}/role.yaml (100%) rename {neonvm/config/rbac => neonvm-controller}/role_binding.yaml (100%) rename {neonvm/config/rbac => neonvm-controller}/service_account.yaml (100%) create mode 100644 neonvm-kernel/.gitignore rename {neonvm/hack/kernel => neonvm-kernel}/Dockerfile.kernel-builder (100%) create mode 100644 neonvm-kernel/README.md rename {neonvm/hack/kernel => neonvm-kernel}/linux-config-6.1.92 (100%) rename {neonvm/runner => neonvm-runner}/Dockerfile (77%) rename {neonvm/runner => neonvm-runner/cmd}/container-mgr/crictl.go (100%) rename {neonvm/runner => neonvm-runner/cmd}/container-mgr/main.go (100%) rename {neonvm/runner => neonvm-runner/cmd}/main.go (100%) rename {neonvm/runner-image-loader => neonvm-runner/image-loader}/bases/daemonset.yaml (100%) rename {neonvm/runner-image-loader => neonvm-runner/image-loader}/bases/kustomization.yaml (100%) create mode 100644 neonvm-runner/image-loader/daemonset.yaml rename {neonvm/runner-image-loader => neonvm-runner/image-loader}/kustomization.yaml (89%) rename {neonvm/runner => neonvm-runner}/ssh_config (100%) rename {neonvm/tools/vxlan => neonvm-vxlan-controller}/Dockerfile (91%) rename {neonvm/tools/vxlan/controller => neonvm-vxlan-controller/cmd}/main.go (100%) rename {neonvm/config/vxlan-controller => neonvm-vxlan-controller}/daemonset.yaml (100%) rename {neonvm/config/vxlan-controller => neonvm-vxlan-controller}/kustomization.yaml (57%) rename {neonvm/config/vxlan-controller => neonvm-vxlan-controller}/rbac.yaml (100%) delete mode 100644 neonvm/config/controller/kustomization.yaml rename {neonvm => pkg/neonvm}/controllers/buildtag/nodelete_false.go (100%) rename {neonvm => pkg/neonvm}/controllers/buildtag/nodelete_true.go (100%) rename {neonvm => pkg/neonvm}/controllers/buildtag/tagnames.go (100%) rename {neonvm => pkg/neonvm}/controllers/catch_panic.go (100%) rename {neonvm => pkg/neonvm}/controllers/config.go (100%) rename {neonvm => pkg/neonvm}/controllers/failurelag/tracker.go (100%) rename {neonvm => pkg/neonvm}/controllers/failurelag/tracker_test.go (97%) rename {neonvm => pkg/neonvm}/controllers/functests/suite_test.go (95%) rename {neonvm => pkg/neonvm}/controllers/functests/vm_controller_test.go (98%) rename {neonvm => pkg/neonvm}/controllers/metrics.go (99%) rename {neonvm => pkg/neonvm}/controllers/vm_controller.go (99%) rename {neonvm => pkg/neonvm}/controllers/vm_controller_unit_test.go (100%) rename {neonvm => pkg/neonvm}/controllers/vm_qmp_queries.go (100%) rename {neonvm => pkg/neonvm}/controllers/vm_qmp_test.go (94%) rename {neonvm => pkg/neonvm}/controllers/vmmigration_controller.go (99%) rename {neonvm => pkg/neonvm}/controllers/webhook.go (100%) rename {neonvm/pkg => pkg/neonvm/ipam}/README.md (66%) rename {neonvm/pkg => pkg/neonvm}/ipam/allocate.go (100%) rename {neonvm/pkg => pkg/neonvm}/ipam/client.go (100%) rename neonvm/pkg/ipam-demo-nad.yaml => pkg/neonvm/ipam/demo/ipam-nad.yaml (100%) rename neonvm/pkg/ipam-demo.go => pkg/neonvm/ipam/demo/ipam.go (97%) rename {neonvm/pkg => pkg/neonvm}/ipam/ipam.go (100%) rename {neonvm/pkg => pkg/neonvm}/ipam/types.go (100%) rename {neonvm/tools/vm-builder => vm-builder}/files/Dockerfile.img (100%) rename {neonvm/tools/vm-builder => vm-builder}/files/chrony.conf (100%) rename {neonvm/tools/vm-builder => vm-builder}/files/helper.move-bins.sh (100%) rename {neonvm/tools/vm-builder => vm-builder}/files/inittab (100%) rename {neonvm/tools/vm-builder => vm-builder}/files/resize-swap.sh (100%) rename {neonvm/tools/vm-builder => vm-builder}/files/sshd_config (100%) rename {neonvm/tools/vm-builder => vm-builder}/files/udev-init.sh (100%) rename {neonvm/tools/vm-builder => vm-builder}/files/vector.yaml (100%) rename {neonvm/tools/vm-builder => vm-builder}/files/vmacpi (100%) rename {neonvm/tools/vm-builder => vm-builder}/files/vminit (100%) rename {neonvm/tools/vm-builder => vm-builder}/files/vmshutdown (100%) rename {neonvm/tools/vm-builder => vm-builder}/files/vmstart (100%) rename {neonvm/tools/vm-builder => vm-builder}/main.go (100%) rename deploy/vmscrape.yaml => vmscrape.yaml (100%) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index a7e19e385..5cf8e2034 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -170,7 +170,7 @@ jobs: run: | docker pull --quiet $IMAGE ID=$(docker create $IMAGE true) - docker cp ${ID}:/vmlinuz neonvm/hack/kernel/vmlinuz + docker cp ${ID}:/vmlinuz neonvm-kernel/vmlinuz docker rm -f ${ID} - name: Build and push neonvm-runner image @@ -179,7 +179,7 @@ jobs: context: . platforms: linux/amd64 push: true - file: neonvm/runner/Dockerfile + file: neonvm-runner/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-runner:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-runner:cache,mode=max' || '' }} tags: ${{ needs.tags.outputs.runner }} @@ -201,7 +201,7 @@ jobs: context: . platforms: linux/amd64 push: true - file: neonvm/Dockerfile + file: neonvm-controller/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-controller:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-controller:cache,mode=max' || '' }} tags: ${{ needs.tags.outputs.controller }} @@ -215,7 +215,7 @@ jobs: context: . platforms: linux/amd64 push: true - file: neonvm/tools/vxlan/Dockerfile + file: neonvm-vxlan-controller/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-vxlan-controller:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-vxlan-controller:cache,mode=max' || '' }} tags: ${{ needs.tags.outputs.vxlan-controller }} @@ -226,7 +226,7 @@ jobs: context: . platforms: linux/amd64 push: true - file: build/autoscale-scheduler/Dockerfile + file: autoscale-scheduler/Dockerfile cache-from: type=registry,ref=cache.neon.build/autoscale-scheduler:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscale-scheduler:cache,mode=max' || '' }} tags: ${{ needs.tags.outputs.scheduler }} @@ -239,7 +239,7 @@ jobs: context: . platforms: linux/amd64 push: true - file: build/autoscaler-agent/Dockerfile + file: autoscaler-agent/Dockerfile cache-from: type=registry,ref=cache.neon.build/autoscaler-agent:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscaler-agent:cache,mode=max' || '' }} tags: ${{ needs.tags.outputs.autoscaler-agent }} diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index 1a238e6a5..b040bdd50 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -174,9 +174,11 @@ jobs: kubectl apply -f $(rendered neonvm-runner-image-loader.yaml) kubectl -n neonvm-system rollout status daemonset neonvm-runner-image-loader kubectl apply -f $(rendered neonvm.yaml) - kubectl -n neonvm-system rollout status daemonset neonvm-device-plugin - kubectl -n neonvm-system rollout status daemonset neonvm-vxlan-controller + kubectl -n neonvm-system rollout status daemonset neonvm-device-plugin + kubectl apply -f $(rendered neonvm-controller.yaml) kubectl -n neonvm-system rollout status deployment neonvm-controller + kubectl apply -f $(rendered neonvm-vxlan-controller.yaml) + kubectl -n neonvm-system rollout status daemonset neonvm-vxlan-controller kubectl apply -f $(rendered autoscale-scheduler.yaml) kubectl -n kube-system rollout status deployment autoscale-scheduler kubectl apply -f $(rendered autoscaler-agent.yaml) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 97bf07d18..e706346ad 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -88,6 +88,8 @@ jobs: rendered_manifests/autoscale-scheduler.yaml rendered_manifests/autoscaler-agent.yaml rendered_manifests/neonvm.yaml + rendered_manifests/neonvm-controller.yaml + rendered_manifests/neonvm-vxlan-controller.yaml rendered_manifests/neonvm-runner-image-loader.yaml rendered_manifests/multus.yaml rendered_manifests/multus-eks.yaml diff --git a/.github/workflows/vm-kernel.yaml b/.github/workflows/vm-kernel.yaml index ea0c129af..cdbfd99e3 100644 --- a/.github/workflows/vm-kernel.yaml +++ b/.github/workflows/vm-kernel.yaml @@ -60,7 +60,7 @@ jobs: -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ --method GET \ - --field path=neonvm/hack/kernel \ + --field path=neonvm-kernel \ --field sha=${COMMIT_SHA} \ --field per_page=1 \ --jq ".[0].sha" \ @@ -165,8 +165,8 @@ jobs: - name: get kernel version id: get-kernel-version run: | - linux_config=$(ls neonvm/hack/kernel/linux-config-*) # returns something like "neonvm/hack/kernel/linux-config-6.1.63" - kernel_version=${linux_config##*-} # returns something like "6.1.63" + linux_config=$(ls neonvm-kernel/linux-config-*) # returns something like "neonvm-kernel/linux-config-6.1.63" + kernel_version=${linux_config##*-} # returns something like "6.1.63" echo VM_KERNEL_VERSION=$kernel_version >> $GITHUB_OUTPUT @@ -192,12 +192,12 @@ jobs: uses: docker/build-push-action@v6 with: build-args: KERNEL_VERSION=${{ steps.get-kernel-version.outputs.VM_KERNEL_VERSION }} - context: neonvm/hack/kernel + context: neonvm-kernel platforms: linux/amd64 # Push kernel image only for scheduled builds or if workflow_dispatch/workflow_call input is true push: true pull: true - file: neonvm/hack/kernel/Dockerfile.kernel-builder + file: neonvm-kernel/Dockerfile.kernel-builder cache-from: type=registry,ref=cache.neon.build/vm-kernel:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/vm-kernel:cache,mode=max' || '' }} tags: ${{ steps.get-tags.outputs.tags }} diff --git a/.gitignore b/.gitignore index 9538b2e24..1d0523c60 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ cover.html *~ *.qcow2 +# todo: remove old kernel location neonvm/hack/kernel/vmlinuz rendered_manifests diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 55326b779..e6f677c67 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -36,20 +36,23 @@ This isn't the only architecture document. You may also want to look at: ## High-level overview -At a high level, this repository provides two components: +At a high level, this repository provides three components: -1. A modified Kubernetes scheduler (using the [plugin interface]) — known as "the (scheduler) +1. A Kubernetes custom resource definition (CRD) and controller for managing resizeable VMs — + NeonVM. +2. A modified Kubernetes scheduler (using the [plugin interface]) — known as "the (scheduler) plugin", `AutoscaleEnforcer`, `autoscale-scheduler` -2. A daemonset responsible for making VM scaling decisions & checking with interested parties +3. A daemonset responsible for making VM scaling decisions & checking with interested parties — known as `autoscaler-agent` or simply `agent` -A third component, a binary running inside of the VM to (a) handle being upscaled +A fourth component, a binary running inside of the VM to (a) handle being upscaled (b) validate that downscaling is ok, and (c) request immediate upscaling due to sharp changes in demand -— known as "the (VM) monitor", lives in -[`neondatabase/vm-monitor`](https://github.com/neondatabase/vm-monitor) +— known as "the (VM) monitor", lives in [`neondatabase/neon/.../vm-monitor`]. [plugin interface]: https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/ +For information on NeonVM, see [README-NeonVM.md](./README-NeonVM.md). + The scheduler plugin is responsible for handling resource requests from the `autoscaler-agent`, capping increases so that node resources aren't overcommitted. @@ -81,48 +84,49 @@ discussed more in the [high-level consequences] section below. ## Repository structure -* `build/` — scripts for building the scheduler (`autoscale-scheduler`) and `autoscaler-agent` -* `cluster-autoscaler/` — patch and Dockerfile for building a NeonVM-compatible [cluster-autoscaler] -* `cmd/` — entrypoints for the `autoscaler-agent` and scheduler plugin. Very little - functionality implemented here. (See: `pkg/agent` and `pkg/plugin`) -* `deploy/` — YAML files used during cluster init. Of these, only the following two are manually - written: - * `deploy/autoscaler-agent.yaml` - * `deploy/autoscale-scheduler.yaml` -* `kind/` — files specific to creating our [kind](https://kind.sigs.k8s.io/) cluster - * `kind/config.yaml` — configuration for the kind cluster -* `neonvm/` — QEMU-based virtualisation API and controllers for k8s - * See [`neonvm/README.md`](./neonvm/README.md) for details -* `pkg/` — core go code from the scheduler plugin and `autoscaler-agent`. Where applicable, the - purpose of individual files is commented at the top. - * `pkg/agent/` — implementation of `autoscaler-agent` - * `pkg/api/` — all types for inter-component communications, plus some protocol-relevant types - independently used by multiple components. - * `pkg/billing/` — consumption metrics API, primarily used in - [`pkg/agent/billing.go`](pkg/agent/billing.go) - * `pkg/plugin/` — implementation of the scheduler plugin - * `pkg/util/` — miscellaneous utilities that are too general to be included in `agent` or - `plugin`. -* `scripts/` — a collection of scripts for common tasks. Items of note: - * `scripts/patch-*.json` — patches for testing live-updating of a VM or config - * `scripts/replace-scheduler.sh` — replaces the currently running scheduler, for quick redeploy - * `scripts/repeat-delete-scheduler.sh` — repeatedly deletes the scheduler (which will be - recreated by the deployment). For debugging. - * `scripts/run-bench.sh` — starts a CPU-intensive pgbench connected to a VM. Useful to watch - the TPS and get confirmation that autoscaled CPUs are being used. - * `scripts/scheduler-logs.sh` — convenience script to tail the scheduler's logs - * `scripts/ssh-into-vm.sh` — `ssh`es into a VM. Useful for debugging. - * `scripts/start-vm-bridge.sh` -* `tests/` — end-to-end tests +At a high level, each component gets its own directory and resulting YAML for its deployment, where +applicable. + +These are: + +* `autoscale-scheduler` — the scheduler (with our plugin) +* `autoscaler-agent` +* `cluster-autoscaler` — patch for building a NeonVM-compatible [cluster-autoscaler] +* `cluster-autoscaler` — our patch to cluster-autoscaler so it understands the resource usage from + NeonVM VMs +* `neonvm` — CRDs and other related YAMLs for NeonVM, alongside Go definitions and a generated + client. Note that the generated YAML includes a dependency on `neonvm-controller` via a webhook + for create/update/delete operations on the CRDs. +* `neonvm-controller` — controller for the NeonVM CRDs +* `neonvm-kernel` — files relating to the virtual machine kernel we use in NeonVM +* `neonvm-runner` — per-VM management process, created by `neonvm-controller` +* `neonvm-vxlan-controller` +* `vm-builder` — binary for building VM images for use by NeonVM + +[cluster-autoscaler]: https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler + +Each component directory contains: + +* `cmd/` — the entrypoint of the component, containing `main.go` +* `Dockerfile` — for building the component, if applicable +* `kustomize.yaml` — if the component is separately deployed, instructions for how Kustomize should + generate the YAML +* `*.yaml` — if the component is separately deployed, there will be other YAML files as the + resources for Kustomize to include -- e.g. `daemonset.yaml` or `config_map.yaml`. + +### Other directories + +* `k3d/` and `kind/` — configuration for local test clusters +* `pkg/` — the bulk of the Go codebase. For more complex components, `cmd/main.go` often just calls + the relevant entrypoint function in its `pkg/` directory. `pkg/` also includes the common + packages shared by multiple components. +* `scripts` — a collection of scripts for common tasks +* `tests` — end-to-end tests * `tests/e2e` — [`kuttl`](https://kuttl.dev/) test scenarios itself -* `scripts-common.sh` — file with a handful of useful functions, used both in `build` and `scripts` -* `vm-deploy.yaml` — sample creation of a single VM, for testing autoscaling * `vm-examples/` — collection of VMs: * `pg16-disk-test/` — VM with Postgres 16 and and ssh access * Refer to [`vm-examples/pg16-disk-test/README.md`](./vm-examples/pg16-disk-test) for more information. -[cluster-autoscaler]: https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler - ## Agent-Scheduler protocol details Broadly speaking, the `autoscaler-agent` _notifies_ on decrease and _requests_ increases. This means diff --git a/Makefile b/Makefile index 37619700d..24fbe6a51 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ build: fmt vet bin/vm-builder ## Build all neonvm binaries. .PHONY: bin/vm-builder bin/vm-builder: ## Build vm-builder binary. - GOOS=linux CGO_ENABLED=0 go build -o bin/vm-builder -ldflags "-X main.Version=${GIT_INFO}" neonvm/tools/vm-builder/main.go + GOOS=linux CGO_ENABLED=0 go build -o bin/vm-builder -ldflags "-X main.Version=${GIT_INFO}" vm-builder/main.go .PHONY: run run: fmt vet ## Run a controller from your host. @@ -147,15 +147,15 @@ docker-push: docker-build ## Push docker images to docker registry .PHONY: docker-build-controller docker-build-controller: ## Build docker image for NeonVM controller - docker build --build-arg VM_RUNNER_IMAGE=$(IMG_RUNNER) --build-arg BUILDTAGS=$(if $(PRESERVE_RUNNER_PODS),nodelete) -t $(IMG_CONTROLLER) -f neonvm/Dockerfile . + docker build --build-arg VM_RUNNER_IMAGE=$(IMG_RUNNER) --build-arg BUILDTAGS=$(if $(PRESERVE_RUNNER_PODS),nodelete) -t $(IMG_CONTROLLER) -f neonvm-controller/Dockerfile . .PHONY: docker-build-runner docker-build-runner: ## Build docker image for NeonVM runner - docker build -t $(IMG_RUNNER) -f neonvm/runner/Dockerfile . + docker build -t $(IMG_RUNNER) -f neonvm-runner/Dockerfile . .PHONY: docker-build-vxlan-controller docker-build-vxlan-controller: ## Build docker image for NeonVM vxlan controller - docker build -t $(IMG_VXLAN_CONTROLLER) -f neonvm/tools/vxlan/Dockerfile . + docker build -t $(IMG_VXLAN_CONTROLLER) -f neonvm-vxlan-controller/Dockerfile . .PHONY: docker-build-autoscaler-agent docker-build-autoscaler-agent: ## Build docker image for autoscaler-agent @@ -163,7 +163,7 @@ docker-build-autoscaler-agent: ## Build docker image for autoscaler-agent --tag $(IMG_AUTOSCALER_AGENT) \ --load \ --build-arg "GIT_INFO=$(GIT_INFO)" \ - --file build/autoscaler-agent/Dockerfile \ + --file autoscaler-agent/Dockerfile \ . .PHONY: docker-build-scheduler @@ -172,7 +172,7 @@ docker-build-scheduler: ## Build docker image for (autoscaling) scheduler --tag $(IMG_SCHEDULER) \ --load \ --build-arg "GIT_INFO=$(GIT_INFO)" \ - --file build/autoscale-scheduler/Dockerfile \ + --file autoscale-scheduler/Dockerfile \ . .PHONY: docker-build-examples @@ -213,8 +213,8 @@ endif .PHONY: kernel kernel: ## Build linux kernel. - rm -f neonvm/hack/kernel/vmlinuz; \ - linux_config=$$(ls neonvm/hack/kernel/linux-config-*) \ + rm -f neonvm-kernel/vmlinuz; \ + linux_config=$$(ls neonvm-kernel/linux-config-*) \ kernel_version=$${linux_config##*-} \ iidfile=$$(mktemp /tmp/iid-XXXXXX); \ trap "rm $$iidfile" EXIT; \ @@ -224,10 +224,10 @@ kernel: ## Build linux kernel. --pull \ --load \ --iidfile $$iidfile \ - --file neonvm/hack/kernel/Dockerfile.kernel-builder \ - neonvm/hack/kernel; \ + --file neonvm-kernel/Dockerfile.kernel-builder \ + neonvm-kernel; \ id=$$(docker create $$(cat $$iidfile)); \ - docker cp $$id:/vmlinuz neonvm/hack/kernel/vmlinuz; \ + docker cp $$id:/vmlinuz neonvm-kernel/vmlinuz; \ docker rm -f $$id .PHONY: check-local-context @@ -249,47 +249,51 @@ $(RENDERED): .PHONY: render-manifests render-manifests: $(RENDERED) kustomize # Prepare: - cd neonvm/config/controller && $(KUSTOMIZE) edit set image controller=$(IMG_CONTROLLER) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force - cd neonvm/config/vxlan-controller && $(KUSTOMIZE) edit set image vxlan-controller=$(IMG_VXLAN_CONTROLLER) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force - cd neonvm/runner-image-loader/bases && $(KUSTOMIZE) edit set image runner=$(IMG_RUNNER) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force - cd deploy/scheduler && $(KUSTOMIZE) edit set image autoscale-scheduler=$(IMG_SCHEDULER) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force - cd deploy/agent && $(KUSTOMIZE) edit set image autoscaler-agent=$(IMG_AUTOSCALER_AGENT) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force + cd neonvm-controller && $(KUSTOMIZE) edit set image controller=$(IMG_CONTROLLER) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force + cd neonvm-vxlan-controller && $(KUSTOMIZE) edit set image vxlan-controller=$(IMG_VXLAN_CONTROLLER) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force + cd neonvm-runner/image-loader/bases && $(KUSTOMIZE) edit set image runner=$(IMG_RUNNER) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force + cd autoscale-scheduler && $(KUSTOMIZE) edit set image autoscale-scheduler=$(IMG_SCHEDULER) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force + cd autoscaler-agent && $(KUSTOMIZE) edit set image autoscaler-agent=$(IMG_AUTOSCALER_AGENT) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force # Build: $(KUSTOMIZE) build neonvm/config/whereabouts > $(RENDERED)/whereabouts.yaml $(KUSTOMIZE) build neonvm/config/multus-eks > $(RENDERED)/multus-eks.yaml $(KUSTOMIZE) build neonvm/config/multus > $(RENDERED)/multus.yaml $(KUSTOMIZE) build neonvm/config > $(RENDERED)/neonvm.yaml - $(KUSTOMIZE) build neonvm/runner-image-loader > $(RENDERED)/neonvm-runner-image-loader.yaml - $(KUSTOMIZE) build deploy/scheduler > $(RENDERED)/autoscale-scheduler.yaml - $(KUSTOMIZE) build deploy/agent > $(RENDERED)/autoscaler-agent.yaml + $(KUSTOMIZE) build neonvm-controller > $(RENDERED)/neonvm-controller.yaml + $(KUSTOMIZE) build neonvm-vxlan-controller > $(RENDERED)/neonvm-vxlan-controller.yaml + $(KUSTOMIZE) build neonvm-runner/image-loader > $(RENDERED)/neonvm-runner-image-loader.yaml + $(KUSTOMIZE) build autoscale-scheduler > $(RENDERED)/autoscale-scheduler.yaml + $(KUSTOMIZE) build autoscaler-agent > $(RENDERED)/autoscaler-agent.yaml # Cleanup: - cd neonvm/config/controller && $(KUSTOMIZE) edit set image controller=controller:dev && $(KUSTOMIZE) edit remove annotation buildtime --ignore-non-existence - cd neonvm/config/vxlan-controller && $(KUSTOMIZE) edit set image vxlan-controller=vxlan-controller:dev && $(KUSTOMIZE) edit remove annotation buildtime --ignore-non-existence - cd neonvm/runner-image-loader/bases && $(KUSTOMIZE) edit set image runner=runner:dev && $(KUSTOMIZE) edit remove annotation buildtime --ignore-non-existence - cd deploy/scheduler && $(KUSTOMIZE) edit set image autoscale-scheduler=autoscale-scheduler:dev && $(KUSTOMIZE) edit remove annotation buildtime --ignore-non-existence - cd deploy/agent && $(KUSTOMIZE) edit set image autoscaler-agent=autoscaler-agent:dev && $(KUSTOMIZE) edit remove annotation buildtime --ignore-non-existence + cd neonvm-controller && $(KUSTOMIZE) edit set image controller=controller:dev && $(KUSTOMIZE) edit remove annotation buildtime --ignore-non-existence + cd neonvm-vxlan-controller && $(KUSTOMIZE) edit set image vxlan-controller=vxlan-controller:dev && $(KUSTOMIZE) edit remove annotation buildtime --ignore-non-existence + cd neonvm-runner/image-loader/bases && $(KUSTOMIZE) edit set image runner=runner:dev && $(KUSTOMIZE) edit remove annotation buildtime --ignore-non-existence + cd autoscale-scheduler && $(KUSTOMIZE) edit set image autoscale-scheduler=autoscale-scheduler:dev && $(KUSTOMIZE) edit remove annotation buildtime --ignore-non-existence + cd autoscaler-agent && $(KUSTOMIZE) edit set image autoscaler-agent=autoscaler-agent:dev && $(KUSTOMIZE) edit remove annotation buildtime --ignore-non-existence render-release: $(RENDERED) kustomize # Prepare: - cd neonvm/config/controller && $(KUSTOMIZE) edit set image controller=$(IMG_CONTROLLER) - cd neonvm/config/vxlan-controller && $(KUSTOMIZE) edit set image vxlan-controller=$(IMG_VXLAN_CONTROLLER) - cd neonvm/runner-image-loader/bases && $(KUSTOMIZE) edit set image runner=$(IMG_RUNNER) - cd deploy/scheduler && $(KUSTOMIZE) edit set image autoscale-scheduler=$(IMG_SCHEDULER) - cd deploy/agent && $(KUSTOMIZE) edit set image autoscaler-agent=$(IMG_AUTOSCALER_AGENT) + cd neonvm-controller && $(KUSTOMIZE) edit set image controller=$(IMG_CONTROLLER) + cd neonvm-vxlan-controller && $(KUSTOMIZE) edit set image vxlan-controller=$(IMG_VXLAN_CONTROLLER) + cd neonvm-runner/image-loader && $(KUSTOMIZE) edit set image runner=$(IMG_RUNNER) + cd autoscale-scheduler && $(KUSTOMIZE) edit set image autoscale-scheduler=$(IMG_SCHEDULER) + cd autoscaler-agent && $(KUSTOMIZE) edit set image autoscaler-agent=$(IMG_AUTOSCALER_AGENT) # Build: $(KUSTOMIZE) build neonvm/config/whereabouts > $(RENDERED)/whereabouts.yaml $(KUSTOMIZE) build neonvm/config/multus-eks > $(RENDERED)/multus-eks.yaml $(KUSTOMIZE) build neonvm/config/multus > $(RENDERED)/multus.yaml $(KUSTOMIZE) build neonvm/config > $(RENDERED)/neonvm.yaml - $(KUSTOMIZE) build neonvm/runner-image-loader > $(RENDERED)/neonvm-runner-image-loader.yaml - $(KUSTOMIZE) build deploy/scheduler > $(RENDERED)/autoscale-scheduler.yaml - $(KUSTOMIZE) build deploy/agent > $(RENDERED)/autoscaler-agent.yaml + $(KUSTOMIZE) build neonvm-controller > $(RENDERED)/neonvm-controller.yaml + $(KUSTOMIZE) build neonvm-vxlan-controller > $(RENDERED)/neonvm-vxlan-controller.yaml + $(KUSTOMIZE) build neonvm-runner/image-loader > $(RENDERED)/neonvm-runner-image-loader.yaml + $(KUSTOMIZE) build autoscale-scheduler > $(RENDERED)/autoscale-scheduler.yaml + $(KUSTOMIZE) build autoscaler-agent > $(RENDERED)/autoscaler-agent.yaml # Cleanup: - cd neonvm/config/controller && $(KUSTOMIZE) edit set image controller=controller:dev - cd neonvm/config/vxlan-controller && $(KUSTOMIZE) edit set image vxlan-controller=vxlan-controller:dev - cd neonvm/runner-image-loader/bases && $(KUSTOMIZE) edit set image runner=runner:dev - cd deploy/scheduler && $(KUSTOMIZE) edit set image autoscale-scheduler=autoscale-scheduler:dev - cd deploy/agent && $(KUSTOMIZE) edit set image autoscaler-agent=autoscaler-agent:dev + cd neonvm-controller && $(KUSTOMIZE) edit set image controller=controller:dev + cd neonvm-vxlan-controller && $(KUSTOMIZE) edit set image vxlan-controller=vxlan-controller:dev + cd neonvm-runner/image-loader/bases && $(KUSTOMIZE) edit set image runner=runner:dev + cd autoscale-scheduler && $(KUSTOMIZE) edit set image autoscale-scheduler=autoscale-scheduler:dev + cd autoscaler-agent && $(KUSTOMIZE) edit set image autoscaler-agent=autoscaler-agent:dev .PHONY: deploy deploy: check-local-context docker-build load-images render-manifests kubectl ## Deploy controller to the K8s cluster specified in ~/.kube/config. @@ -300,9 +304,11 @@ deploy: check-local-context docker-build load-images render-manifests kubectl ## $(KUBECTL) apply -f $(RENDERED)/neonvm-runner-image-loader.yaml $(KUBECTL) -n neonvm-system rollout status daemonset neonvm-runner-image-loader $(KUBECTL) apply -f $(RENDERED)/neonvm.yaml - $(KUBECTL) -n neonvm-system rollout status daemonset neonvm-device-plugin - $(KUBECTL) -n neonvm-system rollout status daemonset neonvm-vxlan-controller + $(KUBECTL) -n neonvm-system rollout status daemonset neonvm-device-plugin + $(KUBECTL) apply -f $(RENDERED)/neonvm-controller.yaml $(KUBECTL) -n neonvm-system rollout status deployment neonvm-controller + $(KUBECTL) apply -f $(RENDERED)/neonvm-vxlan-controller.yaml + $(KUBECTL) -n neonvm-system rollout status daemonset neonvm-vxlan-controller # NB: typical upgrade path requires updated scheduler before autoscaler-agents. $(KUBECTL) apply -f $(RENDERED)/autoscale-scheduler.yaml $(KUBECTL) -n kube-system rollout status deployment autoscale-scheduler diff --git a/neonvm/README.md b/README-NeonVM.md similarity index 87% rename from neonvm/README.md rename to README-NeonVM.md index fb2a79fcb..4e25c3889 100644 --- a/neonvm/README.md +++ b/README-NeonVM.md @@ -18,6 +18,8 @@ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/latest/do kubectl apply -f https://github.com/neondatabase/autoscaling/releases/latest/download/multus.yaml kubectl apply -f https://github.com/neondatabase/autoscaling/releases/latest/download/whereabouts.yaml kubectl apply -f https://github.com/neondatabase/autoscaling/releases/latest/download/neonvm.yaml +kubectl apply -f https://github.com/neondatabase/autoscaling/releases/latest/download/neonvm-vxlan-controller.yaml +kubectl apply -f https://github.com/neondatabase/autoscaling/releases/latest/download/neonvm-controller.yaml ``` ### Run virtual machine @@ -111,16 +113,7 @@ make k3d-setup make kernel ``` -(Alternatively, pull & extract it from Dockerhub) - -To adjust the kernel config: - -``` -cd hack/kernel -docker build --build-arg KERNEL_VERSION=6.1.92 --platform linux/x86_64 --target build-deps -t kernel-build-deps -f Dockerfile.kernel-builder . -docker run --rm -v $PWD:/host --name kernel-build -it kernel-build-deps bash -# inside that bash shell, do the menuconfig, then copy-out the config to /host -``` +For more on the kernel, see [neonvm-kernel/](./neonvm-kernel/). #### 3. Build and deploy controller and VXLAN overlay network to local cluster @@ -290,20 +283,3 @@ which provides a reconcile function responsible for synchronizing resources unti - [x] Simplify VM disk image creation from any docker image - [ ] ARM64 support - -## License - -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - diff --git a/README.md b/README.md index e6e4464af..c73cb6921 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ We've tried a bunch of existing tools and settled on the following: * Use [VM live migration](https://www.qemu.org/docs/master/devel/migration/index.html) to move running postgres instances between physical nodes * QEMU is used as our hypervisor -* [NeonVM](https://github.com/neondatabase/autoscaling/tree/main/neonvm) orchestrates NeonVM VMs as custom resources in +* [NeonVM](./README-NeonVM.md) orchestrates NeonVM VMs as custom resources in K8s, and is responsible for scaling allocated resources (CPU and memory) * A modified K8s scheduler ensures that we don't overcommit resources and triggers migrations when demand is above a pre-configured threshold diff --git a/build/autoscale-scheduler/Dockerfile b/autoscale-scheduler/Dockerfile similarity index 90% rename from build/autoscale-scheduler/Dockerfile rename to autoscale-scheduler/Dockerfile index aa972d9b2..f707a3639 100644 --- a/build/autoscale-scheduler/Dockerfile +++ b/autoscale-scheduler/Dockerfile @@ -12,7 +12,7 @@ COPY neonvm/client neonvm/client COPY pkg/api pkg/api COPY pkg/plugin pkg/plugin COPY pkg/util pkg/util -COPY cmd/autoscale-scheduler cmd/autoscale-scheduler +COPY autoscale-scheduler/cmd autoscale-scheduler/cmd ARG GIT_INFO @@ -22,7 +22,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ -mod readonly \ # -ldflags "-X ..." allows us to overwrite the value of a variable in a package \ -ldflags "-X 'github.com/neondatabase/autoscaling/pkg/util.BuildGitInfo=$GIT_INFO'" \ - cmd/autoscale-scheduler/main.go + autoscale-scheduler/cmd/*.go FROM alpine COPY --from=builder /workspace/main /usr/bin/kube-scheduler diff --git a/cmd/autoscale-scheduler/main.go b/autoscale-scheduler/cmd/main.go similarity index 100% rename from cmd/autoscale-scheduler/main.go rename to autoscale-scheduler/cmd/main.go diff --git a/deploy/scheduler/config_map.yaml b/autoscale-scheduler/config_map.yaml similarity index 100% rename from deploy/scheduler/config_map.yaml rename to autoscale-scheduler/config_map.yaml diff --git a/deploy/scheduler/deployment.yaml b/autoscale-scheduler/deployment.yaml similarity index 100% rename from deploy/scheduler/deployment.yaml rename to autoscale-scheduler/deployment.yaml diff --git a/deploy/scheduler/kustomization.yaml b/autoscale-scheduler/kustomization.yaml similarity index 100% rename from deploy/scheduler/kustomization.yaml rename to autoscale-scheduler/kustomization.yaml diff --git a/deploy/scheduler/role_binding.yaml b/autoscale-scheduler/role_binding.yaml similarity index 100% rename from deploy/scheduler/role_binding.yaml rename to autoscale-scheduler/role_binding.yaml diff --git a/deploy/scheduler/service_account.yaml b/autoscale-scheduler/service_account.yaml similarity index 100% rename from deploy/scheduler/service_account.yaml rename to autoscale-scheduler/service_account.yaml diff --git a/build/autoscaler-agent/Dockerfile b/autoscaler-agent/Dockerfile similarity index 91% rename from build/autoscaler-agent/Dockerfile rename to autoscaler-agent/Dockerfile index 1d95bbd8a..d869937b3 100644 --- a/build/autoscaler-agent/Dockerfile +++ b/autoscaler-agent/Dockerfile @@ -13,7 +13,7 @@ COPY pkg/agent pkg/agent COPY pkg/api pkg/api COPY pkg/billing pkg/billing COPY pkg/util pkg/util -COPY cmd/autoscaler-agent cmd/autoscaler-agent +COPY autoscaler-agent/cmd autoscaler-agent/cmd ARG GIT_INFO @@ -23,7 +23,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ -mod readonly \ # -ldflags "-X ..." allows us to overwrite the value of a variable in a package \ -ldflags "-X 'github.com/neondatabase/autoscaling/pkg/util.BuildGitInfo=$GIT_INFO'" \ - cmd/autoscaler-agent/main.go + autoscaler-agent/cmd/*.go FROM alpine COPY --from=builder /workspace/main /usr/bin/autoscaler-agent diff --git a/cmd/autoscaler-agent/main.go b/autoscaler-agent/cmd/main.go similarity index 100% rename from cmd/autoscaler-agent/main.go rename to autoscaler-agent/cmd/main.go diff --git a/deploy/agent/config_map.yaml b/autoscaler-agent/config_map.yaml similarity index 100% rename from deploy/agent/config_map.yaml rename to autoscaler-agent/config_map.yaml diff --git a/deploy/agent/daemonset.yaml b/autoscaler-agent/daemonset.yaml similarity index 100% rename from deploy/agent/daemonset.yaml rename to autoscaler-agent/daemonset.yaml diff --git a/deploy/agent/kustomization.yaml b/autoscaler-agent/kustomization.yaml similarity index 100% rename from deploy/agent/kustomization.yaml rename to autoscaler-agent/kustomization.yaml diff --git a/deploy/agent/role_binding.yaml b/autoscaler-agent/role_binding.yaml similarity index 100% rename from deploy/agent/role_binding.yaml rename to autoscaler-agent/role_binding.yaml diff --git a/deploy/agent/service_account.yaml b/autoscaler-agent/service_account.yaml similarity index 100% rename from deploy/agent/service_account.yaml rename to autoscaler-agent/service_account.yaml diff --git a/deploy/.gitignore b/deploy/.gitignore deleted file mode 100644 index 74ca1a220..000000000 --- a/deploy/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -flannel.yaml -cert-manager.yaml -multus-daemonset.yaml -neonvm.yaml diff --git a/neonvm/Dockerfile b/neonvm-controller/Dockerfile similarity index 77% rename from neonvm/Dockerfile rename to neonvm-controller/Dockerfile index 550759269..dd86c567f 100644 --- a/neonvm/Dockerfile +++ b/neonvm-controller/Dockerfile @@ -13,21 +13,19 @@ COPY go.sum go.sum RUN go mod download # Copy the go source -COPY pkg/util pkg/util -COPY neonvm/main.go neonvm/main.go -COPY neonvm/apis/ neonvm/apis/ -COPY neonvm/controllers/ neonvm/controllers/ -COPY neonvm/pkg/ neonvm/pkg/ -COPY neonvm/client/ neonvm/client/ -COPY pkg/api/ pkg/api -COPY pkg/util pkg/util +COPY pkg/api/ pkg/api +COPY pkg/util pkg/util +COPY pkg/neonvm/ pkg/neonvm +COPY neonvm/apis neonvm/apis +COPY neonvm/client neonvm/client +COPY neonvm-controller/cmd neonvm-controller/cmd # Build # the GOARCH has not a default value to allow the binary be built according to the host where the command # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -tags=${BUILDTAGS} -o manager neonvm/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -tags=${BUILDTAGS} -o manager neonvm-controller/cmd/*.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/neonvm/main.go b/neonvm-controller/cmd/main.go similarity index 99% rename from neonvm/main.go rename to neonvm-controller/cmd/main.go index 99f2e8ba2..3dd3b78f7 100644 --- a/neonvm/main.go +++ b/neonvm-controller/cmd/main.go @@ -51,7 +51,7 @@ import ( "k8s.io/klog/v2" vmv1 "github.com/neondatabase/autoscaling/neonvm/apis/neonvm/v1" - "github.com/neondatabase/autoscaling/neonvm/controllers" + "github.com/neondatabase/autoscaling/pkg/neonvm/controllers" "github.com/neondatabase/autoscaling/pkg/util" ) diff --git a/neonvm/config/controller/deployment.yaml b/neonvm-controller/deployment.yaml similarity index 100% rename from neonvm/config/controller/deployment.yaml rename to neonvm-controller/deployment.yaml diff --git a/neonvm-controller/kustomization.yaml b/neonvm-controller/kustomization.yaml new file mode 100644 index 000000000..b94e2c1e5 --- /dev/null +++ b/neonvm-controller/kustomization.yaml @@ -0,0 +1,66 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Add neonvm-system namespace to all resources +namespace: neonvm-system +# Prepend 'neonvm-' to all resource names. +namePrefix: neonvm- + +# double-include overlay networks so we can reference them correctly +resources: +- deployment.yaml +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +- ../neonvm/config/network +- ../neonvm-runner/image-loader/bases + +images: +- name: controller + newName: controller + newTag: dev + +# overlay networking variables +vars: +- fieldref: + fieldPath: spec.template.spec.containers.[name=neonvm-runner-loader].image + name: VM_RUNNER_IMAGE + objref: + group: apps + kind: DaemonSet + name: runner-image-loader + version: v1 +- fieldref: + fieldPath: metadata.name + name: NAD_IPAM_NAME + objref: + group: k8s.cni.cncf.io + kind: NetworkAttachmentDefinition + name: overlay-ipam + version: v1 +- fieldref: + fieldPath: metadata.namespace + name: NAD_IPAM_NAMESPACE + objref: + group: k8s.cni.cncf.io + kind: NetworkAttachmentDefinition + name: overlay-ipam + version: v1 +- fieldref: + fieldPath: metadata.name + name: NAD_RUNNER_NAME + objref: + group: k8s.cni.cncf.io + kind: NetworkAttachmentDefinition + name: overlay-for-vms + version: v1 +- fieldref: + fieldPath: metadata.namespace + name: NAD_RUNNER_NAMESPACE + objref: + group: k8s.cni.cncf.io + kind: NetworkAttachmentDefinition + name: overlay-for-vms + version: v1 diff --git a/neonvm/config/rbac/leader_election_role.yaml b/neonvm-controller/leader_election_role.yaml similarity index 100% rename from neonvm/config/rbac/leader_election_role.yaml rename to neonvm-controller/leader_election_role.yaml diff --git a/neonvm/config/rbac/leader_election_role_binding.yaml b/neonvm-controller/leader_election_role_binding.yaml similarity index 100% rename from neonvm/config/rbac/leader_election_role_binding.yaml rename to neonvm-controller/leader_election_role_binding.yaml diff --git a/neonvm/config/rbac/role.yaml b/neonvm-controller/role.yaml similarity index 100% rename from neonvm/config/rbac/role.yaml rename to neonvm-controller/role.yaml diff --git a/neonvm/config/rbac/role_binding.yaml b/neonvm-controller/role_binding.yaml similarity index 100% rename from neonvm/config/rbac/role_binding.yaml rename to neonvm-controller/role_binding.yaml diff --git a/neonvm/config/rbac/service_account.yaml b/neonvm-controller/service_account.yaml similarity index 100% rename from neonvm/config/rbac/service_account.yaml rename to neonvm-controller/service_account.yaml diff --git a/neonvm-kernel/.gitignore b/neonvm-kernel/.gitignore new file mode 100644 index 000000000..00023b26b --- /dev/null +++ b/neonvm-kernel/.gitignore @@ -0,0 +1 @@ +vmlinuz diff --git a/neonvm/hack/kernel/Dockerfile.kernel-builder b/neonvm-kernel/Dockerfile.kernel-builder similarity index 100% rename from neonvm/hack/kernel/Dockerfile.kernel-builder rename to neonvm-kernel/Dockerfile.kernel-builder diff --git a/neonvm-kernel/README.md b/neonvm-kernel/README.md new file mode 100644 index 000000000..533ad24d4 --- /dev/null +++ b/neonvm-kernel/README.md @@ -0,0 +1,54 @@ +# NeonVM kernel + +We build a custom kernel in order to: + +1. Have support for unusual features that are only required for VMs (e.g. hotplugging, kvm-ptp clock + synchronization, etc.) +2. Avoid including features we don't need + +Kernel images are all at `neondatabase/vm-kernel:$tag`, built by the +[vm-kernel](../.github/workflows/vm-kernel.yaml) github workflow. + +Kernel images are typically assigned to a VM based on what was bundled in with the `neonvm-runner` +in use, although this can be overridden on an individual VM basis using the +`.spec.guest.kernelImage` field. + +## Upgrading the kernel + +Assuming a plain upgrade (i.e. no additional features to enable), upgrading the kernel can be done +with the following sequence of actions: + +1. On the host, run: + ```sh + cd neonvm-kernel # this directory + docker build --build-arg KERNEL_VERSION=$NEW_VERSION --platform linux/x86_64 --target build-deps -t kernel-build-deps -f Dockerfile.kernel-builder . + docker run --rm -v $PWD:/host --name kernel-build -it kernel-build-deps bash + ``` +2. Then, inside the container, run: + ```sh + cd linux-$NEW_VERSION + cp /host/linux-config-6.1.92 .config # Copy current config in + make menuconfig + # do nothing; just save and exit, overwriting .config + cp .config /host/linux-config-$NEW_VERSION # NOTE: Different from existing! + ``` +3. Back on the host, finish with: + ```sh + # compare the two versions + diff linux-config-6.1.92 linux-config-$NEW_VERSION + # If all looks good, delete the old version. This is required so auto-selection works. + rm linux-config-6.1.92 + ``` + +Afterwards, it's probably also good to do a search-and-replace repo-wide to update all places that +mention the old kernel version. + +## Adjusting the config + +To adjust the kernel config, try the following from this directory: + +```sh +docker build --build-arg KERNEL_VERSION=6.1.92 --platform linux/x86_64 --target build-deps -t kernel-build-deps -f Dockerfile.kernel-builder . +docker run --rm -v $PWD:/host --name kernel-build -it kernel-build-deps bash +# inside that bash shell, do the menuconfig, then copy-out the config to /host +``` diff --git a/neonvm/hack/kernel/linux-config-6.1.92 b/neonvm-kernel/linux-config-6.1.92 similarity index 100% rename from neonvm/hack/kernel/linux-config-6.1.92 rename to neonvm-kernel/linux-config-6.1.92 diff --git a/neonvm/runner/Dockerfile b/neonvm-runner/Dockerfile similarity index 77% rename from neonvm/runner/Dockerfile rename to neonvm-runner/Dockerfile index 75faaa9a2..27effec66 100644 --- a/neonvm/runner/Dockerfile +++ b/neonvm-runner/Dockerfile @@ -12,20 +12,18 @@ COPY go.sum go.sum RUN go mod download # Copy the go source -COPY neonvm/main.go neonvm/main.go -COPY neonvm/apis/ neonvm/apis/ -COPY neonvm/controllers/ neonvm/controllers/ -COPY neonvm/runner/ neonvm/runner/ -COPY pkg/api/ pkg/api -COPY pkg/util pkg/util +COPY neonvm/apis neonvm/apis +COPY pkg/api pkg/api +COPY pkg/util pkg/util +COPY neonvm-runner/cmd neonvm-runner/cmd # Build # the GOARCH has not a default value to allow the binary be built according to the host where the command # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /runner neonvm/runner/main.go -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /container-mgr neonvm/runner/container-mgr/*.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /runner neonvm-runner/cmd/*.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /container-mgr neonvm-runner/cmd/container-mgr/*.go FROM alpine:3.18 AS crictl @@ -57,10 +55,10 @@ RUN apk add --no-cache \ cgroup-tools \ openssh -COPY --from=builder /runner /usr/bin/runner +COPY --from=builder /runner /usr/bin/runner COPY --from=builder /container-mgr /usr/bin/container-mgr -COPY --from=crictl /crictl /usr/bin/crictl -COPY neonvm/hack/kernel/vmlinuz /vm/kernel/vmlinuz -COPY neonvm/runner/ssh_config /etc/ssh/ssh_config +COPY --from=crictl /crictl /usr/bin/crictl +COPY neonvm-kernel/vmlinuz /vm/kernel/vmlinuz +COPY neonvm-runner/ssh_config /etc/ssh/ssh_config ENTRYPOINT ["/sbin/tini", "--", "runner"] diff --git a/neonvm/runner/container-mgr/crictl.go b/neonvm-runner/cmd/container-mgr/crictl.go similarity index 100% rename from neonvm/runner/container-mgr/crictl.go rename to neonvm-runner/cmd/container-mgr/crictl.go diff --git a/neonvm/runner/container-mgr/main.go b/neonvm-runner/cmd/container-mgr/main.go similarity index 100% rename from neonvm/runner/container-mgr/main.go rename to neonvm-runner/cmd/container-mgr/main.go diff --git a/neonvm/runner/main.go b/neonvm-runner/cmd/main.go similarity index 100% rename from neonvm/runner/main.go rename to neonvm-runner/cmd/main.go diff --git a/neonvm/runner-image-loader/bases/daemonset.yaml b/neonvm-runner/image-loader/bases/daemonset.yaml similarity index 100% rename from neonvm/runner-image-loader/bases/daemonset.yaml rename to neonvm-runner/image-loader/bases/daemonset.yaml diff --git a/neonvm/runner-image-loader/bases/kustomization.yaml b/neonvm-runner/image-loader/bases/kustomization.yaml similarity index 100% rename from neonvm/runner-image-loader/bases/kustomization.yaml rename to neonvm-runner/image-loader/bases/kustomization.yaml diff --git a/neonvm-runner/image-loader/daemonset.yaml b/neonvm-runner/image-loader/daemonset.yaml new file mode 100644 index 000000000..d24f28d28 --- /dev/null +++ b/neonvm-runner/image-loader/daemonset.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: runner-image-loader +spec: + selector: + matchLabels: + name: neonvm-runner-image-loader + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 100% + template: + metadata: + labels: + name: neonvm-runner-image-loader + spec: + containers: + # NOTE: this image is referenced by kustomize in order to set the value of the + # neonvm-controller's VM_RUNNER_IMAGE env var. We do it that way so that we can use + # 'kustomize edit image' to set the runner image, even if it does require some + # spooky action at a distance. + - image: runner:dev + name: neonvm-runner-loader + command: ["sh", "-c", "echo 'image loaded and container started' && sleep 100d"] + resources: + limits: + cpu: 10m + memory: 10Mi + # Sleep never gracefully terminates. No point waiting for it to. + terminationGracePeriodSeconds: 0 diff --git a/neonvm/runner-image-loader/kustomization.yaml b/neonvm-runner/image-loader/kustomization.yaml similarity index 89% rename from neonvm/runner-image-loader/kustomization.yaml rename to neonvm-runner/image-loader/kustomization.yaml index c2a84289c..ad422a9af 100644 --- a/neonvm/runner-image-loader/kustomization.yaml +++ b/neonvm-runner/image-loader/kustomization.yaml @@ -8,7 +8,8 @@ namePrefix: neonvm- resources: - bases -- ../config/namespace +- ../../neonvm/config/namespace + images: - name: runner newName: runner diff --git a/neonvm/runner/ssh_config b/neonvm-runner/ssh_config similarity index 100% rename from neonvm/runner/ssh_config rename to neonvm-runner/ssh_config diff --git a/neonvm/tools/vxlan/Dockerfile b/neonvm-vxlan-controller/Dockerfile similarity index 91% rename from neonvm/tools/vxlan/Dockerfile rename to neonvm-vxlan-controller/Dockerfile index c7f4521d2..ab3d37c26 100644 --- a/neonvm/tools/vxlan/Dockerfile +++ b/neonvm-vxlan-controller/Dockerfile @@ -12,14 +12,14 @@ COPY go.sum go.sum RUN go mod download # Copy the go source -COPY neonvm/tools/vxlan/controller/ neonvm/tools/vxlan/controller/ +COPY neonvm-vxlan-controller/cmd neonvm-vxlan-controller/cmd # Build # the GOARCH has not a default value to allow the binary be built according to the host where the command # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /vxlan-controller neonvm/tools/vxlan/controller/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /vxlan-controller neonvm-vxlan-controller/cmd/*.go FROM alpine:3.18 diff --git a/neonvm/tools/vxlan/controller/main.go b/neonvm-vxlan-controller/cmd/main.go similarity index 100% rename from neonvm/tools/vxlan/controller/main.go rename to neonvm-vxlan-controller/cmd/main.go diff --git a/neonvm/config/vxlan-controller/daemonset.yaml b/neonvm-vxlan-controller/daemonset.yaml similarity index 100% rename from neonvm/config/vxlan-controller/daemonset.yaml rename to neonvm-vxlan-controller/daemonset.yaml diff --git a/neonvm/config/vxlan-controller/kustomization.yaml b/neonvm-vxlan-controller/kustomization.yaml similarity index 57% rename from neonvm/config/vxlan-controller/kustomization.yaml rename to neonvm-vxlan-controller/kustomization.yaml index f07243cb8..3397cfac6 100644 --- a/neonvm/config/vxlan-controller/kustomization.yaml +++ b/neonvm-vxlan-controller/kustomization.yaml @@ -1,6 +1,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +# Add neonvm-system namespace to all resources +namespace: neonvm-system +# Prepend 'neonvm-' to all resource names. +namePrefix: neonvm- + resources: - daemonset.yaml - rbac.yaml diff --git a/neonvm/config/vxlan-controller/rbac.yaml b/neonvm-vxlan-controller/rbac.yaml similarity index 100% rename from neonvm/config/vxlan-controller/rbac.yaml rename to neonvm-vxlan-controller/rbac.yaml diff --git a/neonvm/config/controller/kustomization.yaml b/neonvm/config/controller/kustomization.yaml deleted file mode 100644 index eb633f0a1..000000000 --- a/neonvm/config/controller/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: -- deployment.yaml -images: -- name: controller - newName: controller - newTag: dev diff --git a/neonvm/config/kustomization.yaml b/neonvm/config/kustomization.yaml index c549b2815..82191926a 100644 --- a/neonvm/config/kustomization.yaml +++ b/neonvm/config/kustomization.yaml @@ -16,29 +16,18 @@ bases: - namespace - crd - rbac -- controller - webhook - certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../common/prometheus -- vxlan-controller - network - device-plugin -- ../runner-image-loader/bases patchesStrategicMerge: - webhookcainjection_patch.yaml # the following config is for teaching kustomize how to do var substitution vars: -- name: VM_RUNNER_IMAGE # full name of the neonvm-runner image - objref: - kind: DaemonSet - name: runner-image-loader - group: apps - version: v1 - fieldref: - fieldpath: spec.template.spec.containers.[name=neonvm-runner-loader].image - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR objref: kind: Certificate @@ -65,36 +54,3 @@ vars: kind: Service version: v1 name: webhook-service -# overlay networking variables -- name: NAD_IPAM_NAME - objref: - kind: NetworkAttachmentDefinition - group: k8s.cni.cncf.io - version: v1 - name: overlay-ipam # this name should match the one in overlay-ipam.yaml - fieldref: - fieldpath: metadata.name -- name: NAD_IPAM_NAMESPACE - objref: - kind: NetworkAttachmentDefinition - group: k8s.cni.cncf.io - version: v1 - name: overlay-ipam # this name should match the one in overlay-ipam.yaml - fieldref: - fieldpath: metadata.namespace -- name: NAD_RUNNER_NAME - objref: - kind: NetworkAttachmentDefinition - group: k8s.cni.cncf.io - version: v1 - name: overlay-for-vms # this name should match the one in overlay-for-vms.yaml - fieldref: - fieldpath: metadata.name -- name: NAD_RUNNER_NAMESPACE - objref: - kind: NetworkAttachmentDefinition - group: k8s.cni.cncf.io - version: v1 - name: overlay-for-vms # this name should match the one in overlay-for-vms.yaml - fieldref: - fieldpath: metadata.namespace diff --git a/neonvm/config/rbac/kustomization.yaml b/neonvm/config/rbac/kustomization.yaml index 371aaa137..8a95ef863 100644 --- a/neonvm/config/rbac/kustomization.yaml +++ b/neonvm/config/rbac/kustomization.yaml @@ -4,11 +4,6 @@ resources: # if your manager will use a service account that exists at # runtime. Be sure to update RoleBinding and ClusterRoleBinding # subjects if changing service account names. -- service_account.yaml -- role.yaml -- role_binding.yaml -- leader_election_role.yaml -- leader_election_role_binding.yaml - virtualmachine_viewer_role.yaml - virtualmachine_editor_role.yaml - virtualmachinemigration_viewer_role.yaml diff --git a/neonvm/controllers/buildtag/nodelete_false.go b/pkg/neonvm/controllers/buildtag/nodelete_false.go similarity index 100% rename from neonvm/controllers/buildtag/nodelete_false.go rename to pkg/neonvm/controllers/buildtag/nodelete_false.go diff --git a/neonvm/controllers/buildtag/nodelete_true.go b/pkg/neonvm/controllers/buildtag/nodelete_true.go similarity index 100% rename from neonvm/controllers/buildtag/nodelete_true.go rename to pkg/neonvm/controllers/buildtag/nodelete_true.go diff --git a/neonvm/controllers/buildtag/tagnames.go b/pkg/neonvm/controllers/buildtag/tagnames.go similarity index 100% rename from neonvm/controllers/buildtag/tagnames.go rename to pkg/neonvm/controllers/buildtag/tagnames.go diff --git a/neonvm/controllers/catch_panic.go b/pkg/neonvm/controllers/catch_panic.go similarity index 100% rename from neonvm/controllers/catch_panic.go rename to pkg/neonvm/controllers/catch_panic.go diff --git a/neonvm/controllers/config.go b/pkg/neonvm/controllers/config.go similarity index 100% rename from neonvm/controllers/config.go rename to pkg/neonvm/controllers/config.go diff --git a/neonvm/controllers/failurelag/tracker.go b/pkg/neonvm/controllers/failurelag/tracker.go similarity index 100% rename from neonvm/controllers/failurelag/tracker.go rename to pkg/neonvm/controllers/failurelag/tracker.go diff --git a/neonvm/controllers/failurelag/tracker_test.go b/pkg/neonvm/controllers/failurelag/tracker_test.go similarity index 97% rename from neonvm/controllers/failurelag/tracker_test.go rename to pkg/neonvm/controllers/failurelag/tracker_test.go index f191f53f9..8c44f400a 100644 --- a/neonvm/controllers/failurelag/tracker_test.go +++ b/pkg/neonvm/controllers/failurelag/tracker_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/neondatabase/autoscaling/neonvm/controllers/failurelag" + "github.com/neondatabase/autoscaling/pkg/neonvm/controllers/failurelag" ) type nowMock struct { diff --git a/neonvm/controllers/functests/suite_test.go b/pkg/neonvm/controllers/functests/suite_test.go similarity index 95% rename from neonvm/controllers/functests/suite_test.go rename to pkg/neonvm/controllers/functests/suite_test.go index 2c52d2ce5..12315a312 100644 --- a/neonvm/controllers/functests/suite_test.go +++ b/pkg/neonvm/controllers/functests/suite_test.go @@ -51,7 +51,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "neonvm", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, } diff --git a/neonvm/controllers/functests/vm_controller_test.go b/pkg/neonvm/controllers/functests/vm_controller_test.go similarity index 98% rename from neonvm/controllers/functests/vm_controller_test.go rename to pkg/neonvm/controllers/functests/vm_controller_test.go index 2ec7280fa..f22b7bde0 100644 --- a/neonvm/controllers/functests/vm_controller_test.go +++ b/pkg/neonvm/controllers/functests/vm_controller_test.go @@ -31,7 +31,7 @@ import ( "k8s.io/apimachinery/pkg/types" vmv1 "github.com/neondatabase/autoscaling/neonvm/apis/neonvm/v1" - "github.com/neondatabase/autoscaling/neonvm/controllers" + "github.com/neondatabase/autoscaling/pkg/neonvm/controllers" ) var _ = Describe("VirtualMachine controller", func() { diff --git a/neonvm/controllers/metrics.go b/pkg/neonvm/controllers/metrics.go similarity index 99% rename from neonvm/controllers/metrics.go rename to pkg/neonvm/controllers/metrics.go index b8791baf6..9ec978c59 100644 --- a/neonvm/controllers/metrics.go +++ b/pkg/neonvm/controllers/metrics.go @@ -15,7 +15,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" - "github.com/neondatabase/autoscaling/neonvm/controllers/failurelag" + "github.com/neondatabase/autoscaling/pkg/neonvm/controllers/failurelag" "github.com/neondatabase/autoscaling/pkg/util" ) diff --git a/neonvm/controllers/vm_controller.go b/pkg/neonvm/controllers/vm_controller.go similarity index 99% rename from neonvm/controllers/vm_controller.go rename to pkg/neonvm/controllers/vm_controller.go index aa045798e..ea433c9b4 100644 --- a/neonvm/controllers/vm_controller.go +++ b/pkg/neonvm/controllers/vm_controller.go @@ -53,9 +53,9 @@ import ( "k8s.io/client-go/tools/record" vmv1 "github.com/neondatabase/autoscaling/neonvm/apis/neonvm/v1" - "github.com/neondatabase/autoscaling/neonvm/controllers/buildtag" - "github.com/neondatabase/autoscaling/neonvm/pkg/ipam" "github.com/neondatabase/autoscaling/pkg/api" + "github.com/neondatabase/autoscaling/pkg/neonvm/controllers/buildtag" + "github.com/neondatabase/autoscaling/pkg/neonvm/ipam" "github.com/neondatabase/autoscaling/pkg/util/patch" ) diff --git a/neonvm/controllers/vm_controller_unit_test.go b/pkg/neonvm/controllers/vm_controller_unit_test.go similarity index 100% rename from neonvm/controllers/vm_controller_unit_test.go rename to pkg/neonvm/controllers/vm_controller_unit_test.go diff --git a/neonvm/controllers/vm_qmp_queries.go b/pkg/neonvm/controllers/vm_qmp_queries.go similarity index 100% rename from neonvm/controllers/vm_qmp_queries.go rename to pkg/neonvm/controllers/vm_qmp_queries.go diff --git a/neonvm/controllers/vm_qmp_test.go b/pkg/neonvm/controllers/vm_qmp_test.go similarity index 94% rename from neonvm/controllers/vm_qmp_test.go rename to pkg/neonvm/controllers/vm_qmp_test.go index 2be4ae2a8..d70bc5ea5 100644 --- a/neonvm/controllers/vm_qmp_test.go +++ b/pkg/neonvm/controllers/vm_qmp_test.go @@ -4,7 +4,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/neondatabase/autoscaling/neonvm/controllers" + "github.com/neondatabase/autoscaling/pkg/neonvm/controllers" ) type qmpEvent struct { diff --git a/neonvm/controllers/vmmigration_controller.go b/pkg/neonvm/controllers/vmmigration_controller.go similarity index 99% rename from neonvm/controllers/vmmigration_controller.go rename to pkg/neonvm/controllers/vmmigration_controller.go index ca9367f9a..83205336f 100644 --- a/neonvm/controllers/vmmigration_controller.go +++ b/pkg/neonvm/controllers/vmmigration_controller.go @@ -39,7 +39,7 @@ import ( "k8s.io/client-go/tools/record" vmv1 "github.com/neondatabase/autoscaling/neonvm/apis/neonvm/v1" - "github.com/neondatabase/autoscaling/neonvm/controllers/buildtag" + "github.com/neondatabase/autoscaling/pkg/neonvm/controllers/buildtag" ) const virtualmachinemigrationFinalizer = "vm.neon.tech/finalizer" diff --git a/neonvm/controllers/webhook.go b/pkg/neonvm/controllers/webhook.go similarity index 100% rename from neonvm/controllers/webhook.go rename to pkg/neonvm/controllers/webhook.go diff --git a/neonvm/pkg/README.md b/pkg/neonvm/ipam/README.md similarity index 66% rename from neonvm/pkg/README.md rename to pkg/neonvm/ipam/README.md index 25ea1c832..4abd2d3a5 100644 --- a/neonvm/pkg/README.md +++ b/pkg/neonvm/ipam/README.md @@ -3,17 +3,17 @@ ### Creat Network Attachment Definition ```console -kubectl apply -f ipam-demo-nad.yaml +kubectl apply -f demo/ipam-nad.yaml ``` ### Run example ```console -go run ipam-demo.go +go run demo/ipam.go ``` ### Delete Network Attachment Definition ```console -kubectl delete -f ipam-demo-nad.yaml +kubectl delete -f ipam-nad.yaml ``` diff --git a/neonvm/pkg/ipam/allocate.go b/pkg/neonvm/ipam/allocate.go similarity index 100% rename from neonvm/pkg/ipam/allocate.go rename to pkg/neonvm/ipam/allocate.go diff --git a/neonvm/pkg/ipam/client.go b/pkg/neonvm/ipam/client.go similarity index 100% rename from neonvm/pkg/ipam/client.go rename to pkg/neonvm/ipam/client.go diff --git a/neonvm/pkg/ipam-demo-nad.yaml b/pkg/neonvm/ipam/demo/ipam-nad.yaml similarity index 100% rename from neonvm/pkg/ipam-demo-nad.yaml rename to pkg/neonvm/ipam/demo/ipam-nad.yaml diff --git a/neonvm/pkg/ipam-demo.go b/pkg/neonvm/ipam/demo/ipam.go similarity index 97% rename from neonvm/pkg/ipam-demo.go rename to pkg/neonvm/ipam/demo/ipam.go index a486768e2..bbb8677f2 100644 --- a/neonvm/pkg/ipam-demo.go +++ b/pkg/neonvm/ipam/demo/ipam.go @@ -14,7 +14,7 @@ import ( "k8s.io/klog/v2" - "github.com/neondatabase/autoscaling/neonvm/pkg/ipam" + "github.com/neondatabase/autoscaling/pkg/neonvm/ipam" ) var ( diff --git a/neonvm/pkg/ipam/ipam.go b/pkg/neonvm/ipam/ipam.go similarity index 100% rename from neonvm/pkg/ipam/ipam.go rename to pkg/neonvm/ipam/ipam.go diff --git a/neonvm/pkg/ipam/types.go b/pkg/neonvm/ipam/types.go similarity index 100% rename from neonvm/pkg/ipam/types.go rename to pkg/neonvm/ipam/types.go diff --git a/neonvm/tools/vm-builder/files/Dockerfile.img b/vm-builder/files/Dockerfile.img similarity index 100% rename from neonvm/tools/vm-builder/files/Dockerfile.img rename to vm-builder/files/Dockerfile.img diff --git a/neonvm/tools/vm-builder/files/chrony.conf b/vm-builder/files/chrony.conf similarity index 100% rename from neonvm/tools/vm-builder/files/chrony.conf rename to vm-builder/files/chrony.conf diff --git a/neonvm/tools/vm-builder/files/helper.move-bins.sh b/vm-builder/files/helper.move-bins.sh similarity index 100% rename from neonvm/tools/vm-builder/files/helper.move-bins.sh rename to vm-builder/files/helper.move-bins.sh diff --git a/neonvm/tools/vm-builder/files/inittab b/vm-builder/files/inittab similarity index 100% rename from neonvm/tools/vm-builder/files/inittab rename to vm-builder/files/inittab diff --git a/neonvm/tools/vm-builder/files/resize-swap.sh b/vm-builder/files/resize-swap.sh similarity index 100% rename from neonvm/tools/vm-builder/files/resize-swap.sh rename to vm-builder/files/resize-swap.sh diff --git a/neonvm/tools/vm-builder/files/sshd_config b/vm-builder/files/sshd_config similarity index 100% rename from neonvm/tools/vm-builder/files/sshd_config rename to vm-builder/files/sshd_config diff --git a/neonvm/tools/vm-builder/files/udev-init.sh b/vm-builder/files/udev-init.sh similarity index 100% rename from neonvm/tools/vm-builder/files/udev-init.sh rename to vm-builder/files/udev-init.sh diff --git a/neonvm/tools/vm-builder/files/vector.yaml b/vm-builder/files/vector.yaml similarity index 100% rename from neonvm/tools/vm-builder/files/vector.yaml rename to vm-builder/files/vector.yaml diff --git a/neonvm/tools/vm-builder/files/vmacpi b/vm-builder/files/vmacpi similarity index 100% rename from neonvm/tools/vm-builder/files/vmacpi rename to vm-builder/files/vmacpi diff --git a/neonvm/tools/vm-builder/files/vminit b/vm-builder/files/vminit similarity index 100% rename from neonvm/tools/vm-builder/files/vminit rename to vm-builder/files/vminit diff --git a/neonvm/tools/vm-builder/files/vmshutdown b/vm-builder/files/vmshutdown similarity index 100% rename from neonvm/tools/vm-builder/files/vmshutdown rename to vm-builder/files/vmshutdown diff --git a/neonvm/tools/vm-builder/files/vmstart b/vm-builder/files/vmstart similarity index 100% rename from neonvm/tools/vm-builder/files/vmstart rename to vm-builder/files/vmstart diff --git a/neonvm/tools/vm-builder/main.go b/vm-builder/main.go similarity index 100% rename from neonvm/tools/vm-builder/main.go rename to vm-builder/main.go diff --git a/deploy/vmscrape.yaml b/vmscrape.yaml similarity index 100% rename from deploy/vmscrape.yaml rename to vmscrape.yaml