diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 071d52d93..c10652327 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -30,26 +30,39 @@ on: description: 'Should images be uploaded to neon ECR' type: boolean required: false + arch: + description: 'Architecture to build for' + type: string + required: false + default: 'amd64' + matrix-step-name: + required: false + type: string + matrix-key: + required: false + type: string + outputs: + controller: description: 'neonvm-controller image' - value: ${{ jobs.tags.outputs.controller }} + value: ${{ jobs.save-matrix-results.outputs.controller }} vxlan-controller: description: 'neonvm-vxlan-controller image' - value: ${{ jobs.tags.outputs.vxlan-controller }} + value: ${{ jobs.save-matrix-results.outputs.vxlan-controller}} runner: description: 'neonvm-runner image' - value: ${{ jobs.tags.outputs.runner }} + value: ${{ jobs.save-matrix-results.outputs.runner}} scheduler: description: 'autoscale-scheduler image' - value: ${{ jobs.tags.outputs.scheduler }} + value: ${{ jobs.save-matrix-results.outputs.scheduler}} autoscaler-agent: description: 'autoscaler-agent image' value: ${{ jobs.tags.outputs.autoscaler-agent }} daemon: description: 'neonvm-daemon image' value: ${{ jobs.tags.outputs.daemon }} - + env: IMG_CONTROLLER: "neondatabase/neonvm-controller" IMG_VXLAN_CONTROLLER: "neondatabase/neonvm-vxlan-controller" @@ -62,15 +75,6 @@ env: ECR_DEV: "369495373322.dkr.ecr.eu-central-1.amazonaws.com" ECR_PROD: "093970136003.dkr.ecr.eu-central-1.amazonaws.com" - # Why localhost? We use a local registry so that when docker/build-push-action tries to pull the - # image we built locally, it'll actually have a place to pull from. - # - # Otherwise, if we just try to use a local image, it fails trying to pull it from dockerhub. - # See https://github.com/moby/buildkit/issues/2343 for more information. - GO_BASE_IMG: "localhost:5000/neondatabase/autoscaling-go-base:dev" - # Default architecture to build. In future it would be changed to multi-arch build or separate builds for each arch - TARGET_ARCH: "amd64" - defaults: run: shell: bash -euo pipefail {0} @@ -104,17 +108,33 @@ jobs: with: tag: ${{ inputs.kernel-image || inputs.tag }} return-image-for-tag: ${{ inputs.kernel-image }} + arch: ${{ inputs.arch }} secrets: inherit build: # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'. if: ${{ format('{0}', inputs.skip) != 'true' }} - needs: [ tags, vm-kernel ] - runs-on: [ self-hosted, large ] + needs: [ vm-kernel ] + outputs: + controller: ${{ steps.tags.outputs.controller }} + vxlan-controller: ${{ steps.tags.outputs.vxlan-controller }} + runner: ${{ steps.tags.outputs.runner }} + scheduler: ${{ steps.tags.outputs.scheduler }} + autoscaler-agent: ${{ steps.tags.outputs.autoscaler-agent }} + cluster-autoscaler: ${{ steps.tags.outputs.cluster-autoscaler }} + env: + # Why localhost? We use a local registry so that when docker/build-push-action tries to pull the + # image we built locally, it'll actually have a place to pull from. + # + # Otherwise, if we just try to use a local image, it fails trying to pull it from dockerhub. + # See https://github.com/moby/buildkit/issues/2343 for more information. + GO_BASE_IMG: ${{ format('localhost:5000/neondatabase/autoscaling-go-base-{0}:dev', inputs.arch) }} permissions: contents: read # This is required for actions/checkout id-token: write # This is required for aws-actions/configure-aws-credentials + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} + services: registry: image: registry:2 @@ -122,6 +142,18 @@ jobs: - 5000:5000 steps: + # tags converted to be a step and moved here to be in the same strategy context + + - id: tags + run: | + echo "controller=${{ env.IMG_CONTROLLER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "vxlan-controller=${{ env.IMG_VXLAN_CONTROLLER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "runner=${{ env.IMG_RUNNER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "scheduler=${{ env.IMG_SCHEDULER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "autoscaler-agent=${{ env.IMG_AUTOSCALER_AGENT }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "cluster-autoscaler=${{ env.IMG_CLUSTER_AUTOSCALER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "daemon=${{ env.IMG_DAEMON }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + - uses: actions/checkout@v4 with: fetch-depth: 0 # fetch all, so that we also include tags @@ -164,7 +196,6 @@ jobs: registry: cache.neon.build username: ${{ secrets.NEON_CI_DOCKERCACHE_USERNAME }} password: ${{ secrets.NEON_CI_DOCKERCACHE_PASSWORD }} - - name: Configure dev AWS credentials if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }} uses: aws-actions/configure-aws-credentials@v4 @@ -212,7 +243,7 @@ jobs: id: build-go-dependencies-image with: context: . - platforms: linux/amd64 + push: true file: Dockerfile.go-base cache-from: type=registry,ref=cache.neon.build/autoscaling-go-base:cache @@ -223,12 +254,12 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 push: true + platforms: linux/${{ inputs.arch }} file: neonvm-runner/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-runner:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-runner:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.runner }} + tags: ${{ steps.tags.outputs.runner }} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} @@ -247,41 +278,41 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: neonvm-controller/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-controller:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-controller:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.controller }} + tags: ${{ steps.tags.outputs.controller }} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} - VM_RUNNER_IMAGE=${{ needs.tags.outputs.runner }} + VM_RUNNER_IMAGE=${{ steps.tags.outputs.runner }} BUILDTAGS=${{ steps.controller-build-tags.outputs.buildtags }} - name: Build and push neonvm-vxlan-controller image uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: neonvm-vxlan-controller/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-vxlan-controller:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-vxlan-controller:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.vxlan-controller }} + tags: ${{ steps.tags.outputs.vxlan-controller }} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} - TARGET_ARCH=${{ env.TARGET_ARCH }} + TARGET_ARCH=${{ inputs.arch }} - name: Build and push autoscale-scheduler image uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: autoscale-scheduler/Dockerfile cache-from: type=registry,ref=cache.neon.build/autoscale-scheduler:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscale-scheduler:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.scheduler }} + tags: ${{ steps.tags.outputs.scheduler}} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} GIT_INFO=${{ steps.get-git-info.outputs.info }}:${{ inputs.tag }} @@ -290,12 +321,12 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: autoscaler-agent/Dockerfile cache-from: type=registry,ref=cache.neon.build/autoscaler-agent:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscaler-agent:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.autoscaler-agent }} + tags: ${{ steps.tags.outputs.autoscaler-agent}} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} GIT_INFO=${{ steps.get-git-info.outputs.info }} @@ -304,23 +335,24 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: neonvm-daemon/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-daemon:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-daemon:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.daemon }} + tags: ${{ steps.tags.outputs.daemon }} build-args: | - GO_BASE_IMG=${{ env.GO_BASE_IMG }} + GO_BASE_IMG=${{ env.GO_BASE_IMG }} - name: Build and push cluster-autoscaler image uses: docker/build-push-action@v6 if: ${{ format('{0}', inputs.build-cluster-autoscaler) == 'true' }} with: context: cluster-autoscaler - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true - tags: ${{ needs.tags.outputs.cluster-autoscaler }} + target: ${{format('cluster_autoscaler_{0}', inputs.arch)}} + tags: ${{ steps.tags.outputs.cluster-autoscaler}} cache-from: type=registry,ref=cache.neon.build/cluster-autoscaler-neonvm:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/cluster-autoscaler-neonvm:cache,mode=max' || '' }} @@ -343,3 +375,29 @@ jobs: docker buildx imagetools create -t ${{ env.ECR_PROD }}/${image}:${{ inputs.tag }} \ neondatabase/${image}:${{ inputs.tag }} done + + save-matrix-results: + runs-on: ubuntu-latest + needs: [build] + steps: + ## Write for matrix outputs workaround + - uses: cloudposse/github-action-matrix-outputs-write@v1 + id: out + with: + matrix-step-name: ${{ inputs.matrix-step-name }} + matrix-key: ${{ inputs.matrix-key }} + outputs: |- + controller: ${{ needs.build.outputs.controller }} + vxlan-controller: ${{ needs.build.outputs.vxlan-controller }} + runner: ${{ needs.build.outputs.runner }} + scheduler: ${{ needs.build.outputs.scheduler }} + autoscaler-agent: ${{ needs.build.outputs.autoscaler-agent }} + cluster-autoscaler: ${{ needs.build.outputs.cluster-autoscaler }} + + outputs: + controller: ${{ steps.out.outputs.controller }} + vxlan-controller: ${{ steps.out.outputs.vxlan-controller }} + runner: ${{ steps.out.outputs.runner }} + scheduler: ${{ steps.out.outputs.scheduler }} + autoscaler-agent: ${{ steps.out.outputs.autoscaler-agent }} + cluster-autoscaler: ${{ steps.out.outputs.cluster-autoscaler }} diff --git a/.github/workflows/build-test-vm.yaml b/.github/workflows/build-test-vm.yaml index d6d0eec87..f55fcce13 100644 --- a/.github/workflows/build-test-vm.yaml +++ b/.github/workflows/build-test-vm.yaml @@ -22,10 +22,21 @@ on: type: boolean required: false default: false + arch: + description: 'Architecture to build for' + type: string + required: false + default: 'amd64' + matrix-step-name: + required: false + type: string + matrix-key: + required: false + type: string outputs: vm-postgres-16-bullseye: description: 'image name for postgres:16-bullseye, VM-ified' - value: ${{ jobs.tags.outputs.vm-postgres-16-bullseye }} + value: ${{ jobs.build.outputs.vm-postgres-16-bullseye }} env: IMG_POSTGRES_16_BULLSEYE: "neondatabase/vm-postgres-16-bullseye" @@ -38,25 +49,19 @@ defaults: shell: bash -euo pipefail {0} jobs: - tags: - outputs: - vm-postgres-16-bullseye: ${{ steps.show-tags.outputs.vm-postgres-16-bullseye }} - daemon: ${{ steps.show-tags.outputs.daemon }} - runs-on: ubuntu-latest - steps: - - id: show-tags - run: | - echo "vm-postgres-16-bullseye=${{ env.IMG_POSTGRES_16_BULLSEYE }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - echo "daemon=${{ env.IMG_DAEMON }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - build: # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'. if: ${{ format('{0}', inputs.skip) != 'true' }} - needs: tags - runs-on: [ self-hosted, gen3, large ] - env: - IMG_DAEMON: ${{ needs.tags.outputs.daemon }} + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} + outputs: + vm-postgres-16-bullseye: ${{ steps.tags.outputs.vm-postgres-16-bullseye }} + daemon: ${{ steps.tags.outputs.daemon }} steps: + # tags converted to be a step and moved here to be in the same strategy contextt + - id: tags + run: | + echo "vm-postgres-16-bullseye=${{ env.IMG_POSTGRES_16_BULLSEYE }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "daemon=${{ env.IMG_DAEMON }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: @@ -65,15 +70,34 @@ jobs: cache: false # Sometimes setup-go gets stuck. Without this, it'll keep going until the job gets killed timeout-minutes: 10 - - run: make docker-build-daemon + - name: Build daemon image + run: make docker-build-daemon + env: + IMG_DAEMON: ${{ steps.tags.outputs.daemon }} + # - name: Build daemon image + # uses: docker/build-push-action@v6 + # env: + # GO_BASE_IMG: ${{ format('localhost:5000/neondatabase/autoscaling-go-base-{0}:dev', inputs.arch) }} + # with: + # context: . + # push: false + # platforms: linux/${{ inputs.arch }} + # file: neonvm-daemon/Dockerfile + # cache-from: type=registry,ref=cache.neon.build/neonvm-daemon:cache + # cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-daemon:cache,mode=max' || '' }} + # tags: ${{ steps.tags.outputs.daemon }} + # build-args: | + # GO_BASE_IMG=${{ env.GO_BASE_IMG }} - run: make bin/vm-builder + env: + IMG_DAEMON: ${{ steps.tags.outputs.daemon }} - name: upload vm-builder if: ${{ format('{0}', inputs.upload-vm-builder) == 'true' }} uses: actions/upload-artifact@v4 with: name: vm-builder - path: bin/vm-builder + path: ${{format('bin/vm-builder-{0}', inputs.arch)}} if-no-files-found: error retention-days: 2 @@ -86,9 +110,28 @@ jobs: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - - name: build ${{ needs.tags.outputs.vm-postgres-16-bullseye }} + - name: build ${{ steps.tags.outputs.vm-postgres-16-bullseye }} run: | - ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ needs.tags.outputs.daemon }} -target-arch linux/${TARGET_ARCH} - - name: docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }} + ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ steps.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ steps.tags.outputs.daemon }} -target-arch linux/${{ inputs.arch }} + - name: docker push ${{ steps.tags.outputs.vm-postgres-16-bullseye }} run: | - docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }} + docker push ${{ steps.tags.outputs.vm-postgres-16-bullseye }} + + + save-matrix-results: + runs-on: ubuntu-latest + needs: [build] + steps: + ## Write for matrix outputs workaround + - uses: cloudposse/github-action-matrix-outputs-write@v1 + id: out + with: + matrix-step-name: ${{ inputs.matrix-step-name }} + matrix-key: ${{ inputs.matrix-key }} + outputs: |- + vm-postgres-16-bullseye: ${{ needs.build.outputs.vm-postgres-16-bullseye }} + daemon: ${{ needs.build.outputs.daemon }} + + outputs: + vm-postgres-16-bullseye: ${{ steps.out.outputs.vm-postgres-16-bullseye }} + daemon: ${{ steps.out.outputs.daemon }} diff --git a/.github/workflows/check-ca-builds.yaml b/.github/workflows/check-ca-builds.yaml index 1b8d71bbc..61de71d17 100644 --- a/.github/workflows/check-ca-builds.yaml +++ b/.github/workflows/check-ca-builds.yaml @@ -17,7 +17,11 @@ on: jobs: build-ca: - runs-on: [ self-hosted, gen3, small ] + strategy: + fail-fast: false + matrix: + arch: [ amd64, arm64 ] + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} steps: - uses: actions/checkout@v4 @@ -36,7 +40,8 @@ jobs: uses: docker/build-push-action@v6 with: context: cluster-autoscaler - platforms: linux/amd64 + platforms: ${{format('linux/{0}', matrix.arch)}} push: false + target: ${{format('cluster_autoscaler_{0}', matrix.arch)}} file: cluster-autoscaler/Dockerfile cache-from: type=registry,ref=cache.neon.build/cluster-autoscaler-neonvm:cache diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index 8d50ae5ac..c91cf9a24 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -54,6 +54,9 @@ jobs: build-images: needs: get-tag uses: ./.github/workflows/build-images.yaml + strategy: + matrix: + arch: [amd64, arm64] with: skip: ${{ inputs.tag != '' }} tag: ${{ inputs.tag || needs.get-tag.outputs.tag }} @@ -62,24 +65,59 @@ jobs: # settings and used properly in the tests. But if skip (because inputs.tag != ''), then this # setting will have no effect and the release images will be normal. controller-preserve-runner-pods: true + matrix-step-name: build-images + matrix-key: ${{ matrix.arch }} + arch: ${{ matrix.arch }} secrets: inherit + get-build-images-matrix-results: + runs-on: ubuntu-latest + needs: [build-images] + steps: + - uses: cloudposse/github-action-matrix-outputs-read@v1 + id: read + with: + matrix-step-name: build-images + outputs: + result: "${{ steps.read.outputs.result }}" + build-test-vm: needs: get-tag uses: ./.github/workflows/build-test-vm.yaml + strategy: + matrix: + arch: [amd64, arm64] with: skip: ${{ inputs.tag != '' }} tag: ${{ inputs.tag || needs.get-tag.outputs.tag }} + matrix-step-name: build-test-vm + matrix-key: ${{ matrix.arch }} + arch: ${{ matrix.arch }} secrets: inherit + get-build-test-vm-matrix-results: + runs-on: ubuntu-latest + needs: [build-test-vm] + steps: + - uses: cloudposse/github-action-matrix-outputs-read@v1 + id: read + with: + matrix-step-name: build-test-vm + outputs: + result: "${{ steps.read.outputs.result }}" + e2e-tests: - needs: [ build-images, build-test-vm ] + needs: [ get-build-images-matrix-results, get-build-test-vm-matrix-results ] strategy: fail-fast: false matrix: cluster: - ${{ inputs.cluster || 'k3d' }} - runs-on: [ self-hosted, gen3, large ] + # run tests on amd64 only, since scope of the PR is to build images, there is separate issue for e2e tests + arch: [ arm64 ] + + runs-on: ${{ fromJson(format('["{0}"]', matrix.arch == 'arm64' && 'gha-ubuntu-22.04-16cores-arm' || 'large')) }} + steps: - uses: actions/checkout@v4 with: @@ -109,18 +147,18 @@ jobs: - run: make render-release env: - IMG_CONTROLLER: ${{ needs.build-images.outputs.controller }} - IMG_VXLAN_CONTROLLER: ${{ needs.build-images.outputs.vxlan-controller }} - IMG_RUNNER: ${{ needs.build-images.outputs.runner }} - IMG_SCHEDULER: ${{ needs.build-images.outputs.scheduler }} - IMG_AUTOSCALER_AGENT: ${{ needs.build-images.outputs.autoscaler-agent }} + IMG_CONTROLLER: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).controller[matrix.arch] }} + IMG_VXLAN_CONTROLLER: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).vxlan-controller[matrix.arch] }} + IMG_RUNNER: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).runner[matrix.arch]}} + IMG_SCHEDULER: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).scheduler[matrix.arch]}} + IMG_AUTOSCALER_AGENT: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).autoscaler-agent[matrix.arch]}} - name: upload manifests # nb: use format(..) to catch both inputs.push-yamls = true AND inputs.push-yamls = 'true'. - if: ${{ format('{0}', inputs.push-yamls) == 'true' }} + if: ${{ format('{0}', inputs.push-yamls) == 'true'}} uses: actions/upload-artifact@v4 with: - name: rendered_manifests + name: ${{ format('rendered_manifests-{0}', matrix.arch) }} # nb: prefix before wildcard is removed from the uploaded files, so the artifact should # contain e.g. # - autoscale-scheduler.yaml @@ -167,9 +205,9 @@ jobs: run: | rendered () { echo "rendered_manifests/$1"; } - kubectl apply -f $(rendered multus.yaml) + kubectl apply -f $(rendered multus-${{ matrix.arch}}.yaml) kubectl -n kube-system rollout status daemonset kube-multus-ds - kubectl apply -f $(rendered whereabouts.yaml) + kubectl apply -f $(rendered whereabouts-${{matrix.arch}}.yaml) kubectl -n kube-system rollout status daemonset whereabouts kubectl apply -f $(rendered neonvm-runner-image-loader.yaml) kubectl -n neonvm-system rollout status daemonset neonvm-runner-image-loader @@ -186,7 +224,7 @@ jobs: - name: load e2e test vm image env: - TEST_IMAGE: ${{ needs.build-test-vm.outputs.vm-postgres-16-bullseye }} + TEST_IMAGE: ${{ fromJson(needs.get-build-test-vm-matrix-results.outputs.result).vm-postgres-16-bullseye[matrix.arch]}} timeout-minutes: 2 run: | # Pull the docker image so we can re-tag it, because using a consistent tag inside the diff --git a/.github/workflows/vm-kernel.yaml b/.github/workflows/vm-kernel.yaml index 90d8ae3a3..ddc8de4da 100644 --- a/.github/workflows/vm-kernel.yaml +++ b/.github/workflows/vm-kernel.yaml @@ -28,13 +28,14 @@ on: type: boolean required: false default: false + arch: + description: 'Architecture to build the kernel image for' + type: string + required: true outputs: image: - description: 'vm-kernel Docker image' - value: ${{ jobs.setup-build-vm-kernel-image.outputs.image || jobs.build-vm-kernel-image.outputs.image }} - -env: - VM_KERNEL_IMAGE: "neondatabase/vm-kernel" + description: "Image" + value: ${{ jobs.build-vm-kernel-image.outputs.image }} defaults: run: @@ -43,10 +44,11 @@ defaults: jobs: setup-build-vm-kernel-image: outputs: - image: ${{ steps.get-kernel-image.outputs.image }} + image: ${{ steps.get-kernel-image.outputs.image-arm64 }} kernel-cache-tag: ${{ steps.get-kernel-cache-tag.outputs.kernel-cache-tag }} - runs-on: ubuntu-latest + env: + VM_KERNEL_IMAGE: ${{format('neondatabase/vm-kernel-{0}', inputs.arch)}} steps: - uses: actions/checkout@v4 @@ -132,7 +134,10 @@ jobs: outputs: image: ${{ steps.get-tags.outputs.canonical }}@${{ steps.build-linux-kernel.outputs.digest }} - runs-on: [ self-hosted, gen3, large ] + env: + VM_KERNEL_IMAGE: ${{format('neondatabase/vm-kernel-{0}', inputs.arch)}} + + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} steps: - name: git checkout uses: actions/checkout@v4 @@ -185,14 +190,15 @@ jobs: with: build-args: KERNEL_VERSION=${{ steps.get-kernel-version.outputs.VM_KERNEL_VERSION }} context: neonvm-kernel - platforms: linux/amd64 + platforms: ${{format('linux/{0}', inputs.arch)}} # neonvm-kernel/Dockerfile.kernel-builder has different targets for different architectures # so we need to specify the target explicitly - target: kernel_amd64 + target: ${{format('kernel_{0}', inputs.arch)}} # Push kernel image only for scheduled builds or if workflow_dispatch/workflow_call input is true push: true pull: true file: neonvm-kernel/Dockerfile.kernel-builder - cache-from: type=registry,ref=cache.neon.build/vm-kernel:cache - cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/vm-kernel:cache,mode=max' || '' }} + cache-from: ${{format('type=registry,ref=cache.neon.build/vm-kernel:{0}cache', inputs.arch)}} + # Cache the kernel image for future builds, temporary from all branches + cache-to: ${{ format('type=registry,ref=cache.neon.build/vm-kernel:{0}cache,mode=max', inputs.arch) || '' }} tags: ${{ steps.get-tags.outputs.tags }} diff --git a/Makefile b/Makefile index 2d5ff8577..ac9b4d806 100644 --- a/Makefile +++ b/Makefile @@ -320,10 +320,12 @@ render-manifests: $(RENDERED) kustomize cd autoscale-scheduler && $(KUSTOMIZE) edit set image autoscale-scheduler=$(IMG_SCHEDULER) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force cd autoscaler-agent && $(KUSTOMIZE) edit set image autoscaler-agent=$(IMG_AUTOSCALER_AGENT) && $(KUSTOMIZE) edit add annotation buildtime:$(BUILDTS) --force # Build: - $(KUSTOMIZE) build neonvm/config/whereabouts > $(RENDERED)/whereabouts.yaml + $(KUSTOMIZE) build neonvm/config/whereabouts-amd64 > $(RENDERED)/whereabouts-amd64.yaml + $(KUSTOMIZE) build neonvm/config/whereabouts-arm64 > $(RENDERED)/whereabouts-arm64.yaml $(KUSTOMIZE) build neonvm/config/multus-aks > $(RENDERED)/multus-aks.yaml $(KUSTOMIZE) build neonvm/config/multus-eks > $(RENDERED)/multus-eks.yaml - $(KUSTOMIZE) build neonvm/config/multus > $(RENDERED)/multus.yaml + $(KUSTOMIZE) build neonvm/config/multus-amd64 > $(RENDERED)/multus-amd64.yaml + $(KUSTOMIZE) build neonvm/config/multus-arm64 > $(RENDERED)/multus-arm64.yaml $(KUSTOMIZE) build neonvm/config > $(RENDERED)/neonvm.yaml $(KUSTOMIZE) build neonvm-controller > $(RENDERED)/neonvm-controller.yaml $(KUSTOMIZE) build neonvm-vxlan-controller > $(RENDERED)/neonvm-vxlan-controller.yaml @@ -345,10 +347,12 @@ render-release: $(RENDERED) kustomize cd autoscale-scheduler && $(KUSTOMIZE) edit set image autoscale-scheduler=$(IMG_SCHEDULER) cd autoscaler-agent && $(KUSTOMIZE) edit set image autoscaler-agent=$(IMG_AUTOSCALER_AGENT) # Build: - $(KUSTOMIZE) build neonvm/config/whereabouts > $(RENDERED)/whereabouts.yaml + $(KUSTOMIZE) build neonvm/config/whereabouts-amd64 > $(RENDERED)/whereabouts-amd64.yaml + $(KUSTOMIZE) build neonvm/config/whereabouts-arm64 > $(RENDERED)/whereabouts-arm64.yaml $(KUSTOMIZE) build neonvm/config/multus-aks > $(RENDERED)/multus-aks.yaml $(KUSTOMIZE) build neonvm/config/multus-eks > $(RENDERED)/multus-eks.yaml - $(KUSTOMIZE) build neonvm/config/multus > $(RENDERED)/multus.yaml + $(KUSTOMIZE) build neonvm/config/multus-amd64 > $(RENDERED)/multus-amd64.yaml + $(KUSTOMIZE) build neonvm/config/multus-arm64 > $(RENDERED)/multus-arm64.yaml $(KUSTOMIZE) build neonvm/config > $(RENDERED)/neonvm.yaml $(KUSTOMIZE) build neonvm-controller > $(RENDERED)/neonvm-controller.yaml $(KUSTOMIZE) build neonvm-vxlan-controller > $(RENDERED)/neonvm-vxlan-controller.yaml @@ -364,9 +368,9 @@ render-release: $(RENDERED) kustomize .PHONY: deploy deploy: check-local-context docker-build load-images render-manifests kubectl ## Deploy controller to the K8s cluster specified in ~/.kube/config. - $(KUBECTL) apply -f $(RENDERED)/multus.yaml + $(KUBECTL) apply -f $(RENDERED)/multus-$(TARGET_ARCH).yaml $(KUBECTL) -n kube-system rollout status daemonset kube-multus-ds - $(KUBECTL) apply -f $(RENDERED)/whereabouts.yaml + $(KUBECTL) apply -f $(RENDERED)/whereabouts-$(TARGET_ARCH).yaml $(KUBECTL) -n kube-system rollout status daemonset whereabouts $(KUBECTL) apply -f $(RENDERED)/neonvm-runner-image-loader.yaml $(KUBECTL) -n neonvm-system rollout status daemonset neonvm-runner-image-loader @@ -397,7 +401,7 @@ example-vms: docker-build-examples load-example-vms ## Build and push the testin .PHONY: example-vms-arm64 example-vms-arm64: TARGET_ARCH=arm64 -example-vms-arm64: example-vms +example-vms-arm64: example-vms .PHONY: load-pg16-disk-test load-pg16-disk-test: check-local-context kubectl kind k3d ## Load the pg16-disk-test VM image to the kind/k3d cluster. @@ -561,3 +565,10 @@ cert-manager: check-local-context kubectl ## install cert-manager to cluster python-init: python3 -m venv tests/e2e/.venv tests/e2e/.venv/bin/pip install -r requirements.txt + +# arm doesn't support cpu hot plug and memory hot plug and CI runners are based on qemu so no kvm acceleration as well +arm_patch_e2e: + @find neonvm/samples/*yaml tests/e2e -name "*.yaml" | xargs -I{} ./bin/yq eval '(select(.kind == "VirtualMachine") | .spec.cpuScalingMode = "SysfsScaling") // .' -i {} + # @find neonvm/samples/*yaml tests/e2e -name "*.yaml" | xargs -I{} ./bin/yq eval '(select(.kind == "VirtualMachine") | .spec.enableAcceleration = false) // .' -i {} + @find neonvm/samples/*yaml tests/e2e -name "*.yaml" | xargs -I{} ./bin/yq eval '(select(.kind == "VirtualMachine") | .status.memoryProvider = "VirtioMem") // .' -i {} + @find neonvm/samples/*yaml tests/e2e -name "*.yaml" | xargs -I{} ./bin/yq eval '(select(.kind == "VirtualMachine") | .spec.guest.memoryProvider = "VirtioMem") // .' -i {} diff --git a/cluster-autoscaler/Dockerfile b/cluster-autoscaler/Dockerfile index 0530a1ce8..9acdda310 100644 --- a/cluster-autoscaler/Dockerfile +++ b/cluster-autoscaler/Dockerfile @@ -30,7 +30,15 @@ RUN cd autoscaler/cluster-autoscaler \ # This is adapted from CA's Dockerfile.amd64, here: # https://github.com/kubernetes/autoscaler/blob/cluster-autoscaler-1.24.1/cluster-autoscaler/Dockerfile.amd64 -FROM gcr.io/distroless/static:nonroot-amd64 + +# NB: two build stages, one for each architecture, because I wasn't able to use variable substitution in FROM statements +FROM gcr.io/distroless/static:nonroot-amd64 AS cluster_autoscaler_amd64 + +WORKDIR / +COPY --from=builder /workspace/cluster-autoscaler . +CMD ["/cluster-autoscaler"] + +FROM gcr.io/distroless/static:nonroot-arm64 AS cluster_autoscaler_arm64 WORKDIR / COPY --from=builder /workspace/cluster-autoscaler . diff --git a/neonvm-runner/Dockerfile b/neonvm-runner/Dockerfile index 4eb3df79f..0d2b0a136 100644 --- a/neonvm-runner/Dockerfile +++ b/neonvm-runner/Dockerfile @@ -5,7 +5,6 @@ COPY . . # Build RUN CGO_ENABLED=0 go build -o /runner neonvm-runner/cmd/*.go - FROM alpine:3.19 RUN apk add --no-cache \ @@ -21,12 +20,16 @@ RUN apk add --no-cache \ busybox-extras \ e2fsprogs \ qemu-system-x86_64 \ + qemu-system-aarch64 \ qemu-img \ cgroup-tools \ openssh + COPY --from=builder /runner /usr/bin/runner COPY neonvm-kernel/vmlinuz /vm/kernel/vmlinuz COPY neonvm-runner/ssh_config /etc/ssh/ssh_config +# QEMU_EFI used only by runner running on the arm architecture +RUN wget https://releases.linaro.org/components/kernel/uefi-linaro/16.02/release/qemu64/QEMU_EFI.fd -O /vm/QEMU_EFI_ARM.fd ENTRYPOINT ["/sbin/tini", "--", "runner"] diff --git a/neonvm-runner/cmd/main.go b/neonvm-runner/cmd/main.go index 90ed7732b..44ed63780 100644 --- a/neonvm-runner/cmd/main.go +++ b/neonvm-runner/cmd/main.go @@ -20,6 +20,7 @@ import ( "os/signal" "path/filepath" "regexp" + "runtime" "strings" "sync" "sync/atomic" @@ -52,8 +53,12 @@ import ( ) const ( - QEMU_BIN = "qemu-system-x86_64" - QEMU_IMG_BIN = "qemu-img" + qemuBinArm64 = "qemu-system-aarch64" + qemuBinX8664 = "qemu-system-x86_64" + qemuImgBin = "qemu-img" + + architectureArm64 = "arm64" + architectureAmd64 = "amd64" defaultKernelPath = "/vm/kernel/vmlinuz" rootDiskPath = "/vm/images/rootdisk.qcow2" @@ -403,14 +408,14 @@ func calcDirUsage(dirPath string) (int64, error) { func createSwap(diskPath string, swapSize *resource.Quantity) error { tmpRawFile := "swap.raw" - if err := execFg(QEMU_IMG_BIN, "create", "-q", "-f", "raw", tmpRawFile, fmt.Sprintf("%d", swapSize.Value())); err != nil { + if err := execFg(qemuImgBin, "create", "-q", "-f", "raw", tmpRawFile, fmt.Sprintf("%d", swapSize.Value())); err != nil { return err } if err := execFg("mkswap", "-L", swapName, tmpRawFile); err != nil { return err } - if err := execFg(QEMU_IMG_BIN, "convert", "-q", "-f", "raw", "-O", "qcow2", "-o", "cluster_size=2M,lazy_refcounts=on", tmpRawFile, diskPath); err != nil { + if err := execFg(qemuImgBin, "convert", "-q", "-f", "raw", "-O", "qcow2", "-o", "cluster_size=2M,lazy_refcounts=on", tmpRawFile, diskPath); err != nil { return err } @@ -466,7 +471,7 @@ func createQCOW2(diskName string, diskPath string, diskSize *resource.Quantity, return err } - if err := execFg(QEMU_IMG_BIN, "convert", "-q", "-f", "raw", "-O", "qcow2", "-o", "cluster_size=2M,lazy_refcounts=on", "ext4.raw", diskPath); err != nil { + if err := execFg(qemuImgBin, "convert", "-q", "-f", "raw", "-O", "qcow2", "-o", "cluster_size=2M,lazy_refcounts=on", "ext4.raw", diskPath); err != nil { return err } @@ -618,9 +623,14 @@ type Config struct { appendKernelCmdline string skipCgroupManagement bool diskCacheSettings string - memoryProvider vmv1.MemoryProvider - autoMovableRatio string - cpuScalingMode vmv1.CpuScalingMode + // memoryProvider is a memory provider to use. Validated in newConfig. + memoryProvider vmv1.MemoryProvider + // autoMovableRatio value for VirtioMem provider. Validated in newConfig. + autoMovableRatio string + // cpuScalingMode is a mode to use for CPU scaling. Validated in newConfig. + cpuScalingMode vmv1.CpuScalingMode + // System CPU architecture. Set automatically equal to runtime.GOARCH. + architecture string } func newConfig(logger *zap.Logger) *Config { @@ -631,9 +641,10 @@ func newConfig(logger *zap.Logger) *Config { appendKernelCmdline: "", skipCgroupManagement: false, diskCacheSettings: "cache=none", - memoryProvider: "", // Require that this is explicitly set. We'll check later. - autoMovableRatio: "", // Require that this is explicitly set IFF memoryProvider is VirtioMem. We'll check later. - cpuScalingMode: "", // Require that this is explicitly set. We'll check later. + memoryProvider: "", + autoMovableRatio: "", + cpuScalingMode: "", + architecture: runtime.GOARCH, } flag.StringVar(&cfg.vmSpecDump, "vmspec", cfg.vmSpecDump, "Base64 encoded VirtualMachine json specification") @@ -868,7 +879,7 @@ func resizeRootDisk(logger *zap.Logger, vmSpec *vmv1.VirtualMachineSpec) error { VirtualSize int64 `json:"virtual-size"` } // get current disk size by qemu-img info command - qemuImgOut, err := exec.Command(QEMU_IMG_BIN, "info", "--output=json", rootDiskPath).Output() + qemuImgOut, err := exec.Command(qemuImgBin, "info", "--output=json", rootDiskPath).Output() if err != nil { return fmt.Errorf("could not get root image size: %w", err) } @@ -882,7 +893,7 @@ func resizeRootDisk(logger *zap.Logger, vmSpec *vmv1.VirtualMachineSpec) error { if !vmSpec.Guest.RootDisk.Size.IsZero() { if vmSpec.Guest.RootDisk.Size.Cmp(*imageSizeQuantity) == 1 { logger.Info(fmt.Sprintf("resizing rootDisk from %s to %s", imageSizeQuantity.String(), vmSpec.Guest.RootDisk.Size.String())) - if err := execFg(QEMU_IMG_BIN, "resize", rootDiskPath, fmt.Sprintf("%d", vmSpec.Guest.RootDisk.Size.Value())); err != nil { + if err := execFg(qemuImgBin, "resize", rootDiskPath, fmt.Sprintf("%d", vmSpec.Guest.RootDisk.Size.Value())); err != nil { return fmt.Errorf("failed to resize rootDisk: %w", err) } } else { @@ -904,14 +915,13 @@ func buildQEMUCmd( // prepare qemu command line qemuCmd := []string{ "-runas", "qemu", - "-machine", "q35", + "-machine", getMachineType(cfg.architecture), "-nographic", "-no-reboot", "-nodefaults", "-only-migratable", "-audiodev", "none,id=noaudio", "-serial", "pty", - "-serial", "stdio", "-msg", "timestamp=on", "-qmp", fmt.Sprintf("tcp:0.0.0.0:%d,server,wait=off", vmSpec.QMP), "-qmp", fmt.Sprintf("tcp:0.0.0.0:%d,server,wait=off", vmSpec.QMPManual), @@ -941,6 +951,21 @@ func buildQEMUCmd( } qemuCmd = append(qemuCmd, "-drive", fmt.Sprintf("id=%s,file=%s,if=virtio,media=disk,%s,discard=unmap", swapName, dPath, cfg.diskCacheSettings)) } + switch cfg.architecture { + case architectureArm64: + // add custom firmware to have ACPI working + qemuCmd = append(qemuCmd, "-bios", "/vm/QEMU_EFI_ARM.fd") + // arm virt has only one UART, setup virtio-serial to add more /dev/hvcX + qemuCmd = append(qemuCmd, + "-chardev", "stdio,id=virtio-console", + "-device", "virtconsole,chardev=virtio-console", + ) + case architectureAmd64: + // on amd we have multiple UART ports so we can just use serial stdio + qemuCmd = append(qemuCmd, "-serial", "stdio") + default: + logger.Fatal("unsupported architecture", zap.String("architecture", cfg.architecture)) + } for _, disk := range vmSpec.Disks { switch { @@ -1049,7 +1074,7 @@ func buildQEMUCmd( qemuCmd = append( qemuCmd, "-kernel", cfg.kernelPath, - "-append", makeKernelCmdline(cfg, vmSpec, vmStatus, hostname), + "-append", makeKernelCmdline(cfg, logger, vmSpec, vmStatus, hostname), ) // should runner receive migration ? @@ -1061,12 +1086,12 @@ func buildQEMUCmd( } const ( - baseKernelCmdline = "panic=-1 init=/neonvm/bin/init console=ttyS1 loglevel=7 root=/dev/vda rw" + baseKernelCmdline = "panic=-1 init=/neonvm/bin/init loglevel=7 root=/dev/vda rw" kernelCmdlineDIMMSlots = "memhp_default_state=online_movable" kernelCmdlineVirtioMemTmpl = "memhp_default_state=online memory_hotplug.online_policy=auto-movable memory_hotplug.auto_movable_ratio=%s" ) -func makeKernelCmdline(cfg *Config, vmSpec *vmv1.VirtualMachineSpec, vmStatus *vmv1.VirtualMachineStatus, hostname string) string { +func makeKernelCmdline(cfg *Config, logger *zap.Logger, vmSpec *vmv1.VirtualMachineSpec, vmStatus *vmv1.VirtualMachineStatus, hostname string) string { cmdlineParts := []string{baseKernelCmdline} switch cfg.memoryProvider { @@ -1095,6 +1120,18 @@ func makeKernelCmdline(cfg *Config, vmSpec *vmv1.VirtualMachineSpec, vmStatus *v cmdlineParts = append(cmdlineParts, fmt.Sprintf("maxcpus=%d", vmSpec.Guest.CPUs.Min.RoundedUp())) } + switch cfg.architecture { + case architectureArm64: + // explicitly enable acpi if we run on arm + cmdlineParts = append(cmdlineParts, "acpi=on") + // use virtio-serial device kernel console + cmdlineParts = append(cmdlineParts, "console=hvc0") + case architectureAmd64: + cmdlineParts = append(cmdlineParts, "console=ttyS1") + default: + logger.Fatal("unsupported architecture", zap.String("architecture", cfg.architecture)) + } + return strings.Join(cmdlineParts, " ") } @@ -1141,7 +1178,6 @@ func runQEMU( wg.Add(1) go terminateQemuOnSigterm(ctx, logger, &wg) - var callbacks cpuServerCallbacks // lastValue is used to store last fractional CPU request // we need to store the value as is because we can't convert it back from MilliCPU @@ -1173,13 +1209,14 @@ func runQEMU( wg.Add(1) go forwardLogs(ctx, logger, &wg) + qemuBin := getQemuBinaryName(cfg.architecture) var bin string var cmd []string if !cfg.skipCgroupManagement { bin = "cgexec" - cmd = append([]string{"-g", fmt.Sprintf("cpu:%s", cgroupPath), QEMU_BIN}, qemuCmd...) + cmd = append([]string{"-g", fmt.Sprintf("cpu:%s", cgroupPath), qemuBin}, qemuCmd...) } else { - bin = QEMU_BIN + bin = qemuBin cmd = qemuCmd } @@ -1199,6 +1236,30 @@ func runQEMU( return err } +func getQemuBinaryName(architecture string) string { + switch architecture { + case architectureArm64: + return qemuBinArm64 + case architectureAmd64: + return qemuBinX8664 + default: + panic(fmt.Errorf("unknown architecture %s", architecture)) + } +} + +func getMachineType(architecture string) string { + switch architecture { + case architectureArm64: + // virt is the most up to date and generic ARM machine architecture + return "virt" + case architectureAmd64: + // q35 is the most up to date and generic x86_64 machine architecture + return "q35" + default: + panic(fmt.Errorf("unknown architecture %s", architecture)) + } +} + func handleCPUChange( logger *zap.Logger, w http.ResponseWriter, diff --git a/neonvm-vxlan-controller/daemonset.yaml b/neonvm-vxlan-controller/daemonset.yaml index 6c56ff49b..c92f9ac9c 100644 --- a/neonvm-vxlan-controller/daemonset.yaml +++ b/neonvm-vxlan-controller/daemonset.yaml @@ -38,6 +38,7 @@ spec: operator: In values: - amd64 + - arm64 - key: kubernetes.io/os operator: In values: diff --git a/neonvm/config/multus/daemonset_patch.yaml b/neonvm/config/multus-amd64/daemonset_patch.yaml similarity index 100% rename from neonvm/config/multus/daemonset_patch.yaml rename to neonvm/config/multus-amd64/daemonset_patch.yaml diff --git a/neonvm/config/multus/kustomization.yaml b/neonvm/config/multus-amd64/kustomization.yaml similarity index 100% rename from neonvm/config/multus/kustomization.yaml rename to neonvm/config/multus-amd64/kustomization.yaml diff --git a/neonvm/config/multus-arm64/cluster_role.yaml b/neonvm/config/multus-arm64/cluster_role.yaml new file mode 100644 index 000000000..49c8dbcb8 --- /dev/null +++ b/neonvm/config/multus-arm64/cluster_role.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: multus +rules: +- apiGroups: + - k8s.cni.cncf.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + - pods/status + verbs: + - get + - update + - watch + - list +- apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - patch + - update diff --git a/neonvm/config/multus-arm64/config_map.yaml b/neonvm/config/multus-arm64/config_map.yaml new file mode 100644 index 000000000..190d6141d --- /dev/null +++ b/neonvm/config/multus-arm64/config_map.yaml @@ -0,0 +1,21 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: multus-daemon-config + namespace: kube-system + labels: + tier: node + app: multus +data: + daemon-config.json: | + { + "chrootDir": "/hostroot", + "cniVersion": "0.3.1", + "logLevel": "verbose", + "logToStderr": true, + "cniConfigDir": "/host/etc/cni/net.d", + "multusAutoconfigDir": "/host/etc/cni/net.d", + "multusConfigFile": "auto", + "socketDir": "/host/run/multus/" + } diff --git a/neonvm/config/multus-arm64/daemonset_patch.yaml b/neonvm/config/multus-arm64/daemonset_patch.yaml new file mode 100644 index 000000000..c8e18ddfd --- /dev/null +++ b/neonvm/config/multus-arm64/daemonset_patch.yaml @@ -0,0 +1,112 @@ +--- +# Source: multus-cni/templates/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-multus-ds + namespace: kube-system + labels: + tier: node + app: multus + name: multus +spec: + selector: + matchLabels: + app: multus + updateStrategy: + type: RollingUpdate + template: + metadata: + annotations: + spec: + + hostNetwork: true + serviceAccountName: multus + securityContext: + fsGroup: 0 + affinity: + podAffinity: + + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + priorityClassName: "" + initContainers: + - name: install-multus-binary + image: docker.io/bitnami/multus-cni:3.9.3 + command: + - cp + - "/usr/src/multus-cni/bin/multus" + - "/bitnami/multus-cni/host/opt/cni/bin" + securityContext: + privileged: true + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - name: cni-bin-dir + mountPath: /bitnami/multus-cni/host/opt/cni/bin + mountPropagation: Bidirectional + - name: generate-kubeconfig + image: docker.io/bitnami/multus-cni:3.9.3 + command: + - generate-kubeconfig + args: + - "-k8s-service-host=$(KUBERNETES_SERVICE_HOST)" + - "-k8s-service-port=$(KUBERNETES_SERVICE_PORT)" + - "-cni-config-dir=/bitnami/multus-cni/host/etc/cni/net.d" + securityContext: + privileged: true + runAsNonRoot: false + runAsUser: 0 + volumeMounts: + - name: cni-net-dir + mountPath: /bitnami/multus-cni/host/etc/cni/net.d + mountPropagation: Bidirectional + containers: + - name: kube-multus + image: docker.io/bitnami/multus-cni:3.9.3 + imagePullPolicy: "IfNotPresent" + command: + - multus-daemon + args: + - "-cni-version=0.3.0" + - "-cni-config-dir=/bitnami/multus-cni/host/etc/cni/net.d" + - "-multus-autoconfig-dir=/bitnami/multus-cni/host/etc/cni/net.d" + - "-multus-log-to-stderr=true" + - "-multus-log-level=verbose" + securityContext: + privileged: true + runAsNonRoot: false + runAsUser: 0 + env: + - name: BITNAMI_DEBUG + value: "false" + envFrom: + livenessProbe: + exec: + command: + - pgrep + - multus-daemon + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + resources: + limits: {} + requests: {} + volumeMounts: + - name: cni-net-dir + mountPath: /bitnami/multus-cni/host/etc/cni/net.d + volumes: + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d diff --git a/neonvm/config/multus-arm64/kustomization.yaml b/neonvm/config/multus-arm64/kustomization.yaml new file mode 100644 index 000000000..a945347f7 --- /dev/null +++ b/neonvm/config/multus-arm64/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: +- ../multus-common + +images: +- name: kube-multus + newName: ghcr.io/k8snetworkplumbingwg/multus-cni + newTag: v4.1.3-thick + + +patchesStrategicMerge: +- cluster_role.yaml +- daemonset_patch.yaml + +patches: +- target: + kind: ServiceAccount + name: multus + patch: |- + - op: replace + path: /automountServiceAccountToken + value: true diff --git a/neonvm/config/whereabouts/daemonset_patch.yaml b/neonvm/config/whereabouts-amd64/daemonset_patch.yaml similarity index 100% rename from neonvm/config/whereabouts/daemonset_patch.yaml rename to neonvm/config/whereabouts-amd64/daemonset_patch.yaml diff --git a/neonvm/config/whereabouts/kustomization.yaml b/neonvm/config/whereabouts-amd64/kustomization.yaml similarity index 100% rename from neonvm/config/whereabouts/kustomization.yaml rename to neonvm/config/whereabouts-amd64/kustomization.yaml diff --git a/neonvm/config/whereabouts-arm64/daemonset_patch.yaml b/neonvm/config/whereabouts-arm64/daemonset_patch.yaml new file mode 100644 index 000000000..c9a407e40 --- /dev/null +++ b/neonvm/config/whereabouts-arm64/daemonset_patch.yaml @@ -0,0 +1,24 @@ +# patch the DaemonSet so that it's only running on nodes that we'd support +# +# The image we're is a linux amd64 image; it doesn't work on ARM or non-Linux. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: whereabouts + namespace: kube-system +spec: + template: + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - key: kubernetes.io/os + operator: In + values: + - linux diff --git a/neonvm/config/whereabouts-arm64/kustomization.yaml b/neonvm/config/whereabouts-arm64/kustomization.yaml new file mode 100644 index 000000000..4001d3cbd --- /dev/null +++ b/neonvm/config/whereabouts-arm64/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +bases: +# whereabouts CNI +- https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/v0.6.2/doc/crds/whereabouts.cni.cncf.io_ippools.yaml +- https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/v0.6.2/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml +- https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/v0.6.2/doc/crds/daemonset-install.yaml + +patchesStrategicMerge: +- daemonset_patch.yaml + +images: +- name: ghcr.io/k8snetworkplumbingwg/whereabouts + newTag: v0.6.2-arm64 diff --git a/pkg/agent/billing/prommetrics.go b/pkg/agent/billing/prommetrics.go index f15346674..bebf6a4b9 100644 --- a/pkg/agent/billing/prommetrics.go +++ b/pkg/agent/billing/prommetrics.go @@ -59,8 +59,6 @@ type batchMetricsLabels struct { } func (m PromMetrics) forBatch() batchMetrics { - m.vmsCurrent.Reset() - return batchMetrics{ total: make(map[batchMetricsLabels]int), @@ -88,6 +86,8 @@ func (b batchMetrics) inc(isEndpoint isEndpointFlag, autoscalingEnabled autoscal } func (b batchMetrics) finish() { + b.vmsCurrent.Reset() + for key, count := range b.total { b.vmsCurrent.WithLabelValues(key.isEndpoint, key.autoscalingEnabled, key.phase).Set(float64(count)) } diff --git a/pkg/neonvm/controllers/runner_cpu_limits.go b/pkg/neonvm/controllers/runner_cpu_limits.go index e0b45e34c..e55590895 100644 --- a/pkg/neonvm/controllers/runner_cpu_limits.go +++ b/pkg/neonvm/controllers/runner_cpu_limits.go @@ -13,11 +13,11 @@ import ( "github.com/neondatabase/autoscaling/pkg/api" ) -func setRunnerCPULimits(ctx context.Context, vm *vmv1.VirtualMachine, cpu vmv1.MilliCPU) error { +func setRunnerCPULimits(ctx context.Context, vm *vmv1.VirtualMachine, targetPodIP string, cpu vmv1.MilliCPU) error { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - url := fmt.Sprintf("http://%s:%d/cpu_change", vm.Status.PodIP, vm.Spec.RunnerPort) + url := fmt.Sprintf("http://%s:%d/cpu_change", targetPodIP, vm.Spec.RunnerPort) update := api.VCPUChange{VCPUs: cpu} diff --git a/pkg/neonvm/controllers/vm_controller.go b/pkg/neonvm/controllers/vm_controller.go index 170c65333..e904f1fe7 100644 --- a/pkg/neonvm/controllers/vm_controller.go +++ b/pkg/neonvm/controllers/vm_controller.go @@ -26,6 +26,7 @@ import ( "fmt" "os" "reflect" + sysruntime "runtime" "strconv" "time" @@ -1213,7 +1214,6 @@ func affinityForVirtualMachine(vm *vmv1.VirtualMachine) *corev1.Affinity { if a.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { a.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{} } - // if NodeSelectorTerms list is empty - add default values (arch==amd64 or os==linux) if len(a.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms) == 0 { a.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append( @@ -1223,7 +1223,7 @@ func affinityForVirtualMachine(vm *vmv1.VirtualMachine) *corev1.Affinity { { Key: "kubernetes.io/arch", Operator: "In", - Values: []string{"amd64"}, + Values: []string{sysruntime.GOARCH}, }, { Key: "kubernetes.io/os", diff --git a/pkg/neonvm/controllers/vm_controller_cpu_scaling.go b/pkg/neonvm/controllers/vm_controller_cpu_scaling.go index 9266ae820..b01007802 100644 --- a/pkg/neonvm/controllers/vm_controller_cpu_scaling.go +++ b/pkg/neonvm/controllers/vm_controller_cpu_scaling.go @@ -112,7 +112,7 @@ func (r *VMReconciler) handleCPUScalingSysfs(ctx context.Context, vm *vmv1.Virtu func (r *VMReconciler) handleCgroupCPUUpdate(ctx context.Context, vm *vmv1.VirtualMachine, cgroupUsage *api.VCPUCgroup) (bool, error) { specCPU := vm.Spec.Guest.CPUs.Use - if err := setRunnerCPULimits(ctx, vm, specCPU); err != nil { + if err := setRunnerCPULimits(ctx, vm, vm.Status.PodIP, specCPU); err != nil { return false, err } reason := "ScaleDown" diff --git a/pkg/neonvm/controllers/vmmigration_controller.go b/pkg/neonvm/controllers/vmmigration_controller.go index 0066cdd5a..b77cfaca4 100644 --- a/pkg/neonvm/controllers/vmmigration_controller.go +++ b/pkg/neonvm/controllers/vmmigration_controller.go @@ -309,11 +309,16 @@ func (r *VirtualMachineMigrationReconciler) Reconcile(ctx context.Context, req c migration.Status.SourcePodIP = vm.Status.PodIP migration.Status.TargetPodIP = targetRunner.Status.PodIP - // do hotplugCPU in targetRunner before migration + // do cpu hot plug in targetRunner before migration + // in case of QMP mode, we need to sync CPUs before migration + // in case of Sysfs mode, we need to sync CPUs during migration log.Info("Syncing CPUs in Target runner", "TargetPod.Name", migration.Status.TargetPodName) - if err := QmpSyncCpuToTarget(vm, migration); err != nil { - return ctrl.Result{}, err + if *vm.Spec.CpuScalingMode == vmv1.CpuScalingModeQMP { + if err := QmpSyncCpuToTarget(vm, migration); err != nil { + return ctrl.Result{}, err + } } + log.Info("CPUs in Target runner synced", "TargetPod.Name", migration.Status.TargetPodName) // do hotplug Memory in targetRunner -- only needed for dimm slots; virtio-mem Just Works™ @@ -334,8 +339,8 @@ func (r *VirtualMachineMigrationReconciler) Reconcile(ctx context.Context, req c panic(fmt.Errorf("unexpected vm.status.memoryProvider %q", *vm.Status.MemoryProvider)) } - // Migrate only running VMs to target with plugged devices - if vm.Status.Phase == vmv1.VmPreMigrating { + switch vm.Status.Phase { + case vmv1.VmPreMigrating: // update VM status vm.Status.Phase = vmv1.VmMigrating if err := r.Status().Update(ctx, vm); err != nil { @@ -357,10 +362,22 @@ func (r *VirtualMachineMigrationReconciler) Reconcile(ctx context.Context, req c Reason: "Reconciling", Message: message, }) - // finally update migration phase to Running + return r.updateMigrationStatus(ctx, migration) + case vmv1.VmMigrating: + // migration is in progress so we can scale CPU using sysfs + if *vm.Spec.CpuScalingMode == vmv1.CpuScalingModeSysfs { + if err := setRunnerCPULimits(ctx, + vm, + targetRunner.Status.PodIP, + vm.Spec.Guest.CPUs.Use); err != nil { + return ctrl.Result{}, err + } + } + // if cpu scaling is not sysfs based we just update VM status to Running, since migration is done at the moment migration.Status.Phase = vmv1.VmmRunning return r.updateMigrationStatus(ctx, migration) } + case runnerSucceeded: // target runner pod finished without error? but it shouldn't finish message := fmt.Sprintf("Target Pod (%s) completed suddenly", targetRunner.Name) diff --git a/tests/e2e/autoscaling.cpu-sys-fs-state-scaling/00-assert.yaml b/tests/e2e/autoscaling.cpu-sys-fs-state-scaling/00-assert.yaml index 3a0b27560..10de5a68b 100644 --- a/tests/e2e/autoscaling.cpu-sys-fs-state-scaling/00-assert.yaml +++ b/tests/e2e/autoscaling.cpu-sys-fs-state-scaling/00-assert.yaml @@ -14,4 +14,4 @@ status: status: "True" cpus: 250m memorySize: 1Gi - memoryProvider: DIMMSlots + memoryProvider: VirtioMem diff --git a/tests/e2e/autoscaling.cpu-sys-fs-state-scaling/00-create-vm.yaml b/tests/e2e/autoscaling.cpu-sys-fs-state-scaling/00-create-vm.yaml index efe8f5ca7..33ac11e70 100644 --- a/tests/e2e/autoscaling.cpu-sys-fs-state-scaling/00-create-vm.yaml +++ b/tests/e2e/autoscaling.cpu-sys-fs-state-scaling/00-create-vm.yaml @@ -37,7 +37,7 @@ spec: min: 1 max: 5 use: 1 - memoryProvider: DIMMSlots + memoryProvider: VirtioMem rootDisk: image: vm-postgres:15-bullseye size: 8Gi @@ -56,7 +56,7 @@ spec: - name: monitor port: 10301 extraNetwork: - enable: true + enable: false disks: - name: pgdata mountPath: /var/lib/postgresql diff --git a/vm-builder/files/Dockerfile.img b/vm-builder/files/Dockerfile.img index 0abed8262..9d2fb77e8 100644 --- a/vm-builder/files/Dockerfile.img +++ b/vm-builder/files/Dockerfile.img @@ -13,13 +13,13 @@ FROM busybox:1.35.0-musl AS busybox-loader FROM alpine:3.19 AS vm-runtime ARG TARGET_ARCH -RUN set -e && mkdir -p /neonvm/bin /neonvm/runtime /neonvm/config +RUN set -e && mkdir -p /neonvm/bin /neonvm/runtime /neonvm/config # add busybox COPY --from=busybox-loader /bin/busybox /neonvm/bin/busybox RUN set -e \ chmod +x /neonvm/bin/busybox \ - && /neonvm/bin/busybox --install -s /neonvm/bin + && /neonvm/bin/busybox --install -s /neonvm/bin COPY helper.move-bins.sh /helper.move-bins.sh diff --git a/vm-builder/files/agetty-init-amd64 b/vm-builder/files/agetty-init-amd64 new file mode 100644 index 000000000..894a24ae4 --- /dev/null +++ b/vm-builder/files/agetty-init-amd64 @@ -0,0 +1 @@ +ttyS0::respawn:/neonvm/bin/agetty --8bits --local-line --noissue --noclear --noreset --host console --login-program /neonvm/bin/login --login-pause --autologin root 115200 ttyS0 linux diff --git a/vm-builder/files/agetty-init-arm64 b/vm-builder/files/agetty-init-arm64 new file mode 100644 index 000000000..713a4f669 --- /dev/null +++ b/vm-builder/files/agetty-init-arm64 @@ -0,0 +1 @@ +ttyAMA0::respawn:/neonvm/bin/agetty --8bits --local-line --noissue --noclear --noreset --host console --login-program /neonvm/bin/login --login-pause --autologin root 115200 ttyAMA0 linux