From 52314918cb87dfc523159785d9f0e95b9a40ed10 Mon Sep 17 00:00:00 2001 From: Misha Sakhnov Date: Tue, 22 Oct 2024 18:36:48 +0200 Subject: [PATCH] ci: build images for arm and amd using github-action-matrix-outputs Signed-off-by: Misha Sakhnov --- .github/workflows/build-images.yaml | 151 ++++++++++++++++--------- .github/workflows/build-test-vm.yaml | 82 ++++++++++---- .github/workflows/check-ca-builds.yaml | 9 +- .github/workflows/e2e-test.yaml | 76 ++++++++++--- .github/workflows/vm-kernel.yaml | 35 ++++-- Makefile | 6 +- cluster-autoscaler/Dockerfile | 10 +- tests/e2e/vm-neonvmd/00-create-vm.yaml | 1 + 8 files changed, 257 insertions(+), 113 deletions(-) diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 3784c651e..caa815781 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -30,22 +30,35 @@ on: description: 'Should images be uploaded to neon ECR' type: boolean required: false + arch: + description: 'Architecture to build for' + type: string + required: false + default: 'amd64' + matrix-step-name: + required: false + type: string + matrix-key: + required: false + type: string + outputs: + controller: description: 'neonvm-controller image' - value: ${{ jobs.tags.outputs.controller }} + value: ${{ jobs.save-matrix-results.outputs.controller }} vxlan-controller: description: 'neonvm-vxlan-controller image' - value: ${{ jobs.tags.outputs.vxlan-controller }} + value: ${{ jobs.save-matrix-results.outputs.vxlan-controller}} runner: description: 'neonvm-runner image' - value: ${{ jobs.tags.outputs.runner }} + value: ${{ jobs.save-matrix-results.outputs.runner}} scheduler: description: 'autoscale-scheduler image' - value: ${{ jobs.tags.outputs.scheduler }} + value: ${{ jobs.save-matrix-results.outputs.scheduler}} autoscaler-agent: description: 'autoscaler-agent image' - value: ${{ jobs.tags.outputs.autoscaler-agent }} + value: ${{ jobs.save-matrix-results.outputs.autoscaler-agent}} env: IMG_CONTROLLER: "neondatabase/neonvm-controller" @@ -58,39 +71,11 @@ env: ECR_DEV: "369495373322.dkr.ecr.eu-central-1.amazonaws.com" ECR_PROD: "093970136003.dkr.ecr.eu-central-1.amazonaws.com" - # Why localhost? We use a local registry so that when docker/build-push-action tries to pull the - # image we built locally, it'll actually have a place to pull from. - # - # Otherwise, if we just try to use a local image, it fails trying to pull it from dockerhub. - # See https://github.com/moby/buildkit/issues/2343 for more information. - GO_BASE_IMG: "localhost:5000/neondatabase/autoscaling-go-base:dev" - # Default architecture to build. In future it would be changed to multi-arch build or separate builds for each arch - TARGET_ARCH: "amd64" - defaults: run: shell: bash -euo pipefail {0} jobs: - tags: - outputs: - controller: ${{ steps.show-tags.outputs.controller }} - vxlan-controller: ${{ steps.show-tags.outputs.vxlan-controller }} - runner: ${{ steps.show-tags.outputs.runner }} - scheduler: ${{ steps.show-tags.outputs.scheduler }} - autoscaler-agent: ${{ steps.show-tags.outputs.autoscaler-agent }} - cluster-autoscaler: ${{ steps.show-tags.outputs.cluster-autoscaler }} - runs-on: ubuntu-latest - steps: - - id: show-tags - run: | - echo "controller=${{ env.IMG_CONTROLLER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - echo "vxlan-controller=${{ env.IMG_VXLAN_CONTROLLER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - echo "runner=${{ env.IMG_RUNNER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - echo "scheduler=${{ env.IMG_SCHEDULER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - echo "autoscaler-agent=${{ env.IMG_AUTOSCALER_AGENT }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - echo "cluster-autoscaler=${{ env.IMG_CLUSTER_AUTOSCALER }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - vm-kernel: # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'. if: ${{ format('{0}', inputs.skip) != 'true' }} @@ -98,17 +83,33 @@ jobs: with: tag: ${{ inputs.kernel-image || inputs.tag }} return-image-for-tag: ${{ inputs.kernel-image }} + arch: ${{ inputs.arch }} secrets: inherit - + build: # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'. if: ${{ format('{0}', inputs.skip) != 'true' }} - needs: [ tags, vm-kernel ] - runs-on: [ self-hosted, large ] + needs: [ vm-kernel ] + outputs: + controller: ${{ steps.tags.outputs.controller }} + vxlan-controller: ${{ steps.tags.outputs.vxlan-controller }} + runner: ${{ steps.tags.outputs.runner }} + scheduler: ${{ steps.tags.outputs.scheduler }} + autoscaler-agent: ${{ steps.tags.outputs.autoscaler-agent }} + cluster-autoscaler: ${{ steps.tags.outputs.cluster-autoscaler }} + env: + # Why localhost? We use a local registry so that when docker/build-push-action tries to pull the + # image we built locally, it'll actually have a place to pull from. + # + # Otherwise, if we just try to use a local image, it fails trying to pull it from dockerhub. + # See https://github.com/moby/buildkit/issues/2343 for more information. + GO_BASE_IMG: ${{ format('localhost:5000/neondatabase/autoscaling-go-base-{0}:dev', inputs.arch) }} permissions: contents: read # This is required for actions/checkout id-token: write # This is required for aws-actions/configure-aws-credentials + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} + services: registry: image: registry:2 @@ -116,6 +117,17 @@ jobs: - 5000:5000 steps: + # tags converted to be a step and moved here to be in the same strategy context + + - id: tags + run: | + echo "controller=${{ env.IMG_CONTROLLER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "vxlan-controller=${{ env.IMG_VXLAN_CONTROLLER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "runner=${{ env.IMG_RUNNER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "scheduler=${{ env.IMG_SCHEDULER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "autoscaler-agent=${{ env.IMG_AUTOSCALER_AGENT }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "cluster-autoscaler=${{ env.IMG_CLUSTER_AUTOSCALER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + - uses: actions/checkout@v4 with: fetch-depth: 0 # fetch all, so that we also include tags @@ -164,7 +176,6 @@ jobs: registry: cache.neon.build username: ${{ secrets.NEON_CI_DOCKERCACHE_USERNAME }} password: ${{ secrets.NEON_CI_DOCKERCACHE_PASSWORD }} - - name: Configure dev AWS credentials if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }} uses: aws-actions/configure-aws-credentials@v4 @@ -200,7 +211,8 @@ jobs: - name: Load VM kernel env: - IMAGE: ${{ needs.vm-kernel.outputs.image }} + IMAGE: ${{ needs.vm-kernel.outputs.image }} + # IMAGE: ${{ fromJson(needs.vm-kernel-results.outputs.result).image[ inputs.arch ] }} run: | docker pull --quiet $IMAGE ID=$(docker create $IMAGE true) @@ -212,23 +224,23 @@ jobs: id: build-go-dependencies-image with: context: . - platforms: linux/amd64 + push: true file: Dockerfile.go-base cache-from: type=registry,ref=cache.neon.build/autoscaling-go-base:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscaling-go-base:cache,mode=max' || '' }} tags: ${{ env.GO_BASE_IMG }} - + - name: Build and push neonvm-runner image uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 push: true + platforms: linux/${{ inputs.arch }} file: neonvm-runner/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-runner:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-runner:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.runner }} + tags: ${{ steps.tags.outputs.runner }} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} @@ -247,41 +259,41 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: neonvm-controller/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-controller:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-controller:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.controller }} + tags: ${{ steps.tags.outputs.controller }} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} - VM_RUNNER_IMAGE=${{ needs.tags.outputs.runner }} + VM_RUNNER_IMAGE=${{ steps.tags.outputs.runner }} BUILDTAGS=${{ steps.controller-build-tags.outputs.buildtags }} - name: Build and push neonvm-vxlan-controller image uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: neonvm-vxlan-controller/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-vxlan-controller:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-vxlan-controller:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.vxlan-controller }} + tags: ${{ steps.tags.outputs.vxlan-controller }} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} - TARGET_ARCH=${{ env.TARGET_ARCH }} + TARGET_ARCH=${{ inputs.arch }} - name: Build and push autoscale-scheduler image uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: autoscale-scheduler/Dockerfile cache-from: type=registry,ref=cache.neon.build/autoscale-scheduler:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscale-scheduler:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.scheduler }} + tags: ${{ steps.tags.outputs.scheduler}} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} GIT_INFO=${{ steps.get-git-info.outputs.info }}:${{ inputs.tag }} @@ -290,29 +302,30 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: autoscaler-agent/Dockerfile cache-from: type=registry,ref=cache.neon.build/autoscaler-agent:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscaler-agent:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.autoscaler-agent }} + tags: ${{ steps.tags.outputs.autoscaler-agent}} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} GIT_INFO=${{ steps.get-git-info.outputs.info }} - + - name: Build and push cluster-autoscaler image uses: docker/build-push-action@v6 if: ${{ format('{0}', inputs.build-cluster-autoscaler) == 'true' }} with: context: cluster-autoscaler - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true - tags: ${{ needs.tags.outputs.cluster-autoscaler }} + target: ${{format('cluster_autoscaler_{0}', inputs.arch)}} + tags: ${{ steps.tags.outputs.cluster-autoscaler}} cache-from: type=registry,ref=cache.neon.build/cluster-autoscaler-neonvm:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/cluster-autoscaler-neonvm:cache,mode=max' || '' }} build-args: | CA_GIT_TAG=${{ steps.get-ca-tag.outputs.tag }} - + - name: Copy all images to ECR if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }} run: | @@ -332,3 +345,29 @@ jobs: docker buildx imagetools create -t ${{ env.ECR_PROD }}/${image}:${{ inputs.tag }} \ neondatabase/${image}:${{ inputs.tag }} done + + save-matrix-results: + runs-on: ubuntu-latest + needs: [build] + steps: + ## Write for matrix outputs workaround + - uses: cloudposse/github-action-matrix-outputs-write@v1 + id: out + with: + matrix-step-name: ${{ inputs.matrix-step-name }} + matrix-key: ${{ inputs.matrix-key }} + outputs: |- + controller: ${{ needs.build.outputs.controller }} + vxlan-controller: ${{ needs.build.outputs.vxlan-controller }} + runner: ${{ needs.build.outputs.runner }} + scheduler: ${{ needs.build.outputs.scheduler }} + autoscaler-agent: ${{ needs.build.outputs.autoscaler-agent }} + cluster-autoscaler: ${{ needs.build.outputs.cluster-autoscaler }} + + outputs: + controller: ${{ steps.out.outputs.controller }} + vxlan-controller: ${{ steps.out.outputs.vxlan-controller }} + runner: ${{ steps.out.outputs.runner }} + scheduler: ${{ steps.out.outputs.scheduler }} + autoscaler-agent: ${{ steps.out.outputs.autoscaler-agent }} + cluster-autoscaler: ${{ steps.out.outputs.cluster-autoscaler }} diff --git a/.github/workflows/build-test-vm.yaml b/.github/workflows/build-test-vm.yaml index 265b807bc..2e676ff6c 100644 --- a/.github/workflows/build-test-vm.yaml +++ b/.github/workflows/build-test-vm.yaml @@ -22,39 +22,47 @@ on: type: boolean required: false default: false + arch: + description: 'Architecture to build for' + type: string + required: false + default: 'amd64' + matrix-step-name: + required: false + type: string + matrix-key: + required: false + type: string outputs: vm-postgres-16-bullseye: description: 'image name for postgres:16-bullseye, VM-ified' - value: ${{ jobs.tags.outputs.vm-postgres-16-bullseye }} - + value: ${{ jobs.build.outputs.vm-postgres-16-bullseye }} + env: IMG_POSTGRES_16_BULLSEYE: "neondatabase/vm-postgres-16-bullseye" # using image built in the same workflow - IMG_DAEMON: "daemon:dev" - TARGET_ARCH: "amd64" + IMG_DAEMON: "daemon" defaults: run: shell: bash -euo pipefail {0} jobs: - tags: - outputs: - vm-postgres-16-bullseye: ${{ steps.show-tags.outputs.vm-postgres-16-bullseye }} - daemon: ${{ steps.show-tags.outputs.daemon }} - runs-on: ubuntu-latest - steps: - - id: show-tags - run: | - echo "vm-postgres-16-bullseye=${{ env.IMG_POSTGRES_16_BULLSEYE }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - echo "daemon=${{ env.IMG_DAEMON }}" | tee -a $GITHUB_OUTPUT - + build: # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'. if: ${{ format('{0}', inputs.skip) != 'true' }} - needs: tags - runs-on: [ self-hosted, gen3, large ] + outputs: + vm-postgres-16-bullseye: ${{ steps.tags.outputs.vm-postgres-16-bullseye }} + daemon: ${{ steps.tags.outputs.daemon }} + # TODO: do we need gen3 runners? to clarify with dev exp team + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} steps: + # tags converted to be a step and moved here to be in the same strategy contextt + - id: tags + run: | + echo "vm-postgres-16-bullseye=${{ env.IMG_POSTGRES_16_BULLSEYE }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "daemon=${{ env.IMG_DAEMON }}-${{inputs.arch}}:dev" | tee -a $GITHUB_OUTPUT - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: @@ -63,7 +71,16 @@ jobs: cache: false # Sometimes setup-go gets stuck. Without this, it'll keep going until the job gets killed timeout-minutes: 10 - - run: make docker-build-daemon + - name: Build daemon image + uses: docker/build-push-action@v6 + with: + context: . + push: false + platforms: linux/${{ inputs.arch }} + file: neonvm-daemon/Dockerfile + cache-from: type=registry,ref=cache.neon.build/neonvm-daemon:cache + cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-daemon:cache,mode=max' || '' }} + tags: ${{ steps.tags.outputs.daemon }} - run: make bin/vm-builder - name: upload vm-builder @@ -71,7 +88,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: vm-builder - path: bin/vm-builder + path: ${{format('bin/vm-builder-{0}', inputs.arch)}} if-no-files-found: error retention-days: 2 @@ -84,9 +101,28 @@ jobs: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - - name: build ${{ needs.tags.outputs.vm-postgres-16-bullseye }} + - name: build ${{ steps.tags.outputs.vm-postgres-16-bullseye }} run: | - ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ needs.tags.outputs.daemon }} -target-arch linux/${TARGET_ARCH} - - name: docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }} + ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ steps.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ steps.tags.outputs.daemon }} -target-arch linux/${{ inputs.arch }} + - name: docker push ${{ steps.tags.outputs.vm-postgres-16-bullseye }} run: | - docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }} + docker push ${{ steps.tags.outputs.vm-postgres-16-bullseye }} + + + save-matrix-results: + runs-on: ubuntu-latest + needs: [build] + steps: + ## Write for matrix outputs workaround + - uses: cloudposse/github-action-matrix-outputs-write@v1 + id: out + with: + matrix-step-name: ${{ inputs.matrix-step-name }} + matrix-key: ${{ inputs.matrix-key }} + outputs: |- + vm-postgres-16-bullseye: ${{ needs.build.outputs.vm-postgres-16-bullseye }} + daemon: ${{ needs.build.outputs.daemon }} + + outputs: + vm-postgres-16-bullseye: ${{ steps.out.outputs.vm-postgres-16-bullseye }} + daemon: ${{ steps.out.outputs.daemon }} diff --git a/.github/workflows/check-ca-builds.yaml b/.github/workflows/check-ca-builds.yaml index 895db56db..6bcfe98b8 100644 --- a/.github/workflows/check-ca-builds.yaml +++ b/.github/workflows/check-ca-builds.yaml @@ -17,7 +17,11 @@ on: jobs: build-ca: - runs-on: [ self-hosted, gen3, small ] + strategy: + fail-fast: false + matrix: + arch: [ amd64, arm64 ] + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} steps: - uses: actions/checkout@v4 @@ -41,8 +45,9 @@ jobs: uses: docker/build-push-action@v6 with: context: cluster-autoscaler - platforms: linux/amd64 + platforms: ${{format('linux/{0}', matrix.arch)}} push: false + target: ${{format('cluster_autoscaler_{0}', matrix.arch)}} file: cluster-autoscaler/Dockerfile cache-from: type=registry,ref=cache.neon.build/cluster-autoscaler-neonvm:cache build-args: | diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index 2d2aaf390..d2c413992 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -50,10 +50,13 @@ jobs: test -n "$SHA" sha="${SHA::7}" echo "tag=$sha.$GITHUB_RUN_ID" | tee -a $GITHUB_OUTPUT - + build-images: needs: get-tag uses: ./.github/workflows/build-images.yaml + strategy: + matrix: + arch: [amd64, arm64] with: skip: ${{ inputs.tag != '' }} tag: ${{ inputs.tag || needs.get-tag.outputs.tag }} @@ -62,25 +65,61 @@ jobs: # settings and used properly in the tests. But if skip (because inputs.tag != ''), then this # setting will have no effect and the release images will be normal. controller-preserve-runner-pods: true + matrix-step-name: build-images + matrix-key: ${{ matrix.arch }} + arch: ${{ matrix.arch }} secrets: inherit + get-build-images-matrix-results: + runs-on: ubuntu-latest + needs: [build-images] + steps: + - uses: cloudposse/github-action-matrix-outputs-read@v1 + id: read + with: + matrix-step-name: build-images + outputs: + result: "${{ steps.read.outputs.result }}" + build-test-vm: needs: get-tag uses: ./.github/workflows/build-test-vm.yaml + strategy: + matrix: + arch: [amd64, arm64] with: skip: ${{ inputs.tag != '' }} tag: ${{ inputs.tag || needs.get-tag.outputs.tag }} + matrix-step-name: build-test-vm + matrix-key: ${{ matrix.arch }} + arch: ${{ matrix.arch }} secrets: inherit + get-build-test-vm-matrix-results: + runs-on: ubuntu-latest + needs: [build-test-vm] + steps: + - uses: cloudposse/github-action-matrix-outputs-read@v1 + id: read + with: + matrix-step-name: build-test-vm + outputs: + result: "${{ steps.read.outputs.result }}" + e2e-tests: - needs: [ build-images, build-test-vm ] + needs: [ get-build-images-matrix-results, get-build-test-vm-matrix-results ] strategy: fail-fast: false matrix: - cluster: - - ${{ inputs.cluster || 'k3d' }} - runs-on: [ self-hosted, gen3, large ] + cluster: [ k3d ] + # run tests on amd64 only, since scope of the PR is to build images, there is separate issue for e2e tests + arch: [ amd64 ] + + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} + steps: + - run: echo ${{ needs.get-build-images-matrix-results.outputs.result }} + - run: echo ${{ needs.get-build-test-vm-matrix-results.outputs.result }} - uses: actions/checkout@v4 with: fetch-depth: 0 # fetch all, so that we also include tags @@ -109,18 +148,19 @@ jobs: - run: make render-release env: - IMG_CONTROLLER: ${{ needs.build-images.outputs.controller }} - IMG_VXLAN_CONTROLLER: ${{ needs.build-images.outputs.vxlan-controller }} - IMG_RUNNER: ${{ needs.build-images.outputs.runner }} - IMG_SCHEDULER: ${{ needs.build-images.outputs.scheduler }} - IMG_AUTOSCALER_AGENT: ${{ needs.build-images.outputs.autoscaler-agent }} + # ${{ fromJson(needs.vm-kernel-results.outputs.result).image[ inputs.arch ] }} + IMG_CONTROLLER: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).controller[matrix.arch] }} + IMG_VXLAN_CONTROLLER: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).vxlan-controller[matrix.arch] }} + IMG_RUNNER: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).runner[matrix.arch]}} + IMG_SCHEDULER: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).scheduler[matrix.arch]}} + IMG_AUTOSCALER_AGENT: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).autoscaler-agent[matrix.arch]}} - name: upload manifests # nb: use format(..) to catch both inputs.push-yamls = true AND inputs.push-yamls = 'true'. - if: ${{ format('{0}', inputs.push-yamls) == 'true' }} + if: ${{ format('{0}', inputs.push-yamls) == 'true'}} uses: actions/upload-artifact@v4 with: - name: rendered_manifests + name: ${{ format('rendered_manifests-{0}', matrix.arch) }} # nb: prefix before wildcard is removed from the uploaded files, so the artifact should # contain e.g. # - autoscale-scheduler.yaml @@ -167,9 +207,9 @@ jobs: run: | rendered () { echo "rendered_manifests/$1"; } - kubectl apply -f $(rendered multus-amd64.yaml) + kubectl apply -f $(rendered multus-${{ matrix.arch}}.yaml) kubectl -n kube-system rollout status daemonset kube-multus-ds - kubectl apply -f $(rendered whereabouts-amd64.yaml) + kubectl apply -f $(rendered whereabouts-${{matrix.arch}}.yaml) kubectl -n kube-system rollout status daemonset whereabouts kubectl apply -f $(rendered neonvm-runner-image-loader.yaml) kubectl -n neonvm-system rollout status daemonset neonvm-runner-image-loader @@ -186,7 +226,7 @@ jobs: - name: load e2e test vm image env: - TEST_IMAGE: ${{ needs.build-test-vm.outputs.vm-postgres-16-bullseye }} + TEST_IMAGE: ${{ fromJson(needs.get-build-test-vm-matrix-results.outputs.result).vm-postgres-16-bullseye[matrix.arch]}} timeout-minutes: 2 run: | # Pull the docker image so we can re-tag it, because using a consistent tag inside the @@ -194,7 +234,11 @@ jobs: docker pull "$TEST_IMAGE" docker image tag "$TEST_IMAGE" "$IMG_E2E_TEST" make load-example-vms - + + # - name: Patch e2e tests to not use acceleration in arm runners + # if: matrix.arch == 'arm64' + # run: make arm_patch_e2e + - run: make e2e timeout-minutes: 15 diff --git a/.github/workflows/vm-kernel.yaml b/.github/workflows/vm-kernel.yaml index 67d780fdb..8767a10c3 100644 --- a/.github/workflows/vm-kernel.yaml +++ b/.github/workflows/vm-kernel.yaml @@ -28,13 +28,14 @@ on: type: boolean required: false default: false + arch: + description: 'Architecture to build the kernel image for' + type: string + required: true outputs: image: - description: 'vm-kernel Docker image' - value: ${{ jobs.setup-build-vm-kernel-image.outputs.image || jobs.build-vm-kernel-image.outputs.image }} - -env: - VM_KERNEL_IMAGE: "neondatabase/vm-kernel" + description: "Image" + value: ${{ jobs.build-vm-kernel-image.outputs.image }} defaults: run: @@ -43,10 +44,11 @@ defaults: jobs: setup-build-vm-kernel-image: outputs: - image: ${{ steps.get-kernel-image.outputs.image }} + image: ${{ steps.get-kernel-image.outputs.image-arm64 }} kernel-cache-tag: ${{ steps.get-kernel-cache-tag.outputs.kernel-cache-tag }} - runs-on: ubuntu-latest + env: + VM_KERNEL_IMAGE: ${{format('neondatabase/vm-kernel-{0}', inputs.arch)}} steps: - uses: actions/checkout@v4 @@ -127,11 +129,16 @@ jobs: docker push ${VM_KERNEL_IMAGE}:${NEW_TAG} build-vm-kernel-image: + needs: setup-build-vm-kernel-image if: needs.setup-build-vm-kernel-image.outputs.image == '' outputs: + # image for backward compatibility image: ${{ steps.get-tags.outputs.canonical }}@${{ steps.build-linux-kernel.outputs.digest }} + env: + VM_KERNEL_IMAGE: ${{format('neondatabase/vm-kernel-{0}', inputs.arch)}} + runs-on: [ self-hosted, gen3, large ] steps: - name: git checkout @@ -162,7 +169,7 @@ jobs: echo VM_KERNEL_VERSION=$kernel_version >> $GITHUB_OUTPUT - - name: get docker tags + - name: get docker tags id: get-tags env: KERNEL_VERSION_TAG: ${{ inputs.tag || steps.get-kernel-version.outputs.VM_KERNEL_VERSION }} @@ -185,14 +192,18 @@ jobs: with: build-args: KERNEL_VERSION=${{ steps.get-kernel-version.outputs.VM_KERNEL_VERSION }} context: neonvm-kernel - platforms: linux/amd64 + platforms: ${{format('linux/{0}', inputs.arch)}} # neonvm-kernel/Dockerfile.kernel-builder has different targets for different architectures # so we need to specify the target explicitly - target: kernel_amd64 + target: ${{format('kernel_{0}', inputs.arch)}} # Push kernel image only for scheduled builds or if workflow_dispatch/workflow_call input is true push: true pull: true file: neonvm-kernel/Dockerfile.kernel-builder - cache-from: type=registry,ref=cache.neon.build/vm-kernel:cache - cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/vm-kernel:cache,mode=max' || '' }} + cache-from: ${{format('type=registry,ref=cache.neon.build/vm-kernel:{0}cache', inputs.arch)}} + # Cache the kernel image for future builds, temporary from all branches + cache-to: ${{ format('type=registry,ref=cache.neon.build/vm-kernel:{0}cache,mode=max', inputs.arch) || '' }} tags: ${{ steps.get-tags.outputs.tags }} + + + - run: git diff neonvm-kernel diff --git a/Makefile b/Makefile index 9ef568293..0527b4b26 100644 --- a/Makefile +++ b/Makefile @@ -344,12 +344,10 @@ render-release: $(RENDERED) kustomize cd autoscaler-agent && $(KUSTOMIZE) edit set image autoscaler-agent=$(IMG_AUTOSCALER_AGENT) # Build: $(KUSTOMIZE) build neonvm/config/whereabouts-amd64 > $(RENDERED)/whereabouts-amd64.yaml - # TODO: I ain't sure if we need arm64 for render-release target - # $(KUSTOMIZE) build neonvm/config/whereabouts-arm64 > $(RENDERED)/whereabouts-arm64.yaml + $(KUSTOMIZE) build neonvm/config/whereabouts-arm64 > $(RENDERED)/whereabouts-arm64.yaml $(KUSTOMIZE) build neonvm/config/multus-aks > $(RENDERED)/multus-aks.yaml $(KUSTOMIZE) build neonvm/config/multus-eks > $(RENDERED)/multus-eks.yaml $(KUSTOMIZE) build neonvm/config/multus-amd64 > $(RENDERED)/multus-amd64.yaml - # TODO: I ain't sure if we need arm64 for render-release target $(KUSTOMIZE) build neonvm/config/multus-arm64 > $(RENDERED)/multus-arm64.yaml $(KUSTOMIZE) build neonvm/config > $(RENDERED)/neonvm.yaml $(KUSTOMIZE) build neonvm-controller > $(RENDERED)/neonvm-controller.yaml @@ -507,7 +505,9 @@ else ifeq ($(GOARCH), amd64) else $(error Unsupported architecture: $(GOARCH)) endif + KUBECTL ?= $(LOCALBIN)/kubectl + KUBECTL_VERSION ?= v1.28.12 KIND ?= $(LOCALBIN)/kind diff --git a/cluster-autoscaler/Dockerfile b/cluster-autoscaler/Dockerfile index d868e62e9..e14c39e54 100644 --- a/cluster-autoscaler/Dockerfile +++ b/cluster-autoscaler/Dockerfile @@ -28,7 +28,15 @@ RUN cd autoscaler/cluster-autoscaler \ # This is adapted from CA's Dockerfile.amd64, here: # https://github.com/kubernetes/autoscaler/blob/cluster-autoscaler-1.24.1/cluster-autoscaler/Dockerfile.amd64 -FROM gcr.io/distroless/static:nonroot-amd64 + +# NB: two build stages, one for each architecture, because I wasn't able to use variable substitution in FROM statements +FROM gcr.io/distroless/static:nonroot-amd64 as cluster_autoscaler_amd64 + +WORKDIR / +COPY --from=builder /workspace/cluster-autoscaler . +CMD ["/cluster-autoscaler"] + +FROM gcr.io/distroless/static:nonroot-arm64 as cluster_autoscaler_arm64 WORKDIR / COPY --from=builder /workspace/cluster-autoscaler . diff --git a/tests/e2e/vm-neonvmd/00-create-vm.yaml b/tests/e2e/vm-neonvmd/00-create-vm.yaml index 2bb3c345d..28ab41499 100644 --- a/tests/e2e/vm-neonvmd/00-create-vm.yaml +++ b/tests/e2e/vm-neonvmd/00-create-vm.yaml @@ -7,6 +7,7 @@ kind: VirtualMachine metadata: name: example spec: + cpuScalingMode: sysfsScaling schedulerName: autoscale-scheduler enableSSH: true guest: