diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 071d52d93..c10652327 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -30,26 +30,39 @@ on: description: 'Should images be uploaded to neon ECR' type: boolean required: false + arch: + description: 'Architecture to build for' + type: string + required: false + default: 'amd64' + matrix-step-name: + required: false + type: string + matrix-key: + required: false + type: string + outputs: + controller: description: 'neonvm-controller image' - value: ${{ jobs.tags.outputs.controller }} + value: ${{ jobs.save-matrix-results.outputs.controller }} vxlan-controller: description: 'neonvm-vxlan-controller image' - value: ${{ jobs.tags.outputs.vxlan-controller }} + value: ${{ jobs.save-matrix-results.outputs.vxlan-controller}} runner: description: 'neonvm-runner image' - value: ${{ jobs.tags.outputs.runner }} + value: ${{ jobs.save-matrix-results.outputs.runner}} scheduler: description: 'autoscale-scheduler image' - value: ${{ jobs.tags.outputs.scheduler }} + value: ${{ jobs.save-matrix-results.outputs.scheduler}} autoscaler-agent: description: 'autoscaler-agent image' value: ${{ jobs.tags.outputs.autoscaler-agent }} daemon: description: 'neonvm-daemon image' value: ${{ jobs.tags.outputs.daemon }} - + env: IMG_CONTROLLER: "neondatabase/neonvm-controller" IMG_VXLAN_CONTROLLER: "neondatabase/neonvm-vxlan-controller" @@ -62,15 +75,6 @@ env: ECR_DEV: "369495373322.dkr.ecr.eu-central-1.amazonaws.com" ECR_PROD: "093970136003.dkr.ecr.eu-central-1.amazonaws.com" - # Why localhost? We use a local registry so that when docker/build-push-action tries to pull the - # image we built locally, it'll actually have a place to pull from. - # - # Otherwise, if we just try to use a local image, it fails trying to pull it from dockerhub. - # See https://github.com/moby/buildkit/issues/2343 for more information. - GO_BASE_IMG: "localhost:5000/neondatabase/autoscaling-go-base:dev" - # Default architecture to build. In future it would be changed to multi-arch build or separate builds for each arch - TARGET_ARCH: "amd64" - defaults: run: shell: bash -euo pipefail {0} @@ -104,17 +108,33 @@ jobs: with: tag: ${{ inputs.kernel-image || inputs.tag }} return-image-for-tag: ${{ inputs.kernel-image }} + arch: ${{ inputs.arch }} secrets: inherit build: # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'. if: ${{ format('{0}', inputs.skip) != 'true' }} - needs: [ tags, vm-kernel ] - runs-on: [ self-hosted, large ] + needs: [ vm-kernel ] + outputs: + controller: ${{ steps.tags.outputs.controller }} + vxlan-controller: ${{ steps.tags.outputs.vxlan-controller }} + runner: ${{ steps.tags.outputs.runner }} + scheduler: ${{ steps.tags.outputs.scheduler }} + autoscaler-agent: ${{ steps.tags.outputs.autoscaler-agent }} + cluster-autoscaler: ${{ steps.tags.outputs.cluster-autoscaler }} + env: + # Why localhost? We use a local registry so that when docker/build-push-action tries to pull the + # image we built locally, it'll actually have a place to pull from. + # + # Otherwise, if we just try to use a local image, it fails trying to pull it from dockerhub. + # See https://github.com/moby/buildkit/issues/2343 for more information. + GO_BASE_IMG: ${{ format('localhost:5000/neondatabase/autoscaling-go-base-{0}:dev', inputs.arch) }} permissions: contents: read # This is required for actions/checkout id-token: write # This is required for aws-actions/configure-aws-credentials + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} + services: registry: image: registry:2 @@ -122,6 +142,18 @@ jobs: - 5000:5000 steps: + # tags converted to be a step and moved here to be in the same strategy context + + - id: tags + run: | + echo "controller=${{ env.IMG_CONTROLLER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "vxlan-controller=${{ env.IMG_VXLAN_CONTROLLER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "runner=${{ env.IMG_RUNNER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "scheduler=${{ env.IMG_SCHEDULER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "autoscaler-agent=${{ env.IMG_AUTOSCALER_AGENT }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "cluster-autoscaler=${{ env.IMG_CLUSTER_AUTOSCALER }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "daemon=${{ env.IMG_DAEMON }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + - uses: actions/checkout@v4 with: fetch-depth: 0 # fetch all, so that we also include tags @@ -164,7 +196,6 @@ jobs: registry: cache.neon.build username: ${{ secrets.NEON_CI_DOCKERCACHE_USERNAME }} password: ${{ secrets.NEON_CI_DOCKERCACHE_PASSWORD }} - - name: Configure dev AWS credentials if: ${{ format('{0}', inputs.upload-to-ecr) == 'true' }} uses: aws-actions/configure-aws-credentials@v4 @@ -212,7 +243,7 @@ jobs: id: build-go-dependencies-image with: context: . - platforms: linux/amd64 + push: true file: Dockerfile.go-base cache-from: type=registry,ref=cache.neon.build/autoscaling-go-base:cache @@ -223,12 +254,12 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 push: true + platforms: linux/${{ inputs.arch }} file: neonvm-runner/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-runner:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-runner:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.runner }} + tags: ${{ steps.tags.outputs.runner }} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} @@ -247,41 +278,41 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: neonvm-controller/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-controller:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-controller:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.controller }} + tags: ${{ steps.tags.outputs.controller }} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} - VM_RUNNER_IMAGE=${{ needs.tags.outputs.runner }} + VM_RUNNER_IMAGE=${{ steps.tags.outputs.runner }} BUILDTAGS=${{ steps.controller-build-tags.outputs.buildtags }} - name: Build and push neonvm-vxlan-controller image uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: neonvm-vxlan-controller/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-vxlan-controller:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-vxlan-controller:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.vxlan-controller }} + tags: ${{ steps.tags.outputs.vxlan-controller }} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} - TARGET_ARCH=${{ env.TARGET_ARCH }} + TARGET_ARCH=${{ inputs.arch }} - name: Build and push autoscale-scheduler image uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: autoscale-scheduler/Dockerfile cache-from: type=registry,ref=cache.neon.build/autoscale-scheduler:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscale-scheduler:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.scheduler }} + tags: ${{ steps.tags.outputs.scheduler}} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} GIT_INFO=${{ steps.get-git-info.outputs.info }}:${{ inputs.tag }} @@ -290,12 +321,12 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: autoscaler-agent/Dockerfile cache-from: type=registry,ref=cache.neon.build/autoscaler-agent:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscaler-agent:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.autoscaler-agent }} + tags: ${{ steps.tags.outputs.autoscaler-agent}} build-args: | GO_BASE_IMG=${{ env.GO_BASE_IMG }} GIT_INFO=${{ steps.get-git-info.outputs.info }} @@ -304,23 +335,24 @@ jobs: uses: docker/build-push-action@v6 with: context: . - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true file: neonvm-daemon/Dockerfile cache-from: type=registry,ref=cache.neon.build/neonvm-daemon:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-daemon:cache,mode=max' || '' }} - tags: ${{ needs.tags.outputs.daemon }} + tags: ${{ steps.tags.outputs.daemon }} build-args: | - GO_BASE_IMG=${{ env.GO_BASE_IMG }} + GO_BASE_IMG=${{ env.GO_BASE_IMG }} - name: Build and push cluster-autoscaler image uses: docker/build-push-action@v6 if: ${{ format('{0}', inputs.build-cluster-autoscaler) == 'true' }} with: context: cluster-autoscaler - platforms: linux/amd64 + platforms: linux/${{ inputs.arch }} push: true - tags: ${{ needs.tags.outputs.cluster-autoscaler }} + target: ${{format('cluster_autoscaler_{0}', inputs.arch)}} + tags: ${{ steps.tags.outputs.cluster-autoscaler}} cache-from: type=registry,ref=cache.neon.build/cluster-autoscaler-neonvm:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/cluster-autoscaler-neonvm:cache,mode=max' || '' }} @@ -343,3 +375,29 @@ jobs: docker buildx imagetools create -t ${{ env.ECR_PROD }}/${image}:${{ inputs.tag }} \ neondatabase/${image}:${{ inputs.tag }} done + + save-matrix-results: + runs-on: ubuntu-latest + needs: [build] + steps: + ## Write for matrix outputs workaround + - uses: cloudposse/github-action-matrix-outputs-write@v1 + id: out + with: + matrix-step-name: ${{ inputs.matrix-step-name }} + matrix-key: ${{ inputs.matrix-key }} + outputs: |- + controller: ${{ needs.build.outputs.controller }} + vxlan-controller: ${{ needs.build.outputs.vxlan-controller }} + runner: ${{ needs.build.outputs.runner }} + scheduler: ${{ needs.build.outputs.scheduler }} + autoscaler-agent: ${{ needs.build.outputs.autoscaler-agent }} + cluster-autoscaler: ${{ needs.build.outputs.cluster-autoscaler }} + + outputs: + controller: ${{ steps.out.outputs.controller }} + vxlan-controller: ${{ steps.out.outputs.vxlan-controller }} + runner: ${{ steps.out.outputs.runner }} + scheduler: ${{ steps.out.outputs.scheduler }} + autoscaler-agent: ${{ steps.out.outputs.autoscaler-agent }} + cluster-autoscaler: ${{ steps.out.outputs.cluster-autoscaler }} diff --git a/.github/workflows/build-test-vm.yaml b/.github/workflows/build-test-vm.yaml index d6d0eec87..f55fcce13 100644 --- a/.github/workflows/build-test-vm.yaml +++ b/.github/workflows/build-test-vm.yaml @@ -22,10 +22,21 @@ on: type: boolean required: false default: false + arch: + description: 'Architecture to build for' + type: string + required: false + default: 'amd64' + matrix-step-name: + required: false + type: string + matrix-key: + required: false + type: string outputs: vm-postgres-16-bullseye: description: 'image name for postgres:16-bullseye, VM-ified' - value: ${{ jobs.tags.outputs.vm-postgres-16-bullseye }} + value: ${{ jobs.build.outputs.vm-postgres-16-bullseye }} env: IMG_POSTGRES_16_BULLSEYE: "neondatabase/vm-postgres-16-bullseye" @@ -38,25 +49,19 @@ defaults: shell: bash -euo pipefail {0} jobs: - tags: - outputs: - vm-postgres-16-bullseye: ${{ steps.show-tags.outputs.vm-postgres-16-bullseye }} - daemon: ${{ steps.show-tags.outputs.daemon }} - runs-on: ubuntu-latest - steps: - - id: show-tags - run: | - echo "vm-postgres-16-bullseye=${{ env.IMG_POSTGRES_16_BULLSEYE }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - echo "daemon=${{ env.IMG_DAEMON }}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - build: # nb: use format(..) to catch both inputs.skip = true AND inputs.skip = 'true'. if: ${{ format('{0}', inputs.skip) != 'true' }} - needs: tags - runs-on: [ self-hosted, gen3, large ] - env: - IMG_DAEMON: ${{ needs.tags.outputs.daemon }} + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} + outputs: + vm-postgres-16-bullseye: ${{ steps.tags.outputs.vm-postgres-16-bullseye }} + daemon: ${{ steps.tags.outputs.daemon }} steps: + # tags converted to be a step and moved here to be in the same strategy contextt + - id: tags + run: | + echo "vm-postgres-16-bullseye=${{ env.IMG_POSTGRES_16_BULLSEYE }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT + echo "daemon=${{ env.IMG_DAEMON }}-${{inputs.arch}}:${{ inputs.tag }}" | tee -a $GITHUB_OUTPUT - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: @@ -65,15 +70,34 @@ jobs: cache: false # Sometimes setup-go gets stuck. Without this, it'll keep going until the job gets killed timeout-minutes: 10 - - run: make docker-build-daemon + - name: Build daemon image + run: make docker-build-daemon + env: + IMG_DAEMON: ${{ steps.tags.outputs.daemon }} + # - name: Build daemon image + # uses: docker/build-push-action@v6 + # env: + # GO_BASE_IMG: ${{ format('localhost:5000/neondatabase/autoscaling-go-base-{0}:dev', inputs.arch) }} + # with: + # context: . + # push: false + # platforms: linux/${{ inputs.arch }} + # file: neonvm-daemon/Dockerfile + # cache-from: type=registry,ref=cache.neon.build/neonvm-daemon:cache + # cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-daemon:cache,mode=max' || '' }} + # tags: ${{ steps.tags.outputs.daemon }} + # build-args: | + # GO_BASE_IMG=${{ env.GO_BASE_IMG }} - run: make bin/vm-builder + env: + IMG_DAEMON: ${{ steps.tags.outputs.daemon }} - name: upload vm-builder if: ${{ format('{0}', inputs.upload-vm-builder) == 'true' }} uses: actions/upload-artifact@v4 with: name: vm-builder - path: bin/vm-builder + path: ${{format('bin/vm-builder-{0}', inputs.arch)}} if-no-files-found: error retention-days: 2 @@ -86,9 +110,28 @@ jobs: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - - name: build ${{ needs.tags.outputs.vm-postgres-16-bullseye }} + - name: build ${{ steps.tags.outputs.vm-postgres-16-bullseye }} run: | - ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ needs.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ needs.tags.outputs.daemon }} -target-arch linux/${TARGET_ARCH} - - name: docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }} + ./bin/vm-builder -src postgres:16-bullseye -spec tests/e2e/image-spec.yaml -dst ${{ steps.tags.outputs.vm-postgres-16-bullseye }} -daemon-image ${{ steps.tags.outputs.daemon }} -target-arch linux/${{ inputs.arch }} + - name: docker push ${{ steps.tags.outputs.vm-postgres-16-bullseye }} run: | - docker push ${{ needs.tags.outputs.vm-postgres-16-bullseye }} + docker push ${{ steps.tags.outputs.vm-postgres-16-bullseye }} + + + save-matrix-results: + runs-on: ubuntu-latest + needs: [build] + steps: + ## Write for matrix outputs workaround + - uses: cloudposse/github-action-matrix-outputs-write@v1 + id: out + with: + matrix-step-name: ${{ inputs.matrix-step-name }} + matrix-key: ${{ inputs.matrix-key }} + outputs: |- + vm-postgres-16-bullseye: ${{ needs.build.outputs.vm-postgres-16-bullseye }} + daemon: ${{ needs.build.outputs.daemon }} + + outputs: + vm-postgres-16-bullseye: ${{ steps.out.outputs.vm-postgres-16-bullseye }} + daemon: ${{ steps.out.outputs.daemon }} diff --git a/.github/workflows/check-ca-builds.yaml b/.github/workflows/check-ca-builds.yaml index 1b8d71bbc..61de71d17 100644 --- a/.github/workflows/check-ca-builds.yaml +++ b/.github/workflows/check-ca-builds.yaml @@ -17,7 +17,11 @@ on: jobs: build-ca: - runs-on: [ self-hosted, gen3, small ] + strategy: + fail-fast: false + matrix: + arch: [ amd64, arm64 ] + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} steps: - uses: actions/checkout@v4 @@ -36,7 +40,8 @@ jobs: uses: docker/build-push-action@v6 with: context: cluster-autoscaler - platforms: linux/amd64 + platforms: ${{format('linux/{0}', matrix.arch)}} push: false + target: ${{format('cluster_autoscaler_{0}', matrix.arch)}} file: cluster-autoscaler/Dockerfile cache-from: type=registry,ref=cache.neon.build/cluster-autoscaler-neonvm:cache diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index 5c21ce559..36e377e88 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -54,6 +54,9 @@ jobs: build-images: needs: get-tag uses: ./.github/workflows/build-images.yaml + strategy: + matrix: + arch: [amd64, arm64] with: skip: ${{ inputs.tag != '' }} tag: ${{ inputs.tag || needs.get-tag.outputs.tag }} @@ -62,24 +65,59 @@ jobs: # settings and used properly in the tests. But if skip (because inputs.tag != ''), then this # setting will have no effect and the release images will be normal. controller-preserve-runner-pods: true + matrix-step-name: build-images + matrix-key: ${{ matrix.arch }} + arch: ${{ matrix.arch }} secrets: inherit + get-build-images-matrix-results: + runs-on: ubuntu-latest + needs: [build-images] + steps: + - uses: cloudposse/github-action-matrix-outputs-read@v1 + id: read + with: + matrix-step-name: build-images + outputs: + result: "${{ steps.read.outputs.result }}" + build-test-vm: needs: get-tag uses: ./.github/workflows/build-test-vm.yaml + strategy: + matrix: + arch: [amd64, arm64] with: skip: ${{ inputs.tag != '' }} tag: ${{ inputs.tag || needs.get-tag.outputs.tag }} + matrix-step-name: build-test-vm + matrix-key: ${{ matrix.arch }} + arch: ${{ matrix.arch }} secrets: inherit + get-build-test-vm-matrix-results: + runs-on: ubuntu-latest + needs: [build-test-vm] + steps: + - uses: cloudposse/github-action-matrix-outputs-read@v1 + id: read + with: + matrix-step-name: build-test-vm + outputs: + result: "${{ steps.read.outputs.result }}" + e2e-tests: - needs: [ build-images, build-test-vm ] + needs: [ get-build-images-matrix-results, get-build-test-vm-matrix-results ] strategy: fail-fast: false matrix: cluster: - ${{ inputs.cluster || 'k3d' }} - runs-on: [ self-hosted, gen3, large ] + # run tests on amd64 only, since scope of the PR is to build images, there is separate issue for e2e tests + arch: [ amd64 ] + + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} + steps: - uses: actions/checkout@v4 with: @@ -109,18 +147,18 @@ jobs: - run: make render-release env: - IMG_CONTROLLER: ${{ needs.build-images.outputs.controller }} - IMG_VXLAN_CONTROLLER: ${{ needs.build-images.outputs.vxlan-controller }} - IMG_RUNNER: ${{ needs.build-images.outputs.runner }} - IMG_SCHEDULER: ${{ needs.build-images.outputs.scheduler }} - IMG_AUTOSCALER_AGENT: ${{ needs.build-images.outputs.autoscaler-agent }} + IMG_CONTROLLER: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).controller[matrix.arch] }} + IMG_VXLAN_CONTROLLER: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).vxlan-controller[matrix.arch] }} + IMG_RUNNER: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).runner[matrix.arch]}} + IMG_SCHEDULER: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).scheduler[matrix.arch]}} + IMG_AUTOSCALER_AGENT: ${{ fromJson(needs.get-build-images-matrix-results.outputs.result).autoscaler-agent[matrix.arch]}} - name: upload manifests # nb: use format(..) to catch both inputs.push-yamls = true AND inputs.push-yamls = 'true'. - if: ${{ format('{0}', inputs.push-yamls) == 'true' }} + if: ${{ format('{0}', inputs.push-yamls) == 'true'}} uses: actions/upload-artifact@v4 with: - name: rendered_manifests + name: ${{ format('rendered_manifests-{0}', matrix.arch) }} # nb: prefix before wildcard is removed from the uploaded files, so the artifact should # contain e.g. # - autoscale-scheduler.yaml @@ -167,9 +205,9 @@ jobs: run: | rendered () { echo "rendered_manifests/$1"; } - kubectl apply -f $(rendered multus-amd64.yaml) + kubectl apply -f $(rendered multus-${{ matrix.arch}}.yaml) kubectl -n kube-system rollout status daemonset kube-multus-ds - kubectl apply -f $(rendered whereabouts-amd64.yaml) + kubectl apply -f $(rendered whereabouts-${{matrix.arch}}.yaml) kubectl -n kube-system rollout status daemonset whereabouts kubectl apply -f $(rendered neonvm-runner-image-loader.yaml) kubectl -n neonvm-system rollout status daemonset neonvm-runner-image-loader @@ -186,7 +224,7 @@ jobs: - name: load e2e test vm image env: - TEST_IMAGE: ${{ needs.build-test-vm.outputs.vm-postgres-16-bullseye }} + TEST_IMAGE: ${{ fromJson(needs.get-build-test-vm-matrix-results.outputs.result).vm-postgres-16-bullseye[matrix.arch]}} timeout-minutes: 2 run: | # Pull the docker image so we can re-tag it, because using a consistent tag inside the diff --git a/.github/workflows/vm-kernel.yaml b/.github/workflows/vm-kernel.yaml index 90d8ae3a3..ddc8de4da 100644 --- a/.github/workflows/vm-kernel.yaml +++ b/.github/workflows/vm-kernel.yaml @@ -28,13 +28,14 @@ on: type: boolean required: false default: false + arch: + description: 'Architecture to build the kernel image for' + type: string + required: true outputs: image: - description: 'vm-kernel Docker image' - value: ${{ jobs.setup-build-vm-kernel-image.outputs.image || jobs.build-vm-kernel-image.outputs.image }} - -env: - VM_KERNEL_IMAGE: "neondatabase/vm-kernel" + description: "Image" + value: ${{ jobs.build-vm-kernel-image.outputs.image }} defaults: run: @@ -43,10 +44,11 @@ defaults: jobs: setup-build-vm-kernel-image: outputs: - image: ${{ steps.get-kernel-image.outputs.image }} + image: ${{ steps.get-kernel-image.outputs.image-arm64 }} kernel-cache-tag: ${{ steps.get-kernel-cache-tag.outputs.kernel-cache-tag }} - runs-on: ubuntu-latest + env: + VM_KERNEL_IMAGE: ${{format('neondatabase/vm-kernel-{0}', inputs.arch)}} steps: - uses: actions/checkout@v4 @@ -132,7 +134,10 @@ jobs: outputs: image: ${{ steps.get-tags.outputs.canonical }}@${{ steps.build-linux-kernel.outputs.digest }} - runs-on: [ self-hosted, gen3, large ] + env: + VM_KERNEL_IMAGE: ${{format('neondatabase/vm-kernel-{0}', inputs.arch)}} + + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} steps: - name: git checkout uses: actions/checkout@v4 @@ -185,14 +190,15 @@ jobs: with: build-args: KERNEL_VERSION=${{ steps.get-kernel-version.outputs.VM_KERNEL_VERSION }} context: neonvm-kernel - platforms: linux/amd64 + platforms: ${{format('linux/{0}', inputs.arch)}} # neonvm-kernel/Dockerfile.kernel-builder has different targets for different architectures # so we need to specify the target explicitly - target: kernel_amd64 + target: ${{format('kernel_{0}', inputs.arch)}} # Push kernel image only for scheduled builds or if workflow_dispatch/workflow_call input is true push: true pull: true file: neonvm-kernel/Dockerfile.kernel-builder - cache-from: type=registry,ref=cache.neon.build/vm-kernel:cache - cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/vm-kernel:cache,mode=max' || '' }} + cache-from: ${{format('type=registry,ref=cache.neon.build/vm-kernel:{0}cache', inputs.arch)}} + # Cache the kernel image for future builds, temporary from all branches + cache-to: ${{ format('type=registry,ref=cache.neon.build/vm-kernel:{0}cache,mode=max', inputs.arch) || '' }} tags: ${{ steps.get-tags.outputs.tags }} diff --git a/cluster-autoscaler/Dockerfile b/cluster-autoscaler/Dockerfile index 0530a1ce8..9acdda310 100644 --- a/cluster-autoscaler/Dockerfile +++ b/cluster-autoscaler/Dockerfile @@ -30,7 +30,15 @@ RUN cd autoscaler/cluster-autoscaler \ # This is adapted from CA's Dockerfile.amd64, here: # https://github.com/kubernetes/autoscaler/blob/cluster-autoscaler-1.24.1/cluster-autoscaler/Dockerfile.amd64 -FROM gcr.io/distroless/static:nonroot-amd64 + +# NB: two build stages, one for each architecture, because I wasn't able to use variable substitution in FROM statements +FROM gcr.io/distroless/static:nonroot-amd64 AS cluster_autoscaler_amd64 + +WORKDIR / +COPY --from=builder /workspace/cluster-autoscaler . +CMD ["/cluster-autoscaler"] + +FROM gcr.io/distroless/static:nonroot-arm64 AS cluster_autoscaler_arm64 WORKDIR / COPY --from=builder /workspace/cluster-autoscaler .