test: docker buildx

Kuadrant · Oct 19, 2023 · c960cdd · c960cdd
1 parent a334b3e
commit c960cdd
Show file tree

Hide file tree

Showing 6 changed files with 193 additions and 44 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -2,7 +2,19 @@
 Dockerfile
 *.swp
 .dockerignore
+.gitignore
 .git
-
+.github
+doc
+.idea
+**/*.yaml
+**/*.md
+**/*.json
+**/*.sh
+**/*.png
+**/*.svg
 # OSX files
-.DS_Store
+.DS_Store
+
+# Exception - Used in docker build
+!limitador-server/examples/limits.yaml
diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml
@@ -8,7 +8,6 @@ on:
     tags: ['*']
 
 env:
-  IMG_TAGS: ${{ github.sha }}
   IMG_REGISTRY_HOST: quay.io
   IMG_REGISTRY_ORG: kuadrant
   MAIN_BRANCH_NAME: main
@@ -17,43 +16,119 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - dockerfile: Dockerfile
+            platform: linux/amd64
+          - dockerfile: Dockerfile.aarch64
+            platform: linux/arm64
     steps:
       - name: Check out code
         uses: actions/checkout@v4
-      - name: Add latest tag
-        if: ${{ github.ref_name == env.MAIN_BRANCH_NAME }}
-        run: |
-          echo "IMG_TAGS=${{ env.IMG_TAGS }} latest" >> $GITHUB_ENV
-      - name: Add branch name tag
-        if: ${{ github.ref_name != env.MAIN_BRANCH_NAME }}
-        run: |
-          echo "IMG_TAGS=${{ env.IMG_TAGS }} ${{ github.ref_name }}" >> $GITHUB_ENV
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Build Image
-        id: build-image
-        uses: redhat-actions/buildah-build@v2
+#      - name: Set up QEMU
+#        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.IMG_REGISTRY_HOST}}/${{ env.IMG_REGISTRY_ORG }}/limitador
+#          tags: |
+#            type=raw,value=${{ github.sha }}
+#            # set latest tag for main branch
+#            type=raw,value=latest,enable=${{ github.ref_name == env.MAIN_BRANCH_NAME }}
+#            # set branch name tag for non-main branches
+#            type=raw,value=${{ github.ref_name }},enable=${{ github.ref_name != env.MAIN_BRANCH_NAME }}
+      - name: Login to container registry
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.IMG_REGISTRY_USERNAME }}
+          password: ${{ secrets.IMG_REGISTRY_TOKEN }}
+          registry: ${{ env.IMG_REGISTRY_HOST }}
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v5
         with:
-          image: limitador
-          tags: ${{ env.IMG_TAGS }}
-          platforms: linux/amd64,linux/arm64
+          context: .
+#          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+#          build-args: |
+#            GITHUB_SHA=${{ github.sha }}
+          cache-from: type=gha,scope=${{ matrix.platform == 'linux/amd64' && 'build-amd' || 'build-arm'}}
+          cache-to: type=gha,mode=max,scope=${{ matrix.platform == 'linux/amd64' && 'build-amd' || 'build-arm'}}
+          outputs: type=image,name=${{ env.IMG_REGISTRY_HOST}}/${{ env.IMG_REGISTRY_ORG }}/limitador,push-by-digest=true,name-canonical=true,push=true
           dockerfiles: |
-            ./Dockerfile
-          build-args: |
-            GITHUB_SHA=${{ github.sha }}
-      - name: Smoke Test
+            ${{ matrix.dockerfile }}
+          platforms: |
+            ${{ matrix.platform }}
+#          push: true
+#      - name: Smoke Test
+#        run: |
+#          docker run --rm -t ${{ env.IMG_REGISTRY_HOST}}/${{ env.IMG_REGISTRY_ORG }}/limitador:${{ github.sha }} limitador-server --help
+      - name: Export digest
         run: |
-          podman run --rm -t ${{ steps.build-image.outputs.image }}:${{ github.sha }} limitador-server --help
-      - name: Push Image
-        if: ${{ !env.ACT }}
-        id: push-to-quay
-        uses: redhat-actions/push-to-registry@v2
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+      - name: Upload digest
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+#      - name: Push Image
+#        if: ${{ !env.ACT }}
+#        id: push-to-quay
+#        uses: redhat-actions/push-to-registry@v2
+#        with:
+#          image: ${{ steps.build-image.outputs.image }}
+#          tags: ${{ steps.build-image.outputs.tags }}
+#          registry: ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}
+#          username: ${{ secrets.IMG_REGISTRY_USERNAME }}
+#          password: ${{ secrets.IMG_REGISTRY_TOKEN }}
+#      - name: Print Image URL
+#        run: |
+#          echo "Image pushed to ${{ steps.push-to-quay.outputs.registry-paths }}"
+  merge:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.IMG_REGISTRY_HOST}}/${{ env.IMG_REGISTRY_ORG }}/limitador
+          tags: |
+            # type=raw,value=${{ github.sha }}
+            # set latest tag for main branch
+            # type=raw,value=latest,enable=${{ github.ref_name == env.MAIN_BRANCH_NAME }}
+            # set branch name tag for non-main branches
+            type=raw,value=${{ github.ref_name }},enable=${{ github.ref_name != env.MAIN_BRANCH_NAME }}
+      - name: Login to container registry
+        uses: docker/login-action@v2
         with:
-          image: ${{ steps.build-image.outputs.image }}
-          tags: ${{ steps.build-image.outputs.tags }}
-          registry: ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}
           username: ${{ secrets.IMG_REGISTRY_USERNAME }}
           password: ${{ secrets.IMG_REGISTRY_TOKEN }}
-      - name: Print Image URL
+          registry: ${{ env.IMG_REGISTRY_HOST }}
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.IMG_REGISTRY_HOST}}/${{ env.IMG_REGISTRY_ORG }}/limitador@sha256:%s ' *)
+      - name: Inspect image
         run: |
-          echo "Image pushed to ${{ steps.push-to-quay.outputs.registry-paths }}"
+          docker buildx imagetools inspect ${{ env.IMG_REGISTRY_HOST}}/${{ env.IMG_REGISTRY_ORG }}/limitador:${{ steps.meta.outputs.version }}
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -14,18 +14,29 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            limitador-testing
+          tags: |
+            type=raw,value=latest
       - name: Build Limitador Image
         id: build-image
-        uses: redhat-actions/buildah-build@v2
+        uses: docker/build-push-action@v5
         with:
-          image: limitador-testing
-          tags: latest
-          dockerfiles: |
-            ./Dockerfile
+          context: .
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          load: true
       - name: Create k8s Kind Cluster
-        uses: helm/kind-action@v1.1.0
+        uses: helm/kind-action@v1.8.0
         with:
-          version: v0.11.1
+          version: v0.20.0
           config: limitador-server/script/kind-cluster.yaml
           cluster_name: limitador-local
           wait: 120s
@@ -34,8 +45,7 @@ jobs:
           kubectl cluster-info dump
       - name: Load limitador docker image
         run: |
-          podman save -o image.tar ${{ steps.build-image.outputs.image }}:${{ steps.build-image.outputs.tags }}
-          kind load image-archive --name limitador-local image.tar
+          kind load docker-image ${{ steps.meta.outputs.tags }} -n limitador-local
       - name: Deploy limitador
         run: |
           kubectl apply -f limitador-server/e2e/file-watcher/configmap.yaml

diff --git a/Dockerfile b/Dockerfile
@@ -2,7 +2,7 @@
 # Build Stage
 # ------------------------------------------------------------------------------
 
-FROM registry.access.redhat.com/ubi8/ubi:8.7 as limitador-build
+FROM --platform=${BUILDPLATFORM} registry.access.redhat.com/ubi8/ubi:8.7 as limitador-build
 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
 
 ARG RUSTC_VERSION=1.72.0

diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64
@@ -0,0 +1,52 @@
+# ------------------------------------------------------------------------------
+# Build Stage cross compiling
+# ------------------------------------------------------------------------------
+
+FROM --platform=${BUILDPLATFORM} rust:1.72 as limitador-build
+
+RUN apt update && apt upgrade -y
+RUN apt install -y protobuf-compiler clang
+RUN apt install -y g++-aarch64-linux-gnu libc6-dev-arm64-cross
+
+RUN rustup target add aarch64-unknown-linux-gnu
+RUN rustup toolchain install stable-aarch64-unknown-linux-gnu
+
+WORKDIR /usr/src/limitador
+
+ARG GITHUB_SHA
+ENV GITHUB_SHA=${GITHUB_SHA:-unknown}
+ENV RUSTFLAGS="-C target-feature=-crt-static"
+
+COPY . .
+
+ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
+    CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \
+    CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++
+
+RUN cargo build --release --target aarch64-unknown-linux-gnu
+
+# ------------------------------------------------------------------------------
+# Run Stage
+# ------------------------------------------------------------------------------
+
+FROM --platform=linux/arm64 registry.access.redhat.com/ubi8/ubi-minimal:8.7
+
+# shadow-utils is required for `useradd`
+RUN PKGS="libgcc libstdc++ shadow-utils" \
+    && microdnf --assumeyes install --nodocs $PKGS \
+    && rpm --verify --nogroup --nouser $PKGS \
+    && microdnf -y clean all
+RUN useradd -u 1000 -s /bin/sh -m -d /home/limitador limitador
+
+WORKDIR /home/limitador/bin/
+ENV PATH="/home/limitador/bin:${PATH}"
+
+COPY --from=limitador-build /usr/src/limitador/limitador-server/examples/limits.yaml ../
+COPY --from=limitador-build /usr/src/limitador/target/aarch64-unknown-linux-gnu/release/limitador-server ./limitador-server
+
+RUN chown -R limitador:root /home/limitador \
+    && chmod -R 750 /home/limitador
+
+USER limitador
+
+CMD ["limitador-server"]
diff --git a/limitador-server/e2e/file-watcher/deployment.yaml b/limitador-server/e2e/file-watcher/deployment.yaml
@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: limitador
-          image: localhost/limitador-testing:latest
+          image: limitador-testing:latest
           imagePullPolicy: IfNotPresent
           env:
             - name: RUST_LOG