Build and publish container images onto Docker Cloud/Hub #38
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # File: https://github.com/data-engineering-helpers/dpp-images/blob/main/.github/workflows/container-build-and-publish.yml | |
| # | |
| # On Docker Hub: | |
| # * https://hub.docker.com/repository/docker/infrahelpers/dpp/tags | |
| # | |
| # Dockerfiles: | |
| # * Base: https://github.com/data-engineering-helpers/dpp-images/blob/main/corretto-emr-dbs-universal-base/Dockerfile | |
| # * Usual tags on Docker Hub: infrahelpers/dpp:latest, infrahelpers/dpp:jdk8 | |
| # * Tags for other versions of the JDK: infrahelpers/dpp:jdk11, infrahelpers/dpp:jdk17 | |
| # * Python: https://github.com/data-engineering-helpers/dpp-images/tree/main/corretto-emr-dbs-universal-pyspark/Dockerfile | |
| # * Usual tags on Docker Hub: infrahelpers/dpp:jdkX-python3.Y | |
| # * Scala: https://github.com/data-engineering-helpers/dpp-images/blob/main/corretto-emr-dbs-universal-spark-scala/Dockerfile | |
| # * Usual tags on Docker Hub: infrahelpers/dpp:jdkX-sbt | |
| # | |
| # Docker Cloud builds | |
| # ------------------- | |
| # The number of build minutes are limited per month, and increasing | |
| # that limit is expensive. The activation of Docker Cloud builds is | |
| # therefore commented throughout this CI/CD pipeline. | |
| # References: | |
| # * https://docs.docker.com/build-cloud/ | |
| # * https://docs.docker.com/build-cloud/ci/ | |
| # * Cloud builders: https://app.docker.com/build/accounts/infrahelpers/builders | |
| # | |
| # SBOM and attestations of provenance | |
| # ----------------------------------- | |
| # * https://docs.docker.com/scout/policy/#supply-chain-attestations | |
| # * https://docs.docker.com/build/metadata/attestations/ | |
| # * With GitHub Actions: | |
| # https://docs.docker.com/build/ci/github-actions/attestations/ | |
| # | |
| # Scheduling builds | |
| # ----------------- | |
| # * https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows | |
| # * https://crontab.guru/#0_22_*_*_0,2,4,6 | |
| # | |
| name: Build and publish container images onto Docker Cloud/Hub | |
| on: | |
| push: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: "0 22 * * 0,2,4,6" # Trigger a build at 22:00 UTC on Sun., Tue., Thu., and Sat. | |
| workflow_dispatch: | |
| env: | |
| ORG_NAME: infrahelpers | |
| IMAGE_NAME: infrahelpers/dpp | |
| jobs: | |
| build_base_images: | |
| strategy: | |
| matrix: | |
| # https://www.java.com/releases/matrix/ | |
| jdk_version: [8, 11, 17] | |
| # https://github.com/data-engineering-helpers/dpp-images/settings/environments/1067472804/edit | |
| environment: docker-hub | |
| runs-on: ubuntu-latest | |
| steps: | |
| # https://github.com/actions/checkout | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # https://github.com/docker/setup-qemu-action | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| # https://github.com/docker/login-action | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| # https://github.com/docker/setup-buildx-action | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3 | |
| # Uncomment the following to activate the Docker Cloud builds | |
| #with: | |
| # version: "lab:latest" | |
| # driver: cloud | |
| # endpoint: "${{ env.ORG_NAME}}/default" | |
| # install: true | |
| # https://github.com/docker/metadata-action | |
| - name: Extract metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.IMAGE_NAME }} | |
| - name: Run privileged | |
| run: sudo docker run --privileged --rm tonistiigi/binfmt --install arm64 | |
| # https://github.com/docker/build-push-action | |
| - name: Build base corretto image | |
| id: docker_build_corretto_base | |
| uses: docker/build-push-action@v6 | |
| with: | |
| builder: ${{ steps.buildx.outputs.name }} | |
| context: ./corretto-emr-dbs-universal-base | |
| file: ./corretto-emr-dbs-universal-base/Dockerfile | |
| build-args: | | |
| JDK_VERSION=${{ matrix.jdk_version }} | |
| push: true | |
| provenance: mode=max | |
| sbom: true | |
| tags: | | |
| ${{ env.IMAGE_NAME }}:jdk${{ matrix.jdk_version }} | |
| ${{ env.IMAGE_NAME }}:latest | |
| # For pull requests, export results to the build cache. | |
| # Otherwise, push to a registry. | |
| # Uncomment the following 3 lines to activate the Docker Cloud builds | |
| #outputs: ${{ github.event_name == 'pull_request' && 'type=cacheonly' || 'type=registry' }} | |
| #cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:build-cache-jdk | |
| #cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:build-cache-jdk,mode=max | |
| # Comment the following 2 lines when activating the Docker Cloud builds | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| platforms: linux/amd64,linux/arm64/v8 | |
| build_scala_images: | |
| needs: build_base_images | |
| strategy: | |
| matrix: | |
| sbt_version: [1.10.2] | |
| environment: docker-hub | |
| runs-on: ubuntu-latest | |
| steps: | |
| # https://github.com/actions/checkout | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # https://github.com/docker/login-action | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| # https://github.com/docker/setup-buildx-action | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3 | |
| # Uncomment the following to activate the Docker Cloud builds | |
| #with: | |
| # version: "lab:latest" | |
| # driver: cloud | |
| # endpoint: "${{ env.ORG_NAME}}/default" | |
| # install: true | |
| # https://github.com/benjlevesque/short-sha | |
| - name: Load short SHA1 | |
| uses: benjlevesque/[email protected] | |
| id: short-sha | |
| with: | |
| length: 7 | |
| # https://github.com/docker/build-push-action | |
| - name: Build corretto with sbt installed | |
| id: docker_build_corretto_sbt | |
| uses: docker/build-push-action@v6 | |
| with: | |
| builder: ${{ steps.buildx.outputs.name }} | |
| context: ./corretto-emr-dbs-universal-spark-scala | |
| file: ./corretto-emr-dbs-universal-spark-scala/Dockerfile | |
| # As of end 2024, only Java 8 is supported for Spark | |
| build-args: | | |
| JDK_VERSION=8 | |
| SBT_VERSION=${{ matrix.sbt_version }} | |
| push: true | |
| provenance: mode=max | |
| sbom: true | |
| tags: | | |
| ${{ env.IMAGE_NAME }}:jdk8-sbt${{ matrix.sbt_version }} | |
| ${{ env.IMAGE_NAME }}:jdk8-sbt${{ matrix.sbt_version }}-${{ env.SHA }} | |
| ${{ env.IMAGE_NAME }}:jdk8-sbt | |
| # For pull requests, export results to the build cache. | |
| # Otherwise, push to a registry. | |
| # Uncomment the following 3 lines to activate the Docker Cloud builds | |
| #outputs: ${{ github.event_name == 'pull_request' && 'type=cacheonly' || 'type=registry' }} | |
| #cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:build-cache-jdk8-sbt | |
| #cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:build-cache-jdk8-sbt,mode=max | |
| # Comment the following 2 lines when activating the Docker Cloud builds | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| platforms: linux/amd64,linux/arm64/v8 | |
| build_python_images: | |
| needs: build_base_images | |
| strategy: | |
| matrix: | |
| # https://www.java.com/releases/matrix/ | |
| jdk_version: [8, 11, 17] | |
| # Use the latest releases for each minor version: | |
| # https://www.python.org/downloads/source/ | |
| python_micro_version: [3.8.20, 3.9.20, 3.10.15, 3.11.10, 3.12.7] | |
| environment: docker-hub | |
| runs-on: ubuntu-latest | |
| steps: | |
| # https://github.com/actions/checkout | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # https://github.com/docker/setup-qemu-action | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| # https://github.com/docker/login-action | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| # https://github.com/docker/setup-buildx-action | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3 | |
| # Uncomment the following to activate the Docker Cloud builds | |
| #with: | |
| # version: "lab:latest" | |
| # driver: cloud | |
| # endpoint: "${{ env.ORG_NAME}}/default" | |
| # install: true | |
| - name: Extract Minor Version | |
| id: extract_minor_version | |
| run: | | |
| minor_version=$(echo "${{ matrix.python_micro_version }}" | cut -d. -f1-2) | |
| echo "minor_version=${minor_version}" >> $GITHUB_OUTPUT | |
| # https://github.com/benjlevesque/short-sha | |
| - name: Load short sha | |
| uses: benjlevesque/[email protected] | |
| id: short-sha | |
| with: | |
| length: 7 | |
| # https://github.com/docker/build-push-action | |
| - name: Build Corretto with EMR-/Databricks-compatible Python installed | |
| id: docker_build_corretto_python | |
| uses: docker/build-push-action@v6 | |
| with: | |
| builder: ${{ steps.buildx.outputs.name }} | |
| context: ./corretto-emr-dbs-universal-pyspark | |
| file: ./corretto-emr-dbs-universal-pyspark/Dockerfile | |
| build-args: | | |
| JDK_VERSION=${{ matrix.jdk_version }} | |
| PYTHON_MINOR_VERSION=${{ steps.extract_minor_version.outputs.minor_version }} | |
| PYTHON_MICRO_VERSION=${{ matrix.python_micro_version }} | |
| push: true | |
| provenance: mode=max | |
| sbom: true | |
| tags: | | |
| ${{ env.IMAGE_NAME }}:jdk${{ matrix.jdk_version }}-python${{ steps.extract_minor_version.outputs.minor_version }} | |
| ${{ env.IMAGE_NAME }}:jdk${{ matrix.jdk_version }}-python${{ steps.extract_minor_version.outputs.minor_version }}-${{ env.SHA }} | |
| ${{ env.IMAGE_NAME }}:jdk${{ matrix.jdk_version }}-python${{ matrix.python_micro_version }} | |
| ${{ env.IMAGE_NAME }}:jdk${{ matrix.jdk_version }}-python${{ matrix.python_micro_version }}-${{ env.SHA }} | |
| ${{ env.IMAGE_NAME }}:jdk${{ matrix.jdk_version }}-python | |
| # For pull requests, export results to the build cache. | |
| # Otherwise, push to a registry. | |
| # Uncomment the following 3 lines to activate the Docker Cloud builds | |
| #outputs: ${{ github.event_name == 'pull_request' && 'type=cacheonly' || 'type=registry' }} | |
| #cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:build-cache-jdk-python | |
| #cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:build-cache-jdk-python,mode=max | |
| # Comment the following 2 lines when activating the Docker Cloud builds | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| platforms: linux/amd64,linux/arm64/v8 | |