From 79ae91acd27f5d55e6bd9d3266867d4c4152dc1e Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 6 May 2024 10:02:41 -0600 Subject: [PATCH 1/6] Update manual workflow to be compatible with bake changes --- .github/workflows/build-manual.yaml | 122 ++++++++++------------------ 1 file changed, 41 insertions(+), 81 deletions(-) diff --git a/.github/workflows/build-manual.yaml b/.github/workflows/build-manual.yaml index 89cd4429..9e254314 100644 --- a/.github/workflows/build-manual.yaml +++ b/.github/workflows/build-manual.yaml @@ -14,25 +14,15 @@ on: required: true type: choice options: + - base-images - connect - connect-content-init - - content/base - - content/pro + - content-images - package-manager - - product/base - - product/pro - r-session-complete - workbench + - workbench-for-google-cloud-workstations - workbench-for-microsoft-azure-ml - os: - description: "Which OS to build. WARNING: Not all OSes may be present for all products." - required: false - default: "ubuntu2204" - type: choice - options: - - ubuntu2204 - - ubuntu1804 - - centos7 type: description: "The type of image being built." required: false @@ -42,11 +32,6 @@ on: - preview - daily - release - use_s3_download_url: - description: "Force build to download binaries directly from S3 where applicable." - required: false - default: false - type: boolean version: description: "The version to build. Use 'auto' to target the latest build." required: false @@ -58,19 +43,43 @@ on: default: false type: boolean -name: Manual - Build, Test, Scan, and Push +name: Manual - Build, Test, and Push jobs: - build: + setup: + name: Setup runs-on: ubuntu-latest + + concurrency: + group: bake-setup-${{ github.ref }} + cancel-in-progress: true + + outputs: + GIT_SHA: ${{ steps.checkout.outputs.sha }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get commit SHA + id: get-git-sha + run: | + GIT_SHA=$(git rev-parse --short HEAD) + echo "GIT_SHA=$GIT_SHA" >> $GITHUB_OUTPUT + + build: + runs-on: ubuntu-latest-4x name: manual-build permissions: contents: read packages: write + env: + GIT_SHA: ${{ needs.setup.outputs.GIT_SHA }} + steps: - name: Check Out Repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} @@ -79,6 +88,16 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + - name: Get Version id: get-version run: | @@ -89,70 +108,11 @@ jobs: fi echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - name: Get build args - id: get-build-args - run: | - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - ARGS_CMD="" - USE_S3_DOWNLOAD_URL="" - VERSION_OVERRIDES="" - if [[ "${{inputs.os}}" == "centos7" ]]; then - # FIXME(ianpittwood): This is probably a silly way of doing this. We don't have CentOS builds for later - # Python versions so we need to override them. This is done via matrix in the release CI - # but we don't have that here. We're also limited by the 10 input cap so these can't be - # passed by the user alongside R versions as I would've liked to do. - VERSION_OVERRIDES="PYTHON_VERSION=3.9.14 PYTHON_VERSION_ALT=3.8.15" - fi - if [[ "${{ inputs.type }}" == "release" ]]; then - ARGS_CMD="get-product-args" - if [[ "${{ inputs.use_s3_download_url }}" == "true" ]]; then - USE_S3_DOWNLOAD_URL="${{ inputs.use_s3_download_url }}" - fi - else - ARGS_CMD="get-prerelease-args ${{inputs.type}}" - fi - BUILD_ARGS=$( \ - just -f ci.Justfile \ - ${VERSION_OVERRIDES} \ - ${ARGS_CMD} \ - ${{ inputs.product }} \ - ${{ inputs.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ${USE_S3_DOWNLOAD_URL} \ - ) - echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT - echo "$BUILD_ARGS" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - - - name: Get tags - id: get-tags - run: | - ARGS_CMD="" - if [[ "${{ inputs.type }}" == "release" ]]; then - ARGS_CMD="get-product-tags" - else - ARGS_CMD="get-prerelease-tags ${{inputs.type}}" - fi - IMAGE_TAGS=$( \ - just -f ci.Justfile \ - ${ARGS_CMD} \ - ${{ inputs.product }} \ - ${{ inputs.os }} \ - ${{ steps.get-version.outputs.VERSION }} \ - ) - echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - name: Build/Test/Scan/Push manual build image uses: ./.github/actions/build-test-scan-push with: - context: ./${{ inputs.product }} - os: ${{ inputs.os }} - product: ${{ inputs.product }} - image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} - build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} + target: ${{ inputs.product }} push-image: ${{ inputs.push }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.GITHUB_TOKEN }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} From 96895a18c1a3a7734f92d91b244e1457fcd29bf0 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 6 May 2024 10:13:38 -0600 Subject: [PATCH 2/6] Install Python dependencies --- .github/workflows/build-manual.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-manual.yaml b/.github/workflows/build-manual.yaml index 9e254314..43819489 100644 --- a/.github/workflows/build-manual.yaml +++ b/.github/workflows/build-manual.yaml @@ -92,11 +92,9 @@ jobs: with: python-version: '3.12' - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - id: setup-buildx - with: - buildkitd-config: ./share/buildkitd.toml + - name: Install Python dependencies + run: | + pip install requests - name: Get Version id: get-version @@ -108,6 +106,12 @@ jobs: fi echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + with: + buildkitd-config: ./share/buildkitd.toml + - name: Build/Test/Scan/Push manual build image uses: ./.github/actions/build-test-scan-push with: From baa2244a1675e28391679ff360f5f9d87f279a1e Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 6 May 2024 10:15:16 -0600 Subject: [PATCH 3/6] Revert to original cron for preview builds --- .github/workflows/build-bake-preview.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-bake-preview.yaml b/.github/workflows/build-bake-preview.yaml index aedddb10..b70cce08 100644 --- a/.github/workflows/build-bake-preview.yaml +++ b/.github/workflows/build-bake-preview.yaml @@ -1,6 +1,7 @@ on: schedule: - - cron: '0 * * * *' + - cron: '0 8 * * *' + - cron: '0 9 * * *' push: branches: - main From f663365c43c3d1e456ffc4247a96fbad9bd348c5 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 6 May 2024 10:22:41 -0600 Subject: [PATCH 4/6] Remove `base-images` from manual builds (unnecessary) --- .github/workflows/build-manual.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-manual.yaml b/.github/workflows/build-manual.yaml index 43819489..90160560 100644 --- a/.github/workflows/build-manual.yaml +++ b/.github/workflows/build-manual.yaml @@ -14,7 +14,6 @@ on: required: true type: choice options: - - base-images - connect - connect-content-init - content-images From 4d7fdfbb15fdf68a6e537af49decfe48de9618b9 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 6 May 2024 10:39:15 -0600 Subject: [PATCH 5/6] Fix action name to bake-test-push --- .github/workflows/build-manual.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-manual.yaml b/.github/workflows/build-manual.yaml index 90160560..3f3af6ac 100644 --- a/.github/workflows/build-manual.yaml +++ b/.github/workflows/build-manual.yaml @@ -111,8 +111,8 @@ jobs: with: buildkitd-config: ./share/buildkitd.toml - - name: Build/Test/Scan/Push manual build image - uses: ./.github/actions/build-test-scan-push + - name: Build/Test/Push manual build image + uses: ./.github/actions/bake-test-push with: target: ${{ inputs.product }} push-image: ${{ inputs.push }} From 7a7c4f1bcb10750a582fc2e4601faa424e315d99 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 6 May 2024 13:06:27 -0600 Subject: [PATCH 6/6] Revert cron changes --- .github/workflows/build-bake-preview.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-bake-preview.yaml b/.github/workflows/build-bake-preview.yaml index b70cce08..aedddb10 100644 --- a/.github/workflows/build-bake-preview.yaml +++ b/.github/workflows/build-bake-preview.yaml @@ -1,7 +1,6 @@ on: schedule: - - cron: '0 8 * * *' - - cron: '0 9 * * *' + - cron: '0 * * * *' push: branches: - main