From d5ca50be6a4826be12fd5282f42872da92634cd9 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 20 Jul 2023 14:20:34 -0700 Subject: [PATCH 01/60] Unify R and Python versions --- Justfile | 14 ++++---------- ci.Justfile | 8 ++++---- connect/.env | 8 ++++---- connect/Justfile | 8 ++++---- package-manager/.env | 2 +- package-manager/Justfile | 2 +- package-manager/README.md | 2 +- package-manager/rstudio-pm.gcfg | 2 +- workbench-for-microsoft-azure-ml/Justfile | 4 ++-- workbench/.env | 4 ++-- workbench/Justfile | 4 ++-- 11 files changed, 26 insertions(+), 32 deletions(-) diff --git a/Justfile b/Justfile index af14ef32..44beff68 100644 --- a/Justfile +++ b/Justfile @@ -13,11 +13,11 @@ RSW_VERSION := "2023.06.1+524.pro1" DRIVERS_VERSION := "2023.05.0" DRIVERS_VERSION_RHEL := DRIVERS_VERSION + "-1" -R_VERSION := "3.6.2" -R_VERSION_ALT := "4.1.0" +R_VERSION := "4.2.3" +R_VERSION_ALT := "4.1.3" -PYTHON_VERSION := "3.9.5" -PYTHON_VERSION_ALT := "3.8.10" +PYTHON_VERSION := "3.9.17" +PYTHON_VERSION_ALT := "3.8.17" # just _get-tag-safe-version 2022.07.2+576.pro12 _get-tag-safe-version $VERSION: @@ -134,8 +134,6 @@ update-rsc-versions: set -euxo pipefail sed {{ sed_vars }} "s/RSC_VERSION=.*/RSC_VERSION={{ RSC_VERSION }}/g" \ connect/.env \ - connect/Dockerfile.ubuntu1804 \ - connect-content-init/Dockerfile.ubuntu1804 \ connect/Dockerfile.ubuntu2204 \ connect-content-init/Dockerfile.ubuntu2204 sed {{ sed_vars }} "s/RSC_VERSION:.*/RSC_VERSION: {{ RSC_VERSION }}/g" docker-compose.yml @@ -157,7 +155,6 @@ update-r-versions: workbench/.env \ connect/.env \ package-manager/.env \ - connect/Dockerfile.ubuntu1804 \ package-manager/Dockerfile.ubuntu1804 \ workbench/Dockerfile.ubuntu2204 \ connect/Dockerfile.ubuntu2204 \ @@ -174,7 +171,6 @@ update-r-versions: sed {{ sed_vars }} "s/^R_VERSION_ALT=.*/R_VERSION_ALT={{ R_VERSION_ALT }}/g" \ workbench/.env \ connect/.env \ - connect/Dockerfile.ubuntu1804 \ workbench/Dockerfile.ubuntu2204 \ connect/Dockerfile.ubuntu2204 sed {{ sed_vars }} "s/^R_VERSION_ALT := .*/R_VERSION_ALT := \"{{ R_VERSION_ALT }}\"/g" \ @@ -192,7 +188,6 @@ update-py-versions: sed {{ sed_vars }} "s/^PYTHON_VERSION=.*/PYTHON_VERSION={{ PYTHON_VERSION }}/g" \ workbench/Dockerfile.ubuntu2204 \ workbench/.env \ - connect/Dockerfile.ubuntu1804 \ connect/Dockerfile.ubuntu2204 \ connect/.env \ package-manager/Dockerfile.ubuntu1804 \ @@ -209,7 +204,6 @@ update-py-versions: sed {{ sed_vars }} "s/^PYTHON_VERSION_ALT=.*/PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }}/g" \ workbench/Dockerfile.ubuntu2204 \ workbench/.env \ - connect/Dockerfile.ubuntu1804 \ connect/Dockerfile.ubuntu2204 \ connect/.env sed {{ sed_vars }} "s/^PYTHON_VERSION_ALT := .*/PYTHON_VERSION_ALT := \"{{ PYTHON_VERSION_ALT }}\"/g" \ diff --git a/ci.Justfile b/ci.Justfile index 47777a9e..e0002f46 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -3,11 +3,11 @@ set positional-arguments BUILDX_PATH := "" REGISTRY_NAMESPACE := "rstudio" -R_VERSION := "3.6.2" -R_VERSION_ALT := "4.1.0" +R_VERSION := "4.2.3" +R_VERSION_ALT := "4.1.3" -PYTHON_VERSION := "3.9.5" -PYTHON_VERSION_ALT := "3.8.10" +PYTHON_VERSION := "3.9.17" +PYTHON_VERSION_ALT := "3.8.17" DRIVERS_VERSION := "2023.05.0" DRIVERS_VERSION_RHEL := DRIVERS_VERSION + "-1" diff --git a/connect/.env b/connect/.env index ab47694b..f49808f4 100644 --- a/connect/.env +++ b/connect/.env @@ -1,5 +1,5 @@ RSC_VERSION=2023.06.0 -R_VERSION=3.6.2 -R_VERSION_ALT=4.1.0 -PYTHON_VERSION=3.9.5 -PYTHON_VERSION_ALT=3.8.10 +R_VERSION=4.2.3 +R_VERSION_ALT=4.1.3 +PYTHON_VERSION=3.9.17 +PYTHON_VERSION_ALT=3.8.17 diff --git a/connect/Justfile b/connect/Justfile index a5b77cc4..7a1d9a2c 100644 --- a/connect/Justfile +++ b/connect/Justfile @@ -10,11 +10,11 @@ RSC_VERSION := "2023.06.0" RSC_LICENSE := "" RSC_LICENSE_SERVER := "" -R_VERSION := "3.6.2" -R_VERSION_ALT := "4.1.0" +R_VERSION := "4.2.3" +R_VERSION_ALT := "4.1.3" -PYTHON_VERSION := "3.9.5" -PYTHON_VERSION_ALT := "3.8.10" +PYTHON_VERSION := "3.9.17" +PYTHON_VERSION_ALT := "3.8.17" PERSIST_LICENSE := "false" PERSIST_LICENSE_DIR := join(justfile_directory(), "tmp-lic") diff --git a/package-manager/.env b/package-manager/.env index 64dc554f..05c45629 100644 --- a/package-manager/.env +++ b/package-manager/.env @@ -1,3 +1,3 @@ RSPM_VERSION=2023.04.0-6 -R_VERSION=4.2.0 +R_VERSION=4.2.3 R_VERSION_ALT=3.6.2 diff --git a/package-manager/Justfile b/package-manager/Justfile index 642ca504..9a08b6f0 100644 --- a/package-manager/Justfile +++ b/package-manager/Justfile @@ -10,7 +10,7 @@ RSPM_VERSION := "2023.04.0-6" RSPM_LICENSE := "" RSPM_LICENSE_SERVER := "" -R_VERSION := "3.6.2" +R_VERSION := "4.2.3" R_VERSION_ALT := "4.1.0" PYTHON_VERSION := "3.9.5" diff --git a/package-manager/README.md b/package-manager/README.md index fea1b0a1..5783f37c 100644 --- a/package-manager/README.md +++ b/package-manager/README.md @@ -7,7 +7,7 @@ # Supported tags and respective Dockerfile links * [`2023.04.0`, `bionic`, `ubuntu1804`, `bionic-2023.04.0`, `ubuntu1804-2023.04.0`](https://github.com/rstudio/rstudio-docker-products/blob/main/package-manager/Dockerfile.ubuntu1804) -* [`jammy`, `ubuntu2204`, `jammy-2022.11.4`, `ubuntu2204-2023.04.0`](https://github.com/rstudio/rstudio-docker-products/blob/main/package-manager/Dockerfile.ubuntu2204) +* [`jammy`, `ubuntu2204`, `jammy-2023.04.0`, `ubuntu2204-2023.04.0`](https://github.com/rstudio/rstudio-docker-products/blob/main/package-manager/Dockerfile.ubuntu2204) # What is RStudio Package Manager? diff --git a/package-manager/rstudio-pm.gcfg b/package-manager/rstudio-pm.gcfg index ae878c46..1f1c62cf 100644 --- a/package-manager/rstudio-pm.gcfg +++ b/package-manager/rstudio-pm.gcfg @@ -11,7 +11,7 @@ Address = ; ; Git sources require a configured R installation. R is often installed at `/usr/lib/R` ; or `/usr/lib64/R`. -RVersion = /opt/R/3.6.2/ +RVersion = /opt/R/4.2.3/ ; ; Customize the data directory if necessary. This is where all packages and metadata are ; stored by default. Refer to Admin Guide for details. diff --git a/workbench-for-microsoft-azure-ml/Justfile b/workbench-for-microsoft-azure-ml/Justfile index 8b8b6497..56232d54 100644 --- a/workbench-for-microsoft-azure-ml/Justfile +++ b/workbench-for-microsoft-azure-ml/Justfile @@ -12,8 +12,8 @@ RSW_LICENSE := "" R_VERSION := "4.2.3" R_VERSION_ALT := "4.1.3" -PYTHON_VERSION := "3.9.14" -PYTHON_VERSION_ALT := "3.8.15" +PYTHON_VERSION := "3.9.17" +PYTHON_VERSION_ALT := "3.8.17" _make-default-tag OS=IMAGE_OS: echo "{{IMAGE_PREFIX}}{{PRODUCT}}:{{OS}}-$(just -f ../Justfile _get-tag-safe-version {{RSW_VERSION}})" diff --git a/workbench/.env b/workbench/.env index 9ec2b95b..b89748cb 100644 --- a/workbench/.env +++ b/workbench/.env @@ -1,8 +1,8 @@ RSW_VERSION=2023.06.1+524.pro1 RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/bionic/amd64 RSW_NAME=rstudio-workbench -PYTHON_VERSION=3.9.14 -PYTHON_VERSION_ALT=3.8.15 +PYTHON_VERSION=3.9.17 +PYTHON_VERSION_ALT=3.8.17 JUPYTER_PYTHON_VERSION=3.8.15 R_VERSION_ALT=4.1.3 R_VERSION=4.2.3 diff --git a/workbench/Justfile b/workbench/Justfile index 88203fa3..4937dc9d 100644 --- a/workbench/Justfile +++ b/workbench/Justfile @@ -13,8 +13,8 @@ RSW_LICENSE_SERVER := "" R_VERSION := "4.2.3" R_VERSION_ALT := "4.1.3" -PYTHON_VERSION := "3.9.14" -PYTHON_VERSION_ALT := "3.8.15" +PYTHON_VERSION := "3.9.17" +PYTHON_VERSION_ALT := "3.8.17" PERSIST_LICENSE := "false" PERSIST_LICENSE_DIR := join(justfile_directory(), "tmp-lic") From d6ad8b92834de8afe5a72f5accc9aaf9b49c9de3 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 20 Jul 2023 14:20:55 -0700 Subject: [PATCH 02/60] Add `get-base-tags` and `get-base-args` just targets --- ci.Justfile | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/ci.Justfile b/ci.Justfile index e0002f46..a144321f 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -98,6 +98,51 @@ build-base $OS $TYPE="base" $BRANCH=`git branch --show`: ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}} \ ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} +get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: + #!/usr/bin/env bash + set -euxo pipefail + if [[ $TYPE == "base" || $TYPE == "product-base" ]]; then + SRC_IMAGE_NAME="" + CTX_PATH="./product/base" + FILE_PATH="./product/base/Dockerfile.${OS}" + elif [[ $TYPE == "base-pro" || $TYPE == "pro" || $TYPE == "product-base-pro" ]]; then + SRC_IMAGE_NAME="product-base" + CTX_PATH="./product/pro" + FILE_PATH="./product/pro/Dockerfile.${OS}" + fi + if [[ $BRANCH != "main" ]]; then + IMAGE_NAME="${IMAGE_NAME}-dev" + SRC_IMAGE_NAME="${SRC_IMAGE_NAME}-dev" + fi + + if [[ "${OS}" == "centos7" ]]; then + _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" + else + _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" + fi + echo R_VERSION="{{ R_VERSION }}" \ + R_VERSION_ALT="{{ R_VERSION_ALT }}" \ + PYTHON_VERSION="{{ PYTHON_VERSION }}" \ + PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ + DRIVERS_VERSION="${_DRIVERS_VERSION}" \ + SRC_IMAGE_NAME="${SRC_IMAGE_NAME}" + +get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: + #!/usr/bin/env bash + set -euxo pipefail + if [[ $TYPE == "base" || $TYPE == "product-base" ]]; then + IMAGE_NAME="product-base" + elif [[ $TYPE == "base-pro" || $TYPE == "pro" || $TYPE == "product-base-pro" ]]; then + IMAGE_NAME="product-base-pro" + fi + if [[ $BRANCH != "main" ]]; then + IMAGE_NAME="${IMAGE_NAME}-dev" + fi + + echo ghcr.io/rstudio/${IMAGE_NAME}:${OS} \ + ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}} \ + ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} + # just BUILDX_PATH=~/.buildx test-base ubuntu1804 base test-base $OS $TYPE="base" $BRANCH=`git branch --show`: #!/usr/bin/env bash From 7cea7fc1c20978386fb486dd31eb85be2fb95e9a Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 20 Jul 2023 14:21:19 -0700 Subject: [PATCH 03/60] Change build-preview-webhook.yaml default to "ubuntu2204" --- .github/workflows/build-preview-webhook.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-preview-webhook.yaml b/.github/workflows/build-preview-webhook.yaml index 2064fbbb..e48b3a97 100644 --- a/.github/workflows/build-preview-webhook.yaml +++ b/.github/workflows/build-preview-webhook.yaml @@ -8,7 +8,7 @@ on: os: description: "Which os to build. Default 'bionic'. Options: 'bionic|ubuntu1804', 'jammy|ubuntu2204', 'centos7'" required: true - default: "bionic" + default: "ubuntu2204" type: string type: description: "Which build type. Default 'preview'. Options: 'preview', 'daily'" From 7efbfcd8bb7f441c7bfbd8b35ee808eb8964c07e Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 20 Jul 2023 14:21:33 -0700 Subject: [PATCH 04/60] Begin prototyping generic build/test/scan/push workflow --- .github/workflows/build-release.yaml | 68 ++++++++++ .../generic-build-test-scan-push.yaml | 122 ++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 .github/workflows/build-release.yaml create mode 100644 .github/workflows/generic-build-test-scan-push.yaml diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml new file mode 100644 index 00000000..c2a1b3dc --- /dev/null +++ b/.github/workflows/build-release.yaml @@ -0,0 +1,68 @@ +on: + schedule: + - cron: '0 12 * * 1' + push: + branches: + - main + pull_request: + +name: Build, Test, Push release version +jobs: + build-base: + runs-on: ubuntu-latest + name: build-${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: ['ubuntu2204', 'centos7'] + lang: + - {r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} + - {r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} + - {r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + + steps: + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Get build args + id: get-build-args + run: | + BUILD_ARGS=$( \ + just -f ci.Justfile \ + R_VERSION=${{ matrix.lang.r-primary }} \ + R_VERSION_ALT=${{ matrix.lang.r-alternate }} \ + PYTHON_VERSION=${{ matrix.lang.py-primary }} \ + PYTHON_VERSION_ALT=${{ matrix.lang.py-alternate }} \ + get-base-args ${{ matrix.os }} base \ + ) + echo "BUILD_ARGS=$BUILD_ARGS" >> $GITHUB_OUTPUT + + - name: Get tags + id: get-tags + run: | + IMAGE_TAGS=$( \ + just -f ci.Justfile \ + R_VERSION=${{ matrix.lang.r-primary }} \ + R_VERSION_ALT=${{ matrix.lang.r-alternate }} \ + PYTHON_VERSION=${{ matrix.lang.py-primary }} \ + PYTHON_VERSION_ALT=${{ matrix.lang.py-alternate }} \ + get-base-tags ${{ matrix.lang.os }} base \ + ) + echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT + + - name: Build/Test/Scan/Push base image + uses: ./.github/workflows/generic-build-test-scan-push.yaml + with: + context: ./product/base + os: ${{ matrix.os }} + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + + + + diff --git a/.github/workflows/generic-build-test-scan-push.yaml b/.github/workflows/generic-build-test-scan-push.yaml new file mode 100644 index 00000000..1e22581d --- /dev/null +++ b/.github/workflows/generic-build-test-scan-push.yaml @@ -0,0 +1,122 @@ +on: + workflow_call: + inputs: + context: + description: Path to the directory of the Dockerfile + required: true + type: string + os: + description: Target OS to build, the same as the extension of the Dockerfile + default: ubuntu2204 + type: string + image-tags: + description: List of tags for the built image + required: true + type: string + test-image: + description: Flag to test image once built + default: true + type: boolean + scan-image: + description: Flag to scan image using Snyk + default: true + type: boolean + push-image: + description: Flag to push image once built + default: true + type: boolean + use-ghcr: + description: Flag to authenticate with GHCR.io + default: true + type: boolean + use-dockerhub: + description: Flag to authenticate with Dockerhub + default: true + type: boolean + secrets: inherit + + +jobs: + build-test-push: + runs-on: ubuntu-latest + steps: + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to ghcr.io + if: ${{ inputs.use-ghcr }} + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.BUILD_PAT }} + + - name: Login to Docker Hub + if: ${{ inputs.use-dockerhub }} + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Build + id: image-build + uses: docker/build-push-action@v4 + with: + load: true + context: ${{ inputs.context }} + file: Dockerfile.${{ inputs.os }} + cache-from: type=gha + cache-to: type=gha + build-args: ${{ inputs.build-args }} + tags: ${{ inputs.image-tags }} + + - name: Get first tag + id: first-tag + run: | + echo "FIRST_TAG=$(cut -d " " -f 1 <<< $IMAGE_TAGS)" >> $GITHUB_OUTPUT + + - name: Test + env: + RSC_LICENSE: ${{ secrets.RSC_LICENSE }} + RSPM_LICENSE: ${{ secrets.RSPM_LICENSE }} + RSW_LICENSE: ${{ secrets.RSW_LICENSE }} + if: ${{ inputs.test-image }} + run: | + ${{ inputs.build-args }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut + + - name: Run Snyk to check Docker image for vulnerabilities (main) + if: ${{ github.ref == 'refs/heads/main' }} && ${{ inputs.scan-image }} + continue-on-error: true + uses: snyk/actions/docker@master + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + image: ${{ steps.get-default-tag.outputs.DEFAULT_TAG }} + args: --file=./${{ matrix.config.product }}/Dockerfile.${{ matrix.config.os }} --org=${{ secrets.SNYK_ORG_ID }} --project-name=${{ steps.get-default-tag.outputs.DEFAULT_TAG }} --tags=product=${{ matrix.config.product }},os=${{ matrix.config.os }} --exclude-base-image-vulns --app-vulns + command: monitor + + - name: Run Snyk to check Docker image for vulnerabilities (branch) + if: ${{ github.ref != 'refs/heads/main' }} && ${{ inputs.scan-image }} + continue-on-error: true + uses: snyk/actions/docker@master + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + image: ${{ steps.get-default-tag.outputs.DEFAULT_TAG }} + args: --file=./${{ matrix.config.product }}/Dockerfile.${{ matrix.config.os }} --org=${{ secrets.SNYK_ORG_ID }} --project-name=${{ steps.get-default-tag.outputs.DEFAULT_TAG }} --tags=product=${{ matrix.config.product }},os=${{ matrix.config.os }} --exclude-base-image-vulns --app-vulns + command: test + + - name: Build + uses: docker/build-push-action@v4 + if: ${{ inputs.push-image }} + with: + push: true + context: ${{ inputs.context }} + file: Dockerfile.${{ inputs.os }} + cache-from: type=gha + cache-to: type=gha + build-args: ${{ inputs.build-args }} + tags: ${{ inputs.image-tags }} From e229c8d1108dd5aa48e708878f7bbb3a7d083209 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 20 Jul 2023 14:23:27 -0700 Subject: [PATCH 05/60] Fix just unbound variable --- ci.Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.Justfile b/ci.Justfile index a144321f..1a917a15 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -111,7 +111,6 @@ get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: FILE_PATH="./product/pro/Dockerfile.${OS}" fi if [[ $BRANCH != "main" ]]; then - IMAGE_NAME="${IMAGE_NAME}-dev" SRC_IMAGE_NAME="${SRC_IMAGE_NAME}-dev" fi @@ -130,6 +129,7 @@ get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: #!/usr/bin/env bash set -euxo pipefail + IMAGE_NAME="" if [[ $TYPE == "base" || $TYPE == "product-base" ]]; then IMAGE_NAME="product-base" elif [[ $TYPE == "base-pro" || $TYPE == "pro" || $TYPE == "product-base-pro" ]]; then From 2056c66a31b411a37e7449fd311897a002c9924f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 20 Jul 2023 15:01:49 -0700 Subject: [PATCH 06/60] Switch to composite action --- .../actions/build-test-scan-push/action.yaml | 145 ++++++++++++++++++ .github/workflows/build-release.yaml | 8 +- .../generic-build-test-scan-push.yaml | 122 --------------- 3 files changed, 152 insertions(+), 123 deletions(-) create mode 100644 .github/actions/build-test-scan-push/action.yaml delete mode 100644 .github/workflows/generic-build-test-scan-push.yaml diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml new file mode 100644 index 00000000..5b9db07a --- /dev/null +++ b/.github/actions/build-test-scan-push/action.yaml @@ -0,0 +1,145 @@ +name: 'Build/Test/Scan/Push Image' +inputs: + context: + description: Path to the directory of the Dockerfile + required: true + type: string + os: + description: Target OS to build, the same as the extension of the Dockerfile + default: ubuntu2204 + type: string + product: + description: Product being built + type: string + image-tags: + description: List of tags for the built image + required: true + type: string + test-image: + description: Flag to test image once built + default: true + type: boolean + scan-image: + description: Flag to scan image using Snyk + default: true + type: boolean + snyk-token: + description: Username for authentication with Snyk for scanning images + type: string + snyk-org-id: + description: Snyk Organization ID to publish scans to + type: string + push-image: + description: Flag to push image once built + default: true + type: boolean + use-ghcr: + description: Flag to authenticate with GHCR.io + default: true + type: boolean + ghcr-token: + description: Username for authentication with GHCR.io + type: string + use-dockerhub: + description: Flag to authenticate with Dockerhub + default: true + type: boolean + dockerhub-username: + description: Username for authentication with DockerHub + type: string + dockerhub-token: + description: Username for authentication with DockerHub + type: string + +runs: + using: "composite" + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to ghcr.io + if: ${{ inputs.use-ghcr }} + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.ghcr-token }} + + - name: Login to Docker Hub + if: ${{ inputs.use-dockerhub }} + uses: docker/login-action@v2 + with: + username: ${{ inputs.dockerhub-username }} + password: ${{ inputs.dockerhub-token }} + + - name: Build + id: image-build + uses: docker/build-push-action@v4 + with: + load: true + context: ${{ inputs.context }} + file: ${{ inputs.context }}/Dockerfile.${{ inputs.os }} + cache-from: type=gha + cache-to: type=gha + build-args: ${{ inputs.build-args }} + tags: ${{ inputs.image-tags }} + + - name: Get first tag + id: first-tag + run: | + echo "FIRST_TAG=$(cut -d " " -f 1 <<< $IMAGE_TAGS)" >> $GITHUB_OUTPUT + + - name: Test + env: + RSC_LICENSE: ${{ secrets.RSC_LICENSE }} + RSPM_LICENSE: ${{ secrets.RSPM_LICENSE }} + RSW_LICENSE: ${{ secrets.RSW_LICENSE }} + if: ${{ inputs.test-image }} + run: | + ${{ inputs.build-args }} IMAGE_NAME=${{ steps.first-tag.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut + + - name: Run Snyk to check Docker image for vulnerabilities (main) + if: ${{ github.ref == 'refs/heads/main' }} && ${{ inputs.scan-image }} + continue-on-error: true + uses: snyk/actions/docker@master + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + image: ${{ steps.first-tag.FIRST_TAG }} + args: | + --file=${{ inputs.context }}/Dockerfile.${{ inputs.os }} \ + --org=${{ inputs.snyk-org-id }} \ + --project-name=${{ steps.first-tag.FIRST_TAG }} \ + --tags=product=${{ inputs.product }},os=${{ inputs.os }} \ + --exclude-base-image-vulns \ + --app-vulns + command: monitor + + - name: Run Snyk to check Docker image for vulnerabilities (branch) + if: ${{ github.ref != 'refs/heads/main' }} && ${{ inputs.scan-image }} + continue-on-error: true + uses: snyk/actions/docker@master + env: + SNYK_TOKEN: ${{ inputs.snyk-token }} + with: + image: ${{ steps.first-tag.FIRST_TAG }} + args: | + --file=${{ inputs.context }}/Dockerfile.${{ inputs.os }} \ + --org=${{ secrets.SNYK_ORG_ID }} \ + --project-name=${{ steps.get-default-tag.outputs.DEFAULT_TAG }} \ + --tags=product=${{ inputs.product }},os=${{ inputs.os }} \ + --exclude-base-image-vulns \ + --app-vulns + command: test + + - name: Build + uses: docker/build-push-action@v4 + if: ${{ inputs.push-image }} + with: + push: true + context: ${{ inputs.context }} + file: ${{ inputs.context }}/Dockerfile.${{ inputs.os }} + cache-from: type=gha + cache-to: type=gha + build-args: ${{ inputs.build-args }} + tags: ${{ inputs.image-tags }} diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index c2a1b3dc..86da3b8c 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -57,11 +57,17 @@ jobs: echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - name: Build/Test/Scan/Push base image - uses: ./.github/workflows/generic-build-test-scan-push.yaml + uses: ./.github/actions/build-test-scan-push with: context: ./product/base os: ${{ matrix.os }} + product: base image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} diff --git a/.github/workflows/generic-build-test-scan-push.yaml b/.github/workflows/generic-build-test-scan-push.yaml deleted file mode 100644 index 1e22581d..00000000 --- a/.github/workflows/generic-build-test-scan-push.yaml +++ /dev/null @@ -1,122 +0,0 @@ -on: - workflow_call: - inputs: - context: - description: Path to the directory of the Dockerfile - required: true - type: string - os: - description: Target OS to build, the same as the extension of the Dockerfile - default: ubuntu2204 - type: string - image-tags: - description: List of tags for the built image - required: true - type: string - test-image: - description: Flag to test image once built - default: true - type: boolean - scan-image: - description: Flag to scan image using Snyk - default: true - type: boolean - push-image: - description: Flag to push image once built - default: true - type: boolean - use-ghcr: - description: Flag to authenticate with GHCR.io - default: true - type: boolean - use-dockerhub: - description: Flag to authenticate with Dockerhub - default: true - type: boolean - secrets: inherit - - -jobs: - build-test-push: - runs-on: ubuntu-latest - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to ghcr.io - if: ${{ inputs.use-ghcr }} - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.BUILD_PAT }} - - - name: Login to Docker Hub - if: ${{ inputs.use-dockerhub }} - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Build - id: image-build - uses: docker/build-push-action@v4 - with: - load: true - context: ${{ inputs.context }} - file: Dockerfile.${{ inputs.os }} - cache-from: type=gha - cache-to: type=gha - build-args: ${{ inputs.build-args }} - tags: ${{ inputs.image-tags }} - - - name: Get first tag - id: first-tag - run: | - echo "FIRST_TAG=$(cut -d " " -f 1 <<< $IMAGE_TAGS)" >> $GITHUB_OUTPUT - - - name: Test - env: - RSC_LICENSE: ${{ secrets.RSC_LICENSE }} - RSPM_LICENSE: ${{ secrets.RSPM_LICENSE }} - RSW_LICENSE: ${{ secrets.RSW_LICENSE }} - if: ${{ inputs.test-image }} - run: | - ${{ inputs.build-args }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut - - - name: Run Snyk to check Docker image for vulnerabilities (main) - if: ${{ github.ref == 'refs/heads/main' }} && ${{ inputs.scan-image }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: ${{ steps.get-default-tag.outputs.DEFAULT_TAG }} - args: --file=./${{ matrix.config.product }}/Dockerfile.${{ matrix.config.os }} --org=${{ secrets.SNYK_ORG_ID }} --project-name=${{ steps.get-default-tag.outputs.DEFAULT_TAG }} --tags=product=${{ matrix.config.product }},os=${{ matrix.config.os }} --exclude-base-image-vulns --app-vulns - command: monitor - - - name: Run Snyk to check Docker image for vulnerabilities (branch) - if: ${{ github.ref != 'refs/heads/main' }} && ${{ inputs.scan-image }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: ${{ steps.get-default-tag.outputs.DEFAULT_TAG }} - args: --file=./${{ matrix.config.product }}/Dockerfile.${{ matrix.config.os }} --org=${{ secrets.SNYK_ORG_ID }} --project-name=${{ steps.get-default-tag.outputs.DEFAULT_TAG }} --tags=product=${{ matrix.config.product }},os=${{ matrix.config.os }} --exclude-base-image-vulns --app-vulns - command: test - - - name: Build - uses: docker/build-push-action@v4 - if: ${{ inputs.push-image }} - with: - push: true - context: ${{ inputs.context }} - file: Dockerfile.${{ inputs.os }} - cache-from: type=gha - cache-to: type=gha - build-args: ${{ inputs.build-args }} - tags: ${{ inputs.image-tags }} From 140a38c3a4b5b0abbc250dd6cd02aa18cca7bb02 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 20 Jul 2023 15:04:11 -0700 Subject: [PATCH 07/60] Specify shell to bash --- .github/actions/build-test-scan-push/action.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 5b9db07a..c9de2427 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -85,15 +85,13 @@ runs: tags: ${{ inputs.image-tags }} - name: Get first tag + shell: bash id: first-tag run: | echo "FIRST_TAG=$(cut -d " " -f 1 <<< $IMAGE_TAGS)" >> $GITHUB_OUTPUT - name: Test - env: - RSC_LICENSE: ${{ secrets.RSC_LICENSE }} - RSPM_LICENSE: ${{ secrets.RSPM_LICENSE }} - RSW_LICENSE: ${{ secrets.RSW_LICENSE }} + shell: bash if: ${{ inputs.test-image }} run: | ${{ inputs.build-args }} IMAGE_NAME=${{ steps.first-tag.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut @@ -103,7 +101,7 @@ runs: continue-on-error: true uses: snyk/actions/docker@master env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + SNYK_TOKEN: ${{ inputs.snyk-token }} with: image: ${{ steps.first-tag.FIRST_TAG }} args: | @@ -125,7 +123,7 @@ runs: image: ${{ steps.first-tag.FIRST_TAG }} args: | --file=${{ inputs.context }}/Dockerfile.${{ inputs.os }} \ - --org=${{ secrets.SNYK_ORG_ID }} \ + --org=${{ inputs.snyk-org-id }} \ --project-name=${{ steps.get-default-tag.outputs.DEFAULT_TAG }} \ --tags=product=${{ inputs.product }},os=${{ inputs.os }} \ --exclude-base-image-vulns \ From 42272e841bbb40013f518e1338f46fc7b120625d Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 05:29:56 -0700 Subject: [PATCH 08/60] Use CSV list for tags --- ci.Justfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci.Justfile b/ci.Justfile index 1a917a15..41d8be1b 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -139,8 +139,8 @@ get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: IMAGE_NAME="${IMAGE_NAME}-dev" fi - echo ghcr.io/rstudio/${IMAGE_NAME}:${OS} \ - ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}} \ + echo ghcr.io/rstudio/${IMAGE_NAME}:${OS},\ + ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}},\ ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} # just BUILDX_PATH=~/.buildx test-base ubuntu1804 base From 7ebf3edf20dff45202f2d059722ff2942a710cb3 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 05:42:57 -0700 Subject: [PATCH 09/60] Use printf to get line sep build args --- .github/actions/build-test-scan-push/action.yaml | 3 ++- .github/workflows/build-release.yaml | 1 + ci.Justfile | 12 ++++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index c9de2427..6122a0ce 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -81,7 +81,8 @@ runs: file: ${{ inputs.context }}/Dockerfile.${{ inputs.os }} cache-from: type=gha cache-to: type=gha - build-args: ${{ inputs.build-args }} + build-args: | + ${{ inputs.build-args }} tags: ${{ inputs.image-tags }} - name: Get first tag diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 86da3b8c..c0096f2d 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -63,6 +63,7 @@ jobs: os: ${{ matrix.os }} product: base image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + push-image: ${{ github.ref == 'refs/heads/main' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} diff --git a/ci.Justfile b/ci.Justfile index 41d8be1b..3666aa62 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -119,12 +119,12 @@ get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: else _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" fi - echo R_VERSION="{{ R_VERSION }}" \ - R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - DRIVERS_VERSION="${_DRIVERS_VERSION}" \ - SRC_IMAGE_NAME="${SRC_IMAGE_NAME}" + printf "R_VERSION={{ R_VERSION }} + R_VERSION_ALT={{ R_VERSION_ALT }} + PYTHON_VERSION={{ PYTHON_VERSION }} + PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }} + DRIVERS_VERSION=${_DRIVERS_VERSION} + SRC_IMAGE_NAME=${SRC_IMAGE_NAME}" get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: #!/usr/bin/env bash From e21d99eafc7ea9b04a0aa9e27b32b7f32680dc20 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 05:56:06 -0700 Subject: [PATCH 10/60] Debug echo build args in action --- .github/actions/build-test-scan-push/action.yaml | 4 ++++ ci.Justfile | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 6122a0ce..814f7322 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -72,6 +72,10 @@ runs: username: ${{ inputs.dockerhub-username }} password: ${{ inputs.dockerhub-token }} + - name: echo args + shell: bash + run: echo ${{ inputs.build-args }} + - name: Build id: image-build uses: docker/build-push-action@v4 diff --git a/ci.Justfile b/ci.Justfile index 3666aa62..0023fa0d 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -140,8 +140,8 @@ get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: fi echo ghcr.io/rstudio/${IMAGE_NAME}:${OS},\ - ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}},\ - ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} + ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}},\ + ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} # just BUILDX_PATH=~/.buildx test-base ubuntu1804 base test-base $OS $TYPE="base" $BRANCH=`git branch --show`: From 346801f9be95c75a8b29ec052d8f912082674cbc Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 06:03:50 -0700 Subject: [PATCH 11/60] Feed build args as JSON to action --- .github/actions/build-test-scan-push/action.yaml | 2 +- ci.Justfile | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 814f7322..bba1f47b 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -86,7 +86,7 @@ runs: cache-from: type=gha cache-to: type=gha build-args: | - ${{ inputs.build-args }} + ${{ join(fromJSON(inputs.build-args), '\n') }} tags: ${{ inputs.image-tags }} - name: Get first tag diff --git a/ci.Justfile b/ci.Justfile index 0023fa0d..b518c1a7 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -119,12 +119,12 @@ get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: else _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" fi - printf "R_VERSION={{ R_VERSION }} - R_VERSION_ALT={{ R_VERSION_ALT }} - PYTHON_VERSION={{ PYTHON_VERSION }} - PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }} - DRIVERS_VERSION=${_DRIVERS_VERSION} - SRC_IMAGE_NAME=${SRC_IMAGE_NAME}" + echo "[\"R_VERSION={{ R_VERSION }}\",\ + \"R_VERSION_ALT={{ R_VERSION_ALT }}\",\ + \"PYTHON_VERSION={{ PYTHON_VERSION }}\",\ + \"PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }}\",\ + \"DRIVERS_VERSION=${_DRIVERS_VERSION}\",\ + \"SRC_IMAGE_NAME=${SRC_IMAGE_NAME}\"]" get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: #!/usr/bin/env bash From 438f8f61465c898dc35e041bcece6dd40a18d202 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 08:15:25 -0700 Subject: [PATCH 12/60] Add build-args input option --- .github/actions/build-test-scan-push/action.yaml | 4 ++++ .github/workflows/build-release.yaml | 1 + 2 files changed, 5 insertions(+) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index bba1f47b..f02c8035 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -11,6 +11,10 @@ inputs: product: description: Product being built type: string + build-args: + description: JSON list of build args for the built image + required: true + type: string image-tags: description: List of tags for the built image required: true diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index c0096f2d..ebfa4eb4 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -63,6 +63,7 @@ jobs: os: ${{ matrix.os }} product: base image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} push-image: ${{ github.ref == 'refs/heads/main' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} From 6688b60b3294738fb9f3ab4b5b39b3f2004f3d76 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 08:21:07 -0700 Subject: [PATCH 13/60] Print and pass build args as block --- .github/actions/build-test-scan-push/action.yaml | 2 +- ci.Justfile | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index f02c8035..57510009 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -90,7 +90,7 @@ runs: cache-from: type=gha cache-to: type=gha build-args: | - ${{ join(fromJSON(inputs.build-args), '\n') }} + ${{ inputs.build-args }} tags: ${{ inputs.image-tags }} - name: Get first tag diff --git a/ci.Justfile b/ci.Justfile index b518c1a7..0023fa0d 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -119,12 +119,12 @@ get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: else _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" fi - echo "[\"R_VERSION={{ R_VERSION }}\",\ - \"R_VERSION_ALT={{ R_VERSION_ALT }}\",\ - \"PYTHON_VERSION={{ PYTHON_VERSION }}\",\ - \"PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }}\",\ - \"DRIVERS_VERSION=${_DRIVERS_VERSION}\",\ - \"SRC_IMAGE_NAME=${SRC_IMAGE_NAME}\"]" + printf "R_VERSION={{ R_VERSION }} + R_VERSION_ALT={{ R_VERSION_ALT }} + PYTHON_VERSION={{ PYTHON_VERSION }} + PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }} + DRIVERS_VERSION=${_DRIVERS_VERSION} + SRC_IMAGE_NAME=${SRC_IMAGE_NAME}" get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: #!/usr/bin/env bash From d58cdc3af043a4a6dfe47830f073ce953eb5425f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 08:28:03 -0700 Subject: [PATCH 14/60] Quote block string --- .github/workflows/build-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index ebfa4eb4..f603a6d1 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -41,7 +41,7 @@ jobs: PYTHON_VERSION_ALT=${{ matrix.lang.py-alternate }} \ get-base-args ${{ matrix.os }} base \ ) - echo "BUILD_ARGS=$BUILD_ARGS" >> $GITHUB_OUTPUT + echo "BUILD_ARGS=\"${BUILD_ARGS}\"" >> $GITHUB_OUTPUT - name: Get tags id: get-tags From 2095bb4ae6d30bf6d5cbff4e8413ad90d3b8c0bd Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 08:34:46 -0700 Subject: [PATCH 15/60] Debug build-args pass and fix outputs var --- .github/actions/build-test-scan-push/action.yaml | 2 +- .github/workflows/build-release.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 57510009..8c687ad1 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -103,7 +103,7 @@ runs: shell: bash if: ${{ inputs.test-image }} run: | - ${{ inputs.build-args }} IMAGE_NAME=${{ steps.first-tag.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut + ${{ inputs.build-args }} IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut - name: Run Snyk to check Docker image for vulnerabilities (main) if: ${{ github.ref == 'refs/heads/main' }} && ${{ inputs.scan-image }} diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index f603a6d1..1e97c254 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -43,6 +43,10 @@ jobs: ) echo "BUILD_ARGS=\"${BUILD_ARGS}\"" >> $GITHUB_OUTPUT + - name: echo args + shell: bash + run: echo "${{ steps.get-build-args.outputs.BUILD_ARGS }}" + - name: Get tags id: get-tags run: | From f332d8f57b3e4ad262672c258a9c81899e617f24 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 08:38:52 -0700 Subject: [PATCH 16/60] Try multiline string using delimiters --- .github/workflows/build-release.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 1e97c254..f82ef971 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -33,6 +33,7 @@ jobs: - name: Get build args id: get-build-args run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) BUILD_ARGS=$( \ just -f ci.Justfile \ R_VERSION=${{ matrix.lang.r-primary }} \ @@ -41,7 +42,9 @@ jobs: PYTHON_VERSION_ALT=${{ matrix.lang.py-alternate }} \ get-base-args ${{ matrix.os }} base \ ) - echo "BUILD_ARGS=\"${BUILD_ARGS}\"" >> $GITHUB_OUTPUT + echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT + echo "$BUILD_ARGS" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT - name: echo args shell: bash From 1b11cb3532e46239244f3fcee4186d3cc10b335a Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 08:53:58 -0700 Subject: [PATCH 17/60] Fix build args list for tests and fix first tag cut --- .github/actions/build-test-scan-push/action.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 8c687ad1..1d9d4278 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -97,13 +97,15 @@ runs: shell: bash id: first-tag run: | - echo "FIRST_TAG=$(cut -d " " -f 1 <<< $IMAGE_TAGS)" >> $GITHUB_OUTPUT + FIRST_TAG=$(cut -d "," -f 1 <<< $IMAGE_TAGS) + echo "FIRST_TAG=$FIRST_TAG" >> $GITHUB_OUTPUT - name: Test shell: bash if: ${{ inputs.test-image }} run: | - ${{ inputs.build-args }} IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut + BUILD_ARGS_CLEAN=$(tr '\n' ' ' <<< ${{ inputs.build-args }}) + ${BUILD_ARGS_CLEAN} IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut - name: Run Snyk to check Docker image for vulnerabilities (main) if: ${{ github.ref == 'refs/heads/main' }} && ${{ inputs.scan-image }} From 67913579166e9065fc9fa1eb7a464e95b476f4ca Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 09:01:05 -0700 Subject: [PATCH 18/60] Quote wrap build args input --- .github/actions/build-test-scan-push/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 1d9d4278..d2ac08a8 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -104,7 +104,7 @@ runs: shell: bash if: ${{ inputs.test-image }} run: | - BUILD_ARGS_CLEAN=$(tr '\n' ' ' <<< ${{ inputs.build-args }}) + BUILD_ARGS_CLEAN=$(tr '\n' ' ' <<< "${{ inputs.build-args }}") ${BUILD_ARGS_CLEAN} IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut - name: Run Snyk to check Docker image for vulnerabilities (main) From 0cff240ec2d7d4fa578d8c254390e1a5c9791110 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 09:06:05 -0700 Subject: [PATCH 19/60] Echo variables for debugging --- .github/actions/build-test-scan-push/action.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index d2ac08a8..3a5a7bd7 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -97,7 +97,8 @@ runs: shell: bash id: first-tag run: | - FIRST_TAG=$(cut -d "," -f 1 <<< $IMAGE_TAGS) + FIRST_TAG=$(cut -d "," -f 1 <<< "${{ inputs.image-tags }}") + echo "$FIRST_TAG" echo "FIRST_TAG=$FIRST_TAG" >> $GITHUB_OUTPUT - name: Test @@ -105,6 +106,7 @@ runs: if: ${{ inputs.test-image }} run: | BUILD_ARGS_CLEAN=$(tr '\n' ' ' <<< "${{ inputs.build-args }}") + echo "$BUILD_ARGS_CLEAN" ${BUILD_ARGS_CLEAN} IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut - name: Run Snyk to check Docker image for vulnerabilities (main) From 856a16f7e2efd06a5372af447c10940d6ae64572 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 09:13:40 -0700 Subject: [PATCH 20/60] Fix tagging OS variable --- .github/actions/build-test-scan-push/action.yaml | 4 +--- .github/workflows/build-release.yaml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 3a5a7bd7..d690aca1 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -105,9 +105,7 @@ runs: shell: bash if: ${{ inputs.test-image }} run: | - BUILD_ARGS_CLEAN=$(tr '\n' ' ' <<< "${{ inputs.build-args }}") - echo "$BUILD_ARGS_CLEAN" - ${BUILD_ARGS_CLEAN} IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut + ${{ inputs.build-args }} IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut - name: Run Snyk to check Docker image for vulnerabilities (main) if: ${{ github.ref == 'refs/heads/main' }} && ${{ inputs.scan-image }} diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index f82ef971..e4db899e 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -59,7 +59,7 @@ jobs: R_VERSION_ALT=${{ matrix.lang.r-alternate }} \ PYTHON_VERSION=${{ matrix.lang.py-primary }} \ PYTHON_VERSION_ALT=${{ matrix.lang.py-alternate }} \ - get-base-tags ${{ matrix.lang.os }} base \ + get-base-tags ${{ matrix.os }} base \ ) echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT From 591e30555ed697208468446947109761fa1b5ead Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 09:39:18 -0700 Subject: [PATCH 21/60] Pass OS to test command --- .github/actions/build-test-scan-push/action.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index d690aca1..735f85ee 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -76,10 +76,6 @@ runs: username: ${{ inputs.dockerhub-username }} password: ${{ inputs.dockerhub-token }} - - name: echo args - shell: bash - run: echo ${{ inputs.build-args }} - - name: Build id: image-build uses: docker/build-push-action@v4 @@ -105,7 +101,7 @@ runs: shell: bash if: ${{ inputs.test-image }} run: | - ${{ inputs.build-args }} IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut + ${{ inputs.build-args }} OS=${{ inputs.os }} IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut - name: Run Snyk to check Docker image for vulnerabilities (main) if: ${{ github.ref == 'refs/heads/main' }} && ${{ inputs.scan-image }} From 3c05c99155b05e87ed2508e564ca26dff7285c95 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 11:09:08 -0700 Subject: [PATCH 22/60] Use .env file for tests --- .github/actions/build-test-scan-push/action.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 735f85ee..9f3c2531 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -101,7 +101,10 @@ runs: shell: bash if: ${{ inputs.test-image }} run: | - ${{ inputs.build-args }} OS=${{ inputs.os }} IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut + echo "${{ inputs.build-args }}" > ${{ inputs.context }}/.env + echo "OS=${{ inputs.os }}" >> ${{ inputs.context }}/.env + cat ${{ inputs.context }}/.env + IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut - name: Run Snyk to check Docker image for vulnerabilities (main) if: ${{ github.ref == 'refs/heads/main' }} && ${{ inputs.scan-image }} From e991dc96670f4e55baee23ff0a8d4c8b866f9b0f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 11:30:51 -0700 Subject: [PATCH 23/60] Add `update-quarto-versions` target - Update all Quarto to 1.3.340 - Pass Quarto down as build arg --- Justfile | 22 +++++++++++++++++++++- ci.Justfile | 3 +++ content/base/Justfile | 2 +- content/base/maybe_install_quarto.sh | 2 +- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/Justfile b/Justfile index 44beff68..98aaf520 100644 --- a/Justfile +++ b/Justfile @@ -19,6 +19,8 @@ R_VERSION_ALT := "4.1.3" PYTHON_VERSION := "3.9.17" PYTHON_VERSION_ALT := "3.8.17" +QUARTO_VERSION := "1.3.340" + # just _get-tag-safe-version 2022.07.2+576.pro12 _get-tag-safe-version $VERSION: #!/usr/bin/env bash @@ -88,7 +90,8 @@ update-versions: PYTHON_VERSION={{PYTHON_VERSION}} \ PYTHON_VERSION_ALT={{PYTHON_VERSION_ALT}} \ DRIVERS_VERSION={{DRIVERS_VERSION}} \ - update-rsw-versions update-rspm-versions update-rsc-versions update-r-versions update-py-versions update-drivers-versions + QUARTO_VERSION={{QUARTO_VERSION}} \ + update-rsw-versions update-rspm-versions update-rsc-versions update-r-versions update-py-versions update-drivers-versions update-quarto-versions # just RSW_VERSION=1.2.3 update-rsw-versions update-rsw-versions: @@ -233,6 +236,23 @@ update-drivers-versions: product/pro/Justfile \ ci.Justfile +update-quarto-versions: + #!/usr/bin/env bash + set -euxo pipefail + sed {{ sed_vars }} "s/^QUARTO_VERSION := .*/QUARTO_VERSION := \"{{ QUARTO_VERSION }}\"/g" \ + content/base/Justfile \ + product/base/Justfile \ + ci.Justfile \ + Justfile + sed {{ sed_vars }} "s/^QUARTO_VERSION=.*/QUARTO_VERSION={{ QUARTO_VERSION }}/g" \ + content/base/Dockerfile* \ + product/base/Dockerfile* + sed {{ sed_vars }} "s/^Executable = \/opt\/quarto\/.*\/bin\/quarto/Executable = \/opt\/quarto\/{{ QUARTO_VERSION }}\/bin\/quarto/g" \ + connect/rstudio-connect.gcfg + sed {{ sed_vars }} "s/qver=\${QUARTO_VERSION:-.*}/qver=\${QUARTO_VERSION:-{{ QUARTO_VERSION }}}/g" \ + content/base/maybe_install_quarto.sh + + # just test-image preview workbench 12.0.11-8 tag1 tag2 tag3 ... test-image $PRODUCT $VERSION +IMAGES: #!/usr/bin/env bash diff --git a/ci.Justfile b/ci.Justfile index 0023fa0d..6b5f7d3f 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -12,6 +12,8 @@ PYTHON_VERSION_ALT := "3.8.17" DRIVERS_VERSION := "2023.05.0" DRIVERS_VERSION_RHEL := DRIVERS_VERSION + "-1" +QUARTO_VERSION := "1.3.340" + # just BUILDX_PATH=~/.buildx build-release workbench bionic 12.0.11-11 _get-os-alias OS: #!/usr/bin/env bash @@ -123,6 +125,7 @@ get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: R_VERSION_ALT={{ R_VERSION_ALT }} PYTHON_VERSION={{ PYTHON_VERSION }} PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }} + QUARTO_VERSION={{ QUARTO_VERSION }} DRIVERS_VERSION=${_DRIVERS_VERSION} SRC_IMAGE_NAME=${SRC_IMAGE_NAME}" diff --git a/content/base/Justfile b/content/base/Justfile index d4b6d008..7744929a 100755 --- a/content/base/Justfile +++ b/content/base/Justfile @@ -10,7 +10,7 @@ R_VERSION := "3.6.3" PYTHON_VERSION := "3.9.5" -QUARTO_VERSION := "1.1.251" +QUARTO_VERSION := "1.3.340" DEFAULT_TAG := IMAGE_PREFIX + PRODUCT + ":r" + R_VERSION + "-py" + PYTHON_VERSION + "-" + IMAGE_OS diff --git a/content/base/maybe_install_quarto.sh b/content/base/maybe_install_quarto.sh index 8a0e6889..cb06e6c8 100755 --- a/content/base/maybe_install_quarto.sh +++ b/content/base/maybe_install_quarto.sh @@ -3,7 +3,7 @@ # on bionic, only install quarto if python 3.10 and R 4.1 # TODO: figure out a different hierarchy... if [[ `grep -oE bionic /etc/lsb-release` ]] && [[ `ls /opt/python/ | grep '3\.10\.'` ]] && [[ `ls /opt/R | grep '4\.1\.'` ]]; then - qver=${QUARTO_VERSION:-1.0.37} + qver=${QUARTO_VERSION:-1.3.340} echo '--> Installing Quarto' curl -L -o /quarto.deb https://github.com/quarto-dev/quarto-cli/releases/download/v${qver}/quarto-${qver}-linux-amd64.deb apt install /quarto.deb From 7464d0aaf4ba271bb6fce6927f4a4e4b6fe1c5c2 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 12:21:42 -0700 Subject: [PATCH 24/60] Restrict centos7 to older versions of R and Python Add base-pro build --- .github/workflows/build-release.yaml | 102 +++++++++++++++++++++------ 1 file changed, 80 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index e4db899e..975ee3db 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -10,16 +10,16 @@ name: Build, Test, Push release version jobs: build-base: runs-on: ubuntu-latest - name: build-${{ matrix.os }} + name: product-base-build-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }} strategy: fail-fast: false matrix: - os: ['ubuntu2204', 'centos7'] - lang: - - {r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + config: + - {os: 'centos7', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} + - {os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} + - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} + - {os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} steps: - name: Check Out Repo @@ -36,30 +36,26 @@ jobs: EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) BUILD_ARGS=$( \ just -f ci.Justfile \ - R_VERSION=${{ matrix.lang.r-primary }} \ - R_VERSION_ALT=${{ matrix.lang.r-alternate }} \ - PYTHON_VERSION=${{ matrix.lang.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.lang.py-alternate }} \ - get-base-args ${{ matrix.os }} base \ + R_VERSION=${{ matrix.config.r-primary }} \ + R_VERSION_ALT=${{ matrix.config.r-alternate }} \ + PYTHON_VERSION=${{ matrix.config.py-primary }} \ + PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ + get-base-args ${{ matrix.config.os }} product-base \ ) echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT echo "$BUILD_ARGS" >> $GITHUB_OUTPUT echo "$EOF" >> $GITHUB_OUTPUT - - name: echo args - shell: bash - run: echo "${{ steps.get-build-args.outputs.BUILD_ARGS }}" - - name: Get tags id: get-tags run: | IMAGE_TAGS=$( \ just -f ci.Justfile \ - R_VERSION=${{ matrix.lang.r-primary }} \ - R_VERSION_ALT=${{ matrix.lang.r-alternate }} \ - PYTHON_VERSION=${{ matrix.lang.py-primary }} \ - PYTHON_VERSION_ALT=${{ matrix.lang.py-alternate }} \ - get-base-tags ${{ matrix.os }} base \ + R_VERSION=${{ matrix.config.r-primary }} \ + R_VERSION_ALT=${{ matrix.config.r-alternate }} \ + PYTHON_VERSION=${{ matrix.config.py-primary }} \ + PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ + get-base-tags ${{ matrix.config.os }} product-base \ ) echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT @@ -67,8 +63,8 @@ jobs: uses: ./.github/actions/build-test-scan-push with: context: ./product/base - os: ${{ matrix.os }} - product: base + os: ${{ matrix.config.os }} + product: product-base image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} push-image: ${{ github.ref == 'refs/heads/main' }} @@ -78,6 +74,68 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + build-base-pro: + runs-on: ubuntu-latest + name: product-base-pro-build-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }} + + strategy: + fail-fast: false + matrix: + config: + - {os: 'centos7', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} + - {os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} + - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} + - {os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + steps: + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Get build args + id: get-build-args + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + BUILD_ARGS=$( \ + just -f ci.Justfile \ + R_VERSION=${{ matrix.config.r-primary }} \ + R_VERSION_ALT=${{ matrix.config.r-alternate }} \ + PYTHON_VERSION=${{ matrix.config.py-primary }} \ + PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ + get-base-args ${{ matrix.config.os }} product-base-pro \ + ) + echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT + echo "$BUILD_ARGS" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + - name: Get tags + id: get-tags + run: | + IMAGE_TAGS=$( \ + just -f ci.Justfile \ + R_VERSION=${{ matrix.config.r-primary }} \ + R_VERSION_ALT=${{ matrix.config.r-alternate }} \ + PYTHON_VERSION=${{ matrix.config.py-primary }} \ + PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ + get-base-tags ${{ matrix.config.os }} product-base-pro \ + ) + echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT + - name: Build/Test/Scan/Push base pro image + uses: ./.github/actions/build-test-scan-push + with: + context: ./product/pro + os: ${{ matrix.config.os }} + product: product-base-pro + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} + push-image: ${{ github.ref == 'refs/heads/main' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} From 8c949bc8f142d3ae56c612e43901392bf0a03cb5 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 12:50:21 -0700 Subject: [PATCH 25/60] Trim driver suffix for centos pro builds --- .github/workflows/build-release.yaml | 1 + product/pro/test/goss.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 975ee3db..d570319a 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -83,6 +83,7 @@ jobs: matrix: config: - {os: 'centos7', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} + - {os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - {os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - {os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} diff --git a/product/pro/test/goss.yaml b/product/pro/test/goss.yaml index d689e526..53ea2a34 100644 --- a/product/pro/test/goss.yaml +++ b/product/pro/test/goss.yaml @@ -22,7 +22,7 @@ package: rstudio-drivers: installed: true versions: - - "{{ .Env.DRIVERS_VERSION }}" + - {{ trimSuffix "-1" .Env.DRIVERS_VERSION }} # RHEL driver doesn't print the "-1" suffix in the package name file: /opt/R/{{.Env.R_VERSION}}/bin/R: From 989ad2788d50a26ba83e873293e23864d7e2e584 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 13:27:57 -0700 Subject: [PATCH 26/60] Add product builds to pipeline --- .github/workflows/build-latest.yaml | 289 --------------------------- .github/workflows/build-release.yaml | 89 ++++++++- ci.Justfile | 145 +++----------- 3 files changed, 116 insertions(+), 407 deletions(-) delete mode 100644 .github/workflows/build-latest.yaml diff --git a/.github/workflows/build-latest.yaml b/.github/workflows/build-latest.yaml deleted file mode 100644 index 26445ed0..00000000 --- a/.github/workflows/build-latest.yaml +++ /dev/null @@ -1,289 +0,0 @@ -on: - push: - branches: - - main - - dev - pull_request: - -name: build/test/push (latest) -jobs: - build-base: - runs-on: ubuntu-latest - name: build-${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - os: ['ubuntu1804', 'ubuntu2204', 'centos7'] - lang: - - {r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - just-version: '1.11.0' - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Build base image - id: build-base-image - run: | - TAGS=`just -f ci.Justfile BUILDX_PATH=${{ steps.buildx.outputs.name }} R_VERSION=${{ matrix.lang.r-primary }} R_VERSION_ALT=${{ matrix.lang.r-alternate }} PYTHON_VERSION=${{ matrix.lang.py-primary }} PYTHON_VERSION_ALT=${{ matrix.lang.py-alternate }} build-base ${{ matrix.os }} base` - echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - - - name: Show image size - run: | - docker image ls - - - name: Test image - run: | - just -f ci.Justfile R_VERSION=${{ matrix.lang.r-primary }} R_VERSION_ALT=${{ matrix.lang.r-alternate }} PYTHON_VERSION=${{ matrix.lang.py-primary }} PYTHON_VERSION_ALT=${{ matrix.lang.py-alternate }} test-base ${{ matrix.os }} base - - - name: Run Snyk to check Docker image for vulnerabilities (main) - if: ${{ github.ref == 'refs/heads/main' }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: ${{ steps.build-base-image.outputs.TAGS }} - args: --file=./product/Dockerfile.${{ matrix.os }} - command: monitor - - - name: Run Snyk to check Docker image for vulnerabilities (branch) - if: ${{ github.ref != 'refs/heads/main' }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: ${{ steps.build-base-image.outputs.TAGS }} - args: --file=./product/Dockerfile.${{ matrix.os }} - command: test - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.BUILD_PAT }} - - - name: Push image(s) to registries - if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - run: | - just -f ci.Justfile push-images ${{ steps.build-base-image.outputs.TAGS }} - - - build-pro: - needs: build-base - runs-on: ubuntu-latest - name: build-pro-${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - os: ['ubuntu1804', 'ubuntu2204', 'centos7'] - lang: - - {r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - just-version: '1.11.0' - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.BUILD_PAT }} - - - name: Build pro drivers image - id: build-pro-image - run: | - TAGS=`just -f ci.Justfile BUILDX_PATH=${{ steps.buildx.outputs.name }} R_VERSION=${{ matrix.lang.r-primary }} R_VERSION_ALT=${{ matrix.lang.r-alternate }} PYTHON_VERSION=${{ matrix.lang.py-primary }} PYTHON_VERSION_ALT=${{ matrix.lang.py-alternate }} build-base ${{ matrix.os }} pro` - echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - - - name: Show image size - run: | - docker image ls - - - name: Test image - run: | - just -f ci.Justfile R_VERSION=${{ matrix.lang.r-primary }} R_VERSION_ALT=${{ matrix.lang.r-alternate }} PYTHON_VERSION=${{ matrix.lang.py-primary }} PYTHON_VERSION_ALT=${{ matrix.lang.py-alternate }} test-base ${{ matrix.os }} pro - - - name: Run Snyk to check Docker image for vulnerabilities (main) - if: ${{ github.ref == 'refs/heads/main' }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: ${{ steps.build-pro-image.outputs.TAGS }} - args: --file=./product/pro/Dockerfile.${{ matrix.os }} - command: monitor - - - name: Run Snyk to check Docker image for vulnerabilities (branch) - if: ${{ github.ref != 'refs/heads/main' }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: ${{ steps.build-pro-image.outputs.TAGS }} - args: --file=./product/pro/Dockerfile.${{ matrix.os }} - command: test - - - name: Push image(s) to registries - if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - run: | - just -f ci.Justfile push-images ${{ steps.build-pro-image.outputs.TAGS }} - - - build-products: - needs: [build-base, build-pro] - runs-on: ubuntu-latest - name: build-${{ matrix.config.product }}-${{ matrix.config.os }}-latest - - strategy: - fail-fast: false - matrix: - config: - - {product: 'workbench', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: 'connect', os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {product: 'connect-content-init', os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {product: 'package-manager', os: 'ubuntu1804', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {product: 'package-manager', os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {product: 'r-session-complete', os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: 'r-session-complete', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: 'workbench-for-microsoft-azure-ml', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - just-version: '1.11.0' - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.BUILD_PAT }} - - - name: Get Version - id: get-version - run: | - VERSION=`just -f ci.Justfile get-version ${{ matrix.config.product }} --type=release --local` - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Get default tag - id: get-default-tag - run: | - DEFAULT_TAG=`just -f ci.Justfile R_VERSION=${{ matrix.config.r-primary }} R_VERSION_ALT=${{ matrix.config.r-alternate }} PYTHON_VERSION=${{ matrix.config.py-primary }} PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} _get-default-tag ${{ matrix.config.product }} ${{ matrix.config.os }}` - echo "DEFAULT_TAG=$DEFAULT_TAG" >> $GITHUB_OUTPUT - - - name: Build Image - id: build-image - run: | - TAGS=`just -f ci.Justfile BUILDX_PATH=${{ steps.buildx.outputs.name }} R_VERSION=${{ matrix.config.r-primary }} R_VERSION_ALT=${{ matrix.config.r-alternate }} PYTHON_VERSION=${{ matrix.config.py-primary }} PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} build-release ${{ matrix.config.product }} ${{ matrix.config.os }} ${{ steps.get-version.outputs.VERSION }}` - echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - - - name: Show image size - run: | - docker image ls - - - name: Test image - env: - RSC_LICENSE: ${{ secrets.RSC_LICENSE }} - RSPM_LICENSE: ${{ secrets.RSPM_LICENSE }} - RSW_LICENSE: ${{ secrets.RSW_LICENSE }} - run: | - just R_VERSION=${{ matrix.lang.r-primary }} R_VERSION=${{ matrix.config.r-primary }} R_VERSION_ALT=${{ matrix.config.r-alternate }} PYTHON_VERSION=${{ matrix.config.py-primary }} PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} test-image ${{ matrix.config.product }} ${{ steps.get-version.outputs.VERSION }} ${{ steps.build-image.outputs.TAGS }} - - - name: Run Snyk to check Docker image for vulnerabilities (main) - if: ${{ github.ref == 'refs/heads/main' }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: ${{ steps.get-default-tag.outputs.DEFAULT_TAG }} - args: --file=./${{ matrix.config.product }}/Dockerfile.${{ matrix.config.os }} --org=${{ secrets.SNYK_ORG_ID }} --project-name=${{ steps.get-default-tag.outputs.DEFAULT_TAG }} --tags=product=${{ matrix.config.product }},os=${{ matrix.config.os }} --exclude-base-image-vulns --app-vulns - command: monitor - - - name: Run Snyk to check Docker image for vulnerabilities (branch) - if: ${{ github.ref != 'refs/heads/main' }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: ${{ steps.get-default-tag.outputs.DEFAULT_TAG }} - args: --file=./${{ matrix.config.product }}/Dockerfile.${{ matrix.config.os }} --org=${{ secrets.SNYK_ORG_ID }} --project-name=${{ steps.get-default-tag.outputs.DEFAULT_TAG }} --tags=product=${{ matrix.config.product }},os=${{ matrix.config.os }} --exclude-base-image-vulns --app-vulns - command: test - - - name: Login to Docker Hub - if: ${{ github.ref == 'refs/heads/main' }} - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Push image(s) to registries - if: ${{ github.ref == 'refs/heads/main' }} - run: | - just -f ci.Justfile push-images ${{ steps.build-image.outputs.TAGS }} diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index d570319a..78d64ddf 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -17,7 +17,8 @@ jobs: matrix: config: - {os: 'centos7', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} + - {os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} + - {os: 'ubuntu1804', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - {os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} @@ -75,6 +76,7 @@ jobs: dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} build-base-pro: + needs: build-base runs-on: ubuntu-latest name: product-base-pro-build-${{ matrix.config.os }}-r${{ matrix.config.r-primary }}_${{ matrix.config.r-alternate }}-py${{ matrix.config.py-primary }}_${{ matrix.config.py-alternate }} @@ -84,7 +86,7 @@ jobs: config: - {os: 'centos7', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - {os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {os: 'ubuntu2204', r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} + - {os: 'ubuntu1804', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - {os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - {os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} @@ -140,3 +142,86 @@ jobs: ghcr-token: ${{ secrets.BUILD_PAT }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + build-products: + needs: [ build-base, build-base-pro ] + runs-on: ubuntu-latest + name: build-${{ matrix.config.product }}-${{ matrix.config.os }} + + strategy: + fail-fast: false + matrix: + config: + - {product: 'workbench', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: 'connect', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: 'connect-content-init', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: 'package-manager', os: 'ubuntu1804', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} + - {product: 'package-manager', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: 'r-session-complete', os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} + - {product: 'r-session-complete', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: 'workbench-for-microsoft-azure-ml', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + + steps: + - name: Check Out Repo + uses: actions/checkout@v3 + + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Get Version + id: get-version + run: | + VERSION=`just -f ci.Justfile get-version ${{ matrix.config.product }} --type=release --local` + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + + - name: Get build args + id: get-build-args + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + BUILD_ARGS=$( \ + just -f ci.Justfile \ + R_VERSION=${{ matrix.config.r-primary }} \ + R_VERSION_ALT=${{ matrix.config.r-alternate }} \ + PYTHON_VERSION=${{ matrix.config.py-primary }} \ + PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ + get-product-args \ + ${{ matrix.config.product }} \ + ${{ matrix.config.os }} \ + ${{ steps.get-version.outputs.VERSION }} \ + ) + echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT + echo "$BUILD_ARGS" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + + - name: Get tags + id: get-tags + run: | + IMAGE_TAGS=$( \ + just -f ci.Justfile \ + R_VERSION=${{ matrix.config.r-primary }} \ + R_VERSION_ALT=${{ matrix.config.r-alternate }} \ + PYTHON_VERSION=${{ matrix.config.py-primary }} \ + PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ + get-product-tags \ + ${{ matrix.config.product }} \ + ${{ matrix.config.os }} \ + ${{ steps.get-version.outputs.VERSION }} \ + ) + echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT + + - name: Build/Test/Scan/Push base pro image + uses: ./.github/actions/build-test-scan-push + with: + context: ./product/pro + os: ${{ matrix.config.os }} + product: ${{ matrix.config.product }} + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} + push-image: ${{ github.ref == 'refs/heads/main' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} diff --git a/ci.Justfile b/ci.Justfile index 6b5f7d3f..0293ae1e 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -38,68 +38,6 @@ _get-default-tag PRODUCT OS: echo "{{ REGISTRY_NAMESPACE }}/${IMAGE_PREFIX}{{ PRODUCT }}:{{ OS }}" -# just BUILDX_PATH=~/.buildx build-base ubuntu1804 base -build-base $OS $TYPE="base" $BRANCH=`git branch --show`: - #!/usr/bin/env bash - set -euxo pipefail - - # variable placeholders - BUILDX_ARGS="" - - # set short name - if [[ $TYPE == "base" || $TYPE == "product-base" ]]; then - IMAGE_NAME="product-base" - SRC_IMAGE_NAME="" - CTX_PATH="./product/base" - FILE_PATH="./product/base/Dockerfile.${OS}" - elif [[ $TYPE == "base-pro" || $TYPE == "pro" || $TYPE == "product-base-pro" ]]; then - IMAGE_NAME="product-base-pro" - SRC_IMAGE_NAME="product-base" - CTX_PATH="./product/pro" - FILE_PATH="./product/pro/Dockerfile.${OS}" - fi - if [[ $BRANCH != "main" ]]; then - IMAGE_NAME="${IMAGE_NAME}-dev" - SRC_IMAGE_NAME="${SRC_IMAGE_NAME}-dev" - fi - - if [[ "${OS}" == "centos7" ]]; then - _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" - else - _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" - fi - - # set buildx args - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - docker buildx --builder="{{BUILDX_PATH}}" build --load $BUILDX_ARGS \ - -t rstudio/${IMAGE_NAME}:${OS} \ - -t rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}} \ - -t rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} \ - -t ghcr.io/rstudio/${IMAGE_NAME}:${OS} \ - -t ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}} \ - -t ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg DRIVERS_VERSION="${_DRIVERS_VERSION}" \ - --build-arg SRC_IMAGE_NAME="${SRC_IMAGE_NAME}" \ - --file "${FILE_PATH}" "${CTX_PATH}" - - # echo rstudio/${IMAGE_NAME}:${OS} \ - # rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}} \ - # rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} \ - # ghcr.io/rstudio/${IMAGE_NAME}:${OS} \ - # ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}} \ - # ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} - - echo ghcr.io/rstudio/${IMAGE_NAME}:${OS} \ - ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}} \ - ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} - get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: #!/usr/bin/env bash set -euxo pipefail @@ -121,6 +59,7 @@ get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: else _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" fi + printf "R_VERSION={{ R_VERSION }} R_VERSION_ALT={{ R_VERSION_ALT }} PYTHON_VERSION={{ PYTHON_VERSION }} @@ -146,53 +85,43 @@ get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}},\ ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} -# just BUILDX_PATH=~/.buildx test-base ubuntu1804 base -test-base $OS $TYPE="base" $BRANCH=`git branch --show`: +get-product-args $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`git rev-parse --short HEAD`: #!/usr/bin/env bash set -euxo pipefail + RSW_DOWNLOAD_URL=$(just -f ci.Justfile _get-rsw-download-url release $OS) - # set short name - if [[ $TYPE == "base" ]]; then - IMAGE_NAME="product-base" - if [[ $BRANCH != "main" ]]; then - IMAGE_NAME="${IMAGE_NAME}-dev" - fi - just IMAGE_OS="${OS}" R_VERSION={{R_VERSION}} R_VERSION_ALT={{R_VERSION_ALT}} PYTHON_VERSION={{PYTHON_VERSION}} PYTHON_VERSION_ALT={{PYTHON_VERSION_ALT}} product/base/test ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} - elif [[ $TYPE == "base-pro" || $TYPE == "pro" ]]; then - IMAGE_NAME="product-base-pro" - if [[ $BRANCH != "main" ]]; then - IMAGE_NAME="${IMAGE_NAME}-dev" + # set source image name + SRC_IMAGE_NAME="" + if [[ $PRODUCT == "workbench" || $PRODUCT == "r-session-complete" || $PRODUCT == "workbench-for-microsoft-azure-ml" || $PRODUCT == "connect" ]]; then + if [[ $BRANCH == "main" ]]; then + SRC_IMAGE_NAME="product-base-pro" + else + SRC_IMAGE_NAME="product-base-pro-dev" fi - just IMAGE_OS="${OS}" R_VERSION={{R_VERSION}} R_VERSION_ALT={{R_VERSION_ALT}} PYTHON_VERSION={{PYTHON_VERSION}} PYTHON_VERSION_ALT={{PYTHON_VERSION_ALT}} product/pro/test ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} fi -# just BUILDX_PATH=~/.buildx build-release workbench ubuntu1804 12.0.11-11 -build-release $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`git rev-parse --short HEAD`: + printf "R_VERSION={{ R_VERSION }} + R_VERSION_ALT={{ R_VERSION_ALT }} + PYTHON_VERSION={{ PYTHON_VERSION }} + PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }} + QUARTO_VERSION={{ QUARTO_VERSION }} + DRIVERS_VERSION=${_DRIVERS_VERSION} + SRC_IMAGE_NAME=${SRC_IMAGE_NAME} + RSW_DOWNLOAD_URL=${RSW_DOWNLOAD_URL}" + +get-product-tags $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`git rev-parse --short HEAD`: #!/usr/bin/env bash set -euxo pipefail # variable placeholders - RSW_DOWNLOAD_URL=`just -f ci.Justfile _get-rsw-download-url release $OS` - BUILDX_ARGS="" SHORT_NAME="" - TAG_CLEAN_VERSION=`just _get-clean-version $VERSION` + TAG_CLEAN_VERSION=$(just _get-clean-version $VERSION) # set short name and source image name - SRC_IMAGE_NAME="" if [[ $PRODUCT == "workbench" || $PRODUCT == "r-session-complete" || $PRODUCT == "workbench-for-microsoft-azure-ml" ]]; then SHORT_NAME="RSW" - if [[ $BRANCH == "main" ]]; then - SRC_IMAGE_NAME="product-base-pro" - else - SRC_IMAGE_NAME="product-base-pro-dev" - fi elif [[ $PRODUCT == "connect" ]]; then SHORT_NAME="RSC" - if [[ $BRANCH == "main" ]]; then - SRC_IMAGE_NAME="product-base-pro" - else - SRC_IMAGE_NAME="product-base-pro-dev" - fi elif [[ $PRODUCT == "package-manager" ]]; then SHORT_NAME="RSPM" fi @@ -209,32 +138,16 @@ build-release $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`git for os_name in ${OS_ALIASES[@]}; do tag_array+=( - "-t" "rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}" - "-t" "rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}" - "-t" "rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}--${SHA_SHORT}" - "-t" "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}" - "-t" "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}" - "-t" "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}--${SHA_SHORT}" + "rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}" + "rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}" + "rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}--${SHA_SHORT}" + "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}" + "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}" + "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}--${SHA_SHORT}" ) done - - # set buildx args - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - docker buildx --builder="{{BUILDX_PATH}}" build --load $BUILDX_ARGS \ - ${tag_array[@]} \ - --build-arg "$SHORT_NAME"_VERSION=$VERSION \ - --build-arg RSW_DOWNLOAD_URL=$RSW_DOWNLOAD_URL \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg SRC_IMAGE_NAME="${SRC_IMAGE_NAME}" \ - --file=./${PRODUCT}/Dockerfile.$(just _parse-os ${OS}) ${PRODUCT} - - echo ${tag_array[*]//-t/} + tags=$(IFS="," ; echo "${tag_array[*]}") + echo "${tags}" # just BUILDX_PATH=~/.buildx build-preview preview workbench ubuntu1804 12.0.11-11 build-preview $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: From d80579a3220e2866c4a7964370d2d907bbe677d8 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 14:13:10 -0700 Subject: [PATCH 27/60] Add build-prerelease.yaml replacing build-preview.yaml Cleanup CI naming and Justfile --- .github/workflows/build-prerelease.yaml | 108 ++++++++++++++++ .github/workflows/build-preview.yaml | 112 ---------------- .github/workflows/build-release.yaml | 3 +- .github/workflows/clean-registry.yaml | 2 +- .github/workflows/lint.yaml | 2 +- ci.Justfile | 165 ++++++++++++------------ 6 files changed, 196 insertions(+), 196 deletions(-) create mode 100644 .github/workflows/build-prerelease.yaml delete mode 100644 .github/workflows/build-preview.yaml diff --git a/.github/workflows/build-prerelease.yaml b/.github/workflows/build-prerelease.yaml new file mode 100644 index 00000000..5fab0f8d --- /dev/null +++ b/.github/workflows/build-prerelease.yaml @@ -0,0 +1,108 @@ +on: + schedule: + # every morning at 8am UTC + # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onschedule + # make sure to change the `if` commands below when changing the schedule + - cron: '0 8 * * *' + - cron: '0 9 * * *' + push: + branches: + - main + - dev + - dev-rspm + pull_request: + +name: Prerelease - Build, Test, Scan, and Push +jobs: + build: + runs-on: ubuntu-latest + name: build-${{ matrix.config.type }}-${{ matrix.config.product }}-${{ matrix.config.os }} + + strategy: + fail-fast: false + matrix: + config: + - {product: "workbench", type: "daily", os: "ubuntu2204", r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: "workbench", type: "preview", os: "ubuntu2204", r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: "connect", type: "daily", os: "ubuntu2204", r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: "connect-content-init", type: "daily", os: "ubuntu2204", r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: "package-manager", type: "daily", os: "ubuntu1804", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} + - {product: "package-manager", type: "daily", os: "ubuntu2204", r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: "r-session-complete", type: "daily", os: "ubuntu2204", r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: "r-session-complete", type: "daily", os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} + - {product: "r-session-complete", type: "preview", os: "ubuntu2204", r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: "r-session-complete", type: "preview", os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} + + steps: + - name: Check Out main Branch + if: github.event.schedule == '0 8 * * *' + uses: actions/checkout@v3 + with: + ref: 'main' + + - name: Check Out Repo at Triggered Branch + if: github.event.schedule != '0 8 * * *' + uses: actions/checkout@v3 + + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Get Version + id: get-version + run: | + VERSION=$(just -f ci.Justfile get-version ${{ matrix.config.product }} --type=${{ matrix.config.type }} --local) + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + + - name: Get build args + id: get-build-args + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + BUILD_ARGS=$( \ + just -f ci.Justfile \ + R_VERSION=${{ matrix.config.r-primary }} \ + R_VERSION_ALT=${{ matrix.config.r-alternate }} \ + PYTHON_VERSION=${{ matrix.config.py-primary }} \ + PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ + get-prerelease-args \ + ${{ matrix.config.type }} \ + ${{ matrix.config.product }} \ + ${{ matrix.config.os }} \ + ${{ steps.get-version.outputs.VERSION }} \ + ) + echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT + echo "$BUILD_ARGS" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + + - name: Get tags + id: get-tags + run: | + IMAGE_TAGS=$( \ + just -f ci.Justfile \ + R_VERSION=${{ matrix.config.r-primary }} \ + R_VERSION_ALT=${{ matrix.config.r-alternate }} \ + PYTHON_VERSION=${{ matrix.config.py-primary }} \ + PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} \ + get-prerelease-tags \ + ${{ matrix.config.type }} \ + ${{ matrix.config.product }} \ + ${{ matrix.config.os }} \ + ${{ steps.get-version.outputs.VERSION }} \ + ) + echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT + + - name: Build/Test/Scan/Push base pro image + uses: ./.github/actions/build-test-scan-push + with: + context: ./product/pro + os: ${{ matrix.config.os }} + product: ${{ matrix.config.product }} + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/build-preview.yaml b/.github/workflows/build-preview.yaml deleted file mode 100644 index 8b995be9..00000000 --- a/.github/workflows/build-preview.yaml +++ /dev/null @@ -1,112 +0,0 @@ -on: - schedule: - # every morning at 8am UTC - # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onschedule - # make sure to change the `if` commands below when changing the schedule - - cron: '0 8 * * *' - - cron: '0 9 * * *' - push: - branches: - - main - - dev - - dev-rspm - pull_request: - -name: build/test/push (preview) -jobs: - - - build: - runs-on: ubuntu-latest - name: build-${{ matrix.config.product }}:${{ matrix.config.type }}:${{ matrix.config.os }} - - strategy: - fail-fast: false - matrix: - config: - - {product: "workbench", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: "workbench", type: "preview", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: "connect", type: "daily", os: "ubuntu2204", r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {product: "connect-content-init", type: "daily", os: "ubuntu2204", r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {product: "package-manager", type: "daily", os: "ubuntu1804", r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {product: "package-manager", type: "daily", os: "ubuntu2204", r-primary: "4.2.0", r-alternate: "3.6.2", py-primary: "3.9.5", py-alternate: "3.8.10"} - - {product: "r-session-complete", type: "daily", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: "r-session-complete", type: "daily", os: "centos7", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: "r-session-complete", type: "preview", os: "ubuntu2204", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: "r-session-complete", type: "preview", os: "centos7", r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - steps: - # the schedule triggers on the "default" branch (dev), so we have to specify "main" - # this schedule matching / specification is less than ideal at present and must be - # kept in sync with the schedule specification at the top of the file - - name: Check Out main Branch - if: github.event.schedule == '0 8 * * *' - uses: actions/checkout@v3 - with: - ref: 'main' - - - name: Check Out Repo at Triggered Branch - if: github.event.schedule != '0 8 * * *' - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - just-version: '1.11.0' - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Login to ghcr.io - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.BUILD_PAT }} - - - name: Get Version - id: get-version - run: | - VERSION=`just -f ci.Justfile get-version ${{ matrix.config.product }} --type=${{ matrix.config.type }} --local` - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Build Image - id: build-image - run: | - TAGS=`just -f ci.Justfile BUILDX_PATH=${{ steps.buildx.outputs.name }} R_VERSION=${{ matrix.config.r-primary }} R_VERSION_ALT=${{ matrix.config.r-alternate }} PYTHON_VERSION=${{ matrix.config.py-primary }} PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} build-preview ${{ matrix.config.type }} ${{ matrix.config.product }} ${{ matrix.config.os }} ${{ steps.get-version.outputs.VERSION }}` - echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - - - name: Show image size - run: | - docker image ls - - - name: Test image - env: - RSC_LICENSE: ${{ secrets.RSC_LICENSE }} - RSPM_LICENSE: ${{ secrets.RSPM_LICENSE }} - RSW_LICENSE: ${{ secrets.RSW_LICENSE }} - run: | - just R_VERSION=${{ matrix.config.r-primary }} R_VERSION_ALT=${{ matrix.config.r-alternate }} PYTHON_VERSION=${{ matrix.config.py-primary }} PYTHON_VERSION_ALT=${{ matrix.config.py-alternate }} test-image ${{ matrix.config.product }} ${{ steps.get-version.outputs.VERSION }} ${{ steps.build-image.outputs.TAGS }} - - - name: Login to Docker Hub - if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Push image(s) to registries - if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} - run: | - just -f ci.Justfile push-images ${{ steps.build-image.outputs.TAGS }} diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 78d64ddf..94dac1b6 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -4,9 +4,10 @@ on: push: branches: - main + - dev pull_request: -name: Build, Test, Push release version +name: Release - Build, Test, Scan, and Push jobs: build-base: runs-on: ubuntu-latest diff --git a/.github/workflows/clean-registry.yaml b/.github/workflows/clean-registry.yaml index 3e0322bc..890614d8 100644 --- a/.github/workflows/clean-registry.yaml +++ b/.github/workflows/clean-registry.yaml @@ -2,7 +2,7 @@ on: schedule: - cron: 0 0 * * 2 -name: clean registries +name: Clean Registries jobs: cleanup: diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index d005b4fa..15f71f3e 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -5,7 +5,7 @@ on: - dev pull_request: -name: lint Dockerfiles +name: Lint Dockerfiles jobs: lint: diff --git a/ci.Justfile b/ci.Justfile index 0293ae1e..c6193c1c 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -38,6 +38,26 @@ _get-default-tag PRODUCT OS: echo "{{ REGISTRY_NAMESPACE }}/${IMAGE_PREFIX}{{ PRODUCT }}:{{ OS }}" +# just _get-rsw-download-url release ubuntu1804 +_get-rsw-download-url TYPE OS: + #!/usr/bin/env bash + URL_OS="{{OS}}" + if [[ "{{OS}}" == "ubuntu1804" ]]; then + URL_OS="bionic" + elif [[ "{{OS}}" == "ubuntu2204" ]]; then + URL_OS="jammy" + fi + + if [[ "{{TYPE}}" == "release" ]]; then + echo "https://download2.rstudio.org/server/${URL_OS}/{{ if OS == "centos7" { "x86_64"} else { "amd64" } }}" + else + echo "https://s3.amazonaws.com/rstudio-ide-build/server/${URL_OS}/{{ if OS == "centos7" { "x86_64"} else { "amd64" } }}" + fi + +# just get-version workbench --type=preview --local +get-version +NARGS: + ./tools/get-version.py {{NARGS}} + get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: #!/usr/bin/env bash set -euxo pipefail @@ -90,6 +110,14 @@ get-product-args $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`g set -euxo pipefail RSW_DOWNLOAD_URL=$(just -f ci.Justfile _get-rsw-download-url release $OS) + if [[ $PRODUCT == "workbench" || $PRODUCT == "r-session-complete" || $PRODUCT == "workbench-for-microsoft-azure-ml" ]]; then + SHORT_NAME="RSW" + elif [[ $PRODUCT == "connect" || $PRODUCT == "connect-content-init" ]]; then + SHORT_NAME="RSC" + elif [[ $PRODUCT == "package-manager" ]]; then + SHORT_NAME="RSPM" + fi + # set source image name SRC_IMAGE_NAME="" if [[ $PRODUCT == "workbench" || $PRODUCT == "r-session-complete" || $PRODUCT == "workbench-for-microsoft-azure-ml" || $PRODUCT == "connect" ]]; then @@ -100,7 +128,14 @@ get-product-args $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`g fi fi - printf "R_VERSION={{ R_VERSION }} + if [[ "${OS}" == "centos7" ]]; then + _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" + else + _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" + fi + + printf "${SHORT_NAME}_VERSION=${VERSION} + R_VERSION={{ R_VERSION }} R_VERSION_ALT={{ R_VERSION_ALT }} PYTHON_VERSION={{ PYTHON_VERSION }} PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }} @@ -149,44 +184,58 @@ get-product-tags $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`g tags=$(IFS="," ; echo "${tag_array[*]}") echo "${tags}" -# just BUILDX_PATH=~/.buildx build-preview preview workbench ubuntu1804 12.0.11-11 -build-preview $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: +get-prerelease-args $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: #!/usr/bin/env bash set -euxo pipefail - # variable placeholders - BRANCH_PREFIX="" - RSW_DOWNLOAD_URL=`just -f ci.Justfile _get-rsw-download-url $TYPE $OS` - BUILDX_ARGS="" - SHORT_NAME="" - TAG_CLEAN_VERSION=`just _get-clean-version $VERSION` - TAG_VERSION=`just _get-tag-safe-version $VERSION` + RSW_DOWNLOAD_URL=$(just -f ci.Justfile _get-rsw-download-url release $OS) - # set branch prefix - if [[ $BRANCH == "dev" ]]; then - BRANCH_PREFIX="dev-" - elif [[ $BRANCH == "dev-rspm" ]]; then - BRANCH_PREFIX="dev-rspm-" + if [[ $PRODUCT == "workbench" || $PRODUCT == "r-session-complete" || $PRODUCT == "workbench-for-microsoft-azure-ml" ]]; then + SHORT_NAME="RSW" + elif [[ $PRODUCT == "connect" || $PRODUCT == "connect-content-init" ]]; then + SHORT_NAME="RSC" + elif [[ $PRODUCT == "package-manager" ]]; then + SHORT_NAME="RSPM" fi - # set short name + # set source image name SRC_IMAGE_NAME="" - if [[ $PRODUCT == "workbench" || $PRODUCT == "r-session-complete" || $PRODUCT == "workbench-for-microsoft-azure-ml" ]]; then - SHORT_NAME="RSW" + if [[ $PRODUCT == "workbench" || $PRODUCT == "r-session-complete" || $PRODUCT == "workbench-for-microsoft-azure-ml" || $PRODUCT == "connect" ]]; then if [[ $BRANCH == "main" ]]; then SRC_IMAGE_NAME="product-base-pro" else SRC_IMAGE_NAME="product-base-pro-dev" fi - elif [[ $PRODUCT == "connect" || $PRODUCT == "connect-content-init" ]]; then - SHORT_NAME="RSC" - if [[ $BRANCH == "main" ]]; then - SRC_IMAGE_NAME="product-base" - else - SRC_IMAGE_NAME="product-base-dev" - fi - elif [[ $PRODUCT == "package-manager" ]]; then - SHORT_NAME="RSPM" + fi + + if [[ "${OS}" == "centos7" ]]; then + _DRIVERS_VERSION="{{ DRIVERS_VERSION_RHEL }}" + else + _DRIVERS_VERSION="{{ DRIVERS_VERSION }}" + fi + + printf "${SHORT_NAME}_VERSION=${VERSION} + R_VERSION={{ R_VERSION }} + R_VERSION_ALT={{ R_VERSION_ALT }} + PYTHON_VERSION={{ PYTHON_VERSION }} + PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }} + QUARTO_VERSION={{ QUARTO_VERSION }} + DRIVERS_VERSION=${_DRIVERS_VERSION} + SRC_IMAGE_NAME=${SRC_IMAGE_NAME} + RSW_DOWNLOAD_URL=${RSW_DOWNLOAD_URL}" + +get-prerelease-tags $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: + #!/usr/bin/env bash + set -euxo pipefail + + # variable placeholders + BRANCH_PREFIX="" + TAG_CLEAN_VERSION=$(just _get-clean-version $VERSION) + TAG_VERSION=$(just _get-tag-safe-version $VERSION) + + # set branch prefix + if [[ $BRANCH != "main" ]]; then + BRANCH_PREFIX="${BRANCH}-" fi # set image prefix @@ -201,59 +250,13 @@ build-preview $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: for os_name in ${OS_ALIASES[@]}; do tag_array+=( - "-t" "rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${os_name}-${TAG_VERSION}" - "-t" "rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${os_name}-${TAG_CLEAN_VERSION}" - "-t" "rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${os_name}-${TYPE}" - "-t" "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${os_name}-${TAG_VERSION}" - "-t" "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${os_name}-${TAG_CLEAN_VERSION}" - "-t" "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${os_name}-${TYPE}" + "rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${os_name}-${TAG_VERSION}" + "rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${os_name}-${TAG_CLEAN_VERSION}" + "rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${os_name}-${TYPE}" + "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${os_name}-${TAG_VERSION}" + "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${os_name}-${TAG_CLEAN_VERSION}" + "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${os_name}-${TYPE}" ) done - - # set buildx args - if [[ "{{BUILDX_PATH}}" != "" ]]; then - BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache" - fi - - docker buildx --builder="{{BUILDX_PATH}}" build --load $BUILDX_ARGS \ - ${tag_array[@]} \ - --build-arg ${SHORT_NAME}_VERSION=$VERSION \ - --build-arg RSW_DOWNLOAD_URL=$RSW_DOWNLOAD_URL \ - --build-arg R_VERSION="{{ R_VERSION }}" \ - --build-arg R_VERSION_ALT="{{ R_VERSION_ALT }}" \ - --build-arg PYTHON_VERSION="{{ PYTHON_VERSION }}" \ - --build-arg PYTHON_VERSION_ALT="{{ PYTHON_VERSION_ALT }}" \ - --build-arg SRC_IMAGE_NAME="${SRC_IMAGE_NAME}" \ - --file=./${PRODUCT}/Dockerfile.$(just _parse-os ${OS}) ${PRODUCT} - - # These tags are propogated forward to test-images and push-images in builds. It is important that these tags match the build tags above. - echo ${tag_array[*]//-t/} - -# just push-images tag1 tag2 ... -push-images +IMAGES: - #!/usr/bin/env bash - set -euxo pipefail - for IMAGE in {{IMAGES}} - do - docker push $IMAGE - done - -# just _get-rsw-download-url release ubuntu1804 -_get-rsw-download-url TYPE OS: - #!/usr/bin/env bash - URL_OS="{{OS}}" - if [[ "{{OS}}" == "ubuntu1804" ]]; then - URL_OS="bionic" - elif [[ "{{OS}}" == "ubuntu2204" ]]; then - URL_OS="jammy" - fi - - if [[ "{{TYPE}}" == "release" ]]; then - echo "https://download2.rstudio.org/server/${URL_OS}/{{ if OS == "centos7" { "x86_64"} else { "amd64" } }}" - else - echo "https://s3.amazonaws.com/rstudio-ide-build/server/${URL_OS}/{{ if OS == "centos7" { "x86_64"} else { "amd64" } }}" - fi - -# just get-version workbench --type=preview --local -get-version +NARGS: - ./tools/get-version.py {{NARGS}} + tags=$(IFS="," ; echo "${tag_array[*]}") + echo "${tags}" From b2c4316d581027afe7e5b1e50bcac0b1c426993f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Fri, 21 Jul 2023 14:22:28 -0700 Subject: [PATCH 28/60] Check that $BRANCH is not empty --- ci.Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.Justfile b/ci.Justfile index c6193c1c..03e651da 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -234,7 +234,7 @@ get-prerelease-tags $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: TAG_VERSION=$(just _get-tag-safe-version $VERSION) # set branch prefix - if [[ $BRANCH != "main" ]]; then + if [[ ! -z $BRANCH ]] && [[ $BRANCH != "main" ]]; then BRANCH_PREFIX="${BRANCH}-" fi From b10fa8cd39991929da8e7ec50d6e55889449e64e Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Jul 2023 09:20:37 -0700 Subject: [PATCH 29/60] Update context path for preview builds --- .github/workflows/build-release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 94dac1b6..554dec3a 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -155,7 +155,7 @@ jobs: config: - {product: 'workbench', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} - {product: 'connect', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} - - {product: 'connect-content-init', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: 'connect-content-init', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} - {product: 'package-manager', os: 'ubuntu1804', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - {product: 'package-manager', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} - {product: 'r-session-complete', os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} @@ -215,7 +215,7 @@ jobs: - name: Build/Test/Scan/Push base pro image uses: ./.github/actions/build-test-scan-push with: - context: ./product/pro + context: ./${{ matrix.config.product }} os: ${{ matrix.config.os }} product: ${{ matrix.config.product }} image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} From 8ec90a0dd4d87c0446b525875e2e0580b82268fa Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Jul 2023 09:42:09 -0700 Subject: [PATCH 30/60] Push base images on dev Update context for prerelease --- .github/workflows/build-prerelease.yaml | 2 +- .github/workflows/build-release.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-prerelease.yaml b/.github/workflows/build-prerelease.yaml index 5fab0f8d..d2aa9c96 100644 --- a/.github/workflows/build-prerelease.yaml +++ b/.github/workflows/build-prerelease.yaml @@ -95,7 +95,7 @@ jobs: - name: Build/Test/Scan/Push base pro image uses: ./.github/actions/build-test-scan-push with: - context: ./product/pro + context: ./${{ matrix.config.product }} os: ${{ matrix.config.os }} product: ${{ matrix.config.product }} image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 554dec3a..e5a724b0 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -69,7 +69,7 @@ jobs: product: product-base image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' }} + push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} @@ -137,7 +137,7 @@ jobs: product: product-base-pro image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' }} + push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} From 685de43907b09c924786bc0fda61585dbad9fe36 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Jul 2023 10:12:01 -0700 Subject: [PATCH 31/60] Fix RSW download url for prerelease --- ci.Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.Justfile b/ci.Justfile index 03e651da..62df9363 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -188,7 +188,7 @@ get-prerelease-args $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: #!/usr/bin/env bash set -euxo pipefail - RSW_DOWNLOAD_URL=$(just -f ci.Justfile _get-rsw-download-url release $OS) + RSW_DOWNLOAD_URL=$(just -f ci.Justfile _get-rsw-download-url $TYPE $OS) if [[ $PRODUCT == "workbench" || $PRODUCT == "r-session-complete" || $PRODUCT == "workbench-for-microsoft-azure-ml" ]]; then SHORT_NAME="RSW" From 1896be1a8ceb38ab9b0b4f739adb87cbc6c9594e Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Jul 2023 10:13:19 -0700 Subject: [PATCH 32/60] Temporarily push dev base builds for release --- .github/workflows/build-release.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index e5a724b0..56a6e574 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -69,7 +69,8 @@ jobs: product: product-base image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} + # push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} + push-image: true snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} @@ -137,7 +138,8 @@ jobs: product: product-base-pro image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} + # push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} + push-image: true snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} From a3e4914eb9a983514b7cd42f277caf505617acb5 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Jul 2023 11:07:56 -0700 Subject: [PATCH 33/60] Add PYTHON_VERSION_JUPYTER to ci args --- ci.Justfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci.Justfile b/ci.Justfile index 62df9363..e99629f3 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -139,6 +139,7 @@ get-product-args $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`g R_VERSION_ALT={{ R_VERSION_ALT }} PYTHON_VERSION={{ PYTHON_VERSION }} PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }} + PYTHON_VERSION_JUPYTER={{ PYTHON_VERSION_ALT }} QUARTO_VERSION={{ QUARTO_VERSION }} DRIVERS_VERSION=${_DRIVERS_VERSION} SRC_IMAGE_NAME=${SRC_IMAGE_NAME} @@ -219,6 +220,7 @@ get-prerelease-args $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: R_VERSION_ALT={{ R_VERSION_ALT }} PYTHON_VERSION={{ PYTHON_VERSION }} PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }} + PYTHON_VERSION_JUPYTER={{ PYTHON_VERSION_ALT }} QUARTO_VERSION={{ QUARTO_VERSION }} DRIVERS_VERSION=${_DRIVERS_VERSION} SRC_IMAGE_NAME=${SRC_IMAGE_NAME} From a09fb090c98312575d8e163e1be848d8951bd10a Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Jul 2023 11:18:52 -0700 Subject: [PATCH 34/60] Unify `PYTHON_VERSION_JUPYTER` naming convention --- workbench-for-microsoft-azure-ml/.env | 2 +- workbench-for-microsoft-azure-ml/docker-compose.test.yml | 2 +- workbench-for-microsoft-azure-ml/test/goss.yaml | 2 +- workbench/.env | 2 +- workbench/docker-compose.test.yml | 2 +- workbench/test/goss.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/workbench-for-microsoft-azure-ml/.env b/workbench-for-microsoft-azure-ml/.env index 9ec2b95b..1df43c24 100644 --- a/workbench-for-microsoft-azure-ml/.env +++ b/workbench-for-microsoft-azure-ml/.env @@ -3,6 +3,6 @@ RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/bionic/amd64 RSW_NAME=rstudio-workbench PYTHON_VERSION=3.9.14 PYTHON_VERSION_ALT=3.8.15 -JUPYTER_PYTHON_VERSION=3.8.15 +PYTHON_VERSION_JUPYTER=3.8.15 R_VERSION_ALT=4.1.3 R_VERSION=4.2.3 diff --git a/workbench-for-microsoft-azure-ml/docker-compose.test.yml b/workbench-for-microsoft-azure-ml/docker-compose.test.yml index 87338f04..53a1d86d 100644 --- a/workbench-for-microsoft-azure-ml/docker-compose.test.yml +++ b/workbench-for-microsoft-azure-ml/docker-compose.test.yml @@ -11,7 +11,7 @@ services: - R_VERSION - PYTHON_VERSION - PYTHON_VERSION_ALT - - JUPYTER_PYTHON_VERSION + - PYTHON_VERSION_JUPYTER - R_VERSION_ALT - RSW_LICENSE volumes: diff --git a/workbench-for-microsoft-azure-ml/test/goss.yaml b/workbench-for-microsoft-azure-ml/test/goss.yaml index 09e898da..b7553054 100644 --- a/workbench-for-microsoft-azure-ml/test/goss.yaml +++ b/workbench-for-microsoft-azure-ml/test/goss.yaml @@ -120,7 +120,7 @@ command: title: jupyter_python_version_matches exit-status: 0 stdout: [ - "{{.Env.JUPYTER_PYTHON_VERSION}}" + "{{.Env.PYTHON_VERSION_JUPYTER}}" ] "which openssl": title: uses_system_openssl diff --git a/workbench/.env b/workbench/.env index b89748cb..9459434a 100644 --- a/workbench/.env +++ b/workbench/.env @@ -3,6 +3,6 @@ RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/bionic/amd64 RSW_NAME=rstudio-workbench PYTHON_VERSION=3.9.17 PYTHON_VERSION_ALT=3.8.17 -JUPYTER_PYTHON_VERSION=3.8.15 +PYTHON_VERSION_JUPYTER=3.8.15 R_VERSION_ALT=4.1.3 R_VERSION=4.2.3 diff --git a/workbench/docker-compose.test.yml b/workbench/docker-compose.test.yml index b28c6f5f..db54a879 100644 --- a/workbench/docker-compose.test.yml +++ b/workbench/docker-compose.test.yml @@ -11,7 +11,7 @@ services: - R_VERSION - PYTHON_VERSION - PYTHON_VERSION_ALT - - JUPYTER_PYTHON_VERSION + - PYTHON_VERSION_JUPYTER - R_VERSION_ALT - RSW_LICENSE - RSW_LICENSE_SERVER diff --git a/workbench/test/goss.yaml b/workbench/test/goss.yaml index 86a47063..ccce2166 100644 --- a/workbench/test/goss.yaml +++ b/workbench/test/goss.yaml @@ -123,7 +123,7 @@ command: title: jupyter_python_version_matches exit-status: 0 stdout: [ - "{{.Env.JUPYTER_PYTHON_VERSION}}" + "{{.Env.PYTHON_VERSION_JUPYTER}}" ] "which openssl": title: uses_system_openssl From e5fde0edf21793ea47594dc749b8ba86dd301454 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Jul 2023 11:44:47 -0700 Subject: [PATCH 35/60] Write Python versions to rstudio-connect.gcfg on build --- connect/Dockerfile.ubuntu2204 | 6 ++++++ connect/rstudio-connect.gcfg | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/connect/Dockerfile.ubuntu2204 b/connect/Dockerfile.ubuntu2204 index 83e8c41f..f5608ba0 100644 --- a/connect/Dockerfile.ubuntu2204 +++ b/connect/Dockerfile.ubuntu2204 @@ -9,6 +9,10 @@ LABEL maintainer="RStudio Docker " COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh +ARG R_VERSION=4.2.0 +ARG R_VERSION_ALT=3.6.2 +ARG PYTHON_VERSION=3.9.5 +ARG PYTHON_VERSION_ALT=3.8.10 ARG RSC_VERSION=2023.06.0 SHELL [ "/bin/bash", "-o", "pipefail", "-c"] RUN apt-get update --fix-missing \ @@ -30,6 +34,8 @@ EXPOSE 3939/tcp ENV RSC_LICENSE "" ENV RSC_LICENSE_SERVER "" COPY rstudio-connect.gcfg /etc/rstudio-connect/rstudio-connect.gcfg +RUN sed -i "s/{{PYTHON_VERSION}}/${PYTHON_VERSION}/g" /etc/rstudio-connect/rstudio-connect.gcfg \ + && sed -i "s/{{PYTHON_VERSION_ALT}}/${PYTHON_VERSION_ALT}/g" /etc/rstudio-connect/rstudio-connect.gcfg VOLUME ["/data"] ENTRYPOINT ["tini", "--"] diff --git a/connect/rstudio-connect.gcfg b/connect/rstudio-connect.gcfg index 99407028..376d08c2 100644 --- a/connect/rstudio-connect.gcfg +++ b/connect/rstudio-connect.gcfg @@ -30,8 +30,8 @@ Provider = password [Python] Enabled = true -Executable = /opt/python/3.8.10/bin/python -Executable = /opt/python/3.9.5/bin/python +Executable = /opt/python/{{PYTHON_VERSION}}/bin/python +Executable = /opt/python/{{PYTHON_VERSION_ALT}}/bin/python [Quarto] Enabled = true From e572ed87ab6420990bf4f00a0757d646bac22ab7 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Jul 2023 12:12:28 -0700 Subject: [PATCH 36/60] Default push-image to false Modify logic eval on prerelease push --- .github/actions/build-test-scan-push/action.yaml | 2 +- .github/workflows/build-prerelease.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 9f3c2531..418083fa 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -35,7 +35,7 @@ inputs: type: string push-image: description: Flag to push image once built - default: true + default: false type: boolean use-ghcr: description: Flag to authenticate with GHCR.io diff --git a/.github/workflows/build-prerelease.yaml b/.github/workflows/build-prerelease.yaml index d2aa9c96..03fd7d3a 100644 --- a/.github/workflows/build-prerelease.yaml +++ b/.github/workflows/build-prerelease.yaml @@ -100,7 +100,7 @@ jobs: product: ${{ matrix.config.product }} image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} || ${{ github.ref == 'refs/heads/dev-rspm' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} From 617628c15d4a0092120adefc2249dad7d894cfaf Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Jul 2023 12:15:17 -0700 Subject: [PATCH 37/60] Remove push on base --- .github/workflows/build-release.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 56a6e574..e5a724b0 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -69,8 +69,7 @@ jobs: product: product-base image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - # push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} - push-image: true + push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} @@ -138,8 +137,7 @@ jobs: product: product-base-pro image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - # push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} - push-image: true + push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} From 71c8234dbc735b9e70be1339f2a70569f287b5a3 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Jul 2023 12:28:09 -0700 Subject: [PATCH 38/60] Debug image pushes --- .../actions/build-test-scan-push/action.yaml | 6 +++++- .github/workflows/build-prerelease.yaml | 7 ++++++- .github/workflows/build-release.yaml | 21 ++++++++++++++++--- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 418083fa..8ae1ca5a 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -140,11 +140,15 @@ runs: --app-vulns command: test + - name: Debug push-image + shell: bash + run: echo ${{ inputs.push-image }} + - name: Build uses: docker/build-push-action@v4 if: ${{ inputs.push-image }} with: - push: true + push: false context: ${{ inputs.context }} file: ${{ inputs.context }}/Dockerfile.${{ inputs.os }} cache-from: type=gha diff --git a/.github/workflows/build-prerelease.yaml b/.github/workflows/build-prerelease.yaml index 03fd7d3a..2da004e4 100644 --- a/.github/workflows/build-prerelease.yaml +++ b/.github/workflows/build-prerelease.yaml @@ -92,6 +92,11 @@ jobs: ) echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT + - name: Debug push-image + run: | + echo ${{ github.ref }} + echo ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + - name: Build/Test/Scan/Push base pro image uses: ./.github/actions/build-test-scan-push with: @@ -100,7 +105,7 @@ jobs: product: ${{ matrix.config.product }} image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} || ${{ github.ref == 'refs/heads/dev-rspm' }} + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index e5a724b0..bddcc1c4 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -61,6 +61,11 @@ jobs: ) echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT + - name: Debug push-image + run: | + echo ${{ github.ref }} + echo ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} + - name: Build/Test/Scan/Push base image uses: ./.github/actions/build-test-scan-push with: @@ -69,7 +74,7 @@ jobs: product: product-base image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} @@ -129,6 +134,11 @@ jobs: ) echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT + - name: Debug push-image + run: | + echo ${{ github.ref }} + echo ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} + - name: Build/Test/Scan/Push base pro image uses: ./.github/actions/build-test-scan-push with: @@ -137,7 +147,7 @@ jobs: product: product-base-pro image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' }} || ${{ github.ref == 'refs/heads/dev' }} + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} @@ -212,6 +222,11 @@ jobs: ) echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT + - name: Debug push-image + run: | + echo ${{ github.ref }} + echo ${{ github.ref == 'refs/heads/main' }} + - name: Build/Test/Scan/Push base pro image uses: ./.github/actions/build-test-scan-push with: @@ -220,7 +235,7 @@ jobs: product: ${{ matrix.config.product }} image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: ${{ github.ref == 'refs/heads/main' }} + push-image: false snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} From 19e3df946ff1293c24851d7ba9825978d3b5fcf5 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Jul 2023 12:52:24 -0700 Subject: [PATCH 39/60] Remove any reliance on `if` evaluations --- .../actions/build-test-scan-push/action.yaml | 64 +++++++------------ 1 file changed, 23 insertions(+), 41 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 8ae1ca5a..1678a591 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -23,10 +23,6 @@ inputs: description: Flag to test image once built default: true type: boolean - scan-image: - description: Flag to scan image using Snyk - default: true - type: boolean snyk-token: description: Username for authentication with Snyk for scanning images type: string @@ -37,22 +33,17 @@ inputs: description: Flag to push image once built default: false type: boolean - use-ghcr: - description: Flag to authenticate with GHCR.io - default: true - type: boolean ghcr-token: description: Username for authentication with GHCR.io + required: true type: string - use-dockerhub: - description: Flag to authenticate with Dockerhub - default: true - type: boolean dockerhub-username: description: Username for authentication with DockerHub + required: true type: string dockerhub-token: description: Username for authentication with DockerHub + required: true type: string runs: @@ -62,7 +53,6 @@ runs: uses: docker/setup-buildx-action@v2 - name: Login to ghcr.io - if: ${{ inputs.use-ghcr }} uses: docker/login-action@v2 with: registry: ghcr.io @@ -70,7 +60,6 @@ runs: password: ${{ inputs.ghcr-token }} - name: Login to Docker Hub - if: ${{ inputs.use-dockerhub }} uses: docker/login-action@v2 with: username: ${{ inputs.dockerhub-username }} @@ -99,32 +88,26 @@ runs: - name: Test shell: bash - if: ${{ inputs.test-image }} run: | - echo "${{ inputs.build-args }}" > ${{ inputs.context }}/.env - echo "OS=${{ inputs.os }}" >> ${{ inputs.context }}/.env - cat ${{ inputs.context }}/.env - IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut + if [[ "${{ inputs.test-image }}" == "true" ]]; then + echo "${{ inputs.build-args }}" > ${{ inputs.context }}/.env + echo "OS=${{ inputs.os }}" >> ${{ inputs.context }}/.env + cat ${{ inputs.context }}/.env + IMAGE_NAME=${{ steps.first-tag.outputs.FIRST_TAG }} docker-compose -f ${{ inputs.context }}/docker-compose.test.yml run sut + fi - - name: Run Snyk to check Docker image for vulnerabilities (main) - if: ${{ github.ref == 'refs/heads/main' }} && ${{ inputs.scan-image }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ inputs.snyk-token }} - with: - image: ${{ steps.first-tag.FIRST_TAG }} - args: | - --file=${{ inputs.context }}/Dockerfile.${{ inputs.os }} \ - --org=${{ inputs.snyk-org-id }} \ - --project-name=${{ steps.first-tag.FIRST_TAG }} \ - --tags=product=${{ inputs.product }},os=${{ inputs.os }} \ - --exclude-base-image-vulns \ - --app-vulns - command: monitor + - name: Evaluate Snyk command + id: eval-snyk-command + shell: bash + run: | + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + SNYK_COMMAND="monitor" + else + SNYK_COMMAND="test" + fi + echo "SNYK_COMMAND=$SNYK_COMMAND" >> $GITHUB_OUTPUT - - name: Run Snyk to check Docker image for vulnerabilities (branch) - if: ${{ github.ref != 'refs/heads/main' }} && ${{ inputs.scan-image }} + - name: Run Snyk to check Docker image for vulnerabilities continue-on-error: true uses: snyk/actions/docker@master env: @@ -134,11 +117,11 @@ runs: args: | --file=${{ inputs.context }}/Dockerfile.${{ inputs.os }} \ --org=${{ inputs.snyk-org-id }} \ - --project-name=${{ steps.get-default-tag.outputs.DEFAULT_TAG }} \ + --project-name=${{ steps.first-tag.FIRST_TAG }} \ --tags=product=${{ inputs.product }},os=${{ inputs.os }} \ --exclude-base-image-vulns \ --app-vulns - command: test + command: ${{ steps.eval-snyk-command.SNYK_COMMAND }} - name: Debug push-image shell: bash @@ -146,9 +129,8 @@ runs: - name: Build uses: docker/build-push-action@v4 - if: ${{ inputs.push-image }} with: - push: false + push: ${{ inputs.push-image }} context: ${{ inputs.context }} file: ${{ inputs.context }}/Dockerfile.${{ inputs.os }} cache-from: type=gha From d12c3dd56bb3d2b3cbe6fa69e735704736a91fd5 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Mon, 24 Jul 2023 13:09:18 -0700 Subject: [PATCH 40/60] Remove debugging statements --- .github/workflows/build-release.yaml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index bddcc1c4..30926c88 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -61,11 +61,6 @@ jobs: ) echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - name: Debug push-image - run: | - echo ${{ github.ref }} - echo ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - - name: Build/Test/Scan/Push base image uses: ./.github/actions/build-test-scan-push with: @@ -134,11 +129,6 @@ jobs: ) echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - name: Debug push-image - run: | - echo ${{ github.ref }} - echo ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - - name: Build/Test/Scan/Push base pro image uses: ./.github/actions/build-test-scan-push with: @@ -222,11 +212,6 @@ jobs: ) echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - name: Debug push-image - run: | - echo ${{ github.ref }} - echo ${{ github.ref == 'refs/heads/main' }} - - name: Build/Test/Scan/Push base pro image uses: ./.github/actions/build-test-scan-push with: @@ -235,7 +220,7 @@ jobs: product: ${{ matrix.config.product }} image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} - push-image: false + push-image: ${{ github.ref == 'refs/heads/main' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} From 25461c55166fb91290920cbcd946a2927329dc0b Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 07:06:05 -0700 Subject: [PATCH 41/60] Add manual build workflow --- .github/workflows/build-manual.yaml | 136 +++++++++++++++++++ .github/workflows/build-preview-webhook.yaml | 95 ------------- 2 files changed, 136 insertions(+), 95 deletions(-) create mode 100644 .github/workflows/build-manual.yaml delete mode 100644 .github/workflows/build-preview-webhook.yaml diff --git a/.github/workflows/build-manual.yaml b/.github/workflows/build-manual.yaml new file mode 100644 index 00000000..ac97f89f --- /dev/null +++ b/.github/workflows/build-manual.yaml @@ -0,0 +1,136 @@ +on: + workflow_dispatch: + inputs: + branch: + description: "Branch to build off." + default: "main" + type: choice + options: + - main + - dev + - dev-rspm + product: + description: "The product/path to build." + required: true + type: choice + options: + - connect + - connect-content-init + - content/base + - content/pro + - package-manager + - product/base + - product/pro + - r-session-complete + - workbench + - workbench-for-microsoft-azure-ml + os: + description: "Which OS to build. WARNING: Not all OSes may be present for all products." + required: false + default: "ubuntu2204" + type: choice + options: + - ubuntu2204 + - ubuntu1804 + - centos7 + type: + description: "The type of image being built." + required: false + default: "preview" + type: choice + options: + - preview + - daily + - release + version: + description: "The version to build. Use 'auto' to target the latest build." + required: false + default: "auto" + type: string + push: + description: "Flag to push the image after build." + required: false + default: false + type: boolean + +name: Manual - Build, Test, Scan, and Push +jobs: + build: + runs-on: ubuntu-latest + name: manual-build + + steps: + - name: Check Out Repo + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Get Version + id: get-version + run: | + if [[ "${{ inputs.version }}" == "auto" ]]; then + VERSION=`just -f ci.Justfile get-version ${{ inputs.product }} --type=${{ inputs.type }} --local` + else + VERSION="${{ inputs.version }}" + fi + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + + - name: Get build args + id: get-build-args + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + ARGS_CMD="" + if [[ "${{ inputs.type }}" == "release" ]]; then + ARGS_CMD="get-product-args" + else + ARGS_CMD="get-prerelease-args ${{inputs.type}}" + fi + BUILD_ARGS=$( \ + just -f ci.Justfile \ + ${ARGS_CMD} \ + ${{ inputs.product }} \ + ${{ inputs.os }} \ + ${{ steps.get-version.outputs.VERSION }} \ + ) + echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT + echo "$BUILD_ARGS" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + + - name: Get tags + id: get-tags + run: | + ARGS_CMD="" + if [[ "${{ inputs.type }}" == "release" ]]; then + ARGS_CMD="get-product-tags" + else + ARGS_CMD="get-prerelease-tags ${{inputs.type}}" + fi + IMAGE_TAGS=$( \ + just -f ci.Justfile \ + ${ARGS_CMD} \ + ${{ inputs.product }} \ + ${{ inputs.os }} \ + ${{ steps.get-version.outputs.VERSION }} \ + ) + echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT + + - name: Build/Test/Scan/Push base pro image + uses: ./.github/actions/build-test-scan-push + with: + context: ./${{ inputs.product }} + os: ${{ inputs.os }} + product: ${{ inputs.product }} + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} + push-image: ${{ inputs.push }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + diff --git a/.github/workflows/build-preview-webhook.yaml b/.github/workflows/build-preview-webhook.yaml deleted file mode 100644 index e48b3a97..00000000 --- a/.github/workflows/build-preview-webhook.yaml +++ /dev/null @@ -1,95 +0,0 @@ -on: - workflow_dispatch: - inputs: - product: - description: "Which image to build. Options: 'connect', 'connect-content-init', 'package-manager', 'r-session-complete', 'workbench', 'workbench-for-microsoft-azure-ml'." - required: true - type: string - os: - description: "Which os to build. Default 'bionic'. Options: 'bionic|ubuntu1804', 'jammy|ubuntu2204', 'centos7'" - required: true - default: "ubuntu2204" - type: string - type: - description: "Which build type. Default 'preview'. Options: 'preview', 'daily'" - required: false - default: "preview" - type: string - version: - description: "The version to build. Default 'auto'. Can use any string, but must have which != 'all' if using something other than 'auto'" - required: false - default: "auto" - type: string - -name: build/test/push (adhoc preview) -jobs: - build: - runs-on: ubuntu-latest - name: build-${{ github.event.inputs.product }}:${{ github.event.inputs.os }}-${{ github.event.inputs.type }}-${{ github.event.inputs.version }} - - steps: - - name: Check Out Repo - uses: actions/checkout@v3 - - - name: Set up Just - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - just-version: '1.11.0' - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Get Version - id: get-version - run: | - VERSION=`just -f ci.Justfile get-version ${{ github.event.inputs.product }} --type=${{ github.event.inputs.type }} --local --override=${{ github.event.inputs.version }}` - echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - - - name: Build Image - id: build-image - run: | - TAGS=`just -f ci.Justfile BUILDX_PATH=${{ steps.buildx.outputs.name }} build-preview ${{ github.event.inputs.type }} ${{ github.event.inputs.product }} ${{ github.event.inputs.os }} ${{ steps.get-version.outputs.VERSION }}` - echo "TAGS=$TAGS" >> $GITHUB_OUTPUT - - - name: Show image size - run: | - docker image ls - - - name: Test image - env: - RSC_LICENSE: ${{ secrets.RSC_LICENSE }} - RSPM_LICENSE: ${{ secrets.RSPM_LICENSE }} - RSW_LICENSE: ${{ secrets.RSW_LICENSE }} - run: | - just test-image ${{ github.event.inputs.product }} ${{ steps.get-version.outputs.VERSION }} ${{ steps.build-image.outputs.TAGS }} - - - name: Login to Docker Hub - if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Login to ghcr.io - if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.BUILD_PAT }} - - - name: Push image(s) to registries - if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} - run: | - just -f ci.Justfile push-images ${{ steps.build-image.outputs.TAGS }} From 88b1c2505ec7ef5c9f745f825ecab8539247e10c Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 07:17:27 -0700 Subject: [PATCH 42/60] Use build-test-scan-push action for content builds --- .github/workflows/build-content.yaml | 185 ++++----------------------- 1 file changed, 28 insertions(+), 157 deletions(-) diff --git a/.github/workflows/build-content.yaml b/.github/workflows/build-content.yaml index 79de5553..0fb1d2de 100644 --- a/.github/workflows/build-content.yaml +++ b/.github/workflows/build-content.yaml @@ -5,9 +5,8 @@ on: - dev pull_request: -name: build/push RStudio Content Images +name: Content Images - Build, Test, Scan, and Push jobs: - matrix: runs-on: ubuntu-latest outputs: @@ -22,7 +21,7 @@ jobs: build: runs-on: ubuntu-latest needs: matrix - name: r${{ matrix.config.r }} py${{ matrix.config.py }} ${{ matrix.config.os }} ${{ github.ref }} + name: content-base-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}--${{ github.ref }} strategy: fail-fast: false @@ -33,96 +32,31 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Build and Push - id: docker_build - uses: docker/build-push-action@v3 + - name: Build/Test/Scan/Push content base image + uses: ./.github/actions/build-test-scan-push with: context: ./content/base - file: ./content/base/Dockerfile.${{ matrix.config.os }} - builder: ${{ steps.buildx.outputs.name }} - tags: | - rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - ghcr.io/rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} + os: ${{ matrix.config.os }} + product: content-base + image-tags: | + rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }}, + ghcr.io/rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }}, + rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }}, ghcr.io/rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - load: true - push: false build-args: | R_VERSION=${{ matrix.config.r }} PYTHON_VERSION=${{ matrix.config.py }} - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - - - name: Show image size - run: | - docker image ls - - - name: Run Snyk to check Docker image for vulnerabilities (main) - if: ${{ github.ref == 'refs/heads/main' }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - args: --file=./content/base/Dockerfile.${{ matrix.config.os }} --org=${{ secrets.SNYK_ORG_ID }} --project-name=rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} --tags=product=content-base,os=${{ matrix.config.os }} --app-vulns --exclude-base-image-vulns - command: monitor - - - name: Run Snyk to check Docker image for vulnerabilities (branch) - if: ${{ github.ref != 'refs/heads/main' }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - args: --file=./content/base/Dockerfile.${{ matrix.config.os }} --org=${{ secrets.SNYK_ORG_ID }} --project-name=rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} --tags=product=content-base,os=${{ matrix.config.os }} --app-vulns --exclude-base-image-vulns - command: test - - - name: Login to Docker Hub - if: ${{ github.ref == 'refs/heads/main' }} - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Login to ghcr.io - if: ${{ github.ref == 'refs/heads/main' }} - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.BUILD_PAT }} - - - name: Push image(s) - if: ${{ github.ref == 'refs/heads/main' }} - run: | - docker push rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - docker push ghcr.io/rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - docker push rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} - docker push ghcr.io/rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} - - + push-image: ${{ github.ref == 'refs/heads/main' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} build-pro: runs-on: ubuntu-latest needs: [ matrix, build ] - name: pro r${{ matrix.config.r }} py${{ matrix.config.py }} ${{ matrix.config.os }} ${{ github.ref }} + name: content-pro-${{ matrix.config.os }}-r${{ matrix.config.r }}-py${{ matrix.config.py }}--${{ github.ref }} strategy: fail-fast: false @@ -133,88 +67,25 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Build and Push - id: docker_build - uses: docker/build-push-action@v3 + - name: Build/Test/Scan/Push content pro image + uses: ./.github/actions/build-test-scan-push with: context: ./content/pro - file: ./content/pro/Dockerfile.${{ matrix.config.os }} - builder: ${{ steps.buildx.outputs.name }} - tags: | + os: ${{ matrix.config.os }} + product: content-pro + image-tags: | rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} ghcr.io/rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} ghcr.io/rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - load: true - push: false build-args: | R_VERSION=${{ matrix.config.r }} PYTHON_VERSION=${{ matrix.config.py }} DRIVERS_VERSION=${{ matrix.config.drivers }} BASE_IMAGE=rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - - - name: Show image size - run: | - docker image ls - - - name: Run Snyk to check Docker image for vulnerabilities (main) - if: ${{ github.ref == 'refs/heads/main' }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - args: --file=./content/pro/${{ matrix.config.os }}/Dockerfile --org=${{ secrets.SNYK_ORG_ID }} --project-name=rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} --tags=product=content-pro,os=${{ matrix.config.os }} --app-vulns --exclude-base-image-vulns - command: monitor - - - name: Run Snyk to check Docker image for vulnerabilities (branch) - if: ${{ github.ref != 'refs/heads/main' }} - continue-on-error: true - uses: snyk/actions/docker@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - image: rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - args: --file=./content/pro/${{ matrix.config.os }}/Dockerfile --org=${{ secrets.SNYK_ORG_ID }} --project-name=rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} --tags=product=content-pro,os=${{ matrix.config.os }} --app-vulns --exclude-base-image-vulns - command: test - - - name: Login to Docker Hub - if: ${{ github.ref == 'refs/heads/main' }} - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Login to ghcr.io - if: ${{ github.ref == 'refs/heads/main' }} - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.BUILD_PAT }} - - - name: Push image(s) - if: ${{ github.ref == 'refs/heads/main' }} - run: | - docker push rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - docker push ghcr.io/rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - docker push rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} - docker push ghcr.io/rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} - + push-image: ${{ github.ref == 'refs/heads/main' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} From bb402a32f667de80a0e8fad9372a116e067eba6f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 07:33:10 -0700 Subject: [PATCH 43/60] Trim newlines in image tags Disable testing for content --- .github/actions/build-test-scan-push/action.yaml | 3 ++- .github/workflows/build-content.yaml | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 1678a591..00c5401d 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -82,7 +82,8 @@ runs: shell: bash id: first-tag run: | - FIRST_TAG=$(cut -d "," -f 1 <<< "${{ inputs.image-tags }}") + IMG_TAGS="${{ inputs.image-tags }}" + FIRST_TAG=$(cut -d "," -f 1 <<< "${IMG_TAGS//$'\n'/}") echo "$FIRST_TAG" echo "FIRST_TAG=$FIRST_TAG" >> $GITHUB_OUTPUT diff --git a/.github/workflows/build-content.yaml b/.github/workflows/build-content.yaml index 0fb1d2de..09e826a1 100644 --- a/.github/workflows/build-content.yaml +++ b/.github/workflows/build-content.yaml @@ -46,6 +46,7 @@ jobs: build-args: | R_VERSION=${{ matrix.config.r }} PYTHON_VERSION=${{ matrix.config.py }} + test-image: false push-image: ${{ github.ref == 'refs/heads/main' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} @@ -83,6 +84,7 @@ jobs: PYTHON_VERSION=${{ matrix.config.py }} DRIVERS_VERSION=${{ matrix.config.drivers }} BASE_IMAGE=rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} + test-image: false push-image: ${{ github.ref == 'refs/heads/main' }} snyk-token: ${{ secrets.SNYK_TOKEN }} snyk-org-id: ${{ secrets.SNYK_ORG_ID }} From b49a85e774a9480b95b43cfae1b172bedbb6f11f Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 09:13:23 -0700 Subject: [PATCH 44/60] Revert sweeping release image upgrades --- .github/actions/build-test-scan-push/action.yaml | 10 +++------- .github/workflows/build-release.yaml | 12 ++++++------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 00c5401d..aaff490d 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -87,7 +87,7 @@ runs: echo "$FIRST_TAG" echo "FIRST_TAG=$FIRST_TAG" >> $GITHUB_OUTPUT - - name: Test + - name: Test - ${{ inputs.test-image }} shell: bash run: | if [[ "${{ inputs.test-image }}" == "true" ]]; then @@ -108,7 +108,7 @@ runs: fi echo "SNYK_COMMAND=$SNYK_COMMAND" >> $GITHUB_OUTPUT - - name: Run Snyk to check Docker image for vulnerabilities + - name: Run Snyk ${{ steps.eval-snyk-command.SNYK_COMMAND }} continue-on-error: true uses: snyk/actions/docker@master env: @@ -124,11 +124,7 @@ runs: --app-vulns command: ${{ steps.eval-snyk-command.SNYK_COMMAND }} - - name: Debug push-image - shell: bash - run: echo ${{ inputs.push-image }} - - - name: Build + - name: Push - ${{ inputs.push-image }} uses: docker/build-push-action@v4 with: push: ${{ inputs.push-image }} diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 30926c88..56402177 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -153,14 +153,14 @@ jobs: fail-fast: false matrix: config: - - {product: 'workbench', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} - - {product: 'connect', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} - - {product: 'connect-content-init', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: 'workbench', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} + - {product: 'connect', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} + - {product: 'connect-content-init', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - {product: 'package-manager', os: 'ubuntu1804', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - - {product: 'package-manager', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: 'package-manager', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} - {product: 'r-session-complete', os: 'centos7', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.14", py-alternate: "3.8.15"} - - {product: 'r-session-complete', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} - - {product: 'workbench-for-microsoft-azure-ml', os: 'ubuntu2204', r-primary: "4.3.1", r-alternate: "4.2.3", py-primary: "3.11.4", py-alternate: "3.10.12"} + - {product: 'r-session-complete', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} + - {product: 'workbench-for-microsoft-azure-ml', os: 'ubuntu2204', r-primary: "4.2.3", r-alternate: "4.1.3", py-primary: "3.9.17", py-alternate: "3.8.17"} steps: - name: Check Out Repo From f8e498259cede89343dfa66f0b6832985d313768 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 09:51:25 -0700 Subject: [PATCH 45/60] Write R version to `rstudio-pm.gcfg` on build --- package-manager/Dockerfile.ubuntu1804 | 3 ++- package-manager/Dockerfile.ubuntu2204 | 3 ++- package-manager/rstudio-pm.gcfg | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/package-manager/Dockerfile.ubuntu1804 b/package-manager/Dockerfile.ubuntu1804 index 6217f999..d322f871 100644 --- a/package-manager/Dockerfile.ubuntu1804 +++ b/package-manager/Dockerfile.ubuntu1804 @@ -49,8 +49,9 @@ RUN mkdir -p /var/run/rstudio-pm \ && chown rstudio-pm:rstudio-pm /usr/local/bin/startup.sh \ && chown -R rstudio-pm:rstudio-pm /var/run/rstudio-pm -USER rstudio-pm COPY rstudio-pm.gcfg /etc/rstudio-pm/rstudio-pm.gcfg +RUN sed -i "s/{{R_VERSION}}/${R_VERSION}/g" /etc/rstudio-pm/rstudio-pm.gcfg +USER rstudio-pm # Set up licensing to work in userspace mode. This will not prevent activating a # license as root, but it is required to activate one as the non-root user at diff --git a/package-manager/Dockerfile.ubuntu2204 b/package-manager/Dockerfile.ubuntu2204 index ec29a59a..14ad33c2 100644 --- a/package-manager/Dockerfile.ubuntu2204 +++ b/package-manager/Dockerfile.ubuntu2204 @@ -49,8 +49,9 @@ RUN mkdir -p /var/run/rstudio-pm \ && chown rstudio-pm:rstudio-pm /usr/local/bin/startup.sh \ && chown -R rstudio-pm:rstudio-pm /var/run/rstudio-pm -USER rstudio-pm COPY rstudio-pm.gcfg /etc/rstudio-pm/rstudio-pm.gcfg +RUN sed -i "s/{{R_VERSION}}/${R_VERSION}/g" /etc/rstudio-pm/rstudio-pm.gcfg +USER rstudio-pm # Set up licensing to work in userspace mode. This will not prevent activating a # license as root, but it is required to activate one as the non-root user at diff --git a/package-manager/rstudio-pm.gcfg b/package-manager/rstudio-pm.gcfg index 1f1c62cf..67300acb 100644 --- a/package-manager/rstudio-pm.gcfg +++ b/package-manager/rstudio-pm.gcfg @@ -11,7 +11,7 @@ Address = ; ; Git sources require a configured R installation. R is often installed at `/usr/lib/R` ; or `/usr/lib64/R`. -RVersion = /opt/R/4.2.3/ +RVersion = /opt/R/{{R_VERSION}}/ ; ; Customize the data directory if necessary. This is where all packages and metadata are ; stored by default. Refer to Admin Guide for details. From 58e6e39cd6d84327725faf93c1b0613b44af83a7 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 10:10:09 -0700 Subject: [PATCH 46/60] Fix blindspots in Justfile update-versions Update all versions after changes --- Justfile | 59 ++++++++++++++++----- connect/Dockerfile.ubuntu2204 | 16 +++--- package-manager/.env | 2 +- package-manager/Dockerfile.ubuntu1804 | 4 +- package-manager/Dockerfile.ubuntu1804.draft | 6 +-- package-manager/Dockerfile.ubuntu2204 | 4 +- package-manager/Dockerfile.ubuntu2204.draft | 6 +-- package-manager/Justfile | 6 +-- product/base/Dockerfile.centos7 | 4 +- product/base/Dockerfile.ubuntu1804 | 8 +-- product/base/Dockerfile.ubuntu2204 | 8 +-- product/base/Justfile | 8 +-- product/pro/Dockerfile.centos7 | 8 +-- product/pro/Dockerfile.ubuntu1804 | 16 +++--- product/pro/Dockerfile.ubuntu2204 | 16 +++--- product/pro/Justfile | 8 +-- r-session-complete/Dockerfile.ubuntu2204 | 8 +-- workbench/Dockerfile.ubuntu2204 | 8 +-- 18 files changed, 114 insertions(+), 81 deletions(-) diff --git a/Justfile b/Justfile index 98aaf520..0076ea72 100644 --- a/Justfile +++ b/Justfile @@ -18,6 +18,8 @@ R_VERSION_ALT := "4.1.3" PYTHON_VERSION := "3.9.17" PYTHON_VERSION_ALT := "3.8.17" +PYTHON_VERSION_RHEL := "3.9.14" +PYTHON_VERSION_ALT_RHEL := "3.8.15" QUARTO_VERSION := "1.3.340" @@ -89,6 +91,8 @@ update-versions: R_VERSION_ALT={{R_VERSION_ALT}} \ PYTHON_VERSION={{PYTHON_VERSION}} \ PYTHON_VERSION_ALT={{PYTHON_VERSION_ALT}} \ + PYTHON_VERSION_RHEL={{PYTHON_VERSION_RHEL}} \ + PYTHON_VERSION_ALT_RHEL={{PYTHON_VERSION_ALT_RHEL}} \ DRIVERS_VERSION={{DRIVERS_VERSION}} \ QUARTO_VERSION={{QUARTO_VERSION}} \ update-rsw-versions update-rspm-versions update-rsc-versions update-r-versions update-py-versions update-drivers-versions update-quarto-versions @@ -154,32 +158,42 @@ update-r-versions: #!/usr/bin/env bash set -euxo pipefail # Update primary R versions - sed {{ sed_vars }} "s/^R_VERSION=.*/R_VERSION={{ R_VERSION }}/g" \ + sed {{ sed_vars }} "s/R_VERSION=.*/R_VERSION={{ R_VERSION }}/g" \ workbench/.env \ connect/.env \ package-manager/.env \ - package-manager/Dockerfile.ubuntu1804 \ + package-manager/Dockerfile.ubuntu* \ workbench/Dockerfile.ubuntu2204 \ connect/Dockerfile.ubuntu2204 \ - package-manager/Dockerfile.ubuntu2204 - sed {{ sed_vars }} "s|^RVersion.*=.*|RVersion = /opt/R/{{ R_VERSION }}/|g" package-manager/rstudio-pm.gcfg + product/base/Dockerfile.ubuntu* \ + product/pro/Dockerfile.ubuntu* sed {{ sed_vars }} "s/^R_VERSION := .*/R_VERSION := \"{{ R_VERSION }}\"/g" \ workbench/Justfile \ workbench-for-microsoft-azure-ml/Justfile \ - connect/Justfile package-manager/Justfile \ + connect/Justfile \ + package-manager/Justfile \ + product/base/Justfile \ + product/pro/Justfile \ Justfile \ ci.Justfile # Update alt R versions - sed {{ sed_vars }} "s/^R_VERSION_ALT=.*/R_VERSION_ALT={{ R_VERSION_ALT }}/g" \ + sed {{ sed_vars }} "s/R_VERSION_ALT=.*/R_VERSION_ALT={{ R_VERSION_ALT }}/g" \ workbench/.env \ connect/.env \ + package-manager/.env \ + package-manager/Dockerfile.ubuntu* \ workbench/Dockerfile.ubuntu2204 \ - connect/Dockerfile.ubuntu2204 + connect/Dockerfile.ubuntu2204 \ + product/base/Dockerfile.ubuntu* \ + product/pro/Dockerfile.ubuntu* sed {{ sed_vars }} "s/^R_VERSION_ALT := .*/R_VERSION_ALT := \"{{ R_VERSION_ALT }}\"/g" \ workbench/Justfile \ workbench-for-microsoft-azure-ml/Justfile \ connect/Justfile \ + package-manager/Justfile \ + product/base/Justfile \ + product/pro/Justfile \ Justfile \ ci.Justfile @@ -188,31 +202,50 @@ update-py-versions: #!/usr/bin/env bash set -euxo pipefail # Update primary Python versions - sed {{ sed_vars }} "s/^PYTHON_VERSION=.*/PYTHON_VERSION={{ PYTHON_VERSION }}/g" \ + sed {{ sed_vars }} "s/PYTHON_VERSION=.*/PYTHON_VERSION={{ PYTHON_VERSION }}/g" \ workbench/Dockerfile.ubuntu2204 \ workbench/.env \ connect/Dockerfile.ubuntu2204 \ connect/.env \ - package-manager/Dockerfile.ubuntu1804 \ - package-manager/Dockerfile.ubuntu2204 \ - package-manager/.env + package-manager/Dockerfile.ubuntu* \ + package-manager/.env \ + product/base/Dockerfile.ubuntu* \ + product/pro/Dockerfile.ubuntu* \ + r-session-complete/Dockerfile.ubuntu2204 + sed {{ sed_vars }} "s/PYTHON_VERSION=.*/PYTHON_VERSION={{ PYTHON_VERSION_RHEL }}/g" \ + product/base/Dockerfile.centos7 \ + product/pro/Dockerfile.centos7 \ + r-session-complete/Dockerfile.centos7 sed {{ sed_vars }} "s/^PYTHON_VERSION := .*/PYTHON_VERSION := \"{{ PYTHON_VERSION }}\"/g" \ workbench/Justfile \ workbench-for-microsoft-azure-ml/Justfile \ connect/Justfile \ + package-manager/Justfile \ + product/base/Justfile \ + product/pro/Justfile \ Justfile \ ci.Justfile # Update alt Python versions - sed {{ sed_vars }} "s/^PYTHON_VERSION_ALT=.*/PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }}/g" \ + sed {{ sed_vars }} "s/PYTHON_VERSION_ALT=.*/PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }}/g" \ workbench/Dockerfile.ubuntu2204 \ workbench/.env \ connect/Dockerfile.ubuntu2204 \ - connect/.env + connect/.env \ + product/base/Dockerfile.ubuntu* \ + product/pro/Dockerfile.ubuntu* \ + r-session-complete/Dockerfile.ubuntu2204 + sed {{ sed_vars }} "s/PYTHON_VERSION_ALT=.*/PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT_RHEL }}/g" \ + product/base/Dockerfile.centos7 \ + product/pro/Dockerfile.centos7 \ + r-session-complete/Dockerfile.centos7 sed {{ sed_vars }} "s/^PYTHON_VERSION_ALT := .*/PYTHON_VERSION_ALT := \"{{ PYTHON_VERSION_ALT }}\"/g" \ workbench/Justfile \ workbench-for-microsoft-azure-ml/Justfile \ connect/Justfile \ + package-manager/Justfile \ + product/base/Justfile \ + product/pro/Justfile \ Justfile \ ci.Justfile diff --git a/connect/Dockerfile.ubuntu2204 b/connect/Dockerfile.ubuntu2204 index f5608ba0..75968777 100644 --- a/connect/Dockerfile.ubuntu2204 +++ b/connect/Dockerfile.ubuntu2204 @@ -1,7 +1,7 @@ -ARG R_VERSION=4.2.0 -ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 -ARG PYTHON_VERSION_ALT=3.8.10 +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} @@ -9,10 +9,10 @@ LABEL maintainer="RStudio Docker " COPY --chmod=0775 startup.sh /usr/local/bin/startup.sh -ARG R_VERSION=4.2.0 -ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 -ARG PYTHON_VERSION_ALT=3.8.10 +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG RSC_VERSION=2023.06.0 SHELL [ "/bin/bash", "-o", "pipefail", "-c"] RUN apt-get update --fix-missing \ diff --git a/package-manager/.env b/package-manager/.env index 05c45629..4c45a9de 100644 --- a/package-manager/.env +++ b/package-manager/.env @@ -1,3 +1,3 @@ RSPM_VERSION=2023.04.0-6 R_VERSION=4.2.3 -R_VERSION_ALT=3.6.2 +R_VERSION_ALT=4.1.3 diff --git a/package-manager/Dockerfile.ubuntu1804 b/package-manager/Dockerfile.ubuntu1804 index d322f871..9ef26d0a 100644 --- a/package-manager/Dockerfile.ubuntu1804 +++ b/package-manager/Dockerfile.ubuntu1804 @@ -1,4 +1,4 @@ -ARG R_VERSION=4.2.0 +ARG R_VERSION=4.2.3 FROM rstudio/r-base:${R_VERSION}-bionic LABEL maintainer="RStudio Docker " @@ -18,7 +18,7 @@ RUN curl -L -o /usr/local/bin/tini https://github.com/krallin/tini/releases/down # Add another R version -------------------------------------------------------# -ARG R_VERSION_ALT=3.6.2 +ARG R_VERSION_ALT=4.1.3 RUN apt-get update -qq && \ curl -O https://cdn.rstudio.com/r/ubuntu-1804/pkgs/r-${R_VERSION_ALT}_1_amd64.deb && \ DEBIAN_FRONTEND=noninteractive apt-get install -y ./r-${R_VERSION_ALT}_1_amd64.deb && \ diff --git a/package-manager/Dockerfile.ubuntu1804.draft b/package-manager/Dockerfile.ubuntu1804.draft index d08e962b..88507a1d 100644 --- a/package-manager/Dockerfile.ubuntu1804.draft +++ b/package-manager/Dockerfile.ubuntu1804.draft @@ -1,6 +1,6 @@ -ARG R_VERSION=4.2.0 -ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.10 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io diff --git a/package-manager/Dockerfile.ubuntu2204 b/package-manager/Dockerfile.ubuntu2204 index 14ad33c2..1b5a26b0 100644 --- a/package-manager/Dockerfile.ubuntu2204 +++ b/package-manager/Dockerfile.ubuntu2204 @@ -1,4 +1,4 @@ -ARG R_VERSION=4.2.0 +ARG R_VERSION=4.2.3 FROM rstudio/r-base:${R_VERSION}-jammy LABEL maintainer="RStudio Docker " @@ -18,7 +18,7 @@ RUN curl -L -o /usr/local/bin/tini https://github.com/krallin/tini/releases/down # Add another R version -------------------------------------------------------# -ARG R_VERSION_ALT=3.6.2 +ARG R_VERSION_ALT=4.1.3 RUN apt-get update -qq && \ curl -O https://cdn.rstudio.com/r/ubuntu-2204/pkgs/r-${R_VERSION_ALT}_1_amd64.deb && \ DEBIAN_FRONTEND=noninteractive apt-get install -y ./r-${R_VERSION_ALT}_1_amd64.deb && \ diff --git a/package-manager/Dockerfile.ubuntu2204.draft b/package-manager/Dockerfile.ubuntu2204.draft index 20b47705..67b63c97 100644 --- a/package-manager/Dockerfile.ubuntu2204.draft +++ b/package-manager/Dockerfile.ubuntu2204.draft @@ -1,6 +1,6 @@ -ARG R_VERSION=4.2.0 -ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 ARG PYTHON_VERSION_ALT=3.8.10 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io diff --git a/package-manager/Justfile b/package-manager/Justfile index 9a08b6f0..4df263ed 100644 --- a/package-manager/Justfile +++ b/package-manager/Justfile @@ -11,10 +11,10 @@ RSPM_LICENSE := "" RSPM_LICENSE_SERVER := "" R_VERSION := "4.2.3" -R_VERSION_ALT := "4.1.0" +R_VERSION_ALT := "4.1.3" -PYTHON_VERSION := "3.9.5" -PYTHON_VERSION_ALT := "3.8.10" +PYTHON_VERSION := "3.9.17" +PYTHON_VERSION_ALT := "3.8.17" PERSIST_LICENSE := "false" PERSIST_LICENSE_DIR := join(justfile_directory(), "tmp-lic") diff --git a/product/base/Dockerfile.centos7 b/product/base/Dockerfile.centos7 index cd9201ed..aa15f9c8 100644 --- a/product/base/Dockerfile.centos7 +++ b/product/base/Dockerfile.centos7 @@ -4,8 +4,8 @@ LABEL maintainer="Posit Docker " ### ARG declarations ### ARG R_VERSION=4.2.0 ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 -ARG PYTHON_VERSION_ALT=3.8.10 +ARG PYTHON_VERSION=3.9.14 +ARG PYTHON_VERSION_ALT=3.8.15 ARG TINI_VERSION=0.19.0 ARG QUARTO_VERSION=1.3.340 diff --git a/product/base/Dockerfile.ubuntu1804 b/product/base/Dockerfile.ubuntu1804 index 75a860c3..8c338887 100644 --- a/product/base/Dockerfile.ubuntu1804 +++ b/product/base/Dockerfile.ubuntu1804 @@ -3,10 +3,10 @@ LABEL maintainer="Posit Docker " ### ARG declarations ### ARG DEBIAN_FRONTEND=noninteractive -ARG R_VERSION=4.2.0 -ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 -ARG PYTHON_VERSION_ALT=3.8.10 +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG TINI_VERSION=0.19.0 ARG QUARTO_VERSION=1.3.340 diff --git a/product/base/Dockerfile.ubuntu2204 b/product/base/Dockerfile.ubuntu2204 index 49a8029f..b5f9daec 100644 --- a/product/base/Dockerfile.ubuntu2204 +++ b/product/base/Dockerfile.ubuntu2204 @@ -3,10 +3,10 @@ LABEL maintainer="Posit Docker " ### ARG declarations ### ARG DEBIAN_FRONTEND=noninteractive -ARG R_VERSION=4.2.0 -ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 -ARG PYTHON_VERSION_ALT=3.8.10 +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG TINI_VERSION=0.19.0 ARG QUARTO_VERSION=1.3.340 diff --git a/product/base/Justfile b/product/base/Justfile index 30626cae..b011a4ce 100755 --- a/product/base/Justfile +++ b/product/base/Justfile @@ -7,11 +7,11 @@ IMAGE_OS := "ubuntu1804" IMAGE_REGISTRY := "rstudio" -R_VERSION := "4.1.0" -R_VERSION_ALT := "3.6.2" +R_VERSION := "4.2.3" +R_VERSION_ALT := "4.1.3" -PYTHON_VERSION := "3.9.5" -PYTHON_VERSION_ALT := "3.8.10" +PYTHON_VERSION := "3.9.17" +PYTHON_VERSION_ALT := "3.8.17" TINI_VERSION := "0.19.0" QUARTO_VERSION := "1.3.340" diff --git a/product/pro/Dockerfile.centos7 b/product/pro/Dockerfile.centos7 index 1db485e2..641a3fd4 100644 --- a/product/pro/Dockerfile.centos7 +++ b/product/pro/Dockerfile.centos7 @@ -1,7 +1,7 @@ ARG R_VERSION=4.2.0 ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 -ARG PYTHON_VERSION_ALT=3.8.10 +ARG PYTHON_VERSION=3.9.14 +ARG PYTHON_VERSION_ALT=3.8.15 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:centos7-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} @@ -9,8 +9,8 @@ LABEL maintainer="Posit Docker " ARG R_VERSION=4.2.0 ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 -ARG PYTHON_VERSION_ALT=3.8.10 +ARG PYTHON_VERSION=3.9.14 +ARG PYTHON_VERSION_ALT=3.8.15 ARG DRIVERS_VERSION=2023.05.0-1 RUN yum update -y \ diff --git a/product/pro/Dockerfile.ubuntu1804 b/product/pro/Dockerfile.ubuntu1804 index 3c0c5ad5..cca5be57 100644 --- a/product/pro/Dockerfile.ubuntu1804 +++ b/product/pro/Dockerfile.ubuntu1804 @@ -1,17 +1,17 @@ -ARG R_VERSION=4.2.0 -ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 -ARG PYTHON_VERSION_ALT=3.8.10 +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu1804-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} LABEL maintainer="Posit Docker " ARG DEBIAN_FRONTEND=noninteractive -ARG R_VERSION=4.2.0 -ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 -ARG PYTHON_VERSION_ALT=3.8.10 +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG DRIVERS_VERSION=2023.05.0 RUN apt-get update \ diff --git a/product/pro/Dockerfile.ubuntu2204 b/product/pro/Dockerfile.ubuntu2204 index 2c6f9844..10f0981c 100644 --- a/product/pro/Dockerfile.ubuntu2204 +++ b/product/pro/Dockerfile.ubuntu2204 @@ -1,17 +1,17 @@ -ARG R_VERSION=4.2.0 -ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 -ARG PYTHON_VERSION_ALT=3.8.10 +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base ARG REGISTRY=ghcr.io FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} LABEL maintainer="Posit Docker " ARG DEBIAN_FRONTEND=noninteractive -ARG R_VERSION=4.2.0 -ARG R_VERSION_ALT=3.6.2 -ARG PYTHON_VERSION=3.9.5 -ARG PYTHON_VERSION_ALT=3.8.10 +ARG R_VERSION=4.2.3 +ARG R_VERSION_ALT=4.1.3 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG DRIVERS_VERSION=2023.05.0 RUN apt-get update \ diff --git a/product/pro/Justfile b/product/pro/Justfile index 1dab9599..f67f5ef1 100644 --- a/product/pro/Justfile +++ b/product/pro/Justfile @@ -7,11 +7,11 @@ IMAGE_OS := "ubuntu1804" IMAGE_REGISTRY := "rstudio" -R_VERSION := "4.1.0" -R_VERSION_ALT := "3.6.2" +R_VERSION := "4.2.3" +R_VERSION_ALT := "4.1.3" -PYTHON_VERSION := "3.9.5" -PYTHON_VERSION_ALT := "3.8.10" +PYTHON_VERSION := "3.9.17" +PYTHON_VERSION_ALT := "3.8.17" TINI_VERSION := "0.19.0" QUARTO_VERSION := "1.3.340" diff --git a/r-session-complete/Dockerfile.ubuntu2204 b/r-session-complete/Dockerfile.ubuntu2204 index 14618c87..264c0f9e 100644 --- a/r-session-complete/Dockerfile.ubuntu2204 +++ b/r-session-complete/Dockerfile.ubuntu2204 @@ -1,7 +1,7 @@ ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.9.14 -ARG PYTHON_VERSION_ALT=3.8.15 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} @@ -10,8 +10,8 @@ LABEL maintainer="RStudio Docker " ARG DEBIAN_FRONTEND=noninteractive ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.9.14 -ARG PYTHON_VERSION_ALT=3.8.15 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG RSW_VERSION=2023.06.1+524.pro1 ARG RSW_NAME=rstudio-workbench ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/jammy/amd64 diff --git a/workbench/Dockerfile.ubuntu2204 b/workbench/Dockerfile.ubuntu2204 index c1fe7614..81b3d839 100644 --- a/workbench/Dockerfile.ubuntu2204 +++ b/workbench/Dockerfile.ubuntu2204 @@ -1,7 +1,7 @@ ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.9.14 -ARG PYTHON_VERSION_ALT=3.8.15 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} @@ -10,8 +10,8 @@ LABEL maintainer="RStudio Docker " ARG DEBIAN_FRONTEND=noninteractive ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.9.14 -ARG PYTHON_VERSION_ALT=3.8.15 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG PYTHON_VERSION_JUPYTER=3.8.15 ARG RSW_VERSION=2023.06.1+524.pro1 ARG RSW_NAME=rstudio-workbench From 4e6e65bcb372d5f4b041877db5de28946d60267e Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 11:18:36 -0700 Subject: [PATCH 47/60] Update NEWS.md docs --- NEWS.md | 8 ++++++++ connect/NEWS.md | 3 +++ package-manager/NEWS.md | 3 +++ 3 files changed, 14 insertions(+) diff --git a/NEWS.md b/NEWS.md index 57e3f394..b71783f5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,14 @@ changed in each image. This file only captures pervasive, repository-wide changes. +# 2023-07-25 +- Overhauled the workflows for this repository to use official Github Actions over `just` targets for building, testing, +scanning, and pushing images. +- Universally updated Python versions to 3.9.17 and 3.8.17 for Ubuntu-based images and 3.9.14 and 3.8.15 for CentOS +images. +- **BREAKING:** Universally updated R versions to 4.2.3 and 4.1.3 from 3.6.2 and 4.2.0. +- Added scheduled builds for latest release images to keep images up to date with security patches. + # 2022-11-10 - We replaced Ubuntu codenames with explicit version numbers for ease of use. Images will still be tagged with *both* the OS version number and the codename to retain backwards compatibility. diff --git a/connect/NEWS.md b/connect/NEWS.md index 5710060f..691712a0 100644 --- a/connect/NEWS.md +++ b/connect/NEWS.md @@ -1,3 +1,6 @@ +# 2023-07-25 +- Changed `rstudio-connect.gcfg` Python version numbers behavior from statically defined to dynamically filled on build. + # 2023-07-10 - BREAKING: Deprecate the Ubuntu 18.04 (Bionic Beaver) images. diff --git a/package-manager/NEWS.md b/package-manager/NEWS.md index e59ffc4e..419b0847 100644 --- a/package-manager/NEWS.md +++ b/package-manager/NEWS.md @@ -1,3 +1,6 @@ +# 2023-07-25 +- Changed `rstudio-pm.gcfg` R version number behavior from statically defined to dynamically filled on build. + # 2022.11.4-20 - Mid-release, we accidentally bumped the version of R. To mitigate issues, we added another version of R, From 555f5f5950cd7e59639b9f2889bc8e97eefb8bf5 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 11:40:23 -0700 Subject: [PATCH 48/60] Always order tags from most specific to most generic --- ci.Justfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci.Justfile b/ci.Justfile index e99629f3..5f0d59e9 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -101,9 +101,9 @@ get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: IMAGE_NAME="${IMAGE_NAME}-dev" fi - echo ghcr.io/rstudio/${IMAGE_NAME}:${OS},\ + echo ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}},\ ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}},\ - ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}_{{R_VERSION_ALT}}-py{{PYTHON_VERSION}}_{{PYTHON_VERSION_ALT}} + ghcr.io/rstudio/${IMAGE_NAME}:${OS} get-product-args $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`git rev-parse --short HEAD`: #!/usr/bin/env bash @@ -174,12 +174,12 @@ get-product-tags $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`g for os_name in ${OS_ALIASES[@]}; do tag_array+=( - "rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}" - "rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}" "rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}--${SHA_SHORT}" - "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}" - "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}" + "rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}" + "rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}" "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}--${SHA_SHORT}" + "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}-${TAG_CLEAN_VERSION}" + "ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${os_name}" ) done tags=$(IFS="," ; echo "${tag_array[*]}") From aa61b557e2bfa00c05107e1e57a493e6187f2a4c Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 12:01:04 -0700 Subject: [PATCH 49/60] Fix permissions on rstudio-pm.gcfg --- package-manager/Dockerfile.ubuntu1804 | 3 +++ package-manager/Dockerfile.ubuntu2204 | 3 +++ 2 files changed, 6 insertions(+) diff --git a/package-manager/Dockerfile.ubuntu1804 b/package-manager/Dockerfile.ubuntu1804 index 9ef26d0a..6ac6fb02 100644 --- a/package-manager/Dockerfile.ubuntu1804 +++ b/package-manager/Dockerfile.ubuntu1804 @@ -49,8 +49,11 @@ RUN mkdir -p /var/run/rstudio-pm \ && chown rstudio-pm:rstudio-pm /usr/local/bin/startup.sh \ && chown -R rstudio-pm:rstudio-pm /var/run/rstudio-pm +USER rstudio-pm +USER root COPY rstudio-pm.gcfg /etc/rstudio-pm/rstudio-pm.gcfg RUN sed -i "s/{{R_VERSION}}/${R_VERSION}/g" /etc/rstudio-pm/rstudio-pm.gcfg +RUN chown rstudio-pm:rstudio-pm /etc/rstudio-pm/rstudio-pm.gcfg USER rstudio-pm # Set up licensing to work in userspace mode. This will not prevent activating a diff --git a/package-manager/Dockerfile.ubuntu2204 b/package-manager/Dockerfile.ubuntu2204 index 1b5a26b0..258cd36e 100644 --- a/package-manager/Dockerfile.ubuntu2204 +++ b/package-manager/Dockerfile.ubuntu2204 @@ -49,8 +49,11 @@ RUN mkdir -p /var/run/rstudio-pm \ && chown rstudio-pm:rstudio-pm /usr/local/bin/startup.sh \ && chown -R rstudio-pm:rstudio-pm /var/run/rstudio-pm +USER rstudio-pm +USER root COPY rstudio-pm.gcfg /etc/rstudio-pm/rstudio-pm.gcfg RUN sed -i "s/{{R_VERSION}}/${R_VERSION}/g" /etc/rstudio-pm/rstudio-pm.gcfg +RUN chown rstudio-pm:rstudio-pm /etc/rstudio-pm/rstudio-pm.gcfg USER rstudio-pm # Set up licensing to work in userspace mode. This will not prevent activating a From 0753ccdbd10bfd951ad2933ec534384467e138be Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 13:02:06 -0700 Subject: [PATCH 50/60] Revert changes for dynamic rstudio-pm.gcfg --- package-manager/Dockerfile.ubuntu1804 | 4 ---- package-manager/Dockerfile.ubuntu2204 | 4 ---- package-manager/rstudio-pm.gcfg | 2 +- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/package-manager/Dockerfile.ubuntu1804 b/package-manager/Dockerfile.ubuntu1804 index 6ac6fb02..dfd1b537 100644 --- a/package-manager/Dockerfile.ubuntu1804 +++ b/package-manager/Dockerfile.ubuntu1804 @@ -50,11 +50,7 @@ RUN mkdir -p /var/run/rstudio-pm \ && chown -R rstudio-pm:rstudio-pm /var/run/rstudio-pm USER rstudio-pm -USER root COPY rstudio-pm.gcfg /etc/rstudio-pm/rstudio-pm.gcfg -RUN sed -i "s/{{R_VERSION}}/${R_VERSION}/g" /etc/rstudio-pm/rstudio-pm.gcfg -RUN chown rstudio-pm:rstudio-pm /etc/rstudio-pm/rstudio-pm.gcfg -USER rstudio-pm # Set up licensing to work in userspace mode. This will not prevent activating a # license as root, but it is required to activate one as the non-root user at diff --git a/package-manager/Dockerfile.ubuntu2204 b/package-manager/Dockerfile.ubuntu2204 index 258cd36e..aeffa1c6 100644 --- a/package-manager/Dockerfile.ubuntu2204 +++ b/package-manager/Dockerfile.ubuntu2204 @@ -50,11 +50,7 @@ RUN mkdir -p /var/run/rstudio-pm \ && chown -R rstudio-pm:rstudio-pm /var/run/rstudio-pm USER rstudio-pm -USER root COPY rstudio-pm.gcfg /etc/rstudio-pm/rstudio-pm.gcfg -RUN sed -i "s/{{R_VERSION}}/${R_VERSION}/g" /etc/rstudio-pm/rstudio-pm.gcfg -RUN chown rstudio-pm:rstudio-pm /etc/rstudio-pm/rstudio-pm.gcfg -USER rstudio-pm # Set up licensing to work in userspace mode. This will not prevent activating a # license as root, but it is required to activate one as the non-root user at diff --git a/package-manager/rstudio-pm.gcfg b/package-manager/rstudio-pm.gcfg index 67300acb..1f1c62cf 100644 --- a/package-manager/rstudio-pm.gcfg +++ b/package-manager/rstudio-pm.gcfg @@ -11,7 +11,7 @@ Address = ; ; Git sources require a configured R installation. R is often installed at `/usr/lib/R` ; or `/usr/lib64/R`. -RVersion = /opt/R/{{R_VERSION}}/ +RVersion = /opt/R/4.2.3/ ; ; Customize the data directory if necessary. This is where all packages and metadata are ; stored by default. Refer to Admin Guide for details. From b9ba1d5fef65adf519e2201bb23126b2572ae974 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 13:41:29 -0700 Subject: [PATCH 51/60] Update Workbench for Azure ML Python versions --- workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 index 4f609360..2a824290 100644 --- a/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 +++ b/workbench-for-microsoft-azure-ml/Dockerfile.ubuntu2204 @@ -1,7 +1,7 @@ ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.9.14 -ARG PYTHON_VERSION_ALT=3.8.15 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 ARG SRC_IMAGE_NAME=product-base-pro ARG REGISTRY=ghcr.io FROM ${REGISTRY}/rstudio/${SRC_IMAGE_NAME}:ubuntu2204-r${R_VERSION}_${R_VERSION_ALT}-py${PYTHON_VERSION}_${PYTHON_VERSION_ALT} AS workbench @@ -10,9 +10,9 @@ LABEL maintainer="RStudio Docker " ARG DEBIAN_FRONTEND=noninteractive ARG R_VERSION=4.2.3 ARG R_VERSION_ALT=4.1.3 -ARG PYTHON_VERSION=3.9.14 -ARG PYTHON_VERSION_ALT=3.8.15 -ARG PYTHON_VERSION_JUPYTER=3.8.15 +ARG PYTHON_VERSION=3.9.17 +ARG PYTHON_VERSION_ALT=3.8.17 +ARG PYTHON_VERSION_JUPYTER=3.8.17 ARG RSW_VERSION=2023.06.1+524.pro1 ARG RSW_NAME=rstudio-workbench ARG RSW_DOWNLOAD_URL=https://download2.rstudio.org/server/jammy/amd64 From 4dace239622473c155dc761532a1f08c27cd5ce7 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 13:47:26 -0700 Subject: [PATCH 52/60] Use secrets directly from action.yaml --- .../actions/build-test-scan-push/action.yaml | 28 ++++--------------- .github/workflows/build-content.yaml | 10 ------- .github/workflows/build-manual.yaml | 6 ---- .github/workflows/build-prerelease.yaml | 5 ---- .github/workflows/build-release.yaml | 15 ---------- 5 files changed, 5 insertions(+), 59 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index aaff490d..7ce2caa5 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -23,28 +23,10 @@ inputs: description: Flag to test image once built default: true type: boolean - snyk-token: - description: Username for authentication with Snyk for scanning images - type: string - snyk-org-id: - description: Snyk Organization ID to publish scans to - type: string push-image: description: Flag to push image once built default: false type: boolean - ghcr-token: - description: Username for authentication with GHCR.io - required: true - type: string - dockerhub-username: - description: Username for authentication with DockerHub - required: true - type: string - dockerhub-token: - description: Username for authentication with DockerHub - required: true - type: string runs: using: "composite" @@ -57,13 +39,13 @@ runs: with: registry: ghcr.io username: ${{ github.actor }} - password: ${{ inputs.ghcr-token }} + password: ${{ secrets.BUILD_PAT }} - name: Login to Docker Hub uses: docker/login-action@v2 with: - username: ${{ inputs.dockerhub-username }} - password: ${{ inputs.dockerhub-token }} + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Build id: image-build @@ -112,12 +94,12 @@ runs: continue-on-error: true uses: snyk/actions/docker@master env: - SNYK_TOKEN: ${{ inputs.snyk-token }} + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} with: image: ${{ steps.first-tag.FIRST_TAG }} args: | --file=${{ inputs.context }}/Dockerfile.${{ inputs.os }} \ - --org=${{ inputs.snyk-org-id }} \ + --org=${{ secrets.SNYK_ORG_ID }} \ --project-name=${{ steps.first-tag.FIRST_TAG }} \ --tags=product=${{ inputs.product }},os=${{ inputs.os }} \ --exclude-base-image-vulns \ diff --git a/.github/workflows/build-content.yaml b/.github/workflows/build-content.yaml index 09e826a1..a5fd1537 100644 --- a/.github/workflows/build-content.yaml +++ b/.github/workflows/build-content.yaml @@ -48,11 +48,6 @@ jobs: PYTHON_VERSION=${{ matrix.config.py }} test-image: false push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.BUILD_PAT }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} build-pro: runs-on: ubuntu-latest @@ -86,8 +81,3 @@ jobs: BASE_IMAGE=rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} test-image: false push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.BUILD_PAT }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} diff --git a/.github/workflows/build-manual.yaml b/.github/workflows/build-manual.yaml index ac97f89f..ed94b47a 100644 --- a/.github/workflows/build-manual.yaml +++ b/.github/workflows/build-manual.yaml @@ -128,9 +128,3 @@ jobs: image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} push-image: ${{ inputs.push }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.BUILD_PAT }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - diff --git a/.github/workflows/build-prerelease.yaml b/.github/workflows/build-prerelease.yaml index 2da004e4..1361fcba 100644 --- a/.github/workflows/build-prerelease.yaml +++ b/.github/workflows/build-prerelease.yaml @@ -106,8 +106,3 @@ jobs: image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.BUILD_PAT }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 56402177..e2544b7f 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -70,11 +70,6 @@ jobs: image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.BUILD_PAT }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} build-base-pro: needs: build-base @@ -138,11 +133,6 @@ jobs: image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.BUILD_PAT }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} build-products: needs: [ build-base, build-base-pro ] @@ -221,8 +211,3 @@ jobs: image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} push-image: ${{ github.ref == 'refs/heads/main' }} - snyk-token: ${{ secrets.SNYK_TOKEN }} - snyk-org-id: ${{ secrets.SNYK_ORG_ID }} - ghcr-token: ${{ secrets.BUILD_PAT }} - dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} From 459ef9ed1bfe75f6daad841a63ebf80c1d023c32 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Tue, 25 Jul 2023 13:48:53 -0700 Subject: [PATCH 53/60] Revert "Use secrets directly from action.yaml" This reverts commit 4dace239622473c155dc761532a1f08c27cd5ce7. --- .../actions/build-test-scan-push/action.yaml | 28 +++++++++++++++---- .github/workflows/build-content.yaml | 10 +++++++ .github/workflows/build-manual.yaml | 6 ++++ .github/workflows/build-prerelease.yaml | 5 ++++ .github/workflows/build-release.yaml | 15 ++++++++++ 5 files changed, 59 insertions(+), 5 deletions(-) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 7ce2caa5..aaff490d 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -23,10 +23,28 @@ inputs: description: Flag to test image once built default: true type: boolean + snyk-token: + description: Username for authentication with Snyk for scanning images + type: string + snyk-org-id: + description: Snyk Organization ID to publish scans to + type: string push-image: description: Flag to push image once built default: false type: boolean + ghcr-token: + description: Username for authentication with GHCR.io + required: true + type: string + dockerhub-username: + description: Username for authentication with DockerHub + required: true + type: string + dockerhub-token: + description: Username for authentication with DockerHub + required: true + type: string runs: using: "composite" @@ -39,13 +57,13 @@ runs: with: registry: ghcr.io username: ${{ github.actor }} - password: ${{ secrets.BUILD_PAT }} + password: ${{ inputs.ghcr-token }} - name: Login to Docker Hub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + username: ${{ inputs.dockerhub-username }} + password: ${{ inputs.dockerhub-token }} - name: Build id: image-build @@ -94,12 +112,12 @@ runs: continue-on-error: true uses: snyk/actions/docker@master env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + SNYK_TOKEN: ${{ inputs.snyk-token }} with: image: ${{ steps.first-tag.FIRST_TAG }} args: | --file=${{ inputs.context }}/Dockerfile.${{ inputs.os }} \ - --org=${{ secrets.SNYK_ORG_ID }} \ + --org=${{ inputs.snyk-org-id }} \ --project-name=${{ steps.first-tag.FIRST_TAG }} \ --tags=product=${{ inputs.product }},os=${{ inputs.os }} \ --exclude-base-image-vulns \ diff --git a/.github/workflows/build-content.yaml b/.github/workflows/build-content.yaml index a5fd1537..09e826a1 100644 --- a/.github/workflows/build-content.yaml +++ b/.github/workflows/build-content.yaml @@ -48,6 +48,11 @@ jobs: PYTHON_VERSION=${{ matrix.config.py }} test-image: false push-image: ${{ github.ref == 'refs/heads/main' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} build-pro: runs-on: ubuntu-latest @@ -81,3 +86,8 @@ jobs: BASE_IMAGE=rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} test-image: false push-image: ${{ github.ref == 'refs/heads/main' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} diff --git a/.github/workflows/build-manual.yaml b/.github/workflows/build-manual.yaml index ed94b47a..ac97f89f 100644 --- a/.github/workflows/build-manual.yaml +++ b/.github/workflows/build-manual.yaml @@ -128,3 +128,9 @@ jobs: image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} push-image: ${{ inputs.push }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + diff --git a/.github/workflows/build-prerelease.yaml b/.github/workflows/build-prerelease.yaml index 1361fcba..2da004e4 100644 --- a/.github/workflows/build-prerelease.yaml +++ b/.github/workflows/build-prerelease.yaml @@ -106,3 +106,8 @@ jobs: image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index e2544b7f..56402177 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -70,6 +70,11 @@ jobs: image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} build-base-pro: needs: build-base @@ -133,6 +138,11 @@ jobs: image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} build-products: needs: [ build-base, build-base-pro ] @@ -211,3 +221,8 @@ jobs: image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} push-image: ${{ github.ref == 'refs/heads/main' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} From 093e784de1d400d206437d2af8da3b873a845e21 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 26 Jul 2023 07:28:37 -0700 Subject: [PATCH 54/60] Free up additional space in action.yaml --- .github/actions/build-test-scan-push/action.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index aaff490d..2b2ce7fd 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -52,6 +52,12 @@ runs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Free Disk space + shell: bash + run: | + sudo rm -rf /usr/local/lib/android # will release about 10 GB + sudo rm -rf /usr/share/dotnet # will release about 20GB + - name: Login to ghcr.io uses: docker/login-action@v2 with: From 9f2281bc17ddeba48b15217ac9d9d23c7819beb3 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Wed, 26 Jul 2023 09:34:02 -0700 Subject: [PATCH 55/60] Remove debugging step Fix naming --- .github/workflows/build-manual.yaml | 2 +- .github/workflows/build-prerelease.yaml | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/build-manual.yaml b/.github/workflows/build-manual.yaml index ac97f89f..5ad494d1 100644 --- a/.github/workflows/build-manual.yaml +++ b/.github/workflows/build-manual.yaml @@ -119,7 +119,7 @@ jobs: ) echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - name: Build/Test/Scan/Push base pro image + - name: Build/Test/Scan/Push manual build image uses: ./.github/actions/build-test-scan-push with: context: ./${{ inputs.product }} diff --git a/.github/workflows/build-prerelease.yaml b/.github/workflows/build-prerelease.yaml index 2da004e4..d2aa9c96 100644 --- a/.github/workflows/build-prerelease.yaml +++ b/.github/workflows/build-prerelease.yaml @@ -92,11 +92,6 @@ jobs: ) echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - - name: Debug push-image - run: | - echo ${{ github.ref }} - echo ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }} - - name: Build/Test/Scan/Push base pro image uses: ./.github/actions/build-test-scan-push with: From e12963fdf9b56f724409f2541d5e8bc91df18722 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 27 Jul 2023 06:46:44 -0700 Subject: [PATCH 56/60] Retry builds once on failure --- .github/workflows/build-content.yaml | 70 +++++++++++++++++++++ .github/workflows/build-prerelease.yaml | 29 ++++++++- .github/workflows/build-release.yaml | 81 +++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-content.yaml b/.github/workflows/build-content.yaml index 09e826a1..0194becb 100644 --- a/.github/workflows/build-content.yaml +++ b/.github/workflows/build-content.yaml @@ -33,7 +33,9 @@ jobs: uses: actions/checkout@v3 - name: Build/Test/Scan/Push content base image + id: build1 uses: ./.github/actions/build-test-scan-push + continue-on-error: true with: context: ./content/base os: ${{ matrix.config.os }} @@ -54,6 +56,38 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + # Begin retry logic + + - name: Wait 60s on failure before retrying + if: steps.build1.outcome == 'failure' + run: sleep 60 + + - name: Retry - Build/Test/Scan/Push base pro image + id: build2 + if: steps.build1.outcome == 'failure' + uses: ./.github/actions/build-test-scan-push + with: + context: ./content/base + os: ${{ matrix.config.os }} + product: content-base + image-tags: | + rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }}, + ghcr.io/rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }}, + rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }}, + ghcr.io/rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} + build-args: | + R_VERSION=${{ matrix.config.r }} + PYTHON_VERSION=${{ matrix.config.py }} + test-image: false + push-image: ${{ github.ref == 'refs/heads/main' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + # End retry logic + build-pro: runs-on: ubuntu-latest needs: [ matrix, build ] @@ -69,6 +103,40 @@ jobs: uses: actions/checkout@v3 - name: Build/Test/Scan/Push content pro image + id: build1 + uses: ./.github/actions/build-test-scan-push + continue-on-error: true + with: + context: ./content/pro + os: ${{ matrix.config.os }} + product: content-pro + image-tags: | + rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} + ghcr.io/rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} + rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} + ghcr.io/rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} + build-args: | + R_VERSION=${{ matrix.config.r }} + PYTHON_VERSION=${{ matrix.config.py }} + DRIVERS_VERSION=${{ matrix.config.drivers }} + BASE_IMAGE=rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} + test-image: false + push-image: ${{ github.ref == 'refs/heads/main' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + # Begin retry logic + + - name: Wait 60s on failure before retrying + if: steps.build1.outcome == 'failure' + run: sleep 60 + + - name: Retry - Build/Test/Scan/Push base pro image + id: build2 + if: steps.build1.outcome == 'failure' uses: ./.github/actions/build-test-scan-push with: context: ./content/pro @@ -91,3 +159,5 @@ jobs: ghcr-token: ${{ secrets.BUILD_PAT }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + # End retry logic diff --git a/.github/workflows/build-prerelease.yaml b/.github/workflows/build-prerelease.yaml index d2aa9c96..9fddfec0 100644 --- a/.github/workflows/build-prerelease.yaml +++ b/.github/workflows/build-prerelease.yaml @@ -93,7 +93,9 @@ jobs: echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - name: Build/Test/Scan/Push base pro image + id: build1 uses: ./.github/actions/build-test-scan-push + continue-on-error: true with: context: ./${{ matrix.config.product }} os: ${{ matrix.config.os }} @@ -105,4 +107,29 @@ jobs: snyk-org-id: ${{ secrets.SNYK_ORG_ID }} ghcr-token: ${{ secrets.BUILD_PAT }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} \ No newline at end of file + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + # Begin retry logic + + - name: Wait 60s on failure before retrying + if: steps.build1.outcome == 'failure' + run: sleep 60 + + - name: Retry - Build/Test/Scan/Push base pro image + id: build2 + if: steps.build1.outcome == 'failure' + uses: ./.github/actions/build-test-scan-push + with: + context: ./product/base + os: ${{ matrix.config.os }} + product: product-base + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + # End retry logic diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 56402177..d308062a 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -62,7 +62,9 @@ jobs: echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - name: Build/Test/Scan/Push base image + id: build1 uses: ./.github/actions/build-test-scan-push + continue-on-error: true with: context: ./product/base os: ${{ matrix.config.os }} @@ -76,6 +78,31 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + # Begin retry logic + + - name: Wait 60s on failure before retrying + if: steps.build1.outcome == 'failure' + run: sleep 60 + + - name: Retry - Build/Test/Scan/Push base pro image + id: build2 + if: steps.build1.outcome == 'failure' + uses: ./.github/actions/build-test-scan-push + with: + context: ./product/base + os: ${{ matrix.config.os }} + product: product-base + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + # End retry logic + build-base-pro: needs: build-base runs-on: ubuntu-latest @@ -130,7 +157,9 @@ jobs: echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - name: Build/Test/Scan/Push base pro image + id: build1 uses: ./.github/actions/build-test-scan-push + continue-on-error: true with: context: ./product/pro os: ${{ matrix.config.os }} @@ -144,6 +173,31 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + # Begin retry logic + + - name: Wait 60s on failure before retrying + if: steps.build1.outcome == 'failure' + run: sleep 60 + + - name: Retry - Build/Test/Scan/Push base pro image + id: build2 + if: steps.build1.outcome == 'failure' + uses: ./.github/actions/build-test-scan-push + with: + context: ./product/pro + os: ${{ matrix.config.os }} + product: product-base-pro + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} + push-image: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + # End retry logic + build-products: needs: [ build-base, build-base-pro ] runs-on: ubuntu-latest @@ -213,7 +267,9 @@ jobs: echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT - name: Build/Test/Scan/Push base pro image + id: build1 uses: ./.github/actions/build-test-scan-push + continue-on-error: true with: context: ./${{ matrix.config.product }} os: ${{ matrix.config.os }} @@ -226,3 +282,28 @@ jobs: ghcr-token: ${{ secrets.BUILD_PAT }} dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + # Begin retry logic + + - name: Wait 60s on failure before retrying + if: steps.build1.outcome == 'failure' + run: sleep 60 + + - name: Retry - Build/Test/Scan/Push base pro image + id: build2 + if: steps.build1.outcome == 'failure' + uses: ./.github/actions/build-test-scan-push + with: + context: ./${{ matrix.config.product }} + os: ${{ matrix.config.os }} + product: ${{ matrix.config.product }} + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} + push-image: ${{ github.ref == 'refs/heads/main' }} + snyk-token: ${{ secrets.SNYK_TOKEN }} + snyk-org-id: ${{ secrets.SNYK_ORG_ID }} + ghcr-token: ${{ secrets.BUILD_PAT }} + dockerhub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + # End retry logic From 179140cd363aef441d0c0bfbeabddbd172fa2984 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 27 Jul 2023 07:17:05 -0700 Subject: [PATCH 57/60] Update OS in just target doc Co-authored-by: Benjamin R. J. Schwedler --- ci.Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.Justfile b/ci.Justfile index 5f0d59e9..5629c264 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -38,7 +38,7 @@ _get-default-tag PRODUCT OS: echo "{{ REGISTRY_NAMESPACE }}/${IMAGE_PREFIX}{{ PRODUCT }}:{{ OS }}" -# just _get-rsw-download-url release ubuntu1804 +# just _get-rsw-download-url release ubuntu2204 _get-rsw-download-url TYPE OS: #!/usr/bin/env bash URL_OS="{{OS}}" From accd84837ebf467e071da508cc840c37814c9082 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 27 Jul 2023 07:31:12 -0700 Subject: [PATCH 58/60] Update ci.Justfile target docs --- ci.Justfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ci.Justfile b/ci.Justfile index 5629c264..43168ead 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -14,7 +14,7 @@ DRIVERS_VERSION_RHEL := DRIVERS_VERSION + "-1" QUARTO_VERSION := "1.3.340" -# just BUILDX_PATH=~/.buildx build-release workbench bionic 12.0.11-11 +# just _get-os-alias jammy _get-os-alias OS: #!/usr/bin/env bash if [[ "{{OS}}" == "bionic" || "{{OS}}" == "ubuntu1804" ]]; then @@ -25,6 +25,7 @@ _get-os-alias OS: echo "{{OS}}" fi +# just _get-default-tag connect ubuntu2204 _get-default-tag PRODUCT OS: #!/usr/bin/env bash set -euxo pipefail @@ -58,6 +59,7 @@ _get-rsw-download-url TYPE OS: get-version +NARGS: ./tools/get-version.py {{NARGS}} +# just get-base-args ubuntu2204 base|pro get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: #!/usr/bin/env bash set -euxo pipefail @@ -88,6 +90,7 @@ get-base-args $OS $TYPE="base" $BRANCH=`git branch --show`: DRIVERS_VERSION=${_DRIVERS_VERSION} SRC_IMAGE_NAME=${SRC_IMAGE_NAME}" +# just get-base-tags ubuntu2204 base|pro get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: #!/usr/bin/env bash set -euxo pipefail @@ -105,6 +108,7 @@ get-base-tags $OS $TYPE="base" $BRANCH=`git branch --show`: ghcr.io/rstudio/${IMAGE_NAME}:${OS}-r{{R_VERSION}}-py{{PYTHON_VERSION}},\ ghcr.io/rstudio/${IMAGE_NAME}:${OS} +# just get-product-args connect ubuntu2204 2023.05.0 get-product-args $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`git rev-parse --short HEAD`: #!/usr/bin/env bash set -euxo pipefail @@ -145,6 +149,7 @@ get-product-args $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`g SRC_IMAGE_NAME=${SRC_IMAGE_NAME} RSW_DOWNLOAD_URL=${RSW_DOWNLOAD_URL}" +# just get-product-tags connect ubuntu2204 2023.05.0 get-product-tags $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`git rev-parse --short HEAD`: #!/usr/bin/env bash set -euxo pipefail @@ -185,6 +190,7 @@ get-product-tags $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`g tags=$(IFS="," ; echo "${tag_array[*]}") echo "${tags}" +# just get-prerelease-args preview|daily connect ubuntu2204 2023.05.0 get-prerelease-args $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: #!/usr/bin/env bash set -euxo pipefail @@ -226,6 +232,7 @@ get-prerelease-args $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: SRC_IMAGE_NAME=${SRC_IMAGE_NAME} RSW_DOWNLOAD_URL=${RSW_DOWNLOAD_URL}" +# just get-prerelease-tags preview|daily connect ubuntu2204 2023.05.0 get-prerelease-tags $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: #!/usr/bin/env bash set -euxo pipefail From 58bcbc4956b4e0cf851013437a0a38ae834f6faa Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 27 Jul 2023 07:48:41 -0700 Subject: [PATCH 59/60] Extract content tags and args creation to ci.Justfile --- .github/workflows/build-content.yaml | 109 ++++++++++++++++++--------- ci.Justfile | 17 +++++ 2 files changed, 90 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-content.yaml b/.github/workflows/build-content.yaml index 0194becb..76a294cb 100644 --- a/.github/workflows/build-content.yaml +++ b/.github/workflows/build-content.yaml @@ -32,6 +32,38 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Get build args + id: get-build-args + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + BUILD_ARGS=$( \ + just -f ci.Justfile \ + get-content-args \ + ${{ matrix.config.r }} \ + ${{ matrix.config.py }} \ + ) + echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT + echo "$BUILD_ARGS" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + + - name: Get tags + id: get-tags + run: | + IMAGE_TAGS=$( \ + just -f ci.Justfile \ + get-content-tags \ + content-base \ + ${{ matrix.config.r }} \ + ${{ matrix.config.py }} \ + ${{ matrix.config.os }} \ + ) + echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT + - name: Build/Test/Scan/Push content base image id: build1 uses: ./.github/actions/build-test-scan-push @@ -40,14 +72,8 @@ jobs: context: ./content/base os: ${{ matrix.config.os }} product: content-base - image-tags: | - rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }}, - ghcr.io/rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }}, - rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }}, - ghcr.io/rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} - build-args: | - R_VERSION=${{ matrix.config.r }} - PYTHON_VERSION=${{ matrix.config.py }} + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} test-image: false push-image: ${{ github.ref == 'refs/heads/main' }} snyk-token: ${{ secrets.SNYK_TOKEN }} @@ -70,14 +96,8 @@ jobs: context: ./content/base os: ${{ matrix.config.os }} product: content-base - image-tags: | - rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }}, - ghcr.io/rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }}, - rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }}, - ghcr.io/rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} - build-args: | - R_VERSION=${{ matrix.config.r }} - PYTHON_VERSION=${{ matrix.config.py }} + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} test-image: false push-image: ${{ github.ref == 'refs/heads/main' }} snyk-token: ${{ secrets.SNYK_TOKEN }} @@ -102,6 +122,39 @@ jobs: - name: Check Out Repo uses: actions/checkout@v3 + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Get build args + id: get-build-args + run: | + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + BUILD_ARGS=$( \ + just -f ci.Justfile \ + get-content-args \ + ${{ matrix.config.r }} \ + ${{ matrix.config.py }} \ + ${{ matrix.config.drivers }} \ + ) + echo "BUILD_ARGS<<$EOF" >> $GITHUB_OUTPUT + echo "$BUILD_ARGS" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + + - name: Get tags + id: get-tags + run: | + IMAGE_TAGS=$( \ + just -f ci.Justfile \ + get-content-tags \ + content-pro \ + ${{ matrix.config.r }} \ + ${{ matrix.config.py }} \ + ${{ matrix.config.os }} \ + ) + echo "IMAGE_TAGS=$IMAGE_TAGS" >> $GITHUB_OUTPUT + - name: Build/Test/Scan/Push content pro image id: build1 uses: ./.github/actions/build-test-scan-push @@ -110,16 +163,8 @@ jobs: context: ./content/pro os: ${{ matrix.config.os }} product: content-pro - image-tags: | - rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - ghcr.io/rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} - ghcr.io/rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} - build-args: | - R_VERSION=${{ matrix.config.r }} - PYTHON_VERSION=${{ matrix.config.py }} - DRIVERS_VERSION=${{ matrix.config.drivers }} - BASE_IMAGE=rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} test-image: false push-image: ${{ github.ref == 'refs/heads/main' }} snyk-token: ${{ secrets.SNYK_TOKEN }} @@ -142,16 +187,8 @@ jobs: context: ./content/pro os: ${{ matrix.config.os }} product: content-pro - image-tags: | - rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - ghcr.io/rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} - rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} - ghcr.io/rstudio/content-pro:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os_alt }} - build-args: | - R_VERSION=${{ matrix.config.r }} - PYTHON_VERSION=${{ matrix.config.py }} - DRIVERS_VERSION=${{ matrix.config.drivers }} - BASE_IMAGE=rstudio/content-base:r${{ matrix.config.r }}-py${{ matrix.config.py }}-${{ matrix.config.os }} + image-tags: ${{ steps.get-tags.outputs.IMAGE_TAGS }} + build-args: ${{ steps.get-build-args.outputs.BUILD_ARGS }} test-image: false push-image: ${{ github.ref == 'refs/heads/main' }} snyk-token: ${{ secrets.SNYK_TOKEN }} diff --git a/ci.Justfile b/ci.Justfile index 43168ead..e1bddbc1 100644 --- a/ci.Justfile +++ b/ci.Justfile @@ -269,3 +269,20 @@ get-prerelease-tags $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`: done tags=$(IFS="," ; echo "${tag_array[*]}") echo "${tags}" + +# just get-content-args 4.2.3 3.9.17 +get-content-args r-ver py-ver drivers-ver="": + #!/usr/bin/env bash + printf "R_VERSION={{r-ver}} + PYTHON_VERSION={{py-ver}} + DRIVERS_VERSION={{drivers-ver}}" + +# just get-content-tags content-base|content-pro 4.2.3 3.9.17 ubuntu2204 +get-content-tags image-name r-ver py-ver os: + #!/usr/bin/env bash + OS=$(just _parse-os {{os}}) + OS_ALT=$(just _rev-parse-os {{os}}) + echo rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS},\ + ghcr.io/rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS},\ + rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS_ALT},\ + ghcr.io/rstudio/{{image-name}}:r{{r-ver}}-py{{py-ver}}-${OS_ALT} From 5768cefbbaebf308433de4c1591fd79497fbed71 Mon Sep 17 00:00:00 2001 From: Ian Pittwood Date: Thu, 27 Jul 2023 07:50:23 -0700 Subject: [PATCH 60/60] Note that we can't use "if:" in our action --- .github/actions/build-test-scan-push/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/build-test-scan-push/action.yaml b/.github/actions/build-test-scan-push/action.yaml index 2b2ce7fd..b9007bd7 100644 --- a/.github/actions/build-test-scan-push/action.yaml +++ b/.github/actions/build-test-scan-push/action.yaml @@ -93,6 +93,7 @@ runs: echo "$FIRST_TAG" echo "FIRST_TAG=$FIRST_TAG" >> $GITHUB_OUTPUT + # We have to use bash logic because step "if"s don't work in composite actions - name: Test - ${{ inputs.test-image }} shell: bash run: |