CI: fix zizmor informational severity errors

neondatabase · Dec 8, 2024 · 0a1f2bd · 0a1f2bd
1 parent 51ddcb2
commit 0a1f2bd
Show file tree

Hide file tree

Showing 3 changed files with 98 additions and 61 deletions.
diff --git a/.github/workflows/_build-and-test-locally.yml b/.github/workflows/_build-and-test-locally.yml
@@ -206,39 +206,48 @@ jobs:
             done
           fi
 
-      - name: Run rust tests
-        env:
-          NEXTEST_RETRIES: 3
+      - name: Set PQ_LIB_DIR and LD_LIBRARY_PATH
         run: |
-          PQ_LIB_DIR=$(pwd)/pg_install/v16/lib
-          export PQ_LIB_DIR
-          LD_LIBRARY_PATH=$(pwd)/pg_install/v17/lib
-          export LD_LIBRARY_PATH
+          echo "PQ_LIB_DIR=$(pwd)/pg_install/v17/lib"      | tee -a ${GITHUB_ENV}
+          echo "LD_LIBRARY_PATH=$(pwd)/pg_install/v17/lib" | tee -a ${GITHUB_ENV}
 
+      - name: Run rust doctests
+        run: |
           #nextest does not yet support running doctests
           ${cov_prefix} cargo test --doc $CARGO_FLAGS $CARGO_FEATURES
 
-          # run all non-pageserver tests
+      - name: Run all non-pageserver rust tests
+        env:
+          NEXTEST_RETRIES: 3
+        run: |
           ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E '!package(pageserver)'
 
-          # run pageserver tests with different settings
+      - name: Run pageserver rust tests with different settings
+        env:
+          NEXTEST_RETRIES: 3
+        run: |
           for io_engine in std-fs tokio-epoll-uring ; do
             NEON_PAGESERVER_UNIT_TEST_VIRTUAL_FILE_IOENGINE=$io_engine ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES  -E 'package(pageserver)'
           done
 
-          # Run separate tests for real S3
-          export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty
-          export REMOTE_STORAGE_S3_BUCKET=neon-github-ci-tests
-          export REMOTE_STORAGE_S3_REGION=eu-central-1
+      - name: Run rust tests for real S3
+        env:
+          NEXTEST_RETRIES: 3
+          ENABLE_REAL_S3_REMOTE_STORAGE: nonempty
+          REMOTE_STORAGE_S3_BUCKET: neon-github-ci-tests
+          REMOTE_STORAGE_S3_REGION: eu-central-1
+        run: |
           ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E 'package(remote_storage)' -E 'test(test_real_s3)'
 
-          # Run separate tests for real Azure Blob Storage
-          # XXX: replace region with `eu-central-1`-like region
-          export ENABLE_REAL_AZURE_REMOTE_STORAGE=y
-          export AZURE_STORAGE_ACCOUNT="${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }}"
-          export AZURE_STORAGE_ACCESS_KEY="${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }}"
-          export REMOTE_STORAGE_AZURE_CONTAINER="${{ vars.REMOTE_STORAGE_AZURE_CONTAINER }}"
-          export REMOTE_STORAGE_AZURE_REGION="${{ vars.REMOTE_STORAGE_AZURE_REGION }}"
+      - name: Run rust tests for real Azure Blob Storage
+        env:
+          NEXTEST_RETRIES: 3
+          ENABLE_REAL_AZURE_REMOTE_STORAGE: y
+          AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }}
+          AZURE_STORAGE_ACCESS_KEY: ${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }}
+          REMOTE_STORAGE_AZURE_CONTAINER: ${{ vars.REMOTE_STORAGE_AZURE_CONTAINER }}
+          REMOTE_STORAGE_AZURE_REGION: ${{ vars.REMOTE_STORAGE_AZURE_REGION }}
+        run: |
           ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E 'package(remote_storage)' -E 'test(test_real_azure)'
 
       - name: Install postgres binaries

diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml
@@ -424,6 +424,8 @@ jobs:
 
     - name: Set up Connection String
       id: set-up-connstr
+      env:
+        NEW_NEON_PROJECT_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }}
       run: |
         case "${PLATFORM}" in
           neonvm-captest-reuse)
@@ -433,7 +435,7 @@ jobs:
             CONNSTR=${{ secrets.BENCHMARK_CAPTEST_SHARDING_CONNSTR }}
             ;;
           neonvm-captest-new | neonvm-captest-freetier | neonvm-azure-captest-new | neonvm-azure-captest-freetier)
-            CONNSTR=${{ steps.create-neon-project.outputs.dsn }}
+            CONNSTR=${NEW_NEON_PROJECT_CONNSTR}
             ;;
           rds-aurora)
             CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_CONNSTR }}

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -395,18 +395,25 @@ jobs:
 
       - uses: actions/github-script@v7
         if: ${{ !cancelled() }}
+        env:
+          REPORT_URL: ${{ steps.create-allure-report.outputs.report-url }}
+          REPORT_JSON_URL: ${{ steps.create-allure-report.outputs.report-json-url }}
+          COVERAGE_URL: ${{ needs.coverage-report.outputs.coverage-html }}
+          SUMMARY_JSON_URL: ${{ needs.coverage-report.outputs.coverage-json }}
         with:
           # Retry script for 5XX server errors: https://github.com/actions/github-script#retries
           retries: 5
           script: |
+            const { REPORT_URL, REPORT_JSON_URL, COVERAGE_URL, SUMMARY_JSON_URL } = process.env
+
             const report = {
-              reportUrl:     "${{ steps.create-allure-report.outputs.report-url }}",
-              reportJsonUrl: "${{ steps.create-allure-report.outputs.report-json-url }}",
+              reportUrl:     `${REPORT_URL}`,
+              reportJsonUrl: `${REPORT_JSON_URL}`,
             }
 
             const coverage = {
-              coverageUrl: "${{ needs.coverage-report.outputs.coverage-html }}",
-              summaryJsonUrl: "${{ needs.coverage-report.outputs.coverage-json }}",
+              coverageUrl:    `${COVERAGE_URL}`,
+              summaryJsonUrl: `${SUMMARY_JSON_URL}`,
             }
 
             const script = require("./scripts/comment-test-report.js")
@@ -593,11 +600,13 @@ jobs:
           password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
 
       - name: Create multi-arch image
+        env:
+          BUILD_TAG: ${{ needs.tag.outputs.build-tag }}
         run: |
-          docker buildx imagetools create -t neondatabase/neon:${{ needs.tag.outputs.build-tag }} \
-                                          -t neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm \
-                                             neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-x64 \
-                                             neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-arm64
+          docker buildx imagetools create -t neondatabase/neon:${BUILD_TAG} \
+                                          -t neondatabase/neon:${BUILD_TAG}-bookworm \
+                                             neondatabase/neon:${BUILD_TAG}-bookworm-x64 \
+                                             neondatabase/neon:${BUILD_TAG}-bookworm-arm64
 
       - uses: docker/login-action@v3
         with:
@@ -606,9 +615,11 @@ jobs:
           password: ${{ secrets.AWS_SECRET_KEY_DEV }}
 
       - name: Push multi-arch image to ECR
+        env:
+          BUILD_TAG: ${{ needs.tag.outputs.build-tag }}
         run: |
-          docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{ needs.tag.outputs.build-tag }} \
-                                                                                neondatabase/neon:${{ needs.tag.outputs.build-tag }}
+          docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${BUILD_TAG} \
+                                                                                neondatabase/neon:${BUILD_TAG}
 
   compute-node-image-arch:
     needs: [ check-permissions, build-build-tools-image, tag ]
@@ -745,6 +756,9 @@ jobs:
           - pg: v17
             debian: bookworm
 
+    env:
+      BUILD_TAG: ${{ needs.tag.outputs.build-tag }}
+
     steps:
       - uses: docker/login-action@v3
         with:
@@ -753,26 +767,26 @@ jobs:
 
       - name: Create multi-arch compute-node image
         run: |
-          docker buildx imagetools create -t neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \
-                                          -t neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }} \
-                                             neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \
-                                             neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64
+          docker buildx imagetools create -t neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG} \
+                                          -t neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG}-${{ matrix.version.debian }} \
+                                             neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG}-${{ matrix.version.debian }}-x64 \
+                                             neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG}-${{ matrix.version.debian }}-arm64
 
       - name: Create multi-arch neon-test-extensions image
         if: matrix.version.pg >= 'v16'
         run: |
-          docker buildx imagetools create -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \
-                                          -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }} \
-                                             neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \
-                                             neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64
+          docker buildx imagetools create -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${BUILD_TAG} \
+                                          -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${BUILD_TAG}-${{ matrix.version.debian }} \
+                                             neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${BUILD_TAG}-${{ matrix.version.debian }}-x64 \
+                                             neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${BUILD_TAG}-${{ matrix.version.debian }}-arm64
 
       - name: Create multi-arch compute-tools image
         if: matrix.version.pg == 'v16'
         run: |
-          docker buildx imagetools create -t neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }} \
-                                          -t neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }} \
-                                             neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \
-                                             neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64
+          docker buildx imagetools create -t neondatabase/compute-tools:${BUILD_TAG} \
+                                          -t neondatabase/compute-tools:${BUILD_TAG}-${{ matrix.version.debian }} \
+                                             neondatabase/compute-tools:${BUILD_TAG}-${{ matrix.version.debian }}-x64 \
+                                             neondatabase/compute-tools:${BUILD_TAG}-${{ matrix.version.debian }}-arm64
 
       - uses: docker/login-action@v3
         with:
@@ -782,14 +796,14 @@ jobs:
 
       - name: Push multi-arch compute-node-${{ matrix.version.pg }} image to ECR
         run: |
-          docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \
-                                                                                neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}
+          docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version.pg }}:${BUILD_TAG} \
+                                                                                neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG}
 
       - name: Push multi-arch compute-tools image to ECR
         if: matrix.version.pg == 'v16'
         run: |
-          docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{ needs.tag.outputs.build-tag }} \
-                                                                                neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}
+          docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${BUILD_TAG} \
+                                                                                neondatabase/compute-tools:${BUILD_TAG}
 
   vm-compute-node-image:
     needs: [ check-permissions, tag, compute-node-image ]
@@ -829,20 +843,26 @@ jobs:
       # Note: we need a separate pull step here because otherwise vm-builder will try to pull, and
       # it won't have the proper authentication (written at v0.6.0)
       - name: Pulling compute-node image
+        env:
+          BUILD_TAG: ${{ needs.tag.outputs.build-tag }}
         run: |
-          docker pull neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}
+          docker pull neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG}
 
       - name: Build vm image
+        env:
+          BUILD_TAG: ${{ needs.tag.outputs.build-tag }}
         run: |
           ./vm-builder \
             -size=2G \
             -spec=compute/vm-image-spec-${{ matrix.version.debian }}.yaml \
-            -src=neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \
-            -dst=neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}
+            -src=neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG} \
+            -dst=neondatabase/vm-compute-node-${{ matrix.version.pg }}:${BUILD_TAG}
 
       - name: Pushing vm-compute-node image
+        env:
+          BUILD_TAG: ${{ needs.tag.outputs.build-tag }}
         run: |
-          docker push neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}
+          docker push neondatabase/vm-compute-node-${{ matrix.version.pg }}:${BUILD_TAG}
 
   test-images:
     needs: [ check-permissions, tag, neon-image, compute-node-image ]
@@ -874,8 +894,10 @@ jobs:
       # Ensure that we don't have bad versions.
       - name: Verify image versions
         shell: bash # ensure no set -e for better error messages
+        env:
+          BUILD_TAG: ${{ needs.tag.outputs.build-tag }}
         run: |
-          pageserver_version=$(docker run --rm neondatabase/neon:${{ needs.tag.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version")
+          pageserver_version=$(docker run --rm neondatabase/neon:${BUILD_TAG} "/bin/sh" "-c" "/usr/local/bin/pageserver --version")
 
           echo "Pageserver version string: $pageserver_version"
 
@@ -928,30 +950,32 @@ jobs:
       - name: Copy vm-compute-node images to ECR
         run: |
           for version in ${VERSIONS}; do
-            docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }} \
-                                               neondatabase/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }}
+            docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${version}:${BUILD_TAG} \
+                                               neondatabase/vm-compute-node-${version}:${BUILD_TAG}
           done
 
       - name: Add latest tag to images
         if: github.ref_name == 'main'
+        env:
+          BUILD_TAG: ${{ needs.tag.outputs.build-tag }}
         run: |
           for repo in neondatabase 369495373322.dkr.ecr.eu-central-1.amazonaws.com; do
             docker buildx imagetools create -t $repo/neon:latest \
-                                               $repo/neon:${{ needs.tag.outputs.build-tag }}
+                                               $repo/neon:${BUILD_TAG}
 
             docker buildx imagetools create -t $repo/compute-tools:latest \
-                                               $repo/compute-tools:${{ needs.tag.outputs.build-tag }}
+                                               $repo/compute-tools:${BUILD_TAG}
 
             for version in ${VERSIONS}; do
               docker buildx imagetools create -t $repo/compute-node-${version}:latest \
-                                                 $repo/compute-node-${version}:${{ needs.tag.outputs.build-tag }}
+                                                 $repo/compute-node-${version}:${BUILD_TAG}
 
               docker buildx imagetools create -t $repo/vm-compute-node-${version}:latest \
-                                                 $repo/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }}
+                                                 $repo/vm-compute-node-${version}:${BUILD_TAG}
             done
           done
           docker buildx imagetools create -t neondatabase/neon-test-extensions-v16:latest \
-                                              neondatabase/neon-test-extensions-v16:${{ needs.tag.outputs.build-tag }}
+                                              neondatabase/neon-test-extensions-v16:${BUILD_TAG}
 
       - name: Configure AWS-prod credentials
         if: github.ref_name == 'release'|| github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
@@ -969,10 +993,12 @@ jobs:
 
       - name: Copy all images to prod ECR
         if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
+        env:
+          BUILD_TAG: ${{ needs.tag.outputs.build-tag }}
         run: |
           for image in neon compute-tools {vm-,}compute-node-{v14,v15,v16,v17}; do
-            docker buildx imagetools create -t 093970136003.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }} \
-                                               369495373322.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }}
+            docker buildx imagetools create -t 093970136003.dkr.ecr.eu-central-1.amazonaws.com/${image}:${BUILD_TAG} \
+                                               369495373322.dkr.ecr.eu-central-1.amazonaws.com/${image}:${BUILD_TAG}
           done
 
   push-to-acr-dev: