From 0a1f2bd67124550bb2aed8492aa251c29e0524a2 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Sun, 8 Dec 2024 01:01:05 +0000 Subject: [PATCH] CI: fix zizmor informational severity errors --- .github/workflows/_build-and-test-locally.yml | 49 ++++---- .github/workflows/benchmarking.yml | 4 +- .github/workflows/build_and_test.yml | 106 +++++++++++------- 3 files changed, 98 insertions(+), 61 deletions(-) diff --git a/.github/workflows/_build-and-test-locally.yml b/.github/workflows/_build-and-test-locally.yml index 224740f6307f..78600ffc746d 100644 --- a/.github/workflows/_build-and-test-locally.yml +++ b/.github/workflows/_build-and-test-locally.yml @@ -206,39 +206,48 @@ jobs: done fi - - name: Run rust tests - env: - NEXTEST_RETRIES: 3 + - name: Set PQ_LIB_DIR and LD_LIBRARY_PATH run: | - PQ_LIB_DIR=$(pwd)/pg_install/v16/lib - export PQ_LIB_DIR - LD_LIBRARY_PATH=$(pwd)/pg_install/v17/lib - export LD_LIBRARY_PATH + echo "PQ_LIB_DIR=$(pwd)/pg_install/v17/lib" | tee -a ${GITHUB_ENV} + echo "LD_LIBRARY_PATH=$(pwd)/pg_install/v17/lib" | tee -a ${GITHUB_ENV} + - name: Run rust doctests + run: | #nextest does not yet support running doctests ${cov_prefix} cargo test --doc $CARGO_FLAGS $CARGO_FEATURES - # run all non-pageserver tests + - name: Run all non-pageserver rust tests + env: + NEXTEST_RETRIES: 3 + run: | ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E '!package(pageserver)' - # run pageserver tests with different settings + - name: Run pageserver rust tests with different settings + env: + NEXTEST_RETRIES: 3 + run: | for io_engine in std-fs tokio-epoll-uring ; do NEON_PAGESERVER_UNIT_TEST_VIRTUAL_FILE_IOENGINE=$io_engine ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E 'package(pageserver)' done - # Run separate tests for real S3 - export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty - export REMOTE_STORAGE_S3_BUCKET=neon-github-ci-tests - export REMOTE_STORAGE_S3_REGION=eu-central-1 + - name: Run rust tests for real S3 + env: + NEXTEST_RETRIES: 3 + ENABLE_REAL_S3_REMOTE_STORAGE: nonempty + REMOTE_STORAGE_S3_BUCKET: neon-github-ci-tests + REMOTE_STORAGE_S3_REGION: eu-central-1 + run: | ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E 'package(remote_storage)' -E 'test(test_real_s3)' - # Run separate tests for real Azure Blob Storage - # XXX: replace region with `eu-central-1`-like region - export ENABLE_REAL_AZURE_REMOTE_STORAGE=y - export AZURE_STORAGE_ACCOUNT="${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }}" - export AZURE_STORAGE_ACCESS_KEY="${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }}" - export REMOTE_STORAGE_AZURE_CONTAINER="${{ vars.REMOTE_STORAGE_AZURE_CONTAINER }}" - export REMOTE_STORAGE_AZURE_REGION="${{ vars.REMOTE_STORAGE_AZURE_REGION }}" + - name: Run rust tests for real Azure Blob Storage + env: + NEXTEST_RETRIES: 3 + ENABLE_REAL_AZURE_REMOTE_STORAGE: y + AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }} + AZURE_STORAGE_ACCESS_KEY: ${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }} + REMOTE_STORAGE_AZURE_CONTAINER: ${{ vars.REMOTE_STORAGE_AZURE_CONTAINER }} + REMOTE_STORAGE_AZURE_REGION: ${{ vars.REMOTE_STORAGE_AZURE_REGION }} + run: | ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E 'package(remote_storage)' -E 'test(test_real_azure)' - name: Install postgres binaries diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index af0d4b05dc35..165d7bb96f79 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -424,6 +424,8 @@ jobs: - name: Set up Connection String id: set-up-connstr + env: + NEW_NEON_PROJECT_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }} run: | case "${PLATFORM}" in neonvm-captest-reuse) @@ -433,7 +435,7 @@ jobs: CONNSTR=${{ secrets.BENCHMARK_CAPTEST_SHARDING_CONNSTR }} ;; neonvm-captest-new | neonvm-captest-freetier | neonvm-azure-captest-new | neonvm-azure-captest-freetier) - CONNSTR=${{ steps.create-neon-project.outputs.dsn }} + CONNSTR=${NEW_NEON_PROJECT_CONNSTR} ;; rds-aurora) CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_CONNSTR }} diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a7030b4a7b95..3946493a3f80 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -395,18 +395,25 @@ jobs: - uses: actions/github-script@v7 if: ${{ !cancelled() }} + env: + REPORT_URL: ${{ steps.create-allure-report.outputs.report-url }} + REPORT_JSON_URL: ${{ steps.create-allure-report.outputs.report-json-url }} + COVERAGE_URL: ${{ needs.coverage-report.outputs.coverage-html }} + SUMMARY_JSON_URL: ${{ needs.coverage-report.outputs.coverage-json }} with: # Retry script for 5XX server errors: https://github.com/actions/github-script#retries retries: 5 script: | + const { REPORT_URL, REPORT_JSON_URL, COVERAGE_URL, SUMMARY_JSON_URL } = process.env + const report = { - reportUrl: "${{ steps.create-allure-report.outputs.report-url }}", - reportJsonUrl: "${{ steps.create-allure-report.outputs.report-json-url }}", + reportUrl: `${REPORT_URL}`, + reportJsonUrl: `${REPORT_JSON_URL}`, } const coverage = { - coverageUrl: "${{ needs.coverage-report.outputs.coverage-html }}", - summaryJsonUrl: "${{ needs.coverage-report.outputs.coverage-json }}", + coverageUrl: `${COVERAGE_URL}`, + summaryJsonUrl: `${SUMMARY_JSON_URL}`, } const script = require("./scripts/comment-test-report.js") @@ -593,11 +600,13 @@ jobs: password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - name: Create multi-arch image + env: + BUILD_TAG: ${{ needs.tag.outputs.build-tag }} run: | - docker buildx imagetools create -t neondatabase/neon:${{ needs.tag.outputs.build-tag }} \ - -t neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm \ - neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-x64 \ - neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-arm64 + docker buildx imagetools create -t neondatabase/neon:${BUILD_TAG} \ + -t neondatabase/neon:${BUILD_TAG}-bookworm \ + neondatabase/neon:${BUILD_TAG}-bookworm-x64 \ + neondatabase/neon:${BUILD_TAG}-bookworm-arm64 - uses: docker/login-action@v3 with: @@ -606,9 +615,11 @@ jobs: password: ${{ secrets.AWS_SECRET_KEY_DEV }} - name: Push multi-arch image to ECR + env: + BUILD_TAG: ${{ needs.tag.outputs.build-tag }} run: | - docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{ needs.tag.outputs.build-tag }} \ - neondatabase/neon:${{ needs.tag.outputs.build-tag }} + docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${BUILD_TAG} \ + neondatabase/neon:${BUILD_TAG} compute-node-image-arch: needs: [ check-permissions, build-build-tools-image, tag ] @@ -745,6 +756,9 @@ jobs: - pg: v17 debian: bookworm + env: + BUILD_TAG: ${{ needs.tag.outputs.build-tag }} + steps: - uses: docker/login-action@v3 with: @@ -753,26 +767,26 @@ jobs: - name: Create multi-arch compute-node image run: | - docker buildx imagetools create -t neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \ - -t neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }} \ - neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \ - neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64 + docker buildx imagetools create -t neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG} \ + -t neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG}-${{ matrix.version.debian }} \ + neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG}-${{ matrix.version.debian }}-x64 \ + neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG}-${{ matrix.version.debian }}-arm64 - name: Create multi-arch neon-test-extensions image if: matrix.version.pg >= 'v16' run: | - docker buildx imagetools create -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \ - -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }} \ - neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \ - neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64 + docker buildx imagetools create -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${BUILD_TAG} \ + -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${BUILD_TAG}-${{ matrix.version.debian }} \ + neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${BUILD_TAG}-${{ matrix.version.debian }}-x64 \ + neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${BUILD_TAG}-${{ matrix.version.debian }}-arm64 - name: Create multi-arch compute-tools image if: matrix.version.pg == 'v16' run: | - docker buildx imagetools create -t neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }} \ - -t neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }} \ - neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \ - neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64 + docker buildx imagetools create -t neondatabase/compute-tools:${BUILD_TAG} \ + -t neondatabase/compute-tools:${BUILD_TAG}-${{ matrix.version.debian }} \ + neondatabase/compute-tools:${BUILD_TAG}-${{ matrix.version.debian }}-x64 \ + neondatabase/compute-tools:${BUILD_TAG}-${{ matrix.version.debian }}-arm64 - uses: docker/login-action@v3 with: @@ -782,14 +796,14 @@ jobs: - name: Push multi-arch compute-node-${{ matrix.version.pg }} image to ECR run: | - docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \ - neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} + docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version.pg }}:${BUILD_TAG} \ + neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG} - name: Push multi-arch compute-tools image to ECR if: matrix.version.pg == 'v16' run: | - docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{ needs.tag.outputs.build-tag }} \ - neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }} + docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${BUILD_TAG} \ + neondatabase/compute-tools:${BUILD_TAG} vm-compute-node-image: needs: [ check-permissions, tag, compute-node-image ] @@ -829,20 +843,26 @@ jobs: # Note: we need a separate pull step here because otherwise vm-builder will try to pull, and # it won't have the proper authentication (written at v0.6.0) - name: Pulling compute-node image + env: + BUILD_TAG: ${{ needs.tag.outputs.build-tag }} run: | - docker pull neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} + docker pull neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG} - name: Build vm image + env: + BUILD_TAG: ${{ needs.tag.outputs.build-tag }} run: | ./vm-builder \ -size=2G \ -spec=compute/vm-image-spec-${{ matrix.version.debian }}.yaml \ - -src=neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \ - -dst=neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} + -src=neondatabase/compute-node-${{ matrix.version.pg }}:${BUILD_TAG} \ + -dst=neondatabase/vm-compute-node-${{ matrix.version.pg }}:${BUILD_TAG} - name: Pushing vm-compute-node image + env: + BUILD_TAG: ${{ needs.tag.outputs.build-tag }} run: | - docker push neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} + docker push neondatabase/vm-compute-node-${{ matrix.version.pg }}:${BUILD_TAG} test-images: needs: [ check-permissions, tag, neon-image, compute-node-image ] @@ -874,8 +894,10 @@ jobs: # Ensure that we don't have bad versions. - name: Verify image versions shell: bash # ensure no set -e for better error messages + env: + BUILD_TAG: ${{ needs.tag.outputs.build-tag }} run: | - pageserver_version=$(docker run --rm neondatabase/neon:${{ needs.tag.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version") + pageserver_version=$(docker run --rm neondatabase/neon:${BUILD_TAG} "/bin/sh" "-c" "/usr/local/bin/pageserver --version") echo "Pageserver version string: $pageserver_version" @@ -928,30 +950,32 @@ jobs: - name: Copy vm-compute-node images to ECR run: | for version in ${VERSIONS}; do - docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }} \ - neondatabase/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }} + docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${version}:${BUILD_TAG} \ + neondatabase/vm-compute-node-${version}:${BUILD_TAG} done - name: Add latest tag to images if: github.ref_name == 'main' + env: + BUILD_TAG: ${{ needs.tag.outputs.build-tag }} run: | for repo in neondatabase 369495373322.dkr.ecr.eu-central-1.amazonaws.com; do docker buildx imagetools create -t $repo/neon:latest \ - $repo/neon:${{ needs.tag.outputs.build-tag }} + $repo/neon:${BUILD_TAG} docker buildx imagetools create -t $repo/compute-tools:latest \ - $repo/compute-tools:${{ needs.tag.outputs.build-tag }} + $repo/compute-tools:${BUILD_TAG} for version in ${VERSIONS}; do docker buildx imagetools create -t $repo/compute-node-${version}:latest \ - $repo/compute-node-${version}:${{ needs.tag.outputs.build-tag }} + $repo/compute-node-${version}:${BUILD_TAG} docker buildx imagetools create -t $repo/vm-compute-node-${version}:latest \ - $repo/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }} + $repo/vm-compute-node-${version}:${BUILD_TAG} done done docker buildx imagetools create -t neondatabase/neon-test-extensions-v16:latest \ - neondatabase/neon-test-extensions-v16:${{ needs.tag.outputs.build-tag }} + neondatabase/neon-test-extensions-v16:${BUILD_TAG} - name: Configure AWS-prod credentials if: github.ref_name == 'release'|| github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' @@ -969,10 +993,12 @@ jobs: - name: Copy all images to prod ECR if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' + env: + BUILD_TAG: ${{ needs.tag.outputs.build-tag }} run: | for image in neon compute-tools {vm-,}compute-node-{v14,v15,v16,v17}; do - docker buildx imagetools create -t 093970136003.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }} \ - 369495373322.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }} + docker buildx imagetools create -t 093970136003.dkr.ecr.eu-central-1.amazonaws.com/${image}:${BUILD_TAG} \ + 369495373322.dkr.ecr.eu-central-1.amazonaws.com/${image}:${BUILD_TAG} done push-to-acr-dev: