feat: Use a github variable containing 1password secret references #2386
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rule acceptance tests | |
| on: | |
| pull_request: | |
| branches: [ master ] | |
| paths-ignore: | |
| - '**.md' | |
| - 'Dockerfile' | |
| - '.gitignore' | |
| - 'LICENSE' | |
| - 'docker.yml' | |
| - 'formatting.yml' | |
| - 'test_pack_dock.yml' | |
| - 'triage.yml' | |
| - 'end_to_end.yml' | |
| - 'web/**' | |
| - '.github/workflows/web_**.yml' | |
| - '.github/workflows/stg_web_**.yml' | |
| types: [ opened, synchronize, reopened, ready_for_review ] | |
| concurrency: | |
| group: ${{ github.head_ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| fail_if_pull_request_is_draft: # Fails in order to indicate that pull request needs to be marked as ready to review to pass. | |
| if: github.event.pull_request.draft == true | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Fail if PR is a draft | |
| run: exit 1 | |
| pre_ci: | |
| name: Prepare CI environment | |
| if: github.event.pull_request.draft == false # Skip this job and its dependencies if the PR is draft | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Project | |
| uses: actions/checkout@v4 | |
| with: | |
| # We need to fetch with a depth of 2 for pull_request so we can do HEAD^2 | |
| fetch-depth: 2 | |
| # If this workflow was triggered by a pull request (open or synchronize!) then resolve the commit message from HEAD^2 | |
| # It is stored in output steps, to be referenced with ${{ steps.pr_get_commit_message.outputs.pr_commit_message }} | |
| - name: "[Pull Request] Get commit message" | |
| if: github.event_name == 'pull_request' | |
| id: pr_get_commit_message | |
| # Obtain the last commit from the branch to merge (hence the HEAD^2). | |
| # In case of multi-line commit messages, remove any \n, because the GITHUB_OUTPUT method | |
| # of sending data to other jobs does not like them. | |
| run: echo "pr_commit_message=$(git log --format=%B -n 1 HEAD^2 | tr '\n' ' ')" >> $GITHUB_OUTPUT | |
| # For **Pull Request** events this will resolve to something like "$( [ -z "commit message pr" ] && echo "" || echo "commit message pr" )" which then resolves to just "commit message pr" | |
| outputs: | |
| commit_message: $( [ -z "${{ steps.pr_get_commit_message.outputs.pr_commit_message }}" ] || echo "${{ steps.pr_get_commit_message.outputs.pr_commit_message }}" ) | |
| validate-gradle-wrapper: | |
| if: "!contains(needs.pre_ci.outputs.commit_message, '[acceptance test skip]')" | |
| needs: pre_ci | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: gradle/actions/wrapper-validation@v3 | |
| pack-snapshot: | |
| needs: [ validate-gradle-wrapper ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '11' | |
| distribution: 'temurin' | |
| - name: Cache Gradle packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} | |
| restore-keys: ${{ runner.os }}-gradle | |
| - name: Set up Gradle | |
| uses: gradle/actions/setup-gradle@v3 | |
| - name: Package cli app jar with Gradle | |
| run: ./gradlew shadowJar | |
| - name: Persist gtfs-validator snapshot jar | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gtfs-validator-snapshot | |
| path: cli/build/libs/gtfs-validator-*-cli.jar | |
| - name: Persist comparator snapshot jar | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: comparator-snapshot | |
| path: output-comparator/build/libs/output-comparator-*-cli.jar | |
| pack-master: | |
| needs: [ validate-gradle-wrapper ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: master | |
| - name: Set up JDK 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '11' | |
| distribution: 'temurin' | |
| - name: Cache Gradle packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.gradle/caches | |
| key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} | |
| restore-keys: ${{ runner.os }}-gradle | |
| - name: Set up Gradle | |
| uses: gradle/actions/setup-gradle@v3 | |
| - name: Package cli app jar with Gradle | |
| run: ./gradlew shadowJar | |
| - name: Persist gtfs-validator jar from master branch | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gtfs-validator-master | |
| path: cli/build/libs/gtfs-validator-*-cli.jar | |
| fetch-urls: | |
| if: "!contains(needs.pre_ci.outputs.commit_message, '[acceptance test skip]')" | |
| needs: pre_ci | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository code | |
| uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: | | |
| pip install -r scripts/mobility-database-harvester/requirements.txt | |
| - name: Set URL matrix | |
| id: set-matrix | |
| run: | | |
| DATASETS=$(python3 scripts/mobility-database-harvester/harvest_latest_versions.py -d scripts/mobility-database-harvester/datasets_metadata -l gtfs_latest_versions.json) | |
| echo $DATASETS | |
| echo "matrix=$DATASETS" >> $GITHUB_OUTPUT | |
| - name: Persist metadata | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: datasets_metadata | |
| path: scripts/mobility-database-harvester/datasets_metadata | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| get-reports: | |
| needs: [ fetch-urls, pack-master, pack-snapshot ] | |
| # We use machines with more memory to run validation, as large feeds | |
| # can consume too much heap for default machine instances (see #1304). | |
| runs-on: ubuntu-latest-4-cores | |
| strategy: | |
| matrix: ${{ fromJson(needs.fetch-urls.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download .jar file from master branch | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gtfs-validator-master | |
| path: gtfs-validator-master | |
| - name: Download latest changes .jar file from previous job | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gtfs-validator-snapshot | |
| path: gtfs-validator-snapshot | |
| - name: Extract and concatenate IDs | |
| run: | | |
| concatenated_ids=$(bash ./scripts/extract_ids.sh '${{ matrix.data }}') | |
| echo "CONCATENATED_IDS=$concatenated_ids" >> $GITHUB_ENV | |
| echo "CONCATENATED_IDS=$concatenated_ids" | |
| - name: Run validators on queued URLs | |
| run: | | |
| queue="${{ matrix.data }}" | |
| bash ./scripts/queue_runner.sh --include-master $queue | |
| env: | |
| OUTPUT_BASE: ${{ github.sha }} | |
| - name: Persist reports | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: reports_${{ env.CONCATENATED_IDS }} | |
| path: ${{ github.sha }}/output | |
| merge-reports-artifacts: | |
| runs-on: ubuntu-latest | |
| needs: [ get-reports ] | |
| steps: | |
| - name: Merge Artifacts | |
| uses: actions/upload-artifact/merge@v4 | |
| with: | |
| name: reports_all | |
| pattern: reports_* | |
| delete-merged: true | |
| compare-outputs: | |
| needs: [ merge-reports-artifacts ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download comparator .jar file from previous job | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: comparator-snapshot | |
| - name: Retrieve reports from previous job | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: reports_all | |
| - name: Retrieve gtfs latest versions from previous job | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: datasets_metadata | |
| - name: Generate acceptance report test | |
| run: | | |
| java -jar output-comparator*.jar \ | |
| --report_directory . \ | |
| --source_urls gtfs_latest_versions.json \ | |
| --new_error_threshold 1 \ | |
| --percent_invalid_datasets_threshold 1 \ | |
| --percent_corrupted_sources 2 \ | |
| --reference_report_name reference.json \ | |
| --latest_report_name latest.json \ | |
| --output_base acceptance-test-output \ | |
| --commit_sha ${{ github.sha }} \ | |
| --run_id ${{github.run_id}} | |
| - name: Persist acceptance test reports | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: acceptance_test_report | |
| path: acceptance-test-output | |
| - name: Generate PR comment | |
| id: generate-comment | |
| if: always() | |
| run: | | |
| PR_COMMENT=$(< acceptance-test-output/acceptance_report_summary.md) | |
| echo "PR_COMMENT<<EOF" >> $GITHUB_ENV | |
| echo "$PR_COMMENT" >> $GITHUB_ENV | |
| echo "EOF" >> $GITHUB_ENV | |
| - name: Comment Pull Request | |
| if: always() | |
| uses: thollander/[email protected] | |
| with: | |
| message: ${{ env.PR_COMMENT }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |