WIP: use compute-workflow-parameters.py to get list of weblogs and sc… #11814
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: System Tests | |
| on: | |
| push: | |
| branches: | |
| - "**" | |
| workflow_dispatch: {} | |
| schedule: | |
| - cron: '00 04 * * 2-6' | |
| env: | |
| REGISTRY: ghcr.io | |
| REPO: ghcr.io/datadog/dd-trace-rb | |
| SYSTEM_TESTS_REF: main # This must always be set to `main` on dd-trace-rb's master branch | |
| jobs: | |
| compute-workflow-parameters: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| endtoend_scenarios: ${{ steps.main.outputs.endtoend_scenarios }} | |
| endtoend_weblogs: ${{ steps.main.outputs.endtoend_weblogs }} | |
| graphql_scenarios: ${{ steps.main.outputs.graphql_scenarios }} | |
| graphql_weblogs: ${{ steps.main.outputs.graphql_weblogs }} | |
| libinjection_scenarios: ${{ steps.main.outputs.libinjection_scenarios }} | |
| opentelemetry_scenarios: ${{ steps.main.outputs.opentelemetry_scenarios }} | |
| opentelemetry_weblogs: ${{ steps.main.outputs.opentelemetry_weblogs }} | |
| parametric_scenarios: ${{ steps.main.outputs.parametric_scenarios }} | |
| _experimental_parametric_job_matrix: ${{ steps.main.outputs._experimental_parametric_job_matrix }} | |
| strategy: | |
| fail-fast: false | |
| name: Compute workflow parameters | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.SYSTEM_TESTS_REF }} | |
| persist-credentials: false | |
| - name: Compute workflow parameters | |
| id: main | |
| run: | | |
| source venv/bin/activate | |
| python utils/scripts/compute-workflow-parameters.py ruby -g "all" >> $GITHUB_OUTPUT | |
| env: | |
| PYTHONPATH: "." | |
| build-harness: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - name: runner | |
| internal: system_tests/runner:latest | |
| - name: agent | |
| internal: system_tests/agent:latest | |
| - name: proxy | |
| internal: datadog/system-tests:proxy-v1 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| name: Build (${{ matrix.image.name }}) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.SYSTEM_TESTS_REF }} | |
| persist-credentials: false | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Pull released image | |
| run: | | |
| if docker pull ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest; then | |
| docker tag ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest ${{ matrix.image.internal }} | |
| fi | |
| - name: Build image | |
| run: | | |
| cache_from=() | |
| for tag in latest; do | |
| cache_from+=(--cache-from "${{ env.REPO }}/system-tests/${{ matrix.image.name }}:${tag}") | |
| done | |
| echo "cache args: ${cache_from[*]}" | |
| ./build.sh --images ${{ matrix.image.name }} --docker --extra-docker-args "${cache_from[*]}" | |
| - name: List images | |
| run: | | |
| docker image list | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - name: Tag image for CI run | |
| run: | |
| docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Push image for CI run | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Tag image for commit | |
| run: | |
| docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:g${{ github.sha }} | |
| - name: Push image for commit | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:g${{ github.sha }} | |
| - name: Tag image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | |
| docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest | |
| - name: Push image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest | |
| build-apps: | |
| needs: | |
| - compute-workflow-parameters | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - weblog | |
| library: | |
| - name: ruby | |
| repository: DataDog/dd-trace-rb | |
| path: dd-trace-rb | |
| app: | |
| - ${{ fromJson(needs.compute-workflow-parameters.outputs.endtoend_weblogs) }} | |
| - ${{ fromJson(needs.compute-workflow-parameters.outputs.graphql_weblogs) }} | |
| - ${{ fromJson(needs.compute-workflow-parameters.outputs.opentelemetry_weblogs) }} | |
| runs-on: ubuntu-latest | |
| name: Build (${{ matrix.app }}) | |
| permissions: | |
| packages: write | |
| outputs: | |
| FORCED_TESTS_LIST: ${{steps.read_forced_tests_list.outputs.FORCED_TESTS_LIST}} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.SYSTEM_TESTS_REF }} | |
| persist-credentials: false | |
| - name: Checkout ${{ matrix.library.repository }} | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: '${{ matrix.library.repository }}' | |
| path: 'binaries/${{ matrix.library.path }}' | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: Read forced-tests-list.json file | |
| id: read_forced_tests_list | |
| run: | | |
| echo "FORCED_TESTS_LIST<<EOF" >> $GITHUB_OUTPUT | |
| echo "$(cat binaries/dd-trace-rb/.github/forced-tests-list.json)" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Pull released image | |
| run: | | |
| if docker pull ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest; then | |
| docker tag ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest system_tests/${{ matrix.image }}:latest | |
| fi | |
| parents="$(cd 'binaries/${{ matrix.library.path }}' && git rev-list --parents -n 1 ${{ github.sha }})" | |
| for sha in $parents; do | |
| docker pull "${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${sha}" || true | |
| done | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - name: Build | |
| run: | | |
| cache_from=() | |
| for tag in latest; do | |
| cache_from+=(--cache-from "${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:${tag}") | |
| done | |
| parents="$(cd 'binaries/${{ matrix.library.path }}' && git rev-list --parents -n 1 ${{ github.sha }})" | |
| for sha in $parents; do | |
| cache_from+=(--cache-from ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${sha}) | |
| done | |
| echo "cache args: ${cache_from[*]}" | |
| ./build.sh --library ${{ matrix.library.name }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }} --extra-docker-args "${cache_from[*]}" | |
| - name: Tag image for CI run | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Push image for CI run | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Tag image for commit | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${{ github.sha }} | |
| - name: Push image for commit | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${{ github.sha }} | |
| - name: Tag image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest | |
| - name: Push image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest | |
| end-to-end-tests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| library: | |
| - ruby | |
| app: ${{ fromJson(needs.compute-workflow-parameters.outputs.endtoend_weblogs) }} # This will generate a LOT of jobs, we might want to run all the scenarios for a single weblog in a single job | |
| scenario: ${{ fromJson(needs.compute-workflow-parameters.outputs.endtoend_scenarios) }} # But is it possible without hardcoding them ? Even system-tests is hardcoding them. Maybe naming steps with matrix | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-harness | |
| - build-apps | |
| - compute-workflow-parameters | |
| name: Test (${{ matrix.app }}, ${{ matrix.scenario }}) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.SYSTEM_TESTS_REF }} | |
| persist-credentials: false | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Pull runner image | |
| run: | | |
| docker pull ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} system_tests/runner:latest | |
| - name: Pull proxy image | |
| run: | | |
| docker pull ${{ env.REPO }}/system-tests/proxy:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ${{ env.REPO }}/system-tests/proxy:gha${{ github.run_id }}-g${{ github.sha }} datadog/system-tests:proxy-v1 | |
| - name: Pull agent image | |
| run: | | |
| docker pull ${{ env.REPO }}/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ${{ env.REPO }}/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} system_tests/agent:latest | |
| - name: Pull app image | |
| run: | | |
| docker pull ${{ env.REPO }}/system-tests/${{ matrix.library }}/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ${{ env.REPO }}/system-tests/${{ matrix.library }}/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} system_tests/weblog:latest | |
| - name: List images | |
| run: | | |
| docker image list | |
| - name: Run scenario | |
| # The last escape sequence check if the JSON string exists, | |
| # then check if the current scenario is contained in it, then create a string by chaining -F and forced tests. | |
| # A potential error might come from an empty array in a scenario in the json file. | |
| run: | | |
| ./run.sh ++docker ${{ matrix.scenario }} \ | |
| ${{ env.FORCED_TESTS_LIST && fromJSON(env.FORCED_TESTS_LIST)[matrix.scenario] && format('-F {0}', join(fromJSON(env.FORCED_TESTS_LIST)[matrix.scenario], ' -F ')) || ''}} | |
| env: | |
| DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }} | |
| SYSTEM_TESTS_AWS_ACCESS_KEY_ID: ${{ secrets.SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID }} | |
| SYSTEM_TESTS_AWS_SECRET_ACCESS_KEY: ${{ secrets.SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY }} | |
| FORCED_TESTS_LIST: ${{ needs.build-apps.outputs.FORCED_TESTS_LIST }} | |
| - name: Archive logs | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} | |
| with: | |
| name: system-tests-${{ matrix.library }}-${{ matrix.app }}-${{ matrix.scenario }}-logs-gha${{ github.run_id }}-g${{ github.sha }} | |
| path: logs* | |
| # TODO: Add libinjection_scenarios, opentelemetry_scenarios, graphql_scenarios and maybe parametric (or in a different workflow) | |
| aggregate: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| library: | |
| - ruby | |
| app: ${{ fromJson(needs.compute-workflow-parameters.outputs.endtoend_weblogs) }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-harness | |
| - build-apps | |
| - end-to-end-tests | |
| - compute-workflow-parameters | |
| if: ${{ always() }} | |
| name: Aggregate (${{ matrix.app }}) | |
| steps: | |
| - name: Setup python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.SYSTEM_TESTS_REF }} | |
| persist-credentials: false | |
| - name: Retrieve logs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: system-tests-${{ matrix.library }}-${{ matrix.app }}-*-logs-gha${{ github.run_id }}-g${{ github.sha }} | |
| merge-multiple: true | |
| path: . | |
| - name: Print fancy log report | |
| run: | | |
| find logs* | |
| python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY | |
| cleanup-harness: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - runner | |
| - agent | |
| runs-on: ubuntu-latest | |
| needs: | |
| - end-to-end-tests | |
| if: ${{ always() }} | |
| name: Cleanup (${{ matrix.image }}) | |
| steps: | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - uses: actions/delete-package-versions@v5 | |
| with: | |
| package-version-ids: 'gha${{ github.run_id }}-g${{ github.sha }}' | |
| package-name: 'system-tests/${{ matrix.image }}' | |
| package-type: 'container' | |
| continue-on-error: true | |
| cleanup-apps: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: ${{ fromJson(needs.compute-workflow-parameters.outputs.endtoend_weblogs) }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - end-to-end-tests | |
| - compute-workflow-parameters | |
| if: ${{ always() }} | |
| name: Cleanup (${{ matrix.image }}) | |
| steps: | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - uses: actions/delete-package-versions@v5 | |
| with: | |
| package-version-ids: 'gha${{ github.run_id }}-g${{ github.sha }}' | |
| package-name: 'system-tests/weblog-${{ matrix.image }}' | |
| package-type: 'container' | |
| continue-on-error: true |