Renamed appsec_standalone_reject? to non_billing_reject? #12028
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: System Tests | |
| on: | |
| push: | |
| branches: | |
| - "**" | |
| workflow_dispatch: {} | |
| schedule: | |
| - cron: '00 04 * * 2-6' | |
| env: | |
| REGISTRY: ghcr.io | |
| REPO: ghcr.io/datadog/dd-trace-rb | |
| SYSTEM_TESTS_REF: igor/standalone-sca # This must always be set to `main` on dd-trace-rb's master branch | |
| jobs: | |
| build-harness: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - name: runner | |
| internal: system_tests/runner:latest | |
| - name: agent | |
| internal: system_tests/agent:latest | |
| - name: proxy | |
| internal: datadog/system-tests:proxy-v1 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| name: Build (${{ matrix.image.name }}) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.SYSTEM_TESTS_REF }} | |
| persist-credentials: false | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Pull released image | |
| run: | | |
| if docker pull ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest; then | |
| docker tag ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest ${{ matrix.image.internal }} | |
| fi | |
| - name: Build image | |
| run: | | |
| cache_from=() | |
| for tag in latest; do | |
| cache_from+=(--cache-from "${{ env.REPO }}/system-tests/${{ matrix.image.name }}:${tag}") | |
| done | |
| echo "cache args: ${cache_from[*]}" | |
| ./build.sh --images ${{ matrix.image.name }} --docker --extra-docker-args "${cache_from[*]}" | |
| - name: List images | |
| run: | | |
| docker image list | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - name: Tag image for CI run | |
| run: | |
| docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Push image for CI run | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Tag image for commit | |
| run: | |
| docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:g${{ github.sha }} | |
| - name: Push image for commit | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:g${{ github.sha }} | |
| - name: Tag image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | |
| docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest | |
| - name: Push image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest | |
| build-apps: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - weblog | |
| library: | |
| - name: ruby | |
| repository: DataDog/dd-trace-rb | |
| path: dd-trace-rb | |
| app: | |
| - rack | |
| - sinatra14 | |
| - sinatra20 | |
| - sinatra21 | |
| - sinatra22 | |
| - sinatra30 | |
| - sinatra31 | |
| - sinatra32 | |
| - sinatra40 | |
| - rails42 | |
| - rails50 | |
| - rails51 | |
| - rails52 | |
| - rails60 | |
| - rails61 | |
| - rails70 | |
| - rails71 | |
| - rails72 | |
| - rails80 | |
| - graphql23 | |
| runs-on: ubuntu-latest | |
| name: Build (${{ matrix.app }}) | |
| permissions: | |
| packages: write | |
| outputs: | |
| FORCED_TESTS_LIST: ${{steps.read_forced_tests_list.outputs.FORCED_TESTS_LIST}} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.SYSTEM_TESTS_REF }} | |
| persist-credentials: false | |
| - name: Checkout ${{ matrix.library.repository }} | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: '${{ matrix.library.repository }}' | |
| path: 'binaries/${{ matrix.library.path }}' | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: Read forced-tests-list.json file | |
| id: read_forced_tests_list | |
| run: | | |
| echo "FORCED_TESTS_LIST<<EOF" >> $GITHUB_OUTPUT | |
| echo "$(cat binaries/dd-trace-rb/.github/forced-tests-list.json)" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Pull released image | |
| run: | | |
| if docker pull ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest; then | |
| docker tag ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest system_tests/${{ matrix.image }}:latest | |
| fi | |
| parents="$(cd 'binaries/${{ matrix.library.path }}' && git rev-list --parents -n 1 ${{ github.sha }})" | |
| for sha in $parents; do | |
| docker pull "${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${sha}" || true | |
| done | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - name: Build | |
| run: | | |
| cache_from=() | |
| for tag in latest; do | |
| cache_from+=(--cache-from "${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:${tag}") | |
| done | |
| parents="$(cd 'binaries/${{ matrix.library.path }}' && git rev-list --parents -n 1 ${{ github.sha }})" | |
| for sha in $parents; do | |
| cache_from+=(--cache-from ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${sha}) | |
| done | |
| echo "cache args: ${cache_from[*]}" | |
| ./build.sh --library ${{ matrix.library.name }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }} --extra-docker-args "${cache_from[*]}" | |
| - name: Tag image for CI run | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Push image for CI run | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Tag image for commit | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${{ github.sha }} | |
| - name: Push image for commit | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${{ github.sha }} | |
| - name: Tag image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest | |
| - name: Push image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | | |
| docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest | |
| test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| library: | |
| - ruby | |
| app: | |
| - rack | |
| - sinatra14 | |
| - sinatra20 | |
| - sinatra21 | |
| - sinatra22 | |
| - sinatra30 | |
| - sinatra31 | |
| - sinatra32 | |
| - sinatra40 | |
| - rails42 | |
| - rails50 | |
| - rails51 | |
| - rails52 | |
| - rails60 | |
| - rails61 | |
| - rails70 | |
| - rails71 | |
| - rails72 | |
| - rails80 | |
| scenario: | |
| - DEFAULT | |
| - APPSEC_DISABLED | |
| - APPSEC_BLOCKING_FULL_DENYLIST | |
| - APPSEC_REQUEST_BLOCKING | |
| - APPSEC_STANDALONE | |
| include: | |
| - library: ruby | |
| app: rack | |
| scenario: INTEGRATIONS | |
| - library: ruby | |
| app: rack | |
| scenario: CROSSED_TRACING_LIBRARIES | |
| - library: ruby | |
| app: rack | |
| scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD | |
| - library: ruby | |
| app: rack | |
| scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES | |
| - library: ruby | |
| app: rack | |
| scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE | |
| - library: ruby | |
| app: rack | |
| scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_CUSTOM_RULES | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_MISSING_RULES | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_CORRUPTED_RULES | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_LOW_WAF_TIMEOUT | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_CUSTOM_OBFUSCATION | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_RATE_LIMITER | |
| - library: ruby | |
| app: rails70 | |
| scenario: APPSEC_AUTO_EVENTS_EXTENDED | |
| - library: ruby | |
| app: rails70 | |
| scenario: APPSEC_API_SECURITY | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_RULES_MONITORING_WITH_ERRORS | |
| - library: ruby | |
| app: rails70 | |
| scenario: SCA_STANDALONE | |
| - library: ruby | |
| app: rack | |
| scenario: SAMPLING | |
| - library: ruby | |
| app: rack | |
| scenario: PROFILING | |
| - library: ruby | |
| app: rack | |
| scenario: TELEMETRY_APP_STARTED_PRODUCTS_DISABLED | |
| - library: ruby | |
| app: rack | |
| scenario: TELEMETRY_DEPENDENCY_LOADED_TEST_FOR_DEPENDENCY_COLLECTION_DISABLED | |
| - library: ruby | |
| app: rack | |
| scenario: TELEMETRY_LOG_GENERATION_DISABLED | |
| - library: ruby | |
| app: rack | |
| scenario: TELEMETRY_METRIC_GENERATION_DISABLED | |
| - library: ruby | |
| app: graphql23 | |
| scenario: GRAPHQL_APPSEC | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-harness | |
| - build-apps | |
| name: Test (${{ matrix.app }}, ${{ matrix.scenario }}) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.SYSTEM_TESTS_REF }} | |
| persist-credentials: false | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Pull runner image | |
| run: | | |
| docker pull ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} system_tests/runner:latest | |
| - name: Pull proxy image | |
| run: | | |
| docker pull ${{ env.REPO }}/system-tests/proxy:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ${{ env.REPO }}/system-tests/proxy:gha${{ github.run_id }}-g${{ github.sha }} datadog/system-tests:proxy-v1 | |
| - name: Pull agent image | |
| run: | | |
| docker pull ${{ env.REPO }}/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ${{ env.REPO }}/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} system_tests/agent:latest | |
| - name: Pull app image | |
| run: | | |
| docker pull ${{ env.REPO }}/system-tests/${{ matrix.library }}/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ${{ env.REPO }}/system-tests/${{ matrix.library }}/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} system_tests/weblog:latest | |
| - name: List images | |
| run: | | |
| docker image list | |
| - name: Run scenario | |
| # The last escape sequence check if the JSON string exists, | |
| # then check if the current scenario is contained in it, then create a string by chaining -F and forced tests. | |
| # A potential error might come from an empty array in a scenario in the json file. | |
| run: | | |
| ./run.sh ++docker ${{ matrix.scenario }} \ | |
| ${{ env.FORCED_TESTS_LIST && fromJSON(env.FORCED_TESTS_LIST)[matrix.scenario] && format('-F {0}', join(fromJSON(env.FORCED_TESTS_LIST)[matrix.scenario], ' -F ')) || ''}} | |
| env: | |
| DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }} | |
| SYSTEM_TESTS_AWS_ACCESS_KEY_ID: ${{ secrets.SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID }} | |
| SYSTEM_TESTS_AWS_SECRET_ACCESS_KEY: ${{ secrets.SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY }} | |
| FORCED_TESTS_LIST: ${{ needs.build-apps.outputs.FORCED_TESTS_LIST }} | |
| - name: Archive logs | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} | |
| with: | |
| name: system-tests-${{ matrix.library }}-${{ matrix.app }}-${{ matrix.scenario }}-logs-gha${{ github.run_id }}-g${{ github.sha }} | |
| path: logs* | |
| aggregate: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| library: | |
| - ruby | |
| app: | |
| - rack | |
| - sinatra14 | |
| - sinatra20 | |
| - sinatra21 | |
| - sinatra22 | |
| - sinatra30 | |
| - sinatra31 | |
| - sinatra32 | |
| - sinatra40 | |
| - rails42 | |
| - rails50 | |
| - rails51 | |
| - rails52 | |
| - rails60 | |
| - rails61 | |
| - rails70 | |
| - rails71 | |
| - rails72 | |
| - rails80 | |
| - graphql23 | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-harness | |
| - build-apps | |
| - test | |
| if: ${{ always() }} | |
| name: Aggregate (${{ matrix.app }}) | |
| steps: | |
| - name: Setup python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| ref: ${{ env.SYSTEM_TESTS_REF }} | |
| persist-credentials: false | |
| - name: Retrieve logs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: system-tests-${{ matrix.library }}-${{ matrix.app }}-*-logs-gha${{ github.run_id }}-g${{ github.sha }} | |
| merge-multiple: true | |
| path: . | |
| - name: Print fancy log report | |
| run: | | |
| find logs* | |
| python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY | |
| cleanup: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - runner | |
| - agent | |
| - weblog-rack | |
| - weblog-sinatra14 | |
| - weblog-sinatra20 | |
| - weblog-sinatra21 | |
| - weblog-sinatra22 | |
| - weblog-sinatra30 | |
| - weblog-sinatra31 | |
| - weblog-sinatra32 | |
| - weblog-sinatra40 | |
| - weblog-rails42 | |
| - weblog-rails50 | |
| - weblog-rails51 | |
| - weblog-rails52 | |
| - weblog-rails60 | |
| - weblog-rails61 | |
| - weblog-rails70 | |
| - weblog-rails71 | |
| - weblog-rails72 | |
| - weblog-rails80 | |
| - weblog-graphql23 | |
| runs-on: ubuntu-latest | |
| needs: | |
| - test | |
| if: ${{ always() }} | |
| name: Cleanup (${{ matrix.image }}) | |
| steps: | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - uses: actions/delete-package-versions@v5 | |
| with: | |
| package-version-ids: 'gha${{ github.run_id }}-g${{ github.sha }}' | |
| package-name: 'system-tests/${{ matrix.image }}' | |
| package-type: 'container' | |
| continue-on-error: true |