Skip to content

check positional and keyword arg mix #299

check positional and keyword arg mix

check positional and keyword arg mix #299

Workflow file for this run

name: System Tests
on:
push:
branches:
- "**"
workflow_dispatch: {}
schedule:
- cron: '00 04 * * 2-6'
env:
REGISTRY: ghcr.io
REPO: ghcr.io/datadog/dd-trace-rb
ST_REF: main
FORCE_TESTS:
FORCE_TESTS_SCENARIO:
jobs:
build-harness:
strategy:
fail-fast: false
matrix:
image:
- name: runner
internal: system_tests/runner:latest
- name: agent
internal: system_tests/agent:latest
- name: proxy
internal: datadog/system-tests:proxy-v1
runs-on: ubuntu-latest
permissions:
packages: write
name: Build (${{ matrix.image.name }})
steps:
- name: Checkout
uses: actions/checkout@v4
with:
repository: 'DataDog/system-tests'
ref: ${{ env.ST_REF }}
persist-credentials: false
- name: Pull released image
run: |
if docker pull ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest; then
docker tag ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest ${{ matrix.image.internal }}
fi
- name: Build image
run: |
cache_from=()
for tag in latest; do
cache_from+=(--cache-from "${{ env.REPO }}/system-tests/${{ matrix.image.name }}:${tag}")
done
echo "cache args: ${cache_from[*]}"
./build.sh --images ${{ matrix.image.name }} --docker --extra-docker-args "${cache_from[*]}"
- name: List images
run: |
docker image list
- name: Log in to the Container registry
run: |
echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
- name: Tag image for CI run
run:
docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:gha${{ github.run_id }}-g${{ github.sha }}
- name: Push image for CI run
run: |
docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:gha${{ github.run_id }}-g${{ github.sha }}
- name: Tag image for commit
run:
docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:g${{ github.sha }}
- name: Push image for commit
run: |
docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:g${{ github.sha }}
- name: Tag image for release
if: ${{ github.ref == 'refs/heads/master' }}
run:
docker tag ${{ matrix.image.internal }} ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest
- name: Push image for release
if: ${{ github.ref == 'refs/heads/master' }}
run: |
docker push ${{ env.REPO }}/system-tests/${{ matrix.image.name }}:latest
build-apps:
strategy:
fail-fast: false
matrix:
image:
- weblog
library:
- name: ruby
repository: DataDog/dd-trace-rb
path: dd-trace-rb
app:
- rack
- sinatra14
- sinatra20
- sinatra21
- sinatra22
- sinatra30
- sinatra31
- sinatra32
- sinatra40
- rails42
- rails50
- rails51
- rails52
- rails60
- rails61
- rails70
- rails71
- graphql23
runs-on: ubuntu-latest
name: Build (${{ matrix.app }})
permissions:
packages: write
steps:
- name: Checkout
uses: actions/checkout@v4
with:
repository: 'DataDog/system-tests'
ref: ${{ env.ST_REF }}
persist-credentials: false
- name: Checkout ${{ matrix.library.repository }}
uses: actions/checkout@v4
with:
repository: '${{ matrix.library.repository }}'
path: 'binaries/${{ matrix.library.path }}'
fetch-depth: 2
persist-credentials: false
- name: Pull released image
run: |
if docker pull ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest; then
docker tag ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest system_tests/${{ matrix.image }}:latest
fi
parents="$(cd 'binaries/${{ matrix.library.path }}' && git rev-list --parents -n 1 ${{ github.sha }})"
for sha in $parents; do
docker pull "${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${sha}" || true
done
- name: Log in to the Container registry
run: |
echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
- name: Build
run: |
cache_from=()
for tag in latest; do
cache_from+=(--cache-from "${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:${tag}")
done
parents="$(cd 'binaries/${{ matrix.library.path }}' && git rev-list --parents -n 1 ${{ github.sha }})"
for sha in $parents; do
cache_from+=(--cache-from ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${sha})
done
echo "cache args: ${cache_from[*]}"
./build.sh --library ${{ matrix.library.name }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }} --extra-docker-args "${cache_from[*]}"
- name: Tag image for CI run
run:
docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }}
- name: Push image for CI run
run: |
docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }}
- name: Tag image for commit
run:
docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${{ github.sha }}
- name: Push image for commit
run: |
docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:g${{ github.sha }}
- name: Tag image for release
if: ${{ github.ref == 'refs/heads/master' }}
run:
docker tag system_tests/${{ matrix.image }}:latest ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest
- name: Push image for release
if: ${{ github.ref == 'refs/heads/master' }}
run: |
docker push ${{ env.REPO }}/system-tests/${{ matrix.library.name }}/${{ matrix.image }}-${{ matrix.app }}:latest
test:
strategy:
fail-fast: false
matrix:
library:
- ruby
app:
- rack
- sinatra14
- sinatra20
- sinatra21
- sinatra22
- sinatra30
- sinatra31
- sinatra32
- sinatra40
- rails42
- rails50
- rails51
- rails52
- rails60
- rails61
- rails70
- rails71
scenario:
- DEFAULT
- APPSEC_DISABLED
- APPSEC_BLOCKING_FULL_DENYLIST
- APPSEC_REQUEST_BLOCKING
- APPSEC_STANDALONE
include:
- library: ruby
app: rack
scenario: INTEGRATIONS
- library: ruby
app: rack
scenario: CROSSED_TRACING_LIBRARIES
- library: ruby
app: rack
scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
- library: ruby
app: rack
scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
- library: ruby
app: rack
scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE
- library: ruby
app: rack
scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE
- library: ruby
app: rack
scenario: APPSEC_CUSTOM_RULES
- library: ruby
app: rack
scenario: APPSEC_MISSING_RULES
- library: ruby
app: rack
scenario: APPSEC_CORRUPTED_RULES
- library: ruby
app: rack
scenario: APPSEC_LOW_WAF_TIMEOUT
- library: ruby
app: rack
scenario: APPSEC_CUSTOM_OBFUSCATION
- library: ruby
app: rack
scenario: APPSEC_RATE_LIMITER
- library: ruby
app: rails70
scenario: APPSEC_AUTO_EVENTS_EXTENDED
- library: ruby
app: rails70
scenario: APPSEC_API_SECURITY
- library: ruby
app: rack
scenario: APPSEC_RULES_MONITORING_WITH_ERRORS
- library: ruby
app: rack
scenario: SAMPLING
- library: ruby
app: rack
scenario: PROFILING
- library: ruby
app: rack
scenario: TELEMETRY_APP_STARTED_PRODUCTS_DISABLED
- library: ruby
app: rack
scenario: TELEMETRY_DEPENDENCY_LOADED_TEST_FOR_DEPENDENCY_COLLECTION_DISABLED
- library: ruby
app: rack
scenario: TELEMETRY_LOG_GENERATION_DISABLED
- library: ruby
app: rack
scenario: TELEMETRY_METRIC_GENERATION_DISABLED
- library: ruby
app: graphql23
scenario: GRAPHQL_APPSEC
runs-on: ubuntu-latest
needs:
- build-harness
- build-apps
name: Test (${{ matrix.app }}, ${{ matrix.scenario }})
steps:
- name: Checkout
uses: actions/checkout@v4
with:
repository: 'DataDog/system-tests'
ref: ${{ env.ST_REF }}
persist-credentials: false
- name: Pull runner image
run: |
docker pull ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }}
docker tag ${{ env.REPO }}/system-tests/runner:gha${{ github.run_id }}-g${{ github.sha }} system_tests/runner:latest
- name: Pull proxy image
run: |
docker pull ${{ env.REPO }}/system-tests/proxy:gha${{ github.run_id }}-g${{ github.sha }}
docker tag ${{ env.REPO }}/system-tests/proxy:gha${{ github.run_id }}-g${{ github.sha }} datadog/system-tests:proxy-v1
- name: Pull agent image
run: |
docker pull ${{ env.REPO }}/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }}
docker tag ${{ env.REPO }}/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} system_tests/agent:latest
- name: Pull app image
run: |
docker pull ${{ env.REPO }}/system-tests/${{ matrix.library }}/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }}
docker tag ${{ env.REPO }}/system-tests/${{ matrix.library }}/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} system_tests/weblog:latest
- name: List images
run: |
docker image list
- name: Run scenario
run: |
./run.sh ++docker ${{ matrix.scenario }} ${{matrix.scenario == env.FORCE_TESTS_SCENARIO && env.FORCE_TESTS || ''}}
env:
DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }}
SYSTEM_TESTS_AWS_ACCESS_KEY_ID: ${{ secrets.SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID }}
SYSTEM_TESTS_AWS_SECRET_ACCESS_KEY: ${{ secrets.SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY }}
- name: Archive logs
uses: actions/upload-artifact@v4
if: ${{ always() }}
with:
name: system-tests-${{ matrix.library }}-${{ matrix.app }}-${{ matrix.scenario }}-logs-gha${{ github.run_id }}-g${{ github.sha }}
path: logs*
aggregate:
strategy:
fail-fast: false
matrix:
library:
- ruby
app:
- rack
- sinatra14
- sinatra20
- sinatra21
- sinatra22
- sinatra30
- sinatra31
- sinatra32
- sinatra40
- rails42
- rails50
- rails51
- rails52
- rails60
- rails61
- rails70
- rails71
- graphql23
runs-on: ubuntu-latest
needs:
- build-harness
- build-apps
- test
if: ${{ always() }}
name: Aggregate (${{ matrix.app }})
steps:
- name: Setup python 3.12
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Checkout
uses: actions/checkout@v4
with:
repository: 'DataDog/system-tests'
ref: ${{ env.ST_REF }}
persist-credentials: false
- name: Retrieve logs
uses: actions/download-artifact@v4
with:
pattern: system-tests-${{ matrix.library }}-${{ matrix.app }}-*-logs-gha${{ github.run_id }}-g${{ github.sha }}
merge-multiple: true
path: .
- name: Print fancy log report
run: |
find logs*
python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY
cleanup:
strategy:
fail-fast: false
matrix:
image:
- runner
- agent
- weblog-rack
- weblog-sinatra14
- weblog-sinatra20
- weblog-sinatra21
- weblog-sinatra22
- weblog-sinatra30
- weblog-sinatra31
- weblog-sinatra32
- weblog-sinatra40
- weblog-rails42
- weblog-rails50
- weblog-rails51
- weblog-rails52
- weblog-rails60
- weblog-rails61
- weblog-rails70
- weblog-graphql23
runs-on: ubuntu-latest
needs:
- test
if: ${{ always() }}
name: Cleanup (${{ matrix.image }})
steps:
- name: Log in to the Container registry
run: |
echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin
- uses: actions/delete-package-versions@v5
with:
package-version-ids: 'gha${{ github.run_id }}-g${{ github.sha }}'
package-name: 'system-tests/${{ matrix.image }}'
package-type: 'container'
continue-on-error: true