Skip to content

Commit

Permalink
Migrate automation-nightly-tests.yml
Browse files Browse the repository at this point in the history
  • Loading branch information
@lukaszcl
lukaszcl committed Sep 9, 2024
1 parent 9bc9744 commit cad06ca
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 290 deletions.
6 changes: 3 additions & 3 deletions .github/e2e-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -505,7 +505,7 @@ runner-test-matrix:
E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink
E2E_TEST_CHAINLINK_VERSION: latest
E2E_TEST_CHAINLINK_UPGRADE_IMAGE: '{{ env.QA_CHAINLINK_IMAGE }}'
E2E_TEST_CHAINLINK_UPGRADE_VERSION: develop
E2E_TEST_CHAINLINK_UPGRADE_VERSION: '{{ env.DEFAULT_CHAINLINK_UPGRADE_VERSION }}'
pyroscope_env: ci-smoke-automation-upgrade-tests

- id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1
Expand All @@ -519,7 +519,7 @@ runner-test-matrix:
E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink
E2E_TEST_CHAINLINK_VERSION: latest
E2E_TEST_CHAINLINK_UPGRADE_IMAGE: '{{ env.QA_CHAINLINK_IMAGE }}'
E2E_TEST_CHAINLINK_UPGRADE_VERSION: develop
E2E_TEST_CHAINLINK_UPGRADE_VERSION: '{{ env.DEFAULT_CHAINLINK_UPGRADE_VERSION }}'
pyroscope_env: ci-smoke-automation-upgrade-tests

- id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2
Expand All @@ -533,7 +533,7 @@ runner-test-matrix:
E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink
E2E_TEST_CHAINLINK_VERSION: latest
E2E_TEST_CHAINLINK_UPGRADE_IMAGE: '{{ env.QA_CHAINLINK_IMAGE }}'
E2E_TEST_CHAINLINK_UPGRADE_VERSION: develop
E2E_TEST_CHAINLINK_UPGRADE_VERSION: '{{ env.DEFAULT_CHAINLINK_UPGRADE_VERSION }}'
pyroscope_env: ci-smoke-automation-upgrade-tests

- id: reorg/automation_reorg_test.go^TestAutomationReorg/registry_2_0
Expand Down
315 changes: 28 additions & 287 deletions .github/workflows/automation-nightly-tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,291 +7,32 @@ on:
- "*"
workflow_dispatch:

env:
CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink

jobs:
build-chainlink:
environment: integration
permissions:
id-token: write
contents: read
name: Build Chainlink Image
runs-on: ubuntu22.04-16cores-64GB
steps:
- name: Collect Metrics
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
with:
id: automation-nightly-build-chainlink
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
this-job-name: Build Chainlink Image
continue-on-error: true
- name: Checkout the repo
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
- name: Build Chainlink Image
uses: ./.github/actions/build-chainlink-image
with:
tag_suffix: ""
dockerfile: core/chainlink.Dockerfile
git_commit_sha: ${{ github.sha }}
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

automation-upgrade-tests:
environment: integration
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
needs: [build-chainlink]
env:
CHAINLINK_COMMIT_SHA: ${{ github.sha }}
CHAINLINK_ENV_USER: ${{ github.actor }}
TEST_LOG_LEVEL: info
SELECTED_NETWORKS: "SIMULATED"
strategy:
fail-fast: false
matrix:
tests:
- name: Upgrade 2.0
id: upgrade-2-0
suite: smoke
nodes: 1
os: ubuntu22.04-8cores-32GB
network: SIMULATED
command: -run ^TestAutomationNodeUpgrade/registry_2_0 ./smoke
- name: Upgrade 2.1
id: upgrade-2-1
suite: smoke
nodes: 5
os: ubuntu22.04-8cores-32GB
network: SIMULATED
command: -run ^TestAutomationNodeUpgrade/registry_2_1 ./smoke
- name: Upgrade 2.2
id: upgrade-2-2
suite: smoke
nodes: 5
os: ubuntu22.04-8cores-32GB
network: SIMULATED
command: -run ^TestAutomationNodeUpgrade/registry_2_2 ./smoke
runs-on: ${{ matrix.tests.os }}
name: Automation ${{ matrix.tests.name }} Test
steps:
- name: Checkout the repo
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
ref: ${{ github.head_ref || github.ref_name }}
- name: Setup GAP for Grafana
uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # [email protected]
with:
# aws inputs
aws-region: ${{ secrets.AWS_REGION }}
aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
# other inputs
duplicate-authorization-header: "true"
- name: Run Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32
env:
TEST_SUITE: ${{ matrix.tests.suite }}
E2E_TEST_SELECTED_NETWORK: ${{ env.SELECTED_NETWORKS }}
E2E_TEST_CHAINLINK_IMAGE: "public.ecr.aws/chainlink/chainlink"
E2E_TEST_CHAINLINK_VERSION: "latest"
E2E_TEST_CHAINLINK_UPGRADE_IMAGE: ${{ env.CHAINLINK_IMAGE }}
E2E_TEST_CHAINLINK_UPGRADE_VERSION: ${{ github.sha }}
E2E_TEST_LOGGING_RUN_ID: ${{ github.run_id }}
E2E_TEST_LOG_COLLECT: "true"
E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
E2E_TEST_LOG_STREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary"
E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
with:
test_command_to_run: cd ./integration-tests && go test -timeout 60m -count=1 -json -test.parallel=${{ matrix.tests.nodes }} ${{ matrix.tests.command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
test_download_vendor_packages_command: cd ./integration-tests && go mod download
cl_repo: 'public.ecr.aws/chainlink/chainlink'
cl_image_tag: 'latest'
aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
artifacts_location: ./integration-tests/${{ matrix.tests.suite }}/logs
artifacts_name: testcontainers-logs-${{ matrix.tests.name }}
publish_check_name: Automation Results ${{ matrix.tests.name }}
token: ${{ secrets.GITHUB_TOKEN }}
go_mod_path: ./integration-tests/go.mod
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
- name: Upload test log
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: gotest-logs-${{ matrix.tests.name }}
path: /tmp/gotest.log
retention-days: 7
continue-on-error: true
- name: Collect Metrics
if: always()
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
with:
id: automation-nightly-upgrade-tests-${{ matrix.tests.id }}
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
this-job-name: Automation ${{ matrix.tests.name }} Test
test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
continue-on-error: true
- name: Print failed test summary
if: always()
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17

test-notify:
name: Start Slack Thread
if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }}
environment: integration
outputs:
thread_ts: ${{ steps.slack.outputs.thread_ts }}
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
runs-on: ubuntu-latest
needs: [ automation-upgrade-tests ]
steps:
- name: Debug Result
run: echo ${{ join(needs.*.result, ',') }}
- name: Main Slack Notification
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
id: slack
with:
channel-id: C03KJ5S7KEK
payload: |
{
"attachments": [
{
"color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Automation Nightly Tests ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}",
"emoji": true
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>"
}
}
]
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

test-results:
name: Post Test Results for ${{ matrix.name }}
if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }}
environment: integration
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
runs-on: ubuntu-latest
needs: test-notify
strategy:
fail-fast: false
matrix:
name: [ Upgrade 2.0, Upgrade 2.1, Upgrade 2.2 ]
steps:
- name: Get Results
id: test-results
run: |
# I feel like there's some clever, fully jq way to do this, but I ain't got the motivation to figure it out
echo "Querying test results"
PARSED_RESULTS=$(curl \
-H "Authorization: Bearer ${{ github.token }}" \
'https://api.github.com/repos/${{github.repository}}/actions/runs/${{ github.run_id }}/jobs' \
| jq -r --arg pattern "${{ matrix.name }} Test" '.jobs[]
| select(.name | test($pattern)) as $job
| $job.steps[]
| select(.name == "Run Tests")
| { conclusion: (if .conclusion == "success" then ":white_check_mark:" else ":x:" end), product: ("*" + ($job.name | capture($pattern).product) + "*") }')
echo "Parsed Results:"
echo $PARSED_RESULTS
ALL_SUCCESS=true
for row in $(echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")'); do
success=false
break
done
echo all_success=$ALL_SUCCESS >> $GITHUB_OUTPUT
FORMATTED_RESULTS=$(echo $PARSED_RESULTS | jq -s '[.[]
| {
conclusion: .conclusion,
product: .product
}
]
| map("{\"type\": \"section\", \"text\": {\"type\": \"mrkdwn\", \"text\": \"\(.product): \(.conclusion)\"}}")
| join(",")')
echo "Formatted Results:"
echo $FORMATTED_RESULTS
# Cleans out backslashes and quotes from jq
CLEAN_RESULTS=$(echo "$FORMATTED_RESULTS" | sed 's/\\\"/"/g' | sed 's/^"//;s/"$//')
echo "Clean Results"
echo $CLEAN_RESULTS
echo results=$CLEAN_RESULTS >> $GITHUB_OUTPUT
- name: Test Details
uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
with:
channel-id: C03KJ5S7KEK
payload: |
{
"thread_ts": "${{ needs.test-notify.outputs.thread_ts }}",
"attachments": [
{
"color": "${{ steps.test-results.outputs.all_success && '#2E7D32' || '#C62828' }}",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "${{ matrix.name }} ${{ steps.test-results.outputs.all_success && ':white_check_mark:' || ':x: Notifying <@U02Q14G80TY>'}}",
"emoji": true
}
},
{
"type": "divider"
},
${{ steps.test-results.outputs.results }}
]
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
run-e2e-tests-workflow:
name: Run E2E Tests
uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
with:
test_ids: 'smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1,smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2'
chainlink_version: ${{ github.sha }}
slack_notification_after_tests: true
slack_notification_after_tests_channel_id: "#lf-slack-notification-tests"
slack_notification_after_tests_name: Automation Nightly E2E Tests
# slack_notification_after_tests_notify_user_id_on_failure: U0XXXXXXX
secrets:
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
TEST_SECRETS_OVERRIDE_BASE64: ${{ secrets[inputs.test_secrets_override_key] }}
SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }}
1 change: 1 addition & 0 deletions .github/workflows/run-e2e-tests-reusable-workflow.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ env:
QA_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
DEFAULT_CHAINLINK_VERSION: ${{ inputs.chainlink_version || github.sha }}
DEFAULT_CHAINLINK_PLUGINS_VERSION: ${{ inputs.chainlink_version != '' && format('{0}-plugins', inputs.chainlink_version) || format('{0}-plugins', github.sha) }}
DEFAULT_CHAINLINK_UPGRADE_VERSION: ${{ inputs.chainlink_version || github.sha }}
CHAINLINK_ENV_USER: ${{ github.actor }}
CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
SELECTED_NETWORKS: SIMULATED
Expand Down

0 comments on commit cad06ca

Please sign in to comment.