From 9b67fb73c25310d97000556cbdd635540179771a Mon Sep 17 00:00:00 2001 From: Antonis Stamatiou Date: Fri, 13 Dec 2024 16:03:22 +0200 Subject: [PATCH] feat: Decouple reporting from actual testing (#29587) * feat: Decouple reporting from actual testing * fix: Added also PR number for comment * feat: Add also mmctl publishing --- .github/workflows/mmctl-test-template.yml | 13 ++- .github/workflows/server-ci-report.yml | 95 ++++++++++++++++++++++ .github/workflows/server-test-template.yml | 46 ++--------- 3 files changed, 107 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/server-ci-report.yml diff --git a/.github/workflows/mmctl-test-template.yml b/.github/workflows/mmctl-test-template.yml index 9bd573c6397a..9bceb0b09ebe 100644 --- a/.github/workflows/mmctl-test-template.yml +++ b/.github/workflows/mmctl-test-template.yml @@ -28,6 +28,10 @@ jobs: id: go working-directory: ./server run: echo GO_VERSION=$(cat .go-version) >> "${GITHUB_OUTPUT}" + - name: Store required variables for publishing results + run: | + echo "${{ inputs.name }}" > server/test-name + echo "${{ github.event.pull_request.number }}" > server/pr-number - name: Setup Go uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 with: @@ -80,10 +84,5 @@ jobs: path: | server/gotestsum.json server/report.xml - - name: Publish Test Report - uses: mikepenz/action-junit-report@992d97d6eb2e5f3de985fbf9df6a04386874114d # v5.1.0 - if: success() || failure() # always run even if the previous step fails - with: - report_paths: server/report.xml - check_name: ${{ inputs.name }} (Results) - job_name: ${{ inputs.name }} + server/test-name + server/pr-number diff --git a/.github/workflows/server-ci-report.yml b/.github/workflows/server-ci-report.yml new file mode 100644 index 000000000000..5d667bcca8fc --- /dev/null +++ b/.github/workflows/server-ci-report.yml @@ -0,0 +1,95 @@ +name: Server CI Report + +on: + workflow_run: + workflows: + - "Server CI PR" + - "Server CI Master" + types: + - completed + +jobs: + generate-report-matrix: + runs-on: ubuntu-22.04 + outputs: + REPORT_MATRIX: ${{ steps.report.outputs.REPORT_MATRIX }} + steps: + - name: report/download-artifacts-from-PR-workflow + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + pattern: "*-test-logs" + path: reports + + - name: report/generate-report-matrix + id: report + run: | + find "reports" -type f -name "test-name" | while read -r test_file; do + folder=$(basename "$(dirname "$test_file")") + test_name=$(cat "$test_file") + echo "{\"artifact\": \"$folder\", \"name\": \"$test_name\"}" + done | jq -s '{ "test": . }' | tee /tmp/report-matrix + echo REPORT_MATRIX=$(cat /tmp/report-matrix | jq --compact-output --monochrome-output) >> ${GITHUB_OUTPUT} + + publish-report: + runs-on: ubuntu-22.04 + name: Publish Report ${{ matrix.test.name }} + needs: + - generate-report-matrix + permissions: + pull-requests: write + checks: write + issues: write + strategy: + matrix: ${{ fromJson(needs.generate-report-matrix.outputs.REPORT_MATRIX) }} + steps: + - name: report/download-artifacts-from-PR-workflow + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + name: ${{ matrix.test.artifact }} + path: ${{ matrix.test.artifact }} + - name: report/fetch-pr-number + if: github.event.workflow_run.name == 'Server CI PR' + id: incoming-pr + run: echo "NUMBER=$(cat ${{ matrix.test.artifact }}/pr-number)" >> ${GITHUB_OUTPUT} + - name: Publish test report + id: report + uses: mikepenz/action-junit-report@992d97d6eb2e5f3de985fbf9df6a04386874114d # v5.1.0 + with: + report_paths: ${{ matrix.test.artifact }}/report.xml + check_name: ${{ matrix.test.name }} (Results) + job_name: ${{ matrix.test.name }} + commit: ${{ github.event.workflow_run.head_commit.id }} + require_tests: true + check_retries: true + flaky_summary: true + include_passed: true + check_annotations: true + + - name: Report retried tests via webhook (master || release-*) + if: ${{ steps.report.outputs.flaky_summary != '
TestRetries
' && github.event.workflow_run.name == 'Server CI Master' }} + uses: mattermost/action-mattermost-notify@b7d118e440bf2749cd18a4a8c88e7092e696257a # v2.0.0 + with: + MATTERMOST_WEBHOOK_URL: ${{ secrets.MM_COMMUNITY_DEVELOPERS_INCOMING_WEBHOOK_FROM_GH_ACTIONS }} + TEXT: |- + #### ⚠️ One or more flaky tests detected ⚠️ + * Failing job: [github.com/mattermost/mattermost:${{ matrix.test.name }}](${{ github.event.workflow_run.html_url }}) + * Ideally, this would have been caught in a pull request, but now a volunteer is required. If you're willing to help, submit a separate pull request to skip the flaky tests (e.g. [23360](https://github.com/mattermost/mattermost/pull/23360)) and file JIRA ticket (e.g. [MM-52743](https://mattermost.atlassian.net/browse/MM-52743)) for later investigation. + * Finally, reply to this message with a link to the created JIRA ticket. + + - name: Report retried tests (pull request) + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + if: ${{ steps.report.outputs.flaky_summary != '
TestRetries
' && github.event.workflow_run.name == 'Server CI PR' }} + with: + script: | + const body = `#### ⚠️ One or more flaky tests detected ⚠️\n* Failing job: [github.com/mattermost/mattermost:${{ matrix.test.name }}](${{ github.event.workflow_run.html_url }})\n* Double check your code to ensure you haven't introduced a flaky test.\n* If this seems to be unrelated to your changes, submit a separate pull request to skip the flaky tests (e.g. [23360](https://github.com/mattermost/mattermost/pull/23360)) and file JIRA ticket (e.g. [MM-52743](https://mattermost.atlassian.net/browse/MM-52743)) for later investigation.\n\n${{ steps.report.outputs.flaky_summary }}` + + await github.rest.issues.createComment({ + issue_number: ${{ steps.incoming-pr.outputs.NUMBER }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: body + }) diff --git a/.github/workflows/server-test-template.yml b/.github/workflows/server-test-template.yml index b7d27d21369b..00fa9bc68863 100644 --- a/.github/workflows/server-test-template.yml +++ b/.github/workflows/server-test-template.yml @@ -19,10 +19,6 @@ jobs: test: name: ${{ inputs.name }} runs-on: ubuntu-22.04 - permissions: - pull-requests: write - checks: write - issues: write env: COMPOSE_PROJECT_NAME: ghactions steps: @@ -37,6 +33,10 @@ jobs: with: go-version: ${{ steps.go.outputs.GO_VERSION }} cache-dependency-path: server/go.sum + - name: Store required variables for publishing results + run: | + echo "${{ inputs.name }}" > server/test-name + echo "${{ github.event.pull_request.number }}" > server/pr-number - name: Run docker compose run: | cd server/build @@ -76,39 +76,5 @@ jobs: path: | server/gotestsum.json server/report.xml - - name: Publish test report - id: report - uses: mikepenz/action-junit-report@992d97d6eb2e5f3de985fbf9df6a04386874114d # v5.1.0 - if: success() || failure() # always run even if the previous step fails - with: - report_paths: server/report.xml - check_name: ${{ inputs.name }} (Results) - job_name: ${{ inputs.name }} - require_tests: true - check_retries: true - flaky_summary: true - include_passed: true - check_annotations: true - - name: Report retried tests via webhook (master || release-*) - if: ${{ steps.report.outputs.flaky_summary != '
TestRetries
' && (github.ref_name == 'master' || startsWith(github.ref_name, 'release-')) }} - uses: mattermost/action-mattermost-notify@b7d118e440bf2749cd18a4a8c88e7092e696257a # v2.0.0 - with: - MATTERMOST_WEBHOOK_URL: ${{ secrets.MM_COMMUNITY_DEVELOPERS_INCOMING_WEBHOOK_FROM_GH_ACTIONS }} - TEXT: |- - #### ⚠️ One or more flaky tests detected ⚠️ - * Failing job: [github.com/mattermost/mattermost:${{ inputs.name }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) - * Ideally, this would have been caught in a pull request, but now a volunteer is required. If you're willing to help, submit a separate pull request to skip the flaky tests (e.g. [23360](https://github.com/mattermost/mattermost/pull/23360)) and file JIRA ticket (e.g. [MM-52743](https://mattermost.atlassian.net/browse/MM-52743)) for later investigation. - * Finally, reply to this message with a link to the created JIRA ticket. - - name: Report retried tests (pull request) - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - if: ${{ steps.report.outputs.flaky_summary != '
TestRetries
' && !(github.ref_name == 'master' || startsWith(github.ref_name, 'release-')) }} - with: - script: | - const body = `#### ⚠️ One or more flaky tests detected ⚠️\n* Failing job: [github.com/mattermost/mattermost:${{ inputs.name }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})\n* Double check your code to ensure you haven't introduced a flaky test.\n* If this seems to be unrelated to your changes, submit a separate pull request to skip the flaky tests (e.g. [23360](https://github.com/mattermost/mattermost/pull/23360)) and file JIRA ticket (e.g. [MM-52743](https://mattermost.atlassian.net/browse/MM-52743)) for later investigation.\n\n${{ steps.report.outputs.flaky_summary }}` - - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: body - }) + server/test-name + server/pr-number \ No newline at end of file