e2e tests at push on Automattic/wpcom-site-helper trunk branch #88518
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - '**.md' | |
| repository_dispatch: | |
| types: [ 'e2e tests**' ] | |
| concurrency: | |
| group: e2e-tests-${{ github.event_name }}-${{ github.ref }}-${{ github.event.action }} | |
| cancel-in-progress: true | |
| env: | |
| # Work around a bug in node 18.18.0. See https://github.com/webpack-contrib/thread-loader/issues/191 for details. | |
| UV_USE_IO_URING: 0 | |
| jobs: | |
| create-test-matrix: | |
| name: "Determine tests matrix" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 # 2023-09-15: The pnpm install may take a few minutes on cache miss. | |
| # Only run tests in the main repository | |
| if: github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name | |
| outputs: | |
| matrix: ${{ steps.evaluate.outputs.matrix }} | |
| build-matrix: ${{ steps.evaluate.outputs.build-matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # For pull requests, list-changed-projects.sh needs the merge base. | |
| - name: Deepen to merge base | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/deepen-to-merge-base | |
| - name: Setup tools | |
| uses: ./.github/actions/tool-setup | |
| # Required for list-changed-projects.sh | |
| - name: Install monorepo | |
| run: | | |
| pnpm install | |
| - name: Create test plan | |
| id: evaluate | |
| env: | |
| DISPATCH_REPO: ${{ github.event.client_payload.repository }} | |
| REF_NAME: ${{ github.event.client_payload.ref_name }} | |
| REF_TYPE: ${{ github.event.client_payload.ref_type }} | |
| run: | | |
| MATRIX="$(node .github/files/e2e-tests/e2e-matrix.js)" | |
| echo "matrix: $MATRIX" | |
| echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" | |
| if [[ "$GITHUB_EVENT_NAME" == repository_dispatch ]]; then | |
| echo "No build needed for $GITHUB_EVENT_NAME" | |
| BUILD_MATRIX="[]" | |
| else | |
| BUILD_MATRIX=$(jq -c '[ .[] | select( .suite | startswith( "atomic" ) | not ) | { buildGroup: .buildGroup, path: .path } ] | unique' <<<"$MATRIX") | |
| echo "build matrix: $BUILD_MATRIX" | |
| BAD=$( jq -r '[ group_by( .buildGroup )[] | select( length > 1 ) | .[0].buildGroup ] | unique | join( "," )' <<<"$BUILD_MATRIX" ) | |
| if [[ -n "$BAD" ]]; then | |
| echo "::error::One or more build groups have multiple paths: $BAD" | |
| exit 1 | |
| fi | |
| fi | |
| echo "build-matrix=$BUILD_MATRIX" >> "$GITHUB_OUTPUT" | |
| build-projects: | |
| name: "E2E: Build ${{ matrix.buildGroup }}" | |
| runs-on: ubuntu-latest | |
| needs: create-test-matrix | |
| timeout-minutes: 30 | |
| if: needs.create-test-matrix.outputs.build-matrix != '[]' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson( needs.create-test-matrix.outputs.build-matrix ) }} | |
| steps: | |
| - name: Ensure ${{ matrix.buildGroup }} build cache | |
| id: jetpack-build-cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| lookup-only: true | |
| path: | | |
| . | |
| !./.github/ | |
| key: ${{ matrix.buildGroup }}-${{ github.sha }} | |
| - name: Checkout code | |
| if: steps.jetpack-build-cache.outputs.cache-hit != 'true' | |
| uses: actions/checkout@v4 | |
| - name: Setup tools | |
| if: steps.jetpack-build-cache.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/tool-setup | |
| - name: Install monorepo | |
| if: steps.jetpack-build-cache.outputs.cache-hit != 'true' | |
| run: pnpm install | |
| - name: Build projects | |
| id: build-step | |
| if: steps.jetpack-build-cache.outputs.cache-hit != 'true' | |
| env: | |
| COMPOSER_ROOT_VERSION: "dev-trunk" | |
| BUILD_DIR: ./build-output | |
| PROJECT_PATH: ${{ matrix.path }} | |
| run: | | |
| find . -path ./.github -prune -o -type f -print | sort > /tmp/before.txt | |
| echo "::group::Build plugin(s)" | |
| cd "$PROJECT_PATH" | |
| pnpm run build | |
| cd "$GITHUB_WORKSPACE" | |
| echo "::endgroup::" | |
| # We only want to save the files that were actually created or changed. | |
| # But we can't just list them for actions/cache/save, "Argument list too long". | |
| # So instead we delete all the unchanged files so we can tell actions/cache/save | |
| # to save everything that's left. | |
| git -c core.quotepath=off diff --name-only | sort > /tmp/changed.txt | |
| if [[ -s /tmp/changed.txt ]]; then | |
| grep -F -x -v -f /tmp/changed.txt /tmp/before.txt > /tmp/remove.txt | |
| else | |
| cp /tmp/before.txt /tmp/remove.txt | |
| fi | |
| xargs -d '\n' rm < /tmp/remove.txt | |
| find . -type d -empty -delete | |
| - name: Save ${{ matrix.buildGroup }} build cache | |
| if: steps.jetpack-build-cache.outputs.cache-hit != 'true' | |
| id: jetpack-build-cache-save | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: | | |
| . | |
| !./.github/ | |
| key: ${{ steps.jetpack-build-cache.outputs.cache-primary-key }} | |
| e2e-tests: | |
| name: "${{ matrix.project }} e2e tests" | |
| runs-on: ubuntu-latest | |
| needs: [ create-test-matrix, build-projects ] | |
| # The "always() && ! cancelled() && ! failure()" bit is needed to still run if the build was skipped. | |
| if: > | |
| always() && ! cancelled() && ! failure() && | |
| needs.create-test-matrix.result == 'success' && needs.create-test-matrix.outputs.matrix != '[]' | |
| timeout-minutes: 60 | |
| env: | |
| WP_DEBUG_PATH: '${{ github.workspace }}/tools/docker/wordpress/wp-content/debug.log' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson( needs.create-test-matrix.outputs.matrix ) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Restore ${{ matrix.buildGroup }} build cache | |
| id: jetpack-build-cache | |
| if: needs.build-projects.result == 'success' && ! startsWith( matrix.suite, 'atomic' ) | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| . | |
| !./.github/ | |
| key: ${{ matrix.buildGroup }}-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Setup tools | |
| uses: ./.github/actions/tool-setup | |
| - name: Install monorepo | |
| run: | | |
| pnpm install | |
| - name: Checkout jetpack-production | |
| if: github.event_name == 'repository_dispatch' && github.event.client_payload.repository != 'Automattic/jetpack-production' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Automattic/jetpack-production | |
| path: build-output/build/Automattic/jetpack-production | |
| - name: Checkout mirror repo | |
| if: github.event_name == 'repository_dispatch' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.client_payload.repository }} | |
| ref: ${{ github.event.client_payload.ref_name }} | |
| path: build-output/build/${{ github.event.client_payload.repository }} | |
| - name: Prepare build | |
| env: | |
| COMPOSER_ROOT_VERSION: "dev-trunk" | |
| BUILD_DIR: ./build-output | |
| PROJECT_PATH: ${{ matrix.path }} | |
| SUITE: ${{ matrix.suite }} | |
| BUILD_JOB_RESULT: ${{ needs.build-projects.result }} | |
| run: | | |
| # Only execute these steps if the event name is 'repository_dispatch' | |
| if [[ "$GITHUB_EVENT_NAME" == repository_dispatch ]]; then | |
| if [[ "$BUILD_JOB_RESULT" != 'skipped' ]]; then | |
| echo "::error::Build job ran but result is not being used!" | |
| exit 1 | |
| fi | |
| echo "::group::Update volume mapping" | |
| .github/files/e2e-tests/map-plugins-for-e2e-env.sh | |
| echo "::endgroup::" | |
| elif [[ "$SUITE" != atomic* ]]; then | |
| echo "Using artifact from build-projects job" | |
| fi | |
| - name: Test environment set-up | |
| working-directory: ${{ matrix.path }} | |
| env: | |
| CONFIG_KEY: ${{ secrets.E2E_CONFIG_KEY }} | |
| SUITE: ${{ matrix.suite }} | |
| PROJECT_NAME: ${{ matrix.project }} | |
| run: | | |
| echo "::group::Decrypt config" | |
| pnpm run config:decrypt | |
| echo "::endgroup::" | |
| # Ensure that wp debug.log exists and the web process can write to it | |
| mkdir -p "$(dirname "$WP_DEBUG_PATH")" | |
| touch "$WP_DEBUG_PATH" | |
| chmod 0666 "$WP_DEBUG_PATH" | |
| if [[ "${SUITE}" != atomic* && "${SUITE}" != vip ]]; then | |
| echo "::group::Start docker environment" | |
| pnpm run env:up | |
| echo "::endgroup::" | |
| echo "::group::Create tunnel" | |
| nohup pnpm run tunnel:up "$HOME/tunnel.log" | |
| echo "::endgroup::" | |
| fi | |
| if [ "${SUITE}" == gutenberg ]; then | |
| echo "::group::Setting up Gutenberg" | |
| pnpm e2e-env gb-setup | |
| echo "::endgroup::" | |
| fi | |
| - name: Wait for atomic site to be ready | |
| if: ${{ startsWith(matrix.suite, 'atomic') }} | |
| working-directory: ${{ matrix.path }} | |
| env: | |
| DISPATCH_REF_NAME: ${{ github.event.client_payload.ref_name }} | |
| DISPATCH_REF_TYPE: ${{ github.event.client_payload.ref_type }} | |
| run: | | |
| SITE="atomic" | |
| if [[ "${DISPATCH_REF_NAME}" != trunk ]]; then | |
| SITE="atomicRc" | |
| fi | |
| echo "TEST_SITE=${SITE}" >> $GITHUB_ENV | |
| # Wait for the site to pick up latest tag version (DISPATCH_REF_NAME) | |
| TEST_SITE="${SITE}" node "$GITHUB_WORKSPACE/tools/e2e-commons/bin/update-beta-version.js" $DISPATCH_REF_TYPE $DISPATCH_REF_NAME | |
| - name: Set up VIP site | |
| if: ${{ matrix.suite == 'vip' }} | |
| working-directory: ${{ matrix.path }} | |
| run: | | |
| echo "TEST_SITE=vip" >> $GITHUB_ENV | |
| - name: Run ${{ matrix.project }} tests | |
| working-directory: ${{ matrix.path }} | |
| env: | |
| JSON_ARGS: ${{ toJSON(matrix.testArgs) }} | |
| run: | | |
| mapfile -t TEST_ARGS < <(jq -r '.[]' <<<"$JSON_ARGS") | |
| pnpm run test:run "${TEST_ARGS[@]}" | |
| - name: Write report metadata | |
| if: ${{ always() }} | |
| env: | |
| SUITE: ${{ matrix.suite }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| OUTPUT_PATH: ${{ matrix.path }} | |
| run: .github/files/e2e-tests/report-metadata.sh | |
| - name: Test environment tear-down | |
| if: ${{ always() }} | |
| working-directory: ${{ matrix.path }} | |
| continue-on-error: true | |
| run: | | |
| # Copy debug.log to output folder, if it exists | |
| [[ ! -f "$WP_DEBUG_PATH" ]] || cp "$WP_DEBUG_PATH" "output/logs/php_errors.log" | |
| pnpm run tunnel:down | |
| echo "::group::Tunnel logs" | |
| cat "$HOME/tunnel.log" | |
| echo "::endgroup::" | |
| # Update permissions to workaround https://github.com/actions/cache/issues/753 | |
| sudo chown -R runner:docker "$GITHUB_WORKSPACE/tools/docker" | |
| - name: Upload test artifacts | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-output-${{ matrix.project }} | |
| path: ${{ matrix.path }}/output | |
| include-hidden-files: true | |
| test-report: | |
| name: "Test report" | |
| runs-on: ubuntu-latest | |
| if: ${{ success() || failure() }} | |
| needs: [e2e-tests] | |
| steps: | |
| - name: Set report context | |
| id: report-context | |
| env: | |
| DISPATCH_REF_NAME: ${{ github.event.client_payload.ref_name }} | |
| DISPATCH_REPO: ${{ github.event.client_payload.repository }} | |
| PR_TITLE: ${{ github.event.pull_request.title }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| REPOSITORY=$GITHUB_REPOSITORY | |
| if [ "$GITHUB_EVENT_NAME" == pull_request ]; then | |
| BRANCH=$GITHUB_HEAD_REF | |
| elif [ "$GITHUB_EVENT_NAME" == repository_dispatch ]; then | |
| BRANCH=$DISPATCH_REF_NAME | |
| REPOSITORY=$DISPATCH_REPO | |
| else | |
| BRANCH=${GITHUB_REF:11} | |
| fi | |
| echo "EVENT_TYPE=e2e run $GITHUB_RUN_ID at $GITHUB_EVENT_NAME on $GITHUB_REPOSITORY" >> "$GITHUB_OUTPUT" | |
| echo "CLIENT_PAYLOAD=$( jq -nrc --arg repository "$REPOSITORY" --arg branch "$BRANCH" --arg pr_title "$PR_TITLE" --arg pr_number "$PR_NUMBER" --arg run_id "$GITHUB_RUN_ID" '{ repository: $repository, branch: $branch, pr_title: $pr_title, pr_number: $pr_number, run_id: $run_id }' )" >> "$GITHUB_OUTPUT" | |
| - name: Trigger test report workflow | |
| uses: peter-evans/repository-dispatch@v3 | |
| with: | |
| token: ${{ secrets.E2E_TEST_REPORTS_TOKEN }} | |
| repository: automattic/jetpack-e2e-reports | |
| event-type: ${{ steps.report-context.outputs.event_type }} | |
| client-payload: ${{ steps.report-context.outputs.client_payload }} | |
| slack-notification: | |
| name: "Slack notification" | |
| runs-on: ubuntu-latest | |
| if: ${{ success() || failure() }} | |
| needs: [e2e-tests] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up tools | |
| uses: ./.github/actions/tool-setup | |
| - name: Build action | |
| env: | |
| COMPOSER_ROOT_VERSION: dev-trunk | |
| run: | | |
| pnpm install | |
| composer install | |
| composer build-development | |
| working-directory: ./projects/github-actions/test-results-to-slack | |
| - name: Download test artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: test-output | |
| - name: Decrypt config | |
| env: | |
| CONFIG_KEY: ${{ secrets.E2E_CONFIG_KEY }} | |
| run: ./.github/files/e2e-tests/encryption.sh decrypt | |
| - name: Tag release candidate runs | |
| if: github.event_name == 'repository_dispatch' && github.event.client_payload.ref_type == 'tag' | |
| env: | |
| VERSION: ${{ github.event.client_payload.ref_name }} | |
| run: echo "NOTIFICATION_SUITE=rc-${VERSION}" >> $GITHUB_ENV | |
| - name: "Send notification" | |
| uses: ./projects/github-actions/test-results-to-slack | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| slack_token: ${{ secrets.SLACK_TOKEN }} | |
| slack_channel: ${{ secrets.SLACK_E2E_CHANNEL }} | |
| slack_icon_emoji: ":jetpack:" | |
| suite_name: ${{ env.NOTIFICATION_SUITE }} | |
| playwright_report_path: test-output/**/summary.json | |
| playwright_output_dir: test-output/**/results | |
| rules_configuration_path: .github/files/e2e-tests/notification-rules.json | |