AI Title Optimization: Add error handling for Title Optimization #79032

Workflow file for this run

.github/workflows/e2e-tests.yml at 6548745

	name: E2E Tests

	on:
	pull_request:
	paths-ignore:
	- '**.md'
	repository_dispatch:
	types: [ 'e2e tests**' ]

	concurrency:
	group: e2e-tests-${{ github.event_name }}-${{ github.ref }}-${{ github.event.action }}
	cancel-in-progress: true

	env:
	# Work around a bug in node 18.18.0. See https://github.com/webpack-contrib/thread-loader/issues/191 for details.
	UV_USE_IO_URING: 0

	jobs:
	create-test-matrix:
	name: "Determine tests matrix"
	runs-on: ubuntu-latest
	timeout-minutes: 5 # 2023-09-15: The pnpm install may take a few minutes on cache miss.
	# Only run tests in the main repository
	if: github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name
	outputs:
	matrix: ${{ steps.evaluate.outputs.matrix }}
	build-matrix: ${{ steps.evaluate.outputs.build-matrix }}
	steps:
	- uses: actions/checkout@v4

	# For pull requests, list-changed-projects.sh needs the merge base.
	- name: Deepen to merge base
	if: github.event_name == 'pull_request'
	uses: ./.github/actions/deepen-to-merge-base

	- name: Setup tools
	uses: ./.github/actions/tool-setup

	# Required for list-changed-projects.sh
	- name: Install monorepo
	run: \|
	pnpm install

	- name: Create test plan
	id: evaluate
	env:
	DISPATCH_REPO: ${{ github.event.client_payload.repository }}
	REF_NAME: ${{ github.event.client_payload.ref_name }}
	REF_TYPE: ${{ github.event.client_payload.ref_type }}
	run: \|
	MATRIX="$(node .github/files/e2e-tests/e2e-matrix.js)"
	echo "matrix: $MATRIX"
	echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"
	if [[ "$GITHUB_EVENT_NAME" == repository_dispatch ]]; then
	echo "No build needed for $GITHUB_EVENT_NAME"
	BUILD_MATRIX="[]"
	else
	BUILD_MATRIX=$(jq -c '[ .[] \| select( .suite \| startswith( "atomic" ) \| not ) \| { buildGroup: .buildGroup, path: .path } ] \| unique' <<<"$MATRIX")
	echo "build matrix: $BUILD_MATRIX"
	BAD=$( jq -r '[ group_by( .buildGroup )[] \| select( length > 1 ) \| .[0].buildGroup ] \| unique \| join( "," )' <<<"$BUILD_MATRIX" )
	if [[ -n "$BAD" ]]; then
	echo "::error::One or more build groups have multiple paths: $BAD"
	exit 1
	fi
	fi
	echo "build-matrix=$BUILD_MATRIX" >> "$GITHUB_OUTPUT"

	build-projects:
	name: "E2E: Build ${{ matrix.buildGroup }}"
	runs-on: ubuntu-latest
	needs: create-test-matrix
	timeout-minutes: 30
	if: needs.create-test-matrix.outputs.build-matrix != '[]'
	strategy:
	fail-fast: false
	matrix:
	include: ${{ fromJson( needs.create-test-matrix.outputs.build-matrix ) }}
	steps:
	- name: Ensure ${{ matrix.buildGroup }} build cache
	id: jetpack-build-cache
	uses: actions/cache/restore@v4
	with:
	lookup-only: true
	path: \|
	.
	!./.github/
	key: ${{ matrix.buildGroup }}-${{ github.sha }}

	- name: Checkout code
	if: steps.jetpack-build-cache.outputs.cache-hit != 'true'
	uses: actions/checkout@v4

	- name: Setup tools
	if: steps.jetpack-build-cache.outputs.cache-hit != 'true'
	uses: ./.github/actions/tool-setup

	- name: Install monorepo
	if: steps.jetpack-build-cache.outputs.cache-hit != 'true'
	run: pnpm install

	- name: Build projects
	id: build-step
	if: steps.jetpack-build-cache.outputs.cache-hit != 'true'
	env:
	COMPOSER_ROOT_VERSION: "dev-trunk"
	BUILD_DIR: ./build-output
	PROJECT_PATH: ${{ matrix.path }}
	run: \|
	find . -path ./.github -prune -o -type f -print \| sort > /tmp/before.txt
	echo "::group::Build plugin(s)"
	cd "$PROJECT_PATH"
	pnpm run build
	cd "$GITHUB_WORKSPACE"
	echo "::endgroup::"

	# We only want to save the files that were actually created or changed.
	# But we can't just list them for actions/cache/save, "Argument list too long".
	# So instead we delete all the unchanged files so we can tell actions/cache/save
	# to save everything that's left.
	git -c core.quotepath=off diff --name-only \| sort > /tmp/changed.txt
	if [[ -s /tmp/changed.txt ]]; then
	grep -F -x -v -f /tmp/changed.txt /tmp/before.txt > /tmp/remove.txt
	else
	cp /tmp/before.txt /tmp/remove.txt
	fi
	xargs -d '\n' rm < /tmp/remove.txt
	find . -type d -empty -delete

	- name: Save ${{ matrix.buildGroup }} build cache
	if: steps.jetpack-build-cache.outputs.cache-hit != 'true'
	id: jetpack-build-cache-save
	uses: actions/cache/save@v4
	with:
	path: \|
	.
	!./.github/
	key: ${{ steps.jetpack-build-cache.outputs.cache-primary-key }}

	e2e-tests:
	name: "${{ matrix.project }} e2e tests"
	runs-on: ubuntu-latest
	needs: [ create-test-matrix, build-projects ]
	# The "always() && ! cancelled() && ! failure()" bit is needed to still run if the build was skipped.
	if: >
	always() && ! cancelled() && ! failure() &&
	needs.create-test-matrix.result == 'success' && needs.create-test-matrix.outputs.matrix != '[]'
	timeout-minutes: 60
	strategy:
	fail-fast: false
	matrix:
	include: ${{ fromJson( needs.create-test-matrix.outputs.matrix ) }}
	steps:
	- uses: actions/checkout@v4

	- name: Restore ${{ matrix.buildGroup }} build cache
	id: jetpack-build-cache
	if: needs.build-projects.result == 'success' && ! startsWith( matrix.suite, 'atomic' )
	uses: actions/cache/restore@v4
	with:
	path: \|
	.
	!./.github/
	key: ${{ matrix.buildGroup }}-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Setup tools
	uses: ./.github/actions/tool-setup

	- name: Install monorepo
	run: \|
	pnpm install

	- name: Checkout jetpack-production
	if: github.event_name == 'repository_dispatch' && github.event.client_payload.repository != 'Automattic/jetpack-production'
	uses: actions/checkout@v4
	with:
	repository: Automattic/jetpack-production
	path: build-output/build/Automattic/jetpack-production

	- name: Checkout mirror repo
	if: github.event_name == 'repository_dispatch'
	uses: actions/checkout@v4
	with:
	repository: ${{ github.event.client_payload.repository }}
	ref: ${{ github.event.client_payload.ref_name }}
	path: build-output/build/${{ github.event.client_payload.repository }}

	- name: Prepare build
	env:
	COMPOSER_ROOT_VERSION: "dev-trunk"
	BUILD_DIR: ./build-output
	PROJECT_PATH: ${{ matrix.path }}
	SUITE: ${{ matrix.suite }}
	BUILD_JOB_RESULT: ${{ needs.build-projects.result }}
	run: \|
	# Only execute these steps if the event name is 'repository_dispatch'
	if [[ "$GITHUB_EVENT_NAME" == repository_dispatch ]]; then
	if [[ "$BUILD_JOB_RESULT" != 'skipped' ]]; then
	echo "::error::Build job ran but result is not being used!"
	exit 1
	fi

	echo "::group::Update volume mapping"
	.github/files/e2e-tests/map-plugins-for-e2e-env.sh
	echo "::endgroup::"
	elif [[ "$SUITE" != atomic* ]]; then
	echo "Using artifact from build-projects job"
	fi

	- name: Test environment set-up
	working-directory: ${{ matrix.path }}
	env:
	CONFIG_KEY: ${{ secrets.E2E_CONFIG_KEY }}
	SUITE: ${{ matrix.suite }}
	PROJECT_NAME: ${{ matrix.project }}
	run: \|
	echo "::group::Decrypt config"
	pnpm run config:decrypt
	echo "::endgroup::"

	if [[ "${SUITE}" != atomic* && "${SUITE}" != vip ]]; then
	echo "::group::Start docker environment"
	pnpm run env:up
	echo "::endgroup::"

	echo "::group::Create tunnel"
	nohup pnpm run tunnel:up "$HOME/tunnel.log"
	echo "::endgroup::"
	fi

	if [ "${SUITE}" == gutenberg ]; then
	echo "::group::Setting up Gutenberg"
	pnpm e2e-env gb-setup
	echo "::endgroup::"
	fi

	- name: Wait for atomic site to be ready
	if: ${{ startsWith(matrix.suite, 'atomic') }}
	working-directory: ${{ matrix.path }}
	env:
	DISPATCH_REF_NAME: ${{ github.event.client_payload.ref_name }}
	DISPATCH_REF_TYPE: ${{ github.event.client_payload.ref_type }}
	run: \|
	SITE="atomic"

	if [[ "${DISPATCH_REF_NAME}" != trunk ]]; then
	SITE="atomicRc"
	fi

	echo "TEST_SITE=${SITE}" >> $GITHUB_ENV

	# Wait for the site to pick up latest tag version (DISPATCH_REF_NAME)
	TEST_SITE="${SITE}" node "$GITHUB_WORKSPACE/tools/e2e-commons/bin/update-beta-version.js" $DISPATCH_REF_TYPE $DISPATCH_REF_NAME

	- name: Set up VIP site
	if: ${{ matrix.suite == 'vip' }}
	working-directory: ${{ matrix.path }}
	run: \|
	echo "TEST_SITE=vip" >> $GITHUB_ENV

	- name: Run ${{ matrix.project }} tests
	working-directory: ${{ matrix.path }}
	env:
	JSON_ARGS: ${{ toJSON(matrix.testArgs) }}
	run: \|
	mapfile -t TEST_ARGS < <(jq -r '.[]' <<<"$JSON_ARGS")
	pnpm run test:run "${TEST_ARGS[@]}"

	- name: Write report metadata
	if: ${{ always() }}
	env:
	SUITE: ${{ matrix.suite }}
	PR_NUMBER: ${{ github.event.pull_request.number }}
	OUTPUT_PATH: ${{ matrix.path }}
	run: .github/files/e2e-tests/report-metadata.sh

	- name: Test environment tear-down
	if: ${{ always() }}
	working-directory: ${{ matrix.path }}
	continue-on-error: true
	run: \|
	pnpm run tunnel:down
	echo "::group::Tunnel logs"
	cat "$HOME/tunnel.log"
	echo "::endgroup::"

	# Update permissions to workaround https://github.com/actions/cache/issues/753
	sudo chown -R runner:docker "$GITHUB_WORKSPACE/tools/docker"

	- name: Upload test artifacts
	if: ${{ always() }}
	continue-on-error: true
	uses: actions/upload-artifact@v4
	with:
	name: test-output-${{ matrix.project }}
	path: ${{ matrix.path }}/output

	test-report:
	name: "Test report"
	runs-on: ubuntu-latest
	if: ${{ success() \|\| failure() }}
	needs: [e2e-tests]

	steps:
	- name: Set report context
	id: report-context
	env:
	DISPATCH_REF_NAME: ${{ github.event.client_payload.ref_name }}
	DISPATCH_REPO: ${{ github.event.client_payload.repository }}
	PR_TITLE: ${{ github.event.pull_request.title }}
	PR_NUMBER: ${{ github.event.pull_request.number }}
	run: \|
	REPOSITORY=$GITHUB_REPOSITORY

	if [ "$GITHUB_EVENT_NAME" == pull_request ]; then
	BRANCH=$GITHUB_HEAD_REF
	elif [ "$GITHUB_EVENT_NAME" == repository_dispatch ]; then
	BRANCH=$DISPATCH_REF_NAME
	REPOSITORY=$DISPATCH_REPO
	else
	BRANCH=${GITHUB_REF:11}
	fi

	echo "EVENT_TYPE=e2e run $GITHUB_RUN_ID at $GITHUB_EVENT_NAME on $GITHUB_REPOSITORY" >> "$GITHUB_OUTPUT"
	echo "CLIENT_PAYLOAD=$( jq -nrc --arg repository "$REPOSITORY" --arg branch "$BRANCH" --arg pr_title "$PR_TITLE" --arg pr_number "$PR_NUMBER" --arg run_id "$GITHUB_RUN_ID" '{ repository: $repository, branch: $branch, pr_title: $pr_title, pr_number: $pr_number, run_id: $run_id }' )" >> "$GITHUB_OUTPUT"

	- name: Trigger test report workflow
	uses: peter-evans/repository-dispatch@v3
	with:
	token: ${{ secrets.E2E_TEST_REPORTS_TOKEN }}
	repository: automattic/jetpack-e2e-reports
	event-type: ${{ steps.report-context.outputs.event_type }}
	client-payload: ${{ steps.report-context.outputs.client_payload }}

	slack-notification:
	name: "Slack notification"
	runs-on: ubuntu-latest
	if: ${{ success() \|\| failure() }}
	needs: [e2e-tests]

	steps:
	- uses: actions/checkout@v4

	- name: Set up tools
	uses: ./.github/actions/tool-setup

	- name: Build action
	env:
	COMPOSER_ROOT_VERSION: dev-trunk
	run: \|
	pnpm install
	composer install
	composer build-development
	working-directory: ./projects/github-actions/test-results-to-slack

	- name: Download test artifacts
	uses: actions/download-artifact@v4
	with:
	path: test-output

	- name: Decrypt config
	env:
	CONFIG_KEY: ${{ secrets.E2E_CONFIG_KEY }}
	run: ./.github/files/e2e-tests/encryption.sh decrypt

	- name: Tag release candidate runs
	if: github.event_name == 'repository_dispatch' && github.event.client_payload.ref_type == 'tag'
	env:
	VERSION: ${{ github.event.client_payload.ref_name }}
	run: echo "NOTIFICATION_SUITE=rc-${VERSION}" >> $GITHUB_ENV

	- name: "Send notification"
	uses: ./projects/github-actions/test-results-to-slack
	with:
	github_token: ${{ secrets.GITHUB_TOKEN }}
	slack_token: ${{ secrets.SLACK_TOKEN }}
	slack_channel: ${{ secrets.SLACK_E2E_CHANNEL }}
	slack_icon_emoji: ":jetpack:"
	suite_name: ${{ env.NOTIFICATION_SUITE }}
	playwright_report_path: test-output/**/summary.json
	playwright_output_dir: test-output/**/results
	rules_configuration_path: .github/files/e2e-tests/notification-rules.json

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

AI Title Optimization: Add error handling for Title Optimization #79032

Workflow file

AI Title Optimization: Add error handling for Title Optimization #79032

Jobs

Run details

Workflow file for this run