Merge branch 'trunk' into update/add-better-free-check-for-my-jetpack…

…-products

.github/actions/tool-setup/composer-plugin/.phan/config.php

@@ -8,4 +8,4 @@
 // Require base config.
 require dirname( __DIR__, 5 ) . '/.phan/config.base.php';
 
-return make_phan_config( dirname( __DIR__ ), array( 'is_wordpress' => false ) );
+return make_phan_config( dirname( __DIR__ ), array( 'stubs' => array() ) );

.github/files/e2e-tests/e2e-matrix.js

@@ -58,6 +58,38 @@ const projects = [
 		suite: '',
 		buildGroup: 'jetpack-boost',
 	},
+	{
+		project: 'Jetpack Boost - Page Cache',
+		path: 'projects/plugins/boost/tests/e2e',
+		testArgs: [ 'specs/page-cache', '--retries=1' ],
+		targets: [ 'plugins/boost' ],
+		suite: '',
+		buildGroup: 'jetpack-boost',
+	},
+	{
+		project: 'Jetpack Boost - Concatenate JS/CSS',
+		path: 'projects/plugins/boost/tests/e2e',
+		testArgs: [ 'specs/concatenate', '--retries=1' ],
+		targets: [ 'plugins/boost' ],
+		suite: '',
+		buildGroup: 'jetpack-boost',
+	},
+	{
+		project: 'Jetpack Boost - Image CDN',
+		path: 'projects/plugins/boost/tests/e2e',
+		testArgs: [ 'specs/image-cdn', '--retries=1' ],
+		targets: [ 'plugins/boost' ],
+		suite: '',
+		buildGroup: 'jetpack-boost',
+	},
+	{
+		project: 'Jetpack Boost - Image Guide',
+		path: 'projects/plugins/boost/tests/e2e',
+		testArgs: [ 'specs/image-guide', '--retries=1' ],
+		targets: [ 'plugins/boost' ],
+		suite: '',
+		buildGroup: 'jetpack-boost',
+	},
 	{
 		project: 'Search',
 		path: 'projects/plugins/search/tests/e2e',

.github/files/lint-project-structure.sh

@@ -221,10 +221,6 @@ for PROJECT in projects/*/*; do
 			EXIT=1
 			echo "::error file=$PROJECT/.phan/config.php::Project $SLUG has PHP files but does not contain .phan/config.php. Refer to Static Analysis in docs/monorepo.md."
 		fi
-		if [[ ! -e "$PROJECT/.phan/baseline.php" ]]; then
-			EXIT=1
-			echo "::error file=$PROJECT/.phan/baseline.php::Project $SLUG has PHP files but does not contain .phan/baseline.php. Refer to Static Analysis in docs/monorepo.md."
-		fi
 	fi
 
 	# - composer.json must exist.
@@ -432,6 +428,20 @@ if [[ -n "$DUPS" ]]; then
 	done <<<"$DUPS"
 fi
 
+# - package.json name fields should not be repeated.
+debug "Checking for duplicate package.json names"
+DUPS="$(jq -rn 'reduce inputs as $i ({}; if $i.name then .[$i.name] |= ( . // [] ) + [ input_filename ] else . end) | to_entries[] | .key as $key | .value | select( length > 1 ) | ( [ .[] | capture("^projects/(?<s>.*)/package\\.json$").s ] | .[-1] |= "and " + . | join( if length > 2 then ", " else " " end ) ) as $slugs | .[] | [ ., $key, $slugs ] | @tsv' projects/*/*/package.json projects/*/*/tests/e2e/package.json)"
+if [[ -n "$DUPS" ]]; then
+	while IFS=$'\t' read -r FILE KEY SLUGS; do
+		LINE=$(grep --line-number --max-count=1 '^	"name":' "$FILE" || true)
+		if [[ -n "$LINE" ]]; then
+			LINE=",line=${LINE%%:*}"
+		fi
+		EXIT=1
+		echo "::error file=$FILE$LINE::Name $KEY is in use in package.json by $SLUGS. They must be deduplicated."
+	done <<<"$DUPS"
+fi
+
 # - Text domains from plugins should not be used in packages.
 debug "Checking package textdomain usage vs plugin slugs"
 PLUGDOMAINS="$(jq -n 'reduce inputs as $i ({}; .[$i.extra["wp-plugin-slug"] // $i.extra["wp-theme-slug"] // ""] = ( input_filename | sub("^projects/(?<slug>.*)/composer\\.json$";"\(.slug)"))) | .[""] |= empty' projects/plugins/*/composer.json)"
@@ -499,6 +509,59 @@ for FILE in $(git -c core.quotepath=off ls-files 'projects/packages/**/.eslintrc
 	fi
 done
 
+# - Text domains in block.json should match composer.json.
+debug "Checking textdomain usage in block.json"
+for FILE in $(git -c core.quotepath=off ls-files 'projects/packages/**/block.json' 'projects/plugins/**/block.json'); do
+	[[ "$FILE" == projects/packages/blocks/tests/php/fixtures/* ]] && continue  # Ignore test fixtures
+
+	DOM="$(jq -r '.textdomain' "$FILE")"
+	DIR="$FILE"
+	while ! [[ "$DIR" =~ ^projects/[^/]*/[^/]*$ ]]; do
+		DIR="${DIR%/*}"
+	done
+	SLUG="${DIR#projects/}"
+	if [[ "$SLUG" == plugins/* ]]; then
+		DOM2="$(jq -r '.extra["wp-plugin-slug"] // .extra["wp-theme-slug"] // ""' "$DIR/composer.json")"
+	else
+		DOM2="$(jq -r '.extra.textdomain // ""' "$DIR/composer.json")"
+	fi
+	if [[ "$DOM" != "$DOM2" ]]; then
+		EXIT=1
+		LINE=$(jq --stream 'if length == 1 then .[0][:-1] else .[0] end | if . == ["textdomain"] then input_line_number - 1 else empty end' package.json)
+		if [[ -n "$LINE" ]]; then
+			LINE=",line=${LINE%%:*}"
+		fi
+		if [[ -z "$DOM2" ]]; then
+			echo "::error file=$FILE$LINE::block.json sets textdomain \"$DOM\", but $SLUG's composer.json does not set \`.extra.textdomain\`."
+		else
+			echo "::error file=$FILE$LINE::block.json sets textdomain \"$DOM\", but $SLUG's composer.json sets domain \"$DOM2\"."
+		fi
+	fi
+done
+
+# - In phpcs config, `<rule ref="Standard.Category.Sniff.Message"><severity>0</severity></rule>` doesn't do what you think.
+debug "Checking for bad message exclusions in phpcs configs"
+for FILE in $(git -c core.quotepath=off ls-files .phpcs.config.xml .phpcs.xml.dist .github/files/php-linting-phpcs.xml .github/files/phpcompatibility-dev-phpcs.xml '*/.phpcs.dir.xml' '*/.phpcs.dir.phpcompatibility.xml'); do
+	while IFS=$'\t' read -r LINE REF; do
+		EXIT=1
+		echo "::error file=$FILE,line=$LINE::PHPCS config attempts to set severity 0 for the sniff message \"$REF\". To exclude a single message from a sniff, use \`<rule ref=\"${REF%.*}\"><exclude name=\"$REF\"/></rule>\` instead."
+	done < <( php -- "$FILE" <<-'PHPDOC'
+		<?php
+		$doc = new DOMDocument();
+		$doc->load( $argv[1] );
+		$xpath = new DOMXPath( $doc );
+		function has_message( $v ) {
+			return count( explode(".", $v[0]->value) ) >= 4;
+		}
+		$xpath->registerNamespace("php", "http://php.net/xpath");
+		$xpath->registerPHPFunctions( "has_message" );
+		foreach ( $xpath->evaluate( "//rule[php:function(\"has_message\", @ref)][severity[normalize-space(.)=\"0\"]]" ) as $node ) {
+			echo "{$node->getLineNo()}\t{$node->getAttribute("ref")}\n";
+		}
+		PHPDOC
+	)
+done
+
 # - .nvmrc should match .github/versions.sh.
 debug "Checking .nvmrc vs versions.sh"
 if [[ "$(<.nvmrc)" != "$NODE_VERSION" ]]; then

.github/files/setup-wordpress-env.sh

@@ -40,7 +40,7 @@ case "$WP_BRANCH" in
 	previous)
 		# We hard-code the version here because there's a time near WP releases where
 		# we've dropped the old 'previous' but WP hasn't actually released the new 'latest'
-		TAG=6.3
+		TAG=6.4
 		;;
 	*)
 		echo "Unrecognized value for WP_BRANCH: $WP_BRANCH" >&2

.github/workflows/build.yml

@@ -315,6 +315,7 @@ jobs:
         with:
           source-directory: ${{ github.workspace }}/monorepo
           token: ${{ secrets.API_TOKEN_GITHUB }}
+          upstream-ref-since: '2024-04-10' # No point in checking 12 years of earlier commits from before we started adding "Upstream-Ref".
           username: matticbot
           working-directory: ${{ github.workspace }}/build
-        timeout-minutes: 5  # 2021-01-18: Successful runs seem to take about half a minute.
+        timeout-minutes: 10  # 2024-04-11: Successful runs seem to take about a minute.

.github/workflows/slack-workflow-failed.yml

@@ -12,6 +12,7 @@ on:
       - Post-Build
       - PR is up-to-date
       - Update Jetpack Staging Test Sites
+      - Update Phan stubs
       - k6 Tests for Jetpack Staging Test Sites
     branches: [ 'trunk', 'prerelease', '*/branch-*' ]
 

.github/workflows/tests.yml

@@ -327,20 +327,42 @@ jobs:
           pnpm jetpack phan --all -v --format github
       - name: Check baselines
         run: |
-          # Anything changed (with a side of printing the diff)
-          if git diff --exit-code --ignore-matching-lines='^    // ' .phan/baseline.php 'projects/*/*/.phan/baseline.php'; then
+          # Anything changed? (with a side of printing the diff)
+          if git diff --exit-code --ignore-matching-lines='^    // ' -- .phan/baseline.php '*/.phan/baseline.php'; then
             exit 0
           fi
 
           # Collect which projects changed to suggest the right command.
           PROJECTS=()
-          if ! git diff --exit-code --name-only .phan/baseline.php &>/dev/null; then
-            PROJECTS+=( 'monorepo' )
-          fi
-          for f in $( git -c core.quotepath=off diff --name-only 'projects/*/*/.phan/baseline.php' ); do
-            SLUG=${f%/.phan/baseline.php}
-            SLUG=${SLUG#projects/}
-            PROJECTS+=( "$SLUG" )
+          for f in $( git -c core.quotepath=off diff --name-only -- .phan/baseline.php '*/.phan/baseline.php' ); do
+            # --name-only and --ignore-matching-lines don't combine, so we have to do the check separately.
+            if git diff --quiet --exit-code --ignore-matching-lines='^    // ' -- "$f"; then
+              continue
+            fi
+
+            if [[ "$f" == ".phan/baseline.php" ]]; then
+              SLUG=monorepo
+            elif [[ "$f" == projects/*/*/.phan/baseline.php ]]; then
+              SLUG=${f%/.phan/baseline.php}
+              SLUG=${SLUG#projects/}
+            elif SLUG=$( grep -v '^[ \t]*\/\/' .phan/monorepo-pseudo-projects.jsonc | jq -re --arg f "${f%.phan/baseline.php}" 'to_entries[] | select( .value == $f ) | .key' ); then
+              : # Ok
+            else
+              SLUG=
+            fi
+            if grep -q 'This baseline has no suppressions' "$f"; then
+              if [[ -n "$SLUG" ]]; then
+                echo "::error file=$f::This Phan baseline is now empty (good job!). You may remove it, or if you want to keep it (e.g. if you expect new unfixed issues to be added in the future) you can run \`jetpack phan --update-baseline $SLUG\` to update it."
+              else
+                echo "::error file=$f::This Phan baseline is now empty (good job!). You may remove it."
+              fi
+            elif [[ -n "$SLUG" ]]; then
+              PROJECTS+=( "$SLUG" )
+            else
+              echo "::error file=$f::This Phan baseline has changed and should be updated. This Action was unable to determine the command needed to update it; please report this to the Garage team."
+            fi
           done
-          echo "::error::Phan baselines have changed (good job!). Run \`jetpack phan --update-baseline ${PROJECTS[*]}\` to update them."
+          if [[ ${#PROJECTS[@]} -gt 0 ]]; then
+            echo "::error::Phan baselines have changed (good job!). Run \`jetpack phan --update-baseline ${PROJECTS[*]}\` to update them."
+          fi
           exit 1

.github/workflows/update-phan-stubs.yml

@@ -0,0 +1,75 @@
+name: Update Phan stubs
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '22 0 * * 1,2,3,4,5'
+concurrency:
+  group: update-phan-stubs-${{ github.ref }}
+
+permissions:
+  contents: write
+  pull-requests: write
+
+env:
+  GIT_AUTHOR_NAME: matticbot
+  GIT_AUTHOR_EMAIL: [email protected]
+  GIT_COMMITTER_NAME: matticbot
+  GIT_COMMITTER_EMAIL: [email protected]
+
+jobs:
+  update-stubs:
+    name: Update stubs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Use the matticbot token so CI runs on the created PR.
+          token: ${{ secrets.API_TOKEN_GITHUB }}
+
+      - name: Update stubs
+        run: tools/stubs/update-stubs.sh
+
+      - name: Check for changes
+        id: changes
+        run: |
+          if git diff --exit-code; then
+            echo "::notice::No changes"
+            echo "needed=false" >> "$GITHUB_OUTPUT"
+          elif ! [[ "$(date +%e)" -le 7 && "$(date +%A)" == Thursday ]] && git diff --exit-code --ignore-matching-lines='^ \* Stubs automatically generated from' >/dev/null; then
+            echo "::notice::No changes (other than version bumps) and it's not the first Thursday"
+            echo "needed=false" >> "$GITHUB_OUTPUT"
+          elif git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin update/phan-custom-stubs >/dev/null; then
+            TMP=$( git -c core.quotepath=off diff --name-only )
+            mapfile -t FILES <<<"$TMP"
+            if git diff --quiet --exit-code origin/update/phan-custom-stubs "${FILES[@]}"; then
+              echo "::notice::Branch already exists and is up to date with these changes"
+              echo "needed=false" >> "$GITHUB_OUTPUT"
+            else
+              echo "::notice::Branch needs updating"
+              echo "has-branch=true" >> "$GITHUB_OUTPUT"
+              echo "needed=true" >> "$GITHUB_OUTPUT"
+            fi
+          else
+            echo "::notice::Branch needs creating"
+            echo "has-branch=false" >> "$GITHUB_OUTPUT"
+            echo "needed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Create commit and PR
+        if: steps.changes.outputs.needed == 'true' && steps.changes.outputs.has-branch == 'false'
+        env:
+          GH_TOKEN: ${{ secrets.API_TOKEN_GITHUB }}
+        run: |
+          git checkout -b update/phan-custom-stubs
+          git commit -am 'phan: Update custom stubs'
+          git push origin HEAD
+          gh pr create --title 'phan: Update custom stubs' --body 'This is an automatic update generated by a GitHub Action. If closed it will be recreated the next time the action runs.' --label '[Pri] Normal' --label '[Type] Janitorial' --label '[Status] Needs Review'
+
+      - name: Update existing branch
+        if: steps.changes.outputs.needed == 'true' && steps.changes.outputs.has-branch == 'true'
+        run: |
+          git commit -am 'phan: Update custom stubs'
+          SHA=$( git rev-parse HEAD )
+          git checkout update/phan-custom-stubs
+          git reset --hard "$SHA"
+          git push --force-with-lease origin HEAD

.gitignore

@@ -12,7 +12,7 @@ jetpack_vendor/
 .DS_Store
 *.code-workspace
 *.swp
-# Custom environment for docker-compose (used by docker-compose.yml)
+# Custom environment for Docker compose (used by docker-compose.yml)
 /.env
 /tools/docker/compose-extras.yml
 /tools/docker/compose-volumes.yml
@@ -34,7 +34,7 @@ npm-debug.log
 phpcs.xml
 .phpunit.result.cache
 
-## Things for docker-composer
+## Things for Docker compose
 /tools/docker/data/*
 !/tools/docker/data/.gitkeep
 !/tools/docker/data/ssh.keys/

.npmrc

@@ -13,6 +13,9 @@ public-hoist-pattern=[]
 auto-install-peers = false
 dedupe-peer-dependents = false
 
+# Silence this warning. Our `jetpack dependencies build-order` (also used by `jetpack build --all`) checks for cycles itself, plus it has a way to indicate that a dep is only for testing.
+ignore-workspace-cycles = true
+
 # Opinion seems divided on whether the new default 'lowest-direct' is good or bad.
 #  https://github.com/pnpm/pnpm/issues/6463
 #  https://github.com/pnpm/pnpm/issues/6498