Jetpack MU WPCOM: Blog Privacy: Update Applebot-Extended disallow rule (

#38599) * Update applebot-extended disallow rule * changelog
Automattic · Jul 29, 2024 · c39fc16 · c39fc16
1 parent d2f0cb7
commit c39fc16
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 7 deletions.
diff --git a/projects/packages/jetpack-mu-wpcom/changelog/update-applebot-extended b/projects/packages/jetpack-mu-wpcom/changelog/update-applebot-extended
@@ -0,0 +1,4 @@
+Significance: patch
+Type: changed
+
+Blog Privacy: Update Applebot-Extended disallow rule
diff --git a/projects/packages/jetpack-mu-wpcom/src/features/blog-privacy/blog-privacy.php b/projects/packages/jetpack-mu-wpcom/src/features/blog-privacy/blog-privacy.php
@@ -37,6 +37,7 @@ function robots_txt( string $output, $public ): string {
 		$ai_bots = array(
 			'Amazonbot',
 			'anthropic-ai',
+			'Applebot-Extended',
 			'Bytespider',
 			'CCBot',
 			'ClaudeBot',
@@ -54,10 +55,6 @@ function robots_txt( string $output, $public ): string {
 			$output .= "\nUser-agent: {$ai_bot}\n";
 			$output .= "Disallow: /\n";
 		}
-
-		// https://support.apple.com/en-us/119829#datausage
-		$output .= "\nUser-agent: Applebot-Extended\n";
-		$output .= "Disallow: /private/\n";
 	}
 
 	return $output;

diff --git a/...cts/packages/jetpack-mu-wpcom/tests/php/features/blog-privacy/class-blog-privacy-test.php b/...cts/packages/jetpack-mu-wpcom/tests/php/features/blog-privacy/class-blog-privacy-test.php
@@ -37,6 +37,9 @@ public function provide_robots_txt(): \Iterator {
 User-agent: anthropic-ai
 Disallow: /
 
+User-agent: Applebot-Extended
+Disallow: /
+
 User-agent: Bytespider
 Disallow: /
 
@@ -70,9 +73,6 @@ public function provide_robots_txt(): \Iterator {
 User-agent: sentibot
 Disallow: /
 
-User-agent: Applebot-Extended
-Disallow: /private/
-
 AI_BLOCKS;
 
 		yield 'public, no discourage AI' => array(