Backport #5725 16.x.x (#5916)

plone · Mar 24, 2024 · bc19806 · bc19806
1 parent 3b04e82
commit bc19806
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 7 deletions.
diff --git a/docs/source/robots.txt b/docs/source/robots.txt
@@ -1,7 +1,2 @@
-# Disallow all user agents from the following directories and files
-User-agent: *
-Disallow: /_sources/
-Disallow: /.doctrees/
-Disallow: /*.txt$
-Disallow: /.buildinfo$
-Disallow: /objects.inv$
+# Disallow all user agents from indexing pages and links in the site on Netlify only.
+Disallow: /
diff --git a/netlify.toml b/netlify.toml
@@ -3,3 +3,7 @@
   NODE_VERSION = "16.17.1"
 [build]
   ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF ./docs/"
+[[headers]]
+  for = "/*"
+  [[headers.values]]
+    X-Robots-Tag = "none"
diff --git a/news/5916.documentation b/news/5916.documentation
@@ -0,0 +1 @@
+Block search engines from indexing content on Netlify preview builds. @stevepiercy
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Block search engines from indexing content on Netlify preview builds. @stevepiercy