From bc198061f3411cb3ffd1633058b6ebd9c9fc7cba Mon Sep 17 00:00:00 2001
From: Steve Piercy <web@stevepiercy.com>
Date: Sun, 24 Mar 2024 19:34:29 -0400
Subject: [PATCH] Backport #5725 16.x.x (#5916)

---
 docs/source/robots.txt  | 9 ++-------
 netlify.toml            | 4 ++++
 news/5916.documentation | 1 +
 3 files changed, 7 insertions(+), 7 deletions(-)
 create mode 100644 news/5916.documentation

diff --git a/docs/source/robots.txt b/docs/source/robots.txt
index 88d8608916..2dc87d3d33 100644
--- a/docs/source/robots.txt
+++ b/docs/source/robots.txt
@@ -1,7 +1,2 @@
-# Disallow all user agents from the following directories and files
-User-agent: *
-Disallow: /_sources/
-Disallow: /.doctrees/
-Disallow: /*.txt$
-Disallow: /.buildinfo$
-Disallow: /objects.inv$
+# Disallow all user agents from indexing pages and links in the site on Netlify only.
+Disallow: /
diff --git a/netlify.toml b/netlify.toml
index 981fae29dd..3ef6a9db73 100644
--- a/netlify.toml
+++ b/netlify.toml
@@ -3,3 +3,7 @@
   NODE_VERSION = "16.17.1"
 [build]
   ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF ./docs/"
+[[headers]]
+  for = "/*"
+  [[headers.values]]
+    X-Robots-Tag = "none"
diff --git a/news/5916.documentation b/news/5916.documentation
new file mode 100644
index 0000000000..6264f42515
--- /dev/null
+++ b/news/5916.documentation
@@ -0,0 +1 @@
+Block search engines from indexing content on Netlify preview builds. @stevepiercy