From e1274a94a78d09472ae74ca33f85a96419250c1d Mon Sep 17 00:00:00 2001
From: Steve Piercy <web@stevepiercy.com>
Date: Sun, 24 Mar 2024 19:34:09 -0400
Subject: [PATCH] =?UTF-8?q?Backport=20#5725:=20Block=20search=20engines=20?=
 =?UTF-8?q?from=20indexing=20content=20on=20Netlify=E2=80=A6=20(#5915)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/source/robots.txt  | 9 ++-------
 netlify.toml            | 4 ++++
 news/5915.documentation | 1 +
 3 files changed, 7 insertions(+), 7 deletions(-)
 create mode 100644 news/5915.documentation

diff --git a/docs/source/robots.txt b/docs/source/robots.txt
index 88d8608916..2dc87d3d33 100644
--- a/docs/source/robots.txt
+++ b/docs/source/robots.txt
@@ -1,7 +1,2 @@
-# Disallow all user agents from the following directories and files
-User-agent: *
-Disallow: /_sources/
-Disallow: /.doctrees/
-Disallow: /*.txt$
-Disallow: /.buildinfo$
-Disallow: /objects.inv$
+# Disallow all user agents from indexing pages and links in the site on Netlify only.
+Disallow: /
diff --git a/netlify.toml b/netlify.toml
index 981fae29dd..3ef6a9db73 100644
--- a/netlify.toml
+++ b/netlify.toml
@@ -3,3 +3,7 @@
   NODE_VERSION = "16.17.1"
 [build]
   ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF ./docs/"
+[[headers]]
+  for = "/*"
+  [[headers.values]]
+    X-Robots-Tag = "none"
diff --git a/news/5915.documentation b/news/5915.documentation
new file mode 100644
index 0000000000..6264f42515
--- /dev/null
+++ b/news/5915.documentation
@@ -0,0 +1 @@
+Block search engines from indexing content on Netlify preview builds. @stevepiercy