From 1b768a6ddc99420997d5b67fba46ca1cddc209a2 Mon Sep 17 00:00:00 2001
From: Steve Piercy <web@stevepiercy.com>
Date: Sun, 24 Mar 2024 15:11:03 -0700
Subject: [PATCH 1/2] Backport #5725: Block search engines from indexing
 content on Netlify preview builds

---
 docs/source/robots.txt | 9 ++-------
 netlify.toml           | 4 ++++
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/docs/source/robots.txt b/docs/source/robots.txt
index 88d8608916..2dc87d3d33 100644
--- a/docs/source/robots.txt
+++ b/docs/source/robots.txt
@@ -1,7 +1,2 @@
-# Disallow all user agents from the following directories and files
-User-agent: *
-Disallow: /_sources/
-Disallow: /.doctrees/
-Disallow: /*.txt$
-Disallow: /.buildinfo$
-Disallow: /objects.inv$
+# Disallow all user agents from indexing pages and links in the site on Netlify only.
+Disallow: /
diff --git a/netlify.toml b/netlify.toml
index 981fae29dd..3ef6a9db73 100644
--- a/netlify.toml
+++ b/netlify.toml
@@ -3,3 +3,7 @@
   NODE_VERSION = "16.17.1"
 [build]
   ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF ./docs/"
+[[headers]]
+  for = "/*"
+  [[headers.values]]
+    X-Robots-Tag = "none"

From 27233cb0fbb52c93d385927cd3de10795a8de8a2 Mon Sep 17 00:00:00 2001
From: Steve Piercy <web@stevepiercy.com>
Date: Sun, 24 Mar 2024 15:11:53 -0700
Subject: [PATCH 2/2] Add changelog

---
 news/5915.documentation | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 news/5915.documentation

diff --git a/news/5915.documentation b/news/5915.documentation
new file mode 100644
index 0000000000..6264f42515
--- /dev/null
+++ b/news/5915.documentation
@@ -0,0 +1 @@
+Block search engines from indexing content on Netlify preview builds. @stevepiercy