Block search engines from indexing content on Netlify preview builds

See plone/documentation#1619
plone · Feb 1, 2024 · e454f56 · e454f56
1 parent 4e741b9
commit e454f56
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 6 deletions.
diff --git a/docs/source/robots.txt b/docs/source/robots.txt
@@ -1,7 +1,3 @@
-# Disallow all user agents from the following directories and files
+# Disallow all user agents from indexing pages and links in the site on Netlify only.
 User-agent: *
-Disallow: /_sources/
-Disallow: /.doctrees/
-Disallow: /*.txt$
-Disallow: /.buildinfo$
-Disallow: /objects.inv$
+Disallow: /
diff --git a/netlify.toml b/netlify.toml
@@ -2,3 +2,7 @@
   PYTHON_VERSION = "3.8"
 [build]
   ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF ./docs/"
+[[headers]]
+  for = "/*"
+  [[headers.values]]
+    X-Robots-Tag = "none"