fix(seo): prevent contributors.txt from indexing and crawling (#11149)

* chore(article-footer): set rel="nofollow" on contributors.txt links * chore(cloud-function): set X-Robots-Tag header on /contributors.txt * fix(robots): disallow contributors.txt
mdn · May 27, 2024 · 7164c71 · 7164c71
1 parent d4c41a8
commit 7164c71
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 1 deletion.
diff --git a/client/src/document/organisms/article-footer/index.tsx b/client/src/document/organisms/article-footer/index.tsx
@@ -29,7 +29,11 @@ export function LastModified({ value, locale }) {
 }
 
 export function Authors({ url }) {
-  return <a href={`${url}/contributors.txt`}>MDN contributors</a>;
+  return (
+    <a href={`${url}/contributors.txt`} rel="nofollow">
+      MDN contributors
+    </a>
+  );
 }
 
 enum ArticleFooterView {

diff --git a/cloud-function/src/headers.ts b/cloud-function/src/headers.ts
@@ -38,6 +38,10 @@ export function withContentResponseHeaders(
     xFrame: !isLiveSample,
   });
 
+  if (req.url?.endsWith("/contributors.txt")) {
+    res.setHeader("X-Robots-Tag", "noindex, nofollow");
+  }
+
   if (req.url?.endsWith("/sitemap.xml.gz")) {
     res.setHeader("Content-Type", "application/xml");
     res.setHeader("Content-Encoding", "gzip");

diff --git a/tool/build-robots-txt.ts b/tool/build-robots-txt.ts
@@ -13,6 +13,7 @@ User-agent: *
 Sitemap: https://developer.mozilla.org/sitemap.xml
 
 Disallow: /api/
+Disallow: /*/contributors.txt
 Disallow: /*/files/
 Disallow: /media
 `;