fix(ssr): remove unnecessary robots tags (#11139)

* fix(ssr): remove unnecessary robots tags 1. Robots tag for 404 page is unnecessary, as it is ignored anyways. 2. Robots tag with value "index, follow" is unnecessary, as it is the default. * test(spas): update expectations
mdn · May 16, 2024 · 1207158 · 1207158
1 parent f4c2388
commit 1207158
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 9 deletions.
diff --git a/ssr/render.ts b/ssr/render.ts
@@ -209,13 +209,12 @@ export default function render(
   }</script>`;
 
   const robotsContent =
-    !ALWAYS_ALLOW_ROBOTS ||
-    (doc && doc.noIndexing) ||
-    pageNotFound ||
-    noIndexing
+    !ALWAYS_ALLOW_ROBOTS || (doc && doc.noIndexing) || noIndexing
       ? "noindex, nofollow"
-      : "index, follow";
-  const robotsMeta = `<meta name="robots" content="${robotsContent}">`;
+      : "";
+  const robotsMeta = robotsContent
+    ? `<meta name="robots" content="${robotsContent}">`
+    : "";
   const rssLink = `<link rel="alternate" type="application/rss+xml" title="MDN Blog RSS Feed" href="${BASE_URL}/${DEFAULT_LOCALE}/blog/rss.xml" hreflang="en" />`;
   const ssr_data = [...translations, ...WEBFONT_TAGS, rssLink, robotsMeta];
   let html = buildHtml;

diff --git a/testing/tests/index.test.ts b/testing/tests/index.test.ts
@@ -119,7 +119,7 @@ test("content built foo page", () => {
     `https://developer.mozilla.org${doc.mdn_url}`
   );
 
-  expect($('meta[name="robots"]').attr("content")).toBe("index, follow");
+  expect($('meta[name="robots"]')).toHaveLength(0);
 
   // The HTML should contain the Google Analytics snippet.
   // The ID should match what's set in `.env.testing`.
@@ -1179,7 +1179,7 @@ test("404 page", () => {
   const $ = cheerio.load(html);
   expect($("title").text()).toContain("Page not found");
   expect($("h1").text()).toContain("Page not found");
-  expect($('meta[name="robots"]').attr("content")).toBe("noindex, nofollow");
+  expect($('meta[name="robots"]')).toHaveLength(0);
   expect($('meta[property="og:locale"]').attr("content")).toBe("en_US");
 });
 
@@ -1190,7 +1190,7 @@ test("plus page", () => {
   const html = fs.readFileSync(htmlFile, "utf-8");
   const $ = cheerio.load(html);
   expect($("title").text()).toContain("Plus");
-  expect($('meta[name="robots"]').attr("content")).toBe("index, follow");
+  expect($('meta[name="robots"]')).toHaveLength(0);
 });
 
 test("plus collections page", () => {