From 0126aa71311acbc0c6a679d1620f025ae6ceb980 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 19 Dec 2024 12:35:30 -0800 Subject: [PATCH 1/3] robots: block semver-based URLs --- static/robots.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/static/robots.txt b/static/robots.txt index 7dd3caf3e..3c1673a2c 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1 +1,6 @@ Sitemap: https://docs.rs/sitemap.xml +# Semver-based URL are always redirects, and sometimes +# confuse Google's duplicate detection, so we block crawling them. +# https://docs.rs/about/redirections +Disallow: */^ +Disallow: */~ From 75e0026b550620d331db71157a79682c5acbe66a Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Thu, 19 Dec 2024 22:11:50 -0800 Subject: [PATCH 2/3] Add a User-Agent And remove / from the Disallow rule --- static/robots.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/static/robots.txt b/static/robots.txt index 3c1673a2c..06fb71627 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -2,5 +2,6 @@ Sitemap: https://docs.rs/sitemap.xml # Semver-based URL are always redirects, and sometimes # confuse Google's duplicate detection, so we block crawling them. # https://docs.rs/about/redirections -Disallow: */^ -Disallow: */~ +User-Agent: * +Disallow: *^ +Disallow: *~ From 98a797e2e35df7bcf67cd81b72378566b9bee41e Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 20 Dec 2024 14:22:32 -0800 Subject: [PATCH 3/3] Restore slash to disallow rule --- static/robots.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/static/robots.txt b/static/robots.txt index 06fb71627..04040f2c9 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -3,5 +3,5 @@ Sitemap: https://docs.rs/sitemap.xml # confuse Google's duplicate detection, so we block crawling them. # https://docs.rs/about/redirections User-Agent: * -Disallow: *^ -Disallow: *~ +Disallow: */^ +Disallow: */~