Rollup merge of rust-lang#139593 - lolbinarycat:sitemap, r=jieyouxu

Zalathar · web-flow · commit 8e5f8389b93a · 2025-09-09T14:35:00.000+10:00
add sitemap to rust docs attempt to mitigate rust-lang#104670
diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs
@@ -88,6 +88,7 @@ impl Step for Docs {
         tarball.set_product_name("Rust Documentation");
         tarball.add_bulk_dir(builder.doc_out(host), dest);
         tarball.add_file(builder.src.join("src/doc/robots.txt"), dest, FileType::Regular);
+        tarball.add_file(builder.src.join("src/doc/sitemap.txt"), dest, FileType::Regular);
         Some(tarball.generate())
     }
 
diff --git a/src/doc/robots.txt b/src/doc/robots.txt
@@ -9,3 +9,5 @@ Disallow: /beta/book/first-edition/
 Disallow: /beta/book/second-edition/
 Disallow: /nightly/book/first-edition/
 Disallow: /nightly/book/second-edition/
+
+Sitemap: https://doc.rust-lang.org/sitemap.txt
diff --git a/src/doc/sitemap.txt b/src/doc/sitemap.txt
@@ -0,0 +1,4 @@
+https://doc.rust-lang.org/stable/
+https://doc.rust-lang.org/beta/
+https://doc.rust-lang.org/nightly/
+

Original file line number	Diff line number	Diff line change
`@@ -88,6 +88,7 @@ impl Step for Docs {`
`88`	`88`	`tarball.set_product_name("Rust Documentation");`
`89`	`89`	`tarball.add_bulk_dir(builder.doc_out(host), dest);`
`90`	`90`	`tarball.add_file(builder.src.join("src/doc/robots.txt"), dest, FileType::Regular);`
	`91`	`+ tarball.add_file(builder.src.join("src/doc/sitemap.txt"), dest, FileType::Regular);`
`91`	`92`	`Some(tarball.generate())`
`92`	`93`	`}`
`93`	`94`