Add target crates as inputs to reduce size of intermediates

Add tests for module-path remapping and scrape example options

Find all crates with a given name

Author: willcrichton

src/librustdoc/clean/mod.rs

@@ -1059,20 +1059,22 @@ impl Clean<Item> for ty::AssocItem {
                         ty::ImplContainer(_) => Some(self.defaultness),
                         ty::TraitContainer(_) => None,
                     };
-                    let function = Function {
-                        generics,
-                        decl,
-                        header: hir::FnHeader {
-                            unsafety: sig.unsafety(),
-                            abi: sig.abi(),
-                            constness,
-                            asyncness,
+                    MethodItem(
+                        Function {
+                            generics,
+                            decl,
+                            header: hir::FnHeader {
+                                unsafety: sig.unsafety(),
+                                abi: sig.abi(),
+                                constness,
+                                asyncness,
+                            },
+                            def_id: self.def_id,
                         },
-                        def_id: self.def_id,
-                    };
-                    MethodItem(function, defaultness)
+                        defaultness,
+                    )
                 } else {
-                    let function = Function {
+                    TyMethodItem(Function {
                         generics,
                         decl,
                         header: hir::FnHeader {
@@ -1082,8 +1084,7 @@ impl Clean<Item> for ty::AssocItem {
                             asyncness: hir::IsAsync::NotAsync,
                         },
                         def_id: self.def_id,
-                    };
-                    TyMethodItem(function)
+                    })
                 }
             }
             ty::AssocKind::Type => {

src/librustdoc/config.rs

@@ -25,7 +25,7 @@ use crate::html::render::StylePath;
 use crate::html::static_files;
 use crate::opts;
 use crate::passes::{self, Condition, DefaultPassOption};
-use crate::scrape_examples::AllCallLocations;
+use crate::scrape_examples::{AllCallLocations, ScrapeExamplesOptions};
 use crate::theme;
 
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
@@ -160,9 +160,9 @@ crate struct Options {
     /// Whether to skip capturing stdout and stderr of tests.
     crate nocapture: bool,
 
-    /// Path to output file to write JSON of call sites. If this option is Some(..) then
+    /// Configuration for scraping examples from the current crate. If this option is Some(..) then
     /// the compiler will scrape examples and not generate documentation.
-    crate scrape_examples: Option<PathBuf>,
+    crate scrape_examples_options: Option<ScrapeExamplesOptions>,
 }
 
 impl fmt::Debug for Options {
@@ -207,7 +207,7 @@ impl fmt::Debug for Options {
             .field("run_check", &self.run_check)
             .field("no_run", &self.no_run)
             .field("nocapture", &self.nocapture)
-            .field("scrape_examples", &self.scrape_examples)
+            .field("scrape_examples_options", &self.scrape_examples_options)
             .finish()
     }
 }
@@ -678,7 +678,7 @@ impl Options {
             return Err(1);
         }
 
-        let scrape_examples = matches.opt_str("scrape-examples").map(PathBuf::from);
+        let scrape_examples_options = ScrapeExamplesOptions::new(&matches, &diag)?;
         let with_examples = matches.opt_strs("with-examples");
         let call_locations = crate::scrape_examples::load_call_locations(with_examples, &diag)?;
 
@@ -753,7 +753,7 @@ impl Options {
             crate_name,
             output_format,
             json_unused_externs,
-            scrape_examples,
+            scrape_examples_options,
         })
     }
 

src/librustdoc/lib.rs

@@ -621,12 +621,20 @@ fn opts() -> Vec<RustcOptGroup> {
                 "Make the identifiers in the HTML source code pages navigable",
             )
         }),
-        unstable("scrape-examples", |o| {
+        unstable("scrape-examples-output-path", |o| {
             o.optopt(
                 "",
-                "scrape-examples",
+                "scrape-examples-output-path",
                 "",
-                "collect function call information (for use with `--with-examples`)",
+                "collect function call information and output at the given path",
+            )
+        }),
+        unstable("scrape-examples-target-crate", |o| {
+            o.optmulti(
+                "",
+                "scrape-examples-target-crate",
+                "",
+                "collect function call information for functions from the target crate",
             )
         }),
         unstable("with-examples", |o| {
@@ -750,7 +758,7 @@ fn main_options(options: config::Options) -> MainResult {
     // FIXME: fix this clone (especially render_options)
     let manual_passes = options.manual_passes.clone();
     let render_options = options.render_options.clone();
-    let scrape_examples = options.scrape_examples.clone();
+    let scrape_examples_options = options.scrape_examples_options.clone();
     let config = core::create_config(options);
 
     interface::create_compiler_and_run(config, |compiler| {
@@ -787,8 +795,8 @@ fn main_options(options: config::Options) -> MainResult {
                 });
                 info!("finished with rustc");
 
-                if let Some(example_path) = scrape_examples {
-                    return scrape_examples::run(krate, render_opts, cache, tcx, example_path);
+                if let Some(options) = scrape_examples_options {
+                    return scrape_examples::run(krate, render_opts, cache, tcx, options);
                 }
 
                 cache.crate_version = crate_version;

src/librustdoc/scrape_examples.rs

@@ -20,11 +20,43 @@ use rustc_serialize::{
     opaque::{Decoder, FileEncoder},
     Decodable, Encodable,
 };
-use rustc_span::{def_id::DefId, edition::Edition, BytePos, FileName, SourceFile};
+use rustc_session::getopts;
+use rustc_span::{
+    def_id::{CrateNum, DefId},
+    edition::Edition,
+    BytePos, FileName, SourceFile,
+};
 
 use std::fs;
 use std::path::PathBuf;
 
+#[derive(Debug, Clone)]
+crate struct ScrapeExamplesOptions {
+    output_path: PathBuf,
+    target_crates: Vec<String>,
+}
+
+impl ScrapeExamplesOptions {
+    crate fn new(
+        matches: &getopts::Matches,
+        diag: &rustc_errors::Handler,
+    ) -> Result<Option<Self>, i32> {
+        let output_path = matches.opt_str("scrape-examples-output-path");
+        let target_crates = matches.opt_strs("scrape-examples-target-crate");
+        match (output_path, !target_crates.is_empty()) {
+            (Some(output_path), true) => Ok(Some(ScrapeExamplesOptions {
+                output_path: PathBuf::from(output_path),
+                target_crates,
+            })),
+            (Some(_), false) | (None, true) => {
+                diag.err(&format!("must use --scrape-examples-output-path and --scrape-examples-target-crate together"));
+                Err(1)
+            }
+            (None, false) => Ok(None),
+        }
+    }
+}
+
 #[derive(Encodable, Decodable, Debug, Clone)]
 crate struct SyntaxRange {
     crate byte_span: (u32, u32),
@@ -83,6 +115,7 @@ struct FindCalls<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     map: Map<'tcx>,
     cx: Context<'tcx>,
+    target_crates: Vec<CrateNum>,
     calls: &'a mut AllCallLocations,
 }
 
@@ -130,6 +163,11 @@ where
 
         // Save call site if the function resolves to a concrete definition
         if let ty::FnDef(def_id, _) = ty.kind() {
+            // Ignore functions not from the crate being documented
+            if self.target_crates.iter().all(|krate| *krate != def_id.krate) {
+                return;
+            }
+
             let file = tcx.sess.source_map().lookup_char_pos(span.lo()).file;
             let file_path = match file.name.clone() {
                 FileName::Real(real_filename) => real_filename.into_local_path(),
@@ -143,7 +181,7 @@ where
                     let clean_span = crate::clean::types::Span::new(span);
                     let url = cx.href_from_span(clean_span).unwrap();
                     let display_name = file_path.display().to_string();
-                    let edition = tcx.sess.edition();
+                    let edition = span.edition();
                     CallData { locations: Vec::new(), url, display_name, edition }
                 };
 
@@ -162,19 +200,34 @@ crate fn run(
     renderopts: config::RenderOptions,
     cache: formats::cache::Cache,
     tcx: TyCtxt<'_>,
-    example_path: PathBuf,
+    options: ScrapeExamplesOptions,
 ) -> interface::Result<()> {
     let inner = move || -> Result<(), String> {
         // Generates source files for examples
         let (cx, _) = Context::init(krate, renderopts, cache, tcx).map_err(|e| e.to_string())?;
 
+        // Collect CrateIds corresponding to provided target crates
+        // If two different versions of the crate in the dependency tree, then examples will be collcted from both.
+        let find_crates_with_name = |target_crate: String| {
+            tcx.crates(())
+                .iter()
+                .filter(move |crate_num| tcx.crate_name(**crate_num).as_str() == target_crate)
+                .copied()
+        };
+        let target_crates = options
+            .target_crates
+            .into_iter()
+            .map(find_crates_with_name)
+            .flatten()
+            .collect::<Vec<_>>();
+
         // Run call-finder on all items
         let mut calls = FxHashMap::default();
-        let mut finder = FindCalls { calls: &mut calls, tcx, map: tcx.hir(), cx };
+        let mut finder = FindCalls { calls: &mut calls, tcx, map: tcx.hir(), cx, target_crates };
         tcx.hir().krate().visit_all_item_likes(&mut finder.as_deep_visitor());
 
         // Save output to provided path
-        let mut encoder = FileEncoder::new(example_path).map_err(|e| e.to_string())?;
+        let mut encoder = FileEncoder::new(options.output_path).map_err(|e| e.to_string())?;
         calls.encode(&mut encoder).map_err(|e| e.to_string())?;
         encoder.flush().map_err(|e| e.to_string())?;
 

src/test/run-make/rustdoc-scrape-examples-remap/Makefile

@@ -0,0 +1,22 @@
+-include ../../run-make-fulldeps/tools.mk
+
+OUTPUT_DIR := "$(TMPDIR)/rustdoc"
+
+all:
+# 1. compile the library crate and emit an rmeta
+	$(RUSTC) src/lib.rs --crate-name foobar --crate-type lib --emit=metadata
+
+# 2. scrape examples from the reverse-dependency into an ex.calls file
+	$(RUSTDOC) examples/ex.rs --crate-name ex --crate-type bin \
+	  --extern foobar=$(TMPDIR)/libfoobar.rmeta \
+		-Z unstable-options \
+		--scrape-examples-output-path $(TMPDIR)/ex.calls \
+		--scrape-examples-target-crate foobar
+
+# 3. pass those examples to rustdoc when documenting the library crate
+	$(RUSTDOC) src/lib.rs --crate-name foobar --crate-type lib --output $(OUTPUT_DIR) \
+		-Z unstable-options \
+		--with-examples $(TMPDIR)/ex.calls
+
+# 4. check that the examples were scraped successfully
+	$(HTMLDOCCK) $(OUTPUT_DIR) src/lib.rs

src/test/run-make/rustdoc-scrape-examples-remap/examples/ex.rs

@@ -0,0 +1,4 @@
+fn main() {
+    foobar::b::foo();
+    foobar::c::foo();
+}

src/test/run-make/rustdoc-scrape-examples-remap/src/a.rs

@@ -0,0 +1 @@
+pub fn foo() {}

src/test/run-make/rustdoc-scrape-examples-remap/src/lib.rs

@@ -0,0 +1,8 @@
+// @has foobar/b/fn.foo.html '//*[@class="scraped-example"]' 'ex.rs'
+// @has foobar/c/fn.foo.html '//*[@class="scraped-example"]' 'ex.rs'
+
+#[path = "a.rs"]
+pub mod b;
+
+#[path = "a.rs"]
+pub mod c;

src/test/rustdoc/scrape-examples-wrong-options.rs

@@ -0,0 +1,2 @@
+// should-fail
+// compile-flags: --scrape-examples-target-crate foobar