Make rust-demangler installable

richkadel · richkadel · commit 23325caf43d1 · 2021-04-03T11:42:54.000-07:00
Adds bootstrap rules to support installing rust-demangler.

When compiling with `-Z instrument-coverage`, the coverage reports are
generated by `llvm-cov`. `llvm-cov` includes a built-in demangler for
C++, and an option to supply an alternate demangler. For Rust, we have
`rust-demangler`, currently used in `rustc` coverage tests.

Fuchsia's toolchain for Rust is built via `./x.py install`. Fuchsia is
adding support for Rust coverage, and we need to include the
`rust-demangler` in the installed `bin` directory.

Configured rust-demangler as an in-tree extended tool.

Added tests to support `./x.py test rust-demangler`.

Install with extended tools by default only if `profiler = true`.
diff --git a/config.toml.example b/config.toml.example
@@ -259,10 +259,11 @@ changelog-seen = 2
 # be built if `extended = true`.
 #extended = false
 
-# Installs chosen set of extended tools if `extended = true`. By default builds all.
-# If chosen tool failed to build the installation fails. If `extended = false`, this
-# option is ignored.
-#tools = ["cargo", "rls", "clippy", "rustfmt", "analysis", "src"]
+# Installs chosen set of extended tools if `extended = true`. By default builds
+# all extended tools except `rust-demangler`, unless the target is also being
+# built with `profiler = true`. If chosen tool failed to build the installation
+# fails. If `extended = false`, this option is ignored.
+#tools = ["cargo", "rls", "clippy", "rustfmt", "analysis", "src"] # + "rust-demangler" if `profiler`
 
 # Verbosity level: 0 == not verbose, 1 == verbose, 2 == very verbose
 #verbose = 0
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
@@ -420,6 +420,7 @@ impl<'a> Builder<'a> {
                 test::Rustfmt,
                 test::Miri,
                 test::Clippy,
+                test::RustDemangler,
                 test::CompiletestTest,
                 test::RustdocJSStd,
                 test::RustdocJSNotStd,
@@ -466,6 +467,7 @@ impl<'a> Builder<'a> {
                 dist::Rls,
                 dist::RustAnalyzer,
                 dist::Rustfmt,
+                dist::RustDemangler,
                 dist::Clippy,
                 dist::Miri,
                 dist::LlvmTools,
@@ -481,6 +483,7 @@ impl<'a> Builder<'a> {
                 install::Rls,
                 install::RustAnalyzer,
                 install::Rustfmt,
+                install::RustDemangler,
                 install::Clippy,
                 install::Miri,
                 install::Analysis,
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
@@ -1246,6 +1246,50 @@ impl Step for Rustfmt {
     }
 }
 
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustDemangler {
+    pub compiler: Compiler,
+    pub target: TargetSelection,
+}
+
+impl Step for RustDemangler {
+    type Output = GeneratedTarball;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("rust-demangler")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(RustDemangler {
+            compiler: run.builder.compiler_for(
+                run.builder.top_stage,
+                run.builder.config.build,
+                run.target,
+            ),
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
+        let compiler = self.compiler;
+        let target = self.target;
+        assert!(builder.config.extended);
+
+        let rust_demangler = builder
+            .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
+            .expect("rust-demangler expected to build - in-tree tool");
+
+        // Prepare the image directory
+        let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
+        tarball.set_overlay(OverlayKind::RustDemangler);
+        tarball.is_preview(true);
+        tarball.add_file(&rust_demangler, "bin", 0o755);
+        tarball.add_legal_and_readme_to("share/doc/rust-demangler");
+        tarball.generate()
+    }
+}
+
 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Extended {
     stage: u32,
@@ -1282,6 +1326,14 @@ impl Step for Extended {
         let rustc_installer = builder.ensure(Rustc { compiler: builder.compiler(stage, target) });
         let cargo_installer = builder.ensure(Cargo { compiler, target });
         let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
+        let profiler = builder.config.profiler_enabled(target);
+        let install_rust_demangler =
+            builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler"));
+        let rust_demangler_installer = if install_rust_demangler {
+            Some(builder.ensure(RustDemangler { compiler, target }))
+        } else {
+            None
+        };
         let rls_installer = builder.ensure(Rls { compiler, target });
         let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target });
         let llvm_tools_installer = builder.ensure(LlvmTools { target });
@@ -1312,6 +1364,9 @@ impl Step for Extended {
         tarballs.push(clippy_installer);
         tarballs.extend(miri_installer.clone());
         tarballs.extend(rustfmt_installer.clone());
+        if let Some(rust_demangler_installer) = rust_demangler_installer {
+            tarballs.push(rust_demangler_installer);
+        }
         tarballs.extend(llvm_tools_installer);
         if let Some(analysis_installer) = analysis_installer {
             tarballs.push(analysis_installer);
@@ -1413,6 +1468,9 @@ impl Step for Extended {
             prepare("rust-docs");
             prepare("rust-std");
             prepare("rust-analysis");
+            if install_rust_demangler {
+                prepare("rust-demangler");
+            }
             prepare("clippy");
 
             if rls_installer.is_some() {
@@ -1476,6 +1534,9 @@ impl Step for Extended {
             prepare("rustc");
             prepare("cargo");
             prepare("rust-analysis");
+            if install_rust_demangler {
+                prepare("rust-demangler");
+            }
             prepare("rust-docs");
             prepare("rust-std");
             prepare("clippy");
@@ -1620,6 +1681,25 @@ impl Step for Extended {
                     .arg("-t")
                     .arg(etc.join("msi/remove-duplicates.xsl")),
             );
+            if install_rust_demangler {
+                builder.run(
+                    Command::new(&heat)
+                        .current_dir(&exe)
+                        .arg("dir")
+                        .arg("rust-demangler")
+                        .args(&heat_flags)
+                        .arg("-cg")
+                        .arg("RustDemanglerGroup")
+                        .arg("-dr")
+                        .arg("RustDemangler")
+                        .arg("-var")
+                        .arg("var.RustDemanglerDir")
+                        .arg("-out")
+                        .arg(exe.join("RustDemanglerGroup.wxs"))
+                        .arg("-t")
+                        .arg(etc.join("msi/remove-duplicates.xsl")),
+                );
+            }
             if miri_installer.is_some() {
                 builder.run(
                     Command::new(&heat)
@@ -1715,6 +1795,9 @@ impl Step for Extended {
             candle("CargoGroup.wxs".as_ref());
             candle("StdGroup.wxs".as_ref());
             candle("ClippyGroup.wxs".as_ref());
+            if install_rust_demangler {
+                candle("RustDemanglerGroup.wxs".as_ref());
+            }
             if rls_installer.is_some() {
                 candle("RlsGroup.wxs".as_ref());
             }
@@ -1761,6 +1844,9 @@ impl Step for Extended {
             if rust_analyzer_installer.is_some() {
                 cmd.arg("RustAnalyzerGroup.wixobj");
             }
+            if install_rust_demangler {
+                cmd.arg("RustDemanglerGroup.wixobj");
+            }
             if miri_installer.is_some() {
                 cmd.arg("MiriGroup.wixobj");
             }
diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs
@@ -190,6 +190,26 @@ install!((self, builder, _config),
             );
         }
     };
+    RustDemangler,
+        "rust-demangler",
+        Self::should_build(_config),
+        only_hosts: true,
+    {
+        let profiler = builder.config.profiler_enabled(self.target);
+        let install_rust_demangler =
+            builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler"));
+        if install_rust_demangler {
+            let tarball = builder.ensure(
+                dist::RustDemangler { compiler: self.compiler, target: self.target }
+            );
+            install_sh(builder, "rust-demangler", self.compiler.stage, Some(self.target), &tarball);
+        } else {
+            builder.info(
+                &format!("skipping Install RustDemangler stage{} ({})",
+                         self.compiler.stage, self.target),
+            );
+        }
+    };
     Analysis, "analysis", Self::should_build(_config), only_hosts: false, {
         let tarball = builder.ensure(dist::Analysis {
             // Find the actual compiler (handling the full bootstrap option) which
diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs
@@ -15,6 +15,7 @@ pub(crate) enum OverlayKind {
     Clippy,
     Miri,
     Rustfmt,
+    RustDemangler,
     RLS,
     RustAnalyzer,
 }
@@ -47,6 +48,9 @@ impl OverlayKind {
                 "src/tools/rustfmt/LICENSE-APACHE",
                 "src/tools/rustfmt/LICENSE-MIT",
             ],
+            OverlayKind::RustDemangler => {
+                &["src/tools/rust-demangler/README.md", "LICENSE-APACHE", "LICENSE-MIT"]
+            }
             OverlayKind::RLS => &[
                 "src/tools/rls/README.md",
                 "src/tools/rls/LICENSE-APACHE",
@@ -64,6 +68,7 @@ impl OverlayKind {
         match self {
             OverlayKind::Rust => builder.rust_version(),
             OverlayKind::LLVM => builder.rust_version(),
+            OverlayKind::RustDemangler => builder.rust_version(),
             OverlayKind::Cargo => {
                 builder.cargo_info.version(builder, &builder.release_num("cargo"))
             }
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
@@ -351,6 +351,54 @@ impl Step for Rustfmt {
     }
 }
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct RustDemangler {
+    stage: u32,
+    host: TargetSelection,
+}
+
+impl Step for RustDemangler {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/rust-demangler")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(RustDemangler { stage: run.builder.top_stage, host: run.target });
+    }
+
+    /// Runs `cargo test` for rust-demangler.
+    fn run(self, builder: &Builder<'_>) {
+        let stage = self.stage;
+        let host = self.host;
+        let compiler = builder.compiler(stage, host);
+
+        let rust_demangler = builder
+            .ensure(tool::RustDemangler { compiler, target: self.host, extra_features: Vec::new() })
+            .expect("in-tree tool");
+        let mut cargo = tool::prepare_tool_cargo(
+            builder,
+            compiler,
+            Mode::ToolRustc,
+            host,
+            "test",
+            "src/tools/rust-demangler",
+            SourceType::InTree,
+            &[],
+        );
+
+        let dir = testdir(builder, compiler.host);
+        t!(fs::create_dir_all(&dir));
+
+        cargo.env("RUST_DEMANGLER_DRIVER_PATH", rust_demangler);
+        cargo.add_rustc_lib_path(builder, compiler);
+
+        builder.run(&mut cargo.into());
+    }
+}
+
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Miri {
     stage: u32,
@@ -1126,7 +1174,10 @@ note: if you're sure you want to do this, please open an issue as to why. In the
         }
 
         if mode == "run-make" && suite.ends_with("fulldeps") {
-            cmd.arg("--rust-demangler-path").arg(builder.tool_exe(Tool::RustDemangler));
+            let rust_demangler = builder
+                .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
+                .expect("in-tree tool");
+            cmd.arg("--rust-demangler-path").arg(rust_demangler);
         }
 
         cmd.arg("--src-base").arg(builder.src.join("src/test").join(suite));
diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs
@@ -368,7 +368,6 @@ bootstrap_tool!(
     Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true;
     BuildManifest, "src/tools/build-manifest", "build-manifest";
     RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
-    RustDemangler, "src/tools/rust-demangler", "rust-demangler";
     RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true;
     RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
     ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
@@ -719,6 +718,7 @@ tool_extended!((self, builder),
         });
         self.extra_features.push("clippy".to_owned());
     };
+    RustDemangler, rust_demangler, "src/tools/rust-demangler", "rust-demangler", stable=false, in_tree=true, {};
     Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, {};
     RustAnalyzer, rust_analyzer, "src/tools/rust-analyzer/crates/rust-analyzer", "rust-analyzer", stable=false, {};
 );
diff --git a/src/tools/rust-demangler/Cargo.toml b/src/tools/rust-demangler/Cargo.toml
@@ -8,6 +8,10 @@ edition = "2018"
 regex = "1.0"
 rustc-demangle = "0.1.17"
 
+[lib]
+name = "rust_demangler"
+doctest = false
+
 [[bin]]
 name = "rust-demangler"
-path = "main.rs"
+test = false
diff --git a/src/tools/rust-demangler/README.md b/src/tools/rust-demangler/README.md
@@ -0,0 +1,29 @@
+# rust-demangler
+
+Demangles rustc mangled names.
+
+This tool uses the [rustc-demangle](https://crates.io/crates/rustc-demangle)
+crate to convert an input buffer of newline-separated mangled names into their
+demangled translations.
+
+This tool takes a list of mangled names (one per line) on standard input, and
+prints a corresponding list of demangled names. The tool is designed to support
+programs that can leverage a third-party demangler, such as `llvm-cov`, via the
+`-Xdemangler=<path-to-demangler>` option.
+
+To use `rust-demangler` with `llvm-cov` for example, add the `-Xdemangler=...`
+option:
+
+```shell
+$ TARGET="${PWD}/build/x86_64-unknown-linux-gnu"
+$ "${TARGET}"/llvm/bin/llvm-cov show \
+  --Xdemangler=path/to/rust-demangler \
+  --instr-profile=main.profdata ./main --show-line-counts-or-regions
+```
+
+## License
+
+Rust-demangler is distributed under the terms of both the MIT license and the
+Apache License (Version 2.0).
+
+See [LICENSE-APACHE](/LICENSE-APACHE) and [LICENSE-MIT](/LICENSE-MIT) for details.
diff --git a/src/tools/rust-demangler/src/lib.rs b/src/tools/rust-demangler/src/lib.rs
@@ -0,0 +1,22 @@
+use regex::Regex;
+use rustc_demangle::demangle;
+
+const REPLACE_COLONS: &str = "::";
+
+pub fn create_disambiguator_re() -> Regex {
+    Regex::new(r"\[[a-f0-9]{5,16}\]::").unwrap()
+}
+
+pub fn demangle_lines(buffer: &str, strip_crate_disambiguators: Option<Regex>) -> Vec<String> {
+    let lines = buffer.lines();
+    let mut demangled_lines = Vec::new();
+    for mangled in lines {
+        let mut demangled = demangle(mangled).to_string();
+        if let Some(re) = &strip_crate_disambiguators {
+            demangled = re.replace_all(&demangled, REPLACE_COLONS).to_string();
+        }
+        demangled_lines.push(demangled);
+    }
+    demangled_lines.push("".to_string());
+    demangled_lines
+}
diff --git a/src/tools/rust-demangler/src/main.rs b/src/tools/rust-demangler/src/main.rs
diff --git a/src/tools/rust-demangler/tests/lib.rs b/src/tools/rust-demangler/tests/lib.rs
