Start using serde_derive in a couple places in the compiler.

Author: eddyb

Cargo.lock

@@ -2540,6 +2540,7 @@ version = "0.0.0"
 dependencies = [
  "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -2604,6 +2605,7 @@ dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc_cratesio_shim 0.0.0",
  "rustc_data_structures 0.0.0",
+ "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serialize 0.0.0",
  "syntax_pos 0.0.0",
  "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3217,6 +3219,7 @@ dependencies = [
  "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc_data_structures 0.0.0",
  "scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serialize 0.0.0",
  "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

src/bootstrap/bin/rustc.rs

@@ -109,6 +109,12 @@ fn main() {
         // actually downloaded, so we just always pass the `--sysroot` option.
         cmd.arg("--sysroot").arg(&sysroot);
 
+        // Link crates to the proc macro crate for the target, but use a host proc macro crate
+        // to actually run the macros
+        if env::var_os("RUST_DUAL_PROC_MACROS").is_some() {
+            cmd.arg("-Zdual-proc-macros");
+        }
+
         // When we build Rust dylibs they're all intended for intermediate
         // usage, so make sure we pass the -Cprefer-dynamic flag instead of
         // linking all deps statically into the dylib.

src/bootstrap/builder.rs

@@ -608,6 +608,7 @@ impl<'a> Builder<'a> {
                     .join(self.target)
                     .join("lib");
                 let _ = fs::remove_dir_all(&sysroot);
+                eprintln!("creating sysroot {:?}", sysroot);
                 t!(fs::create_dir_all(&sysroot));
                 INTERNER.intern_path(sysroot)
             }
@@ -811,6 +812,12 @@ impl<'a> Builder<'a> {
             cargo.env("RUST_CHECK", "1");
         }
 
+        // Build proc macros both for the host and the target
+        if target != compiler.host && cmd != "check" {
+            cargo.arg("-Zdual-proc-macros");
+            cargo.env("RUST_DUAL_PROC_MACROS", "1");
+        }
+
         cargo.arg("-j").arg(self.jobs().to_string());
         // Remove make-related flags to ensure Cargo can correctly set things up
         cargo.env_remove("MAKEFLAGS");

src/bootstrap/check.rs

@@ -42,7 +42,8 @@ impl Step for Std {
                   true);
 
         let libdir = builder.sysroot_libdir(compiler, target);
-        add_to_sysroot(&builder, &libdir, &libstd_stamp(builder, compiler, target));
+        let hostdir = builder.sysroot_libdir(compiler, compiler.host);
+        add_to_sysroot(&builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target));
     }
 }
 
@@ -88,7 +89,8 @@ impl Step for Rustc {
                   true);
 
         let libdir = builder.sysroot_libdir(compiler, target);
-        add_to_sysroot(&builder, &libdir, &librustc_stamp(builder, compiler, target));
+        let hostdir = builder.sysroot_libdir(compiler, compiler.host);
+        add_to_sysroot(&builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target));
     }
 }
 
@@ -175,7 +177,8 @@ impl Step for Test {
                   true);
 
         let libdir = builder.sysroot_libdir(compiler, target);
-        add_to_sysroot(builder, &libdir, &libtest_stamp(builder, compiler, target));
+        let hostdir = builder.sysroot_libdir(compiler, compiler.host);
+        add_to_sysroot(builder, &libdir, &hostdir, &libtest_stamp(builder, compiler, target));
     }
 }
 
@@ -222,7 +225,8 @@ impl Step for Rustdoc {
                   true);
 
         let libdir = builder.sysroot_libdir(compiler, target);
-        add_to_sysroot(&builder, &libdir, &rustdoc_stamp(builder, compiler, target));
+        let hostdir = builder.sysroot_libdir(compiler, compiler.host);
+        add_to_sysroot(&builder, &libdir, &hostdir, &rustdoc_stamp(builder, compiler, target));
         builder.cargo(compiler, Mode::ToolRustc, target, "clean");
     }
 }

src/bootstrap/compile.rs

@@ -224,7 +224,8 @@ impl Step for StdLink {
                 target_compiler.host,
                 target));
         let libdir = builder.sysroot_libdir(target_compiler, target);
-        add_to_sysroot(builder, &libdir, &libstd_stamp(builder, compiler, target));
+        let hostdir = builder.sysroot_libdir(target_compiler, compiler.host);
+        add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target));
 
         if builder.config.sanitizers && compiler.stage != 0 && target == "x86_64-apple-darwin" {
             // The sanitizers are only built in stage1 or above, so the dylibs will
@@ -426,8 +427,12 @@ impl Step for TestLink {
                 &compiler.host,
                 target_compiler.host,
                 target));
-        add_to_sysroot(builder, &builder.sysroot_libdir(target_compiler, target),
-                    &libtest_stamp(builder, compiler, target));
+        add_to_sysroot(
+            builder,
+            &builder.sysroot_libdir(target_compiler, target),
+            &builder.sysroot_libdir(target_compiler, compiler.host),
+            &libtest_stamp(builder, compiler, target)
+        );
 
         builder.cargo(target_compiler, Mode::ToolTest, target, "clean");
     }
@@ -491,8 +496,8 @@ impl Step for Rustc {
             return;
         }
 
-        // Ensure that build scripts have a std to link against.
-        builder.ensure(Std {
+        // Ensure that build scripts and proc macros have a std / libproc_macro to link against.
+        builder.ensure(Test {
             compiler: builder.compiler(self.compiler.stage, builder.config.build),
             target: builder.config.build,
         });
@@ -587,8 +592,12 @@ impl Step for RustcLink {
                  &compiler.host,
                  target_compiler.host,
                  target));
-        add_to_sysroot(builder, &builder.sysroot_libdir(target_compiler, target),
-                       &librustc_stamp(builder, compiler, target));
+        add_to_sysroot(
+            builder,
+            &builder.sysroot_libdir(target_compiler, target),
+            &builder.sysroot_libdir(target_compiler, compiler.host),
+            &librustc_stamp(builder, compiler, target)
+        );
         builder.cargo(target_compiler, Mode::ToolRustc, target, "clean");
     }
 }
@@ -996,10 +1005,22 @@ impl Step for Assemble {
 ///
 /// For a particular stage this will link the file listed in `stamp` into the
 /// `sysroot_dst` provided.
-pub fn add_to_sysroot(builder: &Builder, sysroot_dst: &Path, stamp: &Path) {
+pub fn add_to_sysroot(
+    builder: &Builder,
+    sysroot_dst: &Path,
+    sysroot_host_dst: &Path,
+    stamp: &Path
+) {
+    //eprintln!("add_to_sysroot - host dir {:?} - stamp {:?}", sysroot_host_dst, stamp);
     t!(fs::create_dir_all(&sysroot_dst));
-    for path in builder.read_stamp_file(stamp) {
-        builder.copy(&path, &sysroot_dst.join(path.file_name().unwrap()));
+    t!(fs::create_dir_all(&sysroot_host_dst));
+    for (path, host) in builder.read_stamp_file(stamp) {
+        if host {
+            eprintln!("add_to_sysroot host - copying {:?} to {:?}", path, sysroot_host_dst);
+            builder.copy(&path, &sysroot_host_dst.join(path.file_name().unwrap()));
+        } else {
+            builder.copy(&path, &sysroot_dst.join(path.file_name().unwrap()));
+        }
     }
 }
 
@@ -1028,8 +1049,14 @@ pub fn run_cargo(builder: &Builder,
     let mut deps = Vec::new();
     let mut toplevel = Vec::new();
     let ok = stream_cargo(builder, cargo, &mut |msg| {
-        let filenames = match msg {
-            CargoMessage::CompilerArtifact { filenames, .. } => filenames,
+        let (filenames, crate_types) = match msg {
+            CargoMessage::CompilerArtifact {
+                filenames,
+                target: CargoTarget {
+                    crate_types,
+                },
+                ..
+            } => (filenames, crate_types),
             _ => return,
         };
         for filename in filenames {
@@ -1044,15 +1071,19 @@ pub fn run_cargo(builder: &Builder,
             let filename = Path::new(&*filename);
 
             // If this was an output file in the "host dir" we don't actually
-            // worry about it, it's not relevant for us.
+            // worry about it, it's not relevant for us
             if filename.starts_with(&host_root_dir) {
+                // Unless it's a proc macro used in the compiler
+                if crate_types.iter().any(|t| t == "proc-macro") {
+                    deps.push((filename.to_path_buf(), true));
+                }
                 continue;
             }
 
             // If this was output in the `deps` dir then this is a precise file
             // name (hash included) so we start tracking it.
             if filename.starts_with(&target_deps_dir) {
-                deps.push(filename.to_path_buf());
+                deps.push((filename.to_path_buf(), false));
                 continue;
             }
 
@@ -1105,10 +1136,10 @@ pub fn run_cargo(builder: &Builder,
             let candidate = format!("{}.lib", path_to_add);
             let candidate = PathBuf::from(candidate);
             if candidate.exists() {
-                deps.push(candidate);
+                deps.push((candidate, false));
             }
         }
-        deps.push(path_to_add.into());
+        deps.push((path_to_add.into(), false));
     }
 
     // Now we want to update the contents of the stamp file, if necessary. First
@@ -1121,12 +1152,13 @@ pub fn run_cargo(builder: &Builder,
     let mut new_contents = Vec::new();
     let mut max = None;
     let mut max_path = None;
-    for dep in deps.iter() {
+    for (dep, proc_macro) in deps.iter() {
         let mtime = mtime(dep);
         if Some(mtime) > max {
             max = Some(mtime);
             max_path = Some(dep.clone());
         }
+        new_contents.extend(if *proc_macro { b"h" } else { b"t" });
         new_contents.extend(dep.to_str().unwrap().as_bytes());
         new_contents.extend(b"\0");
     }
@@ -1138,15 +1170,15 @@ pub fn run_cargo(builder: &Builder,
     if contents_equal && max <= stamp_mtime {
         builder.verbose(&format!("not updating {:?}; contents equal and {:?} <= {:?}",
                 stamp, max, stamp_mtime));
-        return deps
+        return deps.into_iter().map(|(d, _)| d).collect()
     }
     if max > stamp_mtime {
         builder.verbose(&format!("updating {:?} as {:?} changed", stamp, max_path));
     } else {
         builder.verbose(&format!("updating {:?} as deps changed", stamp));
     }
     t!(fs::write(&stamp, &new_contents));
-    deps
+    deps.into_iter().map(|(d, _)| d).collect()
 }
 
 pub fn stream_cargo(
@@ -1192,13 +1224,19 @@ pub fn stream_cargo(
     status.success()
 }
 
+#[derive(Deserialize)]
+pub struct CargoTarget<'a> {
+    crate_types: Vec<Cow<'a, str>>,
+}
+
 #[derive(Deserialize)]
 #[serde(tag = "reason", rename_all = "kebab-case")]
 pub enum CargoMessage<'a> {
     CompilerArtifact {
         package_id: Cow<'a, str>,
         features: Vec<Cow<'a, str>>,
         filenames: Vec<Cow<'a, str>>,
+        target: CargoTarget<'a>,
     },
     BuildScriptExecuted {
         package_id: Cow<'a, str>,

src/bootstrap/lib.rs

@@ -104,7 +104,7 @@
 //! also check out the `src/bootstrap/README.md` file for more information.
 
 #![deny(bare_trait_objects)]
-#![deny(warnings)]
+//#![deny(warnings)]
 #![feature(core_intrinsics)]
 #![feature(drain_filter)]
 
@@ -1142,7 +1142,7 @@ impl Build {
         ret
     }
 
-    fn read_stamp_file(&self, stamp: &Path) -> Vec<PathBuf> {
+    fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, bool)> {
         if self.config.dry_run {
             return Vec::new();
         }
@@ -1155,8 +1155,9 @@ impl Build {
             if part.is_empty() {
                 continue
             }
-            let path = PathBuf::from(t!(str::from_utf8(part)));
-            paths.push(path);
+            let host = part[0] as char == 'h';
+            let path = PathBuf::from(t!(str::from_utf8(&part[1..])));
+            paths.push((path, host));
         }
         paths
     }

src/bootstrap/tool.rs

@@ -92,7 +92,8 @@ impl Step for ToolBuild {
                 compile::CargoMessage::CompilerArtifact {
                     package_id,
                     features,
-                    filenames
+                    filenames,
+                    target: _,
                 } => {
                     (package_id, features, filenames)
                 }

src/librustc_cratesio_shim/Cargo.toml

@@ -22,4 +22,5 @@ crate-type = ["dylib"]
 [dependencies]
 bitflags = "1.0"
 log = "0.4"
+serde = { version = "1.0", features = ["serde_derive"] }
 unicode-width = "0.1.4"

src/librustc_cratesio_shim/src/lib.rs

@@ -6,4 +6,5 @@
 extern crate bitflags;
 extern crate log;
 extern crate proc_macro;
+extern crate serde;
 extern crate unicode_width;

src/librustc_errors/Cargo.toml

@@ -14,6 +14,7 @@ serialize = { path = "../libserialize" }
 syntax_pos = { path = "../libsyntax_pos" }
 rustc_data_structures = { path = "../librustc_data_structures" }
 rustc_cratesio_shim = { path = "../librustc_cratesio_shim" }
+serde = { version = "1.0", features = ["serde_derive"] }
 unicode-width = "0.1.4"
 atty = "0.2"
 termcolor = "1.0"