add and document --incremental flag along with misc other changes

For example:

- we now support `-vv` to get very verbose output.
- RUSTFLAGS is respected by `x.py`
- better error messages for some cases

Author: nikomatsakis

src/bootstrap/README.md

@@ -116,6 +116,42 @@ compiler. What actually happens when you invoke rustbuild is:
 The goal of each stage is to (a) leverage Cargo as much as possible and failing
 that (b) leverage Rust as much as possible!
 
+## Incremental builds
+
+You can configure rustbuild to use incremental compilation. Because
+incremental is new and evolving rapidly, if you want to use it, it is
+recommended that you replace the snapshot with a locally installed
+nightly build of rustc. You will want to keep this up to date.
+
+To follow this course of action, first thing you will want to do is to
+install a nightly, presumably using `rustup`. You will then want to
+configure your directory to use this build, like so:
+
+```
+# configure to use local rust instead of downloding a beta.
+# `--local-rust-root` is optional here. If elided, we will
+# use whatever rustc we find on your PATH.
+> configure --enable-rustbuild --local-rust-root=~/.cargo/ --enable-local-rebuild
+```
+
+After that, you can use the `--incremental` flag to actually do
+incremental builds:
+
+```
+> ../x.py build --incremental
+```
+
+The `--incremental` flag will store incremental compilation artifacts
+in `build/stage0-incremental`. Note that we only use incremental
+compilation for the stage0 -> stage1 compilation -- this is because
+the stage1 compiler is changing, and we don't try to cache and reuse
+incremental artifacts across different versions of the compiler. For
+this reason, `--incremental` defaults to `--stage 1` (though you can
+manually select a higher stage, if you prefer).
+
+You can always drop the `--incremental` to build as normal (but you
+will still be using the local nightly as your bootstrap).
+
 ## Directory Layout
 
 This build system houses all output under the `build` directory, which looks

src/bootstrap/bin/rustc.rs

@@ -29,6 +29,9 @@ extern crate bootstrap;
 
 use std::env;
 use std::ffi::OsString;
+use std::io;
+use std::io::prelude::*;
+use std::str::FromStr;
 use std::path::PathBuf;
 use std::process::Command;
 
@@ -41,6 +44,11 @@ fn main() {
         .and_then(|w| w[1].to_str());
     let version = args.iter().find(|w| &**w == "-vV");
 
+    let verbose = match env::var("RUSTC_VERBOSE") {
+        Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"),
+        Err(_) => 0,
+    };
+
     // Build scripts always use the snapshot compiler which is guaranteed to be
     // able to produce an executable, whereas intermediate compilers may not
     // have the standard library built yet and may not be able to produce an
@@ -95,6 +103,15 @@ fn main() {
             cmd.args(&s.split(" ").filter(|s| !s.is_empty()).collect::<Vec<_>>());
         }
 
+        // Pass down incremental directory, if any.
+        if let Ok(dir) = env::var("RUSTC_INCREMENTAL") {
+            cmd.arg(format!("-Zincremental={}", dir));
+
+            if verbose > 0 {
+                cmd.arg("-Zincremental-info");
+            }
+        }
+
         // If we're compiling specifically the `panic_abort` crate then we pass
         // the `-C panic=abort` option. Note that we do not do this for any
         // other crate intentionally as this is the only crate for now that we
@@ -176,9 +193,19 @@ fn main() {
             if let Some(rpath) = rpath {
                 cmd.arg("-C").arg(format!("link-args={}", rpath));
             }
+
+            if let Ok(s) = env::var("RUSTFLAGS") {
+                for flag in s.split_whitespace() {
+                    cmd.arg(flag);
+                }
+            }
         }
     }
 
+    if verbose > 1 {
+        writeln!(&mut io::stderr(), "rustc command: {:?}", cmd).unwrap();
+    }
+
     // Actually run the compiler!
     std::process::exit(match cmd.status() {
         Ok(s) => s.code().unwrap_or(1),

src/bootstrap/bootstrap.py

@@ -294,6 +294,8 @@ def build_bootstrap(self):
         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
                       os.pathsep + env["PATH"]
+        if not os.path.isfile(self.cargo()):
+            raise Exception("no cargo executable found at `%s`" % self.cargo())
         args = [self.cargo(), "build", "--manifest-path",
                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
         if self.use_vendored_sources:

src/bootstrap/check.rs

@@ -190,7 +190,7 @@ pub fn compiletest(build: &Build,
 
     cmd.args(&build.flags.cmd.test_args());
 
-    if build.config.verbose || build.flags.verbose {
+    if build.config.verbose() || build.flags.verbose() {
         cmd.arg("--verbose");
     }
 

src/bootstrap/config.rs

@@ -40,7 +40,7 @@ use util::push_exe_path;
 pub struct Config {
     pub ccache: Option<String>,
     pub ninja: bool,
-    pub verbose: bool,
+    pub verbose: usize,
     pub submodules: bool,
     pub compiler_docs: bool,
     pub docs: bool,
@@ -504,6 +504,14 @@ impl Config {
             }
         }
     }
+
+    pub fn verbose(&self) -> bool {
+        self.verbose > 0
+    }
+
+    pub fn very_verbose(&self) -> bool {
+        self.verbose > 1
+    }
 }
 
 #[cfg(not(windows))]

src/bootstrap/flags.rs

@@ -27,7 +27,7 @@ use step;
 
 /// Deserialized version of all flags for this compile.
 pub struct Flags {
-    pub verbose: bool,
+    pub verbose: usize, // verbosity level: 0 == not verbose, 1 == verbose, 2 == very verbose
     pub stage: Option<u32>,
     pub keep_stage: Option<u32>,
     pub build: String,
@@ -37,6 +37,17 @@ pub struct Flags {
     pub src: Option<PathBuf>,
     pub jobs: Option<u32>,
     pub cmd: Subcommand,
+    pub incremental: bool,
+}
+
+impl Flags {
+    pub fn verbose(&self) -> bool {
+        self.verbose > 0
+    }
+
+    pub fn very_verbose(&self) -> bool {
+        self.verbose > 1
+    }
 }
 
 pub enum Subcommand {
@@ -63,7 +74,8 @@ pub enum Subcommand {
 impl Flags {
     pub fn parse(args: &[String]) -> Flags {
         let mut opts = Options::new();
-        opts.optflag("v", "verbose", "use verbose output");
+        opts.optflagmulti("v", "verbose", "use verbose output (-vv for very verbose)");
+        opts.optflag("i", "incremental", "use incremental compilation");
         opts.optopt("", "config", "TOML configuration file for build", "FILE");
         opts.optopt("", "build", "build target of the stage0 compiler", "BUILD");
         opts.optmulti("", "host", "host targets to build", "HOST");
@@ -256,8 +268,18 @@ To learn more about a subcommand, run `./x.py <command> -h`
             }
         });
 
+        let mut stage = m.opt_str("stage").map(|j| j.parse().unwrap());
+
+        let incremental = m.opt_present("i");
+
+        if incremental {
+            if stage.is_none() {
+                stage = Some(1);
+            }
+        }
+
         Flags {
-            verbose: m.opt_present("v"),
+            verbose: m.opt_count("v"),
             stage: m.opt_str("stage").map(|j| j.parse().unwrap()),
             keep_stage: m.opt_str("keep-stage").map(|j| j.parse().unwrap()),
             build: m.opt_str("build").unwrap_or_else(|| {
@@ -269,6 +291,7 @@ To learn more about a subcommand, run `./x.py <command> -h`
             src: m.opt_str("src").map(PathBuf::from),
             jobs: m.opt_str("jobs").map(|j| j.parse().unwrap()),
             cmd: cmd,
+            incremental: incremental,
         }
     }
 }

src/bootstrap/lib.rs

@@ -74,6 +74,7 @@ extern crate rustc_serialize;
 extern crate toml;
 
 use std::collections::HashMap;
+use std::cmp;
 use std::env;
 use std::ffi::OsString;
 use std::fs::{self, File};
@@ -497,6 +498,17 @@ impl Build {
         cargo.env("RUSTC_BOOTSTRAP", "1");
         self.add_rust_test_threads(&mut cargo);
 
+        // Ignore incremental modes except for stage0, since we're
+        // not guaranteeing correctness acros builds if the compiler
+        // is changing under your feet.`
+        if self.flags.incremental && compiler.stage == 0 {
+            let incr_dir = self.incremental_dir(compiler);
+            cargo.env("RUSTC_INCREMENTAL", incr_dir);
+        }
+
+        let verbose = cmp::max(self.config.verbose, self.flags.verbose);
+        cargo.env("RUSTC_VERBOSE", format!("{}", verbose));
+
         // Specify some various options for build scripts used throughout
         // the build.
         //
@@ -516,7 +528,7 @@ impl Build {
         // FIXME: should update code to not require this env var
         cargo.env("CFG_COMPILER_HOST_TRIPLE", target);
 
-        if self.config.verbose || self.flags.verbose {
+        if self.config.verbose() || self.flags.verbose() {
             cargo.arg("-v");
         }
         // FIXME: cargo bench does not accept `--release`
@@ -630,6 +642,12 @@ impl Build {
         }
     }
 
+    /// Get the directory for incremental by-products when using the
+    /// given compiler.
+    fn incremental_dir(&self, compiler: &Compiler) -> PathBuf {
+        self.out.join(compiler.host).join(format!("stage{}-incremental", compiler.stage))
+    }
+
     /// Returns the libdir where the standard library and other artifacts are
     /// found for a compiler's sysroot.
     fn sysroot_libdir(&self, compiler: &Compiler, target: &str) -> PathBuf {
@@ -768,7 +786,7 @@ impl Build {
 
     /// Prints a message if this build is configured in verbose mode.
     fn verbose(&self, msg: &str) {
-        if self.flags.verbose || self.config.verbose {
+        if self.flags.verbose() || self.config.verbose() {
             println!("{}", msg);
         }
     }