auto merge of #19070 : nikomatsakis/rust/crates, r=nikomatsakis

Reduces memory usage significantly and opens opportunities for more parallel compilation.

This PR was previously #19002 but I closed it because bors didn't seem to recognize the `r+` annotations there.

Author: bors

mk/crates.mk

@@ -53,7 +53,7 @@ TARGET_CRATES := libc std green native flate arena term \
                  serialize sync getopts collections test time rand \
                  log regex graphviz core rbml alloc rustrt \
                  unicode
-HOST_CRATES := syntax rustc rustdoc regex_macros fmt_macros \
+HOST_CRATES := syntax rustc rustc_trans rustdoc regex_macros fmt_macros \
 	       rustc_llvm rustc_back
 CRATES := $(TARGET_CRATES) $(HOST_CRATES)
 TOOLS := compiletest rustdoc rustc
@@ -69,11 +69,12 @@ DEPS_graphviz := std
 DEPS_green := std native:context_switch
 DEPS_native := std
 DEPS_syntax := std term serialize log fmt_macros arena libc
+DEPS_rustc_trans := rustc rustc_back rustc_llvm libc
 DEPS_rustc := syntax flate arena serialize getopts rbml \
               time log graphviz rustc_llvm rustc_back
 DEPS_rustc_llvm := native:rustllvm libc std
 DEPS_rustc_back := std syntax rustc_llvm flate log libc
-DEPS_rustdoc := rustc native:hoedown serialize getopts \
+DEPS_rustdoc := rustc rustc_trans native:hoedown serialize getopts \
                 test time
 DEPS_flate := std native:miniz
 DEPS_arena := std
@@ -96,7 +97,7 @@ DEPS_fmt_macros = std
 
 TOOL_DEPS_compiletest := test getopts native
 TOOL_DEPS_rustdoc := rustdoc native
-TOOL_DEPS_rustc := rustc native
+TOOL_DEPS_rustc := rustc_trans native
 TOOL_SOURCE_compiletest := $(S)src/compiletest/compiletest.rs
 TOOL_SOURCE_rustdoc := $(S)src/driver/driver.rs
 TOOL_SOURCE_rustc := $(S)src/driver/driver.rs
@@ -112,8 +113,8 @@ ONLY_RLIB_unicode := 1
 # You should not need to edit below this line
 ################################################################################
 
-DOC_CRATES := $(filter-out rustc, $(filter-out syntax, $(CRATES)))
-COMPILER_DOC_CRATES := rustc syntax
+DOC_CRATES := $(filter-out rustc, $(filter-out rustc_trans, $(filter-out syntax, $(CRATES))))
+COMPILER_DOC_CRATES := rustc rustc_trans syntax
 
 # This macro creates some simple definitions for each crate being built, just
 # some munging of all of the parameters above.

src/driver/driver.rs

@@ -12,6 +12,6 @@
 extern crate "rustdoc" as this;
 
 #[cfg(rustc)]
-extern crate "rustc" as this;
+extern crate "rustc_trans" as this;
 
 fn main() { this::main() }

src/librustc/README.txt

@@ -13,31 +13,62 @@ https://github.com/rust-lang/rust/issues
 
 Your concerns are probably the same as someone else's.
 
-
-High-level concepts
+The crates of rustc
 ===================
 
-Rustc consists of the following subdirectories:
-
-front/    - front-end: attributes, conditional compilation
-middle/   - middle-end: name resolution, typechecking, LLVM code
+Rustc consists of four crates altogether: `libsyntax`, `librustc`,
+`librustc_back`, and `librustc_trans` (the names and divisions are not
+set in stone and may change; in general, a finer-grained division of
+crates is preferable):
+
+- `libsyntax` contains those things concerned purely with syntax --
+  that is, the AST, parser, pretty-printer, lexer, macro expander, and
+  utilities for traversing ASTs -- are in a separate crate called
+  "syntax", whose files are in ./../libsyntax, where . is the current
+  directory (that is, the parent directory of front/, middle/, back/,
+  and so on).
+
+- `librustc` (the current directory) contains the high-level analysis
+  passes, such as the type checker, borrow checker, and so forth.
+  It is the heart of the compiler.
+
+- `librustc_back` contains some very low-level details that are
+  specific to different LLVM targets and so forth.
+
+- `librustc_trans` contains the code to convert from Rust IR into LLVM
+  IR, and then from LLVM IR into machine code, as well as the main
+  driver that orchestrates all the other passes and various other bits
+  of miscellany. In general it contains code that runs towards the
+  end of the compilation process.
+  
+Roughly speaking the "order" of the three crates is as follows:
+
+    libsyntax -> librustc -> librustc_trans
+    |                                     |
+    +-----------------+-------------------+
+                      |
+             librustc_trans/driver
+
+Here the role of `librustc_trans/driver` is to invoke the compiler
+from libsyntax, then the analysis phases from librustc, and finally
+the lowering and codegen passes from librustc_trans.
+
+Modules in the rustc crate
+==========================
+
+The rustc crate itself consists of the following subdirectories
+(mostly, but not entirely, in their own directories):
+
+session  - options and data that pertain to the compilation session as a whole
+middle   - middle-end: name resolution, typechecking, LLVM code
                   generation
-back/     - back-end: linking and ABI
-metadata/ - encoder and decoder for data required by
+metadata - encoder and decoder for data required by
                     separate compilation
-driver/   - command-line processing, main() entrypoint
-util/     - ubiquitous types and helper functions
-lib/      - bindings to LLVM
-
-The files concerned purely with syntax -- that is, the AST, parser,
-pretty-printer, lexer, macro expander, and utilities for traversing
-ASTs -- are in a separate crate called "syntax", whose files are in
-./../libsyntax, where . is the current directory (that is, the parent
-directory of front/, middle/, back/, and so on).
-
-The entry-point for the compiler is main() in lib.rs, and
-this file sequences the various parts together.
+util     - ubiquitous types and helper functions
+lib      - bindings to LLVM
 
+The entry-point for the compiler is main() in the librustc_trans
+crate. But the 
 
 The 3 central data structures:
 ------------------------------
@@ -66,10 +97,10 @@ The 3 central data structures:
     compilation.  Most variants in the ast::ty tag have a
     corresponding variant in the ty::sty tag.
 
-#3: lib/llvm.rs defines the exported types ValueRef, TypeRef,
-    BasicBlockRef, and several others. Each of these is an opaque
-    pointer to an LLVM type, manipulated through the lib::llvm
-    interface.
+#3: lib/llvm.rs (in librustc_trans) defines the exported types
+    ValueRef, TypeRef, BasicBlockRef, and several others. Each of
+    these is an opaque pointer to an LLVM type, manipulated through
+    the lib::llvm interface.
 
 
 Control and information flow within the compiler:
@@ -87,7 +118,7 @@ Control and information flow within the compiler:
   structures. The driver passes environments to each compiler pass
   that needs to refer to them.
 
-- Finally middle/trans.rs translates the Rust AST to LLVM bitcode in a
-  type-directed way. When it's finished synthesizing LLVM values,
-  rustc asks LLVM to write them out in some form (.bc, .o) and
-  possibly run the system linker.
+- Finally, the `trans` module in `librustc_trans` translates the Rust
+  AST to LLVM bitcode in a type-directed way. When it's finished
+  synthesizing LLVM values, rustc asks LLVM to write them out in some
+  form (.bc, .o) and possibly run the system linker.

src/librustc/lib.rs

@@ -62,11 +62,6 @@ pub mod back {
     pub use rustc_back::target_strs;
     pub use rustc_back::x86;
     pub use rustc_back::x86_64;
-
-    pub mod link;
-    pub mod lto;
-    pub mod write;
-
 }
 
 pub mod middle {
@@ -99,11 +94,9 @@ pub mod middle {
     pub mod region;
     pub mod resolve;
     pub mod resolve_lifetime;
-    pub mod save;
     pub mod stability;
     pub mod subst;
     pub mod traits;
-    pub mod trans;
     pub mod ty;
     pub mod ty_fold;
     pub mod typeck;
@@ -112,7 +105,7 @@ pub mod middle {
 
 pub mod metadata;
 
-pub mod driver;
+pub mod session;
 
 pub mod plugin;
 
@@ -142,9 +135,3 @@ __build_diagnostic_array!(DIAGNOSTICS)
 mod rustc {
     pub use lint;
 }
-
-pub fn main() {
-    let args = std::os::args();
-    let result = driver::run(args);
-    std::os::set_exit_status(result);
-}

src/librustc/lint/context.rs

@@ -30,8 +30,7 @@ use middle::subst;
 use middle::ty;
 use middle::typeck::astconv::AstConv;
 use middle::typeck::infer;
-use driver::session::Session;
-use driver::early_error;
+use session::{early_error, Session};
 use lint::{Level, LevelSource, Lint, LintId, LintArray, LintPass, LintPassObject};
 use lint::{Default, CommandLine, Node, Allow, Warn, Deny, Forbid};
 use lint::builtin;

src/librustc/metadata/creader.rs

@@ -13,8 +13,7 @@
 //! Validates all used crates and extern libraries and loads their metadata
 
 use back::svh::Svh;
-use driver::session::Session;
-use driver::driver;
+use session::{config, Session};
 use metadata::cstore;
 use metadata::cstore::{CStore, CrateSource};
 use metadata::decoder;
@@ -455,7 +454,7 @@ impl<'a> PluginMetadataReader<'a> {
     pub fn read_plugin_metadata(&mut self, krate: &ast::ViewItem) -> PluginMetadata {
         let info = extract_crate_info(&self.env, krate).unwrap();
         let target_triple = self.env.sess.opts.target_triple.as_slice();
-        let is_cross = target_triple != driver::host_triple();
+        let is_cross = target_triple != config::host_triple();
         let mut should_link = info.should_link && !is_cross;
         let mut load_ctxt = loader::Context {
             sess: self.env.sess,
@@ -464,7 +463,7 @@ impl<'a> PluginMetadataReader<'a> {
             crate_name: info.name.as_slice(),
             hash: None,
             filesearch: self.env.sess.host_filesearch(),
-            triple: driver::host_triple(),
+            triple: config::host_triple(),
             root: &None,
             rejected_via_hash: vec!(),
             rejected_via_triple: vec!(),
@@ -481,7 +480,7 @@ impl<'a> PluginMetadataReader<'a> {
                 if decoder::get_plugin_registrar_fn(lib.metadata.as_slice()).is_some() {
                     let message = format!("crate `{}` contains a plugin_registrar fn but \
                                   only a version for triple `{}` could be found (need {})",
-                                  info.ident, target_triple, driver::host_triple());
+                                  info.ident, target_triple, config::host_triple());
                     self.env.sess.span_err(krate.span, message.as_slice());
                     // need to abort now because the syntax expansion
                     // code will shortly attempt to load and execute

src/librustc/metadata/encoder.rs

@@ -16,7 +16,7 @@
 pub use self::InlinedItemRef::*;
 
 use back::svh::Svh;
-use driver::config;
+use session::config;
 use metadata::common::*;
 use metadata::cstore;
 use metadata::decoder;

src/librustc/metadata/loader.rs

@@ -214,7 +214,7 @@
 
 use back::archive::{METADATA_FILENAME};
 use back::svh::Svh;
-use driver::session::Session;
+use session::Session;
 use llvm;
 use llvm::{False, ObjectFile, mk_section_iter};
 use llvm::archive_ro::ArchiveRO;

src/librustc/middle/astencode.rs

@@ -14,7 +14,7 @@
 
 use metadata::common as c;
 use metadata::cstore as cstore;
-use driver::session::Session;
+use session::Session;
 use metadata::decoder;
 use middle::def;
 use metadata::encoder as e;

src/librustc/middle/check_loop.rs

@@ -9,7 +9,7 @@
 // except according to those terms.
 use self::Context::*;
 
-use driver::session::Session;
+use session::Session;
 
 use syntax::ast;
 use syntax::codemap::Span;