Add support for parsing with rust-analyzer instead of librustc_parse

This adds a new librustc_parse_ra library that implements the same
interface of librustc_parse, but uses rust-analyzer to parse the code,
and then converts the rust-analyzer AST to the rustc AST, and adds a
new -Z parse-with-rust-analyzer option that makes rustc use it, gated
by a new rust_analyzer feature flag and config.toml option.

The code is preliminary but should be essentially complete, and capable
of building large projects that only use the Rust 2018 edition (Rust 2015
is not currently supported by rust-analyzer).

Note that this is likely to accept invalid code, is not fully complete,
and provides worse error messages than the current parser, so it cannot
be stabilized without significant additional work.

Author: luca-barbieri

Cargo.lock

@@ -740,7 +740,7 @@ dependencies = [
  "crossbeam-utils 0.6.5",
  "lazy_static 1.4.0",
  "memoffset",
- "scopeguard",
+ "scopeguard 1.0.0",
 ]
 
 [[package]]
@@ -952,6 +952,12 @@ dependencies = [
  "rustc-std-workspace-core",
 ]
 
+[[package]]
+name = "drop_bomb"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69b26e475fd29098530e709294e94e661974c851aed42512793f120fed4e199f"
+
 [[package]]
 name = "dtoa"
 version = "0.4.4"
@@ -1404,7 +1410,7 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3753954f7bd71f0e671afb8b5a992d1724cf43b7f95a563cd4a0bde94659ca8"
 dependencies = [
- "scopeguard",
+ "scopeguard 1.0.0",
  "winapi 0.3.8",
 ]
 
@@ -1864,13 +1870,23 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
 
+[[package]]
+name = "lock_api"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c"
+dependencies = [
+ "owning_ref",
+ "scopeguard 0.3.3",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8912e782533a93a167888781b836336a6ca5da6175c05944c86cf28c31104dc"
 dependencies = [
- "scopeguard",
+ "scopeguard 1.0.0",
 ]
 
 [[package]]
@@ -2345,6 +2361,15 @@ dependencies = [
  "winapi 0.3.8",
 ]
 
+[[package]]
+name = "owning_ref"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce"
+dependencies = [
+ "stable_deref_trait",
+]
+
 [[package]]
 name = "packed_simd"
 version = "0.3.1"
@@ -2393,13 +2418,33 @@ dependencies = [
  "winapi 0.3.8",
 ]
 
+[[package]]
+name = "parking_lot"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0802bff09003b291ba756dc7e79313e51cc31667e94afbe847def490424cde5"
+dependencies = [
+ "lock_api 0.1.5",
+ "parking_lot_core 0.3.1",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337"
+dependencies = [
+ "lock_api 0.1.5",
+ "parking_lot_core 0.4.0",
+]
+
 [[package]]
 name = "parking_lot"
 version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252"
 dependencies = [
- "lock_api",
+ "lock_api 0.3.1",
  "parking_lot_core 0.6.2",
  "rustc_version",
 ]
@@ -2410,10 +2455,36 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc"
 dependencies = [
- "lock_api",
+ "lock_api 0.3.1",
  "parking_lot_core 0.7.0",
 ]
 
+[[package]]
+name = "parking_lot_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad7f7e6ebdc79edff6fdcb87a55b620174f7a989e3eb31b65231f4af57f00b8c"
+dependencies = [
+ "libc",
+ "rand 0.5.6",
+ "rustc_version",
+ "smallvec 0.6.10",
+ "winapi 0.3.8",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9"
+dependencies = [
+ "libc",
+ "rand 0.6.1",
+ "rustc_version",
+ "smallvec 0.6.10",
+ "winapi 0.3.8",
+]
+
 [[package]]
 name = "parking_lot_core"
 version = "0.6.2"
@@ -2713,6 +2784,20 @@ dependencies = [
  "proc-macro2 1.0.3",
 ]
 
+[[package]]
+name = "ra_syntax"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be308e1907623f15bc874a4b2742b6a20c44c439f201638fa26564e664276ce9"
+dependencies = [
+ "drop_bomb",
+ "itertools 0.7.8",
+ "parking_lot 0.6.4",
+ "rowan",
+ "text_unit",
+ "unicode-xid 0.1.0",
+]
+
 [[package]]
 name = "racer"
 version = "2.1.31"
@@ -2736,6 +2821,19 @@ dependencies = [
  "rustc-ap-syntax",
 ]
 
+[[package]]
+name = "rand"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9"
+dependencies = [
+ "cloudabi",
+ "fuchsia-cprng",
+ "libc",
+ "rand_core 0.3.0",
+ "winapi 0.3.8",
+]
+
 [[package]]
 name = "rand"
 version = "0.6.1"
@@ -3141,6 +3239,17 @@ dependencies = [
  "rls-span",
 ]
 
+[[package]]
+name = "rowan"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c218b4430ab922850b71b14fa9bca224425097f935f6155c0b6a4b1f398a54f0"
+dependencies = [
+ "parking_lot 0.7.1",
+ "smol_str",
+ "text_unit",
+]
+
 [[package]]
 name = "rustbook"
 version = "0.1.0"
@@ -3849,6 +3958,7 @@ dependencies = [
  "rustc_mir",
  "rustc_mir_build",
  "rustc_parse",
+ "rustc_parse_ra",
  "rustc_passes",
  "rustc_plugin_impl",
  "rustc_privacy",
@@ -4045,6 +4155,24 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "rustc_parse_ra"
+version = "0.0.0"
+dependencies = [
+ "bitflags",
+ "log",
+ "ra_syntax",
+ "rustc_ast",
+ "rustc_ast_pretty",
+ "rustc_data_structures",
+ "rustc_errors",
+ "rustc_parse",
+ "rustc_session",
+ "rustc_span",
+ "smallvec 1.0.0",
+ "unicode-normalization",
+]
+
 [[package]]
 name = "rustc_passes"
 version = "0.0.0"
@@ -4443,6 +4571,12 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8"
 
+[[package]]
+name = "scopeguard"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
+
 [[package]]
 name = "scopeguard"
 version = "1.0.0"
@@ -4625,6 +4759,12 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4ecf3b85f68e8abaa7555aa5abdb1153079387e60b718283d732f03897fcfc86"
 
+[[package]]
+name = "smol_str"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34836c9a295c62c2ce3514471117c5cb269891e8421b2aafdd910050576c4d8b"
+
 [[package]]
 name = "socket2"
 version = "0.3.11"
@@ -4921,6 +5061,12 @@ dependencies = [
  "term 0.6.0",
 ]
 
+[[package]]
+name = "text_unit"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20431e104bfecc1a40872578dbc390e10290a0e9c35fffe3ce6f73c15a9dbfc2"
+
 [[package]]
 name = "textwrap"
 version = "0.11.0"

config.toml.example

@@ -436,6 +436,9 @@
 # override the default allocator for rustc and LLVM.
 #jemalloc = false
 
+# Enable experimental features that depend on rust-analyzer.
+#rust-analyzer = false
+
 # Run tests in various test suites with the "nll compare mode" in addition to
 # running the tests in normal mode. Largely only used on CI and during local
 # development of NLL

src/bootstrap/config.rs

@@ -117,6 +117,7 @@ pub struct Config {
     pub targets: Vec<Interned<String>>,
     pub local_rebuild: bool,
     pub jemalloc: bool,
+    pub rust_analyzer: bool,
     pub control_flow_guard: bool,
 
     // dist misc
@@ -342,6 +343,7 @@ struct Rust {
     thin_lto_import_instr_limit: Option<u32>,
     remap_debuginfo: Option<bool>,
     jemalloc: Option<bool>,
+    rust_analyzer: Option<bool>,
     test_compare_mode: Option<bool>,
     llvm_libunwind: Option<bool>,
     control_flow_guard: Option<bool>,
@@ -571,6 +573,7 @@ impl Config {
             set(&mut config.codegen_tests, rust.codegen_tests);
             set(&mut config.rust_rpath, rust.rpath);
             set(&mut config.jemalloc, rust.jemalloc);
+            set(&mut config.rust_analyzer, rust.rust_analyzer);
             set(&mut config.test_compare_mode, rust.test_compare_mode);
             set(&mut config.llvm_libunwind, rust.llvm_libunwind);
             set(&mut config.backtrace, rust.backtrace);

src/bootstrap/lib.rs

@@ -509,6 +509,9 @@ impl Build {
         if self.config.jemalloc {
             features.push_str("jemalloc");
         }
+        if self.config.rust_analyzer {
+            features.push_str("rust_analyzer");
+        }
         if self.config.llvm_enabled() {
             features.push_str(" llvm");
         }

src/librustc_driver/Cargo.toml

@@ -40,3 +40,4 @@ winapi = { version = "0.3", features = ["consoleapi", "debugapi", "processenv"]
 
 [features]
 llvm = ['rustc_interface/llvm']
+rust_analyzer = ['rustc_interface/rust_analyzer']

src/librustc_interface/Cargo.toml

@@ -18,6 +18,7 @@ rustc_attr = { path = "../librustc_attr" }
 rustc_builtin_macros = { path = "../librustc_builtin_macros" }
 rustc_expand = { path = "../librustc_expand" }
 rustc_parse = { path = "../librustc_parse" }
+rustc_parse_ra = { path = "../librustc_parse_ra", optional = true }
 rustc_session = { path = "../librustc_session" }
 rustc_span = { path = "../librustc_span" }
 rustc_serialize = { path = "../libserialize", package = "serialize" }
@@ -55,3 +56,4 @@ rustc_target = { path = "../librustc_target" }
 
 [features]
 llvm = ['rustc_codegen_llvm']
+rust_analyzer = ['rustc_parse_ra']

src/librustc_interface/passes.rs

@@ -50,13 +50,40 @@ use std::path::PathBuf;
 use std::rc::Rc;
 use std::{env, fs, iter, mem};
 
-pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> {
-    let krate = sess.time("parse_crate", || match input {
-        Input::File(file) => parse_crate_from_file(file, &sess.parse_sess),
+#[cfg(rust_analyzer)]
+fn parse_with_ra<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate>
+{
+    match input {
+        Input::File(file) => {
+            rustc_parse_ra::parse_crate_from_file(file, &sess.parse_sess)
+        },
         Input::Str { input, name } => {
-            parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess)
+            rustc_parse_ra::parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess)
         }
-    })?;
+    }
+}
+
+#[cfg(not(rust_analyzer))]
+fn parse_with_ra<'a>(sess: &'a Session, _input: &Input) -> PResult<'a, ast::Crate>
+{
+    Err(sess.struct_fatal("parsing with rust-analyzer is not supported because this version of rustc was not compiled with the \"rust_analyzer\" feature flag enabled"))
+}
+
+pub fn parse<'a>(sess: &'a Session, input: &Input) -> PResult<'a, ast::Crate> {
+    let krate = sess.time("parse_crate", ||
+        if sess.opts.debugging_opts.parse_with_rust_analyzer {
+            parse_with_ra(sess, input)
+        } else {
+            match input {
+                Input::File(file) => {
+                    parse_crate_from_file(file, &sess.parse_sess)
+                },
+                Input::Str { input, name } => {
+                    parse_crate_from_source_str(name.clone(), input.clone(), &sess.parse_sess)
+                }
+            }
+        }
+    )?;
 
     if sess.opts.debugging_opts.ast_json_noexpand {
         println!("{}", json::as_json(&krate));

src/librustc_parse/lexer/mod.rs

@@ -83,6 +83,7 @@ impl<'a> StringReader<'a> {
         let mut sr = StringReader::new(sess, begin.sf, None);
 
         // Seek the lexer to the right byte range.
+        sr.pos = span.lo();
         sr.end_src_index = sr.src_index(span.hi());
 
         sr