Merge pull request #36 from CAD97/gll-bump

Bump GLL

Author: qmx

Cargo.toml

@@ -5,7 +5,7 @@ authors = ["The Rust Project Developers"]
 edition = "2018"
 
 [dependencies]
-gll = { git = "https://github.com/rust-lang-nursery/gll" }
+gll = "0.0.2"
 proc-macro2 = "0.4.0"
 structopt = "0.2.12"
 walkdir = "2.2.6"
@@ -21,7 +21,7 @@ regex = "1.1.6"
 lazy_static = "1.3.0"
 
 [build-dependencies]
-gll = { git = "https://github.com/rust-lang-nursery/gll" }
+gll = "0.0.2"
 walkdir = "2.2.6"
 
 [[bin]]
@@ -31,3 +31,7 @@ path = "src/bin/coverage.rs"
 [[bin]]
 name = "snapshots"
 path = "src/bin/snapshots.rs"
+
+[patch.'crates-io']
+gll = { git = "https://github.com/rust-lang/gll", rev = "bc72820905eac498c1fb1cc644e931d63e7d07e3" }
+grammer = { git = "https://github.com/lykenware/grammer", rev = "eb47b51a9332c0e82d7c02d988e203d2a01f3654" }

build.rs

@@ -7,7 +7,6 @@ use walkdir::WalkDir;
 fn main() {
     let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
 
-
     // FIXME(eddyb) streamline this process in `gll`.
 
     // Find all the `.lyg` grammar fragments in `grammar/`.
@@ -18,23 +17,9 @@ fn main() {
         .filter(|entry| entry.path().extension().map_or(false, |ext| ext == "lyg"));
 
     // Start with the builtin rules for proc-macro grammars.
-    let mut grammar = gll::proc_macro::builtin();
-
-    // HACK(eddyb) inject a custom builtin - this should be upstreamed to gll!
-    {
-        use gll::proc_macro::{FlatTokenPat, Pat};
-
-        grammar.define(
-            "LIFETIME",
-            gll::grammar::eat(Pat(vec![
-                FlatTokenPat::Punct {
-                    ch: Some('\''),
-                    joint: Some(true),
-                },
-                FlatTokenPat::Ident(None),
-            ])),
-        );
-    }
+    let mut cx = gll::proc_macro::Context::new();
+    let cx = &mut cx;
+    let mut grammar = gll::proc_macro::builtin(cx);
 
     // Add in each grammar fragment to the grammar.
     for fragment in fragments {
@@ -44,10 +29,14 @@ fn main() {
         println!("cargo:rerun-if-changed={}", path.display());
 
         let src = fs::read_to_string(&path).unwrap();
-        let fragment: gll::grammar::Grammar<_> = src.parse().unwrap();
+        let fragment = gll::parse_grammar(cx, src.parse().unwrap()).unwrap();
         grammar.extend(fragment);
     }
 
     // Generate a Rust parser from the combined grammar and write it out.
-    fs::write(&out_dir.join("parse.rs"), grammar.generate_rust()).unwrap();
+    fs::write(
+        &out_dir.join("parse.rs"),
+        gll::generate::rust::generate(cx, &grammar).to_rustfmt_or_pretty_string(),
+    )
+    .unwrap();
 }

src/bin/coverage.rs

@@ -2,17 +2,22 @@
 
 use std::{
     collections::{BTreeSet, VecDeque},
-    fs, io, io::prelude::*,
+    fs, io,
+    io::prelude::*,
     path::{Path, PathBuf},
     time::{Duration, Instant},
 };
-use gll::runtime::{MoreThanOne, ParseNodeKind, ParseNodeShape};
+
+use derive_more::Add;
+use gll::{
+    forest::{MoreThanOne, GrammarReflector},
+    parse_node::ParseNodeShape,
+};
 use rayon::prelude::*;
 use rust_grammar::parse;
+use serde::{Deserialize, Serialize};
 use structopt::StructOpt;
 use walkdir::WalkDir;
-use derive_more::Add;
-use serde::{Serialize, Deserialize};
 
 #[derive(Debug, Default, Serialize, Deserialize)]
 struct Blacklist {
@@ -67,36 +72,44 @@ enum Command {
     },
 }
 
-type ModuleContentsResult<'a, 'i> = parse::ParseResult<
-    'a,
-    'i,
-    proc_macro2::TokenStream,
-    parse::ModuleContents<'a, 'i, proc_macro2::TokenStream>,
->;
+type ModuleContentsResult = Result<ModuleContentsHandle, Error<proc_macro2::Span>>;
 
-type ModuleContentsHandle<'a, 'i> = parse::Handle<
-    'a,
-    'i,
+type ModuleContentsHandle = parse::OwnedHandle<
     proc_macro2::TokenStream,
-    parse::ModuleContents<'a, 'i, proc_macro2::TokenStream>,
+    parse::ModuleContents<'static, 'static, proc_macro2::TokenStream>,
 >;
 
+enum Error<A> {
+    Lex(proc_macro2::LexError),
+    Parse(gll::parser::ParseError<A>),
+}
+
+impl<A> From<proc_macro2::LexError> for Error<A> {
+    fn from(error: proc_macro2::LexError) -> Self {
+        Error::Lex(error)
+    }
+}
+
+impl<A> From<gll::parser::ParseError<A>> for Error<A> {
+    fn from(error: gll::parser::ParseError<A>) -> Self {
+        Error::Parse(error)
+    }
+}
+
 /// Read the contents of the file at the given `path`, parse it
 /// using the `ModuleContents` rule, and pass the result to `f`.
-fn parse_file_with<R>(path: &Path, f: impl FnOnce(ModuleContentsResult<'_, '_>) -> R) -> R {
+fn parse_file(path: &Path) -> ModuleContentsResult {
     let src = fs::read_to_string(path).unwrap();
-    match src.parse::<proc_macro2::TokenStream>() {
-        Ok(tts) => parse::ModuleContents::parse_with(tts, |_, result| f(result)),
-        // FIXME(eddyb) provide more information in this error case.
-        Err(_) => f(Err(parse::ParseError::NoParse)),
-    }
+    let tts = src.parse::<proc_macro2::TokenStream>()?;
+    let res = parse::ModuleContents::parse(tts)?;
+    Ok(res)
 }
 
 /// Output the result of a single file to stderr,
 /// optionally prefixed by a given `path`.
 fn report_file_result(
     path: Option<&Path>,
-    result: ModuleContentsResult<'_, '_>,
+    result: &ModuleContentsResult,
     ambiguity_result: Result<(), MoreThanOne>,
     duration: Option<Duration>,
 ) {
@@ -111,60 +124,62 @@ fn report_file_result(
     match (result, ambiguity_result) {
         (Ok(_), Ok(_)) => eprintln!("OK"),
         (Ok(_), Err(_)) => eprintln!("OK (ambiguous)"),
-        (Err(parse::ParseError::TooShort(handle)), _) => {
+        (Err(Error::Parse(error)), _) => {
             eprint!("FAIL after ");
 
             #[cfg(procmacro2_semver_exempt)]
             {
                 // HACK(eddyb) work around `proc-macro2` `Span` printing limitation
-                let end_location = handle.source_info().end.end();
+                let end_location = error.at.end();
                 eprintln!("{}:{}", end_location.line, end_location.column);
             }
             #[cfg(not(procmacro2_semver_exempt))]
             {
-                let _ = handle;
                 eprintln!(
                     "(missing location information; \
                      set `RUSTFLAGS='--cfg procmacro2_semver_exempt'`)"
                 );
             }
+            eprintln!("Expected: {:?}", error.expected);
         }
-        (Err(parse::ParseError::NoParse), _) => eprintln!("FAIL (lexer error?)"),
+        (Err(Error::Lex(e)), _) => eprintln!("FAIL ({:?})", e),
     }
 }
 
-fn ambiguity_check(handle: ModuleContentsHandle<'_, '_>) -> Result<(), MoreThanOne> {
-    let sppf = &handle.parser.sppf;
+fn ambiguity_check(handle: &ModuleContentsHandle) -> Result<(), MoreThanOne> {
+    handle.with(|handle| {
+        let forest = &handle.forest;
 
-    let mut queue = VecDeque::new();
-    queue.push_back(handle.node);
-    let mut seen: BTreeSet<_> = queue.iter().cloned().collect();
+        let mut queue = VecDeque::new();
+        queue.push_back(handle.node);
+        let mut seen: BTreeSet<_> = queue.iter().cloned().collect();
 
-    while let Some(source) = queue.pop_front() {
-        let mut add_children = |children: &[_]| {
-            for &child in children {
-                if seen.insert(child) {
-                    queue.push_back(child);
+        while let Some(source) = queue.pop_front() {
+            let mut add_children = |children: &[_]| {
+                for &child in children {
+                    if seen.insert(child) {
+                        queue.push_back(child);
+                    }
                 }
-            }
-        };
-        match source.kind.shape() {
-            ParseNodeShape::Opaque => {}
-            ParseNodeShape::Alias(_) => add_children(&[source.unpack_alias()]),
-            ParseNodeShape::Opt(_) => {
-                if let Some(child) = source.unpack_opt() {
-                    add_children(&[child]);
+            };
+            match forest.grammar.parse_node_shape(source.kind) {
+                ParseNodeShape::Opaque => {}
+                ParseNodeShape::Alias(_) => add_children(&[forest.unpack_alias(source)]),
+                ParseNodeShape::Opt(_) => {
+                    if let Some(child) = forest.unpack_opt(source) {
+                        add_children(&[child]);
+                    }
+                }
+                ParseNodeShape::Choice => add_children(&[forest.one_choice(source)?]),
+                ParseNodeShape::Split(..) => {
+                    let (left, right) = forest.one_split(source)?;
+                    add_children(&[left, right])
                 }
-            }
-            ParseNodeShape::Choice => add_children(&[sppf.one_choice(source)?]),
-            ParseNodeShape::Split(..) => {
-                let (left, right) = sppf.one_split(source)?;
-                add_children(&[left, right])
             }
         }
-    }
 
-    Ok(())
+        Ok(())
+    })
 }
 
 #[derive(Debug, Default, Add)]
@@ -199,30 +214,29 @@ fn process(file: walkdir::DirEntry, verbose: bool) -> ParseResult {
     let mut stdout = io::stdout();
     let path = file.into_path();
 
-    parse_file_with(&path, |result| {
-        let mut ambiguity_result = Ok(());
-        let start = Instant::now();
-        let status = match result {
-            Ok(handle) => {
-                ambiguity_result = ambiguity_check(handle);
-                if ambiguity_result.is_ok() {
-                    ParseResult::Unambiguous
-                } else {
-                    ParseResult::Ambiguous
-                }
+    let result = parse_file(&path);
+    let mut ambiguity_result = Ok(());
+    let start = Instant::now();
+    let status = match &result {
+        Ok(handle) => {
+            ambiguity_result = ambiguity_check(handle);
+            if ambiguity_result.is_ok() {
+                ParseResult::Unambiguous
+            } else {
+                ParseResult::Ambiguous
             }
-            Err(parse::ParseError::TooShort(_)) => ParseResult::Partial,
-            Err(parse::ParseError::NoParse) => ParseResult::Error,
-        };
-        let duration = start.elapsed();
-        if verbose {
-            report_file_result(Some(&path), result, ambiguity_result, Some(duration));
-        } else {
-            print!("{}", status.compact_display());
-            stdout.flush().unwrap();
         }
-        status
-    })
+        Err(Error::Parse(_)) => ParseResult::Partial,
+        Err(Error::Lex(_)) => ParseResult::Error,
+    };
+    let duration = start.elapsed();
+    if verbose {
+        report_file_result(Some(&path), &result, ambiguity_result, Some(duration));
+    } else {
+        print!("{}", status.compact_display());
+        stdout.flush().unwrap();
+    }
+    status
 }
 
 fn print_statistics(counters: Counters) {
@@ -253,24 +267,21 @@ fn main() -> Result<(), failure::Error> {
             file,
         } => {
             // Not much to do, try to parse the file and report the result.
-            parse_file_with(&file, |result| {
-                let mut ambiguity_result = Ok(());
-                match result {
-                    Ok(handle) | Err(parse::ParseError::TooShort(handle)) => {
-                        ambiguity_result = ambiguity_check(handle);
-
-                        if let Some(out_path) = graphviz_forest {
-                            handle
-                                .parser
-                                .sppf
-                                .dump_graphviz(&mut fs::File::create(out_path).unwrap())
-                                .unwrap();
-                        }
-                    }
-                    Err(parse::ParseError::NoParse) => {}
+            let result = parse_file(&file);
+            let mut ambiguity_result = Ok(());
+            if let Ok(handle) = &result {
+                ambiguity_result = ambiguity_check(handle);
+
+                if let Some(out_path) = graphviz_forest {
+                    handle.with(|handle| {
+                        handle
+                            .forest
+                            .dump_graphviz(&mut fs::File::create(out_path).unwrap())
+                            .unwrap();
+                    })
                 }
-                report_file_result(None, result, ambiguity_result, None);
-            });
+            }
+            report_file_result(None, &result, ambiguity_result, None);
         }
         Command::Dir { verbose, dir } => {
             let config = Config::load()?;
@@ -288,27 +299,24 @@ fn main() -> Result<(), failure::Error> {
             let counters: Counters = files
                 .par_bridge()
                 .map(|f| process(f, verbose))
-                .fold(
-                    Counters::default,
-                    |mut acc, x| {
-                        acc.total_count += 1;
-                        match x {
-                            ParseResult::Ambiguous => {
-                                acc.ambiguous_count += 1;
-                            }
-                            ParseResult::Unambiguous => {
-                                acc.unambiguous_count += 1;
-                            }
-                            ParseResult::Partial => {
-                                acc.too_short_count += 1;
-                            }
-                            ParseResult::Error => {
-                                acc.no_parse_count += 1;
-                            }
-                        };
-                        acc
-                    },
-                )
+                .fold(Counters::default, |mut acc, x| {
+                    acc.total_count += 1;
+                    match x {
+                        ParseResult::Ambiguous => {
+                            acc.ambiguous_count += 1;
+                        }
+                        ParseResult::Unambiguous => {
+                            acc.unambiguous_count += 1;
+                        }
+                        ParseResult::Partial => {
+                            acc.too_short_count += 1;
+                        }
+                        ParseResult::Error => {
+                            acc.no_parse_count += 1;
+                        }
+                    };
+                    acc
+                })
                 .reduce(Counters::default, |a, b| a + b);
 
             // We're done, time to print out stats!