Skip to content

Commit 2595d75

Browse files
matkladmatklad
committed
Introduce NonterminalKind
It encapsulate the (part of) the interface between the parser and macro by example (macro_rules) parser. The second bit is somewhat more general `parse_ast_fragment`, which is the reason why we keep some `parse_xxx` functions as public.
1 parent dfe1e3b commit 2595d75

File tree

9 files changed

+231
-211
lines changed

9 files changed

+231
-211
lines changed

‎src/librustc_ast/token.rs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,44 @@ pub enum Nonterminal {
760760
#[cfg(target_arch = "x86_64")]
761761
rustc_data_structures::static_assert_size!(Nonterminal, 40);
762762

763+
#[derive(Copy, Clone)]
764+
pub enum NonterminalKind {
765+
Item,
766+
Block,
767+
Stmt,
768+
Pat,
769+
Expr,
770+
Ty,
771+
Ident,
772+
Lifetime,
773+
Literal,
774+
Meta,
775+
Path,
776+
Vis,
777+
TT,
778+
}
779+
780+
impl NonterminalKind {
781+
pub fn from_symbol(symbol: Symbol) -> Option<NonterminalKind> {
782+
Some(match symbol {
783+
sym::item => NonterminalKind::Item,
784+
sym::block => NonterminalKind::Block,
785+
sym::stmt => NonterminalKind::Stmt,
786+
sym::pat => NonterminalKind::Pat,
787+
sym::expr => NonterminalKind::Expr,
788+
sym::ty => NonterminalKind::Ty,
789+
sym::ident => NonterminalKind::Ident,
790+
sym::lifetime => NonterminalKind::Lifetime,
791+
sym::literal => NonterminalKind::Literal,
792+
sym::meta => NonterminalKind::Meta,
793+
sym::path => NonterminalKind::Path,
794+
sym::vis => NonterminalKind::Vis,
795+
sym::tt => NonterminalKind::TT,
796+
_ => return None,
797+
})
798+
}
799+
}
800+
763801
impl Nonterminal {
764802
fn span(&self) -> Span {
765803
match self {

‎src/librustc_expand/mbe/macro_parser.rs‎

Lines changed: 18 additions & 185 deletions
Original file line numberDiff line numberDiff line change
@@ -76,15 +76,11 @@ use TokenTreeOrTokenTreeSlice::*;
7676

7777
use crate::mbe::{self, TokenTree};
7878

79-
use rustc_ast::ptr::P;
80-
use rustc_ast::token::{self, DocComment, Nonterminal, Token};
81-
use rustc_ast_pretty::pprust;
82-
use rustc_parse::parser::{FollowedByType, Parser, PathStyle};
79+
use rustc_ast::token::{self, DocComment, Nonterminal, NonterminalKind, Token};
80+
use rustc_parse::parser::Parser;
8381
use rustc_session::parse::ParseSess;
84-
use rustc_span::symbol::{kw, sym, Ident, MacroRulesNormalizedIdent, Symbol};
82+
use rustc_span::symbol::{kw, MacroRulesNormalizedIdent};
8583

86-
use rustc_errors::PResult;
87-
use rustc_span::Span;
8884
use smallvec::{smallvec, SmallVec};
8985

9086
use rustc_data_structures::fx::FxHashMap;
@@ -576,7 +572,8 @@ fn inner_parse_loop<'root, 'tt>(
576572
TokenTree::MetaVarDecl(_, _, id) => {
577573
// Built-in nonterminals never start with these tokens,
578574
// so we can eliminate them from consideration.
579-
if may_begin_with(token, id.name) {
575+
let kind = NonterminalKind::from_symbol(id.name).unwrap();
576+
if Parser::nonterminal_may_begin_with(kind, token) {
580577
bb_items.push(item);
581578
}
582579
}
@@ -738,8 +735,19 @@ pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> Na
738735
let mut item = bb_items.pop().unwrap();
739736
if let TokenTree::MetaVarDecl(span, _, ident) = item.top_elts.get_tt(item.idx) {
740737
let match_cur = item.match_cur;
741-
let nt = match parse_nt(parser.to_mut(), span, ident.name) {
742-
Err(()) => return ErrorReported,
738+
let kind = NonterminalKind::from_symbol(ident.name).unwrap();
739+
let nt = match parser.to_mut().parse_nonterminal(kind) {
740+
Err(mut err) => {
741+
err.span_label(
742+
span,
743+
format!(
744+
"while parsing argument for this `{}` macro fragment",
745+
ident.name
746+
),
747+
)
748+
.emit();
749+
return ErrorReported;
750+
}
743751
Ok(nt) => nt,
744752
};
745753
item.push_match(match_cur, MatchedNonterminal(Lrc::new(nt)));
@@ -754,178 +762,3 @@ pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> Na
754762
assert!(!cur_items.is_empty());
755763
}
756764
}
757-
758-
/// The token is an identifier, but not `_`.
759-
/// We prohibit passing `_` to macros expecting `ident` for now.
760-
fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
761-
token.ident().filter(|(ident, _)| ident.name != kw::Underscore)
762-
}
763-
764-
/// Checks whether a non-terminal may begin with a particular token.
765-
///
766-
/// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that
767-
/// token. Be conservative (return true) if not sure.
768-
fn may_begin_with(token: &Token, name: Symbol) -> bool {
769-
/// Checks whether the non-terminal may contain a single (non-keyword) identifier.
770-
fn may_be_ident(nt: &token::Nonterminal) -> bool {
771-
match *nt {
772-
token::NtItem(_) | token::NtBlock(_) | token::NtVis(_) | token::NtLifetime(_) => false,
773-
_ => true,
774-
}
775-
}
776-
777-
match name {
778-
sym::expr => {
779-
token.can_begin_expr()
780-
// This exception is here for backwards compatibility.
781-
&& !token.is_keyword(kw::Let)
782-
}
783-
sym::ty => token.can_begin_type(),
784-
sym::ident => get_macro_ident(token).is_some(),
785-
sym::literal => token.can_begin_literal_maybe_minus(),
786-
sym::vis => match token.kind {
787-
// The follow-set of :vis + "priv" keyword + interpolated
788-
token::Comma | token::Ident(..) | token::Interpolated(..) => true,
789-
_ => token.can_begin_type(),
790-
},
791-
sym::block => match token.kind {
792-
token::OpenDelim(token::Brace) => true,
793-
token::Interpolated(ref nt) => match **nt {
794-
token::NtItem(_)
795-
| token::NtPat(_)
796-
| token::NtTy(_)
797-
| token::NtIdent(..)
798-
| token::NtMeta(_)
799-
| token::NtPath(_)
800-
| token::NtVis(_) => false, // none of these may start with '{'.
801-
_ => true,
802-
},
803-
_ => false,
804-
},
805-
sym::path | sym::meta => match token.kind {
806-
token::ModSep | token::Ident(..) => true,
807-
token::Interpolated(ref nt) => match **nt {
808-
token::NtPath(_) | token::NtMeta(_) => true,
809-
_ => may_be_ident(&nt),
810-
},
811-
_ => false,
812-
},
813-
sym::pat => match token.kind {
814-
token::Ident(..) | // box, ref, mut, and other identifiers (can stricten)
815-
token::OpenDelim(token::Paren) | // tuple pattern
816-
token::OpenDelim(token::Bracket) | // slice pattern
817-
token::BinOp(token::And) | // reference
818-
token::BinOp(token::Minus) | // negative literal
819-
token::AndAnd | // double reference
820-
token::Literal(..) | // literal
821-
token::DotDot | // range pattern (future compat)
822-
token::DotDotDot | // range pattern (future compat)
823-
token::ModSep | // path
824-
token::Lt | // path (UFCS constant)
825-
token::BinOp(token::Shl) => true, // path (double UFCS)
826-
token::Interpolated(ref nt) => may_be_ident(nt),
827-
_ => false,
828-
},
829-
sym::lifetime => match token.kind {
830-
token::Lifetime(_) => true,
831-
token::Interpolated(ref nt) => match **nt {
832-
token::NtLifetime(_) | token::NtTT(_) => true,
833-
_ => false,
834-
},
835-
_ => false,
836-
},
837-
_ => match token.kind {
838-
token::CloseDelim(_) => false,
839-
_ => true,
840-
},
841-
}
842-
}
843-
844-
/// A call to the "black-box" parser to parse some Rust non-terminal.
845-
///
846-
/// # Parameters
847-
///
848-
/// - `p`: the "black-box" parser to use
849-
/// - `sp`: the `Span` we want to parse
850-
/// - `name`: the name of the metavar _matcher_ we want to match (e.g., `tt`, `ident`, `block`,
851-
/// etc...)
852-
///
853-
/// # Returns
854-
///
855-
/// The parsed non-terminal.
856-
fn parse_nt(p: &mut Parser<'_>, sp: Span, name: Symbol) -> Result<Nonterminal, ()> {
857-
// FIXME(Centril): Consider moving this to `parser.rs` to make
858-
// the visibilities of the methods used below `pub(super)` at most.
859-
if name == sym::tt {
860-
return Ok(token::NtTT(p.parse_token_tree()));
861-
}
862-
parse_nt_inner(p, sp, name).map_err(|mut err| {
863-
err.span_label(sp, format!("while parsing argument for this `{}` macro fragment", name))
864-
.emit()
865-
})
866-
}
867-
868-
fn parse_nt_inner<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> PResult<'a, Nonterminal> {
869-
// Any `Nonterminal` which stores its tokens (currently `NtItem` and `NtExpr`)
870-
// needs to have them force-captured here.
871-
// A `macro_rules!` invocation may pass a captured item/expr to a proc-macro,
872-
// which requires having captured tokens available. Since we cannot determine
873-
// in advance whether or not a proc-macro will be (transitively) invoked,
874-
// we always capture tokens for any `Nonterminal` which needs them.
875-
Ok(match name {
876-
sym::item => match p.collect_tokens(|this| this.parse_item())? {
877-
(Some(mut item), tokens) => {
878-
// If we captured tokens during parsing (due to outer attributes),
879-
// use those.
880-
if item.tokens.is_none() {
881-
item.tokens = Some(tokens);
882-
}
883-
token::NtItem(item)
884-
}
885-
(None, _) => return Err(p.struct_span_err(p.token.span, "expected an item keyword")),
886-
},
887-
sym::block => token::NtBlock(p.parse_block()?),
888-
sym::stmt => match p.parse_stmt()? {
889-
Some(s) => token::NtStmt(s),
890-
None => return Err(p.struct_span_err(p.token.span, "expected a statement")),
891-
},
892-
sym::pat => token::NtPat(p.parse_pat(None)?),
893-
sym::expr => {
894-
let (mut expr, tokens) = p.collect_tokens(|this| this.parse_expr())?;
895-
// If we captured tokens during parsing (due to outer attributes),
896-
// use those.
897-
if expr.tokens.is_none() {
898-
expr.tokens = Some(tokens);
899-
}
900-
token::NtExpr(expr)
901-
}
902-
sym::literal => token::NtLiteral(p.parse_literal_maybe_minus()?),
903-
sym::ty => token::NtTy(p.parse_ty()?),
904-
// this could be handled like a token, since it is one
905-
sym::ident => {
906-
if let Some((ident, is_raw)) = get_macro_ident(&p.token) {
907-
p.bump();
908-
token::NtIdent(ident, is_raw)
909-
} else {
910-
let token_str = pprust::token_to_string(&p.token);
911-
let msg = &format!("expected ident, found {}", &token_str);
912-
return Err(p.struct_span_err(p.token.span, msg));
913-
}
914-
}
915-
sym::path => token::NtPath(p.parse_path(PathStyle::Type)?),
916-
sym::meta => token::NtMeta(P(p.parse_attr_item()?)),
917-
sym::vis => token::NtVis(p.parse_visibility(FollowedByType::Yes)?),
918-
sym::lifetime => {
919-
if p.check_lifetime() {
920-
token::NtLifetime(p.expect_lifetime().ident)
921-
} else {
922-
let token_str = pprust::token_to_string(&p.token);
923-
let msg = &format!("expected a lifetime, found `{}`", &token_str);
924-
return Err(p.struct_span_err(p.token.span, msg));
925-
}
926-
}
927-
// this is not supposed to happen, since it has been checked
928-
// when compiling the macro.
929-
_ => p.span_bug(sp, "invalid fragment specifier"),
930-
})
931-
}

‎src/librustc_expand/mbe/macro_rules.rs‎

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq};
99
use crate::mbe::transcribe::transcribe;
1010

1111
use rustc_ast::ast;
12-
use rustc_ast::token::{self, NtTT, Token, TokenKind::*};
12+
use rustc_ast::token::{self, NonterminalKind, NtTT, Token, TokenKind::*};
1313
use rustc_ast::tokenstream::{DelimSpan, TokenStream};
1414
use rustc_ast_pretty::pprust;
1515
use rustc_attr::{self as attr, TransparencyError};
@@ -1228,23 +1228,7 @@ fn is_legal_fragment_specifier(
12281228
* for checking against feature gates. See past versions of
12291229
* this function.
12301230
*/
1231-
match frag_name {
1232-
sym::item
1233-
| sym::block
1234-
| sym::stmt
1235-
| sym::expr
1236-
| sym::pat
1237-
| sym::lifetime
1238-
| sym::path
1239-
| sym::ty
1240-
| sym::ident
1241-
| sym::meta
1242-
| sym::tt
1243-
| sym::vis
1244-
| sym::literal
1245-
| kw::Invalid => true,
1246-
_ => false,
1247-
}
1231+
NonterminalKind::from_symbol(frag_name).is_some() || frag_name == kw::Invalid
12481232
}
12491233

12501234
fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {

‎src/librustc_parse/parser/expr.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1450,7 +1450,7 @@ impl<'a> Parser<'a> {
14501450

14511451
/// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`).
14521452
/// Keep this in sync with `Token::can_begin_literal_maybe_minus`.
1453-
pub fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> {
1453+
pub(super) fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> {
14541454
maybe_whole_expr!(self);
14551455

14561456
let lo = self.token.span;

‎src/librustc_parse/parser/mod.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
pub mod attr;
22
mod expr;
33
mod item;
4+
mod nonterminal;
45
mod pat;
56
mod path;
67
mod ty;
@@ -10,6 +11,7 @@ mod generics;
1011
mod stmt;
1112
use diagnostics::Error;
1213

14+
1315
use crate::lexer::UnmatchedBrace;
1416

1517
use log::debug;
@@ -958,7 +960,7 @@ impl<'a> Parser<'a> {
958960
}
959961

960962
/// Parses a single token tree from the input.
961-
pub fn parse_token_tree(&mut self) -> TokenTree {
963+
pub(crate) fn parse_token_tree(&mut self) -> TokenTree {
962964
match self.token.kind {
963965
token::OpenDelim(..) => {
964966
let frame = mem::replace(
@@ -1017,7 +1019,7 @@ impl<'a> Parser<'a> {
10171019
/// If the following element can't be a tuple (i.e., it's a function definition), then
10181020
/// it's not a tuple struct field), and the contents within the parentheses isn't valid,
10191021
/// so emit a proper diagnostic.
1020-
pub fn parse_visibility(&mut self, fbt: FollowedByType) -> PResult<'a, Visibility> {
1022+
pub(crate) fn parse_visibility(&mut self, fbt: FollowedByType) -> PResult<'a, Visibility> {
10211023
maybe_whole!(self, NtVis, |x| x);
10221024

10231025
self.expected_tokens.push(TokenType::Keyword(kw::Crate));

0 commit comments

Comments
 (0)