Skip to content

Commit 1ded7a5

Browse files
Aaron1011Aaron1011
committed
Handle None-delimited groups when parsing macro_rules! macro
When a `macro_rules!` macro expands to another `macro_rules!` macro, we may see `None`-delimited groups in odd places when another crate deserializes the 'inner' macro. This commit 'unwraps' an outer `None`-delimited group to avoid breaking existing code. See #73569 (comment) for more details. The proper fix is to handle `None`-delimited groups systematically throughout the parser, but that will require significant work. In the meantime, this hack lets us fix important hygiene bugs in macros
1 parent c844028 commit 1ded7a5

File tree

3 files changed

+87
-47
lines changed

3 files changed

+87
-47
lines changed

‎src/librustc_expand/mbe/quoted.rs‎

Lines changed: 63 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -90,72 +90,88 @@ pub(super) fn parse(
9090
/// # Parameters
9191
///
9292
/// - `tree`: the tree we wish to convert.
93-
/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish
93+
/// - `outer_trees`: an iterator over trees. We may need to read more tokens from it in order to finish
9494
/// converting `tree`
9595
/// - `expect_matchers`: same as for `parse` (see above).
9696
/// - `sess`: the parsing session. Any errors will be emitted to this session.
9797
/// - `features`, `attrs`: language feature flags and attributes so that we know whether to use
9898
/// unstable features or not.
9999
fn parse_tree(
100100
tree: tokenstream::TokenTree,
101-
trees: &mut impl Iterator<Item = tokenstream::TokenTree>,
101+
outer_trees: &mut impl Iterator<Item = tokenstream::TokenTree>,
102102
expect_matchers: bool,
103103
sess: &ParseSess,
104104
node_id: NodeId,
105105
) -> TokenTree {
106106
// Depending on what `tree` is, we could be parsing different parts of a macro
107107
match tree {
108108
// `tree` is a `$` token. Look at the next token in `trees`
109-
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() {
110-
// `tree` is followed by a delimited set of token trees. This indicates the beginning
111-
// of a repetition sequence in the macro (e.g. `$(pat)*`).
112-
Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
113-
// Must have `(` not `{` or `[`
114-
if delim != token::Paren {
115-
let tok = pprust::token_kind_to_string(&token::OpenDelim(delim));
116-
let msg = format!("expected `(`, found `{}`", tok);
117-
sess.span_diagnostic.span_err(span.entire(), &msg);
118-
}
119-
// Parse the contents of the sequence itself
120-
let sequence = parse(tts, expect_matchers, sess, node_id);
121-
// Get the Kleene operator and optional separator
122-
let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess);
123-
// Count the number of captured "names" (i.e., named metavars)
124-
let name_captures = macro_parser::count_names(&sequence);
125-
TokenTree::Sequence(
126-
span,
127-
Lrc::new(SequenceRepetition {
128-
tts: sequence,
129-
separator,
130-
kleene,
131-
num_captures: name_captures,
132-
}),
133-
)
109+
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => {
110+
// FIXME: Handle `None`-delimited groups in a more systematic way
111+
// during parsing.
112+
let mut next = outer_trees.next();
113+
let mut trees: Box<dyn Iterator<Item = tokenstream::TokenTree>>;
114+
if let Some(tokenstream::TokenTree::Delimited(_, token::NoDelim, tts)) = next {
115+
trees = Box::new(tts.into_trees());
116+
next = trees.next();
117+
} else {
118+
trees = Box::new(outer_trees);
134119
}
135120

136-
// `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special
137-
// metavariable that names the crate of the invocation.
138-
Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => {
139-
let (ident, is_raw) = token.ident().unwrap();
140-
let span = ident.span.with_lo(span.lo());
141-
if ident.name == kw::Crate && !is_raw {
142-
TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span)
143-
} else {
144-
TokenTree::MetaVar(span, ident)
121+
match next {
122+
// `tree` is followed by a delimited set of token trees. This indicates the beginning
123+
// of a repetition sequence in the macro (e.g. `$(pat)*`).
124+
Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
125+
// Must have `(` not `{` or `[`
126+
if delim != token::Paren {
127+
let tok = pprust::token_kind_to_string(&token::OpenDelim(delim));
128+
let msg = format!("expected `(`, found `{}`", tok);
129+
sess.span_diagnostic.span_err(span.entire(), &msg);
130+
}
131+
// Parse the contents of the sequence itself
132+
let sequence = parse(tts, expect_matchers, sess, node_id);
133+
// Get the Kleene operator and optional separator
134+
let (separator, kleene) =
135+
parse_sep_and_kleene_op(&mut trees, span.entire(), sess);
136+
// Count the number of captured "names" (i.e., named metavars)
137+
let name_captures = macro_parser::count_names(&sequence);
138+
TokenTree::Sequence(
139+
span,
140+
Lrc::new(SequenceRepetition {
141+
tts: sequence,
142+
separator,
143+
kleene,
144+
num_captures: name_captures,
145+
}),
146+
)
145147
}
146-
}
147148

148-
// `tree` is followed by a random token. This is an error.
149-
Some(tokenstream::TokenTree::Token(token)) => {
150-
let msg =
151-
format!("expected identifier, found `{}`", pprust::token_to_string(&token),);
152-
sess.span_diagnostic.span_err(token.span, &msg);
153-
TokenTree::MetaVar(token.span, Ident::invalid())
154-
}
149+
// `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special
150+
// metavariable that names the crate of the invocation.
151+
Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => {
152+
let (ident, is_raw) = token.ident().unwrap();
153+
let span = ident.span.with_lo(span.lo());
154+
if ident.name == kw::Crate && !is_raw {
155+
TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span)
156+
} else {
157+
TokenTree::MetaVar(span, ident)
158+
}
159+
}
155160

156-
// There are no more tokens. Just return the `$` we already have.
157-
None => TokenTree::token(token::Dollar, span),
158-
},
161+
// `tree` is followed by a random token. This is an error.
162+
Some(tokenstream::TokenTree::Token(token)) => {
163+
let msg = format!(
164+
"expected identifier, found `{}`",
165+
pprust::token_to_string(&token),
166+
);
167+
sess.span_diagnostic.span_err(token.span, &msg);
168+
TokenTree::MetaVar(token.span, Ident::invalid())
169+
}
170+
171+
// There are no more tokens. Just return the `$` we already have.
172+
None => TokenTree::token(token::Dollar, span),
173+
}
174+
}
159175

160176
// `tree` is an arbitrary token. Keep it.
161177
tokenstream::TokenTree::Token(token) => TokenTree::Token(token),

‎src/test/ui/proc-macro/auxiliary/meta-delim.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
macro_rules! produce_it {
2+
($dollar_one:tt $foo:ident $my_name:ident) => {
3+
#[macro_export]
4+
macro_rules! meta_delim {
5+
($dollar_one ($dollar_one $my_name:ident)*) => {
6+
stringify!($dollar_one ($dollar_one $my_name)*)
7+
}
8+
}
9+
}
10+
}
11+
12+
produce_it!($my_name name);

‎src/test/ui/proc-macro/meta-delim.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
// aux-build:meta-delim.rs
2+
// edition:2018
3+
// run-pass
4+
5+
// Tests that we can properly deserialize a macro with strange delimiters
6+
// See https://github.com/rust-lang/rust/pull/73569#issuecomment-650860457
7+
8+
extern crate meta_delim;
9+
10+
fn main() {
11+
assert_eq!("a bunch of idents", meta_delim::meta_delim!(a bunch of idents));
12+
}

0 commit comments

Comments
 (0)