Skip to content

Commit 78f33c0

Browse files
VeykrilVeykril
committed
Expand unmatched mbe fragments to reasonable default token trees
Currently we expand unmatched fragments by not replacing them at all, leaving us with `$ident`. This trips up the parser or subsequent macro calls. Instead it makes more sense to replace these with some reasonable default depending on the fragment kind which should make more recursive macro calls work better for completions.
1 parent 476d043 commit 78f33c0

File tree

6 files changed

+146
-45
lines changed

6 files changed

+146
-45
lines changed

‎crates/mbe/src/benchmark.rs‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use syntax::{
88
use test_utils::{bench, bench_fixture, skip_slow_tests};
99

1010
use crate::{
11-
parser::{Op, RepeatKind, Separator},
11+
parser::{MetaVarKind, Op, RepeatKind, Separator},
1212
syntax_node_to_token_tree, DeclarativeMacro,
1313
};
1414

@@ -111,35 +111,35 @@ fn invocation_fixtures(rules: &FxHashMap<String, DeclarativeMacro>) -> Vec<(Stri
111111

112112
fn collect_from_op(op: &Op, parent: &mut tt::Subtree, seed: &mut usize) {
113113
return match op {
114-
Op::Var { kind, .. } => match kind.as_ref().map(|it| it.as_str()) {
115-
Some("ident") => parent.token_trees.push(make_ident("foo")),
116-
Some("ty") => parent.token_trees.push(make_ident("Foo")),
117-
Some("tt") => parent.token_trees.push(make_ident("foo")),
118-
Some("vis") => parent.token_trees.push(make_ident("pub")),
119-
Some("pat") => parent.token_trees.push(make_ident("foo")),
120-
Some("path") => parent.token_trees.push(make_ident("foo")),
121-
Some("literal") => parent.token_trees.push(make_literal("1")),
122-
Some("expr") => parent.token_trees.push(make_ident("foo")),
123-
Some("lifetime") => {
114+
Op::Var { kind, .. } => match kind.as_ref() {
115+
Some(MetaVarKind::Ident) => parent.token_trees.push(make_ident("foo")),
116+
Some(MetaVarKind::Ty) => parent.token_trees.push(make_ident("Foo")),
117+
Some(MetaVarKind::Tt) => parent.token_trees.push(make_ident("foo")),
118+
Some(MetaVarKind::Vis) => parent.token_trees.push(make_ident("pub")),
119+
Some(MetaVarKind::Pat) => parent.token_trees.push(make_ident("foo")),
120+
Some(MetaVarKind::Path) => parent.token_trees.push(make_ident("foo")),
121+
Some(MetaVarKind::Literal) => parent.token_trees.push(make_literal("1")),
122+
Some(MetaVarKind::Expr) => parent.token_trees.push(make_ident("foo")),
123+
Some(MetaVarKind::Lifetime) => {
124124
parent.token_trees.push(make_punct('\''));
125125
parent.token_trees.push(make_ident("a"));
126126
}
127-
Some("block") => {
127+
Some(MetaVarKind::Block) => {
128128
parent.token_trees.push(make_subtree(tt::DelimiterKind::Brace, None))
129129
}
130-
Some("item") => {
130+
Some(MetaVarKind::Item) => {
131131
parent.token_trees.push(make_ident("fn"));
132132
parent.token_trees.push(make_ident("foo"));
133133
parent.token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
134134
parent.token_trees.push(make_subtree(tt::DelimiterKind::Brace, None));
135135
}
136-
Some("meta") => {
136+
Some(MetaVarKind::Meta) => {
137137
parent.token_trees.push(make_ident("foo"));
138138
parent.token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
139139
}
140140

141141
None => (),
142-
Some(kind) => panic!("Unhandled kind {}", kind),
142+
Some(kind) => panic!("Unhandled kind {:?}", kind),
143143
},
144144
Op::Leaf(leaf) => parent.token_trees.push(leaf.clone().into()),
145145
Op::Repeat { tokens, kind, separator } => {

‎crates/mbe/src/expander.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ mod transcriber;
88
use rustc_hash::FxHashMap;
99
use syntax::SmolStr;
1010

11-
use crate::{ExpandError, ExpandResult};
11+
use crate::{parser::MetaVarKind, ExpandError, ExpandResult};
1212

1313
pub(crate) fn expand_rules(
1414
rules: &[crate::Rule],
@@ -104,6 +104,7 @@ enum Binding {
104104
Fragment(Fragment),
105105
Nested(Vec<Binding>),
106106
Empty,
107+
Missing(MetaVarKind),
107108
}
108109

109110
#[derive(Debug, Clone, PartialEq, Eq)]

‎crates/mbe/src/expander/matcher.rs‎

Lines changed: 35 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ use syntax::SmolStr;
6666

6767
use crate::{
6868
expander::{Binding, Bindings, ExpandResult, Fragment},
69-
parser::{Op, RepeatKind, Separator},
69+
parser::{MetaVarKind, Op, RepeatKind, Separator},
7070
tt_iter::TtIter,
7171
ExpandError, MetaTemplate,
7272
};
@@ -119,6 +119,7 @@ pub(super) fn match_(pattern: &MetaTemplate, input: &tt::Subtree) -> Match {
119119
.map(|it| match it {
120120
Binding::Fragment(_) => 1,
121121
Binding::Empty => 1,
122+
Binding::Missing(_) => 1,
122123
Binding::Nested(it) => count(it.iter()),
123124
})
124125
.sum()
@@ -130,6 +131,7 @@ enum BindingKind {
130131
Empty(SmolStr),
131132
Optional(SmolStr),
132133
Fragment(SmolStr, Fragment),
134+
Missing(SmolStr, MetaVarKind),
133135
Nested(usize, usize),
134136
}
135137

@@ -190,6 +192,10 @@ impl BindingsBuilder {
190192
.push(LinkNode::Node(Rc::new(BindingKind::Fragment(var.clone(), fragment))));
191193
}
192194

195+
fn push_missing(&mut self, idx: &mut BindingsIdx, var: &SmolStr, kind: MetaVarKind) {
196+
self.nodes[idx.0].push(LinkNode::Node(Rc::new(BindingKind::Missing(var.clone(), kind))));
197+
}
198+
193199
fn push_nested(&mut self, parent: &mut BindingsIdx, child: &BindingsIdx) {
194200
let BindingsIdx(idx, nidx) = self.copy(child);
195201
self.nodes[parent.0].push(LinkNode::Node(Rc::new(BindingKind::Nested(idx, nidx))));
@@ -222,6 +228,9 @@ impl BindingsBuilder {
222228
BindingKind::Fragment(name, fragment) => {
223229
bindings.inner.insert(name.clone(), Binding::Fragment(fragment.clone()));
224230
}
231+
BindingKind::Missing(name, kind) => {
232+
bindings.inner.insert(name.clone(), Binding::Missing(*kind));
233+
}
225234
BindingKind::Nested(idx, nested_idx) => {
226235
let mut nested_nodes = Vec::new();
227236
self.collect_nested(*idx, *nested_idx, &mut nested_nodes);
@@ -458,9 +467,9 @@ fn match_loop_inner<'t>(
458467
}
459468
}
460469
OpDelimited::Op(Op::Var { kind, name, .. }) => {
461-
if let Some(kind) = kind {
470+
if let &Some(kind) = kind {
462471
let mut fork = src.clone();
463-
let match_res = match_meta_var(kind.as_str(), &mut fork);
472+
let match_res = match_meta_var(kind, &mut fork);
464473
match match_res.err {
465474
None => {
466475
// Some meta variables are optional (e.g. vis)
@@ -475,8 +484,15 @@ fn match_loop_inner<'t>(
475484
}
476485
Some(err) => {
477486
res.add_err(err);
478-
if let Some(fragment) = match_res.value {
479-
bindings_builder.push_fragment(&mut item.bindings, name, fragment);
487+
match match_res.value {
488+
Some(fragment) => bindings_builder.push_fragment(
489+
&mut item.bindings,
490+
name,
491+
fragment,
492+
),
493+
None => {
494+
bindings_builder.push_missing(&mut item.bindings, name, kind)
495+
}
480496
}
481497
item.is_error = true;
482498
error_items.push(item);
@@ -668,20 +684,20 @@ fn match_leaf(lhs: &tt::Leaf, src: &mut TtIter<'_>) -> Result<(), ExpandError> {
668684
}
669685
}
670686

671-
fn match_meta_var(kind: &str, input: &mut TtIter<'_>) -> ExpandResult<Option<Fragment>> {
687+
fn match_meta_var(kind: MetaVarKind, input: &mut TtIter<'_>) -> ExpandResult<Option<Fragment>> {
672688
let fragment = match kind {
673-
"path" => parser::PrefixEntryPoint::Path,
674-
"ty" => parser::PrefixEntryPoint::Ty,
689+
MetaVarKind::Path => parser::PrefixEntryPoint::Path,
690+
MetaVarKind::Ty => parser::PrefixEntryPoint::Ty,
675691
// FIXME: These two should actually behave differently depending on the edition.
676692
//
677693
// https://doc.rust-lang.org/edition-guide/rust-2021/or-patterns-macro-rules.html
678-
"pat" | "pat_param" => parser::PrefixEntryPoint::Pat,
679-
"stmt" => parser::PrefixEntryPoint::Stmt,
680-
"block" => parser::PrefixEntryPoint::Block,
681-
"meta" => parser::PrefixEntryPoint::MetaItem,
682-
"item" => parser::PrefixEntryPoint::Item,
683-
"vis" => parser::PrefixEntryPoint::Vis,
684-
"expr" => {
694+
MetaVarKind::Pat | MetaVarKind::PatParam => parser::PrefixEntryPoint::Pat,
695+
MetaVarKind::Stmt => parser::PrefixEntryPoint::Stmt,
696+
MetaVarKind::Block => parser::PrefixEntryPoint::Block,
697+
MetaVarKind::Meta => parser::PrefixEntryPoint::MetaItem,
698+
MetaVarKind::Item => parser::PrefixEntryPoint::Item,
699+
MetaVarKind::Vis => parser::PrefixEntryPoint::Vis,
700+
MetaVarKind::Expr => {
685701
// `expr` should not match underscores.
686702
// HACK: Macro expansion should not be done using "rollback and try another alternative".
687703
// rustc [explicitly checks the next token][0].
@@ -698,17 +714,17 @@ fn match_meta_var(kind: &str, input: &mut TtIter<'_>) -> ExpandResult<Option<Fra
698714
}
699715
_ => {
700716
let tt_result = match kind {
701-
"ident" => input
717+
MetaVarKind::Ident => input
702718
.expect_ident()
703719
.map(|ident| tt::Leaf::from(ident.clone()).into())
704720
.map_err(|()| ExpandError::binding_error("expected ident")),
705-
"tt" => input
721+
MetaVarKind::Tt => input
706722
.expect_tt()
707723
.map_err(|()| ExpandError::binding_error("expected token tree")),
708-
"lifetime" => input
724+
MetaVarKind::Lifetime => input
709725
.expect_lifetime()
710726
.map_err(|()| ExpandError::binding_error("expected lifetime")),
711-
"literal" => {
727+
MetaVarKind::Literal => {
712728
let neg = input.eat_char('-');
713729
input
714730
.expect_literal()

‎crates/mbe/src/expander/transcriber.rs‎

Lines changed: 53 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use tt::{Delimiter, Subtree};
66

77
use crate::{
88
expander::{Binding, Bindings, Fragment},
9-
parser::{Op, RepeatKind, Separator},
9+
parser::{MetaVarKind, Op, RepeatKind, Separator},
1010
ExpandError, ExpandResult, MetaTemplate,
1111
};
1212

@@ -15,7 +15,7 @@ impl Bindings {
1515
self.inner.contains_key(name)
1616
}
1717

18-
fn get(&self, name: &str, nesting: &mut [NestingState]) -> Result<&Fragment, ExpandError> {
18+
fn get(&self, name: &str, nesting: &mut [NestingState]) -> Result<Fragment, ExpandError> {
1919
macro_rules! binding_err {
2020
($($arg:tt)*) => { ExpandError::binding_error(format!($($arg)*)) };
2121
}
@@ -26,6 +26,7 @@ impl Bindings {
2626
nesting_state.hit = true;
2727
b = match b {
2828
Binding::Fragment(_) => break,
29+
Binding::Missing(_) => break,
2930
Binding::Nested(bs) => bs.get(nesting_state.idx).ok_or_else(|| {
3031
nesting_state.at_end = true;
3132
binding_err!("could not find nested binding `{name}`")
@@ -37,7 +38,55 @@ impl Bindings {
3738
};
3839
}
3940
match b {
40-
Binding::Fragment(it) => Ok(it),
41+
Binding::Fragment(it) => Ok(it.clone()),
42+
// emit some reasonable default expansion for missing bindings,
43+
// this gives better recovery than emitting the `$fragment-name` verbatim
44+
Binding::Missing(it) => Ok(match it {
45+
MetaVarKind::Stmt => {
46+
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
47+
id: tt::TokenId::unspecified(),
48+
char: ';',
49+
spacing: tt::Spacing::Alone,
50+
})))
51+
}
52+
MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
53+
delimiter: Some(tt::Delimiter {
54+
id: tt::TokenId::unspecified(),
55+
kind: tt::DelimiterKind::Brace,
56+
}),
57+
token_trees: vec![],
58+
})),
59+
// FIXME: Meta and Item should get proper defaults
60+
MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => {
61+
Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
62+
delimiter: None,
63+
token_trees: vec![],
64+
}))
65+
}
66+
MetaVarKind::Path
67+
| MetaVarKind::Ty
68+
| MetaVarKind::Pat
69+
| MetaVarKind::PatParam
70+
| MetaVarKind::Expr
71+
| MetaVarKind::Ident => {
72+
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
73+
text: SmolStr::new_inline("missing"),
74+
id: tt::TokenId::unspecified(),
75+
})))
76+
}
77+
MetaVarKind::Lifetime => {
78+
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
79+
text: SmolStr::new_inline("'missing"),
80+
id: tt::TokenId::unspecified(),
81+
})))
82+
}
83+
MetaVarKind::Literal => {
84+
Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
85+
text: SmolStr::new_inline("\"missing\""),
86+
id: tt::TokenId::unspecified(),
87+
})))
88+
}
89+
}),
4190
Binding::Nested(_) => {
4291
Err(binding_err!("expected simple binding, found nested binding `{name}`"))
4392
}
@@ -157,7 +206,7 @@ fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandRe
157206
} else {
158207
ctx.bindings.get(v, &mut ctx.nesting).map_or_else(
159208
|e| ExpandResult { value: Fragment::Tokens(tt::TokenTree::empty()), err: Some(e) },
160-
|b| ExpandResult::ok(b.clone()),
209+
|it| ExpandResult::ok(it),
161210
)
162211
}
163212
}

‎crates/mbe/src/lib.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ mod token_map;
2121
use std::fmt;
2222

2323
use crate::{
24-
parser::{MetaTemplate, Op},
24+
parser::{MetaTemplate, MetaVarKind, Op},
2525
tt_iter::TtIter,
2626
};
2727

@@ -291,9 +291,9 @@ fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> {
291291
// Checks that no repetition which could match an empty token
292292
// https://github.com/rust-lang/rust/blob/a58b1ed44f5e06976de2bdc4d7dc81c36a96934f/src/librustc_expand/mbe/macro_rules.rs#L558
293293
let lsh_is_empty_seq = separator.is_none() && subtree.iter().all(|child_op| {
294-
match child_op {
294+
match *child_op {
295295
// vis is optional
296-
Op::Var { kind: Some(kind), .. } => kind == "vis",
296+
Op::Var { kind: Some(kind), .. } => kind == MetaVarKind::Vis,
297297
Op::Repeat {
298298
kind: parser::RepeatKind::ZeroOrMore | parser::RepeatKind::ZeroOrOne,
299299
..

‎crates/mbe/src/parser.rs‎

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ impl MetaTemplate {
5050

5151
#[derive(Clone, Debug, PartialEq, Eq)]
5252
pub(crate) enum Op {
53-
Var { name: SmolStr, kind: Option<SmolStr>, id: tt::TokenId },
53+
Var { name: SmolStr, kind: Option<MetaVarKind>, id: tt::TokenId },
5454
Ignore { name: SmolStr, id: tt::TokenId },
5555
Index { depth: u32 },
5656
Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
@@ -65,6 +65,24 @@ pub(crate) enum RepeatKind {
6565
ZeroOrOne,
6666
}
6767

68+
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
69+
pub(crate) enum MetaVarKind {
70+
Path,
71+
Ty,
72+
Pat,
73+
PatParam,
74+
Stmt,
75+
Block,
76+
Meta,
77+
Item,
78+
Vis,
79+
Expr,
80+
Ident,
81+
Tt,
82+
Lifetime,
83+
Literal,
84+
}
85+
6886
#[derive(Clone, Debug, Eq)]
6987
pub(crate) enum Separator {
7088
Literal(tt::Literal),
@@ -179,13 +197,30 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
179197
Ok(res)
180198
}
181199

182-
fn eat_fragment_kind(src: &mut TtIter<'_>, mode: Mode) -> Result<Option<SmolStr>, ParseError> {
200+
fn eat_fragment_kind(src: &mut TtIter<'_>, mode: Mode) -> Result<Option<MetaVarKind>, ParseError> {
183201
if let Mode::Pattern = mode {
184202
src.expect_char(':').map_err(|()| ParseError::unexpected("missing fragment specifier"))?;
185203
let ident = src
186204
.expect_ident()
187205
.map_err(|()| ParseError::unexpected("missing fragment specifier"))?;
188-
return Ok(Some(ident.text.clone()));
206+
let kind = match ident.text.as_str() {
207+
"path" => MetaVarKind::Path,
208+
"ty" => MetaVarKind::Ty,
209+
"pat" => MetaVarKind::Pat,
210+
"pat_param" => MetaVarKind::PatParam,
211+
"stmt" => MetaVarKind::Stmt,
212+
"block" => MetaVarKind::Block,
213+
"meta" => MetaVarKind::Meta,
214+
"item" => MetaVarKind::Item,
215+
"vis" => MetaVarKind::Vis,
216+
"expr" => MetaVarKind::Expr,
217+
"ident" => MetaVarKind::Ident,
218+
"tt" => MetaVarKind::Tt,
219+
"lifetime" => MetaVarKind::Lifetime,
220+
"literal" => MetaVarKind::Literal,
221+
_ => return Ok(None),
222+
};
223+
return Ok(Some(kind));
189224
};
190225
Ok(None)
191226
}

0 commit comments

Comments
 (0)