Skip to content

Commit f0dbe14

Browse files
petrochenkovpetrochenkov
committed
parser: Ensure that all nonterminals have tokens after parsing
1 parent b224261 commit f0dbe14

File tree

11 files changed

+115
-118
lines changed

11 files changed

+115
-118
lines changed

‎compiler/rustc_ast/src/ast_like.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,8 @@ impl AstLike for crate::token::Nonterminal {
8282
Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
8383
Nonterminal::NtPath(path) => path.tokens_mut(),
8484
Nonterminal::NtVis(vis) => vis.tokens_mut(),
85-
_ => panic!("Called tokens_mut on {:?}", self),
85+
Nonterminal::NtBlock(block) => block.tokens_mut(),
86+
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) | Nonterminal::NtTT(..) => None,
8687
}
8788
}
8889
}

‎compiler/rustc_parse/src/parser/attr_wrapper.rs‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -342,16 +342,10 @@ impl<'a> Parser<'a> {
342342

343343
// If we support tokens at all
344344
if let Some(target_tokens) = ret.tokens_mut() {
345-
if let Some(target_tokens) = target_tokens {
346-
assert!(
347-
!self.capture_cfg,
348-
"Encountered existing tokens with capture_cfg set: {:?}",
349-
target_tokens
350-
);
351-
} else {
345+
if target_tokens.is_none() {
352346
// Store se our newly captured tokens into the AST node
353347
*target_tokens = Some(tokens.clone());
354-
};
348+
}
355349
}
356350

357351
let final_attrs = ret.attrs();

‎compiler/rustc_parse/src/parser/expr.rs‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -94,17 +94,7 @@ impl<'a> Parser<'a> {
9494

9595
/// Parses an expression, forcing tokens to be collected
9696
pub fn parse_expr_force_collect(&mut self) -> PResult<'a, P<Expr>> {
97-
// If we have outer attributes, then the call to `collect_tokens_trailing_token`
98-
// will be made for us.
99-
if matches!(self.token.kind, TokenKind::Pound | TokenKind::DocComment(..)) {
100-
self.parse_expr()
101-
} else {
102-
// If we don't have outer attributes, then we need to ensure
103-
// that collection happens by using `collect_tokens_no_attrs`.
104-
// Expression don't support custom inner attributes, so `parse_expr`
105-
// will never try to collect tokens if we don't have outer attributes.
106-
self.collect_tokens_no_attrs(|this| this.parse_expr())
107-
}
97+
self.collect_tokens_no_attrs(|this| this.parse_expr())
10898
}
10999

110100
pub fn parse_anon_const_expr(&mut self) -> PResult<'a, AnonConst> {

‎compiler/rustc_parse/src/parser/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ enum BlockMode {
6363

6464
/// Whether or not we should force collection of tokens for an AST node,
6565
/// regardless of whether or not it has attributes
66+
#[derive(Clone, Copy, PartialEq)]
6667
pub enum ForceCollect {
6768
Yes,
6869
No,

‎compiler/rustc_parse/src/parser/nonterminal.rs‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
use rustc_ast::ptr::P;
22
use rustc_ast::token::{self, Nonterminal, NonterminalKind, Token};
3+
use rustc_ast::AstLike;
34
use rustc_ast_pretty::pprust;
45
use rustc_errors::PResult;
56
use rustc_span::symbol::{kw, Ident};
@@ -102,7 +103,7 @@ impl<'a> Parser<'a> {
102103
// which requires having captured tokens available. Since we cannot determine
103104
// in advance whether or not a proc-macro will be (transitively) invoked,
104105
// we always capture tokens for any `Nonterminal` which needs them.
105-
Ok(match kind {
106+
let mut nt = match kind {
106107
NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? {
107108
Some(item) => token::NtItem(item),
108109
None => {
@@ -169,7 +170,19 @@ impl<'a> Parser<'a> {
169170
return Err(self.struct_span_err(self.token.span, msg));
170171
}
171172
}
172-
})
173+
};
174+
175+
// If tokens are supported at all, they should be collected.
176+
if matches!(nt.tokens_mut(), Some(None)) {
177+
panic!(
178+
"Missing tokens for nt {:?} at {:?}: {:?}",
179+
nt,
180+
nt.span(),
181+
pprust::nonterminal_to_string(&nt)
182+
);
183+
}
184+
185+
Ok(nt)
173186
}
174187
}
175188

‎compiler/rustc_parse/src/parser/stmt.rs‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,11 @@ impl<'a> Parser<'a> {
7373
// or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something
7474
// that starts like a path (1 token), but it fact not a path.
7575
// Also, we avoid stealing syntax from `parse_item_`.
76-
self.parse_stmt_path_start(lo, attrs, force_collect)?
76+
if force_collect == ForceCollect::Yes {
77+
self.collect_tokens_no_attrs(|this| this.parse_stmt_path_start(lo, attrs))
78+
} else {
79+
self.parse_stmt_path_start(lo, attrs)
80+
}?
7781
} else if let Some(item) =
7882
self.parse_item_common(attrs.clone(), false, true, |_| true, force_collect)?
7983
{
@@ -85,21 +89,22 @@ impl<'a> Parser<'a> {
8589
self.mk_stmt(lo, StmtKind::Empty)
8690
} else if self.token != token::CloseDelim(token::Brace) {
8791
// Remainder are line-expr stmts.
88-
let e = self.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs))?;
92+
let e = if force_collect == ForceCollect::Yes {
93+
self.collect_tokens_no_attrs(|this| {
94+
this.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs))
95+
})
96+
} else {
97+
self.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs))
98+
}?;
8999
self.mk_stmt(lo.to(e.span), StmtKind::Expr(e))
90100
} else {
91101
self.error_outer_attrs(&attrs.take_for_recovery());
92102
return Ok(None);
93103
}))
94104
}
95105

96-
fn parse_stmt_path_start(
97-
&mut self,
98-
lo: Span,
99-
attrs: AttrWrapper,
100-
force_collect: ForceCollect,
101-
) -> PResult<'a, Stmt> {
102-
let stmt = self.collect_tokens_trailing_token(attrs, force_collect, |this, attrs| {
106+
fn parse_stmt_path_start(&mut self, lo: Span, attrs: AttrWrapper) -> PResult<'a, Stmt> {
107+
let stmt = self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
103108
let path = this.parse_path(PathStyle::Expr)?;
104109

105110
if this.eat(&token::Not) {

‎src/test/ui/macros/expr-stmt-nonterminal-tokens.rs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// check-pass
2+
3+
macro_rules! mac {
4+
(expr $expr:expr) => {};
5+
(stmt $stmt:stmt) => {};
6+
}
7+
8+
fn main() {
9+
mac!(expr #[allow(warnings)] 0);
10+
mac!(stmt 0);
11+
mac!(stmt {});
12+
mac!(stmt path);
13+
mac!(stmt 0 + 1);
14+
mac!(stmt path + 1);
15+
}

‎src/test/ui/proc-macro/cfg-eval.stdout‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,23 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
8383
span: $DIR/cfg-eval.rs:17:11: 24:2 (#0),
8484
},
8585
]
86-
PRINT-ATTR INPUT (DISPLAY): #[rustc_dummy] (#[cfg(all())] 1,)
86+
PRINT-ATTR INPUT (DISPLAY): #[rustc_dummy] #[rustc_dummy] (#[cfg(all())] 1,)
8787
PRINT-ATTR INPUT (DEBUG): TokenStream [
88+
Punct {
89+
ch: '#',
90+
spacing: Alone,
91+
span: $DIR/cfg-eval.rs:35:39: 35:40 (#0),
92+
},
93+
Group {
94+
delimiter: Bracket,
95+
stream: TokenStream [
96+
Ident {
97+
ident: "rustc_dummy",
98+
span: $DIR/cfg-eval.rs:35:62: 35:73 (#0),
99+
},
100+
],
101+
span: $DIR/cfg-eval.rs:35:39: 35:40 (#0),
102+
},
88103
Punct {
89104
ch: '#',
90105
spacing: Alone,

‎src/test/ui/proc-macro/inner-attrs.stderr‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,5 +22,36 @@ error: expected non-macro inner attribute, found attribute macro `print_attr`
2222
LL | #![print_attr]
2323
| ^^^^^^^^^^ not a non-macro inner attribute
2424

25-
error: aborting due to 4 previous errors
25+
error[E0308]: mismatched types
26+
--> $DIR/inner-attrs.rs:48:43
27+
|
28+
LL | #[print_target_and_args(tuple_attrs)] (
29+
| ___________________________________________^
30+
LL | | 3, 4, {
31+
LL | | #![cfg_attr(not(FALSE), rustc_dummy(innermost))]
32+
LL | | 5
33+
LL | | }
34+
LL | | );
35+
| |_____^ expected `()`, found tuple
36+
|
37+
= note: expected unit type `()`
38+
found tuple `({integer}, {integer}, {integer})`
39+
40+
error[E0308]: mismatched types
41+
--> $DIR/inner-attrs.rs:55:43
42+
|
43+
LL | #[print_target_and_args(tuple_attrs)] (
44+
| ___________________________________________^
45+
LL | | 3, 4, {
46+
LL | | #![cfg_attr(not(FALSE), rustc_dummy(innermost))]
47+
LL | | 5
48+
LL | | }
49+
LL | | );
50+
| |_____^ expected `()`, found tuple
51+
|
52+
= note: expected unit type `()`
53+
found tuple `({integer}, {integer}, {integer})`
54+
55+
error: aborting due to 6 previous errors
2656

57+
For more information about this error, try `rustc --explain E0308`.

‎src/test/ui/proc-macro/inner-attrs.stdout‎

Lines changed: 16 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -705,7 +705,7 @@ PRINT-ATTR_ARGS INPUT (DEBUG): TokenStream [
705705
span: $DIR/inner-attrs.rs:48:29: 48:40 (#0),
706706
},
707707
]
708-
PRINT-ATTR INPUT (DISPLAY): (3, 4, { #! [cfg_attr(not(FALSE), rustc_dummy(innermost))] 5 }) ;
708+
PRINT-ATTR INPUT (DISPLAY): (3, 4, { #! [rustc_dummy(innermost)] 5 })
709709
PRINT-ATTR INPUT (DEBUG): TokenStream [
710710
Group {
711711
delimiter: Parenthesis,
@@ -737,7 +737,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
737737
stream: TokenStream [
738738
Punct {
739739
ch: '#',
740-
spacing: Joint,
740+
spacing: Alone,
741741
span: $DIR/inner-attrs.rs:50:13: 50:14 (#0),
742742
},
743743
Punct {
@@ -749,50 +749,21 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
749749
delimiter: Bracket,
750750
stream: TokenStream [
751751
Ident {
752-
ident: "cfg_attr",
753-
span: $DIR/inner-attrs.rs:50:16: 50:24 (#0),
752+
ident: "rustc_dummy",
753+
span: $DIR/inner-attrs.rs:50:37: 50:48 (#0),
754754
},
755755
Group {
756756
delimiter: Parenthesis,
757757
stream: TokenStream [
758758
Ident {
759-
ident: "not",
760-
span: $DIR/inner-attrs.rs:50:25: 50:28 (#0),
761-
},
762-
Group {
763-
delimiter: Parenthesis,
764-
stream: TokenStream [
765-
Ident {
766-
ident: "FALSE",
767-
span: $DIR/inner-attrs.rs:50:29: 50:34 (#0),
768-
},
769-
],
770-
span: $DIR/inner-attrs.rs:50:28: 50:35 (#0),
771-
},
772-
Punct {
773-
ch: ',',
774-
spacing: Alone,
775-
span: $DIR/inner-attrs.rs:50:35: 50:36 (#0),
776-
},
777-
Ident {
778-
ident: "rustc_dummy",
779-
span: $DIR/inner-attrs.rs:50:37: 50:48 (#0),
780-
},
781-
Group {
782-
delimiter: Parenthesis,
783-
stream: TokenStream [
784-
Ident {
785-
ident: "innermost",
786-
span: $DIR/inner-attrs.rs:50:49: 50:58 (#0),
787-
},
788-
],
789-
span: $DIR/inner-attrs.rs:50:48: 50:59 (#0),
759+
ident: "innermost",
760+
span: $DIR/inner-attrs.rs:50:49: 50:58 (#0),
790761
},
791762
],
792-
span: $DIR/inner-attrs.rs:50:24: 50:60 (#0),
763+
span: $DIR/inner-attrs.rs:50:48: 50:59 (#0),
793764
},
794765
],
795-
span: $DIR/inner-attrs.rs:50:15: 50:61 (#0),
766+
span: $DIR/inner-attrs.rs:50:13: 50:14 (#0),
796767
},
797768
Literal {
798769
kind: Integer,
@@ -806,11 +777,6 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
806777
],
807778
span: $DIR/inner-attrs.rs:48:43: 53:6 (#0),
808779
},
809-
Punct {
810-
ch: ';',
811-
spacing: Alone,
812-
span: $DIR/inner-attrs.rs:53:6: 53:7 (#0),
813-
},
814780
]
815781
PRINT-ATTR_ARGS INPUT (DISPLAY): tuple_attrs
816782
PRINT-ATTR_ARGS INPUT (DEBUG): TokenStream [
@@ -819,7 +785,7 @@ PRINT-ATTR_ARGS INPUT (DEBUG): TokenStream [
819785
span: $DIR/inner-attrs.rs:55:29: 55:40 (#0),
820786
},
821787
]
822-
PRINT-ATTR INPUT (DISPLAY): (3, 4, { #! [cfg_attr(not(FALSE), rustc_dummy(innermost))] 5 }) ;
788+
PRINT-ATTR INPUT (DISPLAY): (3, 4, { #! [rustc_dummy(innermost)] 5 })
823789
PRINT-ATTR INPUT (DEBUG): TokenStream [
824790
Group {
825791
delimiter: Parenthesis,
@@ -851,7 +817,7 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
851817
stream: TokenStream [
852818
Punct {
853819
ch: '#',
854-
spacing: Joint,
820+
spacing: Alone,
855821
span: $DIR/inner-attrs.rs:57:13: 57:14 (#0),
856822
},
857823
Punct {
@@ -863,50 +829,21 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
863829
delimiter: Bracket,
864830
stream: TokenStream [
865831
Ident {
866-
ident: "cfg_attr",
867-
span: $DIR/inner-attrs.rs:57:16: 57:24 (#0),
832+
ident: "rustc_dummy",
833+
span: $DIR/inner-attrs.rs:57:37: 57:48 (#0),
868834
},
869835
Group {
870836
delimiter: Parenthesis,
871837
stream: TokenStream [
872838
Ident {
873-
ident: "not",
874-
span: $DIR/inner-attrs.rs:57:25: 57:28 (#0),
875-
},
876-
Group {
877-
delimiter: Parenthesis,
878-
stream: TokenStream [
879-
Ident {
880-
ident: "FALSE",
881-
span: $DIR/inner-attrs.rs:57:29: 57:34 (#0),
882-
},
883-
],
884-
span: $DIR/inner-attrs.rs:57:28: 57:35 (#0),
885-
},
886-
Punct {
887-
ch: ',',
888-
spacing: Alone,
889-
span: $DIR/inner-attrs.rs:57:35: 57:36 (#0),
890-
},
891-
Ident {
892-
ident: "rustc_dummy",
893-
span: $DIR/inner-attrs.rs:57:37: 57:48 (#0),
894-
},
895-
Group {
896-
delimiter: Parenthesis,
897-
stream: TokenStream [
898-
Ident {
899-
ident: "innermost",
900-
span: $DIR/inner-attrs.rs:57:49: 57:58 (#0),
901-
},
902-
],
903-
span: $DIR/inner-attrs.rs:57:48: 57:59 (#0),
839+
ident: "innermost",
840+
span: $DIR/inner-attrs.rs:57:49: 57:58 (#0),
904841
},
905842
],
906-
span: $DIR/inner-attrs.rs:57:24: 57:60 (#0),
843+
span: $DIR/inner-attrs.rs:57:48: 57:59 (#0),
907844
},
908845
],
909-
span: $DIR/inner-attrs.rs:57:15: 57:61 (#0),
846+
span: $DIR/inner-attrs.rs:57:13: 57:14 (#0),
910847
},
911848
Literal {
912849
kind: Integer,
@@ -920,11 +857,6 @@ PRINT-ATTR INPUT (DEBUG): TokenStream [
920857
],
921858
span: $DIR/inner-attrs.rs:55:43: 60:6 (#0),
922859
},
923-
Punct {
924-
ch: ';',
925-
spacing: Alone,
926-
span: $DIR/inner-attrs.rs:60:6: 60:7 (#0),
927-
},
928860
]
929861
PRINT-ATTR_ARGS INPUT (DISPLAY): tenth
930862
PRINT-ATTR_ARGS INPUT (DEBUG): TokenStream [

0 commit comments

Comments
 (0)