Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 5b0c39b

Browse files
nnethercotennethercote
committed
Prepare for invisible delimiters.
Current places where `Interpolated` is used are going to change to instead use invisible delimiters. This prepares for that. - It adds invisible delimiter cases to the `can_begin_*`/`may_be_*` methods that are equivalent to the existing `Interpolated` cases. - It adds panics/asserts in some places where invisible delimiters should never occur. - In `Parser::parse_struct_fields` it excludes an ident + invisible delimiter from special consideration in an error message, because that's quite different to an ident + paren/brace/bracket.
1 parent 5721b49 commit 5b0c39b

File tree

4 files changed

+109
-15
lines changed

4 files changed

+109
-15
lines changed

‎compiler/rustc_ast/src/token.rs‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -513,10 +513,11 @@ impl Token {
513513

514514
/// Returns `true` if the token can appear at the start of an expression.
515515
pub fn can_begin_expr(&self) -> bool {
516+
use Delimiter::*;
516517
match self.uninterpolate().kind {
517518
Ident(name, is_raw) =>
518519
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
519-
OpenDelim(..) | // tuple, array or block
520+
OpenDelim(Parenthesis | Brace | Bracket) | // tuple, array or block
520521
Literal(..) | // literal
521522
Not | // operator not
522523
BinOp(Minus) | // unary minus
@@ -527,13 +528,20 @@ impl Token {
527528
// DotDotDot is no longer supported, but we need some way to display the error
528529
DotDot | DotDotDot | DotDotEq | // range notation
529530
Lt | BinOp(Shl) | // associated path
530-
PathSep | // global path
531+
PathSep | // global path
531532
Lifetime(..) | // labeled loop
532533
Pound => true, // expression attributes
533534
Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) |
534535
NtExpr(..) |
535536
NtBlock(..) |
536537
NtPath(..)),
538+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
539+
NonterminalKind::Block |
540+
NonterminalKind::Expr |
541+
NonterminalKind::Expr2021 { .. } |
542+
NonterminalKind::Literal |
543+
NonterminalKind::Path
544+
))) => true,
537545
_ => false,
538546
}
539547
}
@@ -553,11 +561,18 @@ impl Token {
553561
// DotDotDot is no longer supported
554562
| DotDot | DotDotDot | DotDotEq // ranges
555563
| Lt | BinOp(Shl) // associated path
556-
| PathSep => true, // global path
564+
| PathSep => true, // global path
557565
Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) |
558566
NtPat(..) |
559567
NtBlock(..) |
560568
NtPath(..)),
569+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
570+
NonterminalKind::Block |
571+
NonterminalKind::PatParam { .. } |
572+
NonterminalKind::PatWithOr |
573+
NonterminalKind::Path |
574+
NonterminalKind::Literal
575+
))) => true,
561576
_ => false,
562577
}
563578
}
@@ -578,6 +593,10 @@ impl Token {
578593
Lt | BinOp(Shl) | // associated path
579594
PathSep => true, // global path
580595
Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)),
596+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
597+
NonterminalKind::Ty |
598+
NonterminalKind::Path
599+
))) => true,
581600
// For anonymous structs or unions, which only appear in specific positions
582601
// (type of struct fields or union fields), we don't consider them as regular types
583602
_ => false,
@@ -589,7 +608,12 @@ impl Token {
589608
match self.kind {
590609
OpenDelim(Delimiter::Brace) | Literal(..) | BinOp(Minus) => true,
591610
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
592-
Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
611+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
612+
NonterminalKind::Expr
613+
| NonterminalKind::Expr2021 { .. }
614+
| NonterminalKind::Block
615+
| NonterminalKind::Literal,
616+
))) => true,
593617
_ => false,
594618
}
595619
}
@@ -646,6 +670,10 @@ impl Token {
646670
},
647671
_ => false,
648672
},
673+
// njn: too simple compared to what's above?
674+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
675+
NonterminalKind::Literal | NonterminalKind::Expr | NonterminalKind::Expr2021 { .. },
676+
))) => true,
649677
_ => false,
650678
}
651679
}

‎compiler/rustc_parse/src/lexer/tokentrees.rs‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,19 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> {
4242
let mut buf = Vec::new();
4343
loop {
4444
match self.token.kind {
45-
token::OpenDelim(delim) => buf.push(match self.lex_token_tree_open_delim(delim) {
46-
Ok(val) => val,
47-
Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)),
48-
}),
45+
token::OpenDelim(delim) => {
46+
// Invisible delimiters cannot occur here because `TokenTreesReader` parses
47+
// code directly from strings, with no macro expansion involved.
48+
debug_assert!(!matches!(delim, Delimiter::Invisible(_)));
49+
buf.push(match self.lex_token_tree_open_delim(delim) {
50+
Ok(val) => val,
51+
Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)),
52+
})
53+
}
4954
token::CloseDelim(delim) => {
55+
// Invisible delimiters cannot occur here because `TokenTreesReader` parses
56+
// code directly from strings, with no macro expansion involved.
57+
debug_assert!(!matches!(delim, Delimiter::Invisible(_)));
5058
return (
5159
open_spacing,
5260
TokenStream::new(buf),

‎compiler/rustc_parse/src/parser/expr.rs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3572,11 +3572,19 @@ impl<'a> Parser<'a> {
35723572
&& !self.token.is_reserved_ident()
35733573
&& self.look_ahead(1, |t| {
35743574
AssocOp::from_token(t).is_some()
3575-
|| matches!(t.kind, token::OpenDelim(_))
3575+
|| matches!(
3576+
t.kind,
3577+
token::OpenDelim(
3578+
Delimiter::Parenthesis
3579+
| Delimiter::Bracket
3580+
| Delimiter::Brace
3581+
)
3582+
)
35763583
|| t.kind == token::Dot
35773584
})
35783585
{
3579-
// Looks like they tried to write a shorthand, complex expression.
3586+
// Looks like they tried to write a shorthand, complex expression,
3587+
// E.g.: `n + m`, `f(a)`, `a[i]`, `S { x: 3 }`, or `x.y`.
35803588
e.span_suggestion_verbose(
35813589
self.token.span.shrink_to_lo(),
35823590
"try naming a field",

‎compiler/rustc_parse/src/parser/nonterminal.rs‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
use rustc_ast::ptr::P;
2-
use rustc_ast::token::{self, Delimiter, Nonterminal::*, NonterminalKind, Token};
2+
use rustc_ast::token::{
3+
self, Delimiter, InvisibleOrigin, Nonterminal, Nonterminal::*, NonterminalKind, Token,
4+
};
35
use rustc_ast::HasTokens;
46
use rustc_ast_pretty::pprust;
57
use rustc_data_structures::sync::Lrc;
@@ -19,7 +21,31 @@ impl<'a> Parser<'a> {
1921
#[inline]
2022
pub fn nonterminal_may_begin_with(kind: NonterminalKind, token: &Token) -> bool {
2123
/// Checks whether the non-terminal may contain a single (non-keyword) identifier.
22-
fn may_be_ident(nt: &token::Nonterminal) -> bool {
24+
fn may_be_ident(kind: NonterminalKind) -> bool {
25+
use NonterminalKind::*;
26+
match kind {
27+
Stmt
28+
| PatParam { .. }
29+
| PatWithOr
30+
| Expr
31+
| Expr2021 { .. }
32+
| Ty
33+
| Literal // `true`, `false`
34+
| Meta
35+
| Path => true,
36+
37+
Item
38+
| Block
39+
| Vis => false,
40+
41+
Ident
42+
| Lifetime
43+
| TT => unreachable!(),
44+
}
45+
}
46+
47+
/// Old variant of `may_be_ident`. Being phased out.
48+
fn nt_may_be_ident(nt: &Nonterminal) -> bool {
2349
match nt {
2450
NtStmt(_)
2551
| NtPat(_)
@@ -57,7 +83,8 @@ impl<'a> Parser<'a> {
5783
| token::Ident(..)
5884
| token::NtIdent(..)
5985
| token::NtLifetime(..)
60-
| token::Interpolated(_) => true,
86+
| token::Interpolated(_)
87+
| token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => true,
6188
_ => token.can_begin_type(),
6289
},
6390
NonterminalKind::Block => match &token.kind {
@@ -67,11 +94,31 @@ impl<'a> Parser<'a> {
6794
NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
6895
NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false,
6996
},
97+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
98+
NonterminalKind::Block
99+
| NonterminalKind::Stmt
100+
| NonterminalKind::Expr
101+
| NonterminalKind::Expr2021 { .. }
102+
| NonterminalKind::Literal => true,
103+
NonterminalKind::Item
104+
| NonterminalKind::PatParam { .. }
105+
| NonterminalKind::PatWithOr
106+
| NonterminalKind::Ty
107+
| NonterminalKind::Meta
108+
| NonterminalKind::Path
109+
| NonterminalKind::Vis => false,
110+
NonterminalKind::Lifetime | NonterminalKind::Ident | NonterminalKind::TT => {
111+
unreachable!()
112+
}
113+
},
70114
_ => false,
71115
},
72116
NonterminalKind::Path | NonterminalKind::Meta => match &token.kind {
73117
token::PathSep | token::Ident(..) | token::NtIdent(..) => true,
74-
token::Interpolated(nt) => may_be_ident(nt),
118+
token::Interpolated(nt) => nt_may_be_ident(nt),
119+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
120+
may_be_ident(*kind)
121+
}
75122
_ => false,
76123
},
77124
NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => match &token.kind {
@@ -90,7 +137,10 @@ impl<'a> Parser<'a> {
90137
token::BinOp(token::Shl) => true, // path (double UFCS)
91138
// leading vert `|` or-pattern
92139
token::BinOp(token::Or) => matches!(kind, NonterminalKind::PatWithOr),
93-
token::Interpolated(nt) => may_be_ident(nt),
140+
token::Interpolated(nt) => nt_may_be_ident(nt),
141+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
142+
may_be_ident(*kind)
143+
}
94144
_ => false,
95145
},
96146
NonterminalKind::Lifetime => match &token.kind {

0 commit comments

Comments
 (0)