Skip to content

Commit 860b041

Browse files
nnethercotennethercote
committed
Prepare for invisible delimiters.
Current places where `Interpolated` is used are going to change to instead use invisible delimiters. This prepares for that. - It adds invisible delimiter cases to the `can_begin_*`/`may_be_*` methods that are equivalent to the existing `Interpolated` cases. - It adds panics/asserts in some places where invisible delimiters should never occur. - In `Parser::parse_struct_fields` it excludes an ident + invisible delimiter from special consideration in an error message, because that's quite different to an ident + paren/brace/bracket.
1 parent c5bccfd commit 860b041

File tree

4 files changed

+97
-10
lines changed

4 files changed

+97
-10
lines changed

‎compiler/rustc_ast/src/token.rs‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -512,10 +512,11 @@ impl Token {
512512

513513
/// Returns `true` if the token can appear at the start of an expression.
514514
pub fn can_begin_expr(&self) -> bool {
515+
use Delimiter::*;
515516
match self.uninterpolate().kind {
516517
Ident(name, is_raw) =>
517518
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
518-
OpenDelim(..) | // tuple, array or block
519+
OpenDelim(Parenthesis | Brace | Bracket) | // tuple, array or block
519520
Literal(..) | // literal
520521
Not | // operator not
521522
BinOp(Minus) | // unary minus
@@ -526,13 +527,19 @@ impl Token {
526527
// DotDotDot is no longer supported, but we need some way to display the error
527528
DotDot | DotDotDot | DotDotEq | // range notation
528529
Lt | BinOp(Shl) | // associated path
529-
PathSep | // global path
530+
PathSep | // global path
530531
Lifetime(..) | // labeled loop
531532
Pound => true, // expression attributes
532533
Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) |
533534
NtExpr(..) |
534535
NtBlock(..) |
535536
NtPath(..)),
537+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
538+
NonterminalKind::Block |
539+
NonterminalKind::Expr |
540+
NonterminalKind::Literal |
541+
NonterminalKind::Path
542+
))) => true,
536543
_ => false,
537544
}
538545
}
@@ -552,11 +559,18 @@ impl Token {
552559
// DotDotDot is no longer supported
553560
| DotDot | DotDotDot | DotDotEq // ranges
554561
| Lt | BinOp(Shl) // associated path
555-
| PathSep => true, // global path
562+
| PathSep => true, // global path
556563
Interpolated(ref nt) => matches!(&**nt, NtLiteral(..) |
557564
NtPat(..) |
558565
NtBlock(..) |
559566
NtPath(..)),
567+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
568+
NonterminalKind::Block |
569+
NonterminalKind::PatParam { .. } |
570+
NonterminalKind::PatWithOr |
571+
NonterminalKind::Path |
572+
NonterminalKind::Literal
573+
))) => true,
560574
_ => false,
561575
}
562576
}
@@ -577,6 +591,10 @@ impl Token {
577591
Lt | BinOp(Shl) | // associated path
578592
PathSep => true, // global path
579593
Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)),
594+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
595+
NonterminalKind::Ty |
596+
NonterminalKind::Path
597+
))) => true,
580598
// For anonymous structs or unions, which only appear in specific positions
581599
// (type of struct fields or union fields), we don't consider them as regular types
582600
_ => false,
@@ -588,6 +606,9 @@ impl Token {
588606
match self.kind {
589607
OpenDelim(Delimiter::Brace) => true,
590608
Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
609+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
610+
NonterminalKind::Expr | NonterminalKind::Block | NonterminalKind::Literal,
611+
))) => true,
591612
_ => self.can_begin_literal_maybe_minus(),
592613
}
593614
}
@@ -643,6 +664,10 @@ impl Token {
643664
},
644665
_ => false,
645666
},
667+
// njn: too simple compared to what's above?
668+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
669+
NonterminalKind::Literal | NonterminalKind::Expr,
670+
))) => true,
646671
_ => false,
647672
}
648673
}

‎compiler/rustc_parse/src/lexer/tokentrees.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,18 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> {
4343
loop {
4444
match self.token.kind {
4545
token::OpenDelim(delim) => {
46+
// Invisible delimiters cannot occur here because `TokenTreesReader` parses
47+
// code directly from strings, with no macro expansion involved.
48+
debug_assert!(!matches!(delim, Delimiter::Invisible(_)));
4649
buf.push(match self.parse_token_tree_open_delim(delim) {
4750
Ok(val) => val,
4851
Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)),
4952
})
5053
}
5154
token::CloseDelim(delim) => {
55+
// Invisible delimiters cannot occur here because `TokenTreesReader` parses
56+
// code directly from strings, with no macro expansion involved.
57+
debug_assert!(!matches!(delim, Delimiter::Invisible(_)));
5258
return (
5359
open_spacing,
5460
TokenStream::new(buf),

‎compiler/rustc_parse/src/parser/expr.rs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3590,11 +3590,19 @@ impl<'a> Parser<'a> {
35903590
&& !self.token.is_reserved_ident()
35913591
&& self.look_ahead(1, |t| {
35923592
AssocOp::from_token(t).is_some()
3593-
|| matches!(t.kind, token::OpenDelim(_))
3593+
|| matches!(
3594+
t.kind,
3595+
token::OpenDelim(
3596+
Delimiter::Parenthesis
3597+
| Delimiter::Bracket
3598+
| Delimiter::Brace
3599+
)
3600+
)
35943601
|| t.kind == token::Dot
35953602
})
35963603
{
3597-
// Looks like they tried to write a shorthand, complex expression.
3604+
// Looks like they tried to write a shorthand, complex expression,
3605+
// E.g.: `n + m`, `f(a)`, `a[i]`, `S { x: 3 }`, or `x.y`.
35983606
e.span_suggestion_verbose(
35993607
self.token.span.shrink_to_lo(),
36003608
"try naming a field",

‎compiler/rustc_parse/src/parser/nonterminal.rs‎

Lines changed: 53 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
use rustc_ast::ptr::P;
2-
use rustc_ast::token::{self, Delimiter, Nonterminal::*, NonterminalKind, Token};
2+
use rustc_ast::token::{
3+
self, Delimiter, InvisibleOrigin, Nonterminal, Nonterminal::*, NonterminalKind, Token,
4+
};
35
use rustc_ast::HasTokens;
46
use rustc_ast_pretty::pprust;
57
use rustc_data_structures::sync::Lrc;
@@ -19,7 +21,30 @@ impl<'a> Parser<'a> {
1921
#[inline]
2022
pub fn nonterminal_may_begin_with(kind: NonterminalKind, token: &Token) -> bool {
2123
/// Checks whether the non-terminal may contain a single (non-keyword) identifier.
22-
fn may_be_ident(nt: &token::Nonterminal) -> bool {
24+
fn may_be_ident(kind: NonterminalKind) -> bool {
25+
use NonterminalKind::*;
26+
match kind {
27+
Stmt
28+
| PatParam { .. }
29+
| PatWithOr
30+
| Expr
31+
| Ty
32+
| Literal // `true`, `false`
33+
| Meta
34+
| Path => true,
35+
36+
Item
37+
| Block
38+
| Vis => false,
39+
40+
Ident
41+
| Lifetime
42+
| TT => unreachable!(),
43+
}
44+
}
45+
46+
/// Old variant of `may_be_ident`. Being phased out.
47+
fn nt_may_be_ident(nt: &Nonterminal) -> bool {
2348
match nt {
2449
NtStmt(_)
2550
| NtPat(_)
@@ -52,7 +77,8 @@ impl<'a> Parser<'a> {
5277
| token::Ident(..)
5378
| token::NtIdent(..)
5479
| token::NtLifetime(..)
55-
| token::Interpolated(_) => true,
80+
| token::Interpolated(_)
81+
| token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => true,
5682
_ => token.can_begin_type(),
5783
},
5884
NonterminalKind::Block => match &token.kind {
@@ -62,11 +88,30 @@ impl<'a> Parser<'a> {
6288
NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
6389
NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false,
6490
},
91+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
92+
NonterminalKind::Block
93+
| NonterminalKind::Stmt
94+
| NonterminalKind::Expr
95+
| NonterminalKind::Literal => true,
96+
NonterminalKind::Item
97+
| NonterminalKind::PatParam { .. }
98+
| NonterminalKind::PatWithOr
99+
| NonterminalKind::Ty
100+
| NonterminalKind::Meta
101+
| NonterminalKind::Path
102+
| NonterminalKind::Vis => false,
103+
NonterminalKind::Lifetime | NonterminalKind::Ident | NonterminalKind::TT => {
104+
unreachable!()
105+
}
106+
},
65107
_ => false,
66108
},
67109
NonterminalKind::Path | NonterminalKind::Meta => match &token.kind {
68110
token::PathSep | token::Ident(..) | token::NtIdent(..) => true,
69-
token::Interpolated(nt) => may_be_ident(nt),
111+
token::Interpolated(nt) => nt_may_be_ident(nt),
112+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
113+
may_be_ident(*kind)
114+
}
70115
_ => false,
71116
},
72117
NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => match &token.kind {
@@ -85,7 +130,10 @@ impl<'a> Parser<'a> {
85130
token::BinOp(token::Shl) => true, // path (double UFCS)
86131
// leading vert `|` or-pattern
87132
token::BinOp(token::Or) => matches!(kind, NonterminalKind::PatWithOr),
88-
token::Interpolated(nt) => may_be_ident(nt),
133+
token::Interpolated(nt) => nt_may_be_ident(nt),
134+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
135+
may_be_ident(*kind)
136+
}
89137
_ => false,
90138
},
91139
NonterminalKind::Lifetime => match &token.kind {

0 commit comments

Comments
 (0)