Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 9625b10

Browse files
nnethercotecompiler-errors
nnethercote
authored and
compiler-errors
committed
Prepare for invisible delimiters.
Current places where `Interpolated` is used are going to change to instead use invisible delimiters. This prepares for that. - It adds invisible delimiter cases to the `can_begin_*`/`may_be_*` methods and the `failed_to_match_macro` that are equivalent to the existing `Interpolated` cases. - It adds panics/asserts in some places where invisible delimiters should never occur. - In `Parser::parse_struct_fields` it excludes an ident + invisible delimiter from special consideration in an error message, because that's quite different to an ident + paren/brace/bracket.
1 parent 4ce6956 commit 9625b10

File tree

5 files changed

+109
-14
lines changed

5 files changed

+109
-14
lines changed

‎compiler/rustc_ast/src/token.rs‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -592,10 +592,11 @@ impl Token {
592592
/// **NB**: Take care when modifying this function, since it will change
593593
/// the stable set of tokens that are allowed to match an expr nonterminal.
594594
pub fn can_begin_expr(&self) -> bool {
595+
use Delimiter::*;
595596
match self.uninterpolate().kind {
596597
Ident(name, is_raw) =>
597598
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
598-
OpenDelim(..) | // tuple, array or block
599+
OpenDelim(Parenthesis | Brace | Bracket) | // tuple, array or block
599600
Literal(..) | // literal
600601
Not | // operator not
601602
BinOp(Minus) | // unary minus
@@ -606,7 +607,7 @@ impl Token {
606607
// DotDotDot is no longer supported, but we need some way to display the error
607608
DotDot | DotDotDot | DotDotEq | // range notation
608609
Lt | BinOp(Shl) | // associated path
609-
PathSep | // global path
610+
PathSep | // global path
610611
Lifetime(..) | // labeled loop
611612
Pound => true, // expression attributes
612613
Interpolated(ref nt) =>
@@ -616,6 +617,12 @@ impl Token {
616617
NtLiteral(..) |
617618
NtPath(..)
618619
),
620+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
621+
MetaVarKind::Block |
622+
MetaVarKind::Expr { .. } |
623+
MetaVarKind::Literal |
624+
MetaVarKind::Path
625+
))) => true,
619626
_ => false,
620627
}
621628
}
@@ -649,6 +656,14 @@ impl Token {
649656
| NtPath(..)
650657
| NtTy(..)
651658
),
659+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
660+
MetaVarKind::Expr { .. } |
661+
MetaVarKind::Literal |
662+
MetaVarKind::Meta |
663+
MetaVarKind::Pat(_) |
664+
MetaVarKind::Path |
665+
MetaVarKind::Ty
666+
))) => true,
652667
_ => false,
653668
}
654669
}
@@ -669,6 +684,10 @@ impl Token {
669684
Lt | BinOp(Shl) | // associated path
670685
PathSep => true, // global path
671686
Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)),
687+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
688+
MetaVarKind::Ty |
689+
MetaVarKind::Path
690+
))) => true,
672691
// For anonymous structs or unions, which only appear in specific positions
673692
// (type of struct fields or union fields), we don't consider them as regular types
674693
_ => false,
@@ -681,6 +700,9 @@ impl Token {
681700
OpenDelim(Delimiter::Brace) | Literal(..) | BinOp(Minus) => true,
682701
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
683702
Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
703+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
704+
MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
705+
))) => true,
684706
_ => false,
685707
}
686708
}
@@ -737,6 +759,13 @@ impl Token {
737759
},
738760
_ => false,
739761
},
762+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
763+
MetaVarKind::Literal => true,
764+
MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => {
765+
can_begin_literal_maybe_minus
766+
}
767+
_ => false,
768+
},
740769
_ => false,
741770
}
742771
}
@@ -752,6 +781,11 @@ impl Token {
752781
},
753782
_ => false,
754783
},
784+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
785+
MetaVarKind::Literal => true,
786+
MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal,
787+
_ => false,
788+
},
755789
_ => false,
756790
}
757791
}

‎compiler/rustc_expand/src/mbe/diagnostics.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use std::borrow::Cow;
22

3-
use rustc_ast::token::{self, Token, TokenKind};
3+
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
44
use rustc_ast::tokenstream::TokenStream;
55
use rustc_errors::{Applicability, Diag, DiagCtxtHandle, DiagMessage};
66
use rustc_macros::Subdiagnostic;
@@ -68,7 +68,9 @@ pub(super) fn failed_to_match_macro(
6868

6969
if let MatcherLoc::Token { token: expected_token } = &remaining_matcher
7070
&& (matches!(expected_token.kind, TokenKind::Interpolated(_))
71-
|| matches!(token.kind, TokenKind::Interpolated(_)))
71+
|| matches!(token.kind, TokenKind::Interpolated(_))
72+
|| matches!(expected_token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_)))
73+
|| matches!(token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_))))
7274
{
7375
err.note("captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens");
7476
err.note("see <https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment> for more information");

‎compiler/rustc_parse/src/lexer/tokentrees.rs‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,19 @@ impl<'psess, 'src> TokenTreesReader<'psess, 'src> {
4343
let mut buf = Vec::new();
4444
loop {
4545
match self.token.kind {
46-
token::OpenDelim(delim) => buf.push(match self.lex_token_tree_open_delim(delim) {
47-
Ok(val) => val,
48-
Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)),
49-
}),
46+
token::OpenDelim(delim) => {
47+
// Invisible delimiters cannot occur here because `TokenTreesReader` parses
48+
// code directly from strings, with no macro expansion involved.
49+
debug_assert!(!matches!(delim, Delimiter::Invisible(_)));
50+
buf.push(match self.lex_token_tree_open_delim(delim) {
51+
Ok(val) => val,
52+
Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)),
53+
})
54+
}
5055
token::CloseDelim(delim) => {
56+
// Invisible delimiters cannot occur here because `TokenTreesReader` parses
57+
// code directly from strings, with no macro expansion involved.
58+
debug_assert!(!matches!(delim, Delimiter::Invisible(_)));
5159
return (
5260
open_spacing,
5361
TokenStream::new(buf),

‎compiler/rustc_parse/src/parser/expr.rs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3582,11 +3582,19 @@ impl<'a> Parser<'a> {
35823582
&& !self.token.is_reserved_ident()
35833583
&& self.look_ahead(1, |t| {
35843584
AssocOp::from_token(t).is_some()
3585-
|| matches!(t.kind, token::OpenDelim(_))
3585+
|| matches!(
3586+
t.kind,
3587+
token::OpenDelim(
3588+
Delimiter::Parenthesis
3589+
| Delimiter::Bracket
3590+
| Delimiter::Brace
3591+
)
3592+
)
35863593
|| *t == token::Dot
35873594
})
35883595
{
3589-
// Looks like they tried to write a shorthand, complex expression.
3596+
// Looks like they tried to write a shorthand, complex expression,
3597+
// E.g.: `n + m`, `f(a)`, `a[i]`, `S { x: 3 }`, or `x.y`.
35903598
e.span_suggestion_verbose(
35913599
self.token.span.shrink_to_lo(),
35923600
"try naming a field",

‎compiler/rustc_parse/src/parser/nonterminal.rs‎

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ use rustc_ast::ptr::P;
22
use rustc_ast::token::Nonterminal::*;
33
use rustc_ast::token::NtExprKind::*;
44
use rustc_ast::token::NtPatKind::*;
5-
use rustc_ast::token::{self, Delimiter, NonterminalKind, Token};
5+
use rustc_ast::token::{
6+
self, Delimiter, InvisibleOrigin, MetaVarKind, Nonterminal, NonterminalKind, Token,
7+
};
68
use rustc_ast::HasTokens;
79
use rustc_ast_pretty::pprust;
810
use rustc_data_structures::sync::Lrc;
@@ -22,7 +24,29 @@ impl<'a> Parser<'a> {
2224
#[inline]
2325
pub fn nonterminal_may_begin_with(kind: NonterminalKind, token: &Token) -> bool {
2426
/// Checks whether the non-terminal may contain a single (non-keyword) identifier.
25-
fn may_be_ident(nt: &token::Nonterminal) -> bool {
27+
fn may_be_ident(kind: MetaVarKind) -> bool {
28+
use MetaVarKind::*;
29+
match kind {
30+
Stmt
31+
| Pat(_)
32+
| Expr { .. }
33+
| Ty
34+
| Literal // `true`, `false`
35+
| Meta
36+
| Path => true,
37+
38+
Item
39+
| Block
40+
| Vis => false,
41+
42+
Ident
43+
| Lifetime
44+
| TT => unreachable!(),
45+
}
46+
}
47+
48+
/// Old variant of `may_be_ident`. Being phased out.
49+
fn nt_may_be_ident(nt: &Nonterminal) -> bool {
2650
match nt {
2751
NtStmt(_)
2852
| NtPat(_)
@@ -69,7 +93,8 @@ impl<'a> Parser<'a> {
6993
| token::Ident(..)
7094
| token::NtIdent(..)
7195
| token::NtLifetime(..)
72-
| token::Interpolated(_) => true,
96+
| token::Interpolated(_)
97+
| token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => true,
7398
_ => token.can_begin_type(),
7499
},
75100
NonterminalKind::Block => match &token.kind {
@@ -79,11 +104,29 @@ impl<'a> Parser<'a> {
79104
NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
80105
NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false,
81106
},
107+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
108+
MetaVarKind::Block
109+
| MetaVarKind::Stmt
110+
| MetaVarKind::Expr { .. }
111+
| MetaVarKind::Literal => true,
112+
MetaVarKind::Item
113+
| MetaVarKind::Pat(_)
114+
| MetaVarKind::Ty
115+
| MetaVarKind::Meta
116+
| MetaVarKind::Path
117+
| MetaVarKind::Vis => false,
118+
MetaVarKind::Lifetime | MetaVarKind::Ident | MetaVarKind::TT => {
119+
unreachable!()
120+
}
121+
},
82122
_ => false,
83123
},
84124
NonterminalKind::Path | NonterminalKind::Meta => match &token.kind {
85125
token::PathSep | token::Ident(..) | token::NtIdent(..) => true,
86-
token::Interpolated(nt) => may_be_ident(nt),
126+
token::Interpolated(nt) => nt_may_be_ident(nt),
127+
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
128+
may_be_ident(*kind)
129+
}
87130
_ => false,
88131
},
89132
NonterminalKind::Pat(pat_kind) => token.can_begin_pattern(pat_kind),

0 commit comments

Comments
 (0)