@@ -42,11 +42,87 @@ pub enum BinOpToken {
4242 Shr ,
4343}
4444
45+ // This type must not implement `Hash` due to the unusual `PartialEq` impl below.
46+ #[ derive( Copy , Clone , Debug , Encodable , Decodable , HashStable_Generic ) ]
47+ pub enum InvisibleOrigin {
48+ // From the expansion of a metavariable in a declarative macro.
49+ MetaVar ( MetaVarKind ) ,
50+
51+ // Converted from `proc_macro::Delimiter` in
52+ // `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
53+ ProcMacro ,
54+
55+ // Converted from `TokenKind::Interpolated` in
56+ // `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
57+ FlattenToken ,
58+ }
59+
60+ impl PartialEq for InvisibleOrigin {
61+ #[ inline]
62+ fn eq ( & self , _other : & InvisibleOrigin ) -> bool {
63+ // When we had AST-based nonterminals we couldn't compare them, and the
64+ // old `Nonterminal` type had an `eq` that always returned false,
65+ // resulting in this restriction:
66+ // https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment
67+ // This `eq` emulates that behaviour. We could consider lifting this
68+ // restriction now but there are still cases involving invisible
69+ // delimiters that make it harder than it first appears.
70+ false
71+ }
72+ }
73+
74+ /// Annoyingly similar to `NonterminalKind`, but the slight differences are important.
75+ #[ derive( Debug , Copy , Clone , PartialEq , Eq , Encodable , Decodable , Hash , HashStable_Generic ) ]
76+ pub enum MetaVarKind {
77+ Item ,
78+ Block ,
79+ Stmt ,
80+ Pat ( NtPatKind ) ,
81+ Expr {
82+ kind : NtExprKind ,
83+ // This field is needed for `Token::can_begin_literal_maybe_minus`.
84+ can_begin_literal_maybe_minus : bool ,
85+ // This field is needed for `Token::can_begin_string_literal`.
86+ can_begin_string_literal : bool ,
87+ } ,
88+ Ty ,
89+ Ident ,
90+ Lifetime ,
91+ Literal ,
92+ Meta ,
93+ Path ,
94+ Vis ,
95+ TT ,
96+ }
97+
98+ impl fmt:: Display for MetaVarKind {
99+ fn fmt ( & self , f : & mut fmt:: Formatter < ' _ > ) -> fmt:: Result {
100+ use MetaVarKind :: * ;
101+ let sym = match self {
102+ Item => sym:: item,
103+ Block => sym:: block,
104+ Stmt => sym:: stmt,
105+ Pat ( PatParam { inferred : true } | PatWithOr ) => sym:: pat,
106+ Pat ( PatParam { inferred : false } ) => sym:: pat_param,
107+ Expr { kind : Expr2021 { inferred : true } | Expr , .. } => sym:: expr,
108+ Expr { kind : Expr2021 { inferred : false } , .. } => sym:: expr_2021,
109+ Ty => sym:: ty,
110+ Ident => sym:: ident,
111+ Lifetime => sym:: lifetime,
112+ Literal => sym:: literal,
113+ Meta => sym:: meta,
114+ Path => sym:: path,
115+ Vis => sym:: vis,
116+ TT => sym:: tt,
117+ } ;
118+ write ! ( f, "{}" , sym)
119+ }
120+ }
121+
45122/// Describes how a sequence of token trees is delimited.
46123/// Cannot use `proc_macro::Delimiter` directly because this
47124/// structure should implement some additional traits.
48- #[ derive( Copy , Clone , Debug , PartialEq , Eq ) ]
49- #[ derive( Encodable , Decodable , Hash , HashStable_Generic ) ]
125+ #[ derive( Copy , Clone , Debug , PartialEq , Encodable , Decodable , HashStable_Generic ) ]
50126pub enum Delimiter {
51127 /// `( ... )`
52128Parenthesis ,
@@ -59,7 +135,33 @@ pub enum Delimiter {
59135/// "macro variable" `$var`. It is important to preserve operator priorities in cases like
60136/// `$var * 3` where `$var` is `1 + 2`.
61137/// Invisible delimiters might not survive roundtrip of a token stream through a string.
62- Invisible ,
138+ Invisible ( InvisibleOrigin ) ,
139+ }
140+
141+ impl Delimiter {
142+ // Should the parser skip these delimiters? Only happens for certain kinds
143+ // of invisible delimiters. Ideally this function will eventually disappear
144+ // and no invisible delimiters will be skipped.
145+ #[ inline]
146+ pub fn skip ( & self ) -> bool {
147+ use InvisibleOrigin :: * ;
148+ match self {
149+ Delimiter :: Parenthesis | Delimiter :: Bracket | Delimiter :: Brace => false ,
150+ Delimiter :: Invisible ( MetaVar ( _) ) => false ,
151+ Delimiter :: Invisible ( FlattenToken | ProcMacro ) => true ,
152+ }
153+ }
154+
155+ // This exists because `InvisibleOrigin`s should be compared. It is only used for assertions.
156+ pub fn eq_ignoring_invisible_origin ( & self , other : & Delimiter ) -> bool {
157+ match ( self , other) {
158+ ( Delimiter :: Parenthesis , Delimiter :: Parenthesis ) => true ,
159+ ( Delimiter :: Brace , Delimiter :: Brace ) => true ,
160+ ( Delimiter :: Bracket , Delimiter :: Bracket ) => true ,
161+ ( Delimiter :: Invisible ( _) , Delimiter :: Invisible ( _) ) => true ,
162+ _ => false ,
163+ }
164+ }
63165}
64166
65167// Note that the suffix is *not* considered when deciding the `LitKind` in this
@@ -890,7 +992,7 @@ impl PartialEq<TokenKind> for Token {
890992 }
891993}
892994
893- #[ derive( Debug , Copy , Clone , PartialEq , Eq , Encodable , Decodable ) ]
995+ #[ derive( Debug , Copy , Clone , PartialEq , Eq , Encodable , Decodable , Hash , HashStable_Generic ) ]
894996pub enum NtPatKind {
895997 // Matches or-patterns. Was written using `pat` in edition 2021 or later.
896998 PatWithOr ,
@@ -900,7 +1002,7 @@ pub enum NtPatKind {
9001002 PatParam { inferred : bool } ,
9011003}
9021004
903- #[ derive( Debug , Copy , Clone , PartialEq , Eq , Encodable , Decodable ) ]
1005+ #[ derive( Debug , Copy , Clone , PartialEq , Eq , Encodable , Decodable , Hash , HashStable_Generic ) ]
9041006pub enum NtExprKind {
9051007 // Matches expressions using the post-edition 2024. Was written using
9061008 // `expr` in edition 2024 or later.
@@ -927,7 +1029,7 @@ pub enum Nonterminal {
9271029 NtVis ( P < ast:: Visibility > ) ,
9281030}
9291031
930- #[ derive( Debug , Copy , Clone , PartialEq , Encodable , Decodable ) ]
1032+ #[ derive( Debug , Copy , Clone , PartialEq , Eq , Encodable , Decodable , Hash , HashStable_Generic ) ]
9311033pub enum NonterminalKind {
9321034 Item ,
9331035 Block ,
0 commit comments