Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 0016595

Browse files
nnethercotennethercote
committed
XXX: finalizing
1 parent 5bc32bb commit 0016595

File tree

14 files changed

+367
-343
lines changed

14 files changed

+367
-343
lines changed

‎compiler/rustc_ast/src/attr/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use crate::ast::{DelimArgs, Expr, ExprKind, LitKind, MetaItemLit};
77
use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem, NormalAttr};
88
use crate::ast::{Path, PathSegment, DUMMY_NODE_ID};
99
use crate::ptr::P;
10-
use crate::token::{self, CommentKind, Delimiter, InvisibleOrigin, NonterminalKind, Token};
10+
use crate::token::{self, CommentKind, Delimiter, InvisibleOrigin, MetaVarKind, Token};
1111
use crate::tokenstream::{DelimSpan, Spacing, TokenTree};
1212
use crate::tokenstream::{LazyAttrTokenStream, TokenStream};
1313
use crate::util::comments;
@@ -368,7 +368,7 @@ impl MetaItem {
368368
_span,
369369
_spacing,
370370
Delimiter::Invisible(InvisibleOrigin::MetaVar(
371-
NonterminalKind::Meta | NonterminalKind::Path,
371+
MetaVarKind::Meta | MetaVarKind::Path,
372372
)),
373373
_stream,
374374
)) => {

‎compiler/rustc_ast/src/token.rs‎

Lines changed: 84 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ pub enum BinOpToken {
4040
#[derive(Copy, Clone, Debug, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
4141
pub enum InvisibleOrigin {
4242
// From the expansion of a metavariable in a declarative macro.
43-
MetaVar(NonterminalKind),
43+
MetaVar(MetaVarKind),
4444

4545
// Converted from `proc_macro::Delimiter` in
4646
// `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
@@ -51,6 +51,54 @@ pub enum InvisibleOrigin {
5151
FlattenToken,
5252
}
5353

54+
/// Annoyingly similar to `NonterminalKind`, but the slightly differences are important.
55+
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
56+
pub enum MetaVarKind {
57+
Item,
58+
Block,
59+
Stmt,
60+
Pat(NtPatKind),
61+
Expr {
62+
kind: NtExprKind,
63+
// This field is needed for `Token::can_begin_literal_maybe_minus`.
64+
can_begin_literal_maybe_minus: bool,
65+
// This field is needed for `Token::can_begin_string_literal`.
66+
can_begin_string_literal: bool,
67+
},
68+
Ty,
69+
Ident,
70+
Lifetime,
71+
Literal,
72+
Meta,
73+
Path,
74+
Vis,
75+
TT,
76+
}
77+
78+
impl fmt::Display for MetaVarKind {
79+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
80+
use MetaVarKind::*;
81+
let sym = match self {
82+
Item => sym::item,
83+
Block => sym::block,
84+
Stmt => sym::stmt,
85+
Pat(PatParam { inferred: true } | PatWithOr) => sym::pat,
86+
Pat(PatParam { inferred: false }) => sym::pat_param,
87+
Expr { kind: Expr2021 { inferred: true } | Expr, .. } => sym::expr,
88+
Expr { kind: Expr2021 { inferred: false }, .. } => sym::expr_2021,
89+
Ty => sym::ty,
90+
Ident => sym::ident,
91+
Lifetime => sym::lifetime,
92+
Literal => sym::literal,
93+
Meta => sym::meta,
94+
Path => sym::path,
95+
Vis => sym::vis,
96+
TT => sym::tt,
97+
};
98+
write!(f, "{}", sym)
99+
}
100+
}
101+
54102
/// Describes how a sequence of token trees is delimited.
55103
/// Cannot use `proc_macro::Delimiter` directly because this
56104
/// structure should implement some additional traits.
@@ -138,15 +186,17 @@ impl Lit {
138186
match token.uninterpolate().kind {
139187
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
140188
Literal(token_lit) => Some(token_lit),
141-
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(NonterminalKind::Literal))) => {
189+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(MetaVarKind::Literal))) => {
142190
panic!("njn: FROM_TOKEN (1)");
143191
// if let NtExpr(expr) | NtLiteral(expr) = &**nt
144192
// && let ast::ExprKind::Lit(token_lit) = expr.kind =>
145193
// {
146194
// Some(token_lit)
147195
// }
148196
}
149-
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(NonterminalKind::Expr(_)))) => {
197+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(MetaVarKind::Expr {
198+
..
199+
}))) => {
150200
panic!("njn: FROM_TOKEN (2)");
151201
// if let NtExpr(expr) | NtLiteral(expr) = &**nt
152202
// && let ast::ExprKind::Lit(token_lit) = expr.kind =>
@@ -511,10 +561,10 @@ impl Token {
511561
Lifetime(..) | // labeled loop
512562
Pound => true, // expression attributes
513563
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
514-
NonterminalKind::Block |
515-
NonterminalKind::Expr(_) |
516-
NonterminalKind::Literal |
517-
NonterminalKind::Path
564+
MetaVarKind::Block |
565+
MetaVarKind::Expr { .. } |
566+
MetaVarKind::Literal |
567+
MetaVarKind::Path
518568
))) => true,
519569
_ => false,
520570
}
@@ -537,10 +587,10 @@ impl Token {
537587
| Lt | BinOp(Shl) // associated path
538588
| PathSep => true, // global path
539589
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
540-
NonterminalKind::Block |
541-
NonterminalKind::Pat(_) |
542-
NonterminalKind::Path |
543-
NonterminalKind::Literal
590+
MetaVarKind::Block |
591+
MetaVarKind::Pat(_) |
592+
MetaVarKind::Path |
593+
MetaVarKind::Literal
544594
))) => true,
545595
_ => false,
546596
}
@@ -562,8 +612,8 @@ impl Token {
562612
Lt | BinOp(Shl) | // associated path
563613
PathSep => true, // global path
564614
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
565-
NonterminalKind::Ty |
566-
NonterminalKind::Path
615+
MetaVarKind::Ty |
616+
MetaVarKind::Path
567617
))) => true,
568618
// For anonymous structs or unions, which only appear in specific positions
569619
// (type of struct fields or union fields), we don't consider them as regular types
@@ -577,7 +627,7 @@ impl Token {
577627
OpenDelim(Delimiter::Brace) | Literal(..) | BinOp(Minus) => true,
578628
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
579629
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
580-
NonterminalKind::Expr(_) | NonterminalKind::Block | NonterminalKind::Literal,
630+
MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
581631
))) => true,
582632
_ => false,
583633
}
@@ -625,42 +675,25 @@ impl Token {
625675
match self.uninterpolate().kind {
626676
Literal(..) | BinOp(Minus) => true,
627677
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
628-
// njn: fix up
629-
// Interpolated(ref nt) => match &**nt {
630-
// NtLiteral(_) => true,
631-
// NtExpr(e) => match &e.kind {
632-
// ast::ExprKind::Lit(_) => true,
633-
// ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
634-
// matches!(&e.kind, ast::ExprKind::Lit(_))
635-
// }
636-
// _ => false,
637-
// },
638-
// _ => false,
639-
// },
640-
// njn: too simple compared to what's above?
641-
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
642-
NonterminalKind::Literal | NonterminalKind::Expr(_),
643-
))) => true,
678+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
679+
MetaVarKind::Literal => true,
680+
MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => {
681+
can_begin_literal_maybe_minus
682+
}
683+
_ => false,
684+
},
644685
_ => false,
645686
}
646687
}
647688

648689
pub fn can_begin_string_literal(&self) -> bool {
649690
match self.uninterpolate().kind {
650691
Literal(..) => true,
651-
// njn: fix up
652-
// Interpolated(ref nt) => match &**nt {
653-
// NtLiteral(_) => true,
654-
// NtExpr(e) => match &e.kind {
655-
// ast::ExprKind::Lit(_) => true,
656-
// _ => false,
657-
// },
658-
// _ => false,
659-
// },
660-
// njn: too simple compared to what's above?
661-
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
662-
NonterminalKind::Literal | NonterminalKind::Expr(_),
663-
))) => true,
692+
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
693+
MetaVarKind::Literal => true,
694+
MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal,
695+
_ => false,
696+
},
664697
_ => false,
665698
}
666699
}
@@ -715,24 +748,24 @@ impl Token {
715748
self.ident().is_some_and(|(ident, _)| ident.name == name)
716749
}
717750

718-
/// Would `maybe_reparse_metavar_expr` in `parser.rs` return `Ok(..)`?
751+
/// Would `eat_metavar_expr` in `parser.rs` return `Ok(..)`?
719752
/// That is, is this a pre-parsed expression dropped into the token stream
720753
/// (which happens while parsing the result of macro expansion)?
721754
pub fn is_metavar_expr(&self) -> bool {
722755
matches!(
723756
self.is_metavar_seq(),
724757
Some(
725-
NonterminalKind::Expr(_)
726-
| NonterminalKind::Literal
727-
| NonterminalKind::Path
728-
| NonterminalKind::Block
758+
MetaVarKind::Expr { .. }
759+
| MetaVarKind::Literal
760+
| MetaVarKind::Path
761+
| MetaVarKind::Block
729762
)
730763
)
731764
}
732765

733766
/// Are we at a block from a metavar (`$b:block`)?
734767
pub fn is_metavar_block(&self) -> bool {
735-
matches!(self.is_metavar_seq(), Some(NonterminalKind::Block))
768+
matches!(self.is_metavar_seq(), Some(MetaVarKind::Block))
736769
}
737770

738771
/// Returns `true` if the token is either the `mut` or `const` keyword.
@@ -747,7 +780,7 @@ impl Token {
747780
pub fn is_path_start(&self) -> bool {
748781
self == &PathSep
749782
|| self.is_qpath_start()
750-
|| matches!(self.is_metavar_seq(), Some(NonterminalKind::Path))
783+
|| matches!(self.is_metavar_seq(), Some(MetaVarKind::Path))
751784
|| self.is_path_segment_keyword()
752785
|| self.is_ident() && !self.is_reserved_ident()
753786
}
@@ -819,7 +852,7 @@ impl Token {
819852

820853
/// Is this an invisible open delimiter at the start of a token sequence
821854
/// from an expanded metavar?
822-
pub fn is_metavar_seq(&self) -> Option<NonterminalKind> {
855+
pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
823856
match self.kind {
824857
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind),
825858
_ => None,

‎compiler/rustc_builtin_macros/src/cfg_eval.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ impl CfgEval<'_, '_> {
169169
},
170170
Annotatable::Stmt(_) => |parser| {
171171
Ok(Annotatable::Stmt(P(parser
172-
.parse_stmt_without_recovery(false, ForceCollect::Yes)?
172+
.parse_stmt_without_recovery(false, ForceCollect::Yes, false)?
173173
.unwrap())))
174174
},
175175
Annotatable::Expr(_) => {

‎compiler/rustc_expand/src/base.rs‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use crate::module::DirOwnership;
55

66
use rustc_ast::attr::MarkedAttrs;
77
use rustc_ast::ptr::P;
8-
use rustc_ast::token::NonterminalKind;
8+
use rustc_ast::token::MetaVarKind;
99
use rustc_ast::tokenstream::TokenStream;
1010
use rustc_ast::visit::{AssocCtxt, Visitor};
1111
use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, Item, NodeId, PatKind};
@@ -15,7 +15,7 @@ use rustc_data_structures::sync::{self, Lrc};
1515
use rustc_errors::{DiagCtxtHandle, ErrorGuaranteed, PResult};
1616
use rustc_feature::Features;
1717
use rustc_lint_defs::{BufferedEarlyLint, RegisteredTools};
18-
use rustc_parse::parser::{ParseNtResult, Parser};
18+
use rustc_parse::parser::{ForceCollect, Parser};
1919
use rustc_parse::MACRO_ARGUMENTS;
2020
use rustc_session::config::CollapseMacroDebuginfo;
2121
use rustc_session::{parse::ParseSess, Limit, Session};
@@ -1425,24 +1425,27 @@ pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, psess: &
14251425
}
14261426

14271427
pub(crate) fn stream_pretty_printing_compatibility_hack(
1428-
kind: NonterminalKind,
1428+
kind: MetaVarKind,
14291429
stream: &TokenStream,
14301430
psess: &ParseSess,
14311431
) {
14321432
let item = match kind {
1433-
NonterminalKind::Item => {
1433+
MetaVarKind::Item => {
14341434
let mut parser = Parser::new(psess, stream.clone(), None);
1435-
let Ok(ParseNtResult::Item(item)) = parser.parse_nonterminal(kind) else {
1436-
panic!("failed to reparse");
1437-
};
1438-
item
1435+
// No need to collect tokens for this simple check.
1436+
parser
1437+
.parse_item(ForceCollect::No)
1438+
.expect("failed to reparse item")
1439+
.expect("an actual item")
14391440
}
1440-
NonterminalKind::Stmt => {
1441+
MetaVarKind::Stmt => {
14411442
// njn: reparsing and then checking for StmtKind::Item sucks, hmm
14421443
let mut parser = Parser::new(psess, stream.clone(), None);
1443-
let Ok(ParseNtResult::Stmt(stmt)) = parser.parse_nonterminal(kind) else {
1444-
panic!("failed to reparse");
1445-
};
1444+
// No need to collect tokens for this simple check.
1445+
let stmt = parser
1446+
.parse_stmt(ForceCollect::No)
1447+
.expect("failed to reparse")
1448+
.expect("an actual stmt");
14461449
match &stmt.kind {
14471450
ast::StmtKind::Item(item) => item.clone(),
14481451
_ => return,

0 commit comments

Comments
 (0)