Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit e07dc21

Browse files
borsbors
committed
Auto merge of rust-lang#125174 - nnethercote:less-ast-pretty-printing, r=<try>
Print `token::Interpolated` with token stream pretty printing. This is a step towards removing `token::Interpolated` (rust-lang#124141). It unavoidably changes the output of the `stringify!` macro, generally for the better. r? `@petrochenkov`
2 parents 8af67ba + 9c693ad commit e07dc21

29 files changed

+221
-312
lines changed

‎compiler/rustc_ast/src/tokenstream.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -661,11 +661,11 @@ impl TokenStream {
661661
if attr_style == AttrStyle::Inner {
662662
vec![
663663
TokenTree::token_joint(token::Pound, span),
664-
TokenTree::token_alone(token::Not, span),
664+
TokenTree::token_joint_hidden(token::Not, span),
665665
body,
666666
]
667667
} else {
668-
vec![TokenTree::token_alone(token::Pound, span), body]
668+
vec![TokenTree::token_joint_hidden(token::Pound, span), body]
669669
}
670670
}
671671
}

‎compiler/rustc_ast_pretty/src/pprust/state.rs‎

Lines changed: 13 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,10 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
524524
}
525525
}
526526

527-
fn peek_comment<'b>(&'b self) -> Option<&'b Comment> where 'a: 'b {
527+
fn peek_comment<'b>(&'b self) -> Option<&'b Comment>
528+
where
529+
'a: 'b,
530+
{
528531
self.comments().and_then(|c| c.peek())
529532
}
530533

@@ -849,18 +852,11 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
849852
}
850853

851854
fn nonterminal_to_string(&self, nt: &Nonterminal) -> String {
852-
match nt {
853-
token::NtExpr(e) => self.expr_to_string(e),
854-
token::NtMeta(e) => self.attr_item_to_string(e),
855-
token::NtTy(e) => self.ty_to_string(e),
856-
token::NtPath(e) => self.path_to_string(e),
857-
token::NtItem(e) => self.item_to_string(e),
858-
token::NtBlock(e) => self.block_to_string(e),
859-
token::NtStmt(e) => self.stmt_to_string(e),
860-
token::NtPat(e) => self.pat_to_string(e),
861-
token::NtLiteral(e) => self.expr_to_string(e),
862-
token::NtVis(e) => self.vis_to_string(e),
863-
}
855+
// We convert the AST fragment to a token stream and pretty print that,
856+
// rather than using AST pretty printing, because `Nonterminal` is
857+
// slated for removal in #124141. (This method will also then be
858+
// removed.)
859+
self.tts_to_string(&TokenStream::from_nonterminal_ast(nt))
864860
}
865861

866862
/// Print the token kind precisely, without converting `$crate` into its respective crate name.
@@ -994,6 +990,10 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
994990
Self::to_string(|s| s.print_attr_item(ai, ai.path.span))
995991
}
996992

993+
fn tts_to_string(&self, tokens: &TokenStream) -> String {
994+
Self::to_string(|s| s.print_tts(tokens, false))
995+
}
996+
997997
fn to_string(f: impl FnOnce(&mut State<'_>)) -> String {
998998
let mut printer = State::new();
999999
f(&mut printer);
@@ -2039,10 +2039,6 @@ impl<'a> State<'a> {
20392039
})
20402040
}
20412041

2042-
pub(crate) fn tts_to_string(&self, tokens: &TokenStream) -> String {
2043-
Self::to_string(|s| s.print_tts(tokens, false))
2044-
}
2045-
20462042
pub(crate) fn path_segment_to_string(&self, p: &ast::PathSegment) -> String {
20472043
Self::to_string(|s| s.print_path_segment(p, false))
20482044
}

‎compiler/rustc_builtin_macros/src/assert/context.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
153153
fn build_panic(&self, expr_str: &str, panic_path: Path) -> P<Expr> {
154154
let escaped_expr_str = escape_to_fmt(expr_str);
155155
let initial = [
156-
TokenTree::token_joint_hidden(
156+
TokenTree::token_joint(
157157
token::Literal(token::Lit {
158158
kind: token::LitKind::Str,
159159
symbol: Symbol::intern(&if self.fmt_string.is_empty() {
@@ -172,7 +172,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
172172
];
173173
let captures = self.capture_decls.iter().flat_map(|cap| {
174174
[
175-
TokenTree::token_joint_hidden(
175+
TokenTree::token_joint(
176176
token::Ident(cap.ident.name, IdentIsRaw::No),
177177
cap.ident.span,
178178
),

‎compiler/rustc_expand/src/mbe.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ mod transcribe;
1414

1515
use metavar_expr::MetaVarExpr;
1616
use rustc_ast::token::{Delimiter, NonterminalKind, Token, TokenKind};
17-
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan};
17+
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing};
1818
use rustc_macros::{Decodable, Encodable};
1919
use rustc_span::symbol::Ident;
2020
use rustc_span::Span;
@@ -68,7 +68,7 @@ pub(crate) enum KleeneOp {
6868
/// `MetaVarExpr` are "first-class" token trees. Useful for parsing macros.
6969
#[derive(Debug, PartialEq, Encodable, Decodable)]
7070
enum TokenTree {
71-
Token(Token),
71+
Token(Token, Spacing),
7272
/// A delimited sequence, e.g. `($e:expr)` (RHS) or `{ $e }` (LHS).
7373
Delimited(DelimSpan, DelimSpacing, Delimited),
7474
/// A kleene-style repetition sequence, e.g. `$($e:expr)*` (RHS) or `$($e),*` (LHS).
@@ -90,15 +90,15 @@ impl TokenTree {
9090
/// Returns `true` if the given token tree is a token of the given kind.
9191
fn is_token(&self, expected_kind: &TokenKind) -> bool {
9292
match self {
93-
TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind,
93+
TokenTree::Token(Token { kind: actual_kind, .. }, _) => actual_kind == expected_kind,
9494
_ => false,
9595
}
9696
}
9797

9898
/// Retrieves the `TokenTree`'s span.
9999
fn span(&self) -> Span {
100100
match *self {
101-
TokenTree::Token(Token { span, .. })
101+
TokenTree::Token(Token { span, .. }, _)
102102
| TokenTree::MetaVar(span, _)
103103
| TokenTree::MetaVarDecl(span, _, _) => span,
104104
TokenTree::Delimited(span, ..)
@@ -107,7 +107,7 @@ impl TokenTree {
107107
}
108108
}
109109

110-
fn token(kind: TokenKind, span: Span) -> TokenTree {
111-
TokenTree::Token(Token::new(kind, span))
110+
fn token_alone(kind: TokenKind, span: Span) -> TokenTree {
111+
TokenTree::Token(Token::new(kind, span), Spacing::Alone)
112112
}
113113
}

‎compiler/rustc_expand/src/mbe/macro_check.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -412,7 +412,7 @@ fn check_nested_occurrences(
412412
match (state, tt) {
413413
(
414414
NestedMacroState::Empty,
415-
&TokenTree::Token(Token { kind: TokenKind::Ident(name, IdentIsRaw::No), .. }),
415+
&TokenTree::Token(Token { kind: TokenKind::Ident(name, IdentIsRaw::No), .. }, _),
416416
) => {
417417
if name == kw::MacroRules {
418418
state = NestedMacroState::MacroRules;
@@ -422,13 +422,13 @@ fn check_nested_occurrences(
422422
}
423423
(
424424
NestedMacroState::MacroRules,
425-
&TokenTree::Token(Token { kind: TokenKind::Not, .. }),
425+
&TokenTree::Token(Token { kind: TokenKind::Not, .. }, _),
426426
) => {
427427
state = NestedMacroState::MacroRulesNot;
428428
}
429429
(
430430
NestedMacroState::MacroRulesNot,
431-
&TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }),
431+
&TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }, _),
432432
) => {
433433
state = NestedMacroState::MacroRulesNotName;
434434
}
@@ -459,7 +459,7 @@ fn check_nested_occurrences(
459459
}
460460
(
461461
NestedMacroState::Macro,
462-
&TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }),
462+
&TokenTree::Token(Token { kind: TokenKind::Ident(..), .. }, _),
463463
) => {
464464
state = NestedMacroState::MacroName;
465465
}

‎compiler/rustc_expand/src/mbe/macro_parser.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ pub(super) fn compute_locs(matcher: &[TokenTree]) -> Vec<MatcherLoc> {
180180
) {
181181
for tt in tts {
182182
match tt {
183-
TokenTree::Token(token) => {
183+
TokenTree::Token(token, _) => {
184184
locs.push(MatcherLoc::Token { token: token.clone() });
185185
}
186186
TokenTree::Delimited(span, _, delimited) => {

‎compiler/rustc_expand/src/mbe/macro_rules.rs‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use crate::mbe::transcribe::transcribe;
1111
use ast::token::IdentIsRaw;
1212
use rustc_ast as ast;
1313
use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind, TokenKind::*};
14-
use rustc_ast::tokenstream::{DelimSpan, TokenStream};
14+
use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream};
1515
use rustc_ast::{NodeId, DUMMY_NODE_ID};
1616
use rustc_ast_pretty::pprust;
1717
use rustc_attr::{self as attr, TransparencyError};
@@ -402,7 +402,7 @@ pub fn compile_declarative_macro(
402402
mbe::SequenceRepetition {
403403
tts: vec![
404404
mbe::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec),
405-
mbe::TokenTree::token(token::FatArrow, def.span),
405+
mbe::TokenTree::token_alone(token::FatArrow, def.span),
406406
mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec),
407407
],
408408
separator: Some(Token::new(
@@ -417,7 +417,7 @@ pub fn compile_declarative_macro(
417417
mbe::TokenTree::Sequence(
418418
DelimSpan::dummy(),
419419
mbe::SequenceRepetition {
420-
tts: vec![mbe::TokenTree::token(
420+
tts: vec![mbe::TokenTree::token_alone(
421421
if macro_rules { token::Semi } else { token::Comma },
422422
def.span,
423423
)],
@@ -627,10 +627,11 @@ fn is_empty_token_tree(sess: &Session, seq: &mbe::SequenceRepetition) -> bool {
627627
while let Some(tt) = iter.next() {
628628
match tt {
629629
mbe::TokenTree::MetaVarDecl(_, _, Some(NonterminalKind::Vis)) => {}
630-
mbe::TokenTree::Token(t @ Token { kind: DocComment(..), .. }) => {
630+
mbe::TokenTree::Token(t @ Token { kind: DocComment(..), .. }, _) => {
631631
let mut now = t;
632632
while let Some(&mbe::TokenTree::Token(
633633
next @ Token { kind: DocComment(..), .. },
634+
_,
634635
)) = iter.peek()
635636
{
636637
now = next;
@@ -696,10 +697,10 @@ fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
696697
match rhs {
697698
mbe::TokenTree::Delimited(.., d) => {
698699
let has_compile_error = d.tts.array_windows::<3>().any(|[ident, bang, args]| {
699-
if let mbe::TokenTree::Token(ident) = ident
700+
if let mbe::TokenTree::Token(ident, _) = ident
700701
&& let TokenKind::Ident(ident, _) = ident.kind
701702
&& ident == sym::compile_error
702-
&& let mbe::TokenTree::Token(bang) = bang
703+
&& let mbe::TokenTree::Token(bang, _) = bang
703704
&& let TokenKind::Not = bang.kind
704705
&& let mbe::TokenTree::Delimited(.., del) = args
705706
&& del.delim != Delimiter::Invisible
@@ -896,7 +897,9 @@ enum TtHandle<'tt> {
896897

897898
impl<'tt> TtHandle<'tt> {
898899
fn from_token(tok: Token) -> Self {
899-
TtHandle::Token(mbe::TokenTree::Token(tok))
900+
// `Spacing::Alone` is pessimistic but changing it has no effect on the
901+
// current test suite.
902+
TtHandle::Token(mbe::TokenTree::Token(tok, Spacing::Alone))
900903
}
901904

902905
fn from_token_kind(kind: TokenKind, span: Span) -> Self {
@@ -925,8 +928,8 @@ impl<'tt> Clone for TtHandle<'tt> {
925928

926929
// This variant *must* contain a `mbe::TokenTree::Token`, and not
927930
// any other variant of `mbe::TokenTree`.
928-
TtHandle::Token(mbe::TokenTree::Token(tok)) => {
929-
TtHandle::Token(mbe::TokenTree::Token(tok.clone()))
931+
TtHandle::Token(mbe::TokenTree::Token(tok, spacing)) => {
932+
TtHandle::Token(mbe::TokenTree::Token(tok.clone(), *spacing))
930933
}
931934

932935
_ => unreachable!(),
@@ -1143,7 +1146,7 @@ fn check_matcher_core<'tt>(
11431146
// whereas macros from an external crate have a dummy id.
11441147
if def.id != DUMMY_NODE_ID
11451148
&& matches!(kind, NonterminalKind::PatParam { inferred: true })
1146-
&& matches!(next_token, TokenTree::Token(token) if token.kind == BinOp(token::BinOpToken::Or))
1149+
&& matches!(next_token, TokenTree::Token(token, _) if token.kind == BinOp(token::BinOpToken::Or))
11471150
{
11481151
// It is suggestion to use pat_param, for example: $x:pat -> $x:pat_param.
11491152
let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
@@ -1274,7 +1277,7 @@ enum IsInFollow {
12741277
fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
12751278
use mbe::TokenTree;
12761279

1277-
if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok {
1280+
if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }, _) = *tok {
12781281
// closing a token tree can never be matched by any fragment;
12791282
// iow, we always require that `(` and `)` match, etc.
12801283
IsInFollow::Yes
@@ -1293,7 +1296,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
12931296
NonterminalKind::Stmt | NonterminalKind::Expr => {
12941297
const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"];
12951298
match tok {
1296-
TokenTree::Token(token) => match token.kind {
1299+
TokenTree::Token(token, _) => match token.kind {
12971300
FatArrow | Comma | Semi => IsInFollow::Yes,
12981301
_ => IsInFollow::No(TOKENS),
12991302
},
@@ -1303,7 +1306,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13031306
NonterminalKind::PatParam { .. } => {
13041307
const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"];
13051308
match tok {
1306-
TokenTree::Token(token) => match token.kind {
1309+
TokenTree::Token(token, _) => match token.kind {
13071310
FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes,
13081311
Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => {
13091312
IsInFollow::Yes
@@ -1316,7 +1319,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13161319
NonterminalKind::PatWithOr => {
13171320
const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`if`", "`in`"];
13181321
match tok {
1319-
TokenTree::Token(token) => match token.kind {
1322+
TokenTree::Token(token, _) => match token.kind {
13201323
FatArrow | Comma | Eq => IsInFollow::Yes,
13211324
Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => {
13221325
IsInFollow::Yes
@@ -1332,7 +1335,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13321335
"`where`",
13331336
];
13341337
match tok {
1335-
TokenTree::Token(token) => match token.kind {
1338+
TokenTree::Token(token, _) => match token.kind {
13361339
OpenDelim(Delimiter::Brace)
13371340
| OpenDelim(Delimiter::Bracket)
13381341
| Comma
@@ -1369,7 +1372,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13691372
// Explicitly disallow `priv`, on the off chance it comes back.
13701373
const TOKENS: &[&str] = &["`,`", "an ident", "a type"];
13711374
match tok {
1372-
TokenTree::Token(token) => match token.kind {
1375+
TokenTree::Token(token, _) => match token.kind {
13731376
Comma => IsInFollow::Yes,
13741377
Ident(_, IdentIsRaw::Yes) => IsInFollow::Yes,
13751378
Ident(name, _) if name != kw::Priv => IsInFollow::Yes,
@@ -1395,7 +1398,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
13951398

13961399
fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
13971400
match tt {
1398-
mbe::TokenTree::Token(token) => pprust::token_to_string(token).into(),
1401+
mbe::TokenTree::Token(token, _) => pprust::token_to_string(token).into(),
13991402
mbe::TokenTree::MetaVar(_, name) => format!("${name}"),
14001403
mbe::TokenTree::MetaVarDecl(_, name, Some(kind)) => format!("${name}:{kind}"),
14011404
mbe::TokenTree::MetaVarDecl(_, name, None) => format!("${name}:"),

‎compiler/rustc_expand/src/mbe/quoted.rs‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ fn parse_tree<'a>(
180180
err.emit();
181181
// Returns early the same read `$` to avoid spanning
182182
// unrelated diagnostics that could be performed afterwards
183-
return TokenTree::token(token::Dollar, span);
183+
return TokenTree::token_alone(token::Dollar, span);
184184
}
185185
Ok(elem) => {
186186
maybe_emit_macro_metavar_expr_feature(
@@ -220,11 +220,14 @@ fn parse_tree<'a>(
220220

221221
// `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate`
222222
// special metavariable that names the crate of the invocation.
223-
Some(tokenstream::TokenTree::Token(token, _)) if token.is_ident() => {
223+
Some(tokenstream::TokenTree::Token(token, spacing)) if token.is_ident() => {
224224
let (ident, is_raw) = token.ident().unwrap();
225225
let span = ident.span.with_lo(span.lo());
226226
if ident.name == kw::Crate && matches!(is_raw, IdentIsRaw::No) {
227-
TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span)
227+
TokenTree::Token(
228+
Token::new(token::Ident(kw::DollarCrate, is_raw), span),
229+
*spacing,
230+
)
228231
} else {
229232
TokenTree::MetaVar(span, ident)
230233
}
@@ -240,7 +243,7 @@ fn parse_tree<'a>(
240243
} else {
241244
maybe_emit_macro_metavar_expr_feature(features, sess, span);
242245
}
243-
TokenTree::token(token::Dollar, span)
246+
TokenTree::token_alone(token::Dollar, span)
244247
}
245248

246249
// `tree` is followed by some other token. This is an error.
@@ -252,12 +255,12 @@ fn parse_tree<'a>(
252255
}
253256

254257
// There are no more tokens. Just return the `$` we already have.
255-
None => TokenTree::token(token::Dollar, span),
258+
None => TokenTree::token_alone(token::Dollar, span),
256259
}
257260
}
258261

259262
// `tree` is an arbitrary token. Keep it.
260-
tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token.clone()),
263+
tokenstream::TokenTree::Token(token, spacing) => TokenTree::Token(token.clone(), *spacing),
261264

262265
// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
263266
// descend into the delimited set and further parse it.

‎compiler/rustc_expand/src/mbe/transcribe.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -315,10 +315,10 @@ pub(super) fn transcribe<'a>(
315315

316316
// Nothing much to do here. Just push the token to the result, being careful to
317317
// preserve syntax context.
318-
mbe::TokenTree::Token(token) => {
318+
mbe::TokenTree::Token(token, spacing) => {
319319
let mut token = token.clone();
320320
mut_visit::visit_token(&mut token, &mut marker);
321-
let tt = TokenTree::Token(token, Spacing::Alone);
321+
let tt = TokenTree::Token(token, *spacing);
322322
result.push(tt);
323323
}
324324

0 commit comments

Comments
 (0)