Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 1c2b65b

Browse files
Aaron1011Aaron1011
committed
Revert "Move functions to librustc_parse"
This reverts commit 7a4c186.
1 parent 1ae7de9 commit 1c2b65b

File tree

3 files changed

+179
-123
lines changed

3 files changed

+179
-123
lines changed

‎src/librustc_ast/token.rs‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -673,6 +673,62 @@ impl Token {
673673

674674
Some(Token::new(kind, self.span.to(joint.span)))
675675
}
676+
677+
// See comments in `Nonterminal::to_tokenstream` for why we care about
678+
// *probably* equal here rather than actual equality
679+
crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool {
680+
if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) {
681+
return false;
682+
}
683+
match (&self.kind, &other.kind) {
684+
(&Eq, &Eq)
685+
| (&Lt, &Lt)
686+
| (&Le, &Le)
687+
| (&EqEq, &EqEq)
688+
| (&Ne, &Ne)
689+
| (&Ge, &Ge)
690+
| (&Gt, &Gt)
691+
| (&AndAnd, &AndAnd)
692+
| (&OrOr, &OrOr)
693+
| (&Not, &Not)
694+
| (&Tilde, &Tilde)
695+
| (&At, &At)
696+
| (&Dot, &Dot)
697+
| (&DotDot, &DotDot)
698+
| (&DotDotDot, &DotDotDot)
699+
| (&DotDotEq, &DotDotEq)
700+
| (&Comma, &Comma)
701+
| (&Semi, &Semi)
702+
| (&Colon, &Colon)
703+
| (&ModSep, &ModSep)
704+
| (&RArrow, &RArrow)
705+
| (&LArrow, &LArrow)
706+
| (&FatArrow, &FatArrow)
707+
| (&Pound, &Pound)
708+
| (&Dollar, &Dollar)
709+
| (&Question, &Question)
710+
| (&Whitespace, &Whitespace)
711+
| (&Comment, &Comment)
712+
| (&Eof, &Eof) => true,
713+
714+
(&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b,
715+
716+
(&OpenDelim(a), &OpenDelim(b)) | (&CloseDelim(a), &CloseDelim(b)) => a == b,
717+
718+
(&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b,
719+
720+
(&Literal(a), &Literal(b)) => a == b,
721+
722+
(&Lifetime(a), &Lifetime(b)) => a == b,
723+
(&Ident(a, b), &Ident(c, d)) => {
724+
b == d && (a == c || a == kw::DollarCrate || c == kw::DollarCrate)
725+
}
726+
727+
(&Interpolated(_), &Interpolated(_)) => false,
728+
729+
_ => panic!("forgot to add a token?"),
730+
}
731+
}
676732
}
677733

678734
impl PartialEq<TokenKind> for Token {

‎src/librustc_ast/tokenstream.rs‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,23 @@ impl TokenTree {
6868
}
6969
}
7070

71+
// See comments in `Nonterminal::to_tokenstream` for why we care about
72+
// *probably* equal here rather than actual equality
73+
//
74+
// This is otherwise the same as `eq_unspanned`, only recursing with a
75+
// different method.
76+
pub fn probably_equal_for_proc_macro(&self, other: &TokenTree) -> bool {
77+
match (self, other) {
78+
(TokenTree::Token(token), TokenTree::Token(token2)) => {
79+
token.probably_equal_for_proc_macro(token2)
80+
}
81+
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
82+
delim == delim2 && tts.probably_equal_for_proc_macro(&tts2)
83+
}
84+
_ => false,
85+
}
86+
}
87+
7188
/// Retrieves the TokenTree's span.
7289
pub fn span(&self) -> Span {
7390
match self {
@@ -290,7 +307,111 @@ impl TokenStream {
290307
t1.next().is_none() && t2.next().is_none()
291308
}
292309

310+
// See comments in `Nonterminal::to_tokenstream` for why we care about
311+
// *probably* equal here rather than actual equality
312+
//
313+
// This is otherwise the same as `eq_unspanned`, only recursing with a
314+
// different method.
315+
pub fn probably_equal_for_proc_macro(&self, other: &TokenStream) -> bool {
316+
// When checking for `probably_eq`, we ignore certain tokens that aren't
317+
// preserved in the AST. Because they are not preserved, the pretty
318+
// printer arbitrarily adds or removes them when printing as token
319+
// streams, making a comparison between a token stream generated from an
320+
// AST and a token stream which was parsed into an AST more reliable.
321+
fn semantic_tree(tree: &TokenTree) -> bool {
322+
if let TokenTree::Token(token) = tree {
323+
if let
324+
// The pretty printer tends to add trailing commas to
325+
// everything, and in particular, after struct fields.
326+
| token::Comma
327+
// The pretty printer emits `NoDelim` as whitespace.
328+
| token::OpenDelim(DelimToken::NoDelim)
329+
| token::CloseDelim(DelimToken::NoDelim)
330+
// The pretty printer collapses many semicolons into one.
331+
| token::Semi
332+
// The pretty printer collapses whitespace arbitrarily and can
333+
// introduce whitespace from `NoDelim`.
334+
| token::Whitespace
335+
// The pretty printer can turn `$crate` into `::crate_name`
336+
| token::ModSep = token.kind {
337+
return false;
338+
}
339+
}
340+
true
341+
}
293342

343+
// When comparing two `TokenStream`s, we ignore the `IsJoint` information.
344+
//
345+
// However, `rustc_parse::lexer::tokentrees::TokenStreamBuilder` will
346+
// use `Token.glue` on adjacent tokens with the proper `IsJoint`.
347+
// Since we are ignoreing `IsJoint`, a 'glued' token (e.g. `BinOp(Shr)`)
348+
// and its 'split'/'unglued' compoenents (e.g. `Gt, Gt`) are equivalent
349+
// when determining if two `TokenStream`s are 'probably equal'.
350+
//
351+
// Therefore, we use `break_two_token_op` to convert all tokens
352+
// to the 'unglued' form (if it exists). This ensures that two
353+
// `TokenStream`s which differ only in how their tokens are glued
354+
// will be considered 'probably equal', which allows us to keep spans.
355+
//
356+
// This is important when the original `TokenStream` contained
357+
// extra spaces (e.g. `f :: < Vec < _ > > ( ) ;'). These extra spaces
358+
// will be omitted when we pretty-print, which can cause the original
359+
// and reparsed `TokenStream`s to differ in the assignment of `IsJoint`,
360+
// leading to some tokens being 'glued' together in one stream but not
361+
// the other. See #68489 for more details.
362+
fn break_tokens(tree: TokenTree) -> impl Iterator<Item = TokenTree> {
363+
// In almost all cases, we should have either zero or one levels
364+
// of 'unglueing'. However, in some unusual cases, we may need
365+
// to iterate breaking tokens mutliple times. For example:
366+
// '[BinOpEq(Shr)] => [Gt, Ge] -> [Gt, Gt, Eq]'
367+
let mut token_trees: SmallVec<[_; 2]>;
368+
if let TokenTree::Token(token) = &tree {
369+
let mut out = SmallVec::<[_; 2]>::new();
370+
out.push(token.clone());
371+
// Iterate to fixpoint:
372+
// * We start off with 'out' containing our initial token, and `temp` empty
373+
// * If we are able to break any tokens in `out`, then `out` will have
374+
// at least one more element than 'temp', so we will try to break tokens
375+
// again.
376+
// * If we cannot break any tokens in 'out', we are done
377+
loop {
378+
let mut temp = SmallVec::<[_; 2]>::new();
379+
let mut changed = false;
380+
381+
for token in out.into_iter() {
382+
if let Some((first, second)) = token.kind.break_two_token_op() {
383+
temp.push(Token::new(first, DUMMY_SP));
384+
temp.push(Token::new(second, DUMMY_SP));
385+
changed = true;
386+
} else {
387+
temp.push(token);
388+
}
389+
}
390+
out = temp;
391+
if !changed {
392+
break;
393+
}
394+
}
395+
token_trees = out.into_iter().map(|t| TokenTree::Token(t)).collect();
396+
if token_trees.len() != 1 {
397+
debug!("break_tokens: broke {:?} to {:?}", tree, token_trees);
398+
}
399+
} else {
400+
token_trees = SmallVec::new();
401+
token_trees.push(tree);
402+
}
403+
token_trees.into_iter()
404+
}
405+
406+
let mut t1 = self.trees().filter(semantic_tree).flat_map(break_tokens);
407+
let mut t2 = other.trees().filter(semantic_tree).flat_map(break_tokens);
408+
for (t1, t2) in t1.by_ref().zip(t2.by_ref()) {
409+
if !t1.probably_equal_for_proc_macro(&t2) {
410+
return false;
411+
}
412+
}
413+
t1.next().is_none() && t2.next().is_none()
414+
}
294415

295416
pub fn map_enumerated<F: FnMut(usize, TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream {
296417
TokenStream(Lrc::new(

‎src/librustc_parse/lib.rs‎

Lines changed: 2 additions & 123 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,16 @@
77
#![feature(or_patterns)]
88

99
use rustc_ast::ast;
10-
use rustc_ast::token::{self, Nonterminal, Token, TokenKind, DelimToken};
10+
use rustc_ast::token::{self, Nonterminal};
1111
use rustc_ast::tokenstream::{self, TokenStream, TokenTree};
1212
use rustc_ast_pretty::pprust;
1313
use rustc_data_structures::sync::Lrc;
1414
use rustc_errors::{Diagnostic, FatalError, Level, PResult};
1515
use rustc_session::parse::ParseSess;
1616
use rustc_span::{FileName, SourceFile, Span};
17-
use rustc_span::symbol::kw;
1817

1918
use std::path::Path;
2019
use std::str;
21-
use std::mem;
2220

2321
use log::info;
2422

@@ -308,7 +306,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
308306
// modifications, including adding/removing typically non-semantic
309307
// tokens such as extra braces and commas, don't happen.
310308
if let Some(tokens) = tokens {
311-
if tokenstream_probably_equal_for_proc_macro(&tokens, &tokens_for_real) {
309+
if tokens.probably_equal_for_proc_macro(&tokens_for_real) {
312310
return tokens;
313311
}
314312
info!(
@@ -383,122 +381,3 @@ fn prepend_attrs(
383381
builder.push(tokens.clone());
384382
Some(builder.build())
385383
}
386-
387-
// See comments in `Nonterminal::to_tokenstream` for why we care about
388-
// *probably* equal here rather than actual equality
389-
//
390-
// This is otherwise the same as `eq_unspanned`, only recursing with a
391-
// different method.
392-
pub fn tokenstream_probably_equal_for_proc_macro(first: &TokenStream, other: &TokenStream) -> bool {
393-
// When checking for `probably_eq`, we ignore certain tokens that aren't
394-
// preserved in the AST. Because they are not preserved, the pretty
395-
// printer arbitrarily adds or removes them when printing as token
396-
// streams, making a comparison between a token stream generated from an
397-
// AST and a token stream which was parsed into an AST more reliable.
398-
fn semantic_tree(tree: &TokenTree) -> bool {
399-
if let TokenTree::Token(token) = tree {
400-
if let
401-
// The pretty printer tends to add trailing commas to
402-
// everything, and in particular, after struct fields.
403-
| token::Comma
404-
// The pretty printer emits `NoDelim` as whitespace.
405-
| token::OpenDelim(DelimToken::NoDelim)
406-
| token::CloseDelim(DelimToken::NoDelim)
407-
// The pretty printer collapses many semicolons into one.
408-
| token::Semi
409-
// The pretty printer collapses whitespace arbitrarily and can
410-
// introduce whitespace from `NoDelim`.
411-
| token::Whitespace
412-
// The pretty printer can turn `$crate` into `::crate_name`
413-
| token::ModSep = token.kind {
414-
return false;
415-
}
416-
}
417-
true
418-
}
419-
420-
let mut t1 = first.trees().filter(semantic_tree);
421-
let mut t2 = other.trees().filter(semantic_tree);
422-
for (t1, t2) in t1.by_ref().zip(t2.by_ref()) {
423-
if !tokentree_probably_equal_for_proc_macro(&t1, &t2) {
424-
return false;
425-
}
426-
}
427-
t1.next().is_none() && t2.next().is_none()
428-
}
429-
430-
// See comments in `Nonterminal::to_tokenstream` for why we care about
431-
// *probably* equal here rather than actual equality
432-
crate fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool {
433-
use TokenKind::*;
434-
435-
if mem::discriminant(&first.kind) != mem::discriminant(&other.kind) {
436-
return false;
437-
}
438-
match (&first.kind, &other.kind) {
439-
(&Eq, &Eq)
440-
| (&Lt, &Lt)
441-
| (&Le, &Le)
442-
| (&EqEq, &EqEq)
443-
| (&Ne, &Ne)
444-
| (&Ge, &Ge)
445-
| (&Gt, &Gt)
446-
| (&AndAnd, &AndAnd)
447-
| (&OrOr, &OrOr)
448-
| (&Not, &Not)
449-
| (&Tilde, &Tilde)
450-
| (&At, &At)
451-
| (&Dot, &Dot)
452-
| (&DotDot, &DotDot)
453-
| (&DotDotDot, &DotDotDot)
454-
| (&DotDotEq, &DotDotEq)
455-
| (&Comma, &Comma)
456-
| (&Semi, &Semi)
457-
| (&Colon, &Colon)
458-
| (&ModSep, &ModSep)
459-
| (&RArrow, &RArrow)
460-
| (&LArrow, &LArrow)
461-
| (&FatArrow, &FatArrow)
462-
| (&Pound, &Pound)
463-
| (&Dollar, &Dollar)
464-
| (&Question, &Question)
465-
| (&Whitespace, &Whitespace)
466-
| (&Comment, &Comment)
467-
| (&Eof, &Eof) => true,
468-
469-
(&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b,
470-
471-
(&OpenDelim(a), &OpenDelim(b)) | (&CloseDelim(a), &CloseDelim(b)) => a == b,
472-
473-
(&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b,
474-
475-
(&Literal(a), &Literal(b)) => a == b,
476-
477-
(&Lifetime(a), &Lifetime(b)) => a == b,
478-
(&Ident(a, b), &Ident(c, d)) => {
479-
b == d && (a == c || a == kw::DollarCrate || c == kw::DollarCrate)
480-
}
481-
482-
(&Interpolated(_), &Interpolated(_)) => false,
483-
484-
_ => panic!("forgot to add a token?"),
485-
}
486-
}
487-
488-
489-
// See comments in `Nonterminal::to_tokenstream` for why we care about
490-
// *probably* equal here rather than actual equality
491-
//
492-
// This is otherwise the same as `eq_unspanned`, only recursing with a
493-
// different method.
494-
pub fn tokentree_probably_equal_for_proc_macro(first: &TokenTree, other: &TokenTree) -> bool {
495-
match (first, other) {
496-
(TokenTree::Token(token), TokenTree::Token(token2)) => {
497-
token_probably_equal_for_proc_macro(token, token2)
498-
}
499-
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
500-
delim == delim2 && tokenstream_probably_equal_for_proc_macro(&tts, &tts2)
501-
}
502-
_ => false,
503-
}
504-
}

0 commit comments

Comments
 (0)