Break tokens before checking if they are 'probably equal'

Fixes rust-lang#68489

When checking two `TokenStreams` to see if they are 'probably equal',
we ignore the `IsJoint` information associated with each `TokenTree`.
However, the `IsJoint` information determines whether adjacent tokens
will be 'glued' (if possible) when construction the `TokenStream` - e.g.
`[Gt Gt]` can be 'glued' to `BinOp(Shr)`.

Since we are ignoring the `IsJoint` information, 'glued' and 'unglued'
tokens are equivalent for determining if two `TokenStreams` are
'probably equal'. Therefore, we need to 'unglue' all tokens in the
stream to avoid false negatives (which cause us to throw out the cached
tokens, losing span information).

Author: Aaron1011

src/librustc_ast/tokenstream.rs

@@ -338,8 +338,38 @@ impl TokenStream {
             true
         }
 
-        let mut t1 = self.trees().filter(semantic_tree);
-        let mut t2 = other.trees().filter(semantic_tree);
+        // When comparing two `TokenStream`s, we ignore the `IsJoint` information.
+        //
+        // However, `rustc_parse::lexer::tokentrees::TokenStreamBuilder` will
+        // use `Token.glue` on adjacent tokens with the proper `IsJoint`.
+        // Since we are ignoreing `IsJoint`, a 'glued' token (e.g. `BinOp(Shr)`)
+        // and its 'split'/'unglued' compoenents (e.g. `Gt, Gt`) are equivalent
+        // when determining if two `TokenStream`s are 'probably equal'.
+        //
+        // Therefore, we use `break_two_token_op` to convert all tokens
+        // to the 'unglued' form (if it exists). This ensures that two
+        // `TokenStream`s which differ only in how their tokens are glued
+        // will be considered 'probably equal', which allows us to keep spans.
+        //
+        // This is important when the original `TokenStream` contained
+        // extra spaces (e.g. `f :: < Vec < _ > > ( ) ;'). These extra spaces
+        // will be omitted when we pretty-print, which can cause the original
+        // and reparsed `TokenStream`s to differ in the assignment of `IsJoint`,
+        // leading to some tokens being 'glued' together in one stream but not
+        // the other. See #68489 for more details.
+        fn break_tokens(tree: TokenTree) -> impl Iterator<Item = TokenTree> {
+            if let TokenTree::Token(token) = &tree {
+                if let Some((first, second)) = token.kind.break_two_token_op() {
+                    return SmallVec::from_buf([TokenTree::Token(Token::new(first, DUMMY_SP)), TokenTree::Token(Token::new(second, DUMMY_SP))]).into_iter()
+                }
+            }
+            let mut vec = SmallVec::<[_; 2]>::new();
+            vec.push(tree);
+            vec.into_iter()
+        }
+
+        let mut t1 = self.trees().filter(semantic_tree).flat_map(break_tokens);
+        let mut t2 = other.trees().filter(semantic_tree).flat_map(break_tokens);
         for (t1, t2) in t1.by_ref().zip(t2.by_ref()) {
             if !t1.probably_equal_for_proc_macro(&t2) {
                 return false;

src/test/ui/proc-macro/turbo-proc-macro.rs

@@ -0,0 +1,9 @@
+// aux-build:test-macros.rs
+
+extern crate test_macros;
+
+#[test_macros::recollect_attr]
+fn repro() {
+    f :: < Vec < _ > > ( ) ; //~ ERROR cannot find
+}
+fn main() {}

src/test/ui/proc-macro/turbo-proc-macro.stderr

@@ -0,0 +1,9 @@
+error[E0425]: cannot find function `f` in this scope
+  --> $DIR/turbo-proc-macro.rs:7:5
+   |
+LL |     f :: < Vec < _ > > ( ) ;
+   |     ^ not found in this scope
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0425`.