Move jointness censoring to proc_macro

Proc-macro API currently exposes jointness in `Punct` tokens. That is,
`+` in `+one` is **non** joint.

Our lexer produces jointness info for all tokens, so we need to censor
it *somewhere*

Previously we did this in a lexer, but it makes more sense to do this
in a proc-macro server.

Author: matklad

compiler/rustc_expand/src/proc_macro_server.rs

@@ -47,15 +47,21 @@ impl ToInternal<token::DelimToken> for Delimiter {
     }
 }
 
-impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec<Self>)>
+impl FromInternal<(TreeAndJoint, Option<tokenstream::TokenTree>, &'_ ParseSess, &'_ mut Vec<Self>)>
     for TokenTree<Group, Punct, Ident, Literal>
 {
     fn from_internal(
-        ((tree, is_joint), sess, stack): (TreeAndJoint, &ParseSess, &mut Vec<Self>),
+        ((tree, is_joint), look_ahead, sess, stack): (
+            TreeAndJoint,
+            Option<tokenstream::TokenTree>,
+            &ParseSess,
+            &mut Vec<Self>,
+        ),
     ) -> Self {
         use rustc_ast::token::*;
 
-        let joint = is_joint == Joint;
+        let joint = is_joint == Joint
+            && matches!(look_ahead, Some(tokenstream::TokenTree::Token(t)) if t.is_op());
         let Token { kind, span } = match tree {
             tokenstream::TokenTree::Delimited(span, delim, tts) => {
                 let delimiter = Delimiter::from_internal(delim);
@@ -445,7 +451,8 @@ impl server::TokenStreamIter for Rustc<'_> {
         loop {
             let tree = iter.stack.pop().or_else(|| {
                 let next = iter.cursor.next_with_joint()?;
-                Some(TokenTree::from_internal((next, self.sess, &mut iter.stack)))
+                let lookahead = iter.cursor.look_ahead(0);
+                Some(TokenTree::from_internal((next, lookahead, self.sess, &mut iter.stack)))
             })?;
             // A hack used to pass AST fragments to attribute and derive macros
             // as a single nonterminal token instead of a token stream.

compiler/rustc_parse/src/lexer/tokentrees.rs

@@ -262,10 +262,7 @@ impl<'a> TokenTreesReader<'a> {
             }
             _ => {
                 let tt = TokenTree::Token(self.token.take());
-                let mut is_joint = self.bump();
-                if !self.token.is_op() {
-                    is_joint = NonJoint;
-                }
+                let is_joint = self.bump();
                 Ok((tt, is_joint))
             }
         }