Rollup merge of rust-lang#134693 - SpriteOvO:proc-macro-use-to-tokens-in-quote, r=tgross35

proc_macro: Use `ToTokens` trait in `quote` macro

Tracking issues: rust-lang#130977, rust-lang#54722

This PR changed `proc_macro::quote!` to use `ToTokens` trait instead of `TokenStream::from`, and migrated test cases from `quote` crate.

r? `@dtolnay`
CC `@tgross35`

Author: jhpratt

proc_macro/src/lib.rs

@@ -419,7 +419,7 @@ pub mod token_stream {
 /// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term.
 /// To quote `$` itself, use `$$`.
 #[unstable(feature = "proc_macro_quote", issue = "54722")]
-#[allow_internal_unstable(proc_macro_def_site, proc_macro_internals)]
+#[allow_internal_unstable(proc_macro_def_site, proc_macro_internals, proc_macro_totokens)]
 #[rustc_builtin_macro]
 pub macro quote($($t:tt)*) {
     /* compiler built-in */

proc_macro/src/quote.rs

@@ -4,12 +4,14 @@
 //! This quasiquoter uses macros 2.0 hygiene to reliably access
 //! items from `proc_macro`, to build a `proc_macro::TokenStream`.
 
-use crate::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
+use crate::{
+    Delimiter, Group, Ident, Literal, Punct, Spacing, Span, ToTokens, TokenStream, TokenTree,
+};
 
-macro_rules! quote_tt {
-    (($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, quote!($($t)*)) };
-    ([$($t:tt)*]) => { Group::new(Delimiter::Bracket, quote!($($t)*)) };
-    ({$($t:tt)*}) => { Group::new(Delimiter::Brace, quote!($($t)*)) };
+macro_rules! minimal_quote_tt {
+    (($($t:tt)*)) => { Group::new(Delimiter::Parenthesis, minimal_quote!($($t)*)) };
+    ([$($t:tt)*]) => { Group::new(Delimiter::Bracket, minimal_quote!($($t)*)) };
+    ({$($t:tt)*}) => { Group::new(Delimiter::Brace, minimal_quote!($($t)*)) };
     (,) => { Punct::new(',', Spacing::Alone) };
     (.) => { Punct::new('.', Spacing::Alone) };
     (;) => { Punct::new(';', Spacing::Alone) };
@@ -21,21 +23,20 @@ macro_rules! quote_tt {
     ($i:ident) => { Ident::new(stringify!($i), Span::def_site()) };
 }
 
-macro_rules! quote_ts {
+macro_rules! minimal_quote_ts {
     ((@ $($t:tt)*)) => { $($t)* };
     (::) => {
-        [
-            TokenTree::from(Punct::new(':', Spacing::Joint)),
-            TokenTree::from(Punct::new(':', Spacing::Alone)),
-        ].iter()
-            .cloned()
-            .map(|mut x| {
-                x.set_span(Span::def_site());
-                x
-            })
-            .collect::<TokenStream>()
+        {
+            let mut c = (
+                TokenTree::from(Punct::new(':', Spacing::Joint)),
+                TokenTree::from(Punct::new(':', Spacing::Alone))
+            );
+            c.0.set_span(Span::def_site());
+            c.1.set_span(Span::def_site());
+            [c.0, c.1].into_iter().collect::<TokenStream>()
+        }
     };
-    ($t:tt) => { TokenTree::from(quote_tt!($t)) };
+    ($t:tt) => { TokenTree::from(minimal_quote_tt!($t)) };
 }
 
 /// Simpler version of the real `quote!` macro, implemented solely
@@ -46,12 +47,14 @@ macro_rules! quote_ts {
 ///
 /// Note: supported tokens are a subset of the real `quote!`, but
 /// unquoting is different: instead of `$x`, this uses `(@ expr)`.
-macro_rules! quote {
-    () => { TokenStream::new() };
+macro_rules! minimal_quote {
     ($($t:tt)*) => {
-        [
-            $(TokenStream::from(quote_ts!($t)),)*
-        ].iter().cloned().collect::<TokenStream>()
+        {
+            #[allow(unused_mut)] // In case the expansion is empty
+            let mut ts = TokenStream::new();
+            $(ToTokens::to_tokens(&minimal_quote_ts!($t), &mut ts);)*
+            ts
+        }
     };
 }
 
@@ -62,52 +65,66 @@ macro_rules! quote {
 #[unstable(feature = "proc_macro_quote", issue = "54722")]
 pub fn quote(stream: TokenStream) -> TokenStream {
     if stream.is_empty() {
-        return quote!(crate::TokenStream::new());
+        return minimal_quote!(crate::TokenStream::new());
     }
-    let proc_macro_crate = quote!(crate);
+    let proc_macro_crate = minimal_quote!(crate);
     let mut after_dollar = false;
-    let tokens = stream
-        .into_iter()
-        .filter_map(|tree| {
-            if after_dollar {
-                after_dollar = false;
-                match tree {
-                    TokenTree::Ident(_) => {
-                        return Some(quote!(Into::<crate::TokenStream>::into(
-                        Clone::clone(&(@ tree))),));
-                    }
-                    TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
-                    _ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
-                }
-            } else if let TokenTree::Punct(ref tt) = tree {
-                if tt.as_char() == '$' {
-                    after_dollar = true;
-                    return None;
+
+    let mut tokens = crate::TokenStream::new();
+    for tree in stream {
+        if after_dollar {
+            after_dollar = false;
+            match tree {
+                TokenTree::Ident(_) => {
+                    minimal_quote!(crate::ToTokens::to_tokens(&(@ tree), &mut ts);)
+                        .to_tokens(&mut tokens);
+                    continue;
                 }
+                TokenTree::Punct(ref tt) if tt.as_char() == '$' => {}
+                _ => panic!("`$` must be followed by an ident or `$` in `quote!`"),
             }
+        } else if let TokenTree::Punct(ref tt) = tree {
+            if tt.as_char() == '$' {
+                after_dollar = true;
+                continue;
+            }
+        }
 
-            Some(quote!(crate::TokenStream::from((@ match tree {
-                TokenTree::Punct(tt) => quote!(crate::TokenTree::Punct(crate::Punct::new(
+        match tree {
+            TokenTree::Punct(tt) => {
+                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Punct(crate::Punct::new(
                     (@ TokenTree::from(Literal::character(tt.as_char()))),
                     (@ match tt.spacing() {
-                        Spacing::Alone => quote!(crate::Spacing::Alone),
-                        Spacing::Joint => quote!(crate::Spacing::Joint),
+                        Spacing::Alone => minimal_quote!(crate::Spacing::Alone),
+                        Spacing::Joint => minimal_quote!(crate::Spacing::Joint),
                     }),
-                ))),
-                TokenTree::Group(tt) => quote!(crate::TokenTree::Group(crate::Group::new(
+                )), &mut ts);)
+            }
+            TokenTree::Group(tt) => {
+                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Group(crate::Group::new(
                     (@ match tt.delimiter() {
-                        Delimiter::Parenthesis => quote!(crate::Delimiter::Parenthesis),
-                        Delimiter::Brace => quote!(crate::Delimiter::Brace),
-                        Delimiter::Bracket => quote!(crate::Delimiter::Bracket),
-                        Delimiter::None => quote!(crate::Delimiter::None),
+                        Delimiter::Parenthesis => minimal_quote!(crate::Delimiter::Parenthesis),
+                        Delimiter::Brace => minimal_quote!(crate::Delimiter::Brace),
+                        Delimiter::Bracket => minimal_quote!(crate::Delimiter::Bracket),
+                        Delimiter::None => minimal_quote!(crate::Delimiter::None),
                     }),
                     (@ quote(tt.stream())),
-                ))),
-                TokenTree::Ident(tt) => quote!(crate::TokenTree::Ident(crate::Ident::new(
-                    (@ TokenTree::from(Literal::string(&tt.to_string()))),
+                )), &mut ts);)
+            }
+            TokenTree::Ident(tt) => {
+                let literal = tt.to_string();
+                let (literal, ctor) = if let Some(stripped) = literal.strip_prefix("r#") {
+                    (stripped, minimal_quote!(crate::Ident::new_raw))
+                } else {
+                    (literal.as_str(), minimal_quote!(crate::Ident::new))
+                };
+                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Ident((@ ctor)(
+                    (@ TokenTree::from(Literal::string(literal))),
                     (@ quote_span(proc_macro_crate.clone(), tt.span())),
-                ))),
-                TokenTree::Literal(tt) => quote!(crate::TokenTree::Literal({
+                )), &mut ts);)
+            }
+            TokenTree::Literal(tt) => {
+                minimal_quote!(crate::ToTokens::to_tokens(&crate::TokenTree::Literal({
                     let mut iter = (@ TokenTree::from(Literal::string(&tt.to_string())))
                         .parse::<crate::TokenStream>()
                         .unwrap()
@@ -120,22 +137,28 @@ pub fn quote(stream: TokenStream) -> TokenStream {
                     } else {
                         unreachable!()
                     }
-                }))
-            })),))
-        })
-        .collect::<TokenStream>();
-
+                }), &mut ts);)
+            }
+        }
+        .to_tokens(&mut tokens);
+    }
     if after_dollar {
         panic!("unexpected trailing `$` in `quote!`");
     }
 
-    quote!([(@ tokens)].iter().cloned().collect::<crate::TokenStream>())
+    minimal_quote! {
+        {
+            let mut ts = crate::TokenStream::new();
+            (@ tokens)
+            ts
+        }
+    }
 }
 
 /// Quote a `Span` into a `TokenStream`.
 /// This is needed to implement a custom quoter.
 #[unstable(feature = "proc_macro_quote", issue = "54722")]
 pub fn quote_span(proc_macro_crate: TokenStream, span: Span) -> TokenStream {
     let id = span.save_span();
-    quote!((@ proc_macro_crate ) ::Span::recover_proc_macro_span((@ TokenTree::from(Literal::usize_unsuffixed(id)))))
+    minimal_quote!((@ proc_macro_crate ) ::Span::recover_proc_macro_span((@ TokenTree::from(Literal::usize_unsuffixed(id)))))
 }