Move Spacing out of AttrAnnotatedTokenStream.

And into `AttrAnnotatedTokenTree::Token`.

PR rust-lang#99887 did the same thing for `TokenStream`.

Author: nnethercote

compiler/rustc_ast/src/attr/mod.rs

@@ -303,13 +303,12 @@ impl Attribute {
                 .as_ref()
                 .unwrap_or_else(|| panic!("attribute is missing tokens: {:?}", self))
                 .create_token_stream(),
-            AttrKind::DocComment(comment_kind, data) => AttrAnnotatedTokenStream::from((
-                AttrAnnotatedTokenTree::Token(Token::new(
-                    token::DocComment(comment_kind, self.style, data),
-                    self.span,
-                )),
-                Spacing::Alone,
-            )),
+            AttrKind::DocComment(comment_kind, data) => {
+                AttrAnnotatedTokenStream::new(vec![AttrAnnotatedTokenTree::Token(
+                    Token::new(token::DocComment(comment_kind, self.style, data), self.span),
+                    Spacing::Alone,
+                )])
+            }
         }
     }
 }

compiler/rustc_ast/src/mut_visit.rs

@@ -644,7 +644,7 @@ pub fn noop_flat_map_param<T: MutVisitor>(mut param: Param, vis: &mut T) -> Smal
 // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 pub fn visit_attr_annotated_tt<T: MutVisitor>(tt: &mut AttrAnnotatedTokenTree, vis: &mut T) {
     match tt {
-        AttrAnnotatedTokenTree::Token(token) => {
+        AttrAnnotatedTokenTree::Token(token, _) => {
             visit_token(token, vis);
         }
         AttrAnnotatedTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
@@ -696,7 +696,7 @@ pub fn visit_attr_annotated_tts<T: MutVisitor>(
 ) {
     if T::VISIT_TOKENS && !tts.is_empty() {
         let tts = Lrc::make_mut(tts);
-        visit_vec(tts, |(tree, _is_joint)| visit_attr_annotated_tt(tree, vis));
+        visit_vec(tts, |tree| visit_attr_annotated_tt(tree, vis));
     }
 }
 

compiler/rustc_ast/src/tokenstream.rs

@@ -177,12 +177,12 @@ impl<CTX> HashStable<CTX> for LazyTokenStream {
 /// during expansion to perform early cfg-expansion, and to process attributes
 /// during proc-macro invocations.
 #[derive(Clone, Debug, Default, Encodable, Decodable)]
-pub struct AttrAnnotatedTokenStream(pub Lrc<Vec<(AttrAnnotatedTokenTree, Spacing)>>);
+pub struct AttrAnnotatedTokenStream(pub Lrc<Vec<AttrAnnotatedTokenTree>>);
 
 /// Like `TokenTree`, but for `AttrAnnotatedTokenStream`
 #[derive(Clone, Debug, Encodable, Decodable)]
 pub enum AttrAnnotatedTokenTree {
-    Token(Token),
+    Token(Token, Spacing),
     Delimited(DelimSpan, Delimiter, AttrAnnotatedTokenStream),
     /// Stores the attributes for an attribute target,
     /// along with the tokens for that attribute target.
@@ -191,7 +191,7 @@ pub enum AttrAnnotatedTokenTree {
 }
 
 impl AttrAnnotatedTokenStream {
-    pub fn new(tokens: Vec<(AttrAnnotatedTokenTree, Spacing)>) -> AttrAnnotatedTokenStream {
+    pub fn new(tokens: Vec<AttrAnnotatedTokenTree>) -> AttrAnnotatedTokenStream {
         AttrAnnotatedTokenStream(Lrc::new(tokens))
     }
 
@@ -204,9 +204,9 @@ impl AttrAnnotatedTokenStream {
         let trees: Vec<_> = self
             .0
             .iter()
-            .flat_map(|tree| match &tree.0 {
-                AttrAnnotatedTokenTree::Token(inner) => {
-                    smallvec![TokenTree::Token(inner.clone(), tree.1)].into_iter()
+            .flat_map(|tree| match &tree {
+                AttrAnnotatedTokenTree::Token(inner, spacing) => {
+                    smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
                 }
                 AttrAnnotatedTokenTree::Delimited(span, delim, stream) => {
                     smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
@@ -363,12 +363,6 @@ impl TokenStream {
     }
 }
 
-impl From<(AttrAnnotatedTokenTree, Spacing)> for AttrAnnotatedTokenStream {
-    fn from((tree, spacing): (AttrAnnotatedTokenTree, Spacing)) -> AttrAnnotatedTokenStream {
-        AttrAnnotatedTokenStream::new(vec![(tree, spacing)])
-    }
-}
-
 impl iter::FromIterator<TokenTree> for TokenStream {
     fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
         TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>())
@@ -428,10 +422,7 @@ impl TokenStream {
         } else {
             let attr_data =
                 AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() };
-            AttrAnnotatedTokenStream::new(vec![(
-                AttrAnnotatedTokenTree::Attributes(attr_data),
-                Spacing::Alone,
-            )])
+            AttrAnnotatedTokenStream::new(vec![AttrAnnotatedTokenTree::Attributes(attr_data)])
         };
         Some(attr_annotated.to_tokenstream())
     }

compiler/rustc_expand/src/config.rs

@@ -276,9 +276,9 @@ impl<'a> StripUnconfigured<'a> {
     /// Normal cfg-expansion operates on parsed AST nodes via the `configure` method
     fn configure_tokens(&self, stream: &AttrAnnotatedTokenStream) -> AttrAnnotatedTokenStream {
         fn can_skip(stream: &AttrAnnotatedTokenStream) -> bool {
-            stream.0.iter().all(|(tree, _spacing)| match tree {
+            stream.0.iter().all(|tree| match tree {
                 AttrAnnotatedTokenTree::Attributes(_) => false,
-                AttrAnnotatedTokenTree::Token(_) => true,
+                AttrAnnotatedTokenTree::Token(..) => true,
                 AttrAnnotatedTokenTree::Delimited(_, _, inner) => can_skip(inner),
             })
         }
@@ -290,32 +290,32 @@ impl<'a> StripUnconfigured<'a> {
         let trees: Vec<_> = stream
             .0
             .iter()
-            .flat_map(|(tree, spacing)| match tree.clone() {
+            .flat_map(|tree| match tree.clone() {
                 AttrAnnotatedTokenTree::Attributes(mut data) => {
                     data.attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
 
                     if self.in_cfg(&data.attrs) {
                         data.tokens = LazyTokenStream::new(
                             self.configure_tokens(&data.tokens.create_token_stream()),
                         );
-                        Some((AttrAnnotatedTokenTree::Attributes(data), *spacing)).into_iter()
+                        Some(AttrAnnotatedTokenTree::Attributes(data)).into_iter()
                     } else {
                         None.into_iter()
                     }
                 }
                 AttrAnnotatedTokenTree::Delimited(sp, delim, mut inner) => {
                     inner = self.configure_tokens(&inner);
-                    Some((AttrAnnotatedTokenTree::Delimited(sp, delim, inner), *spacing))
+                    Some(AttrAnnotatedTokenTree::Delimited(sp, delim, inner))
                         .into_iter()
                 }
-                AttrAnnotatedTokenTree::Token(ref token) if let TokenKind::Interpolated(ref nt) = token.kind => {
+                AttrAnnotatedTokenTree::Token(ref token, _) if let TokenKind::Interpolated(ref nt) = token.kind => {
                     panic!(
                         "Nonterminal should have been flattened at {:?}: {:?}",
                         token.span, nt
                     );
                 }
-                AttrAnnotatedTokenTree::Token(token) => {
-                    Some((AttrAnnotatedTokenTree::Token(token), *spacing)).into_iter()
+                AttrAnnotatedTokenTree::Token(token, spacing) => {
+                    Some(AttrAnnotatedTokenTree::Token(token, spacing)).into_iter()
                 }
             })
             .collect();
@@ -404,13 +404,13 @@ impl<'a> StripUnconfigured<'a> {
         };
         let pound_span = pound_token.span;
 
-        let mut trees = vec![(AttrAnnotatedTokenTree::Token(pound_token), Spacing::Alone)];
+        let mut trees = vec![AttrAnnotatedTokenTree::Token(pound_token, Spacing::Alone)];
         if attr.style == AttrStyle::Inner {
             // For inner attributes, we do the same thing for the `!` in `#![some_attr]`
             let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = orig_trees.next().unwrap() else {
                 panic!("Bad tokens for attribute {:?}", attr);
             };
-            trees.push((AttrAnnotatedTokenTree::Token(bang_token), Spacing::Alone));
+            trees.push(AttrAnnotatedTokenTree::Token(bang_token, Spacing::Alone));
         }
         // We don't really have a good span to use for the synthesized `[]`
         // in `#[attr]`, so just use the span of the `#` token.
@@ -422,7 +422,7 @@ impl<'a> StripUnconfigured<'a> {
                 .unwrap_or_else(|| panic!("Missing tokens for {:?}", item))
                 .create_token_stream(),
         );
-        trees.push((bracket_group, Spacing::Alone));
+        trees.push(bracket_group);
         let tokens = Some(LazyTokenStream::new(AttrAnnotatedTokenStream::new(trees)));
         let attr = attr::mk_attr_from_item(item, tokens, attr.style, item_span);
         if attr.has_name(sym::crate_type) {

compiler/rustc_parse/src/parser/attr_wrapper.rs

@@ -397,7 +397,7 @@ fn make_token_stream(
     struct FrameData {
         // This is `None` for the first frame, `Some` for all others.
         open_delim_sp: Option<(Delimiter, Span)>,
-        inner: Vec<(AttrAnnotatedTokenTree, Spacing)>,
+        inner: Vec<AttrAnnotatedTokenTree>,
     }
     let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }];
     let mut token_and_spacing = iter.next();
@@ -426,34 +426,34 @@ fn make_token_stream(
                         panic!("Bottom token frame is missing for token: {:?}", token)
                     })
                     .inner
-                    .push((delimited, Spacing::Alone));
+                    .push(delimited);
             }
             FlatToken::Token(token) => stack
                 .last_mut()
                 .expect("Bottom token frame is missing!")
                 .inner
-                .push((AttrAnnotatedTokenTree::Token(token), spacing)),
+                .push(AttrAnnotatedTokenTree::Token(token, spacing)),
             FlatToken::AttrTarget(data) => stack
                 .last_mut()
                 .expect("Bottom token frame is missing!")
                 .inner
-                .push((AttrAnnotatedTokenTree::Attributes(data), spacing)),
+                .push(AttrAnnotatedTokenTree::Attributes(data)),
             FlatToken::Empty => {}
         }
         token_and_spacing = iter.next();
     }
     let mut final_buf = stack.pop().expect("Missing final buf!");
     if break_last_token {
-        let (last_token, spacing) = final_buf.inner.pop().unwrap();
-        if let AttrAnnotatedTokenTree::Token(last_token) = last_token {
+        let last_token = final_buf.inner.pop().unwrap();
+        if let AttrAnnotatedTokenTree::Token(last_token, spacing) = last_token {
             let unglued_first = last_token.kind.break_two_token_op().unwrap().0;
 
             // An 'unglued' token is always two ASCII characters
             let mut first_span = last_token.span.shrink_to_lo();
             first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1));
 
-            final_buf.inner.push((
-                AttrAnnotatedTokenTree::Token(Token::new(unglued_first, first_span)),
+            final_buf.inner.push(AttrAnnotatedTokenTree::Token(
+                Token::new(unglued_first, first_span),
                 spacing,
             ));
         } else {