Auto merge of #16482 - GnomedDev:boxed-subtree, r=lnicola

Swap Subtree::token_trees from Vec to boxed slice

Performs one of the optimizations suggested in #16325, but a little bit more. Boxed slices guarantee `shrink_to_fit` aswell as saving a pointer width as no capacity has to be stored.

Most of the diff is:
- Changing `vec![]` to `Box::new([])`
- Changing initialize -> fill into fill -> into_boxed_slice
- Working around the lack of an owned iterator or automatic iteration over a `Box<[T]>`

I would like to use my own crate, [small-fixed-array](https://lib.rs/small-fixed-array), although I understand if it isn't mature enough for this. If I'm given the go ahead, I can rework this PR to use it instead.

Author: bors

crates/hir-def/src/nameres/collector.rs

@@ -2125,7 +2125,7 @@ impl ModCollector<'_, '_> {
 
         let is_export = export_attr.exists();
         let local_inner = if is_export {
-            export_attr.tt_values().flat_map(|it| &it.token_trees).any(|it| match it {
+            export_attr.tt_values().flat_map(|it| it.token_trees.iter()).any(|it| match it {
                 tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
                     ident.text.contains("local_inner_macros")
                 }

crates/hir-expand/src/attrs.rs

@@ -123,7 +123,7 @@ impl RawAttrs {
                 .filter_map(|(idx, attr)| Attr::from_tt(db, attr, index.with_cfg_attr(idx)));
 
             let cfg_options = &crate_graph[krate].cfg_options;
-            let cfg = Subtree { delimiter: subtree.delimiter, token_trees: cfg.to_vec() };
+            let cfg = Subtree { delimiter: subtree.delimiter, token_trees: Box::from(cfg) };
             let cfg = CfgExpr::parse(&cfg);
             if cfg_options.check(&cfg) == Some(false) {
                 smallvec![]

crates/hir-expand/src/builtin_attr_macro.rs

@@ -137,5 +137,8 @@ pub fn pseudo_derive_attr_expansion(
         token_trees.extend(tt.iter().cloned());
         token_trees.push(mk_leaf(']'));
     }
-    ExpandResult::ok(tt::Subtree { delimiter: tt.delimiter, token_trees })
+    ExpandResult::ok(tt::Subtree {
+        delimiter: tt.delimiter,
+        token_trees: token_trees.into_boxed_slice(),
+    })
 }

crates/hir-expand/src/builtin_fn_macro.rs

@@ -155,10 +155,10 @@ fn line_expand(
     // not incremental
     ExpandResult::ok(tt::Subtree {
         delimiter: tt::Delimiter::invisible_spanned(span),
-        token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
+        token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
             text: "0u32".into(),
             span,
-        }))],
+        }))]),
     })
 }
 
@@ -208,11 +208,11 @@ fn assert_expand(
         [cond, panic_args @ ..] => {
             let comma = tt::Subtree {
                 delimiter: tt::Delimiter::invisible_spanned(call_site_span),
-                token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
+                token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
                     char: ',',
                     spacing: tt::Spacing::Alone,
                     span: call_site_span,
-                }))],
+                }))]),
             };
             let cond = cond.clone();
             let panic_args = itertools::Itertools::intersperse(panic_args.iter().cloned(), comma);
@@ -359,7 +359,10 @@ fn panic_expand(
         close: call_site_span,
         kind: tt::DelimiterKind::Parenthesis,
     };
-    call.token_trees.push(tt::TokenTree::Subtree(subtree));
+
+    // FIXME(slow): quote! have a way to expand to builder to make this a vec!
+    call.push(tt::TokenTree::Subtree(subtree));
+
     ExpandResult::ok(call)
 }
 
@@ -388,7 +391,10 @@ fn unreachable_expand(
         close: call_site_span,
         kind: tt::DelimiterKind::Parenthesis,
     };
-    call.token_trees.push(tt::TokenTree::Subtree(subtree));
+
+    // FIXME(slow): quote! have a way to expand to builder to make this a vec!
+    call.push(tt::TokenTree::Subtree(subtree));
+
     ExpandResult::ok(call)
 }
 
@@ -675,10 +681,10 @@ fn include_bytes_expand(
     // FIXME: actually read the file here if the user asked for macro expansion
     let res = tt::Subtree {
         delimiter: tt::Delimiter::invisible_spanned(span),
-        token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
+        token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
             text: r#"b"""#.into(),
             span,
-        }))],
+        }))]),
     };
     ExpandResult::ok(res)
 }

crates/hir-expand/src/db.rs

@@ -524,7 +524,7 @@ fn macro_expand(
                 return ExpandResult {
                     value: CowArc::Owned(tt::Subtree {
                         delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
-                        token_trees: Vec::new(),
+                        token_trees: Box::new([]),
                     }),
                     // FIXME: We should make sure to enforce an invariant that invalid macro
                     // calls do not reach this call path!
@@ -586,7 +586,7 @@ fn macro_expand(
             return value.map(|()| {
                 CowArc::Owned(tt::Subtree {
                     delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
-                    token_trees: vec![],
+                    token_trees: Box::new([]),
                 })
             });
         }
@@ -601,7 +601,7 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<A
         return ExpandResult {
             value: Arc::new(tt::Subtree {
                 delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
-                token_trees: Vec::new(),
+                token_trees: Box::new([]),
             }),
             // FIXME: We should make sure to enforce an invariant that invalid macro
             // calls do not reach this call path!
@@ -635,7 +635,7 @@ fn expand_proc_macro(db: &dyn ExpandDatabase, id: MacroCallId) -> ExpandResult<A
         return value.map(|()| {
             Arc::new(tt::Subtree {
                 delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
-                token_trees: vec![],
+                token_trees: Box::new([]),
             })
         });
     }

crates/hir-expand/src/fixup.rs

@@ -312,7 +312,7 @@ pub(crate) fn reverse_fixups(tt: &mut Subtree, undo_info: &SyntaxFixupUndoInfo)
 }
 
 fn reverse_fixups_(tt: &mut Subtree, undo_info: &[Subtree]) {
-    let tts = std::mem::take(&mut tt.token_trees);
+    let tts = std::mem::take(&mut tt.token_trees).into_vec();
     tt.token_trees = tts
         .into_iter()
         // delete all fake nodes
@@ -343,7 +343,7 @@ fn reverse_fixups_(tt: &mut Subtree, undo_info: &[Subtree]) {
                     // we have a fake node here, we need to replace it again with the original
                     let original = undo_info[u32::from(leaf.span().range.start()) as usize].clone();
                     if original.delimiter.kind == tt::DelimiterKind::Invisible {
-                        original.token_trees.into()
+                        SmallVec::from(original.token_trees.into_vec())
                     } else {
                         SmallVec::from_const([original.into()])
                     }
@@ -383,7 +383,7 @@ mod tests {
     fn check_subtree_eq(a: &tt::Subtree, b: &tt::Subtree) -> bool {
         a.delimiter.kind == b.delimiter.kind
             && a.token_trees.len() == b.token_trees.len()
-            && a.token_trees.iter().zip(&b.token_trees).all(|(a, b)| check_tt_eq(a, b))
+            && a.token_trees.iter().zip(b.token_trees.iter()).all(|(a, b)| check_tt_eq(a, b))
     }
 
     fn check_tt_eq(a: &tt::TokenTree, b: &tt::TokenTree) -> bool {

crates/hir-expand/src/lib.rs

@@ -66,6 +66,7 @@ pub mod tt {
     pub type Delimiter = ::tt::Delimiter<Span>;
     pub type DelimSpan = ::tt::DelimSpan<Span>;
     pub type Subtree = ::tt::Subtree<Span>;
+    pub type SubtreeBuilder = ::tt::SubtreeBuilder<Span>;
     pub type Leaf = ::tt::Leaf<Span>;
     pub type Literal = ::tt::Literal<Span>;
     pub type Punct = ::tt::Punct<Span>;
@@ -760,7 +761,7 @@ impl ExpansionInfo {
             (
                 Arc::new(tt::Subtree {
                     delimiter: tt::Delimiter::invisible_spanned(loc.call_site),
-                    token_trees: Vec::new(),
+                    token_trees: Box::new([]),
                 }),
                 SyntaxFixupUndoInfo::NONE,
             )

crates/hir-expand/src/quote.rs

@@ -31,7 +31,7 @@ macro_rules! __quote {
                     open: $span,
                     close: $span,
                 },
-                token_trees: $crate::quote::IntoTt::to_tokens(children),
+                token_trees: $crate::quote::IntoTt::to_tokens(children).into_boxed_slice(),
             }
         }
     };
@@ -146,7 +146,7 @@ impl IntoTt for Vec<crate::tt::TokenTree> {
     fn to_subtree(self, span: Span) -> crate::tt::Subtree {
         crate::tt::Subtree {
             delimiter: crate::tt::Delimiter::invisible_spanned(span),
-            token_trees: self,
+            token_trees: self.into_boxed_slice(),
         }
     }
 
@@ -296,8 +296,9 @@ mod tests {
         // }
         let struct_name = mk_ident("Foo");
         let fields = [mk_ident("name"), mk_ident("id")];
-        let fields =
-            fields.iter().flat_map(|it| quote!(DUMMY =>#it: self.#it.clone(), ).token_trees);
+        let fields = fields
+            .iter()
+            .flat_map(|it| quote!(DUMMY =>#it: self.#it.clone(), ).token_trees.into_vec());
 
         let list = crate::tt::Subtree {
             delimiter: crate::tt::Delimiter {

crates/mbe/src/benchmark.rs

@@ -100,17 +100,19 @@ fn invocation_fixtures(
                 // So we just skip any error cases and try again
                 let mut try_cnt = 0;
                 loop {
-                    let mut subtree = tt::Subtree {
+                    let mut token_trees = Vec::new();
+                    for op in rule.lhs.iter() {
+                        collect_from_op(op, &mut token_trees, &mut seed);
+                    }
+
+                    let subtree = tt::Subtree {
                         delimiter: tt::Delimiter {
                             open: DUMMY,
                             close: DUMMY,
                             kind: tt::DelimiterKind::Invisible,
                         },
-                        token_trees: vec![],
+                        token_trees: token_trees.into_boxed_slice(),
                     };
-                    for op in rule.lhs.iter() {
-                        collect_from_op(op, &mut subtree, &mut seed);
-                    }
                     if it.expand(&subtree, |_| (), true, DUMMY).err.is_none() {
                         res.push((name.clone(), subtree));
                         break;
@@ -127,45 +129,45 @@ fn invocation_fixtures(
 
     fn collect_from_op(
         op: &Op<DummyTestSpanData>,
-        parent: &mut tt::Subtree<DummyTestSpanData>,
+        token_trees: &mut Vec<tt::TokenTree<DummyTestSpanData>>,
         seed: &mut usize,
     ) {
         return match op {
             Op::Var { kind, .. } => match kind.as_ref() {
-                Some(MetaVarKind::Ident) => parent.token_trees.push(make_ident("foo")),
-                Some(MetaVarKind::Ty) => parent.token_trees.push(make_ident("Foo")),
-                Some(MetaVarKind::Tt) => parent.token_trees.push(make_ident("foo")),
-                Some(MetaVarKind::Vis) => parent.token_trees.push(make_ident("pub")),
-                Some(MetaVarKind::Pat) => parent.token_trees.push(make_ident("foo")),
-                Some(MetaVarKind::Path) => parent.token_trees.push(make_ident("foo")),
-                Some(MetaVarKind::Literal) => parent.token_trees.push(make_literal("1")),
-                Some(MetaVarKind::Expr) => parent.token_trees.push(make_ident("foo")),
+                Some(MetaVarKind::Ident) => token_trees.push(make_ident("foo")),
+                Some(MetaVarKind::Ty) => token_trees.push(make_ident("Foo")),
+                Some(MetaVarKind::Tt) => token_trees.push(make_ident("foo")),
+                Some(MetaVarKind::Vis) => token_trees.push(make_ident("pub")),
+                Some(MetaVarKind::Pat) => token_trees.push(make_ident("foo")),
+                Some(MetaVarKind::Path) => token_trees.push(make_ident("foo")),
+                Some(MetaVarKind::Literal) => token_trees.push(make_literal("1")),
+                Some(MetaVarKind::Expr) => token_trees.push(make_ident("foo")),
                 Some(MetaVarKind::Lifetime) => {
-                    parent.token_trees.push(make_punct('\''));
-                    parent.token_trees.push(make_ident("a"));
+                    token_trees.push(make_punct('\''));
+                    token_trees.push(make_ident("a"));
                 }
                 Some(MetaVarKind::Block) => {
-                    parent.token_trees.push(make_subtree(tt::DelimiterKind::Brace, None))
+                    token_trees.push(make_subtree(tt::DelimiterKind::Brace, None))
                 }
                 Some(MetaVarKind::Item) => {
-                    parent.token_trees.push(make_ident("fn"));
-                    parent.token_trees.push(make_ident("foo"));
-                    parent.token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
-                    parent.token_trees.push(make_subtree(tt::DelimiterKind::Brace, None));
+                    token_trees.push(make_ident("fn"));
+                    token_trees.push(make_ident("foo"));
+                    token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
+                    token_trees.push(make_subtree(tt::DelimiterKind::Brace, None));
                 }
                 Some(MetaVarKind::Meta) => {
-                    parent.token_trees.push(make_ident("foo"));
-                    parent.token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
+                    token_trees.push(make_ident("foo"));
+                    token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
                 }
 
                 None => (),
                 Some(kind) => panic!("Unhandled kind {kind:?}"),
             },
-            Op::Literal(it) => parent.token_trees.push(tt::Leaf::from(it.clone()).into()),
-            Op::Ident(it) => parent.token_trees.push(tt::Leaf::from(it.clone()).into()),
+            Op::Literal(it) => token_trees.push(tt::Leaf::from(it.clone()).into()),
+            Op::Ident(it) => token_trees.push(tt::Leaf::from(it.clone()).into()),
             Op::Punct(puncts) => {
                 for punct in puncts {
-                    parent.token_trees.push(tt::Leaf::from(*punct).into());
+                    token_trees.push(tt::Leaf::from(*punct).into());
                 }
             }
             Op::Repeat { tokens, kind, separator } => {
@@ -177,20 +179,20 @@ fn invocation_fixtures(
                 };
                 for i in 0..cnt {
                     for it in tokens.iter() {
-                        collect_from_op(it, parent, seed);
+                        collect_from_op(it, token_trees, seed);
                     }
                     if i + 1 != cnt {
                         if let Some(sep) = separator {
                             match sep {
                                 Separator::Literal(it) => {
-                                    parent.token_trees.push(tt::Leaf::Literal(it.clone()).into())
+                                    token_trees.push(tt::Leaf::Literal(it.clone()).into())
                                 }
                                 Separator::Ident(it) => {
-                                    parent.token_trees.push(tt::Leaf::Ident(it.clone()).into())
+                                    token_trees.push(tt::Leaf::Ident(it.clone()).into())
                                 }
                                 Separator::Puncts(puncts) => {
                                     for it in puncts {
-                                        parent.token_trees.push(tt::Leaf::Punct(*it).into())
+                                        token_trees.push(tt::Leaf::Punct(*it).into())
                                     }
                                 }
                             };
@@ -199,11 +201,15 @@ fn invocation_fixtures(
                 }
             }
             Op::Subtree { tokens, delimiter } => {
-                let mut subtree = tt::Subtree { delimiter: *delimiter, token_trees: Vec::new() };
+                let mut subtree = Vec::new();
                 tokens.iter().for_each(|it| {
                     collect_from_op(it, &mut subtree, seed);
                 });
-                parent.token_trees.push(subtree.into());
+
+                let subtree =
+                    tt::Subtree { delimiter: *delimiter, token_trees: subtree.into_boxed_slice() };
+
+                token_trees.push(subtree.into());
             }
             Op::Ignore { .. } | Op::Index { .. } | Op::Count { .. } | Op::Length { .. } => {}
         };
@@ -230,7 +236,7 @@ fn invocation_fixtures(
         ) -> tt::TokenTree<DummyTestSpanData> {
             tt::Subtree {
                 delimiter: tt::Delimiter { open: DUMMY, close: DUMMY, kind },
-                token_trees: token_trees.unwrap_or_default(),
+                token_trees: token_trees.map(Vec::into_boxed_slice).unwrap_or_default(),
             }
             .into()
         }

crates/mbe/src/expander.rs

@@ -58,7 +58,7 @@ pub(crate) fn expand_rules<S: Span>(
         ExpandResult::new(
             tt::Subtree {
                 delimiter: tt::Delimiter::invisible_spanned(call_site),
-                token_trees: vec![],
+                token_trees: Box::new([]),
             },
             ExpandError::NoMatchingRule,
         )