CoAlloc: Vec, VecDeque, RawVec; related str, slice, Box, BinaryHeap, proc_macro...

Author: peter-lyons-kehl

compiler/rustc_middle/src/mir/mod.rs

@@ -2,7 +2,6 @@
 //!
 //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/mir/index.html
 
-use core::alloc::GlobalCoAllocMeta;
 use crate::mir::interpret::{
     AllocRange, ConstAllocation, ConstValue, ErrorHandled, GlobalAlloc, LitToConstInput, Scalar,
 };
@@ -14,6 +13,7 @@ use crate::ty::visit::{TypeVisitable, TypeVisitor};
 use crate::ty::{self, ir, DefIdTree, List, Ty, TyCtxt};
 use crate::ty::{AdtDef, InstanceDef, ScalarInt, UserTypeAnnotationIndex};
 use crate::ty::{GenericArg, InternalSubsts, SubstsRef};
+use core::alloc::GlobalCoAllocMeta;
 
 use rustc_data_structures::captures::Captures;
 use rustc_errors::ErrorGuaranteed;

compiler/rustc_middle/src/mir/syntax.rs

@@ -3,9 +3,9 @@
 //! This is in a dedicated file so that changes to this file can be reviewed more carefully.
 //! The intention is that this file only contains datatype declarations, no code.
 
+use super::{BasicBlock, Constant, Field, Local, SwitchTargets, UserTypeProjection};
 use core::alloc::GlobalCoAllocMeta;
 use core::mem;
-use super::{BasicBlock, Constant, Field, Local, SwitchTargets, UserTypeProjection};
 
 use crate::mir::coverage::{CodeRegion, CoverageKind};
 use crate::traits::Reveal;

compiler/rustc_parse/src/parser/attr_wrapper.rs

@@ -1,6 +1,6 @@
+use super::{Capturing, FlatToken, ForceCollect, Parser, ReplaceRange, TokenCursor, TrailingToken};
 use core::alloc::GlobalCoAllocMeta;
 use core::mem;
-use super::{Capturing, FlatToken, ForceCollect, Parser, ReplaceRange, TokenCursor, TrailingToken};
 use rustc_ast::token::{self, Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::{AttrTokenStream, AttributesData, ToAttrTokenStream};
 use rustc_ast::tokenstream::{AttrTokenTree, DelimSpan, LazyAttrTokenStream, Spacing};

compiler/rustc_parse/src/parser/mod.rs

@@ -11,8 +11,8 @@ mod stmt;
 mod ty;
 
 use crate::lexer::UnmatchedBrace;
-use core::alloc::GlobalCoAllocMeta;
 pub use attr_wrapper::AttrWrapper;
+use core::alloc::GlobalCoAllocMeta;
 pub use diagnostics::AttemptLocalParseRecovery;
 pub(crate) use item::FnParseMode;
 pub use pat::{CommaRecoveryMode, RecoverColon, RecoverComma};

library/alloc/src/boxed.rs

@@ -641,7 +641,8 @@ impl<T> Box<[T]> {
     #[unstable(feature = "new_uninit", issue = "63291")]
     #[must_use]
     pub fn new_uninit_slice(len: usize) -> Box<[mem::MaybeUninit<T>]> {
-        unsafe { RawVec::with_capacity(len).into_box(len) }
+        // false = no need for co-alloc metadata, since it would get lost once converted to Box.
+        unsafe { RawVec::<T, Global, false>::with_capacity(len).into_box(len) }
     }
 
     /// Constructs a new boxed slice with uninitialized contents, with the memory
@@ -666,7 +667,8 @@ impl<T> Box<[T]> {
     #[unstable(feature = "new_uninit", issue = "63291")]
     #[must_use]
     pub fn new_zeroed_slice(len: usize) -> Box<[mem::MaybeUninit<T>]> {
-        unsafe { RawVec::with_capacity_zeroed(len).into_box(len) }
+        // false = no need for co-alloc metadata, since it would get lost once converted to Box.
+        unsafe { RawVec::<T, Global, false>::with_capacity_zeroed(len).into_box(len) }
     }
 
     /// Constructs a new boxed slice with uninitialized contents. Returns an error if
@@ -698,7 +700,12 @@ impl<T> Box<[T]> {
                 Err(_) => return Err(AllocError),
             };
             let ptr = Global.allocate(layout)?;
-            Ok(RawVec::from_raw_parts_in(ptr.as_mut_ptr() as *mut _, len, Global).into_box(len))
+            Ok(RawVec::<T, Global, false>::from_raw_parts_in(
+                ptr.as_mut_ptr() as *mut _,
+                len,
+                Global,
+            )
+            .into_box(len))
         }
     }
 
@@ -730,12 +737,20 @@ impl<T> Box<[T]> {
                 Err(_) => return Err(AllocError),
             };
             let ptr = Global.allocate_zeroed(layout)?;
-            Ok(RawVec::from_raw_parts_in(ptr.as_mut_ptr() as *mut _, len, Global).into_box(len))
+            Ok(RawVec::<T, Global, false>::from_raw_parts_in(
+                ptr.as_mut_ptr() as *mut _,
+                len,
+                Global,
+            )
+            .into_box(len))
         }
     }
 }
 
-impl<T, A: Allocator> Box<[T], A> {
+impl<T, A: Allocator> Box<[T], A>
+where
+    [(); core::alloc::co_alloc_metadata_num_slots::<A>()]:,
+{
     /// Constructs a new boxed slice with uninitialized contents in the provided allocator.
     ///
     /// # Examples
@@ -762,8 +777,13 @@ impl<T, A: Allocator> Box<[T], A> {
     #[unstable(feature = "allocator_api", issue = "32838")]
     // #[unstable(feature = "new_uninit", issue = "63291")]
     #[must_use]
-    pub fn new_uninit_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A> {
-        unsafe { RawVec::with_capacity_in(len, alloc).into_box(len) }
+    #[allow(unused_braces)]
+    pub fn new_uninit_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A>
+    where
+        // false = no need for co-alloc metadata, since it would get lost once converted to Box.
+        [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<A>(false)]:,
+    {
+        unsafe { RawVec::<T, A, false>::with_capacity_in(len, alloc).into_box(len) }
     }
 
     /// Constructs a new boxed slice with uninitialized contents in the provided allocator,
@@ -790,8 +810,13 @@ impl<T, A: Allocator> Box<[T], A> {
     #[unstable(feature = "allocator_api", issue = "32838")]
     // #[unstable(feature = "new_uninit", issue = "63291")]
     #[must_use]
-    pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A> {
-        unsafe { RawVec::with_capacity_zeroed_in(len, alloc).into_box(len) }
+    #[allow(unused_braces)]
+    pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A>
+    where
+        // false = no need for co-alloc metadata, since it would get lost once converted to Box.
+        [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<A>(false)]:,
+    {
+        unsafe { RawVec::<T, A, false>::with_capacity_zeroed_in(len, alloc).into_box(len) }
     }
 }
 
@@ -1496,7 +1521,8 @@ impl<T: Copy> From<&[T]> for Box<[T]> {
     /// ```
     fn from(slice: &[T]) -> Box<[T]> {
         let len = slice.len();
-        let buf = RawVec::with_capacity(len);
+        // false = no need for co-alloc metadata, since it would get lost once converted to Box.
+        let buf = RawVec::<T, Global, false>::with_capacity(len);
         unsafe {
             ptr::copy_nonoverlapping(slice.as_ptr(), buf.ptr(), len);
             buf.into_box(slice.len()).assume_init()
@@ -1661,8 +1687,12 @@ impl<T, const N: usize> TryFrom<Box<[T]>> for Box<[T; N]> {
 
 #[cfg(not(no_global_oom_handling))]
 #[stable(feature = "boxed_array_try_from_vec", since = "1.66.0")]
-impl<T, const N: usize> TryFrom<Vec<T>> for Box<[T; N]> {
-    type Error = Vec<T>;
+impl<T, const N: usize, const COOP_PREFERRED: bool> TryFrom<Vec<T, Global, COOP_PREFERRED>>
+    for Box<[T; N]>
+where
+    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<Global>(COOP_PREFERRED)]:,
+{
+    type Error = Vec<T, Global, COOP_PREFERRED>;
 
     /// Attempts to convert a `Vec<T>` into a `Box<[T; N]>`.
     ///
@@ -1682,7 +1712,7 @@ impl<T, const N: usize> TryFrom<Vec<T>> for Box<[T; N]> {
     /// let state: Box<[f32; 100]> = vec![1.0; 100].try_into().unwrap();
     /// assert_eq!(state.len(), 100);
     /// ```
-    fn try_from(vec: Vec<T>) -> Result<Self, Self::Error> {
+    fn try_from(vec: Vec<T, Global, COOP_PREFERRED>) -> Result<Self, Self::Error> {
         if vec.len() == N {
             let boxed_slice = vec.into_boxed_slice();
             Ok(unsafe { boxed_slice_as_array_unchecked(boxed_slice) })
@@ -2019,10 +2049,14 @@ impl<I> FromIterator<I> for Box<[I]> {
 
 #[cfg(not(no_global_oom_handling))]
 #[stable(feature = "box_slice_clone", since = "1.3.0")]
-impl<T: Clone, A: Allocator + Clone> Clone for Box<[T], A> {
+impl<T: Clone, A: Allocator + Clone> Clone for Box<[T], A>
+where
+    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<A>(false)]:,
+{
     fn clone(&self) -> Self {
         let alloc = Box::allocator(self).clone();
-        self.to_vec_in(alloc).into_boxed_slice()
+        // false = no need for co-alloc metadata, since it would get lost once converted to the boxed slice.
+        self.to_vec_in::<A, false>(alloc).into_boxed_slice()
     }
 
     fn clone_from(&mut self, other: &Self) {

library/alloc/src/collections/binary_heap/mod.rs

@@ -150,9 +150,12 @@ use core::num::NonZeroUsize;
 use core::ops::{Deref, DerefMut};
 use core::ptr;
 
+use crate::alloc::Global;
+
 use crate::collections::TryReserveError;
 use crate::slice;
 use crate::vec::{self, AsVecIntoIter, Vec};
+use crate::DEFAULT_COOP_PREFERRED;
 
 use super::SpecExtend;
 
@@ -1241,7 +1244,8 @@ impl<T> BinaryHeap<T> {
     /// ```
     #[inline]
     #[stable(feature = "drain", since = "1.6.0")]
-    pub fn drain(&mut self) -> Drain<'_, T> {
+    #[allow(unused_braces)]
+    pub fn drain(&mut self) -> Drain<'_, T, { SHORT_TERM_VEC_PREFERS_COOP!() }> {
         Drain { iter: self.data.drain(..) }
     }
 
@@ -1521,12 +1525,18 @@ unsafe impl<T: Ord> TrustedLen for IntoIterSorted<T> {}
 /// [`drain`]: BinaryHeap::drain
 #[stable(feature = "drain", since = "1.6.0")]
 #[derive(Debug)]
-pub struct Drain<'a, T: 'a> {
-    iter: vec::Drain<'a, T>,
+pub struct Drain<'a, T: 'a, const COOP_PREFERRED: bool>
+where
+    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<Global>(COOP_PREFERRED)]:,
+{
+    iter: vec::Drain<'a, T, Global, COOP_PREFERRED>,
 }
 
 #[stable(feature = "drain", since = "1.6.0")]
-impl<T> Iterator for Drain<'_, T> {
+impl<T, const COOP_PREFERRED: bool> Iterator for Drain<'_, T, COOP_PREFERRED>
+where
+    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<Global>(COOP_PREFERRED)]:,
+{
     type Item = T;
 
     #[inline]
@@ -1541,22 +1551,31 @@ impl<T> Iterator for Drain<'_, T> {
 }
 
 #[stable(feature = "drain", since = "1.6.0")]
-impl<T> DoubleEndedIterator for Drain<'_, T> {
+impl<T, const COOP_PREFERRED: bool> DoubleEndedIterator for Drain<'_, T, COOP_PREFERRED>
+where
+    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<Global>(COOP_PREFERRED)]:,
+{
     #[inline]
     fn next_back(&mut self) -> Option<T> {
         self.iter.next_back()
     }
 }
 
 #[stable(feature = "drain", since = "1.6.0")]
-impl<T> ExactSizeIterator for Drain<'_, T> {
+impl<T, const COOP_PREFERRED: bool> ExactSizeIterator for Drain<'_, T, COOP_PREFERRED>
+where
+    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<Global>(COOP_PREFERRED)]:,
+{
     fn is_empty(&self) -> bool {
         self.iter.is_empty()
     }
 }
 
 #[stable(feature = "fused", since = "1.26.0")]
-impl<T> FusedIterator for Drain<'_, T> {}
+impl<T, const COOP_PREFERRED: bool> FusedIterator for Drain<'_, T, COOP_PREFERRED> where
+    [(); core::alloc::co_alloc_metadata_num_slots_with_preference::<Global>(COOP_PREFERRED)]:
+{
+}
 
 /// A draining iterator over the elements of a `BinaryHeap`.
 ///
@@ -1644,7 +1663,8 @@ impl<T: Ord, const N: usize> From<[T; N]> for BinaryHeap<T> {
 }
 
 #[stable(feature = "binary_heap_extras_15", since = "1.5.0")]
-impl<T> From<BinaryHeap<T>> for Vec<T> {
+#[allow(unused_braces)]
+impl<T> From<BinaryHeap<T>> for Vec<T, Global, { DEFAULT_COOP_PREFERRED!() }> {
     /// Converts a `BinaryHeap<T>` into a `Vec<T>`.
     ///
     /// This conversion requires no data movement or allocation, and has