Auto merge of #362 - JustForFun88:check_layout_size, r=Amanieu

 Check that layout size fits in isize in Layout

Since [rust-lang/rust#95295](rust-lang/rust#95295) we need check that Layout `size`, when rounded up to the nearest multiple of `align`, must not overflow isize (i.e., the rounded value must be less than or equal to `isize::MAX`)

Author: bors

src/map.rs

@@ -8044,27 +8044,32 @@ mod test_map {
     fn test_try_reserve() {
         use crate::TryReserveError::{AllocError, CapacityOverflow};
 
-        const MAX_USIZE: usize = usize::MAX;
+        const MAX_ISIZE: usize = isize::MAX as usize;
 
         let mut empty_bytes: HashMap<u8, u8> = HashMap::new();
 
-        if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_USIZE) {
+        if let Err(CapacityOverflow) = empty_bytes.try_reserve(usize::MAX) {
         } else {
             panic!("usize::MAX should trigger an overflow!");
         }
 
-        if let Err(AllocError { .. }) = empty_bytes.try_reserve(MAX_USIZE / 16) {
+        if let Err(CapacityOverflow) = empty_bytes.try_reserve(MAX_ISIZE) {
+        } else {
+            panic!("isize::MAX should trigger an overflow!");
+        }
+
+        if let Err(AllocError { .. }) = empty_bytes.try_reserve(MAX_ISIZE / 5) {
         } else {
             // This may succeed if there is enough free memory. Attempt to
             // allocate a few more hashmaps to ensure the allocation will fail.
             let mut empty_bytes2: HashMap<u8, u8> = HashMap::new();
-            let _ = empty_bytes2.try_reserve(MAX_USIZE / 16);
+            let _ = empty_bytes2.try_reserve(MAX_ISIZE / 5);
             let mut empty_bytes3: HashMap<u8, u8> = HashMap::new();
-            let _ = empty_bytes3.try_reserve(MAX_USIZE / 16);
+            let _ = empty_bytes3.try_reserve(MAX_ISIZE / 5);
             let mut empty_bytes4: HashMap<u8, u8> = HashMap::new();
-            if let Err(AllocError { .. }) = empty_bytes4.try_reserve(MAX_USIZE / 16) {
+            if let Err(AllocError { .. }) = empty_bytes4.try_reserve(MAX_ISIZE / 5) {
             } else {
-                panic!("usize::MAX / 8 should trigger an OOM!");
+                panic!("isize::MAX / 5 should trigger an OOM!");
             }
         }
     }

src/raw/mod.rs

@@ -248,6 +248,12 @@ impl TableLayout {
             size.checked_mul(buckets)?.checked_add(ctrl_align - 1)? & !(ctrl_align - 1);
         let len = ctrl_offset.checked_add(buckets + Group::WIDTH)?;
 
+        // We need an additional check to ensure that the allocation doesn't
+        // exceed `isize::MAX` (https://github.com/rust-lang/rust/pull/95295).
+        if len > isize::MAX as usize - (ctrl_align - 1) {
+            return None;
+        }
+
         Some((
             unsafe { Layout::from_size_align_unchecked(len, ctrl_align) },
             ctrl_offset,
@@ -1078,15 +1084,6 @@ impl<A: Allocator + Clone> RawTableInner<A> {
             None => return Err(fallibility.capacity_overflow()),
         };
 
-        // We need an additional check to ensure that the allocation doesn't
-        // exceed `isize::MAX`. We can skip this check on 64-bit systems since
-        // such allocations will never succeed anyways.
-        //
-        // This mirrors what Vec does in the standard library.
-        if mem::size_of::<usize>() < 8 && layout.size() > isize::MAX as usize {
-            return Err(fallibility.capacity_overflow());
-        }
-
         let ptr: NonNull<u8> = match do_alloc(&alloc, layout) {
             Ok(block) => block.cast(),
             Err(_) => return Err(fallibility.alloc_err(layout)),