Skip to content

Commit 4e886d6

Browse files
borsbors
committed
Auto merge of rust-lang#87827 - eddyb:wrapperless-mem-replace, r=m-ou-se
Avoid using the `copy_nonoverlapping` wrapper through `mem::replace`. This is a much simpler way to achieve the pre-rust-lang#86003 behavior of `mem::replace` not needing dynamically-sized `memcpy`s (at least before inlining), than re-doing rust-lang#81238 (which needs rust-lang#86699 or something similar). I didn't notice it until recently, but `ptr::write` already explicitly avoided using the wrapper, while `ptr::read` just called the wrapper (and was the reason for us observing any behavior change from rust-lang#86003 in Rust-GPU). <hr/> The codegen test I've added fails without the change to `core::ptr::read` like this (ignore the `v0` mangling, I was using a worktree with it turned on by default, for this): ```llvm 13: ; core::intrinsics::copy_nonoverlapping::<u8> 14: ; Function Attrs: inlinehint nonlazybind uwtable 15: define internal void `@_RINvNtCscK5tvALCJol_4core10intrinsics19copy_nonoverlappinghECsaS4X3EinRE8_25mem_replace_direct_memcpy(i8*` %src, i8* %dst, i64 %count) unnamed_addr #0 { 16: start: 17: %0 = mul i64 %count, 1 18: call void `@llvm.memcpy.p0i8.p0i8.i64(i8*` align 1 %dst, i8* align 1 %src, i64 %0, i1 false) not:17 !~~~~~~~~~~~~~~~~~~~~~ error: no match expected 19: ret void 20: } ``` With the `core::ptr::read` change, `core::intrinsics::copy_nonoverlapping` doesn't get instantiated and the test passes. <hr/> r? `@m-ou-se` cc `@nagisa` (codegen test) `@oli-obk` / `@RalfJung` (miri diagnostic changes)
2 parents e8c25f2 + a1d014b commit 4e886d6

File tree

3 files changed

+47
-30
lines changed

3 files changed

+47
-30
lines changed

‎library/core/src/ptr/mod.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -685,6 +685,13 @@ pub const unsafe fn replace<T>(dst: *mut T, mut src: T) -> T {
685685
#[stable(feature = "rust1", since = "1.0.0")]
686686
#[rustc_const_unstable(feature = "const_ptr_read", issue = "80377")]
687687
pub const unsafe fn read<T>(src: *const T) -> T {
688+
// We are calling the intrinsics directly to avoid function calls in the generated code
689+
// as `intrinsics::copy_nonoverlapping` is a wrapper function.
690+
extern "rust-intrinsic" {
691+
#[rustc_const_unstable(feature = "const_intrinsic_copy", issue = "80697")]
692+
fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: usize);
693+
}
694+
688695
let mut tmp = MaybeUninit::<T>::uninit();
689696
// SAFETY: the caller must guarantee that `src` is valid for reads.
690697
// `src` cannot overlap `tmp` because `tmp` was just allocated on

‎src/test/codegen/mem-replace-direct-memcpy.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// This test ensures that `mem::replace::<T>` only ever calls `@llvm.memcpy`
2+
// with `size_of::<T>()` as the size, and never goes through any wrapper that
3+
// may e.g. multiply `size_of::<T>()` with a variable "count" (which is only
4+
// known to be `1` after inlining).
5+
6+
// compile-flags: -C no-prepopulate-passes
7+
8+
#![crate_type = "lib"]
9+
10+
pub fn replace_byte(dst: &mut u8, src: u8) -> u8 {
11+
std::mem::replace(dst, src)
12+
}
13+
14+
// NOTE(eddyb) the `CHECK-NOT`s ensure that the only calls of `@llvm.memcpy` in
15+
// the entire output, are the two direct calls we want, from `ptr::{read,write}`.
16+
17+
// CHECK-NOT: call void @llvm.memcpy
18+
// CHECK: ; core::ptr::read
19+
// CHECK-NOT: call void @llvm.memcpy
20+
// CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{.*}}(i8* align 1 %{{.*}}, i8* align 1 %src, i{{.*}} 1, i1 false)
21+
// CHECK-NOT: call void @llvm.memcpy
22+
// CHECK: ; core::ptr::write
23+
// CHECK-NOT: call void @llvm.memcpy
24+
// CHECK: call void @llvm.memcpy.p0i8.p0i8.i{{.*}}(i8* align 1 %dst, i8* align 1 %src, i{{.*}} 1, i1 false)
25+
// CHECK-NOT: call void @llvm.memcpy

‎src/test/ui/const-ptr/out_of_bounds_read.stderr‎

Lines changed: 15 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,25 @@
11
error[E0080]: evaluation of constant value failed
2-
--> $SRC_DIR/core/src/intrinsics.rs:LL:COL
3-
|
4-
LL | unsafe { copy_nonoverlapping(src, dst, count) }
5-
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
6-
| |
7-
| memory access failed: alloc7 has size 4, so pointer to 4 bytes starting at offset 4 is out-of-bounds
8-
| inside `copy_nonoverlapping::<u32>` at $SRC_DIR/core/src/intrinsics.rs:LL:COL
9-
|
10-
::: $SRC_DIR/core/src/ptr/mod.rs:LL:COL
2+
--> $SRC_DIR/core/src/ptr/mod.rs:LL:COL
113
|
124
LL | copy_nonoverlapping(src, tmp.as_mut_ptr(), 1);
13-
| --------------------------------------------- inside `std::ptr::read::<u32>` at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
5+
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
6+
| |
7+
| memory access failed: alloc7 has size 4, so pointer to 4 bytes starting at offset 4 is out-of-bounds
8+
| inside `std::ptr::read::<u32>` at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
149
|
1510
::: $DIR/out_of_bounds_read.rs:13:33
1611
|
1712
LL | const _READ: u32 = unsafe { ptr::read(PAST_END_PTR) };
1813
| ----------------------- inside `_READ` at $DIR/out_of_bounds_read.rs:13:33
1914

2015
error[E0080]: evaluation of constant value failed
21-
--> $SRC_DIR/core/src/intrinsics.rs:LL:COL
22-
|
23-
LL | unsafe { copy_nonoverlapping(src, dst, count) }
24-
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
25-
| |
26-
| memory access failed: alloc7 has size 4, so pointer to 4 bytes starting at offset 4 is out-of-bounds
27-
| inside `copy_nonoverlapping::<u32>` at $SRC_DIR/core/src/intrinsics.rs:LL:COL
28-
|
29-
::: $SRC_DIR/core/src/ptr/mod.rs:LL:COL
16+
--> $SRC_DIR/core/src/ptr/mod.rs:LL:COL
3017
|
3118
LL | copy_nonoverlapping(src, tmp.as_mut_ptr(), 1);
32-
| --------------------------------------------- inside `std::ptr::read::<u32>` at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
19+
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
20+
| |
21+
| memory access failed: alloc7 has size 4, so pointer to 4 bytes starting at offset 4 is out-of-bounds
22+
| inside `std::ptr::read::<u32>` at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
3323
|
3424
::: $SRC_DIR/core/src/ptr/const_ptr.rs:LL:COL
3525
|
@@ -42,18 +32,13 @@ LL | const _CONST_READ: u32 = unsafe { PAST_END_PTR.read() };
4232
| ------------------- inside `_CONST_READ` at $DIR/out_of_bounds_read.rs:14:39
4333

4434
error[E0080]: evaluation of constant value failed
45-
--> $SRC_DIR/core/src/intrinsics.rs:LL:COL
46-
|
47-
LL | unsafe { copy_nonoverlapping(src, dst, count) }
48-
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
49-
| |
50-
| memory access failed: alloc7 has size 4, so pointer to 4 bytes starting at offset 4 is out-of-bounds
51-
| inside `copy_nonoverlapping::<u32>` at $SRC_DIR/core/src/intrinsics.rs:LL:COL
52-
|
53-
::: $SRC_DIR/core/src/ptr/mod.rs:LL:COL
35+
--> $SRC_DIR/core/src/ptr/mod.rs:LL:COL
5436
|
5537
LL | copy_nonoverlapping(src, tmp.as_mut_ptr(), 1);
56-
| --------------------------------------------- inside `std::ptr::read::<u32>` at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
38+
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
39+
| |
40+
| memory access failed: alloc7 has size 4, so pointer to 4 bytes starting at offset 4 is out-of-bounds
41+
| inside `std::ptr::read::<u32>` at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
5742
|
5843
::: $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
5944
|

0 commit comments

Comments
 (0)