make prefetch intrinsics safe

Author: folkertdev

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -330,10 +330,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     _ => bug!(),
                 };
                 let ptr = args[0].immediate();
+                let locality = fn_args.const_at(1).to_value().valtree.unwrap_leaf().to_u32() as i32;
                 self.call_intrinsic(
                     "llvm.prefetch",
                     &[self.val_ty(ptr)],
-                    &[ptr, self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type)],
+                    &[
+                        ptr,
+                        self.const_i32(rw),
+                        self.const_i32(locality),
+                        self.const_i32(cache_type),
+                    ],
                 )
             }
             sym::carrying_mul_add => {

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -136,6 +136,10 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi
         | sym::round_ties_even_f64
         | sym::round_ties_even_f128
         | sym::autodiff
+        | sym::prefetch_read_data
+        | sym::prefetch_write_data
+        | sym::prefetch_read_instruction
+        | sym::prefetch_write_instruction
         | sym::const_eval_select => hir::Safety::Safe,
         _ => hir::Safety::Unsafe,
     };
@@ -218,7 +222,7 @@ pub(crate) fn check_intrinsic_type(
         | sym::prefetch_write_data
         | sym::prefetch_read_instruction
         | sym::prefetch_write_instruction => {
-            (1, 0, vec![Ty::new_imm_ptr(tcx, param(0)), tcx.types.i32], tcx.types.unit)
+            (1, 1, vec![Ty::new_imm_ptr(tcx, param(0))], tcx.types.unit)
         }
         sym::needs_drop => (1, 0, vec![], tcx.types.bool),
 

library/core/src/intrinsics/mod.rs

@@ -265,71 +265,67 @@ pub unsafe fn atomic_singlethreadfence<const ORD: AtomicOrdering>();
 /// Prefetches have no effect on the behavior of the program but can change its performance
 /// characteristics.
 ///
-/// The `locality` argument must be a constant integer and is a temporal locality specifier
-/// ranging from (0) - no locality, to (3) - extremely local keep in cache.
+/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality,
+/// to (3) - extremely local keep in cache.
 ///
 /// This intrinsic does not have a stable counterpart.
 #[rustc_intrinsic]
 #[rustc_nounwind]
 #[miri::intrinsic_fallback_is_spec]
-pub unsafe fn prefetch_read_data<T>(data: *const T, locality: i32) {
+pub const fn prefetch_read_data<T, const LOCALITY: i32>(data: *const T) {
     // This operation is a no-op, unless it is overridden by the backend.
     let _ = data;
-    let _ = locality;
 }
 
 /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
 /// for the given address if supported; otherwise, it is a no-op.
 /// Prefetches have no effect on the behavior of the program but can change its performance
 /// characteristics.
 ///
-/// The `locality` argument must be a constant integer and is a temporal locality specifier
-/// ranging from (0) - no locality, to (3) - extremely local keep in cache.
+/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality,
+/// to (3) - extremely local keep in cache.
 ///
 /// This intrinsic does not have a stable counterpart.
 #[rustc_intrinsic]
 #[rustc_nounwind]
 #[miri::intrinsic_fallback_is_spec]
-pub unsafe fn prefetch_write_data<T>(data: *const T, locality: i32) {
+pub const fn prefetch_write_data<T, const LOCALITY: i32>(data: *const T) {
     // This operation is a no-op, unless it is overridden by the backend.
     let _ = data;
-    let _ = locality;
 }
 
 /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
 /// for the given address if supported; otherwise, it is a no-op.
 /// Prefetches have no effect on the behavior of the program but can change its performance
 /// characteristics.
 ///
-/// The `locality` argument must be a constant integer and is a temporal locality specifier
-/// ranging from (0) - no locality, to (3) - extremely local keep in cache.
+/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality,
+/// to (3) - extremely local keep in cache.
 ///
 /// This intrinsic does not have a stable counterpart.
 #[rustc_intrinsic]
 #[rustc_nounwind]
 #[miri::intrinsic_fallback_is_spec]
-pub unsafe fn prefetch_read_instruction<T>(data: *const T, locality: i32) {
+pub const fn prefetch_read_instruction<T, const LOCALITY: i32>(data: *const T) {
     // This operation is a no-op, unless it is overridden by the backend.
     let _ = data;
-    let _ = locality;
 }
 
 /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
 /// for the given address if supported; otherwise, it is a no-op.
 /// Prefetches have no effect on the behavior of the program but can change its performance
 /// characteristics.
 ///
-/// The `locality` argument must be a constant integer and is a temporal locality specifier
-/// ranging from (0) - no locality, to (3) - extremely local keep in cache.
+/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality,
+/// to (3) - extremely local keep in cache.
 ///
 /// This intrinsic does not have a stable counterpart.
 #[rustc_intrinsic]
 #[rustc_nounwind]
 #[miri::intrinsic_fallback_is_spec]
-pub unsafe fn prefetch_write_instruction<T>(data: *const T, locality: i32) {
+pub const fn prefetch_write_instruction<T, const LOCALITY: i32>(data: *const T) {
     // This operation is a no-op, unless it is overridden by the backend.
     let _ = data;
-    let _ = locality;
 }
 
 /// Executes a breakpoint trap, for inspection by a debugger.

src/tools/miri/tests/pass/prefetch.rs

@@ -1,23 +1,23 @@
 #![feature(core_intrinsics)]
 
+// Test that these intrinsics work. Their behavior should be a no-op.
+
 fn main() {
     static X: [u8; 8] = [0; 8];
 
-    unsafe {
-        ::std::intrinsics::prefetch_read_data(::std::ptr::null::<u8>(), 1);
-        ::std::intrinsics::prefetch_read_data(::std::ptr::dangling::<u8>(), 2);
-        ::std::intrinsics::prefetch_read_data(X.as_ptr(), 3);
+    ::std::intrinsics::prefetch_read_data::<_, 1>(::std::ptr::null::<u8>());
+    ::std::intrinsics::prefetch_read_data::<_, 2>(::std::ptr::dangling::<u8>());
+    ::std::intrinsics::prefetch_read_data::<_, 3>(X.as_ptr());
 
-        ::std::intrinsics::prefetch_write_data(::std::ptr::null::<u8>(), 1);
-        ::std::intrinsics::prefetch_write_data(::std::ptr::dangling::<u8>(), 2);
-        ::std::intrinsics::prefetch_write_data(X.as_ptr(), 3);
+    ::std::intrinsics::prefetch_write_data::<_, 1>(::std::ptr::null::<u8>());
+    ::std::intrinsics::prefetch_write_data::<_, 2>(::std::ptr::dangling::<u8>());
+    ::std::intrinsics::prefetch_write_data::<_, 3>(X.as_ptr());
 
-        ::std::intrinsics::prefetch_read_instruction(::std::ptr::null::<u8>(), 1);
-        ::std::intrinsics::prefetch_read_instruction(::std::ptr::dangling::<u8>(), 2);
-        ::std::intrinsics::prefetch_read_instruction(X.as_ptr(), 3);
+    ::std::intrinsics::prefetch_read_instruction::<_, 1>(::std::ptr::null::<u8>());
+    ::std::intrinsics::prefetch_read_instruction::<_, 2>(::std::ptr::dangling::<u8>());
+    ::std::intrinsics::prefetch_read_instruction::<_, 3>(X.as_ptr());
 
-        ::std::intrinsics::prefetch_write_instruction(::std::ptr::null::<u8>(), 1);
-        ::std::intrinsics::prefetch_write_instruction(::std::ptr::dangling::<u8>(), 2);
-        ::std::intrinsics::prefetch_write_instruction(X.as_ptr(), 3);
-    }
+    ::std::intrinsics::prefetch_write_instruction::<_, 1>(::std::ptr::null::<u8>());
+    ::std::intrinsics::prefetch_write_instruction::<_, 2>(::std::ptr::dangling::<u8>());
+    ::std::intrinsics::prefetch_write_instruction::<_, 3>(X.as_ptr());
 }

tests/codegen-llvm/intrinsics/prefetch.rs

@@ -9,56 +9,48 @@ use std::intrinsics::{
 
 #[no_mangle]
 pub fn check_prefetch_read_data(data: &[i8]) {
-    unsafe {
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1)
-        prefetch_read_data(data.as_ptr(), 0);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1)
-        prefetch_read_data(data.as_ptr(), 1);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1)
-        prefetch_read_data(data.as_ptr(), 2);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1)
-        prefetch_read_data(data.as_ptr(), 3);
-    }
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1)
+    prefetch_read_data::<_, 0>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1)
+    prefetch_read_data::<_, 1>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1)
+    prefetch_read_data::<_, 2>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1)
+    prefetch_read_data::<_, 3>(data.as_ptr());
 }
 
 #[no_mangle]
 pub fn check_prefetch_write_data(data: &[i8]) {
-    unsafe {
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1)
-        prefetch_write_data(data.as_ptr(), 0);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1)
-        prefetch_write_data(data.as_ptr(), 1);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1)
-        prefetch_write_data(data.as_ptr(), 2);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1)
-        prefetch_write_data(data.as_ptr(), 3);
-    }
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1)
+    prefetch_write_data::<_, 0>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1)
+    prefetch_write_data::<_, 1>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1)
+    prefetch_write_data::<_, 2>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1)
+    prefetch_write_data::<_, 3>(data.as_ptr());
 }
 
 #[no_mangle]
 pub fn check_prefetch_read_instruction(data: &[i8]) {
-    unsafe {
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0)
-        prefetch_read_instruction(data.as_ptr(), 0);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0)
-        prefetch_read_instruction(data.as_ptr(), 1);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0)
-        prefetch_read_instruction(data.as_ptr(), 2);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0)
-        prefetch_read_instruction(data.as_ptr(), 3);
-    }
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0)
+    prefetch_read_instruction::<_, 0>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0)
+    prefetch_read_instruction::<_, 1>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0)
+    prefetch_read_instruction::<_, 2>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0)
+    prefetch_read_instruction::<_, 3>(data.as_ptr());
 }
 
 #[no_mangle]
 pub fn check_prefetch_write_instruction(data: &[i8]) {
-    unsafe {
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0)
-        prefetch_write_instruction(data.as_ptr(), 0);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0)
-        prefetch_write_instruction(data.as_ptr(), 1);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0)
-        prefetch_write_instruction(data.as_ptr(), 2);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0)
-        prefetch_write_instruction(data.as_ptr(), 3);
-    }
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0)
+    prefetch_write_instruction::<_, 0>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0)
+    prefetch_write_instruction::<_, 1>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0)
+    prefetch_write_instruction::<_, 2>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0)
+    prefetch_write_instruction::<_, 3>(data.as_ptr());
 }