Add element-wise atomic memory operations

tmccombs · tmccombs · commit 035fdfc7c2b0 · 2019-03-31T00:18:30.000-06:00
WIP
diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs
@@ -962,6 +962,33 @@ extern "rust-intrinsic" {
     /// value is not necessarily valid to be used to actually access memory.
     pub fn arith_offset<T>(dst: *const T, offset: isize) -> *const T;
 
+    /// Equivalent to the appropriate `llvm.memcpy.element.unordered.atomic.p0i8.p0i8.*` intrinsic, with
+    /// a size of `count` * `size_of::<T>()`, an alignment of
+    /// `min_align_of::<T>()`, and an element size of `size_of::<T>()`.
+    ///
+    /// `size_of::<T>` must be an integer power of two no larger than the
+    /// target-specific atomic access size limit.
+    #[cfg(not(stage0))]
+    pub fn atomic_element_copy_nonoverlapping_memory_unordered<T>(dst: *mut T, src: *const T, count: usize);
+
+    /// Equivalent to the appropriate `llvm.memmove.unordered.atomic.p0i8.p0i8.*` intrinsic, with
+    /// a size of `count` * `size_of::<T>()`, an alignment of
+    /// `min_align_of::<T>()`, and an element size of `size_of::<T>()`.
+    ///
+    /// `size_of::<T>` must be an integer power of two no larger than the
+    /// target-specific atomic access size limit.
+    #[cfg(not(stage0))]
+    pub fn atomic_element_copy_memory_unordered<T>(dst: *mut T, src: *const T, count: usize);
+
+    /// Equivalent to the appropriate `llvm.memset.unordered.atomic.p0i8.p0i8.*` intrinsic, with
+    /// a size of `count` * `size_of::<T>()`, an alignment of
+    /// `min_align_of::<T>()`, and an element size of `size_of::<T>()`.
+    ///
+    /// `size_of::<T>` must be an integer power of two no larger than the
+    /// target-specific atomic access size limit.
+    #[cfg(not(stage0))]
+    pub fn atomic_element_set_memory_unordered<T>(dst: *mut T, val: u8, count: usize);
+
     /// Equivalent to the appropriate `llvm.memcpy.p0i8.0i8.*` intrinsic, with
     /// a size of `count` * `size_of::<T>()` and an alignment of
     /// `min_align_of::<T>()`
diff --git a/src/librustc_codegen_llvm/builder.rs b/src/librustc_codegen_llvm/builder.rs
@@ -966,6 +966,48 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         self.call(llintrinsicfn, &[ptr, fill_byte, size, align, volatile], None);
     }
 
+    fn atomic_element_unordered_memcpy(&mut self, dst: &'ll Value, dst_align: Align,
+                                        src: &'ll Value, src_align: Align,
+                                        size: &'ll Value, element_size: u32) {
+        let size = self.intcast(size, self.type_isize(), false);
+        let dst = self.pointercast(dst, self.type_i8p());
+        let src = self.pointercast(src, self.type_i8p());
+        unsafe {
+            llvm::LLVMRustBuildElementUnorderedAtomicMemCpy(self.llbuilder, dst, dst_align.bytes() as c_uint,
+                                                            src, src_align.bytes() as c_uint, size, element_size);
+        }
+    }
+
+    fn atomic_element_unordered_memmove(&mut self, dst: &'ll Value, dst_align: Align,
+                                        src: &'ll Value, src_align: Align,
+                                        size: &'ll Value, element_size: u32) {
+        let size = self.intcast(size, self.type_isize(), false);
+        let dst = self.pointercast(dst, self.type_i8p());
+        let src = self.pointercast(src, self.type_i8p());
+        let ret_ref = unsafe {
+            llvm::LLVMRustBuildElementUnorderedAtomicMemMove(self.llbuilder,
+                                                             dst, dst_align.bytes() as c_uint,
+                                                             src, src_align.bytes() as c_uint,
+                                                             size, element_size)
+        };
+        if ret_ref.is_none() {
+            bug!("llvm.memmove.element.unordered.atomic.* is not supported with LLVM prior to 7.0");
+        }
+    }
+
+    fn atomic_element_unordered_memset(&mut self, ptr: &'ll Value, fill_byte: &'ll Value,
+                                       size: &'ll Value, align: Align, element_size: u32) {
+        let size = self.intcast(size, self.type_isize(), false);
+        let ptr = self.pointercast(ptr, self.type_i8p());
+        let ret_ref = unsafe {
+            llvm::LLVMRustBuildElementUnorderedAtomicMemSet(self.llbuilder, ptr, fill_byte,
+                                                            size, align.bytes() as c_uint, element_size)
+        };
+        if ret_ref.is_none() {
+            bug!("llvm.memset.element.unordered.atomic.* is not supported with LLVM prior to 7.0");
+        }
+    }
+
     fn select(
         &mut self, cond: &'ll Value,
         then_val: &'ll Value,
diff --git a/src/librustc_codegen_llvm/intrinsic.rs b/src/librustc_codegen_llvm/intrinsic.rs
@@ -102,6 +102,13 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         let llret_ty = self.layout_of(ret_ty).llvm_type(self);
         let result = PlaceRef::new_sized(llresult, fn_ty.ret.layout, fn_ty.ret.layout.align.abi);
 
+        let invalid_integer_monomorphization = |ty| {
+            span_invalid_monomorphization_error(tcx.sess, span,
+                                                &format!("invalid monomorphization of `{}` intrinsic: \
+                                                         expected basic integer type, found `{}`", name, ty));
+        };
+
+
         let simple = get_simple_intrinsic(self, name);
         let llval = match name {
             _ if simple.is_some() => {
@@ -503,10 +510,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                             _ => bug!(),
                         },
                     None => {
-                        span_invalid_monomorphization_error(
-                            tcx.sess, span,
-                            &format!("invalid monomorphization of `{}` intrinsic: \
-                                      expected basic integer type, found `{}`", name, ty));
+                        invalid_integer_monomorphization(ty);
                         return;
                     }
                 }
@@ -548,6 +552,17 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                     Err(()) => return
                 }
             }
+            name if name.starts_with("atomic_element_") => {
+                let ty = substs.type_at(0);
+                if int_type_width_signed(ty, self).is_some() {
+                    atomic_element_intrinsic(self, name,
+                                                    substs.type_at(0),
+                                                    args);
+                    return;
+                } else {
+                    return invalid_integer_monomorphization(ty);
+                }
+            }
             // This requires that atomic intrinsics follow a specific naming pattern:
             // "atomic_<operation>[_<ordering>]", and no ordering means SeqCst
             name if name.starts_with("atomic_") => {
@@ -582,12 +597,6 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                     _ => self.sess().fatal("Atomic intrinsic not in correct format"),
                 };
 
-                let invalid_monomorphization = |ty| {
-                    span_invalid_monomorphization_error(tcx.sess, span,
-                        &format!("invalid monomorphization of `{}` intrinsic: \
-                                  expected basic integer type, found `{}`", name, ty));
-                };
-
                 match split[1] {
                     "cxchg" | "cxchgweak" => {
                         let ty = substs.type_at(0);
@@ -610,7 +619,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                             self.store(success, dest.llval, dest.align);
                             return;
                         } else {
-                            return invalid_monomorphization(ty);
+                            return invalid_integer_monomorphization(ty);
                         }
                     }
 
@@ -620,7 +629,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                             let size = self.size_of(ty);
                             self.atomic_load(args[0].immediate(), order, size)
                         } else {
-                            return invalid_monomorphization(ty);
+                            return invalid_integer_monomorphization(ty);
                         }
                     }
 
@@ -636,7 +645,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                             );
                             return;
                         } else {
-                            return invalid_monomorphization(ty);
+                            return invalid_integer_monomorphization(ty);
                         }
                     }
 
@@ -676,7 +685,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                                 order
                             )
                         } else {
-                            return invalid_monomorphization(ty);
+                            return invalid_integer_monomorphization(ty);
                         }
                     }
                 }
@@ -754,6 +763,54 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
     }
 }
 
+fn atomic_element_intrinsic(
+    bx: &mut Builder<'a, 'll, 'tcx>,
+    name: &str,
+    ty: Ty<'tcx>,
+    args: &[OperandRef<'tcx, &'ll Value>],
+) {
+    let (element_size, align) = bx.size_and_align_of(ty);
+    let element_size = element_size.bytes();
+    assert!(element_size <= u32::max_value() as u64);
+
+    let size = bx.mul(bx.const_usize(element_size), args[2].immediate());
+
+    match name {
+        "atomic_element_copy_nonoverlapping_memory_unordered" => {
+            bx.atomic_element_unordered_memcpy(
+                args[0].immediate(),
+                align,
+                args[1].immediate(),
+                align,
+                size,
+                element_size as u32
+            );
+        }
+        "atomic_element_copy_memory_unordered" => {
+            bx.atomic_element_unordered_memmove(
+                args[0].immediate(),
+                align,
+                args[1].immediate(),
+                align,
+                size,
+                element_size as u32
+            );
+        }
+        "atomic_element_set_memory_unordered" => {
+            bx.atomic_element_unordered_memset(
+                args[0].immediate(),
+                args[1].immediate(),
+                size,
+                align,
+                element_size as u32
+            );
+        }
+        _ => {
+            bug!("unknown intrinsic '{}'", name);
+        }
+    }
+}
+
 fn copy_intrinsic(
     bx: &mut Builder<'a, 'll, 'tcx>,
     allow_overlap: bool,
@@ -777,6 +834,7 @@ fn copy_intrinsic(
     }
 }
 
+
 fn memset_intrinsic(
     bx: &mut Builder<'a, 'll, 'tcx>,
     volatile: bool,
diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs
@@ -1154,6 +1154,28 @@ extern "C" {
                                Size: &'a Value,
                                IsVolatile: bool)
                                -> &'a Value;
+    pub fn LLVMRustBuildElementUnorderedAtomicMemCpy(B: &Builder<'a>,
+                                                     Dst: &'a Value,
+                                                     DstAlign: c_uint,
+                                                     Src: &'a Value,
+                                                     SrcAlign: c_uint,
+                                                     Size: &'a Value,
+                                                     ElementSize: u32)
+                                                     -> &'a Value;
+    pub fn LLVMRustBuildElementUnorderedAtomicMemMove(B: &Builder<'a>,
+                                                      Dst: &'a Value,
+                                                      DstAlign: c_uint,
+                                                      Src: &'a Value,
+                                                      SrcAlign: c_uint,
+                                                      Size: &'a Value,
+                                                      ElementSize: u32) -> Option<&'a Value>;
+    pub fn LLVMRustBuildElementUnorderedAtomicMemSet(B: &Builder<'a>,
+                                                     Ptr: &'a Value,
+                                                     Val: &'a Value,
+                                                     Size: &'a Value,
+                                                     Align: c_uint,
+                                                     ElementSize: u32) -> Option<&'a Value>;
+
     pub fn LLVMRustBuildMemMove(B: &Builder<'a>,
                                 Dst: &'a Value,
                                 DstAlign: c_uint,
diff --git a/src/librustc_codegen_ssa/traits/builder.rs b/src/librustc_codegen_ssa/traits/builder.rs
@@ -190,6 +190,33 @@ pub trait BuilderMethods<'a, 'tcx: 'a>:
         flags: MemFlags,
     );
 
+    fn atomic_element_unordered_memcpy(
+        &mut self,
+        dst: Self::Value,
+        dst_align: Align,
+        src: Self::Value,
+        src_align: Align,
+        size: Self::Value,
+        element_size: u32,
+    );
+    fn atomic_element_unordered_memmove(
+        &mut self,
+        dst: Self::Value,
+        dst_align: Align,
+        src: Self::Value,
+        src_align: Align,
+        size: Self::Value,
+        element_size: u32,
+    );
+    fn atomic_element_unordered_memset(
+        &mut self,
+        ptr: Self::Value,
+        fill_byte: Self::Value,
+        size: Self::Value,
+        align: Align,
+        element_size: u32,
+    );
+
     fn select(
         &mut self,
         cond: Self::Value,
diff --git a/src/librustc_typeck/check/intrinsic.rs b/src/librustc_typeck/check/intrinsic.rs
@@ -92,7 +92,7 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
         })
     };
 
-    let (n_tps, inputs, output, unsafety) = if name.starts_with("atomic_") {
+    let (n_tps, inputs, output, unsafety) = if name.starts_with("atomic_") && ! name.starts_with("atomic_element_") {
         let split : Vec<&str> = name.split('_').collect();
         assert!(split.len() >= 2, "Atomic intrinsic in an incorrect format");
 
@@ -197,7 +197,8 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
                ],
                tcx.mk_unit())
             }
-            "volatile_copy_memory" | "volatile_copy_nonoverlapping_memory" => {
+            "volatile_copy_memory" | "volatile_copy_nonoverlapping_memory" |
+            "atomic_element_copy_memory_unordered" | "atomic_element_copy_nonoverlapping_memory_unordered" => {
               (1,
                vec![
                   tcx.mk_ptr(ty::TypeAndMut {
@@ -212,7 +213,7 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
                ],
                tcx.mk_unit())
             }
-            "write_bytes" | "volatile_set_memory" => {
+            "write_bytes" | "volatile_set_memory" | "atomic_element_set_memory_unordered" => {
               (1,
                vec![
                   tcx.mk_ptr(ty::TypeAndMut {
diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp
@@ -1268,6 +1268,42 @@ extern "C" LLVMValueRef LLVMRustBuildMemMove(LLVMBuilderRef B,
 #endif
 }
 
+extern "C" LLVMValueRef LLVMRustBuildElementUnorderedAtomicMemCpy(LLVMBuilderRef B,
+                                                                  LLVMValueRef Dst, unsigned DstAlign,
+                                                                  LLVMValueRef Src, unsigned SrcAlign,
+                                                                  LLVMValueRef Size, uint32_t ElementSize) {
+  return wrap(unwrap(B)->CreateElementUnorderedAtomicMemCpy(
+      unwrap(Dst), DstAlign,
+      unwrap(Src), SrcAlign,
+      unwrap(Size), ElementSize));
+}
+
+extern "C" LLVMValueRef LLVMRustBuildElementUnorderedAtomicMemMove(LLVMBuilderRef B,
+                                                                   LLVMValueRef Dst, unsigned DstAlign,
+                                                                   LLVMValueRef Src, unsigned SrcAlign,
+                                                                   LLVMValueRef Size, uint32_t ElementSize) {
+#if LLVM_VERSION_GE(7, 0)
+  return wrap(unwrap(B)->CreateElementUnorderedAtomicMemMove(
+      unwrap(Dst), DstAlign,
+      unwrap(Src), SrcAlign,
+      unwrap(Size), ElementSize));
+#else
+  return nullptr;
+#endif
+}
+
+extern "C" LLVMValueRef LLVMRustBuildElementUnorderedAtomicMemSet(LLVMBuilderRef B,
+                                                                  LLVMValueRef Ptr, LLVMValueRef Val,
+                                                                  LLVMValueRef Size, unsigned Align, uint32_t ElementSize) {
+#if LLVM_VERSION_GE(7, 0)
+  return wrap(unwrap(B)->CreateElementUnorderedAtomicMemSet(
+      unwrap(Ptr), unwrap(Val),
+      unwrap(Size), Align, ElementSize));
+#else
+  return nullptr;
+#endif
+}
+
 extern "C" LLVMValueRef
 LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn, LLVMValueRef *Args,
                     unsigned NumArgs, LLVMBasicBlockRef Then,
diff --git a/src/test/run-pass/intrinsics/intrinsic-atomics.rs b/src/test/run-pass/intrinsics/intrinsic-atomics.rs
