Merge pull request #254 from rust-lang/feature/simd_scatter_gather

Feature/simd scatter gather

Author: antoyo

Cargo.lock

@@ -22,8 +22,6 @@ src/test/ui/sepcomp/sepcomp-fns-backwards.rs
 src/test/ui/sepcomp/sepcomp-fns.rs
 src/test/ui/sepcomp/sepcomp-statics.rs
 src/test/ui/simd/intrinsic/generic-arithmetic-pass.rs
-src/test/ui/simd/intrinsic/generic-gather-pass.rs
-src/test/ui/simd/type-generic-monomorphisation-extern-nonnull-ptr.rs
 src/test/ui/sse2.rs
 src/test/ui/target-feature/missing-plusminus.rs
 src/test/ui/asm/x86_64/may_unwind.rs
@@ -54,7 +52,6 @@ src/test/ui/issues/issue-43853.rs
 src/test/ui/issues/issue-47364.rs
 src/test/ui/simd/issue-17170.rs
 src/test/ui/simd/issue-39720.rs
-src/test/ui/simd/issue-85915-simd-ptrs.rs
 src/test/ui/simd/issue-89193.rs
 src/test/ui/statics/issue-91050-1.rs
 src/test/ui/statics/issue-91050-2.rs

failing-ui-tests.txt

@@ -37,3 +37,5 @@ src/test/ui/cfg/cfg-panic.rs
 src/test/ui/generator/size-moved-locals.rs
 src/test/ui/macros/rfc-2011-nicer-assert-messages/all-not-available-cases.rs
 src/test/ui/runtime/rt-explody-panic-payloads.rs
+src/test/ui/simd/intrinsic/generic-gather-pass.rs
+src/test/ui/simd/issue-85915-simd-ptrs.rs

failing-ui-tests12.txt

@@ -1419,7 +1419,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
 impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     #[cfg(feature="master")]
     pub fn shuffle_vector(&mut self, v1: RValue<'gcc>, v2: RValue<'gcc>, mask: RValue<'gcc>) -> RValue<'gcc> {
-        let struct_type = mask.get_type().is_struct().expect("mask of struct type");
+        let struct_type = mask.get_type().is_struct().expect("mask should be of struct type");
 
         // TODO(antoyo): use a recursive unqualified() here.
         let vector_type = v1.get_type().unqualified().dyncast_vector().expect("vector type");

src/builder.rs

@@ -14,6 +14,8 @@ use rustc_span::{Span, Symbol, sym};
 use rustc_target::abi::Align;
 
 use crate::builder::Builder;
+#[cfg(feature="master")]
+use crate::context::CodegenCx;
 
 pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>, name: Symbol, callee_ty: Ty<'tcx>, args: &[OperandRef<'tcx, RValue<'gcc>>], ret_ty: Ty<'tcx>, llret_ty: Type<'gcc>, span: Span) -> Result<RValue<'gcc>, ()> {
     // macros for error handling:
@@ -507,6 +509,286 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
         return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
     }
 
+    #[cfg(feature="master")]
+    fn vector_ty<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, elem_ty: Ty<'tcx>, vec_len: u64) -> Type<'gcc> {
+        // FIXME: use cx.layout_of(ty).llvm_type() ?
+        let elem_ty = match *elem_ty.kind() {
+            ty::Int(v) => cx.type_int_from_ty(v),
+            ty::Uint(v) => cx.type_uint_from_ty(v),
+            ty::Float(v) => cx.type_float_from_ty(v),
+            _ => unreachable!(),
+        };
+        cx.type_vector(elem_ty, vec_len)
+    }
+
+    #[cfg(feature="master")]
+    fn gather<'a, 'gcc, 'tcx>(default: RValue<'gcc>, pointers: RValue<'gcc>, mask: RValue<'gcc>, pointer_count: usize, bx: &mut Builder<'a, 'gcc, 'tcx>, in_len: u64, underlying_ty: Ty<'tcx>, invert: bool) -> RValue<'gcc> {
+        let vector_type =
+            if pointer_count > 1 {
+                bx.context.new_vector_type(bx.usize_type, in_len)
+            }
+            else {
+                vector_ty(bx, underlying_ty, in_len)
+            };
+        let elem_type = vector_type.dyncast_vector().expect("vector type").get_element_type();
+
+        let mut values = vec![];
+        for i in 0..in_len {
+            let index = bx.context.new_rvalue_from_long(bx.i32_type, i as i64);
+            let int = bx.context.new_vector_access(None, pointers, index).to_rvalue();
+
+            let ptr_type = elem_type.make_pointer();
+            let ptr = bx.context.new_bitcast(None, int, ptr_type);
+            let value = ptr.dereference(None).to_rvalue();
+            values.push(value);
+        }
+
+        let vector = bx.context.new_rvalue_from_vector(None, vector_type, &values);
+
+        let mut mask_types = vec![];
+        let mut mask_values = vec![];
+        for i in 0..in_len {
+            let index = bx.context.new_rvalue_from_long(bx.i32_type, i as i64);
+            mask_types.push(bx.context.new_field(None, bx.i32_type, "m"));
+            let mask_value = bx.context.new_vector_access(None, mask, index).to_rvalue();
+            let masked = bx.context.new_rvalue_from_int(bx.i32_type, in_len as i32) & mask_value;
+            let value = index + masked;
+            mask_values.push(value);
+        }
+        let mask_type = bx.context.new_struct_type(None, "mask_type", &mask_types);
+        let mask = bx.context.new_struct_constructor(None, mask_type.as_type(), None, &mask_values);
+
+        if invert {
+            bx.shuffle_vector(vector, default, mask)
+        }
+        else {
+            bx.shuffle_vector(default, vector, mask)
+        }
+    }
+
+    #[cfg(feature="master")]
+    if name == sym::simd_gather {
+        // simd_gather(values: <N x T>, pointers: <N x *_ T>,
+        //             mask: <N x i{M}>) -> <N x T>
+        // * N: number of elements in the input vectors
+        // * T: type of the element to load
+        // * M: any integer width is supported, will be truncated to i1
+
+        // All types must be simd vector types
+        require_simd!(in_ty, "first");
+        require_simd!(arg_tys[1], "second");
+        require_simd!(arg_tys[2], "third");
+        require_simd!(ret_ty, "return");
+
+        // Of the same length:
+        let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
+        require!(
+            in_len == out_len,
+            "expected {} argument with length {} (same as input type `{}`), \
+            found `{}` with length {}",
+            "second",
+            in_len,
+            in_ty,
+            arg_tys[1],
+            out_len
+        );
+        require!(
+            in_len == out_len2,
+            "expected {} argument with length {} (same as input type `{}`), \
+            found `{}` with length {}",
+            "third",
+            in_len,
+            in_ty,
+            arg_tys[2],
+            out_len2
+        );
+
+        // The return type must match the first argument type
+        require!(ret_ty == in_ty, "expected return type `{}`, found `{}`", in_ty, ret_ty);
+
+        // This counts how many pointers
+        fn ptr_count(t: Ty<'_>) -> usize {
+            match t.kind() {
+                ty::RawPtr(p) => 1 + ptr_count(p.ty),
+                _ => 0,
+            }
+        }
+
+        // Non-ptr type
+        fn non_ptr(t: Ty<'_>) -> Ty<'_> {
+            match t.kind() {
+                ty::RawPtr(p) => non_ptr(p.ty),
+                _ => t,
+            }
+        }
+
+        // The second argument must be a simd vector with an element type that's a pointer
+        // to the element type of the first argument
+        let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
+        let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (pointer_count, underlying_ty) = match element_ty1.kind() {
+            ty::RawPtr(p) if p.ty == in_elem => (ptr_count(element_ty1), non_ptr(element_ty1)),
+            _ => {
+                require!(
+                    false,
+                    "expected element type `{}` of second argument `{}` \
+                    to be a pointer to the element type `{}` of the first \
+                        argument `{}`, found `{}` != `*_ {}`",
+                        element_ty1,
+                        arg_tys[1],
+                        in_elem,
+                        in_ty,
+                        element_ty1,
+                        in_elem
+                );
+                unreachable!();
+            }
+        };
+        assert!(pointer_count > 0);
+        assert_eq!(pointer_count - 1, ptr_count(element_ty0));
+        assert_eq!(underlying_ty, non_ptr(element_ty0));
+
+        // The element type of the third argument must be a signed integer type of any width:
+        let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
+        match element_ty2.kind() {
+            ty::Int(_) => (),
+            _ => {
+                require!(
+                    false,
+                    "expected element type `{}` of third argument `{}` \
+                    to be a signed integer type",
+                    element_ty2,
+                    arg_tys[2]
+                );
+            }
+        }
+
+        return Ok(gather(args[0].immediate(), args[1].immediate(), args[2].immediate(), pointer_count, bx, in_len, underlying_ty, false));
+    }
+
+    #[cfg(feature="master")]
+    if name == sym::simd_scatter {
+        // simd_scatter(values: <N x T>, pointers: <N x *mut T>,
+        //             mask: <N x i{M}>) -> ()
+        // * N: number of elements in the input vectors
+        // * T: type of the element to load
+        // * M: any integer width is supported, will be truncated to i1
+
+        // All types must be simd vector types
+        require_simd!(in_ty, "first");
+        require_simd!(arg_tys[1], "second");
+        require_simd!(arg_tys[2], "third");
+
+        // Of the same length:
+        let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
+        require!(
+            in_len == element_len1,
+            "expected {} argument with length {} (same as input type `{}`), \
+            found `{}` with length {}",
+            "second",
+            in_len,
+            in_ty,
+            arg_tys[1],
+            element_len1
+        );
+        require!(
+            in_len == element_len2,
+            "expected {} argument with length {} (same as input type `{}`), \
+            found `{}` with length {}",
+            "third",
+            in_len,
+            in_ty,
+            arg_tys[2],
+            element_len2
+        );
+
+        // This counts how many pointers
+        fn ptr_count(t: Ty<'_>) -> usize {
+            match t.kind() {
+                ty::RawPtr(p) => 1 + ptr_count(p.ty),
+                _ => 0,
+            }
+        }
+
+        // Non-ptr type
+        fn non_ptr(t: Ty<'_>) -> Ty<'_> {
+            match t.kind() {
+                ty::RawPtr(p) => non_ptr(p.ty),
+                _ => t,
+            }
+        }
+
+        // The second argument must be a simd vector with an element type that's a pointer
+        // to the element type of the first argument
+        let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
+        let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
+        let (pointer_count, underlying_ty) = match element_ty1.kind() {
+            ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => {
+                (ptr_count(element_ty1), non_ptr(element_ty1))
+            }
+            _ => {
+                require!(
+                    false,
+                    "expected element type `{}` of second argument `{}` \
+                    to be a pointer to the element type `{}` of the first \
+                        argument `{}`, found `{}` != `*mut {}`",
+                        element_ty1,
+                        arg_tys[1],
+                        in_elem,
+                        in_ty,
+                        element_ty1,
+                        in_elem
+                );
+                unreachable!();
+            }
+        };
+        assert!(pointer_count > 0);
+        assert_eq!(pointer_count - 1, ptr_count(element_ty0));
+        assert_eq!(underlying_ty, non_ptr(element_ty0));
+
+        // The element type of the third argument must be a signed integer type of any width:
+        match element_ty2.kind() {
+            ty::Int(_) => (),
+            _ => {
+                require!(
+                    false,
+                    "expected element type `{}` of third argument `{}` \
+                    be a signed integer type",
+                    element_ty2,
+                    arg_tys[2]
+                );
+            }
+        }
+
+        let result = gather(args[0].immediate(), args[1].immediate(), args[2].immediate(), pointer_count, bx, in_len, underlying_ty, true);
+
+        let pointers = args[1].immediate();
+
+        let vector_type =
+            if pointer_count > 1 {
+                bx.context.new_vector_type(bx.usize_type, in_len)
+            }
+            else {
+                vector_ty(bx, underlying_ty, in_len)
+            };
+        let elem_type = vector_type.dyncast_vector().expect("vector type").get_element_type();
+
+        for i in 0..in_len {
+            let index = bx.context.new_rvalue_from_int(bx.int_type, i as i32);
+            let value = bx.context.new_vector_access(None, result, index);
+
+            let int = bx.context.new_vector_access(None, pointers, index).to_rvalue();
+            let ptr_type = elem_type.make_pointer();
+            let ptr = bx.context.new_bitcast(None, int, ptr_type);
+            bx.llbb().add_assignment(None, ptr.dereference(None), value);
+        }
+
+        return Ok(bx.context.new_rvalue_zero(bx.i32_type));
+    }
+
     arith_binary! {
         simd_add: Uint, Int => add, Float => fadd;
         simd_sub: Uint, Int => sub, Float => fsub;

src/intrinsic/simd.rs

@@ -55,6 +55,14 @@ fn uncached_gcc_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout
         Abi::Scalar(_) => bug!("handled elsewhere"),
         Abi::Vector { ref element, count } => {
             let element = layout.scalar_gcc_type_at(cx, element, Size::ZERO);
+            let element =
+                // NOTE: gcc doesn't allow pointer types in vectors.
+                if element.get_pointee().is_some() {
+                    cx.usize_type
+                }
+                else {
+                    element
+                };
             return cx.context.new_vector_type(element, count);
         },
         Abi::ScalarPair(..) => {