Add intrinsics for bigint helper methods

Author: clarfonthey

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -347,7 +347,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             | sym::rotate_left
             | sym::rotate_right
             | sym::saturating_add
-            | sym::saturating_sub => {
+            | sym::saturating_sub
+            | sym::add_with_carry
+            | sym::sub_with_carry
+            | sym::mul_double
+            | sym::mul_double_add
+            | sym::mul_double_add2 => {
                 let ty = arg_tys[0];
                 match int_type_width_signed(ty, self) {
                     Some((width, signed)) => match name {
@@ -417,6 +422,76 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                             );
                             self.call_intrinsic(llvm_name, &[lhs, rhs])
                         }
+                        sym::add_with_carry | sym::sub_with_carry => {
+                            let llty = self.type_ix(width);
+                            let is_add = name == sym::add_with_carry;
+                            let lhs = args[0].immediate();
+                            let rhs = args[1].immediate();
+
+                            // sign-extending the carry would treat it as -1, not 1
+                            let carry = self.intcast(args[2].immediate(), llty, false);
+
+                            let llvm_name = &format!(
+                                "llvm.{}{}.with.overflow.i{}",
+                                if signed { 's' } else { 'u' },
+                                if is_add { "add" } else { "sub" },
+                                width,
+                            );
+
+                            let ret = self.call_intrinsic(llvm_name, &[lhs, rhs]);
+                            let agg = self.extract_value(ret, 0);
+                            let overflow1 = self.extract_value(ret, 1);
+
+                            let ret = self.call_intrinsic(llvm_name, &[agg, carry]);
+                            let agg = self.extract_value(ret, 0);
+                            let overflow2 = self.extract_value(ret, 1);
+
+                            let overflow = if signed {
+                                self.icmp(IntPredicate::IntNE, overflow1, overflow2)
+                            } else {
+                                self.or(overflow1, overflow2)
+                            };
+
+                            let holder = self.const_struct(
+                                &[self.const_undef(llty), self.const_undef(self.type_i1())],
+                                false,
+                            );
+                            let holder = self.insert_value(holder, agg, 0);
+                            let holder = self.insert_value(holder, overflow, 1);
+                            holder
+                        }
+                        sym::mul_double | sym::mul_double_add | sym::mul_double_add2 => {
+                            let single_ty = self.type_ix(width);
+                            let double_ty = self.type_ix(width * 2);
+                            let lhs = self.intcast(args[0].immediate(), double_ty, signed);
+                            let rhs = self.intcast(args[1].immediate(), double_ty, signed);
+                            let mut ret = self.mul(lhs, rhs);
+                            if name == sym::mul_double_add || name == sym::mul_double_add2 {
+                                let carry = self.intcast(args[2].immediate(), double_ty, signed);
+                                ret = self.add(ret, carry)
+                            }
+                            if name == sym::mul_double_add2 {
+                                let carry2 = self.intcast(args[3].immediate(), double_ty, signed);
+                                ret = self.add(ret, carry2);
+                            }
+
+                            // note: insignificant part is always treated as unsigned, even if we
+                            //   coerce it to signed in the final result to make the intrinsic
+                            //   signature simpler
+                            let lo = self.intcast(ret, single_ty, signed);
+
+                            let bits = self.const_uint(double_ty, width);
+                            let hi = self.ashr(ret, bits);
+                            let hi = self.intcast(hi, single_ty, signed);
+
+                            let holder = self.const_struct(
+                                &[self.const_undef(single_ty), self.const_undef(single_ty)],
+                                false,
+                            );
+                            let holder = self.insert_value(holder, lo, 0);
+                            let holder = self.insert_value(holder, hi, 1);
+                            holder
+                        }
                         _ => bug!(),
                     },
                     None => {

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -178,6 +178,18 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 )?;
                 self.write_scalar(val, dest)?;
             }
+            sym::add_with_carry | sym::sub_with_carry => {
+                let l = self.read_immediate(&args[0])?;
+                let r = self.read_immediate(&args[1])?;
+                let c = self.read_immediate(&args[2])?;
+                let (val, overflowed) = self.carrying_arith(
+                    if intrinsic_name == sym::add_with_carry { BinOp::Add } else { BinOp::Sub },
+                    &l,
+                    &r,
+                    &c,
+                )?;
+                self.write_scalar_pair(val, overflowed, dest)?;
+            }
             sym::discriminant_value => {
                 let place = self.deref_pointer(&args[0])?;
                 let variant = self.read_discriminant(&place)?;
@@ -573,6 +585,39 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         })
     }
 
+    pub fn carrying_arith(
+        &self,
+        mir_op: BinOp,
+        l: &ImmTy<'tcx, M::Provenance>,
+        r: &ImmTy<'tcx, M::Provenance>,
+        c: &ImmTy<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx, (Scalar<M::Provenance>, Scalar<M::Provenance>)> {
+        assert_eq!(l.layout.ty, r.layout.ty);
+        assert_matches!(l.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
+        assert_matches!(c.layout.ty.kind(), ty::Bool);
+        assert_matches!(mir_op, BinOp::Add | BinOp::Sub);
+
+        let mir_op = mir_op.wrapping_to_overflowing().unwrap();
+
+        let (val, overflowed1) = self.binary_op(mir_op, l, r)?.to_scalar_pair();
+
+        let val = ImmTy::from_scalar(val, l.layout);
+        let c = ImmTy::from_scalar(c.to_scalar(), l.layout);
+
+        let (val, overflowed2) = self.binary_op(mir_op, &val, &c)?.to_scalar_pair();
+
+        let overflowed1 = overflowed1.to_bool()?;
+        let overflowed2 = overflowed2.to_bool()?;
+
+        let overflowed = Scalar::from_bool(if l.layout.abi.is_signed() {
+            overflowed1 != overflowed2
+        } else {
+            overflowed1 | overflowed2
+        });
+
+        interp_ok((val, overflowed))
+    }
+
     /// Offsets a pointer by some multiple of its type, returning an error if the pointer leaves its
     /// allocation.
     pub fn ptr_offset_inbounds(

compiler/rustc_const_eval/src/interpret/place.rs

@@ -626,6 +626,17 @@ where
         self.write_immediate(Immediate::Scalar(val.into()), dest)
     }
 
+    /// Write a scalar pair to a place
+    #[inline(always)]
+    pub fn write_scalar_pair(
+        &mut self,
+        val1: impl Into<Scalar<M::Provenance>>,
+        val2: impl Into<Scalar<M::Provenance>>,
+        dest: &impl Writeable<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx> {
+        self.write_immediate(Immediate::ScalarPair(val1.into(), val2.into()), dest)
+    }
+
     /// Write a pointer to a place
     #[inline(always)]
     pub fn write_pointer(

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -103,6 +103,11 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
         | sym::add_with_overflow
         | sym::sub_with_overflow
         | sym::mul_with_overflow
+        | sym::add_with_carry
+        | sym::sub_with_carry
+        | sym::mul_double
+        | sym::mul_double_add
+        | sym::mul_double_add2
         | sym::wrapping_add
         | sym::wrapping_sub
         | sym::wrapping_mul
@@ -433,6 +438,28 @@ pub fn check_intrinsic_type(
                 (1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool]))
             }
 
+            sym::add_with_carry | sym::sub_with_carry => (
+                1,
+                0,
+                vec![param(0), param(0), tcx.types.bool],
+                Ty::new_tup(tcx, &[param(0), tcx.types.bool]),
+            ),
+
+            sym::mul_double => {
+                (1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), param(0)]))
+            }
+
+            sym::mul_double_add => {
+                (1, 0, vec![param(0), param(0), param(0)], Ty::new_tup(tcx, &[param(0), param(0)]))
+            }
+
+            sym::mul_double_add2 => (
+                1,
+                0,
+                vec![param(0), param(0), param(0), param(0)],
+                Ty::new_tup(tcx, &[param(0), param(0)]),
+            ),
+
             sym::ptr_guaranteed_cmp => (
                 1,
                 0,

compiler/rustc_span/src/symbol.rs

@@ -377,6 +377,7 @@ symbols! {
         abort,
         add,
         add_assign,
+        add_with_carry,
         add_with_overflow,
         address,
         adt_const_params,
@@ -1276,6 +1277,9 @@ symbols! {
         move_size_limit,
         mul,
         mul_assign,
+        mul_double,
+        mul_double_add,
+        mul_double_add2,
         mul_with_overflow,
         multiple_supertrait_upcastable,
         must_not_suspend,
@@ -1918,6 +1922,7 @@ symbols! {
         structural_peq,
         sub,
         sub_assign,
+        sub_with_carry,
         sub_with_overflow,
         suggestion,
         surface_async_drop_in_place,

library/core/src/intrinsics.rs

@@ -2444,6 +2444,80 @@ extern "rust-intrinsic" {
     #[rustc_nounwind]
     pub fn mul_with_overflow<T: Copy>(x: T, y: T) -> (T, bool);
 
+    /// Performs integer addition with an explicit carry flag.
+    ///
+    /// Note that, unlike most intrinsics, this is safe to call;
+    /// it does not require an `unsafe` block.
+    /// Therefore, implementations must not require the user to uphold
+    /// any safety invariants.
+    ///
+    /// This intrinsic does not have a stable counterpart.
+    #[cfg(not(bootstrap))]
+    #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")]
+    #[unstable(feature = "core_intrinsics", issue = "none")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn add_with_carry<T: Copy>(x: T, y: T, z: bool) -> (T, bool);
+
+    /// Performs integer subtraction with an explicit carry flag.
+    ///
+    /// Note that, unlike most intrinsics, this is safe to call;
+    /// it does not require an `unsafe` block.
+    /// Therefore, implementations must not require the user to uphold
+    /// any safety invariants.
+    ///
+    /// This intrinsic does not have a stable counterpart.
+    #[cfg(not(bootstrap))]
+    #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")]
+    #[unstable(feature = "core_intrinsics", issue = "none")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn sub_with_carry<T: Copy>(x: T, y: T, z: bool) -> (T, bool);
+
+    /// Performs double-wide integer multiplication.
+    ///
+    /// Note that, unlike most intrinsics, this is safe to call;
+    /// it does not require an `unsafe` block.
+    /// Therefore, implementations must not require the user to uphold
+    /// any safety invariants.
+    ///
+    /// This intrinsic does not have a stable counterpart.
+    #[cfg(not(bootstrap))]
+    #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")]
+    #[unstable(feature = "core_intrinsics", issue = "none")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn mul_double<T: Copy>(x: T, y: T) -> (T, T);
+
+    /// Performs double-wide integer multiplication with an extra addition.
+    ///
+    /// Note that, unlike most intrinsics, this is safe to call;
+    /// it does not require an `unsafe` block.
+    /// Therefore, implementations must not require the user to uphold
+    /// any safety invariants.
+    ///
+    /// This intrinsic does not have a stable counterpart.
+    #[cfg(not(bootstrap))]
+    #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")]
+    #[unstable(feature = "core_intrinsics", issue = "none")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn mul_double_add<T: Copy>(x: T, y: T, z: T) -> (T, T);
+    /// Performs double-wide integer multiplication with an extra two additions.
+    ///
+    /// Note that, unlike most intrinsics, this is safe to call;
+    /// it does not require an `unsafe` block.
+    /// Therefore, implementations must not require the user to uphold
+    /// any safety invariants.
+    ///
+    /// This intrinsic does not have a stable counterpart.
+    #[cfg(not(bootstrap))]
+    #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")]
+    #[unstable(feature = "core_intrinsics", issue = "none")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn mul_double_add2<T: Copy>(x: T, y: T, z: T, w: T) -> (T, T);
+
     /// Performs an exact division, resulting in undefined behavior where
     /// `x % y != 0` or `y == 0` or `x == T::MIN && y == -1`
     ///