Add intrinsics for bigint helper methods

Author: clarfonthey

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -347,7 +347,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             | sym::rotate_left
             | sym::rotate_right
             | sym::saturating_add
-            | sym::saturating_sub => {
+            | sym::saturating_sub
+            | sym::add_with_carry
+            | sym::sub_with_carry
+            | sym::mul_double
+            | sym::mul_double_add
+            | sym::mul_double_add2 => {
                 let ty = arg_tys[0];
                 match int_type_width_signed(ty, self) {
                     Some((width, signed)) => match name {
@@ -417,6 +422,76 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                             );
                             self.call_intrinsic(llvm_name, &[lhs, rhs])
                         }
+                        sym::add_with_carry | sym::sub_with_carry => {
+                            let llty = self.type_ix(width);
+                            let is_add = name == sym::add_with_carry;
+                            let lhs = args[0].immediate();
+                            let rhs = args[1].immediate();
+
+                            // sign-extending the carry would treat it as -1, not 1
+                            let carry = self.intcast(args[2].immediate(), llty, false);
+
+                            let llvm_name = &format!(
+                                "llvm.{}{}.with.overflow.i{}",
+                                if signed { 's' } else { 'u' },
+                                if is_add { "add" } else { "sub" },
+                                width,
+                            );
+
+                            let ret = self.call_intrinsic(llvm_name, &[lhs, rhs]);
+                            let agg = self.extract_value(ret, 0);
+                            let overflow1 = self.extract_value(ret, 1);
+
+                            let ret = self.call_intrinsic(llvm_name, &[agg, carry]);
+                            let agg = self.extract_value(ret, 0);
+                            let overflow2 = self.extract_value(ret, 1);
+
+                            let overflow = if signed {
+                                self.icmp(IntPredicate::IntNE, overflow1, overflow2)
+                            } else {
+                                self.or(overflow1, overflow2)
+                            };
+
+                            let holder = self.const_struct(
+                                &[self.const_undef(llty), self.const_undef(self.type_i1())],
+                                false,
+                            );
+                            let holder = self.insert_value(holder, agg, 0);
+                            let holder = self.insert_value(holder, overflow, 1);
+                            holder
+                        }
+                        sym::mul_double | sym::mul_double_add | sym::mul_double_add2 => {
+                            let single_ty = self.type_ix(width);
+                            let double_ty = self.type_ix(width * 2);
+                            let lhs = self.intcast(args[0].immediate(), double_ty, signed);
+                            let rhs = self.intcast(args[1].immediate(), double_ty, signed);
+                            let mut ret = self.mul(lhs, rhs);
+                            if name == sym::mul_double_add || name == sym::mul_double_add2 {
+                                let carry = self.intcast(args[2].immediate(), double_ty, signed);
+                                ret = self.add(ret, carry)
+                            }
+                            if name == sym::mul_double_add2 {
+                                let carry2 = self.intcast(args[3].immediate(), double_ty, signed);
+                                ret = self.add(ret, carry2);
+                            }
+
+                            // note: insignificant part is always treated as unsigned, even if we
+                            //   coerce it to signed in the final result to make the intrinsic
+                            //   signature simpler
+                            let lo = self.intcast(ret, single_ty, signed);
+
+                            let bits = self.const_uint(double_ty, width);
+                            let hi = self.ashr(ret, bits);
+                            let hi = self.intcast(hi, single_ty, signed);
+
+                            let holder = self.const_struct(
+                                &[self.const_undef(single_ty), self.const_undef(single_ty)],
+                                false,
+                            );
+                            let holder = self.insert_value(holder, lo, 0);
+                            let holder = self.insert_value(holder, hi, 1);
+                            holder
+                        }
                         _ => bug!(),
                     },
                     None => {

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -103,6 +103,11 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
         | sym::add_with_overflow
         | sym::sub_with_overflow
         | sym::mul_with_overflow
+        | sym::add_with_carry
+        | sym::sub_with_carry
+        | sym::mul_double
+        | sym::mul_double_add
+        | sym::mul_double_add2
         | sym::wrapping_add
         | sym::wrapping_sub
         | sym::wrapping_mul
@@ -433,6 +438,28 @@ pub fn check_intrinsic_type(
                 (1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool]))
             }
 
+            sym::add_with_carry | sym::sub_with_carry => (
+                1,
+                0,
+                vec![param(0), param(0), tcx.types.bool],
+                Ty::new_tup(tcx, &[param(0), tcx.types.bool]),
+            ),
+
+            sym::mul_double => {
+                (1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), param(0)]))
+            }
+
+            sym::mul_double_add => {
+                (1, 0, vec![param(0), param(0), param(0)], Ty::new_tup(tcx, &[param(0), param(0)]))
+            }
+
+            sym::mul_double_add2 => (
+                1,
+                0,
+                vec![param(0), param(0), param(0), param(0)],
+                Ty::new_tup(tcx, &[param(0), param(0)]),
+            ),
+
             sym::ptr_guaranteed_cmp => (
                 1,
                 0,

compiler/rustc_span/src/symbol.rs

@@ -377,6 +377,7 @@ symbols! {
         abort,
         add,
         add_assign,
+        add_with_carry,
         add_with_overflow,
         address,
         adt_const_params,
@@ -1276,6 +1277,9 @@ symbols! {
         move_size_limit,
         mul,
         mul_assign,
+        mul_double,
+        mul_double_add,
+        mul_double_add2,
         mul_with_overflow,
         multiple_supertrait_upcastable,
         must_not_suspend,
@@ -1918,6 +1922,7 @@ symbols! {
         structural_peq,
         sub,
         sub_assign,
+        sub_with_carry,
         sub_with_overflow,
         suggestion,
         surface_async_drop_in_place,

library/core/src/intrinsics.rs

@@ -2444,6 +2444,80 @@ extern "rust-intrinsic" {
     #[rustc_nounwind]
     pub fn mul_with_overflow<T: Copy>(x: T, y: T) -> (T, bool);
 
+    /// Performs integer addition with an explicit carry flag.
+    ///
+    /// Note that, unlike most intrinsics, this is safe to call;
+    /// it does not require an `unsafe` block.
+    /// Therefore, implementations must not require the user to uphold
+    /// any safety invariants.
+    ///
+    /// This intrinsic does not have a stable counterpart.
+    #[cfg(not(bootstrap))]
+    #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")]
+    #[unstable(feature = "core_intrinsics", issue = "none")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn add_with_carry<T: Copy>(x: T, y: T, z: bool) -> (T, bool);
+
+    /// Performs integer subtraction with an explicit carry flag.
+    ///
+    /// Note that, unlike most intrinsics, this is safe to call;
+    /// it does not require an `unsafe` block.
+    /// Therefore, implementations must not require the user to uphold
+    /// any safety invariants.
+    ///
+    /// This intrinsic does not have a stable counterpart.
+    #[cfg(not(bootstrap))]
+    #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")]
+    #[unstable(feature = "core_intrinsics", issue = "none")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn sub_with_carry<T: Copy>(x: T, y: T, z: bool) -> (T, bool);
+
+    /// Performs double-wide integer multiplication.
+    ///
+    /// Note that, unlike most intrinsics, this is safe to call;
+    /// it does not require an `unsafe` block.
+    /// Therefore, implementations must not require the user to uphold
+    /// any safety invariants.
+    ///
+    /// This intrinsic does not have a stable counterpart.
+    #[cfg(not(bootstrap))]
+    #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")]
+    #[unstable(feature = "core_intrinsics", issue = "none")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn mul_double<T: Copy>(x: T, y: T) -> (T, T);
+
+    /// Performs double-wide integer multiplication with an extra addition.
+    ///
+    /// Note that, unlike most intrinsics, this is safe to call;
+    /// it does not require an `unsafe` block.
+    /// Therefore, implementations must not require the user to uphold
+    /// any safety invariants.
+    ///
+    /// This intrinsic does not have a stable counterpart.
+    #[cfg(not(bootstrap))]
+    #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")]
+    #[unstable(feature = "core_intrinsics", issue = "none")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn mul_double_add<T: Copy>(x: T, y: T, z: T) -> (T, T);
+    /// Performs double-wide integer multiplication with an extra two additions.
+    ///
+    /// Note that, unlike most intrinsics, this is safe to call;
+    /// it does not require an `unsafe` block.
+    /// Therefore, implementations must not require the user to uphold
+    /// any safety invariants.
+    ///
+    /// This intrinsic does not have a stable counterpart.
+    #[cfg(not(bootstrap))]
+    #[rustc_const_unstable(feature = "const_bigint_helper_methods", issue = "85532")]
+    #[unstable(feature = "core_intrinsics", issue = "none")]
+    #[rustc_safe_intrinsic]
+    #[rustc_nounwind]
+    pub fn mul_double_add2<T: Copy>(x: T, y: T, z: T, w: T) -> (T, T);
+
     /// Performs an exact division, resulting in undefined behavior where
     /// `x % y != 0` or `y == 0` or `x == T::MIN && y == -1`
     ///