Skip to content

Commit 8a0e5fa

Browse files
borsbors
committed
Auto merge of rust-lang#58003 - nikic:saturating-add, r=nagisa
Use LLVM intrinsics for saturating add/sub Use the `[su](add|sub).sat` LLVM intrinsics, if we're compiling against LLVM 8, as they should optimize and codegen better than IR based on `[su](add|sub).with.overlow`. For the fallback for LLVM < 8 I'm using the same expansion that target lowering in LLVM uses, which is not the same as Rust currently uses (in particular due to the use of selects rather than branches). Fixes rust-lang#55286. Fixes rust-lang#52203. Fixes rust-lang#44500. r? @nagisa
2 parents d30b99f + 4a4186e commit 8a0e5fa

File tree

5 files changed

+101
-3
lines changed

5 files changed

+101
-3
lines changed

‎src/libcore/intrinsics.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1493,6 +1493,19 @@ extern "rust-intrinsic" {
14931493
/// [`std::u32::wrapping_mul`](../../std/primitive.u32.html#method.wrapping_mul)
14941494
pub fn overflowing_mul<T>(a: T, b: T) -> T;
14951495

1496+
/// Computes `a + b`, while saturating at numeric bounds.
1497+
/// The stabilized versions of this intrinsic are available on the integer
1498+
/// primitives via the `saturating_add` method. For example,
1499+
/// [`std::u32::saturating_add`](../../std/primitive.u32.html#method.saturating_add)
1500+
#[cfg(not(stage0))]
1501+
pub fn saturating_add<T>(a: T, b: T) -> T;
1502+
/// Computes `a - b`, while saturating at numeric bounds.
1503+
/// The stabilized versions of this intrinsic are available on the integer
1504+
/// primitives via the `saturating_sub` method. For example,
1505+
/// [`std::u32::saturating_sub`](../../std/primitive.u32.html#method.saturating_sub)
1506+
#[cfg(not(stage0))]
1507+
pub fn saturating_sub<T>(a: T, b: T) -> T;
1508+
14961509
/// Returns the value of the discriminant for the variant in 'v',
14971510
/// cast to a `u64`; if `T` has no discriminant, returns 0.
14981511
pub fn discriminant_value<T>(v: &T) -> u64;

‎src/libcore/num/mod.rs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -883,11 +883,16 @@ $EndFeature, "
883883
#[stable(feature = "rust1", since = "1.0.0")]
884884
#[inline]
885885
pub fn saturating_add(self, rhs: Self) -> Self {
886+
#[cfg(stage0)]
886887
match self.checked_add(rhs) {
887888
Some(x) => x,
888889
None if rhs >= 0 => Self::max_value(),
889890
None => Self::min_value(),
890891
}
892+
#[cfg(not(stage0))]
893+
{
894+
intrinsics::saturating_add(self, rhs)
895+
}
891896
}
892897
}
893898

@@ -908,11 +913,16 @@ $EndFeature, "
908913
#[stable(feature = "rust1", since = "1.0.0")]
909914
#[inline]
910915
pub fn saturating_sub(self, rhs: Self) -> Self {
916+
#[cfg(stage0)]
911917
match self.checked_sub(rhs) {
912918
Some(x) => x,
913919
None if rhs >= 0 => Self::min_value(),
914920
None => Self::max_value(),
915921
}
922+
#[cfg(not(stage0))]
923+
{
924+
intrinsics::saturating_sub(self, rhs)
925+
}
916926
}
917927
}
918928

@@ -2744,10 +2754,15 @@ assert_eq!(200u8.saturating_add(127), 255);", $EndFeature, "
27442754
#[stable(feature = "rust1", since = "1.0.0")]
27452755
#[inline]
27462756
pub fn saturating_add(self, rhs: Self) -> Self {
2757+
#[cfg(stage0)]
27472758
match self.checked_add(rhs) {
27482759
Some(x) => x,
27492760
None => Self::max_value(),
27502761
}
2762+
#[cfg(not(stage0))]
2763+
{
2764+
intrinsics::saturating_add(self, rhs)
2765+
}
27512766
}
27522767
}
27532768

@@ -2766,10 +2781,15 @@ assert_eq!(13", stringify!($SelfT), ".saturating_sub(127), 0);", $EndFeature, "
27662781
#[stable(feature = "rust1", since = "1.0.0")]
27672782
#[inline]
27682783
pub fn saturating_sub(self, rhs: Self) -> Self {
2784+
#[cfg(stage0)]
27692785
match self.checked_sub(rhs) {
27702786
Some(x) => x,
27712787
None => Self::min_value(),
27722788
}
2789+
#[cfg(not(stage0))]
2790+
{
2791+
intrinsics::saturating_sub(self, rhs)
2792+
}
27732793
}
27742794
}
27752795

‎src/librustc_codegen_llvm/context.rs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -757,6 +757,30 @@ impl CodegenCx<'b, 'tcx> {
757757
ifn!("llvm.umul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct!{t_i64, i1});
758758
ifn!("llvm.umul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct!{t_i128, i1});
759759

760+
ifn!("llvm.sadd.sat.i8", fn(t_i8, t_i8) -> t_i8);
761+
ifn!("llvm.sadd.sat.i16", fn(t_i16, t_i16) -> t_i16);
762+
ifn!("llvm.sadd.sat.i32", fn(t_i32, t_i32) -> t_i32);
763+
ifn!("llvm.sadd.sat.i64", fn(t_i64, t_i64) -> t_i64);
764+
ifn!("llvm.sadd.sat.i128", fn(t_i128, t_i128) -> t_i128);
765+
766+
ifn!("llvm.uadd.sat.i8", fn(t_i8, t_i8) -> t_i8);
767+
ifn!("llvm.uadd.sat.i16", fn(t_i16, t_i16) -> t_i16);
768+
ifn!("llvm.uadd.sat.i32", fn(t_i32, t_i32) -> t_i32);
769+
ifn!("llvm.uadd.sat.i64", fn(t_i64, t_i64) -> t_i64);
770+
ifn!("llvm.uadd.sat.i128", fn(t_i128, t_i128) -> t_i128);
771+
772+
ifn!("llvm.ssub.sat.i8", fn(t_i8, t_i8) -> t_i8);
773+
ifn!("llvm.ssub.sat.i16", fn(t_i16, t_i16) -> t_i16);
774+
ifn!("llvm.ssub.sat.i32", fn(t_i32, t_i32) -> t_i32);
775+
ifn!("llvm.ssub.sat.i64", fn(t_i64, t_i64) -> t_i64);
776+
ifn!("llvm.ssub.sat.i128", fn(t_i128, t_i128) -> t_i128);
777+
778+
ifn!("llvm.usub.sat.i8", fn(t_i8, t_i8) -> t_i8);
779+
ifn!("llvm.usub.sat.i16", fn(t_i16, t_i16) -> t_i16);
780+
ifn!("llvm.usub.sat.i32", fn(t_i32, t_i32) -> t_i32);
781+
ifn!("llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64);
782+
ifn!("llvm.usub.sat.i128", fn(t_i128, t_i128) -> t_i128);
783+
760784
ifn!("llvm.lifetime.start", fn(t_i64,i8p) -> void);
761785
ifn!("llvm.lifetime.end", fn(t_i64, i8p) -> void);
762786

‎src/librustc_codegen_llvm/intrinsic.rs‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ use type_::Type;
1414
use type_of::LayoutLlvmExt;
1515
use rustc::ty::{self, Ty};
1616
use rustc::ty::layout::{self, LayoutOf, HasTyCtxt, Primitive};
17-
use rustc_codegen_ssa::common::TypeKind;
17+
use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
1818
use rustc::hir;
1919
use syntax::ast::{self, FloatTy};
2020
use syntax::symbol::Symbol;
@@ -28,7 +28,7 @@ use rustc::session::Session;
2828
use syntax_pos::Span;
2929

3030
use std::cmp::Ordering;
31-
use std::iter;
31+
use std::{iter, i128, u128};
3232

3333
fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: &str) -> Option<&'ll Value> {
3434
let llvm_name = match name {
@@ -342,7 +342,7 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
342342
"bitreverse" | "add_with_overflow" | "sub_with_overflow" |
343343
"mul_with_overflow" | "overflowing_add" | "overflowing_sub" | "overflowing_mul" |
344344
"unchecked_div" | "unchecked_rem" | "unchecked_shl" | "unchecked_shr" | "exact_div" |
345-
"rotate_left" | "rotate_right" => {
345+
"rotate_left" | "rotate_right" | "saturating_add" | "saturating_sub" => {
346346
let ty = arg_tys[0];
347347
match int_type_width_signed(ty, self) {
348348
Some((width, signed)) =>
@@ -468,6 +468,44 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
468468
self.or(shift1, shift2)
469469
}
470470
},
471+
"saturating_add" | "saturating_sub" => {
472+
let is_add = name == "saturating_add";
473+
let lhs = args[0].immediate();
474+
let rhs = args[1].immediate();
475+
if llvm_util::get_major_version() >= 8 {
476+
let llvm_name = &format!("llvm.{}{}.sat.i{}",
477+
if signed { 's' } else { 'u' },
478+
if is_add { "add" } else { "sub" },
479+
width);
480+
let llfn = self.get_intrinsic(llvm_name);
481+
self.call(llfn, &[lhs, rhs], None)
482+
} else {
483+
let llvm_name = &format!("llvm.{}{}.with.overflow.i{}",
484+
if signed { 's' } else { 'u' },
485+
if is_add { "add" } else { "sub" },
486+
width);
487+
let llfn = self.get_intrinsic(llvm_name);
488+
let pair = self.call(llfn, &[lhs, rhs], None);
489+
let val = self.extract_value(pair, 0);
490+
let overflow = self.extract_value(pair, 1);
491+
let llty = self.type_ix(width);
492+
493+
let limit = if signed {
494+
let limit_lo = self.const_uint_big(
495+
llty, (i128::MIN >> (128 - width)) as u128);
496+
let limit_hi = self.const_uint_big(
497+
llty, (i128::MAX >> (128 - width)) as u128);
498+
let neg = self.icmp(
499+
IntPredicate::IntSLT, val, self.const_uint(llty, 0));
500+
self.select(neg, limit_hi, limit_lo)
501+
} else if is_add {
502+
self.const_uint_big(llty, u128::MAX >> (128 - width))
503+
} else {
504+
self.const_uint(llty, 0)
505+
};
506+
self.select(overflow, limit, val)
507+
}
508+
},
471509
_ => bug!(),
472510
},
473511
None => {

‎src/librustc_typeck/check/intrinsic.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ pub fn intrisic_operation_unsafety(intrinsic: &str) -> hir::Unsafety {
6868
"size_of" | "min_align_of" | "needs_drop" |
6969
"add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" |
7070
"overflowing_add" | "overflowing_sub" | "overflowing_mul" |
71+
"saturating_add" | "saturating_sub" |
7172
"rotate_left" | "rotate_right" |
7273
"ctpop" | "ctlz" | "cttz" | "bswap" | "bitreverse"
7374
=> hir::Unsafety::Normal,
@@ -307,6 +308,8 @@ pub fn check_intrinsic_type<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
307308

308309
"overflowing_add" | "overflowing_sub" | "overflowing_mul" =>
309310
(1, vec![param(0), param(0)], param(0)),
311+
"saturating_add" | "saturating_sub" =>
312+
(1, vec![param(0), param(0)], param(0)),
310313
"fadd_fast" | "fsub_fast" | "fmul_fast" | "fdiv_fast" | "frem_fast" =>
311314
(1, vec![param(0), param(0)], param(0)),
312315

0 commit comments

Comments
 (0)