Use simd intrinsics for max and min (#1357)

Author: Nugine

crates/core_arch/src/x86/avx2.rs

@@ -1857,7 +1857,9 @@ pub unsafe fn _mm256_maskstore_epi64(mem_addr: *mut i64, mask: __m256i, a: __m25
 #[cfg_attr(test, assert_instr(vpmaxsw))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm256_max_epi16(a: __m256i, b: __m256i) -> __m256i {
-    transmute(pmaxsw(a.as_i16x16(), b.as_i16x16()))
+    let a = a.as_i16x16();
+    let b = b.as_i16x16();
+    transmute(simd_select::<i16x16, _>(simd_gt(a, b), a, b))
 }
 
 /// Compares packed 32-bit integers in `a` and `b`, and returns the packed
@@ -1869,7 +1871,9 @@ pub unsafe fn _mm256_max_epi16(a: __m256i, b: __m256i) -> __m256i {
 #[cfg_attr(test, assert_instr(vpmaxsd))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm256_max_epi32(a: __m256i, b: __m256i) -> __m256i {
-    transmute(pmaxsd(a.as_i32x8(), b.as_i32x8()))
+    let a = a.as_i32x8();
+    let b = b.as_i32x8();
+    transmute(simd_select::<i32x8, _>(simd_gt(a, b), a, b))
 }
 
 /// Compares packed 8-bit integers in `a` and `b`, and returns the packed
@@ -1881,7 +1885,9 @@ pub unsafe fn _mm256_max_epi32(a: __m256i, b: __m256i) -> __m256i {
 #[cfg_attr(test, assert_instr(vpmaxsb))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm256_max_epi8(a: __m256i, b: __m256i) -> __m256i {
-    transmute(pmaxsb(a.as_i8x32(), b.as_i8x32()))
+    let a = a.as_i8x32();
+    let b = b.as_i8x32();
+    transmute(simd_select::<i8x32, _>(simd_gt(a, b), a, b))
 }
 
 /// Compares packed unsigned 16-bit integers in `a` and `b`, and returns
@@ -1893,7 +1899,9 @@ pub unsafe fn _mm256_max_epi8(a: __m256i, b: __m256i) -> __m256i {
 #[cfg_attr(test, assert_instr(vpmaxuw))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm256_max_epu16(a: __m256i, b: __m256i) -> __m256i {
-    transmute(pmaxuw(a.as_u16x16(), b.as_u16x16()))
+    let a = a.as_u16x16();
+    let b = b.as_u16x16();
+    transmute(simd_select::<i16x16, _>(simd_gt(a, b), a, b))
 }
 
 /// Compares packed unsigned 32-bit integers in `a` and `b`, and returns
@@ -1905,7 +1913,9 @@ pub unsafe fn _mm256_max_epu16(a: __m256i, b: __m256i) -> __m256i {
 #[cfg_attr(test, assert_instr(vpmaxud))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm256_max_epu32(a: __m256i, b: __m256i) -> __m256i {
-    transmute(pmaxud(a.as_u32x8(), b.as_u32x8()))
+    let a = a.as_u32x8();
+    let b = b.as_u32x8();
+    transmute(simd_select::<i32x8, _>(simd_gt(a, b), a, b))
 }
 
 /// Compares packed unsigned 8-bit integers in `a` and `b`, and returns
@@ -1917,7 +1927,9 @@ pub unsafe fn _mm256_max_epu32(a: __m256i, b: __m256i) -> __m256i {
 #[cfg_attr(test, assert_instr(vpmaxub))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm256_max_epu8(a: __m256i, b: __m256i) -> __m256i {
-    transmute(pmaxub(a.as_u8x32(), b.as_u8x32()))
+    let a = a.as_u8x32();
+    let b = b.as_u8x32();
+    transmute(simd_select::<i8x32, _>(simd_gt(a, b), a, b))
 }
 
 /// Compares packed 16-bit integers in `a` and `b`, and returns the packed
@@ -1929,7 +1941,9 @@ pub unsafe fn _mm256_max_epu8(a: __m256i, b: __m256i) -> __m256i {
 #[cfg_attr(test, assert_instr(vpminsw))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm256_min_epi16(a: __m256i, b: __m256i) -> __m256i {
-    transmute(pminsw(a.as_i16x16(), b.as_i16x16()))
+    let a = a.as_i16x16();
+    let b = b.as_i16x16();
+    transmute(simd_select::<i16x16, _>(simd_lt(a, b), a, b))
 }
 
 /// Compares packed 32-bit integers in `a` and `b`, and returns the packed
@@ -1941,7 +1955,9 @@ pub unsafe fn _mm256_min_epi16(a: __m256i, b: __m256i) -> __m256i {
 #[cfg_attr(test, assert_instr(vpminsd))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm256_min_epi32(a: __m256i, b: __m256i) -> __m256i {
-    transmute(pminsd(a.as_i32x8(), b.as_i32x8()))
+    let a = a.as_i32x8();
+    let b = b.as_i32x8();
+    transmute(simd_select::<i32x8, _>(simd_lt(a, b), a, b))
 }
 
 /// Compares packed 8-bit integers in `a` and `b`, and returns the packed
@@ -1953,7 +1969,9 @@ pub unsafe fn _mm256_min_epi32(a: __m256i, b: __m256i) -> __m256i {
 #[cfg_attr(test, assert_instr(vpminsb))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm256_min_epi8(a: __m256i, b: __m256i) -> __m256i {
-    transmute(pminsb(a.as_i8x32(), b.as_i8x32()))
+    let a = a.as_i8x32();
+    let b = b.as_i8x32();
+    transmute(simd_select::<i8x32, _>(simd_lt(a, b), a, b))
 }
 
 /// Compares packed unsigned 16-bit integers in `a` and `b`, and returns
@@ -1965,7 +1983,9 @@ pub unsafe fn _mm256_min_epi8(a: __m256i, b: __m256i) -> __m256i {
 #[cfg_attr(test, assert_instr(vpminuw))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm256_min_epu16(a: __m256i, b: __m256i) -> __m256i {
-    transmute(pminuw(a.as_u16x16(), b.as_u16x16()))
+    let a = a.as_u16x16();
+    let b = b.as_u16x16();
+    transmute(simd_select::<i16x16, _>(simd_lt(a, b), a, b))
 }
 
 /// Compares packed unsigned 32-bit integers in `a` and `b`, and returns
@@ -1977,7 +1997,9 @@ pub unsafe fn _mm256_min_epu16(a: __m256i, b: __m256i) -> __m256i {
 #[cfg_attr(test, assert_instr(vpminud))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm256_min_epu32(a: __m256i, b: __m256i) -> __m256i {
-    transmute(pminud(a.as_u32x8(), b.as_u32x8()))
+    let a = a.as_u32x8();
+    let b = b.as_u32x8();
+    transmute(simd_select::<i32x8, _>(simd_lt(a, b), a, b))
 }
 
 /// Compares packed unsigned 8-bit integers in `a` and `b`, and returns
@@ -1989,7 +2011,9 @@ pub unsafe fn _mm256_min_epu32(a: __m256i, b: __m256i) -> __m256i {
 #[cfg_attr(test, assert_instr(vpminub))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm256_min_epu8(a: __m256i, b: __m256i) -> __m256i {
-    transmute(pminub(a.as_u8x32(), b.as_u8x32()))
+    let a = a.as_u8x32();
+    let b = b.as_u8x32();
+    transmute(simd_select::<i8x32, _>(simd_lt(a, b), a, b))
 }
 
 /// Creates mask from the most significant bit of each 8-bit element in `a`,
@@ -3620,30 +3644,6 @@ extern "C" {
     fn maskstoreq(mem_addr: *mut i8, mask: i64x2, a: i64x2);
     #[link_name = "llvm.x86.avx2.maskstore.q.256"]
     fn maskstoreq256(mem_addr: *mut i8, mask: i64x4, a: i64x4);
-    #[link_name = "llvm.x86.avx2.pmaxs.w"]
-    fn pmaxsw(a: i16x16, b: i16x16) -> i16x16;
-    #[link_name = "llvm.x86.avx2.pmaxs.d"]
-    fn pmaxsd(a: i32x8, b: i32x8) -> i32x8;
-    #[link_name = "llvm.x86.avx2.pmaxs.b"]
-    fn pmaxsb(a: i8x32, b: i8x32) -> i8x32;
-    #[link_name = "llvm.x86.avx2.pmaxu.w"]
-    fn pmaxuw(a: u16x16, b: u16x16) -> u16x16;
-    #[link_name = "llvm.x86.avx2.pmaxu.d"]
-    fn pmaxud(a: u32x8, b: u32x8) -> u32x8;
-    #[link_name = "llvm.x86.avx2.pmaxu.b"]
-    fn pmaxub(a: u8x32, b: u8x32) -> u8x32;
-    #[link_name = "llvm.x86.avx2.pmins.w"]
-    fn pminsw(a: i16x16, b: i16x16) -> i16x16;
-    #[link_name = "llvm.x86.avx2.pmins.d"]
-    fn pminsd(a: i32x8, b: i32x8) -> i32x8;
-    #[link_name = "llvm.x86.avx2.pmins.b"]
-    fn pminsb(a: i8x32, b: i8x32) -> i8x32;
-    #[link_name = "llvm.x86.avx2.pminu.w"]
-    fn pminuw(a: u16x16, b: u16x16) -> u16x16;
-    #[link_name = "llvm.x86.avx2.pminu.d"]
-    fn pminud(a: u32x8, b: u32x8) -> u32x8;
-    #[link_name = "llvm.x86.avx2.pminu.b"]
-    fn pminub(a: u8x32, b: u8x32) -> u8x32;
     #[link_name = "llvm.x86.avx2.mpsadbw"]
     fn mpsadbw(a: u8x32, b: u8x32, imm8: i32) -> u16x16;
     #[link_name = "llvm.x86.avx2.pmulhu.w"]

crates/core_arch/src/x86/sse2.rs

@@ -203,7 +203,9 @@ pub unsafe fn _mm_madd_epi16(a: __m128i, b: __m128i) -> __m128i {
 #[cfg_attr(test, assert_instr(pmaxsw))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm_max_epi16(a: __m128i, b: __m128i) -> __m128i {
-    transmute(pmaxsw(a.as_i16x8(), b.as_i16x8()))
+    let a = a.as_i16x8();
+    let b = b.as_i16x8();
+    transmute(simd_select::<i16x8, _>(simd_gt(a, b), a, b))
 }
 
 /// Compares packed unsigned 8-bit integers in `a` and `b`, and returns the
@@ -215,7 +217,9 @@ pub unsafe fn _mm_max_epi16(a: __m128i, b: __m128i) -> __m128i {
 #[cfg_attr(test, assert_instr(pmaxub))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm_max_epu8(a: __m128i, b: __m128i) -> __m128i {
-    transmute(pmaxub(a.as_u8x16(), b.as_u8x16()))
+    let a = a.as_u8x16();
+    let b = b.as_u8x16();
+    transmute(simd_select::<i8x16, _>(simd_gt(a, b), a, b))
 }
 
 /// Compares packed 16-bit integers in `a` and `b`, and returns the packed
@@ -227,7 +231,9 @@ pub unsafe fn _mm_max_epu8(a: __m128i, b: __m128i) -> __m128i {
 #[cfg_attr(test, assert_instr(pminsw))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm_min_epi16(a: __m128i, b: __m128i) -> __m128i {
-    transmute(pminsw(a.as_i16x8(), b.as_i16x8()))
+    let a = a.as_i16x8();
+    let b = b.as_i16x8();
+    transmute(simd_select::<i16x8, _>(simd_lt(a, b), a, b))
 }
 
 /// Compares packed unsigned 8-bit integers in `a` and `b`, and returns the
@@ -239,7 +245,9 @@ pub unsafe fn _mm_min_epi16(a: __m128i, b: __m128i) -> __m128i {
 #[cfg_attr(test, assert_instr(pminub))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm_min_epu8(a: __m128i, b: __m128i) -> __m128i {
-    transmute(pminub(a.as_u8x16(), b.as_u8x16()))
+    let a = a.as_u8x16();
+    let b = b.as_u8x16();
+    transmute(simd_select::<i8x16, _>(simd_lt(a, b), a, b))
 }
 
 /// Multiplies the packed 16-bit integers in `a` and `b`.
@@ -2798,14 +2806,6 @@ extern "C" {
     fn pavgw(a: u16x8, b: u16x8) -> u16x8;
     #[link_name = "llvm.x86.sse2.pmadd.wd"]
     fn pmaddwd(a: i16x8, b: i16x8) -> i32x4;
-    #[link_name = "llvm.x86.sse2.pmaxs.w"]
-    fn pmaxsw(a: i16x8, b: i16x8) -> i16x8;
-    #[link_name = "llvm.x86.sse2.pmaxu.b"]
-    fn pmaxub(a: u8x16, b: u8x16) -> u8x16;
-    #[link_name = "llvm.x86.sse2.pmins.w"]
-    fn pminsw(a: i16x8, b: i16x8) -> i16x8;
-    #[link_name = "llvm.x86.sse2.pminu.b"]
-    fn pminub(a: u8x16, b: u8x16) -> u8x16;
     #[link_name = "llvm.x86.sse2.pmulh.w"]
     fn pmulhw(a: i16x8, b: i16x8) -> i16x8;
     #[link_name = "llvm.x86.sse2.pmulhu.w"]

crates/core_arch/src/x86/sse41.rs

@@ -281,7 +281,9 @@ pub unsafe fn _mm_insert_epi32<const IMM8: i32>(a: __m128i, i: i32) -> __m128i {
 #[cfg_attr(test, assert_instr(pmaxsb))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm_max_epi8(a: __m128i, b: __m128i) -> __m128i {
-    transmute(pmaxsb(a.as_i8x16(), b.as_i8x16()))
+    let a = a.as_i8x16();
+    let b = b.as_i8x16();
+    transmute(simd_select::<i8x16, _>(simd_gt(a, b), a, b))
 }
 
 /// Compares packed unsigned 16-bit integers in `a` and `b`, and returns packed
@@ -293,7 +295,9 @@ pub unsafe fn _mm_max_epi8(a: __m128i, b: __m128i) -> __m128i {
 #[cfg_attr(test, assert_instr(pmaxuw))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm_max_epu16(a: __m128i, b: __m128i) -> __m128i {
-    transmute(pmaxuw(a.as_u16x8(), b.as_u16x8()))
+    let a = a.as_u16x8();
+    let b = b.as_u16x8();
+    transmute(simd_select::<i16x8, _>(simd_gt(a, b), a, b))
 }
 
 /// Compares packed 32-bit integers in `a` and `b`, and returns packed maximum
@@ -305,7 +309,9 @@ pub unsafe fn _mm_max_epu16(a: __m128i, b: __m128i) -> __m128i {
 #[cfg_attr(test, assert_instr(pmaxsd))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm_max_epi32(a: __m128i, b: __m128i) -> __m128i {
-    transmute(pmaxsd(a.as_i32x4(), b.as_i32x4()))
+    let a = a.as_i32x4();
+    let b = b.as_i32x4();
+    transmute(simd_select::<i32x4, _>(simd_gt(a, b), a, b))
 }
 
 /// Compares packed unsigned 32-bit integers in `a` and `b`, and returns packed
@@ -317,7 +323,9 @@ pub unsafe fn _mm_max_epi32(a: __m128i, b: __m128i) -> __m128i {
 #[cfg_attr(test, assert_instr(pmaxud))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm_max_epu32(a: __m128i, b: __m128i) -> __m128i {
-    transmute(pmaxud(a.as_u32x4(), b.as_u32x4()))
+    let a = a.as_u32x4();
+    let b = b.as_u32x4();
+    transmute(simd_select::<i32x4, _>(simd_gt(a, b), a, b))
 }
 
 /// Compares packed 8-bit integers in `a` and `b` and returns packed minimum
@@ -329,7 +337,9 @@ pub unsafe fn _mm_max_epu32(a: __m128i, b: __m128i) -> __m128i {
 #[cfg_attr(test, assert_instr(pminsb))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm_min_epi8(a: __m128i, b: __m128i) -> __m128i {
-    transmute(pminsb(a.as_i8x16(), b.as_i8x16()))
+    let a = a.as_i8x16();
+    let b = b.as_i8x16();
+    transmute(simd_select::<i8x16, _>(simd_lt(a, b), a, b))
 }
 
 /// Compares packed unsigned 16-bit integers in `a` and `b`, and returns packed
@@ -341,7 +351,9 @@ pub unsafe fn _mm_min_epi8(a: __m128i, b: __m128i) -> __m128i {
 #[cfg_attr(test, assert_instr(pminuw))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm_min_epu16(a: __m128i, b: __m128i) -> __m128i {
-    transmute(pminuw(a.as_u16x8(), b.as_u16x8()))
+    let a = a.as_u16x8();
+    let b = b.as_u16x8();
+    transmute(simd_select::<i16x8, _>(simd_lt(a, b), a, b))
 }
 
 /// Compares packed 32-bit integers in `a` and `b`, and returns packed minimum
@@ -353,7 +365,9 @@ pub unsafe fn _mm_min_epu16(a: __m128i, b: __m128i) -> __m128i {
 #[cfg_attr(test, assert_instr(pminsd))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm_min_epi32(a: __m128i, b: __m128i) -> __m128i {
-    transmute(pminsd(a.as_i32x4(), b.as_i32x4()))
+    let a = a.as_i32x4();
+    let b = b.as_i32x4();
+    transmute(simd_select::<i32x4, _>(simd_lt(a, b), a, b))
 }
 
 /// Compares packed unsigned 32-bit integers in `a` and `b`, and returns packed
@@ -365,7 +379,9 @@ pub unsafe fn _mm_min_epi32(a: __m128i, b: __m128i) -> __m128i {
 #[cfg_attr(test, assert_instr(pminud))]
 #[stable(feature = "simd_x86", since = "1.27.0")]
 pub unsafe fn _mm_min_epu32(a: __m128i, b: __m128i) -> __m128i {
-    transmute(pminud(a.as_u32x4(), b.as_u32x4()))
+    let a = a.as_u32x4();
+    let b = b.as_u32x4();
+    transmute(simd_select::<i32x4, _>(simd_lt(a, b), a, b))
 }
 
 /// Converts packed 32-bit integers from `a` and `b` to packed 16-bit integers
@@ -1122,22 +1138,6 @@ extern "C" {
     fn pblendw(a: i16x8, b: i16x8, imm8: u8) -> i16x8;
     #[link_name = "llvm.x86.sse41.insertps"]
     fn insertps(a: __m128, b: __m128, imm8: u8) -> __m128;
-    #[link_name = "llvm.x86.sse41.pmaxsb"]
-    fn pmaxsb(a: i8x16, b: i8x16) -> i8x16;
-    #[link_name = "llvm.x86.sse41.pmaxuw"]
-    fn pmaxuw(a: u16x8, b: u16x8) -> u16x8;
-    #[link_name = "llvm.x86.sse41.pmaxsd"]
-    fn pmaxsd(a: i32x4, b: i32x4) -> i32x4;
-    #[link_name = "llvm.x86.sse41.pmaxud"]
-    fn pmaxud(a: u32x4, b: u32x4) -> u32x4;
-    #[link_name = "llvm.x86.sse41.pminsb"]
-    fn pminsb(a: i8x16, b: i8x16) -> i8x16;
-    #[link_name = "llvm.x86.sse41.pminuw"]
-    fn pminuw(a: u16x8, b: u16x8) -> u16x8;
-    #[link_name = "llvm.x86.sse41.pminsd"]
-    fn pminsd(a: i32x4, b: i32x4) -> i32x4;
-    #[link_name = "llvm.x86.sse41.pminud"]
-    fn pminud(a: u32x4, b: u32x4) -> u32x4;
     #[link_name = "llvm.x86.sse41.packusdw"]
     fn packusdw(a: i32x4, b: i32x4) -> u16x8;
     #[link_name = "llvm.x86.sse41.dppd"]