Merge branch 'main' into implement-vpclmulqdq-512-bits

Author: onethumb

README.md

@@ -297,14 +297,13 @@ AKA `crc32` in many, but not all, implementations.
 
 ### CRC-64/NVME
 
-| Arch    | Brand | CPU             | System                    | Target            | 1KiB (GiB/s) | 1MiB (GiB/s) |
-|:--------|:------|:----------------|:--------------------------|:------------------|-------------:|-------------:|
-| x86_64  | Intel | Sapphire Rapids | EC2 c7i.metal-48xl        | avx512_vpclmulqdq |        ~20.3 |        ~94.1 |
-| x86_64  | AMD   | Genoa           | EC2 c7a.metal-48xl        | avx512_vpclmulqdq |        ~18.3 |        ~53.9 |
-| aarch64 | AWS   | Graviton4       | EC2 c8g.metal-48xl        | neon_pclmulqdq    |        ~16.3 |        ~16.3 |
-| aarch64 | Apple | M3 Ultra        | Mac Studio (32 core)      | neon_pclmulqdq    |        ~44.0 |        ~71.9 |
-| aarch64 | Apple | M4 Max          | MacBook Pro 16" (16 core) | neon_pclmulqdq    |        ~40.3 |        ~72.3 | 
-| aarch64 | Apple | M2 Ultra        | Mac Studio (24 core)      | neon_pclmulqdq    |        ~39.3 |        ~65.0 |
+| Arch    | Brand | CPU             | System               | Target              | 1KiB (GiB/s) | 1MiB (GiB/s) |
+|:--------|:------|:----------------|:---------------------|:--------------------|-------------:|-------------:|
+| x86_64  | Intel | Sapphire Rapids | EC2 c7i.metal-24xl   | avx512_vpclmulqdq   |        ~24.9 |       ~109.7 |
+| x86_64  | AMD   | Genoa           | EC2 c7a.metal-48xl   | avx512_vpclmulqdq   |        ~24.4 |        ~54.6 |
+| aarch64 | AWS   | Graviton4       | EC2 c8g.metal-48xl   | neon_pclmulqdq_eor3 |        ~18.7 |        ~36.8 |
+| aarch64 | AWS   | Graviton2       | EC2 c6g.metal        | neon_pclmulqdq      |         ~9.8 |        ~15.9 |
+| aarch64 | Apple | M3 Ultra        | Mac Studio (32 core) | neon_pclmulqdq_eor3 |        ~49.5 |        ~71.9 |
 
 ## Other CRC widths
 

src/algorithm.rs

@@ -29,7 +29,7 @@ use crate::{crc32, crc64};
 )]
 #[cfg_attr(
     all(target_arch = "x86_64", feature = "vpclmulqdq"),
-    target_feature(enable = "sse2,sse4.1,pclmulqdq,avx2,vpclmulqdq")
+    target_feature(enable = "avx2,vpclmulqdq,avx512f,avx512vl")
 )]
 #[cfg_attr(target_arch = "aarch64", target_feature(enable = "neon,aes"))]
 pub unsafe fn update<T: ArchOps, W: EnhancedCrcWidth>(
@@ -120,6 +120,10 @@ where
     any(target_arch = "x86", target_arch = "x86_64"),
     target_feature(enable = "sse2,sse4.1,pclmulqdq")
 )]
+#[cfg_attr(
+    all(target_arch = "x86_64", feature = "vpclmulqdq"),
+    target_feature(enable = "avx2,vpclmulqdq,avx512f,avx512vl")
+)]
 #[cfg_attr(target_arch = "aarch64", target_feature(enable = "neon,aes"))]
 unsafe fn process_large_aligned<T: ArchOps, W: EnhancedCrcWidth>(
     bytes: &[u8],

src/arch/aarch64.rs

@@ -6,7 +6,6 @@
 
 use crate::traits::ArchOps;
 use std::arch::aarch64::*;
-use std::arch::is_aarch64_feature_detected;
 
 #[derive(Debug, Copy, Clone)]
 pub struct AArch64Ops;
@@ -258,20 +257,27 @@ impl ArchOps for AArch64Ops {
     }
 
     #[inline]
-    #[cfg_attr(target_feature = "sha3", target_feature(enable = "neon,sha3"))]
-    #[cfg_attr(not(target_feature = "sha3"), target_feature(enable = "neon"))]
+    #[cfg(target_feature = "sha3")]
+    #[target_feature(enable = "neon,sha3")]
     unsafe fn xor3_vectors(
         &self,
         a: Self::Vector,
         b: Self::Vector,
         c: Self::Vector,
     ) -> Self::Vector {
-        if is_aarch64_feature_detected!("sha3") {
-            // Use native 3-way XOR instruction when available
-            return veor3q_u8(a, b, c);
-        }
+        veor3q_u8(a, b, c)
+    }
 
-        // Fall back to two XOR operations
+    #[inline]
+    #[cfg(not(target_feature = "sha3"))]
+    #[target_feature(enable = "neon")]
+    unsafe fn xor3_vectors(
+        &self,
+        a: Self::Vector,
+        b: Self::Vector,
+        c: Self::Vector,
+    ) -> Self::Vector {
+        // Fallback for when SHA3 is not available
         veorq_u8(veorq_u8(a, b), c)
     }
 }

src/arch/mod.rs

@@ -32,7 +32,16 @@ mod x86;
 ///
 /// # Safety
 /// May use native CPU features
-#[inline(always)]
+#[inline]
+#[cfg_attr(
+    any(target_arch = "x86", target_arch = "x86_64"),
+    target_feature(enable = "sse2,sse4.1,pclmulqdq")
+)]
+#[cfg_attr(
+    all(target_arch = "x86_64", feature = "vpclmulqdq"),
+    target_feature(enable = "avx2,vpclmulqdq,avx512f,avx512vl")
+)]
+#[cfg_attr(target_arch = "aarch64", target_feature(enable = "neon,aes"))]
 pub(crate) unsafe fn update(state: u64, bytes: &[u8], params: CrcParams) -> u64 {
     #[cfg(target_arch = "aarch64")]
     {
@@ -83,7 +92,10 @@ pub(crate) unsafe fn update(state: u64, bytes: &[u8], params: CrcParams) -> u64
 }
 
 pub fn get_target() -> String {
-    #[cfg(target_arch = "aarch64")]
+    #[cfg(all(target_arch = "aarch64", target_feature = "sha3"))]
+    return "internal-aarch64-neon-eor3".to_string();
+
+    #[cfg(all(target_arch = "aarch64", not(target_feature = "sha3")))]
     return "internal-aarch64-neon".to_string();
 
     #[cfg(all(target_arch = "x86_64", feature = "vpclmulqdq"))]

src/arch/x86.rs

@@ -228,27 +228,28 @@ impl ArchOps for X86Ops {
     }
 
     #[inline]
-    #[cfg_attr(
-        any(feature = "vpclmulqdq", feature = "avx512"),
-        target_feature(enable = "avx512f,avx512vl")
-    )]
-    #[cfg_attr(
-        all(not(feature = "vpclmulqdq"), not(feature = "avx512")),
-        target_feature(enable = "sse2,sse4.1")
-    )]
+    #[cfg(any(feature = "vpclmulqdq", feature = "avx512"))]
+    #[target_feature(enable = "avx512f,avx512vl")]
     unsafe fn xor3_vectors(
         &self,
         a: Self::Vector,
         b: Self::Vector,
         c: Self::Vector,
     ) -> Self::Vector {
-        #[cfg(any(feature = "vpclmulqdq", feature = "avx512"))]
-        if is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("avx512vl") {
-            return _mm_ternarylogic_epi64(
-                a, b, c, 0x96, // XOR3
-            );
-        }
+        _mm_ternarylogic_epi64(
+            a, b, c, 0x96, // XOR3
+        )
+    }
 
+    #[inline]
+    #[cfg(not(any(feature = "vpclmulqdq", feature = "avx512")))]
+    #[target_feature(enable = "sse2,sse4.1")]
+    unsafe fn xor3_vectors(
+        &self,
+        a: Self::Vector,
+        b: Self::Vector,
+        c: Self::Vector,
+    ) -> Self::Vector {
         // x86 doesn't have native XOR3 in SSE, use two XORs
         _mm_xor_si128(_mm_xor_si128(a, b), c)
     }

src/structs.rs

@@ -47,6 +47,7 @@ pub struct CrcState<T> {
 pub(crate) struct Calculator {}
 
 impl CrcCalculator for Calculator {
+    #[inline(always)]
     fn calculate(state: u64, data: &[u8], params: CrcParams) -> u64 {
         unsafe { arch::update(state, data, params) }
     }