Rectify float classification impls for weird FPUs

workingjubilee · workingjubilee · commit b200483412b1 · 2022-04-12T02:27:25.000-07:00
Careful handling does its best to take care of both Armv7's
"unenhanced" Neon as well as the x87 FPU.
diff --git a/library/core/src/num/f32.rs b/library/core/src/num/f32.rs
@@ -449,7 +449,8 @@ impl f32 {
     #[inline]
     #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")]
     pub(crate) const fn abs_private(self) -> f32 {
-        f32::from_bits(self.to_bits() & 0x7fff_ffff)
+        // SAFETY: This transmutation is fine. Probably. For the reasons std is using it.
+        unsafe { mem::transmute::<u32, f32>(mem::transmute::<f32, u32>(self) & 0x7fff_ffff) }
     }
 
     /// Returns `true` if this value is positive infinity or negative infinity, and
@@ -472,7 +473,10 @@ impl f32 {
     #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")]
     #[inline]
     pub const fn is_infinite(self) -> bool {
-        self.abs_private() == Self::INFINITY
+        // Getting clever with transmutation can result in incorrect answers on some FPUs
+        // FIXME: alter the Rust <-> Rust calling convention to prevent this problem.
+        // See https://github.com/rust-lang/rust/issues/72327
+        (self == f32::INFINITY) | (self == f32::NEG_INFINITY)
     }
 
     /// Returns `true` if this number is neither infinite nor `NaN`.
@@ -568,15 +572,53 @@ impl f32 {
     #[stable(feature = "rust1", since = "1.0.0")]
     #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")]
     pub const fn classify(self) -> FpCategory {
+        // A previous implementation tried to only use bitmask-based checks,
+        // using f32::to_bits to transmute the float to its bit repr and match on that.
+        // Unfortunately, floating point numbers can be much worse than that.
+        // This also needs to not result in recursive evaluations of f64::to_bits.
+        //
+        // On some processors, in some cases, LLVM will "helpfully" lower floating point ops,
+        // in spite of a request for them using f32 and f64, to things like x87 operations.
+        // These have an f64's mantissa, but can have a larger than normal exponent.
+        // FIXME(jubilee): Using x87 operations is never necessary in order to function
+        // on x86 processors for Rust-to-Rust calls, so this issue should not happen.
+        // Code generation should be adjusted to use non-C calling conventions, avoiding this.
+        //
+        if self.is_infinite() {
+            // Thus, a value may compare unequal to infinity, despite having a "full" exponent mask.
+            FpCategory::Infinite
+        } else if self.is_nan() {
+            // And it may not be NaN, as it can simply be an "overextended" finite value.
+            FpCategory::Nan
+        } else {
+            // However, std can't simply compare to zero to check for zero, either,
+            // as correctness requires avoiding equality tests that may be Subnormal == -0.0
+            // because it may be wrong under "denormals are zero" and "flush to zero" modes.
+            // Most of std's targets don't use those, but they are used for thumbv7neon".
+            // So, this does use bitpattern matching for the rest.
+
+            // SAFETY: f32 to u32 is fine. Usually.
+            // If classify has gotten this far, the value is definitely in one of these categories.
+            unsafe { f32::partial_classify(self) }
+        }
+    }
+
+    // This doesn't actually return a right answer for NaN on purpose,
+    // seeing as how it cannot correctly discern between a floating point NaN,
+    // and some normal floating point numbers truncated from an x87 FPU.
+    // FIXME(jubilee): This probably could at least answer things correctly for Infinity,
+    // like the f64 version does, but I need to run more checks on how things go on x86.
+    // I fear losing mantissa data that would have answered that differently.
+    #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")]
+    const unsafe fn partial_classify(self) -> FpCategory {
         const EXP_MASK: u32 = 0x7f800000;
         const MAN_MASK: u32 = 0x007fffff;
 
-        let bits = self.to_bits();
-        match (bits & MAN_MASK, bits & EXP_MASK) {
+        // SAFETY: The caller is not asking questions for which this will tell lies.
+        let b = unsafe { mem::transmute::<f32, u32>(self) };
+        match (b & MAN_MASK, b & EXP_MASK) {
             (0, 0) => FpCategory::Zero,
             (_, 0) => FpCategory::Subnormal,
-            (0, EXP_MASK) => FpCategory::Infinite,
-            (_, EXP_MASK) => FpCategory::Nan,
             _ => FpCategory::Normal,
         }
     }
@@ -616,7 +658,8 @@ impl f32 {
     pub const fn is_sign_negative(self) -> bool {
         // IEEE754 says: isSignMinus(x) is true if and only if x has negative sign. isSignMinus
         // applies to zeros and NaNs as well.
-        self.to_bits() & 0x8000_0000 != 0
+        // SAFETY: This is just transmuting to get the sign bit, it's fine.
+        unsafe { mem::transmute::<f32, u32>(self) & 0x8000_0000 != 0 }
     }
 
     /// Takes the reciprocal (inverse) of a number, `1/x`.
@@ -878,7 +921,7 @@ impl f32 {
     pub const fn from_bits(v: u32) -> Self {
         // SAFETY: `u32` is a plain old datatype so we can always transmute from it
         // It turns out the safety issues with sNaN were overblown! Hooray!
-        unsafe { mem::transmute(v) }
+        unsafe { mem::transmute::<u32, f32>(v) }
     }
 
     /// Return the memory representation of this floating point number as a byte array in
diff --git a/library/core/src/num/f64.rs b/library/core/src/num/f64.rs
@@ -448,7 +448,10 @@ impl f64 {
     #[inline]
     #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")]
     pub(crate) const fn abs_private(self) -> f64 {
-        f64::from_bits(self.to_bits() & 0x7fff_ffff_ffff_ffff)
+        // SAFETY: This transmutation is fine. Probably. For the reasons std is using it.
+        unsafe {
+            mem::transmute::<u64, f64>(mem::transmute::<f64, u64>(self) & 0x7fff_ffff_ffff_ffff)
+        }
     }
 
     /// Returns `true` if this value is positive infinity or negative infinity, and
@@ -471,7 +474,10 @@ impl f64 {
     #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")]
     #[inline]
     pub const fn is_infinite(self) -> bool {
-        self.abs_private() == Self::INFINITY
+        // Getting clever with transmutation can result in incorrect answers on some FPUs
+        // FIXME: alter the Rust <-> Rust calling convention to prevent this problem.
+        // See https://github.com/rust-lang/rust/issues/72327
+        (self == f64::INFINITY) | (self == f64::NEG_INFINITY)
     }
 
     /// Returns `true` if this number is neither infinite nor `NaN`.
@@ -567,15 +573,50 @@ impl f64 {
     #[stable(feature = "rust1", since = "1.0.0")]
     #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")]
     pub const fn classify(self) -> FpCategory {
+        // A previous implementation tried to only use bitmask-based checks,
+        // using f64::to_bits to transmute the float to its bit repr and match on that.
+        // Unfortunately, floating point numbers can be much worse than that.
+        // This also needs to not result in recursive evaluations of f64::to_bits.
+        //
+        // On some processors, in some cases, LLVM will "helpfully" lower floating point ops,
+        // in spite of a request for them using f32 and f64, to things like x87 operations.
+        // These have an f64's mantissa, but can have a larger than normal exponent.
+        // FIXME(jubilee): Using x87 operations is never necessary in order to function
+        // on x86 processors for Rust-to-Rust calls, so this issue should not happen.
+        // Code generation should be adjusted to use non-C calling conventions, avoiding this.
+        //
+        // Thus, a value may compare unequal to infinity, despite having a "full" exponent mask.
+        // And it may not be NaN, as it can simply be an "overextended" finite value.
+        if self.is_nan() {
+            FpCategory::Nan
+        } else {
+            // However, std can't simply compare to zero to check for zero, either,
+            // as correctness requires avoiding equality tests that may be Subnormal == -0.0
+            // because it may be wrong under "denormals are zero" and "flush to zero" modes.
+            // Most of std's targets don't use those, but they are used for thumbv7neon".
+            // So, this does use bitpattern matching for the rest.
+
+            // SAFETY: f64 to u64 is fine. Usually.
+            // If control flow has gotten this far, the value is definitely in one of the categories
+            // that f64::partial_classify can correctly analyze.
+            unsafe { f64::partial_classify(self) }
+        }
+    }
+
+    // This doesn't actually return a right answer for NaN on purpose,
+    // seeing as how it cannot correctly discern between a floating point NaN,
+    // and some normal floating point numbers truncated from an x87 FPU.
+    #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")]
+    const unsafe fn partial_classify(self) -> FpCategory {
         const EXP_MASK: u64 = 0x7ff0000000000000;
         const MAN_MASK: u64 = 0x000fffffffffffff;
 
-        let bits = self.to_bits();
-        match (bits & MAN_MASK, bits & EXP_MASK) {
+        // SAFETY: The caller is not asking questions for which this will tell lies.
+        let b = unsafe { mem::transmute::<f64, u64>(self) };
+        match (b & MAN_MASK, b & EXP_MASK) {
+            (0, EXP_MASK) => FpCategory::Infinite,
             (0, 0) => FpCategory::Zero,
             (_, 0) => FpCategory::Subnormal,
-            (0, EXP_MASK) => FpCategory::Infinite,
-            (_, EXP_MASK) => FpCategory::Nan,
             _ => FpCategory::Normal,
         }
     }
@@ -622,7 +663,10 @@ impl f64 {
     #[rustc_const_unstable(feature = "const_float_classify", issue = "72505")]
     #[inline]
     pub const fn is_sign_negative(self) -> bool {
-        self.to_bits() & 0x8000_0000_0000_0000 != 0
+        // IEEE754 says: isSignMinus(x) is true if and only if x has negative sign. isSignMinus
+        // applies to zeros and NaNs as well.
+        // SAFETY: This is just transmuting to get the sign bit, it's fine.
+        unsafe { mem::transmute::<f64, u64>(self) & 0x8000_0000_0000_0000 != 0 }
     }
 
     #[must_use]
@@ -894,7 +938,7 @@ impl f64 {
     pub const fn from_bits(v: u64) -> Self {
         // SAFETY: `u64` is a plain old datatype so we can always transmute from it
         // It turns out the safety issues with sNaN were overblown! Hooray!
-        unsafe { mem::transmute(v) }
+        unsafe { mem::transmute::<u64, f64>(v) }
     }
 
     /// Return the memory representation of this floating point number as a byte array in
