Make most SIMD intrinsics const-compatible (Port the Miri implementation to rustc_const_eval)

Remaining: Math functions (`fsqrt`, `fsin`, `fcos`, `fexp`, `fexp2`, `flog`, `flog2`, `flog10`),  Funnel Shifts, `dyn` extract-inserts and FMA

Author: sayantn

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -9,8 +9,8 @@ use rustc_apfloat::ieee::{Double, Half, Quad, Single};
 use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint};
 use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
 use rustc_middle::ty::layout::TyAndLayout;
-use rustc_middle::ty::{Ty, TyCtxt};
-use rustc_middle::{bug, ty};
+use rustc_middle::ty::{FloatTy, Ty, TyCtxt};
+use rustc_middle::{bug, span_bug, ty};
 use rustc_span::{Symbol, sym};
 use tracing::trace;
 
@@ -22,6 +22,15 @@ use super::{
     throw_ub_custom, throw_ub_format,
 };
 use crate::fluent_generated as fluent;
+use crate::interpret::Projectable;
+
+#[derive(Copy, Clone)]
+pub(super) enum MinMax {
+    MinNum,
+    MaxNum,
+    Minimum,
+    Maximum,
+}
 
 /// Directly returns an `Allocation` containing an absolute path representation of the given type.
 pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, u64) {
@@ -123,6 +132,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         let intrinsic_name = self.tcx.item_name(instance.def_id());
         let tcx = self.tcx.tcx;
 
+        if intrinsic_name.as_str().starts_with("simd_") {
+            return self.eval_simd_intrinsic(intrinsic_name, instance, args, dest, ret);
+        }
+
         match intrinsic_name {
             sym::type_name => {
                 let tp_ty = instance.args.type_at(0);
@@ -453,38 +466,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 let size = ImmTy::from_int(pointee_layout.size.bytes(), ret_layout);
                 self.exact_div(&val, &size, dest)?;
             }
-
-            sym::simd_insert => {
-                let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
-                let elem = &args[2];
-                let (input, input_len) = self.project_to_simd(&args[0])?;
-                let (dest, dest_len) = self.project_to_simd(dest)?;
-                assert_eq!(input_len, dest_len, "Return vector length must match input length");
-                // Bounds are not checked by typeck so we have to do it ourselves.
-                if index >= input_len {
-                    throw_ub_format!(
-                        "`simd_insert` index {index} is out-of-bounds of vector with length {input_len}"
-                    );
-                }
-
-                for i in 0..dest_len {
-                    let place = self.project_index(&dest, i)?;
-                    let value =
-                        if i == index { elem.clone() } else { self.project_index(&input, i)? };
-                    self.copy_op(&value, &place)?;
-                }
-            }
-            sym::simd_extract => {
-                let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
-                let (input, input_len) = self.project_to_simd(&args[0])?;
-                // Bounds are not checked by typeck so we have to do it ourselves.
-                if index >= input_len {
-                    throw_ub_format!(
-                        "`simd_extract` index {index} is out-of-bounds of vector with length {input_len}"
-                    );
-                }
-                self.copy_op(&self.project_index(&input, index)?, dest)?;
-            }
             sym::black_box => {
                 // These just return their argument
                 self.copy_op(&args[0], dest)?;
@@ -510,25 +491,33 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
             }
 
-            sym::minnumf16 => self.float_min_intrinsic::<Half>(args, dest)?,
-            sym::minnumf32 => self.float_min_intrinsic::<Single>(args, dest)?,
-            sym::minnumf64 => self.float_min_intrinsic::<Double>(args, dest)?,
-            sym::minnumf128 => self.float_min_intrinsic::<Quad>(args, dest)?,
+            sym::minnumf16 => self.float_minmax_intrinsic::<Half>(args, dest, MinMax::MinNum)?,
+            sym::minnumf32 => self.float_minmax_intrinsic::<Single>(args, dest, MinMax::MinNum)?,
+            sym::minnumf64 => self.float_minmax_intrinsic::<Double>(args, dest, MinMax::MinNum)?,
+            sym::minnumf128 => self.float_minmax_intrinsic::<Quad>(args, dest, MinMax::MinNum)?,
 
-            sym::minimumf16 => self.float_minimum_intrinsic::<Half>(args, dest)?,
-            sym::minimumf32 => self.float_minimum_intrinsic::<Single>(args, dest)?,
-            sym::minimumf64 => self.float_minimum_intrinsic::<Double>(args, dest)?,
-            sym::minimumf128 => self.float_minimum_intrinsic::<Quad>(args, dest)?,
+            sym::minimumf16 => self.float_minmax_intrinsic::<Half>(args, dest, MinMax::Minimum)?,
+            sym::minimumf32 => {
+                self.float_minmax_intrinsic::<Single>(args, dest, MinMax::Minimum)?
+            }
+            sym::minimumf64 => {
+                self.float_minmax_intrinsic::<Double>(args, dest, MinMax::Minimum)?
+            }
+            sym::minimumf128 => self.float_minmax_intrinsic::<Quad>(args, dest, MinMax::Minimum)?,
 
-            sym::maxnumf16 => self.float_max_intrinsic::<Half>(args, dest)?,
-            sym::maxnumf32 => self.float_max_intrinsic::<Single>(args, dest)?,
-            sym::maxnumf64 => self.float_max_intrinsic::<Double>(args, dest)?,
-            sym::maxnumf128 => self.float_max_intrinsic::<Quad>(args, dest)?,
+            sym::maxnumf16 => self.float_minmax_intrinsic::<Half>(args, dest, MinMax::MaxNum)?,
+            sym::maxnumf32 => self.float_minmax_intrinsic::<Single>(args, dest, MinMax::MaxNum)?,
+            sym::maxnumf64 => self.float_minmax_intrinsic::<Double>(args, dest, MinMax::MaxNum)?,
+            sym::maxnumf128 => self.float_minmax_intrinsic::<Quad>(args, dest, MinMax::MaxNum)?,
 
-            sym::maximumf16 => self.float_maximum_intrinsic::<Half>(args, dest)?,
-            sym::maximumf32 => self.float_maximum_intrinsic::<Single>(args, dest)?,
-            sym::maximumf64 => self.float_maximum_intrinsic::<Double>(args, dest)?,
-            sym::maximumf128 => self.float_maximum_intrinsic::<Quad>(args, dest)?,
+            sym::maximumf16 => self.float_minmax_intrinsic::<Half>(args, dest, MinMax::Maximum)?,
+            sym::maximumf32 => {
+                self.float_minmax_intrinsic::<Single>(args, dest, MinMax::Maximum)?
+            }
+            sym::maximumf64 => {
+                self.float_minmax_intrinsic::<Double>(args, dest, MinMax::Maximum)?
+            }
+            sym::maximumf128 => self.float_minmax_intrinsic::<Quad>(args, dest, MinMax::Maximum)?,
 
             sym::copysignf16 => self.float_copysign_intrinsic::<Half>(args, dest)?,
             sym::copysignf32 => self.float_copysign_intrinsic::<Single>(args, dest)?,
@@ -917,78 +906,45 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
     }
 
-    fn float_min_intrinsic<F>(
+    fn float_minmax_intrinsic<F>(
         &mut self,
         args: &[OpTy<'tcx, M::Provenance>],
         dest: &PlaceTy<'tcx, M::Provenance>,
+        op: MinMax,
     ) -> InterpResult<'tcx, ()>
     where
         F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
     {
-        let a: F = self.read_scalar(&args[0])?.to_float()?;
-        let b: F = self.read_scalar(&args[1])?.to_float()?;
-        let res = if a == b {
-            // They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
-            // Let the machine decide which one to return.
-            M::equal_float_min_max(self, a, b)
-        } else {
-            self.adjust_nan(a.min(b), &[a, b])
-        };
+        let res = self.float_minmax::<F>(&args[0], &args[1], op)?;
         self.write_scalar(res, dest)?;
         interp_ok(())
     }
 
-    fn float_max_intrinsic<F>(
-        &mut self,
-        args: &[OpTy<'tcx, M::Provenance>],
-        dest: &PlaceTy<'tcx, M::Provenance>,
-    ) -> InterpResult<'tcx, ()>
+    pub(super) fn float_minmax<F>(
+        &self,
+        a: &impl Projectable<'tcx, M::Provenance>,
+        b: &impl Projectable<'tcx, M::Provenance>,
+        op: MinMax,
+    ) -> InterpResult<'tcx, Scalar<M::Provenance>>
     where
         F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
     {
-        let a: F = self.read_scalar(&args[0])?.to_float()?;
-        let b: F = self.read_scalar(&args[1])?.to_float()?;
-        let res = if a == b {
+        let a: F = self.read_scalar(a)?.to_float()?;
+        let b: F = self.read_scalar(b)?.to_float()?;
+        let res = if matches!(op, MinMax::MaxNum | MinMax::MinNum) && a == b {
             // They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
             // Let the machine decide which one to return.
             M::equal_float_min_max(self, a, b)
         } else {
-            self.adjust_nan(a.max(b), &[a, b])
+            let res = match op {
+                MinMax::MinNum => a.min(b),
+                MinMax::MaxNum => a.max(b),
+                MinMax::Minimum => a.minimum(b),
+                MinMax::Maximum => a.maximum(b),
+            };
+            self.adjust_nan(res, &[a, b])
         };
-        self.write_scalar(res, dest)?;
-        interp_ok(())
-    }
-
-    fn float_minimum_intrinsic<F>(
-        &mut self,
-        args: &[OpTy<'tcx, M::Provenance>],
-        dest: &PlaceTy<'tcx, M::Provenance>,
-    ) -> InterpResult<'tcx, ()>
-    where
-        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
-    {
-        let a: F = self.read_scalar(&args[0])?.to_float()?;
-        let b: F = self.read_scalar(&args[1])?.to_float()?;
-        let res = a.minimum(b);
-        let res = self.adjust_nan(res, &[a, b]);
-        self.write_scalar(res, dest)?;
-        interp_ok(())
-    }
-
-    fn float_maximum_intrinsic<F>(
-        &mut self,
-        args: &[OpTy<'tcx, M::Provenance>],
-        dest: &PlaceTy<'tcx, M::Provenance>,
-    ) -> InterpResult<'tcx, ()>
-    where
-        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
-    {
-        let a: F = self.read_scalar(&args[0])?.to_float()?;
-        let b: F = self.read_scalar(&args[1])?.to_float()?;
-        let res = a.maximum(b);
-        let res = self.adjust_nan(res, &[a, b]);
-        self.write_scalar(res, dest)?;
-        interp_ok(())
+        interp_ok(res.into())
     }
 
     fn float_copysign_intrinsic<F>(
@@ -1035,4 +991,66 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         self.write_scalar(res, dest)?;
         interp_ok(())
     }
+
+    fn float_to_int_inner<F: rustc_apfloat::Float>(
+        &self,
+        src: F,
+        cast_to: TyAndLayout<'tcx>,
+        round: rustc_apfloat::Round,
+    ) -> (Scalar<M::Provenance>, rustc_apfloat::Status) {
+        let int_size = cast_to.layout.size;
+        match cast_to.ty.kind() {
+            // Unsigned
+            ty::Uint(_) => {
+                let res = src.to_u128_r(int_size.bits_usize(), round, &mut false);
+                (Scalar::from_uint(res.value, int_size), res.status)
+            }
+            // Signed
+            ty::Int(_) => {
+                let res = src.to_i128_r(int_size.bits_usize(), round, &mut false);
+                (Scalar::from_int(res.value, int_size), res.status)
+            }
+            // Nothing else
+            _ => span_bug!(
+                self.cur_span(),
+                "attempted float-to-int conversion with non-int output type {}",
+                cast_to.ty,
+            ),
+        }
+    }
+
+    /// Converts `src` from floating point to integer type `dest_ty`
+    /// after rounding with mode `round`.
+    /// Returns `None` if `f` is NaN or out of range.
+    pub fn float_to_int_checked(
+        &self,
+        src: &ImmTy<'tcx, M::Provenance>,
+        cast_to: TyAndLayout<'tcx>,
+        round: rustc_apfloat::Round,
+    ) -> InterpResult<'tcx, Option<ImmTy<'tcx, M::Provenance>>> {
+        let ty::Float(fty) = src.layout.ty.kind() else {
+            bug!("float_to_int_checked: non-float input type {}", src.layout.ty)
+        };
+
+        let (val, status) = match fty {
+            FloatTy::F16 => self.float_to_int_inner(src.to_scalar().to_f16()?, cast_to, round),
+            FloatTy::F32 => self.float_to_int_inner(src.to_scalar().to_f32()?, cast_to, round),
+            FloatTy::F64 => self.float_to_int_inner(src.to_scalar().to_f64()?, cast_to, round),
+            FloatTy::F128 => self.float_to_int_inner(src.to_scalar().to_f128()?, cast_to, round),
+        };
+
+        if status.intersects(
+            rustc_apfloat::Status::INVALID_OP
+                | rustc_apfloat::Status::OVERFLOW
+                | rustc_apfloat::Status::UNDERFLOW,
+        ) {
+            // Floating point value is NaN (flagged with INVALID_OP) or outside the range
+            // of values of the integer type (flagged with OVERFLOW or UNDERFLOW).
+            interp_ok(None)
+        } else {
+            // Floating point value can be represented by the integer type after rounding.
+            // The INEXACT flag is ignored on purpose to allow rounding.
+            interp_ok(Some(ImmTy::from_scalar(val, cast_to)))
+        }
+    }
 }

compiler/rustc_const_eval/src/interpret/mod.rs

@@ -12,6 +12,7 @@ mod operand;
 mod operator;
 mod place;
 mod projection;
+mod simd_intrinsics;
 mod stack;
 mod step;
 mod traits;