Let miri and const eval execute intrinsics' fallback bodies

oli-obk · oli-obk · commit 351658ae66a6 · 2024-05-03T09:01:12.000Z
diff --git a/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs b/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs
@@ -105,7 +105,7 @@ impl<'mir, 'tcx: 'mir> interpret::Machine<'mir, 'tcx> for DummyMachine {
         _destination: &interpret::MPlaceTy<'tcx, Self::Provenance>,
         _target: Option<BasicBlock>,
         _unwind: UnwindAction,
-    ) -> interpret::InterpResult<'tcx> {
+    ) -> interpret::InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
         unimplemented!()
     }
 
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -459,16 +459,25 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
         dest: &MPlaceTy<'tcx, Self::Provenance>,
         target: Option<mir::BasicBlock>,
         _unwind: mir::UnwindAction,
-    ) -> InterpResult<'tcx> {
+    ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
         // Shared intrinsics.
         if ecx.emulate_intrinsic(instance, args, dest, target)? {
-            return Ok(());
+            return Ok(None);
         }
         let intrinsic_name = ecx.tcx.item_name(instance.def_id());
 
         // CTFE-specific intrinsics.
         let Some(ret) = target else {
-            throw_unsup_format!("intrinsic `{intrinsic_name}` is not supported at compile-time");
+            // Handle diverging intrinsics.
+            if ecx.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {
+                throw_unsup_format!(
+                    "intrinsic `{intrinsic_name}` is not supported at compile-time"
+                );
+            }
+            return Ok(Some(ty::Instance {
+                def: ty::InstanceDef::Item(instance.def_id()),
+                args: instance.args,
+            }));
         };
         match intrinsic_name {
             sym::ptr_guaranteed_cmp => {
@@ -536,14 +545,21 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
             // not the optimization stage.)
             sym::is_val_statically_known => ecx.write_scalar(Scalar::from_bool(false), dest)?,
             _ => {
-                throw_unsup_format!(
-                    "intrinsic `{intrinsic_name}` is not supported at compile-time"
-                );
+                // We haven't handled the intrinsic, let's see if we can use a fallback body.
+                if ecx.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {
+                    throw_unsup_format!(
+                        "intrinsic `{intrinsic_name}` is not supported at compile-time"
+                    );
+                }
+                return Ok(Some(ty::Instance {
+                    def: ty::InstanceDef::Item(instance.def_id()),
+                    args: instance.args,
+                }));
             }
         }
 
         ecx.go_to_block(ret);
-        Ok(())
+        Ok(None)
     }
 
     fn assert_panic(
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -414,7 +414,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 }
                 self.copy_op(&self.project_index(&input, index)?, dest)?;
             }
-            sym::likely | sym::unlikely | sym::black_box => {
+            sym::black_box => {
                 // These just return their argument
                 self.copy_op(&args[0], dest)?;
             }
diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs
@@ -216,14 +216,17 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
 
     /// Directly process an intrinsic without pushing a stack frame. It is the hook's
     /// responsibility to advance the instruction pointer as appropriate.
+    ///
+    /// Returns `None` if the intrinsic was fully handled.
+    /// Otherwise, returns an `Instance` of the function that implements the intrinsic.
     fn call_intrinsic(
         ecx: &mut InterpCx<'mir, 'tcx, Self>,
         instance: ty::Instance<'tcx>,
         args: &[OpTy<'tcx, Self::Provenance>],
         destination: &MPlaceTy<'tcx, Self::Provenance>,
         target: Option<mir::BasicBlock>,
         unwind: mir::UnwindAction,
-    ) -> InterpResult<'tcx>;
+    ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>>;
 
     /// Called to evaluate `Assert` MIR terminators that trigger a panic.
     fn assert_panic(
diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs
@@ -539,14 +539,28 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             ty::InstanceDef::Intrinsic(def_id) => {
                 assert!(self.tcx.intrinsic(def_id).is_some());
                 // FIXME: Should `InPlace` arguments be reset to uninit?
-                M::call_intrinsic(
+                if let Some(fallback) = M::call_intrinsic(
                     self,
                     instance,
                     &self.copy_fn_args(args),
                     destination,
                     target,
                     unwind,
-                )
+                )? {
+                    assert!(!self.tcx.intrinsic(fallback.def_id()).unwrap().must_be_overridden);
+                    assert!(matches!(fallback.def, ty::InstanceDef::Item(_)));
+                    return self.eval_fn_call(
+                        FnVal::Instance(fallback),
+                        (caller_abi, caller_fn_abi),
+                        args,
+                        with_caller_location,
+                        destination,
+                        target,
+                        unwind,
+                    );
+                } else {
+                    Ok(())
+                }
             }
             ty::InstanceDef::VTableShim(..)
             | ty::InstanceDef::ReifyShim(..)
diff --git a/src/tools/miri/src/machine.rs b/src/tools/miri/src/machine.rs
@@ -986,7 +986,7 @@ impl<'mir, 'tcx> Machine<'mir, 'tcx> for MiriMachine<'mir, 'tcx> {
         dest: &MPlaceTy<'tcx, Provenance>,
         ret: Option<mir::BasicBlock>,
         unwind: mir::UnwindAction,
-    ) -> InterpResult<'tcx> {
+    ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
         ecx.call_intrinsic(instance, args, dest, ret, unwind)
     }
 
diff --git a/src/tools/miri/src/shims/intrinsics/atomic.rs b/src/tools/miri/src/shims/intrinsics/atomic.rs
@@ -19,7 +19,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
         intrinsic_name: &str,
         args: &[OpTy<'tcx, Provenance>],
         dest: &MPlaceTy<'tcx, Provenance>,
-    ) -> InterpResult<'tcx> {
+    ) -> InterpResult<'tcx, bool> {
         let this = self.eval_context_mut();
 
         let intrinsic_structure: Vec<_> = intrinsic_name.split('_').collect();
@@ -113,9 +113,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
                 this.atomic_rmw_op(args, dest, AtomicOp::Max, rw_ord(ord)?)?;
             }
 
-            _ => throw_unsup_format!("unimplemented intrinsic: `atomic_{intrinsic_name}`"),
+            _ => return Ok(false),
         }
-        Ok(())
+        Ok(true)
     }
 }
 
diff --git a/src/tools/miri/src/shims/intrinsics/mod.rs b/src/tools/miri/src/shims/intrinsics/mod.rs
@@ -26,12 +26,12 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
         dest: &MPlaceTy<'tcx, Provenance>,
         ret: Option<mir::BasicBlock>,
         _unwind: mir::UnwindAction,
-    ) -> InterpResult<'tcx> {
+    ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
         let this = self.eval_context_mut();
 
         // See if the core engine can handle this intrinsic.
         if this.emulate_intrinsic(instance, args, dest, ret)? {
-            return Ok(());
+            return Ok(None);
         }
         let intrinsic_name = this.tcx.item_name(instance.def_id());
         let intrinsic_name = intrinsic_name.as_str();
@@ -54,16 +54,27 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
 
         // Some intrinsics are special and need the "ret".
         match intrinsic_name {
-            "catch_unwind" => return this.handle_catch_unwind(args, dest, ret),
+            "catch_unwind" => {
+                this.handle_catch_unwind(args, dest, ret)?;
+                return Ok(None);
+            }
             _ => {}
         }
 
         // The rest jumps to `ret` immediately.
-        this.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, dest)?;
+        if !this.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, dest)? {
+            if this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {
+                throw_unsup_format!("unimplemented intrinsic: `{intrinsic_name}`")
+            }
+            return Ok(Some(ty::Instance {
+                def: ty::InstanceDef::Item(instance.def_id()),
+                args: instance.args,
+            }))
+        }
 
         trace!("{:?}", this.dump_place(&dest.clone().into()));
         this.go_to_block(ret);
-        Ok(())
+        Ok(None)
     }
 
     /// Emulates a Miri-supported intrinsic (not supported by the core engine).
@@ -73,7 +84,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
         generic_args: ty::GenericArgsRef<'tcx>,
         args: &[OpTy<'tcx, Provenance>],
         dest: &MPlaceTy<'tcx, Provenance>,
-    ) -> InterpResult<'tcx> {
+    ) -> InterpResult<'tcx, bool> {
         let this = self.eval_context_mut();
 
         if let Some(name) = intrinsic_name.strip_prefix("atomic_") {
@@ -84,24 +95,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
         }
 
         match intrinsic_name {
-            // Miri overwriting CTFE intrinsics.
-            "ptr_guaranteed_cmp" => {
-                let [left, right] = check_arg_count(args)?;
-                let left = this.read_immediate(left)?;
-                let right = this.read_immediate(right)?;
-                let val = this.wrapping_binary_op(mir::BinOp::Eq, &left, &right)?;
-                // We're type punning a bool as an u8 here.
-                this.write_scalar(val.to_scalar(), dest)?;
-            }
-            "const_allocate" => {
-                // For now, for compatibility with the run-time implementation of this, we just return null.
-                // See <https://github.com/rust-lang/rust/issues/93935>.
-                this.write_null(dest)?;
-            }
-            "const_deallocate" => {
-                // complete NOP
-            }
-
             // Raw memory accesses
             "volatile_load" => {
                 let [place] = check_arg_count(args)?;
@@ -425,9 +418,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
                 throw_machine_stop!(TerminationInfo::Abort(format!("trace/breakpoint trap")))
             }
 
-            name => throw_unsup_format!("unimplemented intrinsic: `{name}`"),
+            _ => return Ok(false),
         }
 
-        Ok(())
+        Ok(true)
     }
 }
diff --git a/src/tools/miri/src/shims/intrinsics/simd.rs b/src/tools/miri/src/shims/intrinsics/simd.rs
@@ -22,7 +22,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
         generic_args: ty::GenericArgsRef<'tcx>,
         args: &[OpTy<'tcx, Provenance>],
         dest: &MPlaceTy<'tcx, Provenance>,
-    ) -> InterpResult<'tcx> {
+    ) -> InterpResult<'tcx, bool> {
         let this = self.eval_context_mut();
         match intrinsic_name {
             #[rustfmt::skip]
@@ -743,9 +743,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
                 }
             }
 
-            name => throw_unsup_format!("unimplemented intrinsic: `simd_{name}`"),
+            _ => return Ok(false),
         }
-        Ok(())
+        Ok(true)
     }
 
     fn fminmax_op(

Original file line number	Diff line number	Diff line change
`@@ -105,7 +105,7 @@ impl<'mir, 'tcx: 'mir> interpret::Machine<'mir, 'tcx> for DummyMachine {`
`105`	`105`	`_destination: &interpret::MPlaceTy<'tcx, Self::Provenance>,`
`106`	`106`	`_target: Option<BasicBlock>,`
`107`	`107`	`_unwind: UnwindAction,`
`108`		`- ) -> interpret::InterpResult<'tcx> {`
	`108`	`+ ) -> interpret::InterpResult<'tcx, Option<ty::Instance<'tcx>>> {`
`109`	`109`	`unimplemented!()`
`110`	`110`	`}`
`111`	`111`
Original file line number	Diff line number	Diff line change
`@@ -414,7 +414,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {`
`414`	`414`	`}`
`415`	`415`	`self.copy_op(&self.project_index(&input, index)?, dest)?;`
`416`	`416`	`}`
`417`		`- sym::likely \| sym::unlikely \| sym::black_box => {`
	`417`	`+ sym::black_box => {`
`418`	`418`	`// These just return their argument`
`419`	`419`	`self.copy_op(&args[0], dest)?;`
`420`	`420`	`}`
Original file line number	Diff line number	Diff line change
`@@ -986,7 +986,7 @@ impl<'mir, 'tcx> Machine<'mir, 'tcx> for MiriMachine<'mir, 'tcx> {`
`986`	`986`	`dest: &MPlaceTy<'tcx, Provenance>,`
`987`	`987`	`ret: Option<mir::BasicBlock>,`
`988`	`988`	`unwind: mir::UnwindAction,`
`989`		`- ) -> InterpResult<'tcx> {`
	`989`	`+ ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {`
`990`	`990`	`ecx.call_intrinsic(instance, args, dest, ret, unwind)`
`991`	`991`	`}`
`992`	`992`
Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {`
`19`	`19`	`intrinsic_name: &str,`
`20`	`20`	`args: &[OpTy<'tcx, Provenance>],`
`21`	`21`	`dest: &MPlaceTy<'tcx, Provenance>,`
`22`		`- ) -> InterpResult<'tcx> {`
	`22`	`+ ) -> InterpResult<'tcx, bool> {`
`23`	`23`	`let this = self.eval_context_mut();`
`24`	`24`
`25`	`25`	`let intrinsic_structure: Vec<_> = intrinsic_name.split('_').collect();`
`@@ -113,9 +113,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {`
`113`	`113`	`this.atomic_rmw_op(args, dest, AtomicOp::Max, rw_ord(ord)?)?;`
`114`	`114`	`}`
`115`	`115`
`116`		- _ => throw_unsup_format!("unimplemented intrinsic: `atomic_{intrinsic_name}`"),
	`116`	`+ _ => return Ok(false),`
`117`	`117`	`}`
`118`		`- Ok(())`
	`118`	`+ Ok(true)`
`119`	`119`	`}`
`120`	`120`	`}`
`121`	`121`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {`
`22`	`22`	`generic_args: ty::GenericArgsRef<'tcx>,`
`23`	`23`	`args: &[OpTy<'tcx, Provenance>],`
`24`	`24`	`dest: &MPlaceTy<'tcx, Provenance>,`
`25`		`- ) -> InterpResult<'tcx> {`
	`25`	`+ ) -> InterpResult<'tcx, bool> {`
`26`	`26`	`let this = self.eval_context_mut();`
`27`	`27`	`match intrinsic_name {`
`28`	`28`	`#[rustfmt::skip]`
`@@ -743,9 +743,9 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {`
`743`	`743`	`}`
`744`	`744`	`}`
`745`	`745`
`746`		- name => throw_unsup_format!("unimplemented intrinsic: `simd_{name}`"),
	`746`	`+ _ => return Ok(false),`
`747`	`747`	`}`
`748`		`- Ok(())`
	`748`	`+ Ok(true)`
`749`	`749`	`}`
`750`	`750`
`751`	`751`	`fn fminmax_op(`