Auto merge of #82124 - tmiasko:op-ty-ref, r=oli-obk

Pass large interpreter types by reference, not value

r? `@ghost`

Author: bors

compiler/rustc_mir/src/const_eval/eval_queries.rs

@@ -56,7 +56,7 @@ fn eval_body_using_ecx<'mir, 'tcx>(
     ecx.push_stack_frame(
         cid.instance,
         body,
-        Some(ret.into()),
+        Some(&ret.into()),
         StackPopCleanup::None { cleanup: false },
     )?;
 
@@ -72,7 +72,7 @@ fn eval_body_using_ecx<'mir, 'tcx>(
             None => InternKind::Constant,
         }
     };
-    intern_const_alloc_recursive(ecx, intern_kind, ret)?;
+    intern_const_alloc_recursive(ecx, intern_kind, &ret)?;
 
     debug!("eval_body_using_ecx done: {:?}", *ret);
     Ok(ret)
@@ -105,7 +105,7 @@ pub(super) fn mk_eval_cx<'mir, 'tcx>(
 /// type system.
 pub(super) fn op_to_const<'tcx>(
     ecx: &CompileTimeEvalContext<'_, 'tcx>,
-    op: OpTy<'tcx>,
+    op: &OpTy<'tcx>,
 ) -> ConstValue<'tcx> {
     // We do not have value optimizations for everything.
     // Only scalars and slices, since they are very common.
@@ -137,7 +137,7 @@ pub(super) fn op_to_const<'tcx>(
         op.try_as_mplace(ecx)
     };
 
-    let to_const_value = |mplace: MPlaceTy<'_>| match mplace.ptr {
+    let to_const_value = |mplace: &MPlaceTy<'_>| match mplace.ptr {
         Scalar::Ptr(ptr) => {
             let alloc = ecx.tcx.global_alloc(ptr.alloc_id).unwrap_memory();
             ConstValue::ByRef { alloc, offset: ptr.offset }
@@ -155,12 +155,12 @@ pub(super) fn op_to_const<'tcx>(
         }
     };
     match immediate {
-        Ok(mplace) => to_const_value(mplace),
+        Ok(ref mplace) => to_const_value(mplace),
         // see comment on `let try_as_immediate` above
         Err(imm) => match *imm {
             Immediate::Scalar(x) => match x {
                 ScalarMaybeUninit::Scalar(s) => ConstValue::Scalar(s),
-                ScalarMaybeUninit::Uninit => to_const_value(op.assert_mem_place(ecx)),
+                ScalarMaybeUninit::Uninit => to_const_value(&op.assert_mem_place(ecx)),
             },
             Immediate::ScalarPair(a, b) => {
                 let (data, start) = match a.check_init().unwrap() {
@@ -201,7 +201,7 @@ fn turn_into_const_value<'tcx>(
         "the `eval_to_const_value_raw` query should not be used for statics, use `eval_to_allocation` instead"
     );
     // Turn this into a proper constant.
-    op_to_const(&ecx, mplace.into())
+    op_to_const(&ecx, &mplace.into())
 }
 
 pub fn eval_to_const_value_raw_provider<'tcx>(
@@ -348,7 +348,7 @@ pub fn eval_to_allocation_raw_provider<'tcx>(
                         Some(_) => CtfeValidationMode::Regular, // a `static`
                         None => CtfeValidationMode::Const { inner, allow_static_ptrs: false },
                     };
-                    ecx.const_validate_operand(mplace.into(), path, &mut ref_tracking, mode)?;
+                    ecx.const_validate_operand(&mplace.into(), path, &mut ref_tracking, mode)?;
                     inner = true;
                 }
             };

compiler/rustc_mir/src/const_eval/machine.rs

@@ -39,8 +39,8 @@ impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> {
             // &str
             assert!(args.len() == 1);
 
-            let msg_place = self.deref_operand(args[0])?;
-            let msg = Symbol::intern(self.read_str(msg_place)?);
+            let msg_place = self.deref_operand(&args[0])?;
+            let msg = Symbol::intern(self.read_str(&msg_place)?);
             let span = self.find_closest_untracked_caller_location();
             let (file, line, col) = self.location_triple_for_span(span);
             Err(ConstEvalErrKind::Panic { msg, file, line, col }.into())
@@ -222,7 +222,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
         instance: ty::Instance<'tcx>,
         _abi: Abi,
         args: &[OpTy<'tcx>],
-        _ret: Option<(PlaceTy<'tcx>, mir::BasicBlock)>,
+        _ret: Option<(&PlaceTy<'tcx>, mir::BasicBlock)>,
         _unwind: Option<mir::BasicBlock>, // unwinding is not supported in consts
     ) -> InterpResult<'tcx, Option<&'mir mir::Body<'tcx>>> {
         debug!("find_mir_or_eval_fn: {:?}", instance);
@@ -262,7 +262,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
         ecx: &mut InterpCx<'mir, 'tcx, Self>,
         instance: ty::Instance<'tcx>,
         args: &[OpTy<'tcx>],
-        ret: Option<(PlaceTy<'tcx>, mir::BasicBlock)>,
+        ret: Option<(&PlaceTy<'tcx>, mir::BasicBlock)>,
         _unwind: Option<mir::BasicBlock>,
     ) -> InterpResult<'tcx> {
         // Shared intrinsics.
@@ -284,8 +284,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
         };
         match intrinsic_name {
             sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => {
-                let a = ecx.read_immediate(args[0])?.to_scalar()?;
-                let b = ecx.read_immediate(args[1])?.to_scalar()?;
+                let a = ecx.read_immediate(&args[0])?.to_scalar()?;
+                let b = ecx.read_immediate(&args[1])?.to_scalar()?;
                 let cmp = if intrinsic_name == sym::ptr_guaranteed_eq {
                     ecx.guaranteed_eq(a, b)
                 } else {
@@ -294,8 +294,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
                 ecx.write_scalar(Scalar::from_bool(cmp), dest)?;
             }
             sym::const_allocate => {
-                let size = ecx.read_scalar(args[0])?.to_machine_usize(ecx)?;
-                let align = ecx.read_scalar(args[1])?.to_machine_usize(ecx)?;
+                let size = ecx.read_scalar(&args[0])?.to_machine_usize(ecx)?;
+                let align = ecx.read_scalar(&args[1])?.to_machine_usize(ecx)?;
 
                 let align = match Align::from_bytes(align) {
                     Ok(a) => a,
@@ -330,7 +330,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
         use rustc_middle::mir::AssertKind::*;
         // Convert `AssertKind<Operand>` to `AssertKind<Scalar>`.
         let eval_to_int =
-            |op| ecx.read_immediate(ecx.eval_operand(op, None)?).map(|x| x.to_const_int());
+            |op| ecx.read_immediate(&ecx.eval_operand(op, None)?).map(|x| x.to_const_int());
         let err = match msg {
             BoundsCheck { ref len, ref index } => {
                 let len = eval_to_int(len)?;
@@ -358,15 +358,15 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
     fn binary_ptr_op(
         _ecx: &InterpCx<'mir, 'tcx, Self>,
         _bin_op: mir::BinOp,
-        _left: ImmTy<'tcx>,
-        _right: ImmTy<'tcx>,
+        _left: &ImmTy<'tcx>,
+        _right: &ImmTy<'tcx>,
     ) -> InterpResult<'tcx, (Scalar, bool, Ty<'tcx>)> {
         Err(ConstEvalErrKind::NeedsRfc("pointer arithmetic or comparison".to_string()).into())
     }
 
     fn box_alloc(
         _ecx: &mut InterpCx<'mir, 'tcx, Self>,
-        _dest: PlaceTy<'tcx>,
+        _dest: &PlaceTy<'tcx>,
     ) -> InterpResult<'tcx> {
         Err(ConstEvalErrKind::NeedsRfc("heap allocations via `box` keyword".to_string()).into())
     }

compiler/rustc_mir/src/const_eval/mod.rs

@@ -29,7 +29,7 @@ pub(crate) fn const_caller_location(
     let mut ecx = mk_eval_cx(tcx, DUMMY_SP, ty::ParamEnv::reveal_all(), false);
 
     let loc_place = ecx.alloc_caller_location(file, line, col);
-    if intern_const_alloc_recursive(&mut ecx, InternKind::Constant, loc_place).is_err() {
+    if intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &loc_place).is_err() {
         bug!("intern_const_alloc_recursive should not error in this case")
     }
     ConstValue::Scalar(loc_place.ptr)
@@ -55,17 +55,17 @@ pub(crate) fn destructure_const<'tcx>(
             return mir::DestructuredConst { variant: None, fields: &[] };
         }
         ty::Adt(def, _) => {
-            let variant = ecx.read_discriminant(op).unwrap().1;
-            let down = ecx.operand_downcast(op, variant).unwrap();
+            let variant = ecx.read_discriminant(&op).unwrap().1;
+            let down = ecx.operand_downcast(&op, variant).unwrap();
             (def.variants[variant].fields.len(), Some(variant), down)
         }
         ty::Tuple(substs) => (substs.len(), None, op),
         _ => bug!("cannot destructure constant {:?}", val),
     };
 
     let fields_iter = (0..field_count).map(|i| {
-        let field_op = ecx.operand_field(down, i).unwrap();
-        let val = op_to_const(&ecx, field_op);
+        let field_op = ecx.operand_field(&down, i).unwrap();
+        let val = op_to_const(&ecx, &field_op);
         ty::Const::from_value(tcx, val, field_op.layout.ty)
     });
     let fields = tcx.arena.alloc_from_iter(fields_iter);
@@ -81,7 +81,7 @@ pub(crate) fn deref_const<'tcx>(
     trace!("deref_const: {:?}", val);
     let ecx = mk_eval_cx(tcx, DUMMY_SP, param_env, false);
     let op = ecx.const_to_op(val, None).unwrap();
-    let mplace = ecx.deref_operand(op).unwrap();
+    let mplace = ecx.deref_operand(&op).unwrap();
     if let Scalar::Ptr(ptr) = mplace.ptr {
         assert_eq!(
             ecx.memory.get_raw(ptr.alloc_id).unwrap().mutability,
@@ -106,5 +106,5 @@ pub(crate) fn deref_const<'tcx>(
         },
     };
 
-    tcx.mk_const(ty::Const { val: ty::ConstKind::Value(op_to_const(&ecx, mplace.into())), ty })
+    tcx.mk_const(ty::Const { val: ty::ConstKind::Value(op_to_const(&ecx, &mplace.into())), ty })
 }

compiler/rustc_mir/src/interpret/cast.rs

@@ -17,10 +17,10 @@ use super::{
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     pub fn cast(
         &mut self,
-        src: OpTy<'tcx, M::PointerTag>,
+        src: &OpTy<'tcx, M::PointerTag>,
         cast_kind: CastKind,
         cast_ty: Ty<'tcx>,
-        dest: PlaceTy<'tcx, M::PointerTag>,
+        dest: &PlaceTy<'tcx, M::PointerTag>,
     ) -> InterpResult<'tcx> {
         use rustc_middle::mir::CastKind::*;
         // FIXME: In which cases should we trigger UB when the source is uninit?
@@ -32,7 +32,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
             Misc => {
                 let src = self.read_immediate(src)?;
-                let res = self.misc_cast(src, cast_ty)?;
+                let res = self.misc_cast(&src, cast_ty)?;
                 self.write_immediate(res, dest)?;
             }
 
@@ -107,7 +107,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
     fn misc_cast(
         &self,
-        src: ImmTy<'tcx, M::PointerTag>,
+        src: &ImmTy<'tcx, M::PointerTag>,
         cast_ty: Ty<'tcx>,
     ) -> InterpResult<'tcx, Immediate<M::PointerTag>> {
         use rustc_middle::ty::TyKind::*;
@@ -158,13 +158,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             let dest_layout = self.layout_of(cast_ty)?;
             if dest_layout.size == src.layout.size {
                 // Thin or fat pointer that just hast the ptr kind of target type changed.
-                return Ok(*src);
+                return Ok(**src);
             } else {
                 // Casting the metadata away from a fat ptr.
                 assert_eq!(src.layout.size, 2 * self.memory.pointer_size());
                 assert_eq!(dest_layout.size, self.memory.pointer_size());
                 assert!(src.layout.ty.is_unsafe_ptr());
-                return match *src {
+                return match **src {
                     Immediate::ScalarPair(data, _) => Ok(data.into()),
                     Immediate::Scalar(..) => span_bug!(
                         self.cur_span(),
@@ -259,8 +259,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
     fn unsize_into_ptr(
         &mut self,
-        src: OpTy<'tcx, M::PointerTag>,
-        dest: PlaceTy<'tcx, M::PointerTag>,
+        src: &OpTy<'tcx, M::PointerTag>,
+        dest: &PlaceTy<'tcx, M::PointerTag>,
         // The pointee types
         source_ty: Ty<'tcx>,
         cast_ty: Ty<'tcx>,
@@ -300,9 +300,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
     fn unsize_into(
         &mut self,
-        src: OpTy<'tcx, M::PointerTag>,
+        src: &OpTy<'tcx, M::PointerTag>,
         cast_ty: TyAndLayout<'tcx>,
-        dest: PlaceTy<'tcx, M::PointerTag>,
+        dest: &PlaceTy<'tcx, M::PointerTag>,
     ) -> InterpResult<'tcx> {
         trace!("Unsizing {:?} of type {} into {:?}", *src, src.layout.ty, cast_ty.ty);
         match (&src.layout.ty.kind(), &cast_ty.ty.kind()) {
@@ -340,9 +340,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     let src_field = self.operand_field(src, i)?;
                     let dst_field = self.place_field(dest, i)?;
                     if src_field.layout.ty == cast_ty_field.ty {
-                        self.copy_op(src_field, dst_field)?;
+                        self.copy_op(&src_field, &dst_field)?;
                     } else {
-                        self.unsize_into(src_field, cast_ty_field, dst_field)?;
+                        self.unsize_into(&src_field, cast_ty_field, &dst_field)?;
                     }
                 }
                 Ok(())

compiler/rustc_mir/src/interpret/eval_context.rs

@@ -548,8 +548,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     /// This can fail to provide an answer for extern types.
     pub(super) fn size_and_align_of(
         &self,
-        metadata: MemPlaceMeta<M::PointerTag>,
-        layout: TyAndLayout<'tcx>,
+        metadata: &MemPlaceMeta<M::PointerTag>,
+        layout: &TyAndLayout<'tcx>,
     ) -> InterpResult<'tcx, Option<(Size, Align)>> {
         if !layout.is_unsized() {
             return Ok(Some((layout.size, layout.align.abi)));
@@ -577,24 +577,25 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 // the last field).  Can't have foreign types here, how would we
                 // adjust alignment and size for them?
                 let field = layout.field(self, layout.fields.count() - 1)?;
-                let (unsized_size, unsized_align) = match self.size_and_align_of(metadata, field)? {
-                    Some(size_and_align) => size_and_align,
-                    None => {
-                        // A field with extern type.  If this field is at offset 0, we behave
-                        // like the underlying extern type.
-                        // FIXME: Once we have made decisions for how to handle size and alignment
-                        // of `extern type`, this should be adapted.  It is just a temporary hack
-                        // to get some code to work that probably ought to work.
-                        if sized_size == Size::ZERO {
-                            return Ok(None);
-                        } else {
-                            span_bug!(
-                                self.cur_span(),
-                                "Fields cannot be extern types, unless they are at offset 0"
-                            )
+                let (unsized_size, unsized_align) =
+                    match self.size_and_align_of(metadata, &field)? {
+                        Some(size_and_align) => size_and_align,
+                        None => {
+                            // A field with extern type.  If this field is at offset 0, we behave
+                            // like the underlying extern type.
+                            // FIXME: Once we have made decisions for how to handle size and alignment
+                            // of `extern type`, this should be adapted.  It is just a temporary hack
+                            // to get some code to work that probably ought to work.
+                            if sized_size == Size::ZERO {
+                                return Ok(None);
+                            } else {
+                                span_bug!(
+                                    self.cur_span(),
+                                    "Fields cannot be extern types, unless they are at offset 0"
+                                )
+                            }
                         }
-                    }
-                };
+                    };
 
                 // FIXME (#26403, #27023): We should be adding padding
                 // to `sized_size` (to accommodate the `unsized_align`
@@ -645,24 +646,24 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     #[inline]
     pub fn size_and_align_of_mplace(
         &self,
-        mplace: MPlaceTy<'tcx, M::PointerTag>,
+        mplace: &MPlaceTy<'tcx, M::PointerTag>,
     ) -> InterpResult<'tcx, Option<(Size, Align)>> {
-        self.size_and_align_of(mplace.meta, mplace.layout)
+        self.size_and_align_of(&mplace.meta, &mplace.layout)
     }
 
     pub fn push_stack_frame(
         &mut self,
         instance: ty::Instance<'tcx>,
         body: &'mir mir::Body<'tcx>,
-        return_place: Option<PlaceTy<'tcx, M::PointerTag>>,
+        return_place: Option<&PlaceTy<'tcx, M::PointerTag>>,
         return_to_block: StackPopCleanup,
     ) -> InterpResult<'tcx> {
         // first push a stack frame so we have access to the local substs
         let pre_frame = Frame {
             body,
             loc: Err(body.span), // Span used for errors caused during preamble.
             return_to_block,
-            return_place,
+            return_place: return_place.copied(),
             // empty local array, we fill it in below, after we are inside the stack frame and
             // all methods actually know about the frame
             locals: IndexVec::new(),
@@ -777,10 +778,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
         if !unwinding {
             // Copy the return value to the caller's stack frame.
-            if let Some(return_place) = frame.return_place {
+            if let Some(ref return_place) = frame.return_place {
                 let op = self.access_local(&frame, mir::RETURN_PLACE, None)?;
-                self.copy_op_transmute(op, return_place)?;
-                trace!("{:?}", self.dump_place(*return_place));
+                self.copy_op_transmute(&op, return_place)?;
+                trace!("{:?}", self.dump_place(**return_place));
             } else {
                 throw_ub!(Unreachable);
             }