Directly intern values instead of copying them.

Author: cjgillot

compiler/rustc_const_eval/src/interpret/intern.rs

@@ -450,6 +450,50 @@ pub fn intern_const_alloc_recursive<
     Ok(())
 }
 
+/// Intern `ret`, checking it references no other allocation.
+#[instrument(level = "debug", skip(ecx))]
+pub fn intern_const_alloc_for_constprop<
+    'mir,
+    'tcx: 'mir,
+    T,
+    M: CompileTimeMachine<'mir, 'tcx, T>,
+>(
+    ecx: &mut InterpCx<'mir, 'tcx, M>,
+    ret: &MPlaceTy<'tcx>,
+) -> InterpResult<'tcx, ()> {
+    let Some((size, _align)) = ecx.size_and_align_of_mplace(ret)? else {
+        throw_inval!(ConstPropNonsense)
+    };
+
+    let alloc_ref = ecx.get_ptr_alloc(ret.ptr(), size)?.unwrap();
+    // Do not try interning a value that contains provenance.
+    if alloc_ref.has_provenance() {
+        throw_inval!(ConstPropNonsense)
+    }
+
+    // remove allocation
+    let alloc_id = ret.ptr().provenance.unwrap();
+    let Some((_, mut alloc)) = ecx.memory.alloc_map.remove(&alloc_id) else {
+        // Pointer not found in local memory map. It is either a pointer to the global
+        // map, or dangling.
+        if ecx.tcx.try_get_global_alloc(alloc_id).is_none() {
+            throw_ub!(DeadLocal)
+        }
+        // The constant is already in global memory. Do nothing.
+        return Ok(());
+    };
+
+    alloc.mutability = Mutability::Not;
+
+    // link the alloc id to the actual allocation
+    assert!(alloc.provenance().ptrs().is_empty());
+
+    let alloc = ecx.tcx.mk_const_alloc(alloc);
+    ecx.tcx.set_alloc_id_memory(alloc_id, alloc);
+
+    Ok(())
+}
+
 impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx, !>>
     InterpCx<'mir, 'tcx, M>
 {

compiler/rustc_const_eval/src/interpret/mod.rs

@@ -21,7 +21,9 @@ mod visitor;
 pub use rustc_middle::mir::interpret::*; // have all the `interpret` symbols in one place: here
 
 pub use self::eval_context::{Frame, FrameInfo, InterpCx, StackPopCleanup};
-pub use self::intern::{intern_const_alloc_recursive, InternKind};
+pub use self::intern::{
+    intern_const_alloc_for_constprop, intern_const_alloc_recursive, InternKind,
+};
 pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, StackPopJump};
 pub use self::memory::{AllocKind, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind};
 pub use self::operand::{ImmTy, Immediate, OpTy, Readable};

compiler/rustc_mir_transform/src/gvn.rs

@@ -53,7 +53,7 @@
 //! _c = *_b // replaced by _c = _a
 //! ```
 
-use rustc_const_eval::interpret::MemoryKind;
+use rustc_const_eval::interpret::{intern_const_alloc_for_constprop, MemoryKind};
 use rustc_const_eval::interpret::{ImmTy, InterpCx, MemPlaceMeta, OpTy, Projectable, Scalar};
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
@@ -840,10 +840,13 @@ fn op_to_prop_const<'tcx>(
     if let Either::Left(mplace) = op.as_mplace_or_imm()
         && let MemPlaceMeta::None = mplace.meta()
     {
+        intern_const_alloc_for_constprop(ecx, &mplace).ok()?;
         let pointer = mplace.ptr().into_pointer_or_addr().ok()?;
         let (alloc_id, offset) = pointer.into_parts();
-        if matches!(ecx.tcx.try_get_global_alloc(alloc_id), Some(GlobalAlloc::Memory(_))) {
-            return Some(ConstValue::Indirect { alloc_id, offset })
+        match ecx.tcx.global_alloc(alloc_id) {
+            GlobalAlloc::Memory(_) => return Some(ConstValue::Indirect { alloc_id, offset }),
+            // Fallthrough to copying the data.
+            _ => {}
         }
     }