MIR move elimination [6/6]: Miri support

compiler/rustc_borrowck/src/dataflow.rs

@@ -530,6 +530,7 @@ impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> {
 
             mir::StatementKind::FakeRead(..)
             | mir::StatementKind::SetDiscriminant { .. }
+            | mir::StatementKind::StorageAlloc(_)
             | mir::StatementKind::StorageLive(..)
             | mir::StatementKind::PlaceMention(..)
             | mir::StatementKind::AscribeUserType(..)

compiler/rustc_borrowck/src/def_use.rs

@@ -76,7 +76,8 @@ pub(crate) fn categorize(context: PlaceContext) -> Option<DefUse> {
         // Backwards incompatible drop hint is not a use, just a marker for linting.
         PlaceContext::NonUse(NonUseContext::BackwardIncompatibleDropHint) => None,
 
-        PlaceContext::MutatingUse(MutatingUseContext::SetDiscriminant) => {
+        PlaceContext::NonUse(NonUseContext::StorageAlloc)
+        | PlaceContext::MutatingUse(MutatingUseContext::SetDiscriminant) => {
             bug!("These statements are not allowed in this MIR phase")
         }
     }

compiler/rustc_borrowck/src/lib.rs

@@ -874,7 +874,9 @@ impl<'a, 'tcx> ResultsVisitor<'tcx, Borrowck<'a, 'tcx>> for MirBorrowckCtxt<'a,
                     state,
                 );
             }
-            StatementKind::Nop | StatementKind::SetDiscriminant { .. } => {
+            StatementKind::Nop
+            | StatementKind::StorageAlloc(..)
+            | StatementKind::SetDiscriminant { .. } => {
                 bug!("Statement not allowed in this MIR phase")
             }
         }

compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs

@@ -83,6 +83,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'a, 'tcx> {
             }
             StatementKind::ConstEvalCounter
             | StatementKind::Nop
+            | StatementKind::StorageAlloc(..)
             | StatementKind::SetDiscriminant { .. } => {
                 bug!("Statement not allowed in this MIR phase")
             }

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -730,6 +730,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             | StatementKind::BackwardIncompatibleDropHint { .. }
             | StatementKind::Nop => {}
             StatementKind::Intrinsic(NonDivergingIntrinsic::CopyNonOverlapping(..))
+            | StatementKind::StorageAlloc(..)
             | StatementKind::SetDiscriminant { .. } => {
                 bug!("Statement not allowed in this MIR phase")
             }

compiler/rustc_codegen_cranelift/src/base.rs

@@ -614,7 +614,9 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
     fx.set_debug_loc(stmt.source_info);
 
     match &stmt.kind {
-        StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful
+        StatementKind::StorageAlloc(..)
+        | StatementKind::StorageLive(..)
+        | StatementKind::StorageDead(..) => {} // Those are not very useful
         _ => {
             if fx.clif_comments.enabled() {
                 let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
@@ -928,7 +930,8 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                 Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in codegen"),
             }
         }
-        StatementKind::StorageLive(_)
+        StatementKind::StorageAlloc(_)
+        | StatementKind::StorageLive(_)
         | StatementKind::StorageDead(_)
         | StatementKind::ConstEvalCounter
         | StatementKind::Nop

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -571,6 +571,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                         StatementKind::Assign(_)
                         | StatementKind::FakeRead(_)
                         | StatementKind::SetDiscriminant { .. }
+                        | StatementKind::StorageAlloc(_)
                         | StatementKind::StorageLive(_)
                         | StatementKind::StorageDead(_)
                         | StatementKind::AscribeUserType(_, _)

compiler/rustc_codegen_ssa/src/mir/statement.rs

@@ -70,6 +70,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::StatementKind::SetDiscriminant { ref place, variant_index } => {
                 self.codegen_place(bx, (**place).as_ref()).codegen_set_discr(bx, variant_index);
             }
+            mir::StatementKind::StorageAlloc(_) => {
+                // This is a no-op in codegen.
+            }
             mir::StatementKind::StorageLive(local) => {
                 if let LocalRef::Place(cg_place) = self.locals[local] {
                     cg_place.storage_live(bx);

compiler/rustc_const_eval/src/check_consts/check.rs

@@ -707,6 +707,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             StatementKind::Assign(..)
             | StatementKind::SetDiscriminant { .. }
             | StatementKind::FakeRead(..)
+            | StatementKind::StorageAlloc(_)
             | StatementKind::StorageLive(_)
             | StatementKind::StorageDead(_)
             | StatementKind::PlaceMention(..)

compiler/rustc_const_eval/src/const_eval/machine.rs

@@ -941,8 +941,10 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
     ) -> InterpResult<'tcx> {
         use rustc_middle::mir::AssertKind::*;
         // Convert `AssertKind<Operand>` to `AssertKind<Scalar>`.
-        let eval_to_int =
-            |op| ecx.read_immediate(&ecx.eval_operand(op, None)?).map(|x| x.to_const_int());
+        let mut eval_to_int = |op| {
+            let op = ecx.eval_operand(op, None)?;
+            ecx.read_immediate(&op).map(|x| x.to_const_int())
+        };
         let err = match msg {
             BoundsCheck { len, index } => {
                 let len = eval_to_int(len)?;

compiler/rustc_const_eval/src/interpret/call.rs

@@ -1,8 +1,8 @@
 //! Manages calling a concrete function (with known MIR body) with argument passing,
 //! and returning the return value to the caller.
 
-use std::assert_matches;
 use std::borrow::Cow;
+use std::{assert_matches, debug_assert_matches};
 
 use either::{Left, Right};
 use rustc_abi::{self as abi, ExternAbi, FieldIdx, Integer, VariantIdx};
@@ -432,8 +432,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             self.storage_live_dyn(local, meta)?;
         }
         // Now we can finally actually evaluate the callee place.
-        let callee_arg =
-            self.eval_place(*callee_arg, /* skip_validity_for_simple_deref */ false)?;
+        let callee_arg = self
+            .eval_place_for_write(*callee_arg, /* skip_validity_for_simple_deref */ false)?;
         // We allow some transmutes here.
         // FIXME: Depending on the PassMode, this should reset some padding to uninitialized. (This
         // is true for all `copy_op`, but there are a lot of special cases for argument passing
@@ -516,6 +516,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         // *Before* pushing the new frame, determine whether the return destination is in memory.
         // Need to use `place_to_op` to be *sure* we get the mplace if there is one.
         let destination_mplace = self.place_to_op(destination)?.as_mplace_or_imm().left();
+        if destination_mplace.is_none() {
+            // Protection below clears in-memory destinations. Clear immediate destinations too,
+            // while we can still access the caller's locals. Evaluated arguments already hold
+            // copies of any immediate values, so this cannot affect argument passing.
+            self.write_uninit(destination)?;
+        }
 
         // Push the "raw" frame -- this leaves locals uninitialized.
         self.push_stack_frame_raw(instance, body, destination, cont)?;
@@ -599,6 +605,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     let (callee_arg_idx, callee_abi) = callee_args_abis.next().unwrap();
                     assert!(callee_abi.layout.is_1zst() && callee_abi.is_ignore());
                     ecx.storage_live(local)?;
+                    ecx.allocate_local_for_write(local)?;
                     // And skip it in the caller, if present. We can tell whether it is present by
                     // comparing the number of arguments on the caller and callee side.
                     if caller_fn_abi.args.len() == callee_fn_abi.args.len() {
@@ -618,8 +625,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     // This argument is a VaList holding the remaining caller-side arguments.
                     ecx.storage_live(local)?;
 
-                    let place =
-                        ecx.eval_place(dest, /* skip_validity_for_simple_deref */ false)?;
+                    let place = ecx.eval_place_for_write(
+                        dest, /* skip_validity_for_simple_deref */ false,
+                    )?;
                     let mplace = ecx.force_allocation(&place)?;
 
                     // Consume the remaining arguments by putting them into the variable argument
@@ -645,6 +653,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 } else if Some(local) == body.spread_arg {
                     // Make the local live once, then fill in the value field by field.
                     ecx.storage_live(local)?;
+                    // Function arguments start allocated, including an empty spread tuple for
+                    // which the loop below has no fields to initialize.
+                    ecx.allocate_local_for_write(local)?;
                     // Must be a tuple
                     let ty::Tuple(fields) = ty.kind() else {
                         span_bug!(ecx.cur_span(), "non-tuple type for `spread_arg`: {ty}")
@@ -997,10 +1008,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         // as that "executes" the goto to the return block, but we don't want to,
         // only the tail called function should return to the current return block.
 
-        // The arguments need to all be copied since the current stack frame will be removed
-        // before the callee even starts executing.
-        // FIXME(explicit_tail_calls,#144855): does this match what codegen does?
-        let args = args.iter().map(|fn_arg| FnArg::Copy(fn_arg.copy_fn_arg())).collect::<Vec<_>>();
+        // Tail-call arguments are evaluated as ordinary operands, so none of them may donate a
+        // place in the frame that is about to be destroyed.
+        for arg in args {
+            debug_assert_matches!(arg, FnArg::Copy(_));
+        }
         // Remove the frame from the stack.
         let frame = self.pop_stack_frame_raw()?;
         // Remember where this frame would have returned to.
@@ -1017,7 +1029,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         self.init_fn_call(
             fn_val,
             (caller_abi, caller_fn_abi),
-            &*args,
+            args,
             with_caller_location,
             frame.return_place(),
             ret,
@@ -1127,11 +1139,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         // Get out the return value. Must happen *before* the frame is popped as we have to get the
         // local's value out.
         let return_op =
-            self.local_to_op(mir::RETURN_PLACE, None).expect("return place should always be live");
+            if unwinding { None } else { Some(self.local_to_op(mir::RETURN_PLACE, None)?) };
         // Remove the frame from the stack.
         let frame = self.pop_stack_frame_raw()?;
         // Copy the return value and remember the return continuation.
-        if !unwinding {
+        if let Some(return_op) = return_op {
             self.copy_op_allow_transmute(&return_op, frame.return_place())?;
             trace!("return value: {:?}", self.dump_place(frame.return_place()));
         }

compiler/rustc_const_eval/src/interpret/eval_context.rs

@@ -48,6 +48,12 @@ pub struct InterpCx<'tcx, M: Machine<'tcx>> {
     /// The virtual memory system.
     pub memory: Memory<'tcx, M>,
 
+    /// Temporary operand snapshots that survive deallocation of their source
+    /// local.
+    ///
+    /// These are freed after each MIR statement or terminator.
+    pub(super) operand_temps: Vec<MPlaceTy<'tcx, M::Provenance>>,
+
     /// The recursion limit (cached from `tcx.recursion_limit(())`)
     pub recursion_limit: Limit,
 }
@@ -254,6 +260,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             typing_env,
             layout_cache: RefCell::new(FxHashMap::default()),
             memory: Memory::new(),
+            operand_temps: Vec::new(),
             recursion_limit: tcx.recursion_limit(),
         }
     }

compiler/rustc_const_eval/src/interpret/machine.rs

@@ -165,6 +165,12 @@ pub trait Machine<'tcx>: Sized {
     /// Whether memory accesses should be alignment-checked.
     fn enforce_alignment(ecx: &InterpCx<'tcx, Self>) -> bool;
 
+    /// Whether to enforce the local allocation semantics required by MIR move elimination.
+    #[inline(always)]
+    fn move_elimination_semantics(ecx: &InterpCx<'tcx, Self>) -> bool {
+        ecx.tcx.sess.opts.unstable_opts.mir_move_elimination
+    }
+
     /// Gives the machine a chance to detect more misalignment than the built-in checks would catch.
     #[inline(always)]
     fn alignment_check(

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -1,7 +1,7 @@
 //! Functions concerning immediate values and operands, and reading from operands.
 //! All high-level functions to read from memory work on operands as sources.
 
-use std::assert_matches;
+use std::{assert_matches, mem};
 
 use either::{Either, Left, Right};
 use rustc_abi as abi;
@@ -17,7 +17,7 @@ use tracing::trace;
 
 use super::{
     CtfeProvenance, Frame, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta,
-    OffsetMode, PlaceTy, Pointer, Projectable, Provenance, Scalar, alloc_range, err_ub,
+    MemoryKind, OffsetMode, PlaceTy, Pointer, Projectable, Provenance, Scalar, alloc_range, err_ub,
     from_known_layout, interp_ok, mir_assign_valid_types, throw_ub,
 };
 use crate::enter_trace_span;
@@ -830,21 +830,85 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         interp_ok(op)
     }
 
-    /// Evaluate the operand, returning a place where you can then find the data.
-    /// If you already know the layout, you can save two table lookups
-    /// by passing it in here.
+    /// Capture a fixed copy of an operand that is independent of any further
+    /// changes to its backing allocation.
+    pub(super) fn snapshot_operand(
+        &mut self,
+        op: OpTy<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
+        // Load into an immediate if possible.
+        if let Right(imm) = self.read_immediate_raw(&op)? {
+            return interp_ok(imm.into());
+        }
+
+        // Otherwise use a temporary allocation that is freed at the end of the
+        // machine step.
+        let temp = self.allocate(op.layout, MemoryKind::Stack)?;
+        self.copy_op_no_validate(&op, &temp, /*allow_transmute*/ false)?;
+        self.operand_temps.push(temp.clone());
+        interp_ok(temp.into())
+    }
+
+    /// Deallocate operand snapshots created in the current MIR step.
+    pub(super) fn clear_operand_temps(&mut self) -> InterpResult<'tcx> {
+        for temp in mem::take(&mut self.operand_temps) {
+            self.deallocate_ptr(temp.ptr(), None, MemoryKind::Stack)?;
+        }
+        interp_ok(())
+    }
+
+    /// Evaluate the operand, returning a place where you can then find the
+    /// data.
+    ///
+    /// If you already know the layout, you can save two table lookups by
+    /// passing it in here.
+    ///
+    /// Under move-elimination semantics, the result is a snapshot that remains
+    /// valid if later operand evaluation frees its source storage.
     #[inline]
     pub fn eval_operand(
-        &self,
+        &mut self,
         mir_op: &mir::Operand<'tcx>,
         layout: Option<TyAndLayout<'tcx>>,
     ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
-        let _trace =
-            enter_trace_span!(M, step::eval_operand, ?mir_op, tracing_separate_thread = Empty);
+        let op = self.eval_operand_no_snapshot(mir_op, layout)?;
+        // Whole-local moves already detached their values; constants cannot change.
+        let needs_snapshot = match mir_op {
+            mir::Operand::Copy(_) => true,
+            mir::Operand::Move(place) => !place.projection.is_empty(),
+            mir::Operand::Constant(_) | mir::Operand::RuntimeChecks(_) => false,
+        };
+        if M::move_elimination_semantics(self) && needs_snapshot {
+            self.snapshot_operand(op)
+        } else {
+            interp_ok(op)
+        }
+    }
+
+    /// Evaluate an operand without snapshotting.
+    ///
+    /// Whole-local moves still copy out the value and deallocate the local.
+    ///
+    /// The caller must not evaluate any other operand before consuming the
+    /// result, since those may cause this operand's backing local to be freed.
+    #[inline]
+    pub fn eval_operand_no_snapshot(
+        &mut self,
+        mir_op: &mir::Operand<'tcx>,
+        layout: Option<TyAndLayout<'tcx>>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
+        let _trace = enter_trace_span!(
+            M,
+            step::eval_operand_no_snapshot,
+            ?mir_op,
+            tracing_separate_thread = Empty
+        );
 
         use rustc_middle::mir::Operand::*;
         let op = match mir_op {
-            // FIXME: do some more logic on `move` to invalidate the old location
+            &Move(place) if M::move_elimination_semantics(self) && place.projection.is_empty() => {
+                self.move_out_local(place.local, layout)?
+            }
             &Copy(place) | &Move(place) => self.eval_place_to_op(place, layout)?,
 
             &RuntimeChecks(checks) => {

compiler/rustc_const_eval/src/interpret/place.rs

@@ -644,6 +644,19 @@ where
         interp_ok(place)
     }
 
+    /// Computes a destination place, allocating its base local if it is currently live but without
+    /// an allocation.
+    pub fn eval_place_for_write(
+        &mut self,
+        mir_place: mir::Place<'tcx>,
+        skip_validity_for_simple_deref: bool,
+    ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
+        if M::move_elimination_semantics(self) && !mir_place.is_indirect_first_projection() {
+            self.allocate_local_for_write(mir_place.local)?;
+        }
+        self.eval_place(mir_place, skip_validity_for_simple_deref)
+    }
+
     /// Given a place, returns either the underlying mplace or a reference to where the value of
     /// this place is stored.
     #[inline(always)]