diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs index ea557d5321598..0a39ac0d6918b 100644 --- a/compiler/rustc_borrowck/src/dataflow.rs +++ b/compiler/rustc_borrowck/src/dataflow.rs @@ -530,6 +530,7 @@ impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> { mir::StatementKind::FakeRead(..) | mir::StatementKind::SetDiscriminant { .. } + | mir::StatementKind::StorageAlloc(_) | mir::StatementKind::StorageLive(..) | mir::StatementKind::PlaceMention(..) | mir::StatementKind::AscribeUserType(..) diff --git a/compiler/rustc_borrowck/src/def_use.rs b/compiler/rustc_borrowck/src/def_use.rs index 942d2e59fb02b..cd9dd4fcfc9b7 100644 --- a/compiler/rustc_borrowck/src/def_use.rs +++ b/compiler/rustc_borrowck/src/def_use.rs @@ -76,7 +76,8 @@ pub(crate) fn categorize(context: PlaceContext) -> Option { // Backwards incompatible drop hint is not a use, just a marker for linting. PlaceContext::NonUse(NonUseContext::BackwardIncompatibleDropHint) => None, - PlaceContext::MutatingUse(MutatingUseContext::SetDiscriminant) => { + PlaceContext::NonUse(NonUseContext::StorageAlloc) + | PlaceContext::MutatingUse(MutatingUseContext::SetDiscriminant) => { bug!("These statements are not allowed in this MIR phase") } } diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index c35c7ab311572..6306cb99121f5 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -874,7 +874,9 @@ impl<'a, 'tcx> ResultsVisitor<'tcx, Borrowck<'a, 'tcx>> for MirBorrowckCtxt<'a, state, ); } - StatementKind::Nop | StatementKind::SetDiscriminant { .. } => { + StatementKind::Nop + | StatementKind::StorageAlloc(..) + | StatementKind::SetDiscriminant { .. } => { bug!("Statement not allowed in this MIR phase") } } diff --git a/compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs b/compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs index e4d1419b11d83..6da8b168e0b86 100644 --- a/compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs +++ b/compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs @@ -83,6 +83,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'a, 'tcx> { } StatementKind::ConstEvalCounter | StatementKind::Nop + | StatementKind::StorageAlloc(..) | StatementKind::SetDiscriminant { .. } => { bug!("Statement not allowed in this MIR phase") } diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 0a6fb9d5da47a..5986351badf31 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -730,6 +730,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { | StatementKind::BackwardIncompatibleDropHint { .. } | StatementKind::Nop => {} StatementKind::Intrinsic(NonDivergingIntrinsic::CopyNonOverlapping(..)) + | StatementKind::StorageAlloc(..) | StatementKind::SetDiscriminant { .. } => { bug!("Statement not allowed in this MIR phase") } diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index e752189130290..3a5f5a48f8595 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -614,7 +614,9 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: fx.set_debug_loc(stmt.source_info); match &stmt.kind { - StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful + StatementKind::StorageAlloc(..) + | StatementKind::StorageLive(..) + | StatementKind::StorageDead(..) => {} // Those are not very useful _ => { if fx.clif_comments.enabled() { let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap(); @@ -928,7 +930,8 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in codegen"), } } - StatementKind::StorageLive(_) + StatementKind::StorageAlloc(_) + | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::ConstEvalCounter | StatementKind::Nop diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index f1e4fd818ebeb..14aa141025700 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -571,6 +571,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( StatementKind::Assign(_) | StatementKind::FakeRead(_) | StatementKind::SetDiscriminant { .. } + | StatementKind::StorageAlloc(_) | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::AscribeUserType(_, _) diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs index 14eec4a0195e0..3de9ff9220070 100644 --- a/compiler/rustc_codegen_ssa/src/mir/statement.rs +++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs @@ -70,6 +70,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::StatementKind::SetDiscriminant { ref place, variant_index } => { self.codegen_place(bx, (**place).as_ref()).codegen_set_discr(bx, variant_index); } + mir::StatementKind::StorageAlloc(_) => { + // This is a no-op in codegen. + } mir::StatementKind::StorageLive(local) => { if let LocalRef::Place(cg_place) = self.locals[local] { cg_place.storage_live(bx); diff --git a/compiler/rustc_const_eval/src/check_consts/check.rs b/compiler/rustc_const_eval/src/check_consts/check.rs index 192bb8556a75d..52bc6d6fa873d 100644 --- a/compiler/rustc_const_eval/src/check_consts/check.rs +++ b/compiler/rustc_const_eval/src/check_consts/check.rs @@ -707,6 +707,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { StatementKind::Assign(..) | StatementKind::SetDiscriminant { .. } | StatementKind::FakeRead(..) + | StatementKind::StorageAlloc(_) | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::PlaceMention(..) diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 843cd7cb4930d..ec688dfb54aa3 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -941,8 +941,10 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> { ) -> InterpResult<'tcx> { use rustc_middle::mir::AssertKind::*; // Convert `AssertKind` to `AssertKind`. - let eval_to_int = - |op| ecx.read_immediate(&ecx.eval_operand(op, None)?).map(|x| x.to_const_int()); + let mut eval_to_int = |op| { + let op = ecx.eval_operand(op, None)?; + ecx.read_immediate(&op).map(|x| x.to_const_int()) + }; let err = match msg { BoundsCheck { len, index } => { let len = eval_to_int(len)?; diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs index c7615a216312e..f7f6cbaa97313 100644 --- a/compiler/rustc_const_eval/src/interpret/call.rs +++ b/compiler/rustc_const_eval/src/interpret/call.rs @@ -1,8 +1,8 @@ //! Manages calling a concrete function (with known MIR body) with argument passing, //! and returning the return value to the caller. -use std::assert_matches; use std::borrow::Cow; +use std::{assert_matches, debug_assert_matches}; use either::{Left, Right}; use rustc_abi::{self as abi, ExternAbi, FieldIdx, Integer, VariantIdx}; @@ -432,8 +432,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.storage_live_dyn(local, meta)?; } // Now we can finally actually evaluate the callee place. - let callee_arg = - self.eval_place(*callee_arg, /* skip_validity_for_simple_deref */ false)?; + let callee_arg = self + .eval_place_for_write(*callee_arg, /* skip_validity_for_simple_deref */ false)?; // We allow some transmutes here. // FIXME: Depending on the PassMode, this should reset some padding to uninitialized. (This // is true for all `copy_op`, but there are a lot of special cases for argument passing @@ -516,6 +516,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // *Before* pushing the new frame, determine whether the return destination is in memory. // Need to use `place_to_op` to be *sure* we get the mplace if there is one. let destination_mplace = self.place_to_op(destination)?.as_mplace_or_imm().left(); + if destination_mplace.is_none() { + // Protection below clears in-memory destinations. Clear immediate destinations too, + // while we can still access the caller's locals. Evaluated arguments already hold + // copies of any immediate values, so this cannot affect argument passing. + self.write_uninit(destination)?; + } // Push the "raw" frame -- this leaves locals uninitialized. self.push_stack_frame_raw(instance, body, destination, cont)?; @@ -599,6 +605,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let (callee_arg_idx, callee_abi) = callee_args_abis.next().unwrap(); assert!(callee_abi.layout.is_1zst() && callee_abi.is_ignore()); ecx.storage_live(local)?; + ecx.allocate_local_for_write(local)?; // And skip it in the caller, if present. We can tell whether it is present by // comparing the number of arguments on the caller and callee side. if caller_fn_abi.args.len() == callee_fn_abi.args.len() { @@ -618,8 +625,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // This argument is a VaList holding the remaining caller-side arguments. ecx.storage_live(local)?; - let place = - ecx.eval_place(dest, /* skip_validity_for_simple_deref */ false)?; + let place = ecx.eval_place_for_write( + dest, /* skip_validity_for_simple_deref */ false, + )?; let mplace = ecx.force_allocation(&place)?; // Consume the remaining arguments by putting them into the variable argument @@ -645,6 +653,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } else if Some(local) == body.spread_arg { // Make the local live once, then fill in the value field by field. ecx.storage_live(local)?; + // Function arguments start allocated, including an empty spread tuple for + // which the loop below has no fields to initialize. + ecx.allocate_local_for_write(local)?; // Must be a tuple let ty::Tuple(fields) = ty.kind() else { span_bug!(ecx.cur_span(), "non-tuple type for `spread_arg`: {ty}") @@ -997,10 +1008,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // as that "executes" the goto to the return block, but we don't want to, // only the tail called function should return to the current return block. - // The arguments need to all be copied since the current stack frame will be removed - // before the callee even starts executing. - // FIXME(explicit_tail_calls,#144855): does this match what codegen does? - let args = args.iter().map(|fn_arg| FnArg::Copy(fn_arg.copy_fn_arg())).collect::>(); + // Tail-call arguments are evaluated as ordinary operands, so none of them may donate a + // place in the frame that is about to be destroyed. + for arg in args { + debug_assert_matches!(arg, FnArg::Copy(_)); + } // Remove the frame from the stack. let frame = self.pop_stack_frame_raw()?; // Remember where this frame would have returned to. @@ -1017,7 +1029,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.init_fn_call( fn_val, (caller_abi, caller_fn_abi), - &*args, + args, with_caller_location, frame.return_place(), ret, @@ -1127,11 +1139,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // Get out the return value. Must happen *before* the frame is popped as we have to get the // local's value out. let return_op = - self.local_to_op(mir::RETURN_PLACE, None).expect("return place should always be live"); + if unwinding { None } else { Some(self.local_to_op(mir::RETURN_PLACE, None)?) }; // Remove the frame from the stack. let frame = self.pop_stack_frame_raw()?; // Copy the return value and remember the return continuation. - if !unwinding { + if let Some(return_op) = return_op { self.copy_op_allow_transmute(&return_op, frame.return_place())?; trace!("return value: {:?}", self.dump_place(frame.return_place())); } diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index a74b323c395e6..1869e8dd95a79 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -48,6 +48,12 @@ pub struct InterpCx<'tcx, M: Machine<'tcx>> { /// The virtual memory system. pub memory: Memory<'tcx, M>, + /// Temporary operand snapshots that survive deallocation of their source + /// local. + /// + /// These are freed after each MIR statement or terminator. + pub(super) operand_temps: Vec>, + /// The recursion limit (cached from `tcx.recursion_limit(())`) pub recursion_limit: Limit, } @@ -254,6 +260,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { typing_env, layout_cache: RefCell::new(FxHashMap::default()), memory: Memory::new(), + operand_temps: Vec::new(), recursion_limit: tcx.recursion_limit(), } } diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index 8d9b26cd18423..72c5e3efe79dd 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -165,6 +165,12 @@ pub trait Machine<'tcx>: Sized { /// Whether memory accesses should be alignment-checked. fn enforce_alignment(ecx: &InterpCx<'tcx, Self>) -> bool; + /// Whether to enforce the local allocation semantics required by MIR move elimination. + #[inline(always)] + fn move_elimination_semantics(ecx: &InterpCx<'tcx, Self>) -> bool { + ecx.tcx.sess.opts.unstable_opts.mir_move_elimination + } + /// Gives the machine a chance to detect more misalignment than the built-in checks would catch. #[inline(always)] fn alignment_check( diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index 84cf61c06e790..c73267d8b18b1 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -1,7 +1,7 @@ //! Functions concerning immediate values and operands, and reading from operands. //! All high-level functions to read from memory work on operands as sources. -use std::assert_matches; +use std::{assert_matches, mem}; use either::{Either, Left, Right}; use rustc_abi as abi; @@ -17,7 +17,7 @@ use tracing::trace; use super::{ CtfeProvenance, Frame, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, - OffsetMode, PlaceTy, Pointer, Projectable, Provenance, Scalar, alloc_range, err_ub, + MemoryKind, OffsetMode, PlaceTy, Pointer, Projectable, Provenance, Scalar, alloc_range, err_ub, from_known_layout, interp_ok, mir_assign_valid_types, throw_ub, }; use crate::enter_trace_span; @@ -830,21 +830,85 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { interp_ok(op) } - /// Evaluate the operand, returning a place where you can then find the data. - /// If you already know the layout, you can save two table lookups - /// by passing it in here. + /// Capture a fixed copy of an operand that is independent of any further + /// changes to its backing allocation. + pub(super) fn snapshot_operand( + &mut self, + op: OpTy<'tcx, M::Provenance>, + ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + // Load into an immediate if possible. + if let Right(imm) = self.read_immediate_raw(&op)? { + return interp_ok(imm.into()); + } + + // Otherwise use a temporary allocation that is freed at the end of the + // machine step. + let temp = self.allocate(op.layout, MemoryKind::Stack)?; + self.copy_op_no_validate(&op, &temp, /*allow_transmute*/ false)?; + self.operand_temps.push(temp.clone()); + interp_ok(temp.into()) + } + + /// Deallocate operand snapshots created in the current MIR step. + pub(super) fn clear_operand_temps(&mut self) -> InterpResult<'tcx> { + for temp in mem::take(&mut self.operand_temps) { + self.deallocate_ptr(temp.ptr(), None, MemoryKind::Stack)?; + } + interp_ok(()) + } + + /// Evaluate the operand, returning a place where you can then find the + /// data. + /// + /// If you already know the layout, you can save two table lookups by + /// passing it in here. + /// + /// Under move-elimination semantics, the result is a snapshot that remains + /// valid if later operand evaluation frees its source storage. #[inline] pub fn eval_operand( - &self, + &mut self, mir_op: &mir::Operand<'tcx>, layout: Option>, ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { - let _trace = - enter_trace_span!(M, step::eval_operand, ?mir_op, tracing_separate_thread = Empty); + let op = self.eval_operand_no_snapshot(mir_op, layout)?; + // Whole-local moves already detached their values; constants cannot change. + let needs_snapshot = match mir_op { + mir::Operand::Copy(_) => true, + mir::Operand::Move(place) => !place.projection.is_empty(), + mir::Operand::Constant(_) | mir::Operand::RuntimeChecks(_) => false, + }; + if M::move_elimination_semantics(self) && needs_snapshot { + self.snapshot_operand(op) + } else { + interp_ok(op) + } + } + + /// Evaluate an operand without snapshotting. + /// + /// Whole-local moves still copy out the value and deallocate the local. + /// + /// The caller must not evaluate any other operand before consuming the + /// result, since those may cause this operand's backing local to be freed. + #[inline] + pub fn eval_operand_no_snapshot( + &mut self, + mir_op: &mir::Operand<'tcx>, + layout: Option>, + ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + let _trace = enter_trace_span!( + M, + step::eval_operand_no_snapshot, + ?mir_op, + tracing_separate_thread = Empty + ); use rustc_middle::mir::Operand::*; let op = match mir_op { - // FIXME: do some more logic on `move` to invalidate the old location + &Move(place) if M::move_elimination_semantics(self) && place.projection.is_empty() => { + self.move_out_local(place.local, layout)? + } &Copy(place) | &Move(place) => self.eval_place_to_op(place, layout)?, &RuntimeChecks(checks) => { diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index ed73a76da8bf0..6e67ebf1d798f 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -644,6 +644,19 @@ where interp_ok(place) } + /// Computes a destination place, allocating its base local if it is currently live but without + /// an allocation. + pub fn eval_place_for_write( + &mut self, + mir_place: mir::Place<'tcx>, + skip_validity_for_simple_deref: bool, + ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> { + if M::move_elimination_semantics(self) && !mir_place.is_indirect_first_projection() { + self.allocate_local_for_write(mir_place.local)?; + } + self.eval_place(mir_place, skip_validity_for_simple_deref) + } + /// Given a place, returns either the underlying mplace or a reference to where the value of /// this place is stored. #[inline(always)] diff --git a/compiler/rustc_const_eval/src/interpret/stack.rs b/compiler/rustc_const_eval/src/interpret/stack.rs index 8a731e25d8f4b..b8d2e4975cc68 100644 --- a/compiler/rustc_const_eval/src/interpret/stack.rs +++ b/compiler/rustc_const_eval/src/interpret/stack.rs @@ -18,7 +18,7 @@ use tracing::{info_span, instrument, trace}; use super::{ AllocId, CtfeProvenance, FnArg, Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, - MemPlaceMeta, MemoryKind, Operand, PlaceTy, Pointer, Provenance, ReturnAction, Scalar, + MemPlaceMeta, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, ReturnAction, Scalar, from_known_layout, interp_ok, throw_ub, throw_unsup, }; use crate::{diagnostics, enter_trace_span}; @@ -153,6 +153,8 @@ impl std::fmt::Debug for LocalState<'_, Prov> { pub(super) enum LocalValue { /// This local is not currently alive, and cannot be used at all. Dead, + /// This local is alive, but does not currently have an allocation. + LiveUnallocated, /// A normal, live local. /// Mostly for convenience, we re-use the `Operand` type here. /// This is an optimization over just always having a pointer here; @@ -174,7 +176,7 @@ impl<'tcx, Prov: Provenance> LocalState<'tcx, Prov> { &self, ) -> Option>, MemPlaceMeta), Immediate>> { match self.value { - LocalValue::Dead => None, + LocalValue::Dead | LocalValue::LiveUnallocated => None, LocalValue::Live(Operand::Indirect(mplace)) => Some(Left((mplace.ptr, mplace.meta))), LocalValue::Live(Operand::Immediate(imm)) => Some(Right(imm)), } @@ -185,6 +187,7 @@ impl<'tcx, Prov: Provenance> LocalState<'tcx, Prov> { pub(super) fn access(&self) -> InterpResult<'tcx, &Operand> { match &self.value { LocalValue::Dead => throw_ub!(DeadLocal), // could even be "invalid program"? + LocalValue::LiveUnallocated => throw_ub!(UnallocatedLocal), LocalValue::Live(val) => interp_ok(val), } } @@ -195,6 +198,7 @@ impl<'tcx, Prov: Provenance> LocalState<'tcx, Prov> { pub(super) fn access_mut(&mut self) -> InterpResult<'tcx, &mut Operand> { match &mut self.value { LocalValue::Dead => throw_ub!(DeadLocal), // could even be "invalid program"? + LocalValue::LiveUnallocated => throw_ub!(UnallocatedLocal), LocalValue::Live(val) => interp_ok(val), } } @@ -557,26 +561,35 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { if layout.is_sized() { None } else { Some(layout) } }; - let local_val = LocalValue::Live(if let Some(layout) = unsized_ { - if !meta.has_meta() { - throw_unsup!(UnsizedLocal); - } - // Need to allocate some memory, since `Immediate::Uninit` cannot be unsized. - let dest_place = self.allocate_dyn(layout, MemoryKind::Stack, meta)?; - Operand::Indirect(*dest_place.mplace()) + // `LiveUnallocated` cannot preserve the metadata needed to allocate an unsized local + // later. Unsized locals are only supported as function arguments, where the metadata is + // available here and the local is initialized immediately after being made live, so keep + // allocating them eagerly. + let local_val = if M::move_elimination_semantics(self) && unsized_.is_none() { + assert!(!meta.has_meta()); + LocalValue::LiveUnallocated } else { - // Just make this an efficient immediate. - assert!(!meta.has_meta()); // we're dropping the metadata - // Make sure the machine knows this "write" is happening. (This is important so that - // races involving local variable allocation can be detected by Miri.) - M::after_local_write(self, local, /*storage_live*/ true)?; - // Note that not calling `layout_of` here does have one real consequence: - // if the type is too big, we'll only notice this when the local is actually initialized, - // which is a bit too late -- we should ideally notice this already here, when the memory - // is conceptually allocated. But given how rare that error is and that this is a hot function, - // we accept this downside for now. - Operand::Immediate(Immediate::Uninit) - }); + LocalValue::Live(if let Some(layout) = unsized_ { + if !meta.has_meta() { + throw_unsup!(UnsizedLocal); + } + // Need to allocate some memory, since `Immediate::Uninit` cannot be unsized. + let dest_place = self.allocate_dyn(layout, MemoryKind::Stack, meta)?; + Operand::Indirect(*dest_place.mplace()) + } else { + // Just make this an efficient immediate. + assert!(!meta.has_meta()); // we're dropping the metadata + // Make sure the machine knows this "write" is happening. (This is important so that + // races involving local variable allocation can be detected by Miri.) + M::after_local_write(self, local, /*storage_live*/ true)?; + // Note that not calling `layout_of` here does have one real consequence: + // if the type is too big, we'll only notice this when the local is actually initialized, + // which is a bit too late -- we should ideally notice this already here, when the memory + // is conceptually allocated. But given how rare that error is and that this is a hot function, + // we accept this downside for now. + Operand::Immediate(Immediate::Uninit) + }) + }; // If the local is already live, deallocate its old memory. let old = mem::replace(&mut self.frame_mut().locals[local].value, local_val); @@ -590,6 +603,17 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.storage_live_dyn(local, MemPlaceMeta::None) } + /// Ensure that a live local has an allocation. + pub fn storage_alloc(&mut self, local: mir::Local) -> InterpResult<'tcx> { + match self.frame().locals[local].value { + LocalValue::Dead => throw_ub!(DeadLocal), + LocalValue::LiveUnallocated | LocalValue::Live(_) => { + self.allocate_local_for_write(local)? + } + } + interp_ok(()) + } + pub fn storage_dead(&mut self, local: mir::Local) -> InterpResult<'tcx> { assert!(local != mir::RETURN_PLACE, "Cannot make return place dead"); trace!("{:?} is now dead", local); @@ -600,6 +624,31 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { interp_ok(()) } + /// Ensure that a direct destination local has an allocation. + pub(super) fn allocate_local_for_write(&mut self, local: mir::Local) -> InterpResult<'tcx> { + let local_value = &mut self.frame_mut().locals[local].value; + if matches!(local_value, LocalValue::LiveUnallocated) { + *local_value = LocalValue::Live(Operand::Immediate(Immediate::Uninit)); + M::after_local_write(self, local, /*storage_live*/ true)?; + } + interp_ok(()) + } + + /// Move an entire local into a detached value and leave the local live but unallocated. + pub(super) fn move_out_local( + &mut self, + local: mir::Local, + layout: Option>, + ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + let op = self.local_to_op(local, layout)?; + + let moved_op = self.snapshot_operand(op)?; + let old = + mem::replace(&mut self.frame_mut().locals[local].value, LocalValue::LiveUnallocated); + self.deallocate_local(old)?; + interp_ok(moved_op) + } + fn deallocate_local(&mut self, local: LocalValue) -> InterpResult<'tcx> { if let LocalValue::Live(Operand::Indirect(MemPlace { ptr, .. })) = local { // All locals have a backing allocation, even if the allocation is empty @@ -661,10 +710,12 @@ impl<'a, 'tcx: 'a, M: Machine<'tcx>> InterpCx<'tcx, M> { // provided so it should not be possible to get a mismatch here. let (_idx, callee_abi) = callee_abis.next().unwrap(); assert!(self.check_argument_compat(caller_abi, callee_abi)?); - // FIXME: do we have to worry about in-place argument passing? let op = fn_arg.copy_fn_arg(); let mplace = self.allocate(op.layout, MemoryKind::Stack)?; self.copy_op(&op, &mplace)?; + if let FnArg::InPlace(mplace) = fn_arg { + M::protect_in_place_function_argument(self, mplace)?; + } varargs.push(mplace); } @@ -702,6 +753,7 @@ impl<'tcx, Prov: Provenance> LocalState<'tcx, Prov> { ) -> std::fmt::Result { match self.value { LocalValue::Dead => write!(fmt, " is dead")?, + LocalValue::LiveUnallocated => write!(fmt, " is live but unallocated")?, LocalValue::Live(Operand::Immediate(Immediate::Uninit)) => { write!(fmt, " is uninitialized")? } diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index e9f1d06187cd6..74ddde431654e 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -7,8 +7,9 @@ use std::iter; use either::Either; use rustc_abi::{FIRST_VARIANT, FieldIdx}; use rustc_data_structures::fx::FxHashSet; -use rustc_index::IndexSlice; +use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::mir; +use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{self, Instance, Ty}; use rustc_span::{Spanned, bug, span_bug}; use rustc_target::callconv::FnAbi; @@ -17,10 +18,23 @@ use tracing::{info, instrument, trace}; use super::{ EnteredTraceSpan, FnArg, FnVal, ImmTy, Immediate, InterpCx, InterpResult, Machine, - MemPlaceMeta, PlaceTy, Projectable, RetagMode, interp_ok, throw_ub, throw_unsup_format, + MemPlaceMeta, OpTy, PlaceTy, Projectable, Provenance, RetagMode, Scalar, interp_ok, throw_ub, + throw_unsup_format, }; use crate::{enter_trace_span, util}; +/// An evaluated rvalue, with destination-dependent operations deferred until writing. +enum EvaluatedRvalue<'tcx, Prov: Provenance> { + Use(OpTy<'tcx, Prov>, mir::WithRetag), + Immediate(ImmTy<'tcx, Prov>), + Ref(ImmTy<'tcx, Prov>, RetagMode), + RawPtr { val: ImmTy<'tcx, Prov>, needs_retag: bool }, + Copy { op: OpTy<'tcx, Prov>, allow_transmute: bool }, + Cast(OpTy<'tcx, Prov>, mir::CastKind, Ty<'tcx>), + Aggregate(mir::AggregateKind<'tcx>, IndexVec>), + Repeat(OpTy<'tcx, Prov>), +} + struct EvaluatedCalleeAndArgs<'tcx, M: Machine<'tcx>> { callee: FnVal<'tcx, M::ExtraFnVal>, args: Vec>, @@ -55,6 +69,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { if let Some(stmt) = basic_block.statements.get(loc.statement_index) { let old_frames = self.frame_idx(); self.eval_statement(stmt)?; + self.clear_operand_temps()?; // Make sure we are not updating `statement_index` of the wrong frame. assert_eq!(old_frames, self.frame_idx()); // Advance the program counter. @@ -66,6 +81,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let terminator = basic_block.terminator(); self.eval_terminator(terminator)?; + self.clear_operand_temps()?; if !self.stack().is_empty() { if let Either::Left(loc) = self.frame().loc { info!("// executing {:?}", loc.block); @@ -94,8 +110,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { Assign((place, rvalue)) => self.eval_rvalue_into_place(rvalue, *place)?, SetDiscriminant { place, variant_index } => { - let dest = - self.eval_place(**place, /* skip_validity_for_simple_deref */ false)?; + let dest = self.eval_place_for_write( + **place, /* skip_validity_for_simple_deref */ false, + )?; self.write_discriminant(*variant_index, &dest)?; } @@ -109,6 +126,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.storage_dead(*local)?; } + // Allocate storage for a local + StorageAlloc(local) => { + self.storage_alloc(*local)?; + } + // No dynamic semantics attached to `FakeRead`; MIR // interpreter is solely intended for borrowck'ed code. FakeRead(..) => {} @@ -154,75 +176,152 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } /// Evaluate an assignment statement. - /// - /// There is no separate `eval_rvalue` function. Instead, the code for handling each rvalue - /// type writes its results directly into the memory specified by the place. pub fn eval_rvalue_into_place( &mut self, rvalue: &mir::Rvalue<'tcx>, place: mir::Place<'tcx>, ) -> InterpResult<'tcx> { - // We can skip validity because we'll write to the place which checks everything we care - // about for references, and the pointee must be sized so there's nothing to check for raw - // pointers. - let dest = self.eval_place(place, /* skip_validity_for_simple_deref */ true)?; - // FIXME: ensure some kind of non-aliasing between LHS and RHS? - // Also see https://github.com/rust-lang/rust/issues/68364. + // We can skip validity when evaluating the destination place because + // we'll write to it which checks everything we care about for + // references, and the pointee must be sized so there's nothing to check + // for raw pointers. + if M::move_elimination_semantics(self) { + // Evaluate the destination place last with move-elimination semantics. + let ty = self.instantiate_from_current_frame_and_normalize_erasing_regions( + place.ty(&self.frame().body.local_decls, *self.tcx).ty, + )?; + let layout = self.layout_of(ty)?; + let value = self.eval_rvalue(rvalue, layout)?; + let dest = + self.eval_place_for_write(place, /* skip_validity_for_simple_deref */ true)?; + self.write_rvalue(value, dest) + } else { + // Preserve destination-first evaluation without move-elimination semantics. + let dest = + self.eval_place_for_write(place, /* skip_validity_for_simple_deref */ true)?; + let value = self.eval_rvalue(rvalue, dest.layout)?; + self.write_rvalue(value, dest) + } + } + /// Evaluate an rvalue, leaving destination-dependent operations for `write_rvalue`. + fn eval_rvalue( + &mut self, + rvalue: &mir::Rvalue<'tcx>, + layout: TyAndLayout<'tcx>, + ) -> InterpResult<'tcx, EvaluatedRvalue<'tcx, M::Provenance>> { use rustc_middle::mir::Rvalue::*; - match *rvalue { + interp_ok(match *rvalue { + Use(ref operand, with_retag) => EvaluatedRvalue::Use( + self.eval_operand_no_snapshot(operand, Some(layout))?, + with_retag, + ), + Repeat(ref operand, _) => { + EvaluatedRvalue::Repeat(self.eval_operand_no_snapshot(operand, None)?) + } + Ref(_, borrow_kind, place) => { + // `x = &*ptr` does not need a validity check on `ptr` because we will already + // check `x` when writing to the destination. + let src = self.eval_place(place, /* skip_validity_for_simple_deref */ true)?; + let place = self.force_allocation(&src)?; + let val = ImmTy::from_immediate(place.to_ref(self), layout); + let mode = if borrow_kind.is_two_phase_borrow() { + RetagMode::TwoPhase + } else { + RetagMode::Default + }; + EvaluatedRvalue::Ref(val, mode) + } + RawPtr(kind, place) => { + let place_base_raw = place.is_indirect_first_projection() + && self.frame().body.local_decls[place.local].ty.is_raw_ptr(); + let src = + self.eval_place(place, /* skip_validity_for_simple_deref */ false)?; + let place = self.force_allocation(&src)?; + let val = ImmTy::from_immediate(place.to_ref(self), layout); + // Retag unless the place was already raw or this is a "fake" raw borrow. + EvaluatedRvalue::RawPtr { val, needs_retag: !place_base_raw && !kind.is_fake() } + } ThreadLocalRef(did) => { let ptr = M::thread_local_static_pointer(self, did)?; - self.write_pointer(ptr, &dest)?; + EvaluatedRvalue::Immediate(ImmTy::from_scalar( + Scalar::from_maybe_pointer(ptr.into(), self), + layout, + )) } - - Use(ref operand, with_retag) => { - // Avoid recomputing the layout - let op = self.eval_operand(operand, Some(dest.layout))?; - let mode = if with_retag.yes() { RetagMode::Default } else { RetagMode::None }; - M::with_retag_mode(self, mode, |ecx| ecx.copy_op(&op, &dest))?; + Cast(kind, ref operand, ty) => { + let op = self.eval_operand_no_snapshot(operand, None)?; + let ty = self.instantiate_from_current_frame_and_normalize_erasing_regions(ty)?; + EvaluatedRvalue::Cast(op, kind, ty) } - - CopyForDeref(_) => bug!("`CopyForDeref` in runtime MIR"), - BinaryOp(bin_op, (ref left, ref right)) => { - let layout = util::binop_left_homogeneous(bin_op).then_some(dest.layout); - let left = self.read_immediate(&self.eval_operand(left, layout)?)?; - let layout = util::binop_right_homogeneous(bin_op).then_some(left.layout); - let right = self.read_immediate(&self.eval_operand(right, layout)?)?; + let operand_layout = util::binop_left_homogeneous(bin_op).then_some(layout); + let left = self.eval_operand(left, operand_layout)?; + let left = self.read_immediate(&left)?; + let operand_layout = util::binop_right_homogeneous(bin_op).then_some(left.layout); + let right = self.eval_operand(right, operand_layout)?; + let right = self.read_immediate(&right)?; let result = self.binary_op(bin_op, &left, &right)?; - assert_eq!(result.layout, dest.layout, "layout mismatch for result of {bin_op:?}"); - self.write_immediate(*result, &dest)?; + assert_eq!(result.layout, layout, "layout mismatch for result of {bin_op:?}"); + EvaluatedRvalue::Immediate(result) } - UnaryOp(un_op, ref operand) => { - let layout = util::unop_homogeneous(un_op).then_some(dest.layout); - let val = self.read_immediate(&self.eval_operand(operand, layout)?)?; + let operand_layout = util::unop_homogeneous(un_op).then_some(layout); + let val = self.eval_operand(operand, operand_layout)?; + let val = self.read_immediate(&val)?; let result = self.unary_op(un_op, &val)?; - assert_eq!(result.layout, dest.layout, "layout mismatch for result of {un_op:?}"); - self.write_immediate(*result, &dest)?; + assert_eq!(result.layout, layout, "layout mismatch for result of {un_op:?}"); + EvaluatedRvalue::Immediate(result) + } + Discriminant(place) => { + let op = self.eval_place_to_op(place, None)?; + let variant = self.read_discriminant(&op)?; + EvaluatedRvalue::Immediate(self.discriminant_for_variant(op.layout.ty, variant)?) + } + Reborrow(_, mutability, place) => { + // Shared generic reborrows use `CoerceShared`: a bitwise copy into a + // distinct same-layout target ADT. + EvaluatedRvalue::Copy { + op: self.eval_place_to_op(place, None)?, + allow_transmute: mutability.is_not(), + } } - Aggregate(ref kind, ref operands) => { - self.write_aggregate(kind, operands, &dest)?; + let operands = operands + .iter() + .map(|operand| self.eval_operand(operand, None)) + .collect::>>()?; + EvaluatedRvalue::Aggregate((**kind).clone(), operands) } - - Repeat(ref operand, _) => { - self.write_repeat(operand, &dest)?; + WrapUnsafeBinder(ref operand, _) => { + // Constructing an unsafe binder acts like a transmute + // since the operand's layout does not change. + EvaluatedRvalue::Copy { + op: self.eval_operand_no_snapshot(operand, None)?, + allow_transmute: true, + } } + CopyForDeref(_) => bug!("`CopyForDeref` in runtime MIR"), + }) + } - Ref(_, borrow_kind, place) => { - // `x = &*ptr` does not need a validity check on `ptr` because we will already - // check `x` below. - let src = self.eval_place(place, /* skip_validity_for_simple_deref */ true)?; - let place = self.force_allocation(&src)?; - let mut val = ImmTy::from_immediate(place.to_ref(self), dest.layout); + /// Write an evaluated rvalue, performing destination-dependent conversion and validation. + fn write_rvalue( + &mut self, + value: EvaluatedRvalue<'tcx, M::Provenance>, + dest: PlaceTy<'tcx, M::Provenance>, + ) -> InterpResult<'tcx> { + // FIXME: ensure some kind of non-aliasing between LHS and RHS? + // Also see https://github.com/rust-lang/rust/issues/68364. + + match value { + EvaluatedRvalue::Use(op, with_retag) => { + let mode = if with_retag.yes() { RetagMode::Default } else { RetagMode::None }; + M::with_retag_mode(self, mode, |ecx| ecx.copy_op(&op, &dest))?; + } + EvaluatedRvalue::Immediate(val) => self.write_immediate(*val, &dest)?, + EvaluatedRvalue::Ref(mut val, mode) => { // A fresh reference was created, make sure it gets retagged with the right mode. - let mode = if borrow_kind.is_two_phase_borrow() { - RetagMode::TwoPhase - } else { - RetagMode::Default - }; M::with_retag_mode(self, mode, |ecx| { // If validation is disabled, we still want to do this retag. This is because // const-eval disables validation for performance reasons but wants to retag @@ -233,71 +332,31 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { val = new_val; } } - // Now do the actual write. ecx.write_immediate(*val, &dest) })?; } - - Reborrow(_, mutability, place) => { - let op = self.eval_place_to_op(place, None)?; - if mutability.is_not() { - // Shared generic reborrows use `CoerceShared`: a bitwise copy into a - // distinct same-layout target ADT. - self.copy_op_allow_transmute(&op, &dest)?; - } else { - self.copy_op(&op, &dest)?; - } - } - - RawPtr(kind, place) => { - // Figure out whether this is an addr_of of an already raw place. - let place_base_raw = if place.is_indirect_first_projection() { - let ty = self.frame().body.local_decls[place.local].ty; - ty.is_raw_ptr() - } else { - // Not a deref, and thus not raw. - false - }; - - let src = - self.eval_place(place, /* skip_validity_for_simple_deref */ false)?; - let place = self.force_allocation(&src)?; - let mut val = ImmTy::from_immediate(place.to_ref(self), dest.layout); - if !place_base_raw && !kind.is_fake() { - // If this was not already raw, it needs retagging -- except for "fake" - // raw borrows whose defining property is that they do not get retagged. + EvaluatedRvalue::RawPtr { mut val, needs_retag } => { + if needs_retag { val = M::with_retag_mode(self, RetagMode::Raw, |ecx| { interp_ok(M::retag_ptr_value(ecx, &val, val.layout.ty)?.unwrap_or(val)) })?; } - // This writes a raw pointer so it will not do any retags. + // Writing a raw pointer does not retag it during validation. self.write_immediate(*val, &dest)?; } - - Cast(cast_kind, ref operand, cast_ty) => { - let src = self.eval_operand(operand, None)?; - let cast_ty = - self.instantiate_from_current_frame_and_normalize_erasing_regions(cast_ty)?; - self.cast(&src, cast_kind, cast_ty, &dest)?; - } - - Discriminant(place) => { - let op = self.eval_place_to_op(place, None)?; - let variant = self.read_discriminant(&op)?; - let discr = self.discriminant_for_variant(op.layout.ty, variant)?; - self.write_immediate(*discr, &dest)?; + EvaluatedRvalue::Copy { op, allow_transmute } => { + if allow_transmute { + self.copy_op_allow_transmute(&op, &dest)?; + } else { + self.copy_op(&op, &dest)?; + } } - - WrapUnsafeBinder(ref op, _ty) => { - // Constructing an unsafe binder acts like a transmute - // since the operand's layout does not change. - let op = self.eval_operand(op, None)?; - self.copy_op_allow_transmute(&op, &dest)?; + EvaluatedRvalue::Cast(op, kind, ty) => self.cast(&op, kind, ty, &dest)?, + EvaluatedRvalue::Aggregate(kind, operands) => { + self.write_aggregate(&kind, &operands, &dest)?; } + EvaluatedRvalue::Repeat(op) => self.write_repeat(&op, &dest)?, } - - trace!("{:?}", self.dump_place(&dest)); - interp_ok(()) } @@ -306,7 +365,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { fn write_aggregate( &mut self, kind: &mir::AggregateKind<'tcx>, - operands: &IndexSlice>, + operands: &IndexSlice>, dest: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx> { let (variant_index, variant_dest, active_field_index) = match *kind { @@ -322,13 +381,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let [data, meta] = &operands.raw else { bug!("{kind:?} should have 2 operands, had {operands:?}"); }; - let data = self.eval_operand(data, None)?; - let data = self.read_pointer(&data)?; - let meta = self.eval_operand(meta, None)?; + let data = self.read_pointer(data)?; let meta = if meta.layout.is_zst() { MemPlaceMeta::None } else { - MemPlaceMeta::Meta(self.read_scalar(&meta)?) + MemPlaceMeta::Meta(self.read_scalar(meta)?) }; let ptr_imm = Immediate::new_pointer_with_meta(data, meta, self); let ptr = ImmTy::from_immediate(ptr_imm, dest.layout); @@ -343,9 +400,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { for (field_index, operand) in operands.iter_enumerated() { let field_index = active_field_index.unwrap_or(field_index); let field_dest = self.project_field(&variant_dest, field_index)?; - let op = self.eval_operand(operand, Some(field_dest.layout))?; // We validate manually below so we don't have to do it here. - self.copy_op_no_validate(&op, &field_dest, /*allow_transmute*/ false)?; + self.copy_op_no_validate(operand, &field_dest, /*allow_transmute*/ false)?; } self.write_discriminant(variant_index, dest)?; // Validate that the entire thing is valid, and reset padding that might be in between the @@ -360,14 +416,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { interp_ok(()) } - /// Repeats `operand` into the destination. `dest` must have array type, and that type - /// determines how often `operand` is repeated. + /// Repeats `src` into the destination. `dest` must have array type, and that type + /// determines how often `src` is repeated. fn write_repeat( &mut self, - operand: &mir::Operand<'tcx>, + src: &OpTy<'tcx, M::Provenance>, dest: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx> { - let src = self.eval_operand(operand, None)?; assert!(src.layout.is_sized()); let dest = self.force_allocation(&dest)?; let length = dest.len(self)?; @@ -378,7 +433,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } else { // Write the src to the first element. let first = self.project_index(&dest, 0)?; - self.copy_op(&src, &first)?; + self.copy_op(src, &first)?; // This is performance-sensitive code for big static/const arrays! So we // avoid writing each operand individually and instead just make many copies @@ -399,7 +454,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { interp_ok(()) } - /// Evaluate the arguments of a function call + /// Evaluate the arguments of a function call. + /// + /// This is not used for tail calls: those always use normal operand + /// evaluation since they cannot use `FnArg::InPlace`. fn eval_fn_call_argument( &mut self, op: &mir::Operand<'tcx>, @@ -407,15 +465,27 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { ) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> { interp_ok(match op { mir::Operand::Copy(_) | mir::Operand::Constant(_) | mir::Operand::RuntimeChecks(_) => { - // Make a regular copy. let op = self.eval_operand(op, None)?; FnArg::Copy(op) } - mir::Operand::Move(place) => { + mir::Operand::Move(mir_place) => { + // With move elimination semantics, evaluating a whole-local + // move snapshots the value and frees its original allocation. + // This is stronger than the protector applied to InPlace + // arguments: the local is already unallocated when evaluating + // later arguments, and accesses through aliases to its old + // allocation fail. Direct destinations rooted in the moved + // local are rejected as malformed MIR, preventing destination + // evaluation from allocating fresh storage and hiding the + // overlap. + if M::move_elimination_semantics(self) && mir_place.as_local().is_some() { + return interp_ok(FnArg::Copy(self.eval_operand(op, None)?)); + } + // We will read from this place, which checks everything there is to check, // so we can skip the extra validity check here. let place = - self.eval_place(*place, /* skip_validity_for_simple_deref */ true)?; + self.eval_place(*mir_place, /* skip_validity_for_simple_deref */ true)?; if move_definitely_disjoint { // We still have to ensure that no *other* pointers are used to access this place, // so *if* it is in memory then we have to treat it as `InPlace`. @@ -438,46 +508,54 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { fn eval_callee_and_args( &mut self, terminator: &mir::Terminator<'tcx>, + is_tail_call: bool, func: &mir::Operand<'tcx>, args: &[Spanned>], dest: &mir::Place<'tcx>, ) -> InterpResult<'tcx, EvaluatedCalleeAndArgs<'tcx, M>> { let func = self.eval_operand(func, None)?; - // Evaluating function call arguments. The tricky part here is dealing with `Move` - // arguments: we have to ensure no two such arguments alias. This would be most easily done - // by just forcing them all into memory and then doing the usual in-place argument - // protection, but then we'd force *a lot* of arguments into memory. So we do some syntactic - // pre-processing here where if all `move` arguments are syntactically distinct local - // variables (and none is indirect), we can skip the in-memory forcing. - // We have to include `dest` in that list so that we can detect aliasing of an in-place - // argument with the return place. - let move_definitely_disjoint = 'move_definitely_disjoint: { - let mut previous_locals = FxHashSet::::default(); - for place in args - .iter() - .filter_map(|a| { - // We only have to care about `Move` arguments. - if let mir::Operand::Move(place) = &a.node { Some(place) } else { None } - }) - .chain(iter::once(dest)) - { - if place.is_indirect_first_projection() { - // An indirect in-place argument could alias with anything else... - break 'move_definitely_disjoint false; - } - if !previous_locals.insert(place.local) { - // This local is the base for two arguments! They might overlap. - break 'move_definitely_disjoint false; + let args = if is_tail_call { + // The current frame is destroyed by a tail call, so its argument places cannot be + // donated to the callee. Evaluate them as ordinary operands instead. + args.iter() + .map(|arg| self.eval_operand(&arg.node, None).map(FnArg::Copy)) + .collect::>>()? + } else { + // Evaluating function call arguments. The tricky part here is dealing with `Move` + // arguments: we have to ensure no two such arguments alias. This would be most easily + // done by just forcing them all into memory and then doing the usual in-place argument + // protection, but then we'd force *a lot* of arguments into memory. So we do some + // syntactic pre-processing here where if all `move` arguments are syntactically + // distinct local variables (and none is indirect), we can skip the in-memory forcing. + // We have to include `dest` in that list so that we can detect aliasing of an in-place + // argument with the return place. + let move_definitely_disjoint = 'move_definitely_disjoint: { + let mut previous_locals = FxHashSet::::default(); + for place in args + .iter() + .filter_map(|a| { + // We only have to care about `Move` arguments. + if let mir::Operand::Move(place) = &a.node { Some(place) } else { None } + }) + .chain(iter::once(dest)) + { + if place.is_indirect_first_projection() { + // An indirect in-place argument could alias with anything else... + break 'move_definitely_disjoint false; + } + if !previous_locals.insert(place.local) { + // This local is the base for two arguments! They might overlap. + break 'move_definitely_disjoint false; + } } - } - // We found no violation so they are all definitely disjoint. - true + // We found no violation so they are all definitely disjoint. + true + }; + args.iter() + .map(|arg| self.eval_fn_call_argument(&arg.node, move_definitely_disjoint)) + .collect::>>()? }; - let args = args - .iter() - .map(|arg| self.eval_fn_call_argument(&arg.node, move_definitely_disjoint)) - .collect::>>()?; let fn_sig_binder = { let _trace = enter_trace_span!(M, "fn_sig", ty = ?func.layout.ty.kind()); @@ -535,7 +613,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { Goto { target } => self.go_to_block(target), SwitchInt { ref discr, ref targets } => { - let discr = self.read_immediate(&self.eval_operand(discr, None)?)?; + let discr = self.eval_operand(discr, None)?; + let discr = self.read_immediate(&discr)?; trace!("SwitchInt({:?})", *discr); // Branch to the `otherwise` case by default, if no match is found. @@ -570,11 +649,37 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let old_stack = self.frame_idx(); let old_loc = self.frame().loc; - // Evaluation order consistent with assignment: destination first. - let dest_place = - self.eval_place(destination, /* skip_validity_for_simple_deref */ false)?; + let (dest_place, evaluated) = if M::move_elimination_semantics(self) { + // With move-elimination semantics, evaluate the destination last. + let evaluated = self.eval_callee_and_args( + terminator, + /* is_tail_call */ false, + func, + args, + &destination, + )?; + let dest_place = self.eval_place_for_write( + destination, + /* skip_validity_for_simple_deref */ false, + )?; + (dest_place, evaluated) + } else { + // Without move-elimination semantics, evaluate the destination first. + let dest_place = self.eval_place_for_write( + destination, + /* skip_validity_for_simple_deref */ false, + )?; + let evaluated = self.eval_callee_and_args( + terminator, + /* is_tail_call */ false, + func, + args, + &destination, + )?; + (dest_place, evaluated) + }; let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = - self.eval_callee_and_args(terminator, func, args, &destination)?; + evaluated; self.init_fn_call( callee, @@ -604,7 +709,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let old_frame_idx = self.frame_idx(); let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = - self.eval_callee_and_args(terminator, func, args, &mir::Place::return_place())?; + self.eval_callee_and_args( + terminator, + /* is_tail_call */ true, + func, + args, + &mir::Place::return_place(), + )?; self.init_fn_tail_call( callee, @@ -648,7 +759,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { Assert { ref cond, expected, ref msg, target, unwind } => { let ignored = M::ignore_optional_overflow_checks(self) && msg.is_optional_overflow_check(); - let cond_val = self.read_scalar(&self.eval_operand(cond, None)?)?.to_bool()?; + let cond = self.eval_operand(cond, None)?; + let cond_val = self.read_scalar(&cond)?.to_bool()?; if ignored || expected == cond_val { self.go_to_block(target); } else { diff --git a/compiler/rustc_index/src/interval.rs b/compiler/rustc_index/src/interval.rs index b7b1531e50857..475c81e66ab09 100644 --- a/compiler/rustc_index/src/interval.rs +++ b/compiler/rustc_index/src/interval.rs @@ -1,6 +1,7 @@ use std::iter::Step; use std::marker::PhantomData; -use std::ops::{Bound, Range, RangeBounds}; +use std::ops::{Bound, RangeBounds}; +use std::range::RangeInclusive; use smallvec::SmallVec; @@ -59,11 +60,14 @@ impl IntervalSet { } /// Iterates through intervals stored in the set, in order. - pub fn iter_intervals(&self) -> impl Iterator> + pub fn iter_intervals(&self) -> impl Iterator> where I: Step, { - self.map.iter().map(|&(start, end)| I::new(start as usize)..I::new(end as usize + 1)) + self.map.iter().map(|&(start, end)| RangeInclusive { + start: I::new(start as usize), + last: I::new(end as usize), + }) } /// Returns true if we increased the number of elements present. @@ -204,17 +208,38 @@ impl IntervalSet { needle <= *prev_end } + /// Returns whether any point in `range` is contained in the set. + pub fn intersects_range(&self, range: impl RangeBounds + Clone) -> bool { + let start = inclusive_start(range.clone()); + let Some(end) = inclusive_end(self.domain, range) else { + // empty range + return false; + }; + if start > end { + return false; + } + + // Find the last interval whose start is <= end. + let Some(last) = self.map.partition_point(|r| r.0 <= end).checked_sub(1) else { + // All ranges in the map start after the new range's end + return false; + }; + let (_, prev_end) = &self.map[last]; + start <= *prev_end + } + pub fn superset(&self, other: &IntervalSet) -> bool where I: Step, { let mut sup_iter = self.iter_intervals(); let mut current = None; - let contains = |sup: Range, sub: Range, current: &mut Option>| { - if sup.end < sub.start { - // if `sup.end == sub.start`, the next sup doesn't contain `sub.start` + let contains = |sup: RangeInclusive, + sub: RangeInclusive, + current: &mut Option>| { + if sup.last < sub.start { None // continue to the next sup - } else if sup.end >= sub.end && sup.start <= sub.start { + } else if sup.last >= sub.last && sup.start <= sub.start { *current = Some(sup); // save the current sup Some(true) } else { @@ -224,8 +249,8 @@ impl IntervalSet { other.iter_intervals().all(|sub| { current .take() - .and_then(|sup| contains(sup, sub.clone(), &mut current)) - .or_else(|| sup_iter.find_map(|sup| contains(sup, sub.clone(), &mut current))) + .and_then(|sup| contains(sup, sub, &mut current)) + .or_else(|| sup_iter.find_map(|sup| contains(sup, sub, &mut current))) .unwrap_or(false) }) } @@ -242,11 +267,11 @@ impl IntervalSet { let mut other_current = other_iter.next()?; loop { - if self_current.end <= other_current.start { + if self_current.last < other_current.start { self_current = self_iter.next()?; continue; } - if other_current.end <= self_current.start { + if other_current.last < self_current.start { other_current = other_iter.next()?; continue; } @@ -374,6 +399,12 @@ impl SparseIntervalMatrix { self.rows.iter_enumerated() } + pub fn clear_row(&mut self, row: R) { + if let Some(row) = self.rows.get_mut(row) { + row.clear(); + } + } + fn ensure_row(&mut self, row: R) -> &mut IntervalSet { self.rows.ensure_contains_elem(row, || IntervalSet::new(self.column_size)) } @@ -397,6 +428,16 @@ impl SparseIntervalMatrix { write_row.union(read_row) } + pub fn disjoint_rows(&self, a: R, b: R) -> bool + where + C: Step, + { + let (Some(a), Some(b)) = (self.rows.get(a), self.rows.get(b)) else { + return true; + }; + a.disjoint(b) + } + pub fn insert_all_into_row(&mut self, row: R) { self.ensure_row(row).insert_all(); } diff --git a/compiler/rustc_index/src/interval/tests.rs b/compiler/rustc_index/src/interval/tests.rs index 375af60f66207..cf3222e6c6572 100644 --- a/compiler/rustc_index/src/interval/tests.rs +++ b/compiler/rustc_index/src/interval/tests.rs @@ -5,7 +5,7 @@ fn insert_collapses() { let mut set = IntervalSet::::new(10000); set.insert_range(9831..=9837); set.insert_range(43..=9830); - assert_eq!(set.iter_intervals().collect::>(), [43..9838]); + assert_eq!(set.iter_intervals().collect::>(), [(43..=9837).into()]); } #[test] diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs index 049e1ec22e547..d4ab835a8c65d 100644 --- a/compiler/rustc_middle/src/mir/interpret/error.rs +++ b/compiler/rustc_middle/src/mir/interpret/error.rs @@ -410,6 +410,8 @@ pub enum UndefinedBehaviorInfo<'tcx> { InvalidUninitBytes(Option<(AllocId, BadBytesAccess)>), /// Working with a local that is not currently live. DeadLocal, + /// Working with a local that is live but does not currently have an allocation. + UnallocatedLocal, /// A discriminant of an uninhabited enum variant is written. UninhabitedEnumVariantWritten(VariantIdx), /// An uninhabited enum variant is projected. @@ -616,6 +618,7 @@ impl<'tcx> fmt::Display for UndefinedBehaviorInfo<'tcx> { uninit = info.bad, ), DeadLocal => write!(f, "accessing a dead local variable"), + UnallocatedLocal => write!(f, "accessing a live but unallocated local variable"), UninhabitedEnumVariantWritten(_) => { write!(f, "writing discriminant of an uninhabited enum variant") } diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index 159a0f9883db3..7c994767e53ea 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -873,6 +873,7 @@ impl Debug for StatementKind<'_> { FakeRead((ref cause, ref place)) => { write!(fmt, "FakeRead({cause:?}, {place:?})") } + StorageAlloc(ref local) => write!(fmt, "StorageAlloc({local:?})"), StorageLive(ref place) => write!(fmt, "StorageLive({place:?})"), StorageDead(ref place) => write!(fmt, "StorageDead({place:?})"), SetDiscriminant { ref place, variant_index } => { diff --git a/compiler/rustc_middle/src/mir/statement.rs b/compiler/rustc_middle/src/mir/statement.rs index b6542fb328aac..d706693095f82 100644 --- a/compiler/rustc_middle/src/mir/statement.rs +++ b/compiler/rustc_middle/src/mir/statement.rs @@ -57,6 +57,7 @@ impl<'tcx> StatementKind<'tcx> { StatementKind::SetDiscriminant { .. } => "SetDiscriminant", StatementKind::StorageLive(..) => "StorageLive", StatementKind::StorageDead(..) => "StorageDead", + StatementKind::StorageAlloc(..) => "StorageAlloc", StatementKind::PlaceMention(..) => "PlaceMention", StatementKind::AscribeUserType(..) => "AscribeUserType", StatementKind::Coverage(..) => "Coverage", diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 6bb0190cd70b8..09d246c9118b4 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -125,7 +125,6 @@ pub enum RuntimePhase { /// disallowed: /// * [`TerminatorKind::Yield`] /// * [`TerminatorKind::CoroutineDrop`] - /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array` /// * [`Rvalue::CopyForDeref`] /// * [`PlaceElem::OpaqueCast`] /// * [`LocalInfo::DerefTemp`](super::LocalInfo::DerefTemp) @@ -333,7 +332,11 @@ pub enum StatementKind<'tcx> { /// [`Scalar::Initialized`][rustc_abi::Scalar::Initialized]. As a part of this discussion, it is /// also unclear in what order the components are evaluated. /// + /// With `-Zmir-move-elimination`, the rvalue's operands and source places are evaluated before + /// the destination place. See [RFC 3943]. + /// /// [#68364]: https://github.com/rust-lang/rust/issues/68364 + /// [RFC 3943]: https://github.com/rust-lang/rfcs/pull/3943 /// /// See [`Rvalue`] documentation for details on each of those. Assign(Box<(Place<'tcx>, Rvalue<'tcx>)>), @@ -380,11 +383,31 @@ pub enum StatementKind<'tcx> { /// If the local is already allocated, calling `StorageLive` again will implicitly free the /// local and then allocate fresh uninitialized memory. If a local is already deallocated, /// calling `StorageDead` again is a NOP. + /// + /// With `-Zmir-move-elimination`, `StorageLive` leaves locals live but unallocated. Storage is + /// allocated when a destination place directly based on the local is evaluated, or by + /// [`StorageAlloc`](StatementKind::StorageAlloc). See [RFC 3943]. + /// + /// [RFC 3943]: https://github.com/rust-lang/rfcs/pull/3943 StorageLive(Local), /// See `StorageLive` above. StorageDead(Local), + /// If the local is live but unallocated, allocates backing storage containing uninitialized + /// bytes for it. This has no effect if the local already has an allocation. + /// + /// Calling this on a dead local is UB. + /// + /// This only has an effect when `-Zmir-move-elimination` is enabled since that is the only way + /// to get a live-but-unallocated local. This can be used to ensure a local has storage without + /// writing a value, for example before taking its address. See [RFC 3943]. + /// + /// This statement is only permitted in runtime MIR. + /// + /// [RFC 3943]: https://github.com/rust-lang/rfcs/pull/3943 + StorageAlloc(Local), + /// This statement exists to preserve a trace of a scrutinee matched against a wildcard binding. /// This is especially useful for `let _ = PLACE;` bindings that desugar to a single /// `PlaceMention(PLACE)`. @@ -784,7 +807,13 @@ pub enum TerminatorKind<'tcx> { /// The evaluation order is currently "first compute destination place, then `func` operand, /// then the arguments in left-to-right order". /// + /// [RFC 3943] semantics (enabled with -Z mir-move-elimination) changes the evaluation order to + /// evaluate the destination place last instead. Additionally, a direct destination must not be + /// rooted in a local that is also a whole-local move argument. Violating this restriction makes + /// the MIR malformed. + /// /// [#71117]: https://github.com/rust-lang/rust/issues/71117 + /// [RFC 3943]: https://github.com/rust-lang/rfcs/pull/3943 Call { /// The function that’s being called. func: Operand<'tcx>, @@ -1323,7 +1352,12 @@ pub enum Operand<'tcx> { /// inherently tied to a function call. Are these the semantics we want for MIR? Is this /// something we can even decide without knowing more about Rust's memory model? /// + /// With `-Zmir-move-elimination`, moving a whole local loads its value, frees its allocation, + /// and leaves it live but unallocated. This happens during operand evaluation, including for + /// call arguments, before evaluating subsequent operands. See [RFC 3943]. + /// /// [UCG#188]: https://github.com/rust-lang/unsafe-code-guidelines/issues/188 + /// [RFC 3943]: https://github.com/rust-lang/rfcs/pull/3943 Move(Place<'tcx>), /// Constants are already semantically values, and remain unchanged. @@ -1447,9 +1481,6 @@ pub enum Rvalue<'tcx> { /// This is needed because dataflow analysis needs to distinguish /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo` /// has a destructor. - /// - /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After - /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too. Aggregate(Box>, IndexVec>), /// A CopyForDeref is equivalent to a read from a place at the diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 1b45378c86c0a..0dc9e82db171b 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -445,6 +445,13 @@ macro_rules! make_mir_visitor { location ); } + StatementKind::StorageAlloc(local) => { + self.visit_local( + $(& $mutability)? *local, + PlaceContext::NonUse(NonUseContext::StorageAlloc), + location + ); + } StatementKind::StorageDead(local) => { self.visit_local( $(& $mutability)? *local, @@ -1365,6 +1372,8 @@ pub enum NonUseContext { StorageLive, /// Ending a storage live range. StorageDead, + /// Ensuring a local has allocated storage. + StorageAlloc, /// User type annotation assertions for NLL. AscribeUserTy(ty::Variance), /// The data of a user variable, for debug info. @@ -1464,7 +1473,11 @@ impl PlaceContext { match self { PlaceContext::MutatingUse(_) => ty::Invariant, PlaceContext::NonUse( - StorageDead | StorageLive | VarDebugInfo | BackwardIncompatibleDropHint, + StorageDead + | StorageLive + | StorageAlloc + | VarDebugInfo + | BackwardIncompatibleDropHint, ) => ty::Invariant, PlaceContext::NonMutatingUse( Inspect | Copy | Move | PlaceMention | SharedBorrow | FakeBorrow | RawBorrow diff --git a/compiler/rustc_mir_build/src/builder/cfg.rs b/compiler/rustc_mir_build/src/builder/cfg.rs index a8e68963ec724..8325acd3cb988 100644 --- a/compiler/rustc_mir_build/src/builder/cfg.rs +++ b/compiler/rustc_mir_build/src/builder/cfg.rs @@ -74,11 +74,7 @@ impl<'tcx> CFG<'tcx> { source_info, place, Rvalue::Use( - Operand::Constant(Box::new(ConstOperand { - span: source_info.span, - user_ty: None, - const_: Const::zero_sized(tcx.types.unit), - })), + Operand::zero_sized_constant(tcx.types.unit, source_info.span), WithRetag::Yes, ), ); diff --git a/compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs b/compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs index 5454a7bef67a7..0785f31e88466 100644 --- a/compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs +++ b/compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs @@ -18,6 +18,9 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> { @call(mir_storage_live, args) => { Ok(StatementKind::StorageLive(self.parse_local(args[0])?)) }, + @call(mir_storage_alloc, args) => { + Ok(StatementKind::StorageAlloc(self.parse_local(args[0])?)) + }, @call(mir_storage_dead, args) => { Ok(StatementKind::StorageDead(self.parse_local(args[0])?)) }, diff --git a/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs index 09c76d6beb4a1..da8cbb0dc9d97 100644 --- a/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs +++ b/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs @@ -348,11 +348,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { ExprKind::Assign { .. } | ExprKind::AssignOp { .. } => { block = this.stmt_expr(block, expr_id, None).into_block(); block.and(Rvalue::Use( - Operand::Constant(Box::new(ConstOperand { - span: expr_span, - user_ty: None, - const_: Const::zero_sized(this.tcx.types.unit), - })), + Operand::zero_sized_constant(this.tcx.types.unit, expr_span), WithRetag::Yes, )) } diff --git a/compiler/rustc_mir_build/src/builder/expr/into.rs b/compiler/rustc_mir_build/src/builder/expr/into.rs index 6ca12032362be..4855bf8b91c12 100644 --- a/compiler/rustc_mir_build/src/builder/expr/into.rs +++ b/compiler/rustc_mir_build/src/builder/expr/into.rs @@ -409,8 +409,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { sym::write_via_move => { // `write_via_move(ptr, val)` becomes `*ptr = val` but without any dropping. - // The destination must have unit type (so we don't actually have to store anything - // into it). + // The destination must have unit type. assert!(destination.ty(&this.local_decls, this.tcx).ty.is_unit()); // Compile this to an assignment of the argument into the destination. @@ -422,7 +421,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { span_bug!(expr_span, "invalid write_via_move call") }; let ptr_deref = ptr.project_deeper(&[ProjectionElem::Deref], this.tcx); - this.expr_into_dest(ptr_deref, block, val) + unpack!(block = this.expr_into_dest(ptr_deref, block, val)); + this.cfg.push_assign_unit(block, source_info, destination, this.tcx); + block.unit() } sym::write_box_via_move => { // The signature is: diff --git a/compiler/rustc_mir_dataflow/src/framework/direction.rs b/compiler/rustc_mir_dataflow/src/framework/direction.rs index 39bd2be9409a3..bb0685cb7ff08 100644 --- a/compiler/rustc_mir_dataflow/src/framework/direction.rs +++ b/compiler/rustc_mir_dataflow/src/framework/direction.rs @@ -127,6 +127,8 @@ impl Direction for Backward { analysis.apply_primary_statement_effect(state, stmt, loc); vis.visit_after_primary_statement_effect(state, stmt, loc); } + + vis.visit_block_exit(state, block); } } @@ -242,5 +244,7 @@ impl Direction for Forward { vis.visit_after_early_terminator_effect(state, term, loc); analysis.apply_primary_terminator_effect(state, term, loc); vis.visit_after_primary_terminator_effect(state, term, loc); + + vis.visit_block_exit(state, block); } } diff --git a/compiler/rustc_mir_dataflow/src/framework/visitor.rs b/compiler/rustc_mir_dataflow/src/framework/visitor.rs index e4b840a73e502..5b8a3374e04d9 100644 --- a/compiler/rustc_mir_dataflow/src/framework/visitor.rs +++ b/compiler/rustc_mir_dataflow/src/framework/visitor.rs @@ -34,6 +34,13 @@ pub trait ResultsVisitor<'tcx, A> where A: Analysis<'tcx>, { + /// Called after all effects in a block have been applied in the direction + /// of the analysis. + /// + /// In a forwards analysis, `state` is from the block's end. In a backwards + /// analysis, `state` is from the block's start. + fn visit_block_exit(&mut self, _state: &A::Domain, _block: BasicBlock) {} + /// Called after the "early" effect of the given statement is applied to `state`. fn visit_after_early_statement_effect( &mut self, diff --git a/compiler/rustc_mir_dataflow/src/impls/liveness.rs b/compiler/rustc_mir_dataflow/src/impls/liveness.rs index 2787365c17be1..1bc4997f8d72c 100644 --- a/compiler/rustc_mir_dataflow/src/impls/liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/liveness.rs @@ -239,6 +239,9 @@ impl<'a> MaybeTransitiveLiveLocals<'a> { StatementKind::SetDiscriminant { place, .. } => { (!debuginfo_locals.contains(place.local)).then_some(**place) } + StatementKind::StorageAlloc(local) => { + (!debuginfo_locals.contains(*local)).then_some((*local).into()) + } StatementKind::FakeRead(_) | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) diff --git a/compiler/rustc_mir_dataflow/src/impls/mod.rs b/compiler/rustc_mir_dataflow/src/impls/mod.rs index 1e12e41ce1fb4..495858c776b09 100644 --- a/compiler/rustc_mir_dataflow/src/impls/mod.rs +++ b/compiler/rustc_mir_dataflow/src/impls/mod.rs @@ -1,6 +1,7 @@ mod borrowed_locals; mod initialized; mod liveness; +mod precise_liveness; mod storage_liveness; pub use self::borrowed_locals::{MaybeBorrowedLocals, borrowed_locals}; @@ -11,6 +12,9 @@ pub use self::initialized::{ pub use self::liveness::{ DefUse, LivenessTransferFunction, MaybeLiveLocals, MaybeTransitiveLiveLocals, }; +pub use self::precise_liveness::{ + SplitPointEffect, SplitPointIndex, dump_liveness_matrix, liveness_matrix, +}; pub use self::storage_liveness::{ MaybeRequiresStorage, MaybeStorageDead, MaybeStorageLive, always_storage_live_locals, }; diff --git a/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs new file mode 100644 index 0000000000000..d39a87c26210e --- /dev/null +++ b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs @@ -0,0 +1,592 @@ +//! Computes the points where each local must have a distinct allocation. +//! +//! The result is a [`SparseIntervalMatrix`] with one row per local. Two locals +//! may share the same address only if their rows are disjoint. To model MIR +//! statements where a source operand and destination place may share an +//! address, each statement and terminator is split into an early point, where +//! operands are read, and a late point, where destinations are written. +//! +//! A local live range starts at the late point of any statement or terminator +//! that writes to it without a `Deref` projection. It ends at `StorageLive` or +//! `StorageDead`, or at the early point of a whole-local move operand or the +//! last use of that local on a control-flow path (only for locals whose address +//! is never observed). This happens at `StorageLive` because it resets a local +//! to an uninitialized state. +//! +//! `Call` terminators are handled specially: move operands are kept live +//! through the late point of the terminator so they conflict with each other +//! and with the destination place. This matches the runtime behavior where the +//! place is donated to the callee for the duration of the call. + +use rustc_index::IndexVec; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::interval::SparseIntervalMatrix; +use rustc_middle::mir::visit::{ + MutatingUseContext, NonMutatingUseContext, NonUseContext, PlaceContext, VisitPlacesWith, + Visitor, +}; +use rustc_middle::mir::{self, BasicBlock, Local, Location, MirDumper, PassWhere, Place}; +use rustc_middle::ty::TyCtxt; +use tracing::trace; + +use crate::impls::{DefUse, MaybeLiveLocals, borrowed_locals}; +use crate::points::{DenseLocationMap, PointIndex}; +use crate::{Analysis, GenKill, ResultsVisitor, visit_results}; + +//////////////////////////////////////////////////////////////////////////////// +// Backward dataflow pass +// +// This pass computes "kill points" for each local, indicating the location of +// their last use in a particular control flow branch. These are later used in +// the forward pass later to end the live range of locals that are never +// borrowed at their last direct use. +// +// Borrowed locals are treated as always live by this pass since those need to +// remain allocated until `StorageLive`, `StorageDead`, or a whole-local move. +// +// This pass has 2 outputs: a set of kill points that mark the last use +// locations of locals and a per-block bitset indicating which locals are live +// on entry to that block. + +struct KillPoints<'a> { + live_on_entry: IndexVec>, + kill_points_map: IndexVec, +} + +impl<'a> KillPoints<'a> { + fn compute<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + pass_name: Option<&'static str>, + points: &DenseLocationMap, + kill_points: &'a mut Vec<(Local, Location)>, + ) -> Self { + let maybe_live_locals = MaybeLiveLocals.iterate_to_fixpoint(tcx, body, pass_name); + let borrowed_locals = borrowed_locals(body); + + // Initialize all borrowed locals as live on entry. We never try to kill + // those. + let mut live_on_entry = + IndexVec::from_elem_n(borrowed_locals.clone(), body.basic_blocks.len()); + + // Collect kill points and live-on-entry states from the results of + // MaybeLiveLocals. + kill_points.clear(); + let mut visitor = KillPointsVisitor { + kill_points, + live_on_entry: &mut live_on_entry, + borrowed_locals: &borrowed_locals, + }; + visit_results( + body, + mir::traversal::reachable(body).map(|(block, _)| block), + &maybe_live_locals, + &mut visitor, + ); + trace!(?kill_points); + trace!(?live_on_entry); + + // Create a mapping of `PointIndex` to the set of killed locals at that + // location. + let mut kill_points_map = IndexVec::from_elem_n(&[][..], points.num_points()); + for chunk in kill_points.chunk_by(|a, b| a.1 == b.1) { + let point = points.point_from_location(chunk[0].1); + trace!("Kill points at {:?}: {:?}", chunk[0].1, chunk); + kill_points_map[point] = chunk; + } + + Self { live_on_entry, kill_points_map } + } +} + +struct KillPointsVisitor<'a> { + kill_points: &'a mut Vec<(Local, Location)>, + live_on_entry: &'a mut IndexVec>, + borrowed_locals: &'a DenseBitSet, +} + +impl<'tcx> ResultsVisitor<'tcx, MaybeLiveLocals> for KillPointsVisitor<'_> { + fn visit_block_exit(&mut self, state: &DenseBitSet, block: BasicBlock) { + // Borrowed locals are already marked as live when live_on_entry was + // initialized. This adds the non-borrowed locals that we have + // determined are live on entry to this block. + self.live_on_entry[block].union(state); + } + + fn visit_after_early_statement_effect( + &mut self, + state: &DenseBitSet, + statement: &mir::Statement<'tcx>, + location: Location, + ) { + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // Ignore non-uses. + match ctxt { + PlaceContext::NonMutatingUse(_) | PlaceContext::MutatingUse(_) => {} + // We later specifically start a live range at StorageAlloc. + // Emit a kill point if it is never used after this. + PlaceContext::NonUse(NonUseContext::StorageAlloc) => {} + PlaceContext::NonUse(_) => return, + } + + // If a local is used in a statement but is dead after it then this + // location is a kill point. Don't emit a kill point for borrowed + // locals. + if !state.contains(place.local) && !self.borrowed_locals.contains(place.local) { + self.kill_points.push((place.local, location)); + } + }) + .visit_statement(statement, location); + } + + fn visit_after_early_terminator_effect( + &mut self, + state: &DenseBitSet, + terminator: &mir::Terminator<'tcx>, + location: Location, + ) { + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // Ignore non-uses (they don't do anything) and edge uses + // (implicitly killed though live_on_entry at the start of the + // corresponding successor). + match ctxt { + PlaceContext::MutatingUse( + MutatingUseContext::AsmOutput + | MutatingUseContext::Call + | MutatingUseContext::Yield, + ) + | PlaceContext::NonUse(_) => return, + PlaceContext::NonMutatingUse(_) | PlaceContext::MutatingUse(_) => {} + } + + // If a local is used in a terminator but is dead after it then this + // location is a kill point. Don't emit a kill point for borrowed + // locals. + if !state.contains(place.local) && !self.borrowed_locals.contains(place.local) { + self.kill_points.push((place.local, location)); + } + }) + .visit_terminator(terminator, location); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Forward dataflow pass + +struct PreciseLiveness<'a> { + kill_points: &'a KillPoints<'a>, + points: &'a DenseLocationMap, +} + +impl PreciseLiveness<'_> { + fn apply_block_start_effect(&self, state: &mut DenseBitSet, block: BasicBlock) { + // Notably this kills any dead results produced by a predecessor's + // terminator. + state.intersect(&self.kill_points.live_on_entry[block]); + } +} + +impl<'tcx> Analysis<'tcx> for PreciseLiveness<'_> { + type Domain = DenseBitSet; + + const NAME: &'static str = "precise_liveness"; + + fn bottom_value(&self, body: &mir::Body<'tcx>) -> DenseBitSet { + DenseBitSet::new_empty(body.local_decls.len()) + } + + fn initialize_start_block(&self, body: &mir::Body<'tcx>, state: &mut DenseBitSet) { + // Function arguments start out as live. + for arg in body.args_iter() { + state.gen_(arg); + } + } + + fn apply_primary_statement_effect( + &self, + state: &mut DenseBitSet, + statement: &mir::Statement<'tcx>, + location: Location, + ) { + if location.statement_index == 0 { + self.apply_block_start_effect(state, location.block); + } + + // StorageLive and StorageDead free the old allocation, even if it has + // been borrowed. + if let mir::StatementKind::StorageLive(local) | mir::StatementKind::StorageDead(local) = + statement.kind + { + state.kill(local); + return; + } + + // Kill moved operands if the whole local was moved. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + if ctxt == PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) { + if let Some(local) = place.as_local() { + state.kill(local); + } + } + }) + .visit_statement(statement, location); + + // Gen destination places. + if let mir::StatementKind::StorageAlloc(local) = statement.kind { + state.gen_(local); + } else { + VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => state.gen_(place.local), + DefUse::Use | DefUse::NonUse => {} + }) + .visit_statement(statement, location); + } + + // Apply kill points at this statement: if a variable is dead then it + // doesn't need storage. + let point = self.points.point_from_location(location); + for &(local, _) in self.kill_points.kill_points_map[point] { + state.kill(local); + } + } + + fn apply_primary_terminator_effect( + &self, + state: &mut DenseBitSet, + terminator: &mir::Terminator<'tcx>, + location: Location, + ) { + if location.statement_index == 0 { + self.apply_block_start_effect(state, location.block); + } + + // Kill moved operands if the whole local was moved. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) = ctxt { + if let Some(local) = place.as_local() { + state.kill(local); + } + } + }) + .visit_terminator(terminator, location); + + // Gen destination places. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // The resume place of a Yield is only evaluated after the coroutine + // is resumed. This is in contrast to call destinations and inline + // asm outputs which are evaluated before the call and therefore + // remain allocated on both normal and unwind edges. + if ctxt == PlaceContext::MutatingUse(MutatingUseContext::Yield) { + return; + } + + match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => state.gen_(place.local), + DefUse::Use | DefUse::NonUse => {} + } + }) + .visit_terminator(terminator, location); + } + + fn apply_call_return_effect( + &self, + state: &mut DenseBitSet, + _block: BasicBlock, + return_places: mir::CallReturnPlaces<'_, 'tcx>, + ) { + if let mir::CallReturnPlaces::Yield(place) = return_places + && !place.is_indirect() + { + state.gen_(place.local); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Matrix construction + +/// Different "phases" of a single MIR statement, used to describe how +/// overlapping operands are handled. +/// +/// As a general rule, source operands are read in the `Early` phase and +/// destination places are written in the `Late` phase. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum SplitPointEffect { + Early = 0, + Late = 1, +} + +rustc_index::newtype_index! { + /// A `PointIndex` with the lower bit encoding early/late inside a + /// statement. + /// + /// This is used to model overlap constraints within a MIR statement: if a + /// source/destination are allowed to overlap then the source is read in + /// `SplitPointEffect::Early` and the write is done in + /// `SplitPointEffect::Late`. + #[orderable] + #[debug_format = "SplitPointIndex({})"] + pub struct SplitPointIndex {} +} + +impl SplitPointIndex { + pub fn new(point: PointIndex, effect: SplitPointEffect) -> SplitPointIndex { + let index = (point.as_u32() << 1) | (effect as u32); + SplitPointIndex::from_u32(index) + } + + pub fn point(self) -> PointIndex { + PointIndex::from_u32(self.as_u32() >> 1) + } + + pub fn effect(self) -> SplitPointEffect { + match self.as_u32() & 1 { + 0 => SplitPointEffect::Early, + 1 => SplitPointEffect::Late, + _ => unreachable!(), + } + } +} + +/// Helper type to construct a `SparseIntervalMatrix`. +struct MatrixBuilder { + matrix: SparseIntervalMatrix, + range_start: IndexVec>, + + // Track locals that have been live at any point in a block so that at the + // end of a block we don't need to iterate over all locals. This + // significantly speeds up matrix building. + maybe_live_locals: Vec, +} + +impl MatrixBuilder { + fn gen_(&mut self, local: Local, point: PointIndex, effect: SplitPointEffect) { + let split_point = SplitPointIndex::new(point, effect); + + // No-op if the local is already live. + if self.range_start[local].is_none() { + self.range_start[local] = Some(split_point); + self.maybe_live_locals.push(local); + } + } + + fn kill(&mut self, local: Local, point: PointIndex, effect: SplitPointEffect) { + let end = SplitPointIndex::new(point, effect); + + // No-op if the local is already dead. + if let Some(start) = self.range_start[local].take() { + debug_assert!(end >= start); + self.matrix.append_range(local, start..=end); + } + } + + fn kill_all(&mut self, point: PointIndex, effect: SplitPointEffect) { + while let Some(local) = self.maybe_live_locals.pop() { + self.kill(local, point, effect); + } + } + + fn kill_all_except(&mut self, except: Local, point: PointIndex, effect: SplitPointEffect) { + while let Some(local) = self.maybe_live_locals.pop() { + if local != except { + self.kill(local, point, effect); + } + } + self.maybe_live_locals.push(except); + } +} + +pub fn liveness_matrix<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + points: &DenseLocationMap, + pass_name: Option<&'static str>, +) -> SparseIntervalMatrix { + let mut kill_points_vec = vec![]; + let kill_points = KillPoints::compute(tcx, body, pass_name, points, &mut kill_points_vec); + let mut results = PreciseLiveness { kill_points: &kill_points, points } + .iterate_to_fixpoint(tcx, body, pass_name); + + let mut builder = MatrixBuilder { + matrix: SparseIntervalMatrix::new(points.num_points() * 2), + range_start: IndexVec::from_elem_n(None, body.local_decls.len()), + maybe_live_locals: Vec::new(), + }; + for (block, block_data) in body.basic_blocks.iter_enumerated() { + // We can mutate the state in-place since we're not using it any more + // after this point. + let state = &mut results.entry_states[block]; + + // Notably this kills any dead results produced by a predecessor's + // terminator. + state.intersect(&kill_points.live_on_entry[block]); + + // Gen any locals that are live at the start of the block. If this block + // only consists of a return terminator then instead of gen the return + // place. This ensures that StorageDead for all other locals are + // inserted before the return terminator. + let terminator = block_data.terminator(); + if let mir::TerminatorKind::Return = terminator.kind + && block_data.statements.is_empty() + { + if state.contains(mir::RETURN_PLACE) { + builder.gen_(mir::RETURN_PLACE, points.entry_point(block), SplitPointEffect::Early); + } + } else { + for local in state.iter() { + builder.gen_(local, points.entry_point(block), SplitPointEffect::Early); + } + } + + for (statement_index, statement) in block_data.statements.iter().enumerate() { + let location = Location { block, statement_index }; + let point = points.point_from_location(location); + + // StorageLive and StorageDead free the old allocation, even if it + // has been borrowed. + if let mir::StatementKind::StorageLive(local) | mir::StatementKind::StorageDead(local) = + statement.kind + { + builder.kill(local, point, SplitPointEffect::Late); + continue; + } + + // Kill moved operands if the whole local was moved. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + if ctxt == PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) { + if let Some(local) = place.as_local() { + builder.kill(local, point, SplitPointEffect::Early); + } + } + }) + .visit_statement(statement, location); + + // Kill any locals which are no longer used after this statement. + for &(local, _) in kill_points.kill_points_map[point] { + builder.kill(local, point, SplitPointEffect::Early); + } + + // Gen destination places. + if let mir::StatementKind::StorageAlloc(local) = statement.kind { + builder.gen_(local, point, SplitPointEffect::Late); + } else { + VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => { + builder.gen_(place.local, point, SplitPointEffect::Late) + } + DefUse::Use | DefUse::NonUse => {} + }) + .visit_statement(statement, location); + } + + // Kill any dead destination places: they will only appear at the + // late point of the statement they are generated in, which is + // sufficient for determining overlap. + for &(local, _) in kill_points.kill_points_map[point] { + builder.kill(local, point, SplitPointEffect::Late); + } + } + + // If this block ends in a return terminator, end all live ranges before + // the terminator so that StorageDead statements are inserted before it. + // + // This is useful after inlining so that the lifetime of locals in the + // inlined callee don't extend past the call in the callee. + if let mir::TerminatorKind::Return = terminator.kind + && !block_data.statements.is_empty() + { + // Blocks with only a return terminator are handled above. + let location = Location { block, statement_index: block_data.statements.len() - 1 }; + let point = points.point_from_location(location); + builder.kill_all_except(mir::RETURN_PLACE, point, SplitPointEffect::Late); + } + + let location = Location { block, statement_index: block_data.statements.len() }; + let point = points.point_from_location(location); + + // Kill moved operands if the whole local was moved. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) = ctxt { + if let Some(local) = place.as_local() { + builder.kill(local, point, SplitPointEffect::Early); + } + } + }) + .visit_terminator(terminator, location); + + // Kill any locals which are no longer used after this terminator. + for &(local, _) in kill_points.kill_points_map[point] { + builder.kill(local, point, SplitPointEffect::Early); + } + + // Gen destination places. + VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => { + builder.gen_(place.local, point, SplitPointEffect::Late) + } + DefUse::Use | DefUse::NonUse => {} + }) + .visit_terminator(terminator, location); + + // Move arguments to a call are treated specially: the place that they + // represent is passed directly to the callee, which means that they are + // not allowed to alias any other move operand or the destination place. + // This is represented here by extending their live range to the late + // part, making it overlap with that of the destination place. + // + // Notably, this *doesn't* apply to TailCall. + if let mir::TerminatorKind::Call { + func: _, + args, + destination: _, + target: _, + unwind: _, + call_source: _, + fn_span: _, + } = &terminator.kind + { + for arg in args { + if let mir::Operand::Move(place) = arg.node { + builder.gen_(place.local, point, SplitPointEffect::Late); + builder.kill(place.local, point, SplitPointEffect::Late); + } + } + } + + // End the lifetimes of all locals at the end of the block. Successor + // blocks (which may not be continuous in the index space!) will + // initialize the lifetimes again from their entry state. + builder.kill_all(point, SplitPointEffect::Late); + } + + builder.matrix +} + +pub fn dump_liveness_matrix<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + pass_name: &'static str, + points: &DenseLocationMap, + matrix: &SparseIntervalMatrix, +) { + let locals_live_at = |split_point| { + matrix.rows().filter(|&r| matrix.contains(r, split_point)).collect::>() + }; + + if let Some(dumper) = MirDumper::new(tcx, pass_name, body) { + let extra_data = &|pass_where, w: &mut dyn std::io::Write| { + if let PassWhere::BeforeLocation(loc) = pass_where { + let point = points.point_from_location(loc); + let split_point = SplitPointIndex::new(point, SplitPointEffect::Early); + let live = locals_live_at(split_point); + writeln!(w, " // {loc:?}-early => {live:?}")?; + let split_point = SplitPointIndex::new(point, SplitPointEffect::Late); + let live = locals_live_at(split_point); + writeln!(w, " // {loc:?}-late => {live:?}")?; + } + Ok(()) + }; + + dumper.set_extra_data(extra_data).dump_mir(body) + } +} diff --git a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs index 9f6d3ec6ed08e..0ac8f3b127822 100644 --- a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs @@ -4,6 +4,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; +use rustc_span::bug; use smallvec::SmallVec; use super::MaybeBorrowedLocals; @@ -192,6 +193,9 @@ impl<'tcx> Analysis<'tcx> for MaybeRequiresStorage { StatementKind::SetDiscriminant { place, .. } => { state.gen_(place.local); } + StatementKind::StorageAlloc(_) => { + bug!("StorageAlloc is not allowed before coroutine lowering") + } // Nothing to do for these. Match exhaustively so this fails to compile when new // variants are added. @@ -226,6 +230,9 @@ impl<'tcx> Analysis<'tcx> for MaybeRequiresStorage { StatementKind::SetDiscriminant { place, .. } => { state.gen_(place.local); } + StatementKind::StorageAlloc(_) => { + bug!("StorageAlloc is not allowed before coroutine lowering") + } StatementKind::StorageDead(_) | StatementKind::AscribeUserType(..) diff --git a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs index 3c1c631285f96..e18bb74669055 100644 --- a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs +++ b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs @@ -398,7 +398,7 @@ impl<'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> MoveDataBuilder<'a, 'tcx, F> { StatementKind::FakeRead((_, place)) => { self.create_move_path(*place); } - StatementKind::StorageLive(_) => {} + StatementKind::StorageAlloc(_) | StatementKind::StorageLive(_) => {} StatementKind::StorageDead(local) => { // DerefTemp locals (results of CopyForDeref) don't actually move anything. if !self.body.local_decls[*local].is_deref_temp() { diff --git a/compiler/rustc_mir_dataflow/src/points.rs b/compiler/rustc_mir_dataflow/src/points.rs index 8568f325f1306..a4e5a3a79f04a 100644 --- a/compiler/rustc_mir_dataflow/src/points.rs +++ b/compiler/rustc_mir_dataflow/src/points.rs @@ -61,6 +61,15 @@ impl DenseLocationMap { PointIndex::new(start_index) } + /// Returns the `PointIndex` for the terminator in the given `BasicBlock`. O(1). + #[inline] + pub fn terminator(&self, block: BasicBlock) -> PointIndex { + let next_block = BasicBlock::new(block.index() + 1); + let next_start_index = + *self.statements_before_block.get(next_block).unwrap_or(&self.num_points); + PointIndex::new(next_start_index - 1) + } + /// Return the PointIndex for the block start of this index. #[inline] pub fn to_block_start(&self, index: PointIndex) -> PointIndex { diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs index d22a5c5cc4eef..575562eb9d196 100644 --- a/compiler/rustc_mir_transform/src/copy_prop.rs +++ b/compiler/rustc_mir_transform/src/copy_prop.rs @@ -167,7 +167,8 @@ struct StorageChecker<'a, 'tcx> { impl<'a, 'tcx> Visitor<'tcx> for StorageChecker<'a, 'tcx> { fn visit_local(&mut self, local: Local, context: PlaceContext, loc: Location) { - if !context.is_use() { + // StorageAlloc requires the local to be live. + if !context.is_use() && context != PlaceContext::NonUse(NonUseContext::StorageAlloc) { return; } diff --git a/compiler/rustc_mir_transform/src/coroutine/drop.rs b/compiler/rustc_mir_transform/src/coroutine/drop.rs index 44ea6aef0e02d..f0ea94d1f4ba6 100644 --- a/compiler/rustc_mir_transform/src/coroutine/drop.rs +++ b/compiler/rustc_mir_transform/src/coroutine/drop.rs @@ -208,8 +208,18 @@ pub(super) fn create_coroutine_drop_shim<'tcx>( } } - // Replace the return variable + // Replace the return variable and initialize it once for all return paths. body.local_decls[RETURN_PLACE] = LocalDecl::with_source_info(tcx.types.unit, source_info); + body.basic_blocks_mut()[START_BLOCK].statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + Place::return_place(), + Rvalue::Use( + Operand::zero_sized_constant(tcx.types.unit, source_info.span), + WithRetag::Yes, + ), + ))), + )); make_coroutine_state_argument_indirect(tcx, &mut body); diff --git a/compiler/rustc_mir_transform/src/coroutine/mod.rs b/compiler/rustc_mir_transform/src/coroutine/mod.rs index 1e63dca78adbb..4b52ec03672cf 100644 --- a/compiler/rustc_mir_transform/src/coroutine/mod.rs +++ b/compiler/rustc_mir_transform/src/coroutine/mod.rs @@ -461,8 +461,10 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { #[tracing::instrument(level = "trace", skip(self, stmt), ret)] fn visit_statement(&mut self, stmt: &mut Statement<'tcx>, location: Location) { - // Remove StorageLive and StorageDead statements for remapped locals - if let StatementKind::StorageLive(l) | StatementKind::StorageDead(l) = stmt.kind + // Remove storage statements for remapped locals + if let StatementKind::StorageLive(l) + | StatementKind::StorageDead(l) + | StatementKind::StorageAlloc(l) = stmt.kind && self.remap.contains(l) { stmt.make_nop(true); @@ -760,11 +762,7 @@ fn return_poll_ready_assign<'tcx>(tcx: TyCtxt<'tcx>, source_info: SourceInfo) -> // Poll::Ready(()) let poll_def_id = tcx.require_lang_item(LangItem::Poll, source_info.span); let args = tcx.mk_args(&[tcx.types.unit.into()]); - let val = Operand::Constant(Box::new(ConstOperand { - span: source_info.span, - user_ty: None, - const_: Const::zero_sized(tcx.types.unit), - })); + let val = Operand::zero_sized_constant(tcx.types.unit, source_info.span); let ready_val = Rvalue::Aggregate( Box::new(AggregateKind::Adt(poll_def_id, VariantIdx::from_usize(0), args, None, None)), indexvec![val], @@ -935,6 +933,25 @@ fn create_coroutine_resume_function<'tcx>( } } + // Iterator::next has no resume argument, so initialize the unit resume + // local at the start of each invocation. + if let CoroutineKind::Desugared(CoroutineDesugaring::Gen, _) = transform.coroutine_kind { + let source_info = SourceInfo::outermost(body.span); + body.basic_blocks_mut()[START_BLOCK].statements.insert( + 0, + Statement::new( + source_info, + StatementKind::Assign(Box::new(( + CTX_ARG.into(), + Rvalue::Use( + Operand::zero_sized_constant(tcx.types.unit, source_info.span), + WithRetag::Yes, + ), + ))), + ), + ); + } + // Make sure we remove dead blocks to remove // unrelated code from the drop part of the function simplify::remove_dead_blocks(body); @@ -1300,6 +1317,7 @@ impl<'tcx> Visitor<'tcx> for EnsureCoroutineFieldAssignmentsNeverAlias<'_> { StatementKind::FakeRead(..) | StatementKind::SetDiscriminant { .. } + | StatementKind::StorageAlloc(_) | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::AscribeUserType(..) diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index 53d58439b0434..e9cbebd6891b6 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -136,9 +136,12 @@ struct CostChecker<'b, 'tcx> { impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) { - // Don't count StorageLive/StorageDead in the inlining cost. + // Don't count storage statements in the inlining cost. match statement.kind { - StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::Nop => {} + StatementKind::StorageAlloc(_) + | StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Nop => {} _ => self.statements += 1, } } diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index 21ea02c8ff601..62238d303cbd3 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -201,6 +201,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { | StatementKind::PlaceMention(..) | StatementKind::Coverage(..) | StatementKind::BackwardIncompatibleDropHint { .. } + | StatementKind::StorageAlloc(_) | StatementKind::AscribeUserType(..) => {} } } diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 7f9bba03ff345..5ea98a8fb92f5 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -155,7 +155,9 @@ pub(super) struct DestinationPropagation; impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { fn policy(&self, ctx: &crate::PassCtx<'_>) -> PassPolicy { - PassPolicy::optional(ctx.mir_opt_level() >= 2) + PassPolicy::optional( + ctx.mir_opt_level() >= 2 && !ctx.opts.unstable_opts.mir_move_elimination, + ) } #[tracing::instrument(level = "trace", skip(self, tcx, body))] diff --git a/compiler/rustc_mir_transform/src/elaborate_drop.rs b/compiler/rustc_mir_transform/src/elaborate_drop.rs index 9b81f55ab1145..178f0a88032f7 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drop.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drop.rs @@ -590,7 +590,7 @@ where unwind, vec![self.assign( entry_resume_local.into(), - Rvalue::Use(Operand::Move(resume_place), WithRetag::Yes), + Rvalue::Use(Operand::Copy(resume_place), WithRetag::Yes), )], TerminatorKind::Call { func: Operand::function_handle( diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 9547b5bb93dff..265895ded370b 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -881,6 +881,15 @@ fn inline_call<'tcx, I: Inliner<'tcx>>( None }; + if !destination.is_indirect() { + // With -Zmir-move-elimination we need to ensure the destination place + // is allocated by the time the callee returns, *even if it unwinds*. + caller_body[callsite.block].statements.push(Statement::new( + callsite.source_info, + StatementKind::StorageAlloc(destination.local), + )); + } + // If the call is something like `a[*i] = f(i)`, where // `i : &mut usize`, then just duplicating the `a[*i]` // Place could result in two different locations if `f` diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs index e02e29a2e23e0..ecc2b74a4a532 100644 --- a/compiler/rustc_mir_transform/src/instsimplify.rs +++ b/compiler/rustc_mir_transform/src/instsimplify.rs @@ -59,7 +59,7 @@ impl<'tcx> crate::MirPass<'tcx> for InstSimplify { ctx.simplify_primitive_clone(terminator, &mut block.statements); ctx.simplify_size_or_align_of_val(terminator, &mut block.statements); ctx.simplify_raw_eq(terminator, &mut block.statements); - ctx.simplify_intrinsic_assert(terminator); + ctx.simplify_intrinsic_assert(terminator, &mut block.statements); ctx.simplify_nounwind_call(terminator); simplify_duplicate_switch_targets(terminator); } @@ -397,9 +397,17 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { } } - fn simplify_intrinsic_assert(&self, terminator: &mut Terminator<'tcx>) { - let TerminatorKind::Call { ref func, target: ref mut target @ Some(target_block), .. } = - terminator.kind + fn simplify_intrinsic_assert( + &self, + terminator: &mut Terminator<'tcx>, + statements: &mut Vec>, + ) { + let TerminatorKind::Call { + ref func, + destination, + target: ref mut target @ Some(target_block), + .. + } = terminator.kind else { return; }; @@ -420,7 +428,21 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { *target = None; } Some(false) => { - // If we know the assert does not panic, turn the call into a Goto + // If we know the assert does not panic, turn the call into a Goto. We still need to + // initialize its unit result. + statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( + destination, + Rvalue::Use( + Operand::zero_sized_constant( + self.tcx.types.unit, + terminator.source_info.span, + ), + WithRetag::Yes, + ), + ))), + )); terminator.kind = TerminatorKind::Goto { target: target_block }; } } diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 05616da42505b..e2e46362e02db 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -405,6 +405,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { | StatementKind::ConstEvalCounter | StatementKind::PlaceMention(..) | StatementKind::BackwardIncompatibleDropHint { .. } + | StatementKind::StorageAlloc(_) | StatementKind::Nop => None, } } diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index d0b9c686c3037..697071a31429f 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -164,6 +164,7 @@ declare_passes! { mod lower_slice_len : LowerSliceLenCalls; mod match_branches : MatchBranchSimplification; mod mentioned_items : MentionedItems; + mod move_elimination : MoveElimination; mod multiple_return_terminators : MultipleReturnTerminators; mod post_drop_elaboration : CheckLiveDrops; mod prettify : ReorderBasicBlocks, ReorderLocals; @@ -763,6 +764,7 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' &dead_store_elimination::DeadStoreElimination::Final, &dest_prop::DestinationPropagation, &tail_copy_to_move::TailCopyToMove, + &move_elimination::MoveElimination, &simplify::SimplifyLocals::Final, &multiple_return_terminators::MultipleReturnTerminators, // Some cleanup necessary at least for LLVM and potentially other codegen backends. diff --git a/compiler/rustc_mir_transform/src/liveness.rs b/compiler/rustc_mir_transform/src/liveness.rs index 7725daa276cc6..a764e81b195dc 100644 --- a/compiler/rustc_mir_transform/src/liveness.rs +++ b/compiler/rustc_mir_transform/src/liveness.rs @@ -743,7 +743,8 @@ impl<'a, 'tcx> AssignmentResult<'a, 'tcx> { live, ); } - StatementKind::StorageLive(_) + StatementKind::StorageAlloc(_) + | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::Coverage(_) | StatementKind::Intrinsic(_) @@ -1561,6 +1562,7 @@ impl DefUse { PlaceContext::NonUse( NonUseContext::StorageLive + | NonUseContext::StorageAlloc | NonUseContext::StorageDead | NonUseContext::AscribeUserTy(_) | NonUseContext::BackwardIncompatibleDropHint diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index c0f906330d8af..62947dea6598d 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -47,11 +47,10 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { StatementKind::Assign(Box::new(( *destination, Rvalue::Use( - Operand::Constant(Box::new(ConstOperand { - span: terminator.source_info.span, - user_ty: None, - const_: Const::zero_sized(tcx.types.unit), - })), + Operand::zero_sized_constant( + tcx.types.unit, + terminator.source_info.span, + ), WithRetag::Yes, ), ))), @@ -61,20 +60,31 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { sym::copy_nonoverlapping => { let target = target.unwrap(); let Ok([src, dst, count]) = take_array(args) else { - bug!("Wrong arguments for copy_non_overlapping intrinsic"); + bug!("Wrong arguments for copy_nonoverlapping intrinsic"); }; block.statements.push(Statement::new( terminator.source_info, StatementKind::Intrinsic(Box::new( - NonDivergingIntrinsic::CopyNonOverlapping( - rustc_middle::mir::CopyNonOverlapping { - src: src.node, - dst: dst.node, - count: count.node, - }, - ), + NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping { + src: src.node, + dst: dst.node, + count: count.node, + }), )), )); + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( + *destination, + Rvalue::Use( + Operand::zero_sized_constant( + tcx.types.unit, + terminator.source_info.span, + ), + WithRetag::Yes, + ), + ))), + )); terminator.kind = TerminatorKind::Goto { target }; } sym::assume => { @@ -88,6 +98,19 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { arg.node, ))), )); + block.statements.push(Statement::new( + terminator.source_info, + StatementKind::Assign(Box::new(( + *destination, + Rvalue::Use( + Operand::zero_sized_constant( + tcx.types.unit, + terminator.source_info.span, + ), + WithRetag::Yes, + ), + ))), + )); terminator.kind = TerminatorKind::Goto { target }; } sym::wrapping_add diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs index 48a0caa4979bb..c06f6a89a434a 100644 --- a/compiler/rustc_mir_transform/src/match_branches.rs +++ b/compiler/rustc_mir_transform/src/match_branches.rs @@ -65,9 +65,10 @@ impl<'tcx, 'a> SimplifyMatch<'tcx, 'a> { otherwise: Option<&ConstOperand<'tcx>>, ) -> Option> { let (_, first_const, mut others) = split_first_case(consts, otherwise); - let first_scalar_int = first_const.const_.try_eval_scalar_int(self.tcx, self.typing_env)?; + let first_value = + first_const.const_.eval(self.tcx, self.typing_env, first_const.span).ok()?; if others.all(|const_| { - const_.const_.try_eval_scalar_int(self.tcx, self.typing_env) == Some(first_scalar_int) + const_.const_.eval(self.tcx, self.typing_env, const_.span).ok() == Some(first_value) }) { Some(StatementKind::Assign(Box::new(( dest, diff --git a/compiler/rustc_mir_transform/src/move_elimination.rs b/compiler/rustc_mir_transform/src/move_elimination.rs new file mode 100644 index 0000000000000..f40c4f48fe395 --- /dev/null +++ b/compiler/rustc_mir_transform/src/move_elimination.rs @@ -0,0 +1,1258 @@ +//! Eliminates copies and moves by unifying MIR places whose allocation ranges +//! are disjoint. +//! +//! See RFC 3943 for the local-lifetime semantics that make this optimization +//! possible. +//! +//! # Motivation +//! +//! MIR building can insert a lot of redundant copies, and Rust code in general +//! often tends to move values around a lot. The result is a lot of assignments +//! of the form `dest = {move} src;` in MIR. MIR building for constants in +//! particular tends to create additional locals that are only used inside a +//! single block to shuffle a value around unnecessarily. +//! +//! Additionally, Rust constructs nested aggregates by repeatedly moving values +//! into fields. For example, a function may build an inner value in a local, +//! move it into an outer aggregate, then move that aggregate into the caller's +//! destination. If these intermediate source and destination places have +//! different addresses, each layer needs an actual copy or move of the bytes. +//! +//! LLVM cannot remove these copies when both the source and destination +//! addresses are observed because merging the allocations would be an +//! observable change: the program could see that two addresses which were +//! previously distinct have become the same. This pass removes the copies +//! earlier, while MIR still has the information needed to prove that the two +//! allocation ranges do not overlap. +//! +//! # Optimization +//! +//! The basis of this optimization is place unification. If the source and +//! destination of an assignment have the same address, then the assignment is a +//! no-op. The same idea applies to aggregate construction: if a field operand +//! is already located at the corresponding field of the destination, then the +//! aggregate assignment does not need to copy those fields. +//! +//! The pass represents each unification as a mapping from a local to the place +//! that should replace it. Mappings are transitive, so `_3` can be resolved +//! through `_2.1` to `_1.0.1` if earlier mappings established those +//! relationships. +//! +//! The mapping is built by scanning the MIR for assignment statements. For +//! simple `Use` assignments, it tries to unify the source and destination +//! places. For `Aggregate` assignments, it tries to map each field operand to +//! the corresponding field in the assignment destination. Once all mappings +//! have been chosen, they are applied with one rewrite pass over the body. +//! +//! # Constraints +//! +//! Adding a mapping must preserve these conditions: +//! +//! * At least one side of the candidate pair must be a bare local. The pass can +//! map a local to a place with projections, but it cannot map between two +//! places that both already have projections. +//! +//! * Any projections in the mapped place must be stable everywhere the local is +//! used. `Deref` and `Index` projections are rejected because they may refer +//! to different memory at different points in the function. +//! +//! * The allocation ranges of the source and destination places must not +//! overlap. This is checked using `PreciseLiveness`, which computes the +//! points where each local must have a distinct allocation. The non-overlap +//! proof is required so that the operational semantics can allow both places +//! to have the same address. +//! +//! * Special-use locals such as arguments and the return place must keep their +//! roles. Temps may be mapped into an argument or return place, but two +//! special-use locals are not mapped into each other. +//! +//! * Some locals are used in contexts where projections cannot be added, such +//! as `Index` projections. These locals may only be replaced by another bare +//! local. +//! +//! # Storage reconstruction +//! +//! The original `StorageLive` and `StorageDead` statements no longer describe +//! the merged liveness produced by unification, so they are removed and rebuilt +//! from the liveness matrix when lifetime markers are emitted. This is done for +//! all locals, even ones that have not been merged, which has the additional +//! benefit of tightening the storage lifetime passed to LLVM. +//! +//! # Aliasing fixup +//! +//! MIR assignments currently require source and destination places not to +//! overlap for types that are not treated as scalars in codegen. After local +//! unification, some assignments may violate that invariant, so a final phase +//! rewrites them into a form codegen can handle. For each assignment: +//! +//! * Self-assignments, such as `_1 = _1`, are deleted. +//! +//! * Simple `Use` assignments whose source and destination overlap but are not +//! identical are routed through a temporary: the source is read into the +//! temporary first, and the temporary is then moved into the destination. +//! +//! * Aggregate assignments with any operand that aliases the destination are +//! decomposed into per-field assignments. Field self-assignments are dropped. +//! Other aliasing fields are read into temporaries first, then all +//! destination fields are written. For enum aggregates the discriminant is +//! set after fields are written. +//! +//! * Other rvalues, such as `Repeat` and `Cast`, are hoisted into a temporary +//! if any place they access aliases the destination. +//! +//! * Rvalues that operate only on scalar types, such as binary and unary ops, +//! `Discriminant`, `Ref`, and `RawPtr`, are left untouched because their +//! codegen does not rely on the no-aliasing assumption. + +use rustc_abi::{ExternAbi, FieldIdx, VariantIdx}; +use rustc_const_eval::util::most_packed_projection; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::thin_vec::ThinVec; +use rustc_index::IndexVec; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::interval::SparseIntervalMatrix; +use rustc_middle::mir::visit::{MutVisitor, NonUseContext, PlaceContext, VisitPlacesWith, Visitor}; +use rustc_middle::mir::*; +use rustc_middle::ty::{Ty, TyCtxt}; +use rustc_mir_dataflow::impls::{ + DefUse, SplitPointEffect, SplitPointIndex, dump_liveness_matrix, liveness_matrix, +}; +use rustc_mir_dataflow::points::DenseLocationMap; +use rustc_mir_dataflow::{Analysis, Backward, GenKill, ResultsVisitor, visit_results}; +use tracing::{debug, trace}; + +use crate::PassPolicy; +use crate::patch::MirPatch; + +pub(super) struct MoveElimination; + +impl<'tcx> crate::MirPass<'tcx> for MoveElimination { + fn policy(&self, ctx: &crate::PassCtx<'_>) -> PassPolicy { + PassPolicy::optional( + ctx.mir_opt_level() >= 2 && ctx.opts.unstable_opts.mir_move_elimination, + ) + } + + #[tracing::instrument(level = "trace", skip(self, tcx, body))] + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let def_id = body.source.def_id(); + trace!(?def_id); + + let points = DenseLocationMap::new(body); + let mut liveness_matrix = + liveness_matrix(tcx, body, &points, Some("MoveElimination.liveness")); + + dump_liveness_matrix(tcx, body, "MoveElimination.pre-liveness", &points, &liveness_matrix); + + let unprojectable_locals = UnprojectableLocals::find(body); + trace!(?unprojectable_locals); + + let rust_call_tuples = find_rust_call_tuples(tcx, body); + trace!(?rust_call_tuples); + + let remapped_locals = PlaceUnification::run( + tcx, + body, + &mut liveness_matrix, + unprojectable_locals, + rust_call_tuples, + ); + + apply_mappings(tcx, body, &remapped_locals); + + dump_liveness_matrix(tcx, body, "MoveElimination.post-liveness", &points, &liveness_matrix); + + if tcx.sess.emit_lifetime_markers() { + reconstruct_storage(tcx, body, &points, &liveness_matrix); + } + + apply_alias_fixup(tcx, body); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Unprojectable locals + +/// Set of locals which can only be replaced with another local, instead of +/// an arbitrary place. This is usually because it is used directly as a +/// `Local` outside of a place (e.g. `Index` projections). +#[derive(Debug)] +struct UnprojectableLocals { + locals: DenseBitSet, +} + +impl UnprojectableLocals { + fn find(body: &Body<'_>) -> DenseBitSet { + let mut out = Self { locals: DenseBitSet::new_empty(body.local_decls.len()) }; + + // Arguments and return places have fixed roles and cannot be replaced + // with projected locals. + out.locals.insert(RETURN_PLACE); + for arg in body.args_iter() { + out.locals.insert(arg); + } + + out.visit_body(body); + out.locals + } +} + +impl<'tcx> Visitor<'tcx> for UnprojectableLocals { + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) { + // We can't add more projections before a first position Deref projection. + if place.is_indirect() { + trace!( + "unprojectable local {:?} due to use as deref base at {location:?}", + place.local + ); + self.locals.insert(place.local); + } + + // Only call visit_local for projections, not the base local. + self.visit_projection(place.as_ref(), context, location); + } + + fn visit_local(&mut self, local: Local, context: PlaceContext, location: Location) { + // Storage markers are removed, and StorageAlloc is rewritten explicitly. + if let PlaceContext::NonUse( + NonUseContext::StorageLive | NonUseContext::StorageDead | NonUseContext::StorageAlloc, + ) = context + { + return; + } + + // If this is reached, it means that this is a bare local used outside + // of a place, which means it cannot be replaced with a projection of + // another local. + trace!("unprojectable local {local:?} at {location:?} ({context:?})"); + self.locals.insert(local); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// "rust-call" tuple handling + +/// Search for tuple locals passed to calls using the "rust-call" ABI. +/// +/// For rust-call ABI calls, caller-side MIR passes the logical arguments as a +/// tuple operand. We want to avoid remapping other locals into fields of that +/// tuple, especially if one of those locals is borrowed. +/// +/// Since the tuple itself is never borrowed, it is trivial for LLVM alias +/// analysis to see that accesses to one argument do not affect the others, but +/// merging the arguments into tuple fields from the start can hide that +/// independence. +fn find_rust_call_tuples<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> DenseBitSet { + let mut rust_call_tuples = DenseBitSet::new_empty(body.local_decls.len()); + + for block in body.basic_blocks.iter() { + let terminator = block.terminator(); + let (func, args) = match &terminator.kind { + TerminatorKind::Call { func, args, .. } + | TerminatorKind::TailCall { func, args, .. } => (func, args), + _ => continue, + }; + + let sig = func.ty(&body.local_decls, tcx).fn_sig(tcx); + if sig.abi() != ExternAbi::RustCall { + continue; + } + + let arg_tuple = args.last().expect("rust-call ABI requires a tuple argument"); + let (Operand::Copy(place) | Operand::Move(place)) = arg_tuple.node else { + continue; + }; + if let Some(local) = place.as_local() { + rust_call_tuples.insert(local); + } + } + + rust_call_tuples +} + +//////////////////////////////////////////////////////////////////////////////// +// Local unification + +struct PlaceUnification<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + body: &'a Body<'tcx>, + liveness_matrix: &'a mut SparseIntervalMatrix, + unprojectable_locals: DenseBitSet, + rust_call_tuples: DenseBitSet, + remapped_locals: IndexVec>>, +} + +impl<'tcx> PlaceUnification<'_, 'tcx> { + fn run( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + liveness_matrix: &mut SparseIntervalMatrix, + unprojectable_locals: DenseBitSet, + rust_call_tuples: DenseBitSet, + ) -> IndexVec>> { + let mut visitor = PlaceUnification { + tcx, + body, + liveness_matrix, + unprojectable_locals, + rust_call_tuples, + remapped_locals: IndexVec::from_elem_n(None, body.local_decls.len()), + }; + visitor.visit_body(body); + + // Finalize the mappings by transitively resolving all locals to their + // new final place. + for local in visitor.remapped_locals.indices() { + if let Some(place) = visitor.remapped_locals[local] { + let place = visitor.resolve_place(place); + visitor.remapped_locals[local] = Some(place); + trace!("Remapped {local:?} to {place:?}"); + } + } + + visitor.remapped_locals + } + + #[tracing::instrument(ret, level = "trace", skip(self))] + fn resolve_place(&self, mut place: Place<'tcx>) -> Place<'tcx> { + while let Some(new_place) = self.remapped_locals[place.local] { + place = new_place.project_deeper(place.projection, self.tcx); + } + place + } + + #[tracing::instrument(ret, level = "trace", skip(self))] + fn can_unify_places(&self, a: Place<'tcx>, b: Place<'tcx>) -> Option<(Local, Place<'tcx>)> { + let a = self.resolve_place(a); + let b = self.resolve_place(b); + + if a.local == b.local { + if a.projection != b.projection { + trace!("cannot unify same local with different projections"); + } + return None; + } + + if self.rust_call_tuples.contains(a.local) || self.rust_call_tuples.contains(b.local) { + trace!("cannot unify {a:?} and {b:?} involving a rust-call tuple argument"); + return None; + } + + let (local, place) = match (a.as_local(), b.as_local()) { + (None, None) => { + trace!("cannot unify 2 places that both have projections"); + return None; + } + (None, Some(b)) => { + if self.unprojectable_locals.contains(b) { + trace!("cannot unify {b:?} which cannot be projected"); + return None; + } + (b, a) + } + (Some(a), None) => { + if self.unprojectable_locals.contains(a) { + trace!("cannot unify {a:?} which cannot be projected"); + return None; + } + (a, b) + } + (Some(a), Some(b)) => match (self.body.local_kind(a), self.body.local_kind(b)) { + ( + LocalKind::Arg | LocalKind::ReturnPointer, + LocalKind::Arg | LocalKind::ReturnPointer, + ) => { + trace!("cannot unify {a:?} and {b:?} which are both arguments or return place"); + return None; + } + (LocalKind::Arg | LocalKind::ReturnPointer, LocalKind::Temp) => (b, a.into()), + (LocalKind::Temp, _) => (a, b.into()), + }, + }; + + if most_packed_projection(self.tcx, &self.body.local_decls, place).is_some() { + trace!("cannot unify {place:?} which has packed field projections"); + return None; + } + + if !self.liveness_matrix.disjoint_rows(local, place.local) { + trace!("cannot unify {a:?} and {b:?} which have overlapping live ranges"); + return None; + } + + // FIXME(#112651): This can be removed afterwards. + let local_ty = self.body.local_decls[local].ty; + let place_ty = place.ty(&self.body.local_decls, self.tcx).ty; + if local_ty != place_ty { + trace!( + "cannot unify {a:?} and {b:?} which have different types due to subtyping ({local_ty:?} vs {place_ty:?})" + ); + return None; + } + + Some((local, place)) + } + + #[tracing::instrument(level = "trace", skip(self))] + fn remap_local(&mut self, local: Local, place: Place<'tcx>) { + self.remapped_locals[local] = Some(place); + + self.liveness_matrix.union_rows(local, place.local); + self.liveness_matrix.clear_row(local); + + // If the original local was unprojectable then this now also applies to + // the mapped local. + if self.unprojectable_locals.contains(local) { + debug_assert!(place.projection.is_empty()); + self.unprojectable_locals.insert(place.local); + } + } + + fn visit_aggregate_assign( + &mut self, + dest: Place<'tcx>, + project_field: impl Fn(TyCtxt<'tcx>, Place<'tcx>, FieldIdx, Ty<'tcx>) -> Place<'tcx>, + operands: &IndexVec>, + location: Location, + ) { + // Attempt to unify each field operand with the corresponding field in + // the destination place. + let mut candidates = vec![]; + for (idx, operand) in operands.iter_enumerated() { + let (Operand::Copy(src) | Operand::Move(src)) = *operand else { + continue; + }; + let Some(src) = src.as_local() else { + continue; + }; + let dest = project_field(self.tcx, dest, idx, self.body.local_decls[src].ty); + trace!("Attempting to unify {dest:?} and {src:?} at {location:?}"); + if let Some((local, place)) = self.can_unify_places(dest, src.into()) { + candidates.push((local, place)); + } + } + + // Do the actual remapping *after* checking for live range overlaps. + // This is necessary because the input operands necessarily have + // overlapping live ranges. + for (local, place) in candidates { + self.remap_local(local, place); + } + } +} + +/// Since we are replacing all uses of a local with another place, we need to +/// ensure that the projections on that place are stable no matter where it is +/// used in the body. Additional this local may be used in debuginfo, so ensure +/// that the projections are compatible with usage in debuginfo. +fn check_projections(place: Place<'_>) -> bool { + place.projection.iter().all(|elem| elem.is_stable_offset() && elem.can_use_in_debuginfo()) +} + +impl<'tcx> Visitor<'tcx> for PlaceUnification<'_, 'tcx> { + fn visit_assign(&mut self, dest: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) { + if !check_projections(*dest) { + return; + } + match rvalue { + Rvalue::Use(Operand::Copy(src) | Operand::Move(src), _) => { + if !check_projections(*src) { + return; + } + + trace!("Attempting to unify {dest:?} and {src:?} at {location:?}"); + if let Some((local, place)) = self.can_unify_places(*src, *dest) { + self.remap_local(local, place); + } + } + Rvalue::Aggregate(aggregate_kind, operands) => match *aggregate_kind { + AggregateKind::Array(_) => self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, _field_ty| { + place.project_deeper( + &[PlaceElem::ConstantIndex { + offset: field_idx.as_u32().into(), + min_length: field_idx.as_u32() as u64 + 1, + from_end: false, + }], + tcx, + ) + }, + operands, + location, + ), + AggregateKind::Tuple => self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, field_ty| { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + }, + operands, + location, + ), + AggregateKind::Adt(_, _, _, _, Some(union_field_idx)) => { + debug_assert_eq!(operands.len(), 1); + self.visit_aggregate_assign( + *dest, + |tcx, place, _, field_ty| { + place + .project_deeper(&[PlaceElem::Field(union_field_idx, field_ty)], tcx) + }, + operands, + location, + ) + } + AggregateKind::Adt(adt_did, var_idx, _, _, None) => { + let def = self.tcx.adt_def(adt_did); + if def.repr().simd() { + // MCP#838 banned projections into SIMD types. + return; + } + self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, field_ty| { + if def.is_enum() { + place.project_deeper( + &[ + PlaceElem::Downcast(None, var_idx), + PlaceElem::Field(field_idx, field_ty), + ], + tcx, + ) + } else { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + } + }, + operands, + location, + ) + } + _ => {} + }, + _ => {} + }; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Apply place mappings to the MIR body. + +fn apply_mappings<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + remapped_locals: &IndexVec>>, +) { + let mut rewriter = PlaceUpdater { tcx, remapped_locals }; + rewriter.visit_body_preserves_cfg(body); +} + +struct PlaceUpdater<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + remapped_locals: &'a IndexVec>>, +} + +impl<'tcx> MutVisitor<'tcx> for PlaceUpdater<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_local(&mut self, local: &mut Local, context: PlaceContext, location: Location) { + if let Some(new_place) = self.remapped_locals[*local] { + trace!("replacing {local:?} with {new_place:?} at {location:?} ({context:?})"); + *local = new_place.as_local().expect("mapped place shouldn't have projections"); + } + } + + fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { + if let Some(new_place) = self.remapped_locals[place.local] { + trace!("replacing {place:?} with {new_place:?} at {location:?} ({context:?})"); + *place = new_place.project_deeper(place.projection, self.tcx) + } + + // Only call visit_local for projections, not the base local. + if let Some(new_projection) = self.process_projection(&place.projection, location) { + place.projection = self.tcx().mk_place_elems(&new_projection); + } + } + + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + match statement.kind { + // Rewrite StorageAlloc to the base local of the remapped place. + StatementKind::StorageAlloc(ref mut local) => { + if let Some(place) = self.remapped_locals[*local] { + *local = place.local; + } + return; + } + + // Remove storage lifetime markers. These are rebuilt from liveness + // information later. Also, since we've preserved StorageDead in + // unwind paths until now, we will want to remove those since they + // hurt LLVM's codegen. + StatementKind::StorageDead(_) | StatementKind::StorageLive(_) => { + statement.make_nop(true); + return; + } + _ => {} + } + + self.super_statement(statement, location); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Storage reconstruction + +/// Backward dataflow analysis which answers the question: from this point, is +/// there a path whose first access to a local is an initialization? +/// +/// This is used when a local is dead in a predecessor but maybe-live in its +/// successor. A `StorageLive` is inserted on that edge only if some continuation +/// initializes the local before reading it. If every continuation instead +/// reads the local first or never accesses it, `StorageLive` would be +/// unnecessary: it only allocates uninitialized storage and cannot make the +/// read valid. +/// +/// For example: +/// +/// ```text +/// bb1 bb2 +/// StorageLive(_1); _1 = ... // _1 is dead +/// _flag = true _flag = false +/// \ / +/// \ / +/// bb3 +/// switchInt(_flag) +/// / \ +/// bb4: use(_1) bb5: no use +/// ``` +/// +/// Liveness is path-insensitive, so `_1` is maybe-live in `bb3`: it is live in +/// `bb1` and `bb4`, but dead in `bb2` and `bb5`. Nevertheless, no `StorageLive` +/// is needed on `bb2 -> bb3`. The continuation to `bb5` never accesses `_1`, +/// while the continuation to `bb4` reads `_1` without initializing it first and +/// is therefore already UB. (The `_flag` assignments make that latter +/// continuation dynamically impossible, but this analysis does not need to +/// prove the correlation.) +/// +/// Live ranges which start in the middle of a block do not need this analysis: +/// such ranges always start at an initialization, so a `StorageLive` is +/// unconditionally required there. +/// +/// This analysis deliberately ignores the reconstructed `StorageDead` +/// boundaries. This can cause an initialization from a later allocation range +/// to propagate into an earlier range and result in an unnecessary +/// `StorageLive`, but cannot cause a required `StorageLive` to be omitted. +struct InitializedBeforeUse; + +impl<'tcx> Analysis<'tcx> for InitializedBeforeUse { + type Domain = DenseBitSet; + type Direction = Backward; + + const NAME: &'static str = "initialized-before-use"; + + fn bottom_value(&self, body: &Body<'tcx>) -> Self::Domain { + DenseBitSet::new_empty(body.local_decls.len()) + } + + fn initialize_start_block(&self, _body: &Body<'tcx>, _state: &mut Self::Domain) {} + + fn apply_primary_statement_effect( + &self, + state: &mut Self::Domain, + statement: &Statement<'tcx>, + location: Location, + ) { + // In backward order, process writes before reads. + if let StatementKind::StorageAlloc(local) = statement.kind { + state.gen_(local); + return; + } + VisitPlacesWith(|place: Place<'tcx>, context| { + if matches!(DefUse::for_place(place, context), DefUse::Def | DefUse::PartialWrite) { + state.gen_(place.local); + } + }) + .visit_statement(statement, location); + VisitPlacesWith(|place: Place<'tcx>, context| { + if matches!(DefUse::for_place(place, context), DefUse::Use) { + state.kill(place.local); + } + }) + .visit_statement(statement, location); + } + + fn apply_primary_terminator_effect( + &self, + state: &mut Self::Domain, + terminator: &Terminator<'tcx>, + location: Location, + ) { + // In backward order, process writes before reads. + VisitPlacesWith(|place: Place<'tcx>, context| { + if matches!(DefUse::for_place(place, context), DefUse::Def | DefUse::PartialWrite) { + state.gen_(place.local); + } + }) + .visit_terminator(terminator, location); + VisitPlacesWith(|place: Place<'tcx>, context| { + if matches!(DefUse::for_place(place, context), DefUse::Use) { + state.kill(place.local); + } + }) + .visit_terminator(terminator, location); + } +} + +impl InitializedBeforeUse { + /// Computes the analysis state at the start of each block. + fn compute<'tcx>( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + ) -> IndexVec> { + let results = + Self.iterate_to_fixpoint(tcx, body, Some("MoveElimination.initialized-before-use")); + let mut block_start = IndexVec::from_elem_n( + DenseBitSet::new_empty(body.local_decls.len()), + body.basic_blocks.len(), + ); + + struct BlockStartVisitor<'a> { + block_start: &'a mut IndexVec>, + } + + impl<'tcx> ResultsVisitor<'tcx, InitializedBeforeUse> for BlockStartVisitor<'_> { + fn visit_block_exit(&mut self, state: &DenseBitSet, block: BasicBlock) { + self.block_start[block].clone_from(state); + } + } + + visit_results( + body, + rustc_middle::mir::traversal::reachable(body).map(|(block, _)| block), + &results, + &mut BlockStartVisitor { block_start: &mut block_start }, + ); + block_start + } +} + +/// Helper function to split a critical edge if necessary. +fn get_or_split_edge<'tcx>( + patcher: &mut MirPatch<'tcx>, + body: &Body<'tcx>, + split_edges: &mut FxHashMap<(BasicBlock, BasicBlock), BasicBlock>, + pred: BasicBlock, + succ: BasicBlock, +) -> BasicBlock { + if let Some(&split_bb) = split_edges.get(&(pred, succ)) { + return split_bb; + } + let source_info = body.basic_blocks[pred].terminator().source_info; + let split_bb = patcher.new_block(BasicBlockData::new( + Some(Terminator { + source_info, + kind: TerminatorKind::Goto { target: succ }, + loop_hint_attrs: ThinVec::new(), + }), + body.basic_blocks[succ].is_cleanup, + )); + patcher.mutate_terminator(body, pred, |kind| { + kind.successors_mut(|t| { + if *t == succ { + *t = split_bb; + } + }); + }); + split_edges.insert((pred, succ), split_bb); + split_bb +} + +/// Don't insert `StorageDead` statements in cleanup blocks and unreachable blocks. +fn should_insert_storage_dead<'tcx>(block_data: &BasicBlockData<'tcx>) -> bool { + !block_data.is_cleanup && !matches!(block_data.terminator().kind, TerminatorKind::Unreachable) +} + +/// Re-constructs storage statements for all locals. +fn reconstruct_storage<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + points: &DenseLocationMap, + liveness_matrix: &SparseIntervalMatrix, +) { + let initialized_before_use = InitializedBeforeUse::compute(tcx, body); + let mut patcher = MirPatch::new(body); + let mut split_edges: FxHashMap<(BasicBlock, BasicBlock), BasicBlock> = Default::default(); + let mut storage_lives = Vec::new(); + + for local in body.local_decls.indices() { + // Arguments and return values don't use storage statements. + match body.local_kind(local) { + LocalKind::Arg | LocalKind::ReturnPointer => continue, + LocalKind::Temp => {} + } + + // Ignore dead locals. + let Some(row) = liveness_matrix.row(local) else { continue }; + if row.is_empty() { + continue; + } + + let mut emit_storage_live_in_preds = + |body: &mut Body<'tcx>, + patcher: &mut MirPatch<'tcx>, + storage_lives: &mut Vec<(Location, Local)>, + local: Local, + block: BasicBlock| { + if !initialized_before_use[block].contains(local) { + // No continuation initializes the local before reading it, + // so allocating storage cannot make any such read valid. + return; + } + + for &pred in &body.basic_blocks.predecessors()[block].clone() { + // If the local is live at any point in the predecessor's + // terminator then no StorageLive is needed. + let term = points.terminator(pred); + let term_early = SplitPointIndex::new(term, SplitPointEffect::Early); + let term_late = SplitPointIndex::new(term, SplitPointEffect::Late); + if !row.intersects_range(term_early..=term_late) { + // The local must be live on at least one predecessor, + // so if this is the only one then there is nothing to + // do. + debug_assert!(body.basic_blocks.predecessors()[block].len() > 1); + + // If the predecessor block has multiple successors then + // we need to split the critical edge before inserting + // StorageLive, otherwise the local would end up live on + // paths where it is supposed to be dead. + let loc = if body.basic_blocks[pred].terminator().successors().count() > 1 { + get_or_split_edge(patcher, body, &mut split_edges, pred, block) + .start_location() + } else { + body.terminator_loc(pred) + }; + storage_lives.push((loc, local)); + } + } + }; + let emit_storage_dead_in_succs = + |body: &mut Body<'tcx>, + patcher: &mut MirPatch<'tcx>, + local: Local, + block: BasicBlock| { + for succ in body.basic_blocks[block].terminator().successors() { + if !should_insert_storage_dead(&body.basic_blocks[succ]) { + continue; + } + + if !row.contains(SplitPointIndex::new( + points.entry_point(succ), + SplitPointEffect::Early, + )) { + // We don't care about critical edges here: if the local + // is already dead in the successor then it doesn't + // matter if we emit a redundant StorageDead. + + patcher.add_statement( + succ.start_location(), + StatementKind::StorageDead(local), + ); + } + } + }; + + // Iterate through the live range of the local and insert `StorageLive` + // and `StorageDead` at the points where it transitions from dead to + // live and vice versa. + // + // Note that the range here is an *inclusive range*. + for range in row.iter_intervals() { + let start = points.to_location(range.start.point()); + let end = points.to_location(range.last.point()); + + // If the live range starts at the `Early` point then it means that + // the value came from a predecessor block. A write from the first + // statement would happen at the `Late` point instead. + if range.start.effect() == SplitPointEffect::Early && start.statement_index == 0 { + // If the local is dead at the end of any predecessor block then + // emit a `StorageLive` before the terminator. + emit_storage_live_in_preds( + body, + &mut patcher, + &mut storage_lives, + local, + start.block, + ); + } else { + // Otherwise just add `StorageLive` before the statement that + // starts the live range. + storage_lives.push((start, local)); + } + + // The live range may span multiple blocks because + // `SparseIntervalMatrix` will coalesce adjacent ranges. If this + // happens then we need to repeat the start of block logic (see + // above) and end of block logic (see below) at each block boundary. + let mut current_block = start.block; + debug_assert!(start.block <= end.block); + while current_block != end.block { + if should_insert_storage_dead(&body.basic_blocks[current_block]) { + emit_storage_dead_in_succs(body, &mut patcher, local, current_block); + } + current_block = BasicBlock::from_usize(current_block.index() + 1); + emit_storage_live_in_preds( + body, + &mut patcher, + &mut storage_lives, + local, + current_block, + ); + } + + // We need to insert `StorageDead` after the last statement that + // uses a local. If this is a terminator then we need to instead + // insert it at the start of every successor block where the local + // is dead on entry. + if should_insert_storage_dead(&body.basic_blocks[end.block]) { + if range.last.point() == points.terminator(end.block) { + emit_storage_dead_in_succs(body, &mut patcher, local, current_block); + } else { + patcher.add_statement( + end.successor_within_block(), + StatementKind::StorageDead(local), + ); + } + } + } + } + + // Queue all `StorageLive` statements after `StorageDead` so that, when + // both are inserted at the same location, `StorageDead` always precedes + // `StorageLive` to avoid false overlaps. + for (loc, local) in storage_lives { + patcher.add_statement(loc, StatementKind::StorageLive(local)); + } + + patcher.apply(body); +} + +//////////////////////////////////////////////////////////////////////////////// +// Aliasing assignment fixup +// +// MIR assignments currently do not allow source and destination to alias, so +// fix this in post-processing. + +fn apply_alias_fixup<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let mut patcher = MirPatch::new(body); + let mut fixup = AliasFixup { tcx, local_decls: &body.local_decls, patcher: &mut patcher }; + for (block, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { + fixup.visit_basic_block_data(block, data); + } + patcher.apply(body); +} + +/// Returns whether 2 places alias, ignoring indirect places. +fn places_directly_alias<'tcx>( + tcx: TyCtxt<'tcx>, + local_decls: &IndexVec>, + a: Place<'tcx>, + b: Place<'tcx>, +) -> bool { + // This function doesn't handle indirect aliasing. + if a.local != b.local || a.is_indirect_first_projection() || b.is_indirect_first_projection() { + return false; + } + + for ((prefix, elem_a), (_, elem_b)) in a.iter_projections().zip(b.iter_projections()) { + // Continue until we find the first mismatching projection. + if elem_a == elem_b { + continue; + } + + match (elem_a, elem_b) { + // Disjoint fields don't alias except if they are union fields. + (PlaceElem::Field(_, _), PlaceElem::Field(_, _)) => { + let ty = prefix.ty(local_decls, tcx).ty; + return ty.is_union(); + } + + // Disjoint slice elements don't alias. + ( + PlaceElem::ConstantIndex { offset: offset_a, min_length: _, from_end: from_end_a }, + PlaceElem::ConstantIndex { offset: offset_b, min_length: _, from_end: from_end_b }, + ) if from_end_a == from_end_b && offset_a != offset_b => { + return false; + } + + // Conservatively assume the places may alias. + _ => return true, + } + } + + // If the projections are identical *or* one is a prefix of the other then + // the places alias. + true +} + +struct AliasFixup<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + local_decls: &'a IndexVec>, + patcher: &'a mut MirPatch<'tcx>, +} + +impl<'tcx> AliasFixup<'_, 'tcx> { + fn isolate_rvalue_to_local( + &mut self, + rvalue: Rvalue<'tcx>, + source_info: SourceInfo, + location: Location, + ) -> Place<'tcx> { + let ty = rvalue.ty(self.local_decls, self.tcx); + let temp = Place::from(self.patcher.new_temp(ty, source_info.span)); + trace!("isolating {rvalue:?} to {temp:?} due to conflict"); + self.patcher.add_statement(location, StatementKind::StorageLive(temp.local)); + self.patcher.add_assign(location, Place::from(temp), rvalue); + self.patcher.add_statement( + location.successor_within_block(), + StatementKind::StorageDead(temp.local), + ); + temp + } + + fn visit_aggregate_assign( + &mut self, + dest: Place<'tcx>, + enum_variant: Option, + project_field: impl Fn(TyCtxt<'tcx>, Place<'tcx>, FieldIdx, Ty<'tcx>) -> Place<'tcx>, + operands: &IndexVec>, + source_info: SourceInfo, + location: Location, + ) { + // Fast path: if no direct operand aliases the destination, we're done. + // + // We only look for direct aliases here, which is sufficient because we + // know the input MIR did not have any aliasing and we didn't introduce + // any indirect aliasing in this pass. + // + // If the destination place is indirect then it cannot be the start of + // a lifetime as per the RFC 3943 MIR semantics. This means that the + // lifetime of the underlying allocation must have started earlier, + // which overlaps the early point of the assignment statement. Therefore + // we couldn't have unified any source operand with this destination + // place. + // + // If the source place is indirect then a similar reasoning applies. The + // only exception is if there are multiple source places (e.g. + // aggregates). In that situation it's possible for an indirect source + // to overlap the destination if and only if there is also a direct + // source that overlaps it: + // + // _2 = &_1 + // _3 = (copy *_2, move _1) // _1 becomes _3.1 after unification + // + // We handle this here in 2 ways: if there is no direct alias, then + // we're fine. Otherwise, treat all indirect sources as potentially + // aliasing with the destination operand. + let has_direct_alias = operands.iter().any(|op| match op { + Operand::Copy(src) | Operand::Move(src) => { + places_directly_alias(self.tcx, self.local_decls, dest, *src) + } + Operand::Constant(_) | Operand::RuntimeChecks(_) => false, + }); + if !has_direct_alias { + return; + } + + debug!("splitting aggregate assignment at {location:?}"); + + // Split into per-field assignments. + let mut assignments = vec![]; + for (idx, op) in operands.iter_enumerated() { + let field_ty = op.ty(self.local_decls, self.tcx); + let dest_field = project_field(self.tcx, dest, idx, field_ty); + + let emit_op = match op { + Operand::Copy(src) | Operand::Move(src) => { + if *src == dest_field { + // Skip identity assignments. + continue; + } else if src.is_indirect_first_projection() + || places_directly_alias(self.tcx, self.local_decls, dest, *src) + { + // Partial alias: hoist the source to a temp first so the + // per-field write no longer overlaps the dest. Indirect + // sources also need hoisting here because they may point + // at one of the direct aliasing operands. + Operand::Move(self.isolate_rvalue_to_local( + Rvalue::Use(op.clone(), WithRetag::No), + source_info, + location, + )) + } else { + op.clone() + } + } + Operand::Constant(_) | Operand::RuntimeChecks(_) => op.clone(), + }; + assignments.push((dest_field, emit_op)); + } + + // Perform assignments *after* all aliasing fields have been read into + // temporary locals. + for (dest_field, emit_op) in assignments { + self.patcher.add_assign(location, dest_field, Rvalue::Use(emit_op, WithRetag::No)); + } + + // Delete the original aggregate assignment. + self.patcher.nop_statement(location); + + // For enum variants, set the discriminant after all field writes. + if let Some(variant_index) = enum_variant { + self.patcher.add_statement( + location, + StatementKind::SetDiscriminant { place: Box::new(dest), variant_index }, + ); + } + } +} + +impl<'tcx> MutVisitor<'tcx> for AliasFixup<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + // Fixup the MIR to remove aliasing assignments. + if let StatementKind::Assign((dest, rvalue)) = &mut statement.kind { + match *rvalue { + Rvalue::Use(Operand::Copy(src) | Operand::Move(src), with_retag) => { + if places_directly_alias(self.tcx, self.local_decls, *dest, src) { + if src == *dest { + debug!("{:?} turned into self-assignment, deleting", location); + statement.make_nop(true); + } else { + let temp = self.isolate_rvalue_to_local( + rvalue.clone(), + statement.source_info, + location, + ); + *rvalue = Rvalue::Use(Operand::Move(temp), with_retag); + } + } + } + Rvalue::Aggregate(AggregateKind::Array(_), ref mut operands) => self + .visit_aggregate_assign( + *dest, + None, + |tcx, place, field_idx, _field_ty| { + place.project_deeper( + &[PlaceElem::ConstantIndex { + offset: field_idx.as_u32().into(), + min_length: field_idx.as_u32() as u64 + 1, + from_end: false, + }], + tcx, + ) + }, + operands, + statement.source_info, + location, + ), + Rvalue::Aggregate(AggregateKind::Tuple, ref mut operands) => self + .visit_aggregate_assign( + *dest, + None, + |tcx, place, field_idx, field_ty| { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + }, + operands, + statement.source_info, + location, + ), + Rvalue::Aggregate( + AggregateKind::Adt(_, _, _, _, Some(union_field_idx)), + ref mut operands, + ) => { + debug_assert_eq!(operands.len(), 1); + self.visit_aggregate_assign( + *dest, + None, + |tcx, place, _, field_ty| { + place + .project_deeper(&[PlaceElem::Field(union_field_idx, field_ty)], tcx) + }, + operands, + statement.source_info, + location, + ) + } + Rvalue::Aggregate( + AggregateKind::Adt(adt_did, var_idx, _, _, None), + ref mut operands, + ) => { + let def = self.tcx.adt_def(adt_did); + if def.repr().simd() { + // MCP#838 banned projections into SIMD types. + return; + } + self.visit_aggregate_assign( + *dest, + def.is_enum().then_some(var_idx), + |tcx, place, field_idx, field_ty| { + if def.is_enum() { + place.project_deeper( + &[ + PlaceElem::Downcast(None, var_idx), + PlaceElem::Field(field_idx, field_ty), + ], + tcx, + ) + } else { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + } + }, + operands, + statement.source_info, + location, + ) + } + + // For other rvalues, don't try to split them into components + // and instead just introduce a temporary if there is any + // aliasing + Rvalue::Aggregate(..) + | Rvalue::Repeat(..) + | Rvalue::Cast(..) + | Rvalue::CopyForDeref(..) + | Rvalue::WrapUnsafeBinder(..) => { + let mut overlaps_dest = false; + VisitPlacesWith(|place, _ctxt| { + if places_directly_alias(self.tcx, self.local_decls, *dest, place) { + overlaps_dest = true; + } + }) + .visit_rvalue(rvalue, location); + if overlaps_dest { + let temp = self.isolate_rvalue_to_local( + rvalue.clone(), + statement.source_info, + location, + ); + *rvalue = Rvalue::Use(Operand::Move(temp), WithRetag::No); + } + } + + // These either cannot have aliasing, or allow it because they + // only operate on scalar backend types. + Rvalue::Use(Operand::Constant(..) | Operand::RuntimeChecks(..), _) + | Rvalue::Ref(..) + | Rvalue::ThreadLocalRef(..) + | Rvalue::BinaryOp(..) + | Rvalue::UnaryOp(..) + | Rvalue::Discriminant(..) + | Rvalue::RawPtr(..) + | Rvalue::Reborrow(..) => {} + } + } + } +} diff --git a/compiler/rustc_mir_transform/src/patch.rs b/compiler/rustc_mir_transform/src/patch.rs index 6a2d5101067e7..a2bc424feac9b 100644 --- a/compiler/rustc_mir_transform/src/patch.rs +++ b/compiler/rustc_mir_transform/src/patch.rs @@ -215,6 +215,21 @@ impl<'tcx> MirPatch<'tcx> { self.term_patch_map.insert(block, new); } + /// Modifies the terminator of a block, reading the existing patch if one exists or + /// cloning from the body otherwise. + pub(crate) fn mutate_terminator( + &mut self, + body: &Body<'tcx>, + bb: BasicBlock, + f: impl FnOnce(&mut TerminatorKind<'tcx>), + ) { + let kind = self + .term_patch_map + .entry(bb) + .or_insert_with(|| body.basic_blocks[bb].terminator().kind.clone()); + f(kind); + } + /// Mark given statement to be replaced by a `Nop`. /// /// This method only works on statements from the initial body, and cannot be used to remove diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs index 92c2f9a1c20e1..6d5e7133f4824 100644 --- a/compiler/rustc_mir_transform/src/promote_consts.rs +++ b/compiler/rustc_mir_transform/src/promote_consts.rs @@ -817,11 +817,10 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> { rhs.clone() } else { let unit = Rvalue::Use( - Operand::Constant(Box::new(ConstOperand { - span: statement.source_info.span, - user_ty: None, - const_: Const::zero_sized(self.tcx.types.unit), - })), + Operand::zero_sized_constant( + self.tcx.types.unit, + statement.source_info.span, + ), WithRetag::Yes, ); mem::replace(rhs, unit) diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs index c340d82d0a80b..13712b843f770 100644 --- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs +++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs @@ -86,6 +86,7 @@ impl RemoveNoopLandingPads { StatementKind::FakeRead(..) | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) + | StatementKind::StorageAlloc(_) | StatementKind::PlaceMention(..) | StatementKind::AscribeUserType(..) | StatementKind::Coverage(..) diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs index 98f4009c15665..5d3243b22e3e2 100644 --- a/compiler/rustc_mir_transform/src/remove_zsts.rs +++ b/compiler/rustc_mir_transform/src/remove_zsts.rs @@ -127,17 +127,25 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> { Some(place) } StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { - Some(local.into()) + if self.known_to_be_zst(self.local_decls[local].ty) { + statement.make_nop(true); + } + return; } StatementKind::Coverage(_) | StatementKind::Intrinsic(_) + | StatementKind::StorageAlloc(_) | StatementKind::Nop | StatementKind::BackwardIncompatibleDropHint { .. } | StatementKind::ConstEvalCounter => None, }; + // Under move-elimination semantics, direct ZST writes still serve to + // allocate the base local and give it an address. Genuinely unused + // assignments are later deleted by DSE. if let Some(place_for_ty) = place_for_ty && let ty = place_for_ty.ty(self.local_decls, self.tcx).ty && self.known_to_be_zst(ty) + && place_for_ty.is_indirect() { statement.make_nop(true); } else { diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index faf7039d498b1..e2b9d937fde25 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -279,7 +279,7 @@ pub fn build_drop_shim<'tcx>( let return_block = BasicBlock::new(1); let mut blocks = IndexVec::with_capacity(2); - let block = |blocks: &mut IndexVec<_, _>, kind| { + let block = |blocks: &mut IndexVec<_, _>, kind| -> BasicBlock { blocks.push(BasicBlockData::new( Some(Terminator { source_info, kind, loop_hint_attrs: ThinVec::new() }), false, @@ -288,7 +288,14 @@ pub fn build_drop_shim<'tcx>( if ty.is_some() { block(&mut blocks, TerminatorKind::Goto { target: return_block }); } - block(&mut blocks, TerminatorKind::Return); + let ret = block(&mut blocks, TerminatorKind::Return); + blocks[ret].statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + Place::return_place(), + Rvalue::Use(Operand::zero_sized_constant(tcx.types.unit, span), WithRetag::Yes), + ))), + )); let source = MirSource::from_shim(ty::ShimKind::DropGlue(def_id, ty)); let mut body = @@ -708,10 +715,18 @@ impl<'tcx> CloneShimBuilder<'tcx> { unwind } - fn tuple_like_shim(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I) - where - I: IntoIterator>, - { + fn tuple_like_shim( + &mut self, + dest: Place<'tcx>, + src: Place<'tcx>, + tys: &'tcx ty::List>, + ) { + if tys.is_empty() { + // With no fields to clone, initialize the return local directly. + self.copy_shim(); + return; + } + self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false); let unwind = self.block(vec![], TerminatorKind::UnwindResume, true); let target = self.block(vec![], TerminatorKind::Return, false); diff --git a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs index fcc28b20d25a0..93d31ef3a3d90 100644 --- a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs +++ b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs @@ -84,7 +84,7 @@ pub(super) fn build_async_drop_shim<'tcx>( let return_block = BasicBlock::new(1); let mut blocks = IndexVec::with_capacity(2); - let block = |blocks: &mut IndexVec<_, _>, kind| { + let block = |blocks: &mut IndexVec<_, _>, kind| -> BasicBlock { blocks.push(BasicBlockData::new( Some(Terminator { source_info, kind, loop_hint_attrs: ThinVec::new() }), false, @@ -104,7 +104,14 @@ pub(super) fn build_async_drop_shim<'tcx>( TerminatorKind::Goto { target: return_block } }, ); - block(&mut blocks, TerminatorKind::Return); + let ret = block(&mut blocks, TerminatorKind::Return); + blocks[ret].statements.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + Place::return_place(), + Rvalue::Use(Operand::zero_sized_constant(tcx.types.unit, span), WithRetag::Yes), + ))), + )); let source = MirSource::from_shim(ty::ShimKind::AsyncDropGlue(def_id, ty)); let mut body = diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 32504fc72fecd..eb3464c95e3d8 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -576,7 +576,8 @@ impl<'tcx> Visitor<'tcx> for UsedLocals { StatementKind::ConstEvalCounter | StatementKind::Nop | StatementKind::StorageLive(..) - | StatementKind::StorageDead(..) => {} + | StatementKind::StorageDead(..) + | StatementKind::StorageAlloc(..) => {} StatementKind::Assign((ref place, ref rvalue)) => { if rvalue.is_safe_to_remove() { self.visit_lhs(place, location); @@ -618,12 +619,12 @@ fn remove_unused_definitions_helper(used_locals: &mut UsedLocals, body: &mut Bod modified = false; for data in body.basic_blocks.as_mut_preserves_cfg() { - // Remove unnecessary StorageLive and StorageDead annotations. + // Remove storage statements for unused locals. for statement in data.statements.iter_mut() { let keep_statement = match &statement.kind { - StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { - used_locals.is_used(*local) - } + StatementKind::StorageLive(local) + | StatementKind::StorageDead(local) + | StatementKind::StorageAlloc(local) => used_locals.is_used(*local), StatementKind::Assign((place, _)) => used_locals.is_used(place.local), StatementKind::SetDiscriminant { place, .. } | StatementKind::BackwardIncompatibleDropHint { place, .. } => { @@ -688,12 +689,12 @@ impl UsedInStmtLocals { pub(crate) fn remove_unused_storage_annotations<'tcx>(&self, body: &mut Body<'tcx>) { for data in body.basic_blocks.as_mut_preserves_cfg() { - // Remove unnecessary StorageLive and StorageDead annotations. + // Remove storage statements for unused locals. for statement in data.statements.iter_mut() { let keep_statement = match &statement.kind { - StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { - self.locals.contains(*local) - } + StatementKind::StorageLive(local) + | StatementKind::StorageDead(local) + | StatementKind::StorageAlloc(local) => self.locals.contains(*local), _ => continue, }; if keep_statement { diff --git a/compiler/rustc_mir_transform/src/single_use_consts.rs b/compiler/rustc_mir_transform/src/single_use_consts.rs index aed5325f4c52b..55d3dc780d29f 100644 --- a/compiler/rustc_mir_transform/src/single_use_consts.rs +++ b/compiler/rustc_mir_transform/src/single_use_consts.rs @@ -143,7 +143,9 @@ impl<'tcx> Visitor<'tcx> for SingleUseConstsFinder { fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { match &statement.kind { // Storage markers are irrelevant to this. - StatementKind::StorageLive(_) | StatementKind::StorageDead(_) => {} + StatementKind::StorageAlloc(_) + | StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) => {} _ => self.super_statement(statement, location), } } diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs index 591c84bb17689..1799797707bee 100644 --- a/compiler/rustc_mir_transform/src/sroa.rs +++ b/compiler/rustc_mir_transform/src/sroa.rs @@ -6,6 +6,7 @@ use rustc_index::bit_set::{DenseBitSet, GrowableBitSet}; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_mir_dataflow::impls::DefUse; use rustc_mir_dataflow::value_analysis::{excluded_locals, iter_fields}; use rustc_span::bug; use tracing::{debug, instrument}; @@ -130,7 +131,9 @@ fn escaping_locals<'tcx>( fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { match statement.kind { // Storage statements are expanded in run_pass. - StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => return, + StatementKind::StorageLive(..) + | StatementKind::StorageDead(..) + | StatementKind::StorageAlloc(..) => return, _ => self.super_statement(statement, location), } } @@ -297,6 +300,18 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> { fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { if let Some(repl) = self.replacements.replace_place(self.tcx, place.as_ref()) { + // Under RFC 3943, a write has the side effect of allocating the + // underlying local. When splitting, we need to replicate this + // effect for all other fields using StorageAlloc. + if matches!(DefUse::for_place(*place, context), DefUse::Def | DefUse::PartialWrite) { + let fragments = self.replacements.place_fragments(place.local.into()).unwrap(); + for (_, _, local) in fragments { + if local != repl.local { + self.patch.add_statement(location, StatementKind::StorageAlloc(local)); + } + } + } + *place = repl } else { self.super_place(place, context, location) @@ -325,6 +340,15 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> { } return; } + StatementKind::StorageAlloc(l) => { + if let Some(final_locals) = self.replacements.place_fragments(l.into()) { + for (_, _, fl) in final_locals { + self.patch.add_statement(location, StatementKind::StorageAlloc(fl)); + } + statement.make_nop(true); + } + return; + } // We have `a = Struct { 0: x, 1: y, .. }`. // We replace it by diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 28a508bde9eca..786df3bab0308 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -309,6 +309,11 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { ); } } + StatementKind::StorageAlloc(_) => { + if self.body.phase < MirPhase::Runtime(RuntimePhase::Initial) { + self.fail(location, "`StorageAlloc` is only allowed in runtime MIR"); + } + } StatementKind::SetDiscriminant { .. } => { if self.body.phase < MirPhase::Runtime(RuntimePhase::Initial) { self.fail(location, "`SetDiscriminant`is not allowed until deaggregation"); @@ -383,6 +388,23 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { } self.check_unwind_edge(location, unwind); + // With move-elimination semantics from RFC 3943, the + // destination is evaluated last. It must not allocate fresh + // storage for a moved argument's local: that would hide an + // overlap forbidden by in-place argument and return + // passing. + // + // We forbid this as malformed MIR instead of treating as + // runtime UB. + if !destination.is_indirect() + && args.iter().any(|arg| { + matches!(&arg.node, Operand::Move(place) + if place.as_local() == Some(destination.local)) + }) + { + self.fail(location, "return destination refers to a moved local"); + } + // The code generation assumes that there are no critical call edges. The // assumption is used to simplify inserting code that should be executed along // the return edge from the call. FIXME(tmiasko): Since this is a strictly code @@ -1588,7 +1610,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ); } } - StatementKind::StorageLive(_) + StatementKind::StorageAlloc(_) + | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::Coverage(_) | StatementKind::ConstEvalCounter diff --git a/compiler/rustc_public/src/mir/body.rs b/compiler/rustc_public/src/mir/body.rs index a65798aff1a00..5205bb2e141ae 100644 --- a/compiler/rustc_public/src/mir/body.rs +++ b/compiler/rustc_public/src/mir/body.rs @@ -530,6 +530,7 @@ pub enum StatementKind { SetDiscriminant { place: Place, variant_index: VariantIdx }, StorageLive(Local), StorageDead(Local), + StorageAlloc(Local), PlaceMention(Place), AscribeUserType { place: Place, projections: UserTypeProjection, variance: Variance }, Coverage(Coverage), @@ -551,9 +552,6 @@ pub enum Rvalue { /// This is needed because dataflow analysis needs to distinguish /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo` /// has a destructor. - /// - /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After - /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too. Aggregate(AggregateKind, Vec), /// * `Offset` has the same semantics as `<*const T>::offset`, except that the second diff --git a/compiler/rustc_public/src/mir/pretty.rs b/compiler/rustc_public/src/mir/pretty.rs index 3bef4d5cb38df..01282ebf291b2 100644 --- a/compiler/rustc_public/src/mir/pretty.rs +++ b/compiler/rustc_public/src/mir/pretty.rs @@ -105,6 +105,9 @@ fn pretty_statement(writer: &mut W, statement: &StatementKind) -> io:: StatementKind::StorageLive(local) => { writeln!(writer, "{INDENT}StorageLive(_{local});") } + StatementKind::StorageAlloc(local) => { + writeln!(writer, "{INDENT}StorageAlloc(_{local});") + } StatementKind::StorageDead(local) => { writeln!(writer, "{INDENT}StorageDead(_{local});") } diff --git a/compiler/rustc_public/src/mir/visit.rs b/compiler/rustc_public/src/mir/visit.rs index d80c769cd8169..45dc5ba09d7c4 100644 --- a/compiler/rustc_public/src/mir/visit.rs +++ b/compiler/rustc_public/src/mir/visit.rs @@ -172,7 +172,9 @@ macro_rules! make_mir_visitor { StatementKind::SetDiscriminant { place, .. } => { self.visit_place(place, PlaceContext::MUTATING, location); } - StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { + StatementKind::StorageLive(local) + | StatementKind::StorageDead(local) + | StatementKind::StorageAlloc(local) => { self.visit_local(local, PlaceContext::NON_USE, location); } StatementKind::AscribeUserType { place, projections, variance: _ } => { diff --git a/compiler/rustc_public/src/unstable/convert/stable/mir.rs b/compiler/rustc_public/src/unstable/convert/stable/mir.rs index c464c310cddf4..b38cc71f195c3 100644 --- a/compiler/rustc_public/src/unstable/convert/stable/mir.rs +++ b/compiler/rustc_public/src/unstable/convert/stable/mir.rs @@ -173,6 +173,9 @@ impl<'tcx> Stable<'tcx> for mir::StatementKind<'tcx> { crate::mir::StatementKind::StorageLive(place.stable(tables, cx)) } + mir::StatementKind::StorageAlloc(local) => { + crate::mir::StatementKind::StorageAlloc(local.stable(tables, cx)) + } mir::StatementKind::StorageDead(place) => { crate::mir::StatementKind::StorageDead(place.stable(tables, cx)) } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 3debf3248ff80..ef63d6f0e6c14 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1380,6 +1380,7 @@ symbols! { mir_set_discriminant, mir_static, mir_static_mut, + mir_storage_alloc, mir_storage_dead, mir_storage_live, mir_tail_call, diff --git a/library/core/src/intrinsics/mir.rs b/library/core/src/intrinsics/mir.rs index 929e56872674d..7793620107d2c 100644 --- a/library/core/src/intrinsics/mir.rs +++ b/library/core/src/intrinsics/mir.rs @@ -229,7 +229,7 @@ //! //! #### Statements //! - Assign statements work via normal Rust assignment. -//! - [`StorageLive`], [`StorageDead`] statements have an associated function. +//! - [`StorageLive`], [`StorageDead`], [`StorageAlloc`] statements have an associated function. //! //! #### Rvalues //! @@ -401,6 +401,12 @@ define!("mir_unwind_resume", define!("mir_storage_live", fn StorageLive(local: T)); define!("mir_storage_dead", fn StorageDead(local: T)); +define!( + "mir_storage_alloc", + /// If the local is live but unallocated, allocates backing storage containing uninitialized + /// bytes for it. This has no effect if the local already has an allocation. + fn StorageAlloc(local: T) +); define!("mir_assume", fn Assume(operand: bool)); define!("mir_checked", fn Checked(binop: T) -> (T, bool)); define!( diff --git a/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs b/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs index 33853a3c5ed91..132477eb727d6 100644 --- a/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs +++ b/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs @@ -255,7 +255,8 @@ fn check_statement<'tcx>( check_operand(cx, count, span, body, msrv) }, // These are all NOPs - StatementKind::StorageLive(_) + StatementKind::StorageAlloc(_) + | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::AscribeUserType(..) | StatementKind::PlaceMention(..) diff --git a/src/tools/miri/src/shims/panic.rs b/src/tools/miri/src/shims/panic.rs index 50e32cffaee3f..c308087054a96 100644 --- a/src/tools/miri/src/shims/panic.rs +++ b/src/tools/miri/src/shims/panic.rs @@ -58,9 +58,11 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Forward to `panic_bounds_check` lang item. // First arg: index. - let index = this.read_immediate(&this.eval_operand(index, None)?)?; + let index = this.eval_operand(index, None)?; + let index = this.read_immediate(&index)?; // Second arg: len. - let len = this.read_immediate(&this.eval_operand(len, None)?)?; + let len = this.eval_operand(len, None)?; + let len = this.read_immediate(&len)?; // Call the lang item. let panic_bounds_check = this.tcx.lang_items().panic_bounds_check_fn().unwrap(); @@ -77,9 +79,11 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Forward to `panic_misaligned_pointer_dereference` lang item. // First arg: required. - let required = this.read_immediate(&this.eval_operand(required, None)?)?; + let required = this.eval_operand(required, None)?; + let required = this.read_immediate(&required)?; // Second arg: found. - let found = this.read_immediate(&this.eval_operand(found, None)?)?; + let found = this.eval_operand(found, None)?; + let found = this.read_immediate(&found)?; // Call the lang item. let panic_misaligned_pointer_dereference = diff --git a/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias.move_elimination.stderr b/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias.move_elimination.stderr new file mode 100644 index 0000000000000..0a700083069a8 --- /dev/null +++ b/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias.move_elimination.stderr @@ -0,0 +1,13 @@ +error: Undefined Behavior: accessing a live but unallocated local variable + --> tests/fail/function_calls/arg_inplace_locals_alias.rs:LL:CC + | +LL | Call(_unit = callee(Move(non_copy), Move(non_copy)), ReturnTo(after_call), UnwindContinue()) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias.rs b/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias.rs index 3f6cd583aaff0..c4a360feaed36 100644 --- a/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias.rs +++ b/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias.rs @@ -1,7 +1,8 @@ //! Ensure we detect aliasing of two in-place arguments for the tricky case where they do not -//! live in memory. -//@revisions: stack tree +//! live in memory. Move elimination must also reject this when whole moves free their source. +//@revisions: stack tree move_elimination //@[tree]compile-flags: -Zmiri-tree-borrows +//@[move_elimination]compile-flags: -Zmir-move-elimination #![feature(custom_mir, core_intrinsics)] @@ -17,8 +18,9 @@ fn main() { let staging = S(42); // This forces `staging` into memory... let non_copy = staging; // ... so we move it to a non-inmemory local here. // This specifically uses a type with scalar representation to tempt Miri to use the - // efficient way of storing local variables (outside adressable memory). + // efficient way of storing local variables (outside addressable memory). Call(_unit = callee(Move(non_copy), Move(non_copy)), ReturnTo(after_call), UnwindContinue()) + //~[move_elimination]^ ERROR: accessing a live but unallocated local variable } after_call = { Return() diff --git a/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias_ret.rs b/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias_ret.rs index 2b6649b346e57..1e8961ab8f639 100644 --- a/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias_ret.rs +++ b/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias_ret.rs @@ -1,4 +1,4 @@ -//! Ensure we detect aliasing of a in-place argument with the return place for the tricky case where +//! Ensure we detect aliasing of an in-place argument with the return place for the tricky case where //! they do not live in memory. //@revisions: stack tree //@[tree]compile-flags: -Zmiri-tree-borrows @@ -6,6 +6,7 @@ use std::intrinsics::mir::*; #[allow(unused)] +#[repr(transparent)] pub struct S(i32); #[custom_mir(dialect = "runtime", phase = "optimized")] @@ -16,8 +17,9 @@ fn main() { let staging = S(42); // This forces `staging` into memory... let _non_copy = staging; // ... so we move it to a non-inmemory local here. // This specifically uses a type with scalar representation to tempt Miri to use the - // efficient way of storing local variables (outside adressable memory). - Call(_non_copy = callee(Move(_non_copy)), ReturnTo(after_call), UnwindContinue()) + // efficient way of storing local variables (outside addressable memory). + // Also use a projection otherwise this would be considered malformed MIR. + Call(_non_copy = callee(Move(_non_copy.0)), ReturnTo(after_call), UnwindContinue()) } after_call = { Return() @@ -25,8 +27,8 @@ fn main() { } } -fn callee(x: S) -> S { +fn callee(x: i32) -> S { //~[stack]^ ERROR: not granting access //~[tree]| ERROR: /reborrow .* forbidden/ - x + S(x) } diff --git a/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias_ret.stack.stderr b/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias_ret.stack.stderr index 21fcfb96fd17d..fb53b4d6cfe5b 100644 --- a/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias_ret.stack.stderr +++ b/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias_ret.stack.stderr @@ -1,20 +1,20 @@ error: Undefined Behavior: not granting access to tag because that would remove [Unique for ] which is strongly protected --> tests/fail/function_calls/arg_inplace_locals_alias_ret.rs:LL:CC | -LL | fn callee(x: S) -> S { - | ^ Undefined Behavior occurred here +LL | fn callee(x: i32) -> S { + | ^ Undefined Behavior occurred here | = help: this indicates a potential bug in the program: it performed an invalid operation, but the Stacked Borrows rules it violated are still experimental = help: see https://github.com/rust-lang/unsafe-code-guidelines/blob/master/wip/stacked-borrows.md for further information help: was created here, as the root tag for ALLOC --> tests/fail/function_calls/arg_inplace_locals_alias_ret.rs:LL:CC | -LL | Call(_non_copy = callee(Move(_non_copy)), ReturnTo(after_call), UnwindContinue()) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +LL | Call(_non_copy = callee(Move(_non_copy.0)), ReturnTo(after_call), UnwindContinue()) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: is this argument --> tests/fail/function_calls/arg_inplace_locals_alias_ret.rs:LL:CC | -LL | fn callee(x: S) -> S { +LL | fn callee(x: i32) -> S { | ^ = note: stack backtrace: 0: callee diff --git a/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias_ret.tree.stderr b/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias_ret.tree.stderr index e4d4e87625a9e..32b552c6bd6a1 100644 --- a/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias_ret.tree.stderr +++ b/src/tools/miri/tests/fail/function_calls/arg_inplace_locals_alias_ret.tree.stderr @@ -1,8 +1,8 @@ error: Undefined Behavior: reborrow through (root of the allocation) at ALLOC[0x0] is forbidden --> tests/fail/function_calls/arg_inplace_locals_alias_ret.rs:LL:CC | -LL | fn callee(x: S) -> S { - | ^ Undefined Behavior occurred here +LL | fn callee(x: i32) -> S { + | ^ Undefined Behavior occurred here | = help: this indicates a potential bug in the program: it performed an invalid operation, but the Tree Borrows rules it violated are still experimental = help: see https://github.com/rust-lang/unsafe-code-guidelines/blob/master/wip/tree-borrows.md for further information @@ -12,17 +12,17 @@ LL | fn callee(x: S) -> S { help: the accessed tag was created here --> tests/fail/function_calls/arg_inplace_locals_alias_ret.rs:LL:CC | -LL | Call(_non_copy = callee(Move(_non_copy)), ReturnTo(after_call), UnwindContinue()) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +LL | Call(_non_copy = callee(Move(_non_copy.0)), ReturnTo(after_call), UnwindContinue()) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: the protected tag was created here, in the initial state Reserved --> tests/fail/function_calls/arg_inplace_locals_alias_ret.rs:LL:CC | -LL | fn callee(x: S) -> S { +LL | fn callee(x: i32) -> S { | ^ help: the protected tag later transitioned to Unique due to a child write access at offsets [0x0..0x4] --> tests/fail/function_calls/arg_inplace_locals_alias_ret.rs:LL:CC | -LL | fn callee(x: S) -> S { +LL | fn callee(x: i32) -> S { | ^ = help: this transition corresponds to the first write to a 2-phase borrowed mutable reference = note: stack backtrace: diff --git a/src/tools/miri/tests/fail/function_calls/arg_inplace_varargs.none.stderr b/src/tools/miri/tests/fail/function_calls/arg_inplace_varargs.none.stderr new file mode 100644 index 0000000000000..f69e200077df7 --- /dev/null +++ b/src/tools/miri/tests/fail/function_calls/arg_inplace_varargs.none.stderr @@ -0,0 +1,23 @@ +error: Undefined Behavior: reading memory at ALLOC[0x0..0x8], but memory is uninitialized at [0x0..0x8], and this operation requires initialized memory + --> tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + | +LL | unsafe extern "C" fn consume(_: ...) { + | ^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + = note: stack backtrace: + 0: consume + at tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + 1: main + at tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + +Uninitialized memory occurred at ALLOC[0x0..0x8], in this allocation: +ALLOC (stack variable, size: 8, align: 8) { + __ __ __ __ __ __ __ __ │ ░░░░░░░░ +} + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/function_calls/arg_inplace_varargs.rs b/src/tools/miri/tests/fail/function_calls/arg_inplace_varargs.rs new file mode 100644 index 0000000000000..e229a55b2114c --- /dev/null +++ b/src/tools/miri/tests/fail/function_calls/arg_inplace_varargs.rs @@ -0,0 +1,27 @@ +//@revisions: stack tree none +//@[tree]compile-flags: -Zmiri-tree-borrows +//@[none]compile-flags: -Zmiri-disable-stacked-borrows + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; + +// A moved variadic argument must be inaccessible before the next argument is copied. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let ptr: *const u64; + let unit: (); + { + let value = 1u64; + ptr = &raw const value; + Call(unit = consume(Move(value), *ptr), ReturnTo(done), UnwindContinue()) + } + done = { Return() } + } +} + +unsafe extern "C" fn consume(_: ...) { + //~[stack]^ ERROR: tag does not exist in the borrow stack + //~[tree]| ERROR: /read access .* forbidden/ + //~[none]| ERROR: uninitialized +} diff --git a/src/tools/miri/tests/fail/function_calls/arg_inplace_varargs.stack.stderr b/src/tools/miri/tests/fail/function_calls/arg_inplace_varargs.stack.stderr new file mode 100644 index 0000000000000..b60de65815b08 --- /dev/null +++ b/src/tools/miri/tests/fail/function_calls/arg_inplace_varargs.stack.stderr @@ -0,0 +1,28 @@ +error: Undefined Behavior: attempting a read access using at ALLOC[0x0], but that tag does not exist in the borrow stack for this location + --> tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + | +LL | unsafe extern "C" fn consume(_: ...) { + | ^ this error occurs as part of an access at ALLOC[0x0..0x8] + | + = help: this indicates a potential bug in the program: it performed an invalid operation, but the Stacked Borrows rules it violated are still experimental + = help: see https://github.com/rust-lang/unsafe-code-guidelines/blob/master/wip/stacked-borrows.md for further information +help: was created by a SharedReadOnly retag at offsets [0x0..0x8] + --> tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + | +LL | ptr = &raw const value; + | ^^^^^^^^^^^^^^^^^^^^^^ +help: was later invalidated at offsets [0x0..0x8] by a Unique in-place function argument/return passing protection + --> tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + | +LL | unsafe extern "C" fn consume(_: ...) { + | ^ + = note: stack backtrace: + 0: consume + at tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + 1: main + at tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/function_calls/arg_inplace_varargs.tree.stderr b/src/tools/miri/tests/fail/function_calls/arg_inplace_varargs.tree.stderr new file mode 100644 index 0000000000000..c050872b57578 --- /dev/null +++ b/src/tools/miri/tests/fail/function_calls/arg_inplace_varargs.tree.stderr @@ -0,0 +1,37 @@ +error: Undefined Behavior: read access through (root of the allocation) at ALLOC[0x0] is forbidden + --> tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + | +LL | unsafe extern "C" fn consume(_: ...) { + | ^ Undefined Behavior occurred here + | + = help: this indicates a potential bug in the program: it performed an invalid operation, but the Tree Borrows rules it violated are still experimental + = help: see https://github.com/rust-lang/unsafe-code-guidelines/blob/master/wip/tree-borrows.md for further information + = help: the accessed tag (root of the allocation) is foreign to the protected tag (i.e., it is not a child) + = help: this foreign read access would cause the protected tag (currently Unique) to become Disabled + = help: protected tags must never be Disabled +help: the accessed tag was created here + --> tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + | +LL | ptr = &raw const value; + | ^^^^^^^^^^^^^^^^^^^^^^ +help: the protected tag was created here, in the initial state Reserved + --> tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + | +LL | unsafe extern "C" fn consume(_: ...) { + | ^ +help: the protected tag later transitioned to Unique due to a child write access at offsets [0x0..0x8] + --> tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + | +LL | unsafe extern "C" fn consume(_: ...) { + | ^ + = help: this transition corresponds to the first write to a 2-phase borrowed mutable reference + = note: stack backtrace: + 0: consume + at tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + 1: main + at tests/fail/function_calls/arg_inplace_varargs.rs:LL:CC + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/function_calls/return_local_on_unwind.rs b/src/tools/miri/tests/fail/function_calls/return_local_on_unwind.rs new file mode 100644 index 0000000000000..04c70fb390f5e --- /dev/null +++ b/src/tools/miri/tests/fail/function_calls/return_local_on_unwind.rs @@ -0,0 +1,33 @@ +//@compile-flags: -Zmiri-disable-stacked-borrows +#![feature(custom_mir, core_intrinsics)] + +use std::intrinsics::mir::*; + +// Call setup clears the return destination even if it is stored as an immediate. +// Arguments must still receive its old value before it is cleared. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn call() -> u32 { + mir! { + { + let value = 42u32; + Call(value = callee(value), ReturnTo(done), UnwindCleanup(cleanup)) + } + done = { + RET = value; + Return() + } + cleanup (cleanup) = { + RET = value; //~ERROR: uninitialized + UnwindResume() + } + } +} + +fn callee(value: u32) -> u32 { + assert_eq!(value, 42); + panic!("unwind"); +} + +fn main() { + let _ = std::panic::catch_unwind(call); +} diff --git a/src/tools/miri/tests/fail/function_calls/return_local_on_unwind.stderr b/src/tools/miri/tests/fail/function_calls/return_local_on_unwind.stderr new file mode 100644 index 0000000000000..d4a0d8a899c30 --- /dev/null +++ b/src/tools/miri/tests/fail/function_calls/return_local_on_unwind.stderr @@ -0,0 +1,31 @@ + +thread 'main' ($TID) panicked at tests/fail/function_calls/return_local_on_unwind.rs:LL:CC: +unwind +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +note: in Miri, you may have to set `MIRIFLAGS=-Zmiri-env-forward=RUST_BACKTRACE` for the environment variable to have an effect +error: Undefined Behavior: constructing invalid value of type u32: encountered uninitialized memory, but expected an integer + --> tests/fail/function_calls/return_local_on_unwind.rs:LL:CC + | +LL | RET = value; + | ^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + = note: stack backtrace: + 0: call + at tests/fail/function_calls/return_local_on_unwind.rs:LL:CC + 1: u32 {call} as std::ops::FnOnce<()>>::call_once - shim(fn() -> u32 {call}) + at RUSTLIB/core/src/ops/function.rs:LL:CC + 2: std::panicking::catch_unwind::do_call:: u32 {call}, u32> + at RUSTLIB/std/src/panicking.rs:LL:CC + 3: std::panicking::catch_unwind:: u32 {call}> + at RUSTLIB/std/src/panicking.rs:LL:CC + 4: std::panic::catch_unwind:: u32 {call}, u32> + at RUSTLIB/std/src/panic.rs:LL:CC + 5: main + at tests/fail/function_calls/return_local_on_unwind.rs:LL:CC + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/address_after_call_move.move_elimination.stderr b/src/tools/miri/tests/fail/move_elimination/address_after_call_move.move_elimination.stderr new file mode 100644 index 0000000000000..79bc3fe41caa9 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/address_after_call_move.move_elimination.stderr @@ -0,0 +1,13 @@ +error: Undefined Behavior: accessing a live but unallocated local variable + --> tests/fail/move_elimination/address_after_call_move.rs:LL:CC + | +LL | ptr = &raw const value; + | ^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/address_after_call_move.rs b/src/tools/miri/tests/fail/move_elimination/address_after_call_move.rs new file mode 100644 index 0000000000000..3ab3d63124538 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/address_after_call_move.rs @@ -0,0 +1,28 @@ +//@revisions: normal move_elimination +//@[normal]check-pass +//@[move_elimination]compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] + +use std::intrinsics::mir::*; + +// A whole-local call move leaves the caller's local unallocated after return, +// so even taking its address must fail without reading any bytes. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let value: (u8, u8); + let unit: (); + let ptr: *const (u8, u8); + { + value = (1, 2); + Call(unit = consume(Move(value)), ReturnTo(after_call), UnwindContinue()) + } + after_call = { + ptr = &raw const value; //~[move_elimination] ERROR: live but unallocated + Return() + } + } +} + +fn consume(_: (u8, u8)) {} diff --git a/src/tools/miri/tests/fail/move_elimination/call_argument_before_destination.move_elimination.stderr b/src/tools/miri/tests/fail/move_elimination/call_argument_before_destination.move_elimination.stderr new file mode 100644 index 0000000000000..6c2b974b5be34 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_argument_before_destination.move_elimination.stderr @@ -0,0 +1,13 @@ +error: Undefined Behavior: accessing a live but unallocated local variable + --> tests/fail/move_elimination/call_argument_before_destination.rs:LL:CC + | +LL | Call(value = identity(value), ReturnTo(done), UnwindContinue()) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/call_argument_before_destination.rs b/src/tools/miri/tests/fail/move_elimination/call_argument_before_destination.rs new file mode 100644 index 0000000000000..ebc970e697eac --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_argument_before_destination.rs @@ -0,0 +1,26 @@ +//@revisions: normal move_elimination +//@[normal]check-pass +//@[move_elimination]compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; +use std::mem::MaybeUninit; + +// Reject the unallocated argument before destination evaluation allocates its local. +// MaybeUninit permits uninitialized bytes, isolating the allocation-state check. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let value: MaybeUninit; + { + StorageLive(value); + Call(value = identity(value), ReturnTo(done), UnwindContinue()) + //~[move_elimination]^ ERROR: accessing a live but unallocated local variable + } + done = { Return() } + } +} + +fn identity(value: MaybeUninit) -> MaybeUninit { + value +} diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_argument_order.move_elimination.stderr b/src/tools/miri/tests/fail/move_elimination/call_move_argument_order.move_elimination.stderr new file mode 100644 index 0000000000000..1079b077b420c --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_argument_order.move_elimination.stderr @@ -0,0 +1,13 @@ +error: Undefined Behavior: accessing a live but unallocated local variable + --> tests/fail/move_elimination/call_move_argument_order.rs:LL:CC + | +LL | Call(unit = consume(Move(ptr), *ptr), ReturnTo(done), UnwindContinue()) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_argument_order.rs b/src/tools/miri/tests/fail/move_elimination/call_move_argument_order.rs new file mode 100644 index 0000000000000..720f823f1af8a --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_argument_order.rs @@ -0,0 +1,23 @@ +//@revisions: normal move_elimination +//@[normal]check-pass +//@[move_elimination]compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; + +// A later argument cannot read the pointer local consumed by an earlier move. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let ptr: *const u32; + let unit: (); + { + let value = 42u32; + ptr = &raw const value; + Call(unit = consume(Move(ptr), *ptr), ReturnTo(done), UnwindContinue()) + //~[move_elimination]^ ERROR: accessing a live but unallocated local variable + } + done = { Return() } + } +} +fn consume(_: *const u32, _: u32) {} diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_destination_indirect.rs b/src/tools/miri/tests/fail/move_elimination/call_move_destination_indirect.rs new file mode 100644 index 0000000000000..e943eb2645122 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_destination_indirect.rs @@ -0,0 +1,22 @@ +//@compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; + +// A destination pointing into moved storage must fail even if the callee never returns. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let ptr: *mut u32; + { + let value = 42u32; + ptr = &raw mut value; + Call(*ptr = consume(Move(value)), ReturnTo(done), UnwindContinue()) + } + done = { Return() } + } +} +fn consume(_: u32) -> u32 { + //~^ ERROR: has been freed + std::process::exit(0) +} diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_destination_indirect.stderr b/src/tools/miri/tests/fail/move_elimination/call_move_destination_indirect.stderr new file mode 100644 index 0000000000000..16bad2c4b2e39 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_destination_indirect.stderr @@ -0,0 +1,35 @@ +error: Undefined Behavior: pointer not dereferenceable: ALLOC has been freed, so this pointer is dangling + --> tests/fail/move_elimination/call_move_destination_indirect.rs:LL:CC + | +LL | fn consume(_: u32) -> u32 { + | ^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information +help: ALLOC was allocated here: + --> tests/fail/move_elimination/call_move_destination_indirect.rs:LL:CC + | +LL | / mir! { +LL | | let ptr: *mut u32; +LL | | { +LL | | let value = 42u32; +... | +LL | | done = { Return() } +LL | | } + | |_____^ +help: ALLOC was deallocated here: + --> tests/fail/move_elimination/call_move_destination_indirect.rs:LL:CC + | +LL | Call(*ptr = consume(Move(value)), ReturnTo(done), UnwindContinue()) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + = note: stack backtrace: + 0: consume + at tests/fail/move_elimination/call_move_destination_indirect.rs:LL:CC + 1: main + at tests/fail/move_elimination/call_move_destination_indirect.rs:LL:CC + = note: this error originates in the macro `::core::intrinsics::mir::__internal_extract_let` which comes from the expansion of the macro `mir` (in Nightly builds, run with -Z macro-backtrace for more info) + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_destination_order.move_elimination.stderr b/src/tools/miri/tests/fail/move_elimination/call_move_destination_order.move_elimination.stderr new file mode 100644 index 0000000000000..33bc6d279fc2d --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_destination_order.move_elimination.stderr @@ -0,0 +1,13 @@ +error: Undefined Behavior: accessing a live but unallocated local variable + --> tests/fail/move_elimination/call_move_destination_order.rs:LL:CC + | +LL | Call(*ptr = consume(Move(ptr)), ReturnTo(done), UnwindContinue()) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_destination_order.rs b/src/tools/miri/tests/fail/move_elimination/call_move_destination_order.rs new file mode 100644 index 0000000000000..f9441e90f22fb --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_destination_order.rs @@ -0,0 +1,24 @@ +//@revisions: normal move_elimination +//@[normal]check-pass +//@[move_elimination]compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; + +// Destination evaluation cannot read a pointer local moved by an argument. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let ptr: *mut u32; + { + let value = 0u32; + ptr = &raw mut value; + Call(*ptr = consume(Move(ptr)), ReturnTo(done), UnwindContinue()) + //~[move_elimination]^ ERROR: accessing a live but unallocated local variable + } + done = { Return() } + } +} +fn consume(_: *mut u32) -> u32 { + 42 +} diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_offset.move_elimination.stderr b/src/tools/miri/tests/fail/move_elimination/call_move_offset.move_elimination.stderr new file mode 100644 index 0000000000000..b8f339eceed62 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_offset.move_elimination.stderr @@ -0,0 +1,35 @@ +error: Undefined Behavior: in-bounds pointer arithmetic failed: ALLOC has been freed, so this pointer is dangling + --> tests/fail/move_elimination/call_move_offset.rs:LL:CC + | +LL | let next = unsafe { ptr.cast::().add(1) }; + | ^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information +help: ALLOC was allocated here: + --> tests/fail/move_elimination/call_move_offset.rs:LL:CC + | +LL | / mir! { +LL | | let value: [u8; 2]; +LL | | let ptr: *const [u8; 2]; +LL | | let unit: (); +... | +LL | | done = { Return() } +LL | | } + | |_____^ +help: ALLOC was deallocated here: + --> tests/fail/move_elimination/call_move_offset.rs:LL:CC + | +LL | Call(unit = consume(Move(value), ptr), ReturnTo(done), UnwindContinue()) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + = note: stack backtrace: + 0: consume + at tests/fail/move_elimination/call_move_offset.rs:LL:CC + 1: main + at tests/fail/move_elimination/call_move_offset.rs:LL:CC + = note: this error originates in the macro `mir` (in Nightly builds, run with -Z macro-backtrace for more info) + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_offset.rs b/src/tools/miri/tests/fail/move_elimination/call_move_offset.rs new file mode 100644 index 0000000000000..7306750d76f55 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_offset.rs @@ -0,0 +1,28 @@ +//@revisions: normal move_elimination +//@[normal]check-pass +//@[move_elimination]compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; + +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let value: [u8; 2]; + let ptr: *const [u8; 2]; + let unit: (); + { + value = [1, 2]; + ptr = &raw const value; + Call(unit = consume(Move(value), ptr), ReturnTo(done), UnwindContinue()) + } + done = { Return() } + } +} + +fn consume(_: [u8; 2], ptr: *const [u8; 2]) { + // The call move frees the source before the callee runs, so even in-bounds + // pointer arithmetic without a memory access is invalid. + let next = unsafe { ptr.cast::().add(1) }; //~[move_elimination] ERROR: has been freed + assert_eq!(next.addr(), ptr.addr() + 1); +} diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_overlap_field_first.move_elimination.stderr b/src/tools/miri/tests/fail/move_elimination/call_move_overlap_field_first.move_elimination.stderr new file mode 100644 index 0000000000000..61d55c252441c --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_overlap_field_first.move_elimination.stderr @@ -0,0 +1,35 @@ +error: Undefined Behavior: memory access failed: ALLOC has been freed, so this pointer is dangling + --> tests/fail/move_elimination/call_move_overlap_field_first.rs:LL:CC + | +LL | fn field_whole(_: MaybeUninit, _: Value) { + | ^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information +help: ALLOC was allocated here: + --> tests/fail/move_elimination/call_move_overlap_field_first.rs:LL:CC + | +LL | / mir! { +LL | | let unit: (); +LL | | { +LL | | let value = const { (MaybeUninit::new(1u32), MaybeUninit::new(2u32)) }; +... | +LL | | done = { Return() } +LL | | } + | |_____^ +help: ALLOC was deallocated here: + --> tests/fail/move_elimination/call_move_overlap_field_first.rs:LL:CC + | +LL | Call(unit = field_whole(Move(value.0), Move(value)), ReturnTo(done), UnwindContinue()) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + = note: stack backtrace: + 0: field_whole + at tests/fail/move_elimination/call_move_overlap_field_first.rs:LL:CC + 1: main + at tests/fail/move_elimination/call_move_overlap_field_first.rs:LL:CC + = note: this error originates in the macro `::core::intrinsics::mir::__internal_extract_let` which comes from the expansion of the macro `mir` (in Nightly builds, run with -Z macro-backtrace for more info) + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_overlap_field_first.normal.stderr b/src/tools/miri/tests/fail/move_elimination/call_move_overlap_field_first.normal.stderr new file mode 100644 index 0000000000000..ffbbfe8dfe88b --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_overlap_field_first.normal.stderr @@ -0,0 +1,35 @@ +error: Undefined Behavior: not granting access to tag because that would remove [Unique for ] which is strongly protected + --> tests/fail/move_elimination/call_move_overlap_field_first.rs:LL:CC + | +LL | fn field_whole(_: MaybeUninit, _: Value) { + | ^ Undefined Behavior occurred here + | + = help: this indicates a potential bug in the program: it performed an invalid operation, but the Stacked Borrows rules it violated are still experimental + = help: see https://github.com/rust-lang/unsafe-code-guidelines/blob/master/wip/stacked-borrows.md for further information +help: was created here, as the root tag for ALLOC + --> tests/fail/move_elimination/call_move_overlap_field_first.rs:LL:CC + | +LL | / mir! { +LL | | let unit: (); +LL | | { +LL | | let value = const { (MaybeUninit::new(1u32), MaybeUninit::new(2u32)) }; +... | +LL | | done = { Return() } +LL | | } + | |_____^ +help: is this argument + --> tests/fail/move_elimination/call_move_overlap_field_first.rs:LL:CC + | +LL | fn field_whole(_: MaybeUninit, _: Value) { + | ^ + = note: stack backtrace: + 0: field_whole + at tests/fail/move_elimination/call_move_overlap_field_first.rs:LL:CC + 1: main + at tests/fail/move_elimination/call_move_overlap_field_first.rs:LL:CC + = note: this error originates in the macro `::core::intrinsics::mir::__internal_remove_let` which comes from the expansion of the macro `mir` (in Nightly builds, run with -Z macro-backtrace for more info) + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_overlap_field_first.rs b/src/tools/miri/tests/fail/move_elimination/call_move_overlap_field_first.rs new file mode 100644 index 0000000000000..92b790c11a397 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_overlap_field_first.rs @@ -0,0 +1,28 @@ +//@revisions: normal move_elimination +//@[move_elimination]compile-flags: -Zmir-move-elimination + +// A projected move retains its source place: a later whole move cannot hide the overlap. + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; +use std::mem::MaybeUninit; + +type Value = (MaybeUninit, MaybeUninit); + +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let unit: (); + { + let value = const { (MaybeUninit::new(1u32), MaybeUninit::new(2u32)) }; + Call(unit = field_whole(Move(value.0), Move(value)), ReturnTo(done), UnwindContinue()) + } + done = { Return() } + } +} + +// Permit uninitialized bytes so the normal revision tests protection, not value validity. +fn field_whole(_: MaybeUninit, _: Value) { + //~[normal]^ ERROR: protected + //~[move_elimination]| ERROR: has been freed +} diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_overlap_whole_first.move_elimination.stderr b/src/tools/miri/tests/fail/move_elimination/call_move_overlap_whole_first.move_elimination.stderr new file mode 100644 index 0000000000000..ba9bf71893ca4 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_overlap_whole_first.move_elimination.stderr @@ -0,0 +1,13 @@ +error: Undefined Behavior: accessing a live but unallocated local variable + --> tests/fail/move_elimination/call_move_overlap_whole_first.rs:LL:CC + | +LL | Call(unit = whole_field(Move(value), Move(value.0)), ReturnTo(done), UnwindContinue()) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_overlap_whole_first.normal.stderr b/src/tools/miri/tests/fail/move_elimination/call_move_overlap_whole_first.normal.stderr new file mode 100644 index 0000000000000..e73e627972109 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_overlap_whole_first.normal.stderr @@ -0,0 +1,35 @@ +error: Undefined Behavior: not granting access to tag because that would remove [Unique for ] which is strongly protected + --> tests/fail/move_elimination/call_move_overlap_whole_first.rs:LL:CC + | +LL | fn whole_field(_: Value, _: MaybeUninit) { + | ^ Undefined Behavior occurred here + | + = help: this indicates a potential bug in the program: it performed an invalid operation, but the Stacked Borrows rules it violated are still experimental + = help: see https://github.com/rust-lang/unsafe-code-guidelines/blob/master/wip/stacked-borrows.md for further information +help: was created here, as the root tag for ALLOC + --> tests/fail/move_elimination/call_move_overlap_whole_first.rs:LL:CC + | +LL | / mir! { +LL | | let unit: (); +LL | | { +LL | | let value = const { (MaybeUninit::new(1u32), MaybeUninit::new(2u32)) }; +... | +LL | | done = { Return() } +LL | | } + | |_____^ +help: is this argument + --> tests/fail/move_elimination/call_move_overlap_whole_first.rs:LL:CC + | +LL | fn whole_field(_: Value, _: MaybeUninit) { + | ^ + = note: stack backtrace: + 0: whole_field + at tests/fail/move_elimination/call_move_overlap_whole_first.rs:LL:CC + 1: main + at tests/fail/move_elimination/call_move_overlap_whole_first.rs:LL:CC + = note: this error originates in the macro `::core::intrinsics::mir::__internal_remove_let` which comes from the expansion of the macro `mir` (in Nightly builds, run with -Z macro-backtrace for more info) + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_overlap_whole_first.rs b/src/tools/miri/tests/fail/move_elimination/call_move_overlap_whole_first.rs new file mode 100644 index 0000000000000..d543add20b96c --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_overlap_whole_first.rs @@ -0,0 +1,28 @@ +//@revisions: normal move_elimination +//@[move_elimination]compile-flags: -Zmir-move-elimination + +// Moving a whole local makes its storage inaccessible to a later field move, +// through protection normally or deallocation under move-elimination semantics. + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; +use std::mem::MaybeUninit; + +type Value = (MaybeUninit, MaybeUninit); + +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let unit: (); + { + let value = const { (MaybeUninit::new(1u32), MaybeUninit::new(2u32)) }; + Call(unit = whole_field(Move(value), Move(value.0)), ReturnTo(done), UnwindContinue()) + //~[move_elimination]^ ERROR: accessing a live but unallocated local variable + } + done = { Return() } + } +} + +fn whole_field(_: Value, _: MaybeUninit) { + //~[normal]^ ERROR: protected +} diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_then_copy.rs b/src/tools/miri/tests/fail/move_elimination/call_move_then_copy.rs new file mode 100644 index 0000000000000..00816ee2ce53e --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_then_copy.rs @@ -0,0 +1,24 @@ +//@compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; +use std::mem::MaybeUninit; + +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let ptr: *const MaybeUninit<[u64; 4]>; + let unit: (); + { + let value = const { MaybeUninit::new([1u64; 4]) }; + ptr = &raw const value; + Call(unit = consume(Move(value), *ptr), ReturnTo(done), UnwindContinue()) + //~^ ERROR: has been freed + } + done = { Return() } + } +} + +// Merely writing uninit into the source would not reject this copy. Freeing the +// source must prevent reading it, even though MaybeUninit permits uninit bytes. +fn consume(_: MaybeUninit<[u64; 4]>, _: MaybeUninit<[u64; 4]>) {} diff --git a/src/tools/miri/tests/fail/move_elimination/call_move_then_copy.stderr b/src/tools/miri/tests/fail/move_elimination/call_move_then_copy.stderr new file mode 100644 index 0000000000000..fc68bb7a848b5 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/call_move_then_copy.stderr @@ -0,0 +1,29 @@ +error: Undefined Behavior: memory access failed: ALLOC has been freed, so this pointer is dangling + --> tests/fail/move_elimination/call_move_then_copy.rs:LL:CC + | +LL | Call(unit = consume(Move(value), *ptr), ReturnTo(done), UnwindContinue()) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information +help: ALLOC was allocated here: + --> tests/fail/move_elimination/call_move_then_copy.rs:LL:CC + | +LL | / mir! { +LL | | let ptr: *const MaybeUninit<[u64; 4]>; +LL | | let unit: (); +... | +LL | | done = { Return() } +LL | | } + | |_____^ +help: ALLOC was deallocated here: + --> tests/fail/move_elimination/call_move_then_copy.rs:LL:CC + | +LL | Call(unit = consume(Move(value), *ptr), ReturnTo(done), UnwindContinue()) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + = note: this error originates in the macro `::core::intrinsics::mir::__internal_extract_let` which comes from the expansion of the macro `mir` (in Nightly builds, run with -Z macro-backtrace for more info) + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/self_address_unallocated.raw.stderr b/src/tools/miri/tests/fail/move_elimination/self_address_unallocated.raw.stderr new file mode 100644 index 0000000000000..87d0127b8a6e5 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/self_address_unallocated.raw.stderr @@ -0,0 +1,13 @@ +error: Undefined Behavior: accessing a live but unallocated local variable + --> tests/fail/move_elimination/self_address_unallocated.rs:LL:CC + | +LL | value.ptr = &raw const value; + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/self_address_unallocated.reference.stderr b/src/tools/miri/tests/fail/move_elimination/self_address_unallocated.reference.stderr new file mode 100644 index 0000000000000..6c74a064bc774 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/self_address_unallocated.reference.stderr @@ -0,0 +1,13 @@ +error: Undefined Behavior: accessing a live but unallocated local variable + --> tests/fail/move_elimination/self_address_unallocated.rs:LL:CC + | +LL | value.ptr = &value; + | ^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/self_address_unallocated.rs b/src/tools/miri/tests/fail/move_elimination/self_address_unallocated.rs new file mode 100644 index 0000000000000..54ecab1697542 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/self_address_unallocated.rs @@ -0,0 +1,43 @@ +//@revisions: normal raw reference +//@[normal]check-pass +//@[raw]compile-flags: -Zmir-move-elimination +//@[reference]compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] + +use std::intrinsics::mir::*; + +// Evaluating the destination must not allocate the source before taking its address. +#[cfg(any(normal, raw))] +struct Node { + ptr: *const Node, +} + +#[cfg(reference)] +struct Node { + ptr: &'static Node, +} + +#[cfg(any(normal, raw))] +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let value: Node; + { + value.ptr = &raw const value; //~[raw] ERROR: live but unallocated + Return() + } + } +} + +#[cfg(reference)] +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let value: Node; + { + value.ptr = &value; //~[reference] ERROR: live but unallocated + Return() + } + } +} diff --git a/src/tools/miri/tests/fail/move_elimination/storage_alloc.rs b/src/tools/miri/tests/fail/move_elimination/storage_alloc.rs new file mode 100644 index 0000000000000..99af6663fb157 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/storage_alloc.rs @@ -0,0 +1,21 @@ +//@compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; + +// StorageAlloc provides storage, but does not initialize the value. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let value: u32; + let ptr: *const u32; + let result: u32; + { + StorageAlloc(value); + ptr = &raw const value; + result = *ptr; + //~^ ERROR: uninitialized + Return() + } + } +} diff --git a/src/tools/miri/tests/fail/move_elimination/storage_alloc.stderr b/src/tools/miri/tests/fail/move_elimination/storage_alloc.stderr new file mode 100644 index 0000000000000..bddf35ba0e73b --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/storage_alloc.stderr @@ -0,0 +1,18 @@ +error: Undefined Behavior: reading memory at ALLOC[0x0..0x4], but memory is uninitialized at [0x0..0x4], and this operation requires initialized memory + --> tests/fail/move_elimination/storage_alloc.rs:LL:CC + | +LL | result = *ptr; + | ^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + +Uninitialized memory occurred at ALLOC[0x0..0x4], in this allocation: +ALLOC (stack variable, size: 4, align: 4) { + __ __ __ __ │ ░░░░ +} + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/storage_alloc_dead.rs b/src/tools/miri/tests/fail/move_elimination/storage_alloc_dead.rs new file mode 100644 index 0000000000000..87ab6f80f8b2a --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/storage_alloc_dead.rs @@ -0,0 +1,17 @@ +//@compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; + +// StorageAlloc cannot start a storage lifetime. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let value: u32; + { + StorageAlloc(value); //~ ERROR: accessing a dead local variable + StorageLive(value); + Return() + } + } +} diff --git a/src/tools/miri/tests/fail/move_elimination/storage_alloc_dead.stderr b/src/tools/miri/tests/fail/move_elimination/storage_alloc_dead.stderr new file mode 100644 index 0000000000000..a5a9ce49a33d4 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/storage_alloc_dead.stderr @@ -0,0 +1,13 @@ +error: Undefined Behavior: accessing a dead local variable + --> tests/fail/move_elimination/storage_alloc_dead.rs:LL:CC + | +LL | StorageAlloc(value); + | ^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/unallocated_return.rs b/src/tools/miri/tests/fail/move_elimination/unallocated_return.rs new file mode 100644 index 0000000000000..0a5180a4051e7 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/unallocated_return.rs @@ -0,0 +1,18 @@ +//@compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; + +// Returning from a function requires its return local to be allocated. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn uninitialized() -> u32 { + mir! { + { + Return() //~ ERROR: accessing a live but unallocated local variable + } + } +} + +fn main() { + let _ = uninitialized(); +} diff --git a/src/tools/miri/tests/fail/move_elimination/unallocated_return.stderr b/src/tools/miri/tests/fail/move_elimination/unallocated_return.stderr new file mode 100644 index 0000000000000..89b212faca9e3 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/unallocated_return.stderr @@ -0,0 +1,18 @@ +error: Undefined Behavior: accessing a live but unallocated local variable + --> tests/fail/move_elimination/unallocated_return.rs:LL:CC + | +LL | Return() + | ^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + = note: stack backtrace: + 0: uninitialized + at tests/fail/move_elimination/unallocated_return.rs:LL:CC + 1: main + at tests/fail/move_elimination/unallocated_return.rs:LL:CC + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/use_after_move_in_statement.move_elimination.stderr b/src/tools/miri/tests/fail/move_elimination/use_after_move_in_statement.move_elimination.stderr new file mode 100644 index 0000000000000..cb81052db28e2 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/use_after_move_in_statement.move_elimination.stderr @@ -0,0 +1,29 @@ +error: Undefined Behavior: memory access failed: ALLOC has been freed, so this pointer is dangling + --> tests/fail/move_elimination/use_after_move_in_statement.rs:LL:CC + | +LL | result = (Move(value), (*ptr).0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information +help: ALLOC was allocated here: + --> tests/fail/move_elimination/use_after_move_in_statement.rs:LL:CC + | +LL | / mir! { +LL | | let value: (u8, u8); +LL | | let ptr: *const (u8, u8); +LL | | let result: ((u8, u8), u8); +... | +LL | | } + | |_____^ +help: ALLOC was deallocated here: + --> tests/fail/move_elimination/use_after_move_in_statement.rs:LL:CC + | +LL | result = (Move(value), (*ptr).0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + = note: this error originates in the macro `mir` (in Nightly builds, run with -Z macro-backtrace for more info) + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/use_after_move_in_statement.rs b/src/tools/miri/tests/fail/move_elimination/use_after_move_in_statement.rs new file mode 100644 index 0000000000000..db103d79dcf8e --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/use_after_move_in_statement.rs @@ -0,0 +1,22 @@ +//@revisions: normal move_elimination +//@[normal]check-pass +//@[move_elimination]compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] + +use std::intrinsics::mir::*; + +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let value: (u8, u8); + let ptr: *const (u8, u8); + let result: ((u8, u8), u8); + { + value = (1, 2); + ptr = &raw const value; + result = (Move(value), (*ptr).0); //~[move_elimination] ERROR: has been freed + Return() + } + } +} diff --git a/src/tools/miri/tests/pass/move_elimination/calls.rs b/src/tools/miri/tests/pass/move_elimination/calls.rs new file mode 100644 index 0000000000000..0b96cf9a0b338 --- /dev/null +++ b/src/tools/miri/tests/pass/move_elimination/calls.rs @@ -0,0 +1,77 @@ +//@revisions: stack tree +//@[tree]compile-flags: -Zmiri-tree-borrows +//@compile-flags: -Zmir-move-elimination + +// Check that valid calls preserve argument values without UB or leaked allocations. + +#![feature(core_intrinsics, custom_mir, fn_traits, unboxed_closures)] +use std::intrinsics::mir::*; + +// An earlier copy must complete before a later move frees the same source. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn copy_before_move() { + mir! { + let ptr: *const [u64; 4]; + { + let value = [1u64; 4]; + // Force the source into memory to exercise snapshotting. + ptr = &raw const value; + Call(RET = compare(value, Move(value)), ReturnTo(done), UnwindContinue()) + } + done = { Return() } + } +} +fn compare(a: [u64; 4], b: [u64; 4]) { + assert_eq!(a, [1; 4]); + assert_eq!(a, b); +} + +// Rust-call untupling must preserve every field of the moved tuple. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn call_once u64>(f: F, args: ([u64; 4], [u64; 4])) -> u64 { + mir! { + let f_ptr: *const F; + let args_ptr: *const ([u64; 4], [u64; 4]); + { + // Force both moved arguments into memory. + f_ptr = &raw const f; + args_ptr = &raw const args; + Call(RET = F::call_once(Move(f), Move(args)), ReturnTo(done), UnwindContinue()) + } + done = { Return() } + } +} + +// Capturing a projected move must not retain a dependency on its pointer local. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn projected_before_pointer() { + mir! { + let ptr: *mut u32; + { + let value = 42u32; + ptr = &raw mut value; + Call(RET = value_and_pointer(Move(*ptr), Move(ptr)), ReturnTo(done), UnwindContinue()) + } + done = { Return() } + } +} +fn value_and_pointer(value: u32, _: *mut u32) { + assert_eq!(value, 42); +} + +fn main() { + projected_before_pointer(); + copy_before_move(); + + let captured = Box::new([10u64; 4]); + let closure = move |a: [u64; 4], b: [u64; 4]| { + assert_eq!(a, [1; 4]); + assert_eq!(b, [2; 4]); + a.iter().sum::() + b.iter().sum::() + captured.iter().sum::() + }; + assert_eq!(call_once(closure, ([1; 4], [2; 4])), 52); + + // A call may unwind without having allocated its return local. + std::panic::set_hook(Box::new(|_| {})); + assert!(std::panic::catch_unwind(|| panic!("caught")).is_err()); +} diff --git a/src/tools/miri/tests/pass/move_elimination/operands.rs b/src/tools/miri/tests/pass/move_elimination/operands.rs new file mode 100644 index 0000000000000..1bc798ab1fa21 --- /dev/null +++ b/src/tools/miri/tests/pass/move_elimination/operands.rs @@ -0,0 +1,120 @@ +//@revisions: normal stack tree +//@[stack]compile-flags: -Zmir-move-elimination +//@[tree]compile-flags: -Zmir-move-elimination -Zmiri-tree-borrows + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; + +// A tail call must preserve an earlier copy before a later move frees its source. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn copy_before_move(value: [u64; 4]) -> u64 { + mir! { + let ptr: *const [u64; 4]; + { + // Force the source into memory to exercise snapshotting. + ptr = &raw const value; + TailCall(compare(value, Move(value))) + } + } +} +fn compare(a: [u64; 4], b: [u64; 4]) -> u64 { + assert_eq!(a, [42; 4]); + assert_eq!(a, b); + a[0] +} + +// A tail call must preserve a moved field before a later move frees the whole local. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn field_before_move(value: ([u64; 4], [u64; 4])) -> u64 { + mir! { + { + TailCall(compare_field(Move(value.0), Move(value))) + } + } +} +fn compare_field(field: [u64; 4], whole: ([u64; 4], [u64; 4])) -> u64 { + assert_eq!(field, whole.0); + whole.1[0] +} + +// Evaluate the function pointer before a tail-call argument frees its storage. +#[derive(Clone, Copy)] +struct Callback(fn(Callback) -> u64); +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn moved_callee_storage(callback: Callback) -> u64 { + mir! { + let ptr: *const Callback; + { + // Force the function pointer into memory before evaluating the call. + ptr = &raw const callback; + TailCall((callback.0)(Move(callback))) + } + } +} +fn callback(_: Callback) -> u64 { + 42 +} + +// Lowering produces a CopyNonOverlapping statement. Capture the source pointer +// before moving the local containing it; a zero count avoids overlapping accesses. +#[custom_mir(dialect = "runtime", phase = "initial")] +fn copy_zero(p: *mut u32) { + mir! { + let pp: *const *mut u32; + { + // Force the pointer local into memory to exercise snapshotting. + pp = &raw const p; + Call(RET = std::intrinsics::copy_nonoverlapping(p, Move(p), 0_usize), ReturnTo(done), UnwindContinue()) + } + done = { Return() } + } +} + +// Aggregate construction must preserve earlier fields before later operands free their sources. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn aggregate_before_move(value: [u64; 4]) -> ([u64; 4], [u64; 4]) { + mir! { + { + RET = (value, Move(value)); + Return() + } + } +} + +// A repeat keeps the moved value until all elements have been written. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn repeat_move(value: [u64; 4]) -> [[u64; 4]; 2] { + mir! { + { + RET = [Move(value); 2]; + Return() + } + } +} + +// Move elimination evaluates the move before reusing the same local as the destination. +// Without it, copying an aggregate onto itself is rejected as an overlapping copy. +#[cfg(any(stack, tree))] +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn self_move(mut value: [u64; 4]) -> [u64; 4] { + mir! { + { + value = Move(value); + RET = Move(value); + Return() + } + } +} + +fn main() { + assert_eq!(aggregate_before_move([1, 2, 3, 4]), ([1, 2, 3, 4], [1, 2, 3, 4])); + assert_eq!(repeat_move([1, 2, 3, 4]), [[1, 2, 3, 4]; 2]); + #[cfg(any(stack, tree))] + assert_eq!(self_move([1, 2, 3, 4]), [1, 2, 3, 4]); + assert_eq!(copy_before_move([42; 4]), 42); + assert_eq!(field_before_move(([1; 4], [2; 4])), 2); + assert_eq!(moved_callee_storage(Callback(callback)), 42); + let mut value = 42; + copy_zero(&raw mut value); + assert_eq!(value, 42); +} diff --git a/src/tools/miri/tests/pass/move_elimination/storage_alloc.rs b/src/tools/miri/tests/pass/move_elimination/storage_alloc.rs new file mode 100644 index 0000000000000..f9e78a6283f51 --- /dev/null +++ b/src/tools/miri/tests/pass/move_elimination/storage_alloc.rs @@ -0,0 +1,44 @@ +//@revisions: normal stack tree +//@[stack]compile-flags: -Zmir-move-elimination +//@[tree]compile-flags: -Zmir-move-elimination -Zmiri-tree-borrows + +#![feature(core_intrinsics, custom_mir)] +use std::intrinsics::mir::*; + +// Allocate before taking an address, then preserve the value and pointer across +// another StorageAlloc. It must not reset storage or perform a conflicting access. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn allocated() -> u32 { + mir! { + let value: u32; + let ptr: *mut u32; + { + StorageLive(value); + StorageAlloc(value); + ptr = &raw mut value; + *ptr = 42; + StorageAlloc(value); + RET = *ptr; + StorageDead(value); + Return() + } + } +} + +// Allocation must also preserve a value held as an immediate. +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn immediate() -> u32 { + mir! { + { + let value = 42_u32; + StorageAlloc(value); + RET = value; + Return() + } + } +} + +fn main() { + assert_eq!(allocated(), 42); + assert_eq!(immediate(), 42); +} diff --git a/tests/codegen-llvm/hint/cold_path-target_feature.rs b/tests/codegen-llvm/hint/cold_path-target_feature.rs index 9e06fd0ceafa1..899f8765ab17b 100644 --- a/tests/codegen-llvm/hint/cold_path-target_feature.rs +++ b/tests/codegen-llvm/hint/cold_path-target_feature.rs @@ -28,10 +28,10 @@ pub fn test1(x: bool) { } // CHECK-LABEL: @test1( - // CHECK: br i1 %x, label %bb1, label %bb2, !prof ![[NUM:[0-9]+]] - // CHECK: bb2: + // CHECK: br i1 %x, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM:[0-9]+]] + // CHECK: [[FALSE]]: // CHECK: path_b - // CHECK: bb1: + // CHECK: [[TRUE]]: // CHECK: path_a } @@ -46,10 +46,10 @@ pub fn with_target_feature(x: bool) { } // CHECK-LABEL: @with_target_feature( - // CHECK: br i1 %x, label %bb1, label %bb2, !prof ![[NUM]] - // CHECK: bb2: + // CHECK: br i1 %x, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM]] + // CHECK: [[FALSE]]: // CHECK: path_b - // CHECK: bb1: + // CHECK: [[TRUE]]: // CHECK: path_a } diff --git a/tests/codegen-llvm/hint/cold_path.rs b/tests/codegen-llvm/hint/cold_path.rs index e6e7379322ddf..c8da7af840790 100644 --- a/tests/codegen-llvm/hint/cold_path.rs +++ b/tests/codegen-llvm/hint/cold_path.rs @@ -25,10 +25,10 @@ pub fn test1(x: bool) { } // CHECK-LABEL: @test1( - // CHECK: br i1 %x, label %bb1, label %bb2, !prof ![[NUM:[0-9]+]] - // CHECK: bb2: + // CHECK: br i1 %x, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM:[0-9]+]] + // CHECK: [[FALSE]]: // CHECK: path_b - // CHECK: bb1: + // CHECK: [[TRUE]]: // CHECK: path_a } @@ -43,10 +43,10 @@ pub fn test2(x: i32) { } // CHECK-LABEL: @test2( - // CHECK: br i1 %_2, label %bb2, label %bb1, !prof ![[NUM]] - // CHECK: bb1: + // CHECK: br i1 %_2, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM]] + // CHECK: [[FALSE]]: // CHECK: path_b - // CHECK: bb2: + // CHECK: [[TRUE]]: // CHECK: path_a } diff --git a/tests/codegen-llvm/hint/likely.rs b/tests/codegen-llvm/hint/likely.rs index 75f9e7aae367d..3f1b2bfbb8091 100644 --- a/tests/codegen-llvm/hint/likely.rs +++ b/tests/codegen-llvm/hint/likely.rs @@ -25,10 +25,10 @@ pub fn test1(x: bool) { } // CHECK-LABEL: @test1( - // CHECK: br i1 %x, label %bb2, label %bb3, !prof ![[NUM:[0-9]+]] - // CHECK: bb3: + // CHECK: br i1 %x, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM:[0-9]+]] + // CHECK: [[FALSE]]: // CHECK: path_b - // CHECK: bb2: + // CHECK: [[TRUE]]: // CHECK: path_a } @@ -40,10 +40,10 @@ pub fn test2(x: i32) { } // CHECK-LABEL: @test2( - // CHECK: br i1 %_2, label %bb2, label %bb3, !prof ![[NUM]] - // CHECK: bb3: + // CHECK: br i1 %_2, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM]] + // CHECK: [[FALSE]]: // CHECK: path_b - // CHECK: bb2: + // CHECK: [[TRUE]]: // CHECK: path_a } @@ -55,10 +55,10 @@ pub fn test3(x: i8) { } // CHECK-LABEL: @test3( - // CHECK: br i1 %_2, label %bb2, label %bb3, !prof ![[NUM]] - // CHECK: bb3: + // CHECK: br i1 %_2, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM]] + // CHECK: [[FALSE]]: // CHECK: path_b - // CHECK: bb2: + // CHECK: [[TRUE]]: // CHECK: path_a } @@ -70,10 +70,10 @@ pub fn test4(x: u64) { } // CHECK-LABEL: @test4( - // CHECK: br i1 %0, label %bb3, label %bb2, !prof ![[NUM2:[0-9]+]] - // CHECK: bb3: + // CHECK: br i1 %0, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM2:[0-9]+]] + // CHECK: [[TRUE]]: // CHECK: path_a - // CHECK: bb2: + // CHECK: [[FALSE]]: // CHECK: path_b } diff --git a/tests/codegen-llvm/hint/unlikely.rs b/tests/codegen-llvm/hint/unlikely.rs index 248b1e2537e96..0376e2ebb23f4 100644 --- a/tests/codegen-llvm/hint/unlikely.rs +++ b/tests/codegen-llvm/hint/unlikely.rs @@ -25,10 +25,10 @@ pub fn test1(x: bool) { } // CHECK-LABEL: @test1( - // CHECK: br i1 %x, label %bb2, label %bb4, !prof ![[NUM:[0-9]+]] - // CHECK: bb4: + // CHECK: br i1 %x, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM:[0-9]+]] + // CHECK: [[FALSE]]: // CHECK: path_b - // CHECK: bb2: + // CHECK: [[TRUE]]: // CHECK: path_a } @@ -40,10 +40,10 @@ pub fn test2(x: i32) { } // CHECK-LABEL: @test2( - // CHECK: br i1 %_2, label %bb2, label %bb4, !prof ![[NUM]] - // CHECK: bb4: + // CHECK: br i1 %_2, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM]] + // CHECK: [[FALSE]]: // CHECK: path_b - // CHECK: bb2: + // CHECK: [[TRUE]]: // CHECK: path_a } @@ -55,10 +55,10 @@ pub fn test3(x: i8) { } // CHECK-LABEL: @test3( - // CHECK: br i1 %_2, label %bb2, label %bb4, !prof ![[NUM]] - // CHECK: bb4: + // CHECK: br i1 %_2, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM]] + // CHECK: [[FALSE]]: // CHECK: path_b - // CHECK: bb2: + // CHECK: [[TRUE]]: // CHECK: path_a } @@ -70,10 +70,10 @@ pub fn test4(x: u64) { } // CHECK-LABEL: @test4( - // CHECK: br i1 %0, label %bb4, label %bb2, !prof ![[NUM2:[0-9]+]] - // CHECK: bb4: + // CHECK: br i1 %0, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM2:[0-9]+]] + // CHECK: [[TRUE]]: // CHECK: path_a - // CHECK: bb2: + // CHECK: [[FALSE]]: // CHECK: path_b } diff --git a/tests/codegen-llvm/intrinsics/cold_path2.rs b/tests/codegen-llvm/intrinsics/cold_path2.rs index 0891c878fd9c5..e22bdbb96437b 100644 --- a/tests/codegen-llvm/intrinsics/cold_path2.rs +++ b/tests/codegen-llvm/intrinsics/cold_path2.rs @@ -27,10 +27,10 @@ pub fn test(x: Option) { // CHECK-LABEL: void @test(i8{{.+}}%x) // CHECK: %[[IS_NONE:.+]] = icmp eq i8 %x, 2 - // CHECK: br i1 %[[IS_NONE]], label %bb2, label %bb1, !prof ![[NUM:[0-9]+]] - // CHECK: bb1: + // CHECK: br i1 %[[IS_NONE]], label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM:[0-9]+]] + // CHECK: [[FALSE]]: // CHECK: path_a - // CHECK: bb2: + // CHECK: [[TRUE]]: // CHECK: path_b } diff --git a/tests/codegen-llvm/intrinsics/likely.rs b/tests/codegen-llvm/intrinsics/likely.rs index c5e3c466f4525..72fbe0c1b4b56 100644 --- a/tests/codegen-llvm/intrinsics/likely.rs +++ b/tests/codegen-llvm/intrinsics/likely.rs @@ -26,10 +26,10 @@ pub fn test_likely(x: bool) { } // CHECK-LABEL: @test_likely( -// CHECK: br i1 %x, label %bb2, label %bb3, !prof ![[NUM:[0-9]+]] -// CHECK: bb3: +// CHECK: br i1 %x, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM:[0-9]+]] +// CHECK: [[FALSE]]: // CHECK-NOT: cold_path // CHECK: path_b -// CHECK: bb2: +// CHECK: [[TRUE]]: // CHECK: path_a // CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 2000, i32 1} diff --git a/tests/codegen-llvm/intrinsics/unlikely.rs b/tests/codegen-llvm/intrinsics/unlikely.rs index 90ebf070d2700..6a95476e626dc 100644 --- a/tests/codegen-llvm/intrinsics/unlikely.rs +++ b/tests/codegen-llvm/intrinsics/unlikely.rs @@ -26,10 +26,10 @@ pub fn test_unlikely(x: bool) { } // CHECK-LABEL: @test_unlikely( -// CHECK: br i1 %x, label %bb2, label %bb4, !prof ![[NUM:[0-9]+]] -// CHECK: bb4: +// CHECK: br i1 %x, label %[[TRUE:bb[0-9]+]], label %[[FALSE:bb[0-9]+]], !prof ![[NUM:[0-9]+]] +// CHECK: [[FALSE]]: // CHECK: path_b -// CHECK: bb2: +// CHECK: [[TRUE]]: // CHECK-NOT: cold_path // CHECK: path_a // CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 1, i32 2000} diff --git a/tests/codegen-llvm/lib-optimizations/append-elements.rs b/tests/codegen-llvm/lib-optimizations/append-elements.rs index 47211e3f388a0..a17108d0de0ca 100644 --- a/tests/codegen-llvm/lib-optimizations/append-elements.rs +++ b/tests/codegen-llvm/lib-optimizations/append-elements.rs @@ -1,4 +1,4 @@ -//@ compile-flags: -O -Zmerge-functions=disabled +//@ compile-flags: -O -Zmerge-functions=disabled -Zinline-mir-threshold=52 //@ needs-deterministic-layouts //@ ignore-std-debug-assertions (causes different value naming) #![crate_type = "lib"] diff --git a/tests/incremental/hashes/loop_expressions.rs b/tests/incremental/hashes/loop_expressions.rs index cb21fca7579be..c85748d89f356 100644 --- a/tests/incremental/hashes/loop_expressions.rs +++ b/tests/incremental/hashes/loop_expressions.rs @@ -106,7 +106,7 @@ pub fn add_loop_label_to_break() { #[cfg(not(any(bpass1,bpass4)))] #[rustc_clean(cfg="bpass2", except="hir_owner")] #[rustc_clean(cfg="bpass3")] -#[rustc_clean(cfg="bpass5", except="hir_owner")] +#[rustc_clean(cfg="bpass5", except="hir_owner,optimized_mir")] #[rustc_clean(cfg="bpass6")] pub fn add_loop_label_to_break() { let mut _x = 0; diff --git a/tests/incremental/hashes/trait_impls.rs b/tests/incremental/hashes/trait_impls.rs index 9698ec0473c4b..3bb1e12ed6677 100644 --- a/tests/incremental/hashes/trait_impls.rs +++ b/tests/incremental/hashes/trait_impls.rs @@ -81,7 +81,7 @@ impl ChangeMethodBodyTrait for Foo { impl ChangeMethodBodyTrait for Foo { #[rustc_clean(except="hir_owner,typeck_root", cfg="bpass2")] #[rustc_clean(cfg="bpass3")] - #[rustc_clean(except="hir_owner,typeck_root", cfg="bpass5")] + #[rustc_clean(except="hir_owner,typeck_root,optimized_mir", cfg="bpass5")] #[rustc_clean(cfg="bpass6")] fn method_name() { () diff --git a/tests/incremental/hashes/while_let_loops.rs b/tests/incremental/hashes/while_let_loops.rs index 780775b51930f..18cc32296fa92 100644 --- a/tests/incremental/hashes/while_let_loops.rs +++ b/tests/incremental/hashes/while_let_loops.rs @@ -31,7 +31,7 @@ pub fn change_loop_body() { #[cfg(not(any(bpass1,bpass4)))] #[rustc_clean(cfg="bpass2", except="hir_owner")] #[rustc_clean(cfg="bpass3")] -#[rustc_clean(cfg="bpass5", except="hir_owner")] +#[rustc_clean(cfg="bpass5", except="hir_owner,optimized_mir")] #[rustc_clean(cfg="bpass6")] pub fn change_loop_body() { let mut _x = 0; @@ -56,7 +56,7 @@ pub fn change_loop_condition() { #[cfg(not(any(bpass1,bpass4)))] #[rustc_clean(cfg="bpass2", except="hir_owner")] #[rustc_clean(cfg="bpass3")] -#[rustc_clean(cfg="bpass5", except="hir_owner")] +#[rustc_clean(cfg="bpass5", except="hir_owner,optimized_mir")] #[rustc_clean(cfg="bpass6")] pub fn change_loop_condition() { let mut _x = 0; @@ -81,7 +81,7 @@ pub fn add_break() { #[cfg(not(any(bpass1,bpass4)))] #[rustc_clean(cfg="bpass2", except="hir_owner, typeck_root")] #[rustc_clean(cfg="bpass3")] -#[rustc_clean(cfg="bpass5", except="hir_owner, typeck_root")] +#[rustc_clean(cfg="bpass5", except="hir_owner, typeck_root, optimized_mir")] #[rustc_clean(cfg="bpass6")] pub fn add_break() { let mut _x = 0; @@ -106,7 +106,7 @@ pub fn add_loop_label() { #[cfg(not(any(bpass1,bpass4)))] #[rustc_clean(cfg="bpass2", except="hir_owner")] #[rustc_clean(cfg="bpass3")] -#[rustc_clean(cfg="bpass5", except="hir_owner")] +#[rustc_clean(cfg="bpass5", except="hir_owner,optimized_mir")] #[rustc_clean(cfg="bpass6")] pub fn add_loop_label() { let mut _x = 0; @@ -131,7 +131,7 @@ pub fn add_loop_label_to_break() { #[cfg(not(any(bpass1,bpass4)))] #[rustc_clean(cfg="bpass2", except="hir_owner")] #[rustc_clean(cfg="bpass3")] -#[rustc_clean(cfg="bpass5", except="hir_owner")] +#[rustc_clean(cfg="bpass5", except="hir_owner,optimized_mir")] #[rustc_clean(cfg="bpass6")] pub fn add_loop_label_to_break() { let mut _x = 0; @@ -158,7 +158,7 @@ pub fn change_break_label() { #[cfg(not(any(bpass1,bpass4)))] #[rustc_clean(cfg="bpass2", except="hir_owner")] #[rustc_clean(cfg="bpass3")] -#[rustc_clean(cfg="bpass5", except="hir_owner")] +#[rustc_clean(cfg="bpass5", except="hir_owner,optimized_mir")] #[rustc_clean(cfg="bpass6")] pub fn change_break_label() { let mut _x = 0; @@ -183,7 +183,7 @@ pub fn add_loop_label_to_continue() { #[cfg(not(any(bpass1,bpass4)))] #[rustc_clean(cfg="bpass2", except="hir_owner")] #[rustc_clean(cfg="bpass3")] -#[rustc_clean(cfg="bpass5", except="hir_owner")] +#[rustc_clean(cfg="bpass5", except="hir_owner,optimized_mir")] #[rustc_clean(cfg="bpass6")] pub fn add_loop_label_to_continue() { let mut _x = 0; @@ -210,7 +210,7 @@ pub fn change_continue_label() { #[cfg(not(any(bpass1,bpass4)))] #[rustc_clean(cfg="bpass2", except="hir_owner")] #[rustc_clean(cfg="bpass3")] -#[rustc_clean(cfg="bpass5", except="hir_owner")] +#[rustc_clean(cfg="bpass5", except="hir_owner,optimized_mir")] #[rustc_clean(cfg="bpass6")] pub fn change_continue_label() { let mut _x = 0; @@ -237,7 +237,7 @@ pub fn change_continue_to_break() { #[cfg(not(any(bpass1,bpass4)))] #[rustc_clean(cfg="bpass2", except="hir_owner")] #[rustc_clean(cfg="bpass3")] -#[rustc_clean(cfg="bpass5", except="hir_owner")] +#[rustc_clean(cfg="bpass5", except="hir_owner,optimized_mir")] #[rustc_clean(cfg="bpass6")] pub fn change_continue_to_break() { let mut _x = 0; diff --git a/tests/incremental/hashes/while_loops.rs b/tests/incremental/hashes/while_loops.rs index 4b5af798c0efa..903e1017bd2a4 100644 --- a/tests/incremental/hashes/while_loops.rs +++ b/tests/incremental/hashes/while_loops.rs @@ -131,7 +131,7 @@ pub fn add_loop_label_to_break() { #[cfg(not(any(bpass1,bpass4)))] #[rustc_clean(cfg="bpass2", except="hir_owner")] #[rustc_clean(cfg="bpass3")] -#[rustc_clean(cfg="bpass5", except="hir_owner")] +#[rustc_clean(cfg="bpass5", except="hir_owner,optimized_mir")] #[rustc_clean(cfg="bpass6")] pub fn add_loop_label_to_break() { let mut _x = 0; diff --git a/tests/mir-opt/building/write_via_move.box_new.CleanupPostBorrowck.after.mir b/tests/mir-opt/building/write_via_move.box_new.CleanupPostBorrowck.after.mir index 6a6e984023b75..7a13128131664 100644 --- a/tests/mir-opt/building/write_via_move.box_new.CleanupPostBorrowck.after.mir +++ b/tests/mir-opt/building/write_via_move.box_new.CleanupPostBorrowck.after.mir @@ -44,6 +44,7 @@ fn box_new(_1: T) -> Box<[T; 1024]> { _8 = copy _1; (*_7) = [move _8; 1024]; StorageDead(_8); + _6 = const (); StorageDead(_7); StorageDead(_6); StorageLive(_9); diff --git a/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff b/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff index 2ecf41638125e..0c257693d4bc3 100644 --- a/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff +++ b/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff @@ -14,6 +14,7 @@ } bb2: { + _0 = const (); return; } } diff --git a/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff b/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff index 06287b670dd4f..4be6f1d3ed90a 100644 --- a/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff +++ b/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff @@ -14,6 +14,7 @@ } bb2: { + _0 = const (); return; } } diff --git a/tests/mir-opt/const_prop/control_flow_simplification.hello.runtime-optimized.after.panic-abort.mir b/tests/mir-opt/const_prop/control_flow_simplification.hello.runtime-optimized.after.panic-abort.mir index 86eda4942a7cb..b27b099456e47 100644 --- a/tests/mir-opt/const_prop/control_flow_simplification.hello.runtime-optimized.after.panic-abort.mir +++ b/tests/mir-opt/const_prop/control_flow_simplification.hello.runtime-optimized.after.panic-abort.mir @@ -4,6 +4,7 @@ fn hello() -> () { let mut _0: (); bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/const_prop/control_flow_simplification.hello.runtime-optimized.after.panic-unwind.mir b/tests/mir-opt/const_prop/control_flow_simplification.hello.runtime-optimized.after.panic-unwind.mir index 86eda4942a7cb..b27b099456e47 100644 --- a/tests/mir-opt/const_prop/control_flow_simplification.hello.runtime-optimized.after.panic-unwind.mir +++ b/tests/mir-opt/const_prop/control_flow_simplification.hello.runtime-optimized.after.panic-unwind.mir @@ -4,6 +4,7 @@ fn hello() -> () { let mut _0: (); bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/const_prop/control_flow_simplification.rs b/tests/mir-opt/const_prop/control_flow_simplification.rs index 640579a3ebeca..9865c626aaed5 100644 --- a/tests/mir-opt/const_prop/control_flow_simplification.rs +++ b/tests/mir-opt/const_prop/control_flow_simplification.rs @@ -13,6 +13,7 @@ impl NeedsDrop for This {} fn hello() { // CHECK-LABEL: fn hello( // CHECK: bb0: + // CHECK-NEXT: _0 = const (); // CHECK-NEXT: return; if ::NEEDS { panic!() diff --git a/tests/mir-opt/const_prop/inherit_overflow.main.GVN.panic-abort.diff b/tests/mir-opt/const_prop/inherit_overflow.main.GVN.panic-abort.diff index ab39c5be4e37d..a4016145fa7c5 100644 --- a/tests/mir-opt/const_prop/inherit_overflow.main.GVN.panic-abort.diff +++ b/tests/mir-opt/const_prop/inherit_overflow.main.GVN.panic-abort.diff @@ -14,6 +14,7 @@ bb0: { StorageLive(_1); + StorageAlloc(_1); StorageLive(_2); _2 = const u8::MAX; StorageLive(_3); diff --git a/tests/mir-opt/const_prop/inherit_overflow.main.GVN.panic-unwind.diff b/tests/mir-opt/const_prop/inherit_overflow.main.GVN.panic-unwind.diff index 1aea0dbb5d531..cbddcc05e12d3 100644 --- a/tests/mir-opt/const_prop/inherit_overflow.main.GVN.panic-unwind.diff +++ b/tests/mir-opt/const_prop/inherit_overflow.main.GVN.panic-unwind.diff @@ -14,6 +14,7 @@ bb0: { StorageLive(_1); + StorageAlloc(_1); StorageLive(_2); _2 = const u8::MAX; StorageLive(_3); diff --git a/tests/mir-opt/const_prop/invalid_constant.main.GVN.diff b/tests/mir-opt/const_prop/invalid_constant.main.GVN.diff index 3dc6fc9e3f65b..71e63d852ca0c 100644 --- a/tests/mir-opt/const_prop/invalid_constant.main.GVN.diff +++ b/tests/mir-opt/const_prop/invalid_constant.main.GVN.diff @@ -49,12 +49,13 @@ StorageLive(_8); - _8 = NoVariants { int: const 0_u32 }; + _8 = const NoVariants {{ int: 0_u32, empty: ZeroSized: Empty }}; - nop; - nop; + _7 = const ZeroSized: Empty; +- _6 = [const ZeroSized: Empty]; ++ _6 = const [ZeroSized: Empty]; nop; StorageDead(_8); nop; - nop; + _0 = const (); nop; nop; StorageDead(_3); diff --git a/tests/mir-opt/const_prop/invalid_constant.main.RemoveZsts.diff b/tests/mir-opt/const_prop/invalid_constant.main.RemoveZsts.diff index c16cbaf4c0f27..3f0d1de0da83d 100644 --- a/tests/mir-opt/const_prop/invalid_constant.main.RemoveZsts.diff +++ b/tests/mir-opt/const_prop/invalid_constant.main.RemoveZsts.diff @@ -50,17 +50,16 @@ - _7 = copy (_8.1: Empty); - _6 = [move _7]; - StorageDead(_7); -+ nop; -+ nop; ++ _7 = const ZeroSized: Empty; ++ _6 = [const ZeroSized: Empty]; + nop; StorageDead(_8); - StorageLive(_9); -- _0 = const (); ++ nop; + _0 = const (); - StorageDead(_9); - StorageDead(_6); + nop; -+ nop; -+ nop; + nop; StorageDead(_3); StorageDead(_1); diff --git a/tests/mir-opt/const_prop/trivial_const.unwrap_unchecked.SimplifyConstCondition-after-inst-simplify.diff b/tests/mir-opt/const_prop/trivial_const.unwrap_unchecked.SimplifyConstCondition-after-inst-simplify.diff index 4714e159396e3..9179fdcebb0dc 100644 --- a/tests/mir-opt/const_prop/trivial_const.unwrap_unchecked.SimplifyConstCondition-after-inst-simplify.diff +++ b/tests/mir-opt/const_prop/trivial_const.unwrap_unchecked.SimplifyConstCondition-after-inst-simplify.diff @@ -26,6 +26,7 @@ StorageLive(_2); StorageLive(_3); _3 = copy (*_1); + StorageAlloc(_2); StorageLive(_4); StorageLive(_5); _4 = discriminant(_3); diff --git a/tests/mir-opt/const_prop/union.main.GVN.diff b/tests/mir-opt/const_prop/union.main.GVN.diff index 4c3e0bb68fd43..959c8ccbeed55 100644 --- a/tests/mir-opt/const_prop/union.main.GVN.diff +++ b/tests/mir-opt/const_prop/union.main.GVN.diff @@ -18,6 +18,7 @@ bb0: { StorageLive(_1); StorageLive(_2); + StorageAlloc(_2); _2 = const 1_u32; - _1 = Un { us: move _2 }; + _1 = const Un {{ us: 1_u32 }}; @@ -26,6 +27,8 @@ StorageLive(_4); - _4 = copy (_1.0: u32); + _4 = const 1_u32; + StorageAlloc(_3); + _3 = const (); StorageDead(_4); StorageDead(_3); _0 = const (); diff --git a/tests/mir-opt/copy-prop/storage_alloc.before_source.CopyProp.diff b/tests/mir-opt/copy-prop/storage_alloc.before_source.CopyProp.diff new file mode 100644 index 0000000000000..3c8f6993081b9 --- /dev/null +++ b/tests/mir-opt/copy-prop/storage_alloc.before_source.CopyProp.diff @@ -0,0 +1,23 @@ +- // MIR for `before_source` before CopyProp ++ // MIR for `before_source` after CopyProp + + fn before_source() -> u8 { + let mut _0: u8; + let mut _1: u8; + let mut _2: u8; + + bb0: { +- StorageLive(_2); +- StorageAlloc(_2); +- StorageLive(_1); ++ StorageAlloc(_1); + _1 = const 42_u8; +- _2 = copy _1; +- _0 = Add(copy _2, const 1_u8); +- StorageDead(_1); +- StorageDead(_2); ++ _0 = Add(copy _1, const 1_u8); + return; + } + } + diff --git a/tests/mir-opt/copy-prop/storage_alloc.rs b/tests/mir-opt/copy-prop/storage_alloc.rs new file mode 100644 index 0000000000000..0a147ec5612fa --- /dev/null +++ b/tests/mir-opt/copy-prop/storage_alloc.rs @@ -0,0 +1,36 @@ +//@ test-mir-pass: CopyProp + +#![crate_type = "lib"] +#![feature(custom_mir, core_intrinsics)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +// Ensure the copy source's storage lifetime covers the redirected StorageAlloc. +// EMIT_MIR storage_alloc.before_source.CopyProp.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn before_source() -> u8 { + // CHECK-LABEL: fn before_source( + // CHECK: bb0: { + // CHECK-NOT: StorageLive + // CHECK: StorageAlloc([[SOURCE:_[0-9]+]]); + // CHECK-NOT: StorageLive + // CHECK: [[SOURCE]] = const 42_u8; + // CHECK-NOT: StorageDead + // CHECK: return; + mir! { + let x: u8; + let y: u8; + { + StorageLive(y); + StorageAlloc(y); + StorageLive(x); + x = 42; + y = x; + RET = y + 1; + StorageDead(x); + StorageDead(y); + Return() + } + } +} diff --git a/tests/mir-opt/coroutine/async_drop.array-{closure#0}.ElaborateDrops.diff b/tests/mir-opt/coroutine/async_drop.array-{closure#0}.ElaborateDrops.diff index bad684407ae57..4dbde57214c7e 100644 --- a/tests/mir-opt/coroutine/async_drop.array-{closure#0}.ElaborateDrops.diff +++ b/tests/mir-opt/coroutine/async_drop.array-{closure#0}.ElaborateDrops.diff @@ -162,7 +162,7 @@ + } + + bb24: { -+ _12 = move _2; ++ _12 = copy _2; + _11 = std::future::get_context::<'_, '_>(move _12) -> [return: bb23, unwind: bb16]; + } + @@ -198,7 +198,7 @@ + } + + bb31: { -+ _19 = move _2; ++ _19 = copy _2; + _18 = std::future::get_context::<'_, '_>(move _19) -> [return: bb30, unwind: bb16]; + } + diff --git a/tests/mir-opt/coroutine/async_drop.array-{closure#0}.StateTransform.diff b/tests/mir-opt/coroutine/async_drop.array-{closure#0}.StateTransform.diff index f80266a1b2d00..d1ae79a134a20 100644 --- a/tests/mir-opt/coroutine/async_drop.array-{closure#0}.StateTransform.diff +++ b/tests/mir-opt/coroutine/async_drop.array-{closure#0}.StateTransform.diff @@ -176,7 +176,7 @@ bb16: { - unreachable; -+ _19 = move _26; ++ _19 = copy _26; + _18 = copy (_19.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb15; } @@ -195,7 +195,7 @@ } bb19: { -- _12 = move _2; +- _12 = copy _2; - _11 = std::future::get_context::<'_, '_>(move _12) -> [return: bb18, unwind: bb11]; + _22 = &mut (((*_25) as variant#4).1: [AsyncInt; 2]); + _21 = Pin::<&mut [AsyncInt; 2]>::new_unchecked(move _22) -> [return: bb18, unwind: bb5]; @@ -244,7 +244,7 @@ - } - - bb26: { -- _19 = move _2; +- _19 = copy _2; - _18 = std::future::get_context::<'_, '_>(move _19) -> [return: bb25, unwind: bb11]; - } - diff --git a/tests/mir-opt/coroutine/async_drop.array-{closure#0}.coroutine_drop_async.0.mir b/tests/mir-opt/coroutine/async_drop.array-{closure#0}.coroutine_drop_async.0.mir index 632fc80a968c2..f61a09a488508 100644 --- a/tests/mir-opt/coroutine/async_drop.array-{closure#0}.coroutine_drop_async.0.mir +++ b/tests/mir-opt/coroutine/async_drop.array-{closure#0}.coroutine_drop_async.0.mir @@ -97,7 +97,7 @@ fn array::{closure#0}(_1: Pin<&mut {async fn body of array()}>, _2: &mut Context } bb12: { - _12 = move _26; + _12 = copy _26; _11 = copy (_12.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); goto -> bb11; } diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncEnum.MentionedItems.after.mir b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncEnum.MentionedItems.after.mir index 0041816abd0ff..b85ebe9515073 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncEnum.MentionedItems.after.mir +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncEnum.MentionedItems.after.mir @@ -59,6 +59,7 @@ yields () } bb1: { + _0 = const (); return; } @@ -119,7 +120,7 @@ yields () } bb14: { - _10 = move _2; + _10 = copy _2; _9 = std::future::get_context::<'_, '_>(move _10) -> [return: bb13, unwind: bb6]; } @@ -184,7 +185,7 @@ yields () } bb27: { - _20 = move _2; + _20 = copy _2; _19 = std::future::get_context::<'_, '_>(move _20) -> [return: bb26, unwind: bb20]; } @@ -220,7 +221,7 @@ yields () } bb34: { - _27 = move _2; + _27 = copy _2; _26 = std::future::get_context::<'_, '_>(move _27) -> [return: bb33, unwind: bb20]; } @@ -316,7 +317,7 @@ yields () } bb54: { - _40 = move _2; + _40 = copy _2; _39 = std::future::get_context::<'_, '_>(move _40) -> [return: bb53, unwind: bb47]; } @@ -352,7 +353,7 @@ yields () } bb61: { - _47 = move _2; + _47 = copy _2; _46 = std::future::get_context::<'_, '_>(move _47) -> [return: bb60, unwind: bb47]; } diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncEnum.StateTransform.diff b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncEnum.StateTransform.diff index 8e6c68598e311..be9956f5eddaa 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncEnum.StateTransform.diff +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncEnum.StateTransform.diff @@ -97,6 +97,8 @@ } bb1: { +- _0 = const (); ++ _51 = const (); + _0 = Poll::<()>::Ready(move _51); + discriminant((*_53)) = 1; return; @@ -179,7 +181,7 @@ } bb13: { -- _10 = move _2; +- _10 = copy _2; - _9 = std::future::get_context::<'_, '_>(move _10) -> [return: bb12, unwind: bb5]; + StorageLive(_22); + _0 = Poll::<()>::Pending; @@ -204,7 +206,7 @@ bb16: { - _13 = &mut (((*_3) as A).0: AsyncInt); - _12 = Pin::<&mut AsyncInt>::new_unchecked(move _13) -> [return: bb15, unwind: bb2]; -+ _27 = move _61; ++ _27 = copy _61; + _26 = copy (_27.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb15; } @@ -280,7 +282,7 @@ } bb26: { -- _20 = move _2; +- _20 = copy _2; - _19 = std::future::get_context::<'_, '_>(move _20) -> [return: bb25, unwind: bb19]; + _61 = move _35; + StorageDead(_35); @@ -323,7 +325,7 @@ bb31: { - _24 = discriminant(_23); - switchInt(move _24) -> [0: bb17, 1: bb30, otherwise: bb10]; -+ _47 = move _61; ++ _47 = copy _61; + _46 = copy (_47.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb30; } @@ -335,7 +337,7 @@ } bb33: { -- _27 = move _2; +- _27 = copy _2; - _26 = std::future::get_context::<'_, '_>(move _27) -> [return: bb32, unwind: bb19]; + nop; + (((*_53) as variant#7).1: impl std::future::Future) = ::drop(move _49) -> [return: bb32, unwind: bb24]; @@ -444,7 +446,7 @@ - } - - bb50: { -- _40 = move _2; +- _40 = copy _2; - _39 = std::future::get_context::<'_, '_>(move _40) -> [return: bb49, unwind: bb43]; - } - @@ -480,7 +482,7 @@ - } - - bb57: { -- _47 = move _2; +- _47 = copy _2; - _46 = std::future::get_context::<'_, '_>(move _47) -> [return: bb56, unwind: bb43]; - } - diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncInt.MentionedItems.after.mir b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncInt.MentionedItems.after.mir index 533dc2634ff2a..d52457faecaf1 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncInt.MentionedItems.after.mir +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncInt.MentionedItems.after.mir @@ -29,6 +29,7 @@ yields () } bb1: { + _0 = const (); return; } @@ -86,7 +87,7 @@ yields () } bb13: { - _10 = move _2; + _10 = copy _2; _9 = std::future::get_context::<'_, '_>(move _10) -> [return: bb12, unwind: bb5]; } @@ -122,7 +123,7 @@ yields () } bb20: { - _17 = move _2; + _17 = copy _2; _16 = std::future::get_context::<'_, '_>(move _17) -> [return: bb19, unwind: bb5]; } diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncInt.StateTransform.diff b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncInt.StateTransform.diff index 015c275b529f5..ca19a52e41030 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncInt.StateTransform.diff +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncInt.StateTransform.diff @@ -54,6 +54,8 @@ } bb1: { +- _0 = const (); ++ _21 = const (); + _0 = Poll::<()>::Ready(move _21); + discriminant((*_23)) = 1; return; @@ -132,13 +134,13 @@ bb12: { - _6 = as Future>::poll(move _8, move _9) -> [return: bb11, unwind: bb5]; -+ _17 = move _24; ++ _17 = copy _24; + _16 = copy (_17.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb11; } bb13: { -- _10 = move _2; +- _10 = copy _2; - _9 = std::future::get_context::<'_, '_>(move _10) -> [return: bb12, unwind: bb5]; + _18 = &mut (((*_23) as variant#4).0: impl std::future::Future); + _15 = Pin::<&mut impl Future>::new_unchecked(move _18) -> [return: bb12, unwind: bb4]; @@ -189,7 +191,7 @@ } bb20: { -- _17 = move _2; +- _17 = copy _2; - _16 = std::future::get_context::<'_, '_>(move _17) -> [return: bb19, unwind: bb5]; - } - diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncReference_'__.MentionedItems.after.mir b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncReference_'__.MentionedItems.after.mir index fe1e2d66c2cbf..da437744e80cf 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncReference_'__.MentionedItems.after.mir +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncReference_'__.MentionedItems.after.mir @@ -29,6 +29,7 @@ yields () } bb1: { + _0 = const (); return; } @@ -86,7 +87,7 @@ yields () } bb13: { - _10 = move _2; + _10 = copy _2; _9 = std::future::get_context::<'_, '_>(move _10) -> [return: bb12, unwind: bb5]; } @@ -122,7 +123,7 @@ yields () } bb20: { - _17 = move _2; + _17 = copy _2; _16 = std::future::get_context::<'_, '_>(move _17) -> [return: bb19, unwind: bb5]; } diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncReference_'__.StateTransform.diff b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncReference_'__.StateTransform.diff index 93b76ad125bb3..8d5d5aad5f56e 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncReference_'__.StateTransform.diff +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncReference_'__.StateTransform.diff @@ -54,6 +54,8 @@ } bb1: { +- _0 = const (); ++ _21 = const (); + _0 = Poll::<()>::Ready(move _21); + discriminant((*_23)) = 1; return; @@ -132,13 +134,13 @@ bb12: { - _6 = as Future>::poll(move _8, move _9) -> [return: bb11, unwind: bb5]; -+ _17 = move _24; ++ _17 = copy _24; + _16 = copy (_17.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb11; } bb13: { -- _10 = move _2; +- _10 = copy _2; - _9 = std::future::get_context::<'_, '_>(move _10) -> [return: bb12, unwind: bb5]; + _18 = &mut (((*_23) as variant#4).0: impl std::future::Future); + _15 = Pin::<&mut impl Future>::new_unchecked(move _18) -> [return: bb12, unwind: bb4]; @@ -189,7 +191,7 @@ } bb20: { -- _17 = move _2; +- _17 = copy _2; - _16 = std::future::get_context::<'_, '_>(move _17) -> [return: bb19, unwind: bb5]; - } - diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncStruct.MentionedItems.after.mir b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncStruct.MentionedItems.after.mir index dcb038b587155..606a3735edf38 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncStruct.MentionedItems.after.mir +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncStruct.MentionedItems.after.mir @@ -83,6 +83,7 @@ yields () } bb1: { + _0 = const (); return; } @@ -143,7 +144,7 @@ yields () } bb14: { - _10 = move _2; + _10 = copy _2; _9 = std::future::get_context::<'_, '_>(move _10) -> [return: bb13, unwind: bb6]; } @@ -203,7 +204,7 @@ yields () } bb26: { - _20 = move _2; + _20 = copy _2; _19 = std::future::get_context::<'_, '_>(move _20) -> [return: bb25, unwind: bb19]; } @@ -268,7 +269,7 @@ yields () } bb39: { - _30 = move _2; + _30 = copy _2; _29 = std::future::get_context::<'_, '_>(move _30) -> [return: bb38, unwind: bb32]; } @@ -304,7 +305,7 @@ yields () } bb46: { - _37 = move _2; + _37 = copy _2; _36 = std::future::get_context::<'_, '_>(move _37) -> [return: bb45, unwind: bb32]; } @@ -369,7 +370,7 @@ yields () } bb59: { - _47 = move _2; + _47 = copy _2; _46 = std::future::get_context::<'_, '_>(move _47) -> [return: bb58, unwind: bb52]; } @@ -405,7 +406,7 @@ yields () } bb66: { - _54 = move _2; + _54 = copy _2; _53 = std::future::get_context::<'_, '_>(move _54) -> [return: bb65, unwind: bb52]; } @@ -470,7 +471,7 @@ yields () } bb79: { - _64 = move _2; + _64 = copy _2; _63 = std::future::get_context::<'_, '_>(move _64) -> [return: bb78, unwind: bb72]; } @@ -506,7 +507,7 @@ yields () } bb86: { - _71 = move _2; + _71 = copy _2; _70 = std::future::get_context::<'_, '_>(move _71) -> [return: bb85, unwind: bb72]; } diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncStruct.StateTransform.diff b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncStruct.StateTransform.diff index 339661ef2ce1d..32fa35bb5ca00 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncStruct.StateTransform.diff +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.AsyncStruct.StateTransform.diff @@ -124,6 +124,8 @@ } bb1: { +- _0 = const (); ++ _75 = const (); + _0 = Poll::<()>::Ready(move _75); + discriminant((*_77)) = 1; return; @@ -212,7 +214,7 @@ } bb14: { -- _10 = move _2; +- _10 = copy _2; - _9 = std::future::get_context::<'_, '_>(move _10) -> [return: bb13, unwind: bb6]; + assert(const false, "`async fn` resumed after async drop") -> [success: bb14, unwind: bb13]; } @@ -259,7 +261,7 @@ bb20: { - assert(const false, "`async fn` resumed after async drop") -> [success: bb20, unwind: bb19]; -+ _37 = move _83; ++ _37 = copy _83; + _36 = copy (_37.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb19; } @@ -303,7 +305,7 @@ } bb26: { -- _20 = move _2; +- _20 = copy _2; - _19 = std::future::get_context::<'_, '_>(move _20) -> [return: bb25, unwind: bb19]; + _83 = move _42; + StorageDead(_42); @@ -345,7 +347,7 @@ - StorageDead(_24); - goto -> bb2; + bb31: { -+ _54 = move _83; ++ _54 = copy _83; + _53 = copy (_54.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb30; } @@ -396,7 +398,7 @@ } bb38: { -- _30 = move _2; +- _30 = copy _2; - _29 = std::future::get_context::<'_, '_>(move _30) -> [return: bb37, unwind: bb31]; + _83 = move _66; + StorageDead(_66); @@ -431,7 +433,7 @@ bb42: { - StorageLive(_32); - _32 = yield(const ()) -> [resume: bb40, drop: bb41]; -+ _71 = move _83; ++ _71 = copy _83; + _70 = copy (_71.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb41; } @@ -450,7 +452,7 @@ } - bb45: { -- _37 = move _2; +- _37 = copy _2; - _36 = std::future::get_context::<'_, '_>(move _37) -> [return: bb44, unwind: bb31]; + bb45 (cleanup): { + discriminant((*_77)) = 2; @@ -542,7 +544,7 @@ - } - - bb57: { -- _47 = move _2; +- _47 = copy _2; - _46 = std::future::get_context::<'_, '_>(move _47) -> [return: bb56, unwind: bb50]; - } - @@ -578,7 +580,7 @@ - } - - bb64: { -- _54 = move _2; +- _54 = copy _2; - _53 = std::future::get_context::<'_, '_>(move _54) -> [return: bb63, unwind: bb50]; - } - @@ -640,7 +642,7 @@ - } - - bb76: { -- _64 = move _2; +- _64 = copy _2; - _63 = std::future::get_context::<'_, '_>(move _64) -> [return: bb75, unwind: bb69]; - } - @@ -676,7 +678,7 @@ - } - - bb83: { -- _71 = move _2; +- _71 = copy _2; - _70 = std::future::get_context::<'_, '_>(move _71) -> [return: bb82, unwind: bb69]; - } - diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.Int.MentionedItems.after.mir b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.Int.MentionedItems.after.mir index 7823a862bbded..13d5e10d2855a 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.Int.MentionedItems.after.mir +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.Int.MentionedItems.after.mir @@ -10,6 +10,7 @@ yields () } bb1: { + _0 = const (); return; } } diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.Int.StateTransform.diff b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.Int.StateTransform.diff index be51f11489222..0e2b6b46a91f7 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.Int.StateTransform.diff +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.Int.StateTransform.diff @@ -22,6 +22,7 @@ + let mut _7: std::ptr::NonNull>; bb0: { +- _0 = const (); + _7 = move _2 as std::ptr::NonNull> (Transmute); + _6 = std::future::ResumeTy(move _7); + _5 = copy (_1.0: &mut {async fn body of std::future::async_drop_in_place()}); @@ -39,6 +40,7 @@ + } + + bb3: { ++ _3 = const (); + _0 = Poll::<()>::Ready(move _3); + discriminant((*_5)) = 1; return; diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncInt.MentionedItems.after.mir b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncInt.MentionedItems.after.mir index b958b843c6df0..91efeec1a5977 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncInt.MentionedItems.after.mir +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncInt.MentionedItems.after.mir @@ -12,6 +12,7 @@ yields () } bb1: { + _0 = const (); return; } } diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncInt.StateTransform.diff b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncInt.StateTransform.diff index 2d127c495aba2..cc759752b575b 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncInt.StateTransform.diff +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncInt.StateTransform.diff @@ -33,6 +33,8 @@ } bb1: { +- _0 = const (); ++ _4 = const (); + _0 = Poll::<()>::Ready(move _4); + discriminant((*_6)) = 1; return; diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncThenAsync.MentionedItems.after.mir b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncThenAsync.MentionedItems.after.mir index ad37224d34f9b..52c663a4afd4e 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncThenAsync.MentionedItems.after.mir +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncThenAsync.MentionedItems.after.mir @@ -68,6 +68,7 @@ yields () } bb1: { + _0 = const (); return; } @@ -132,7 +133,7 @@ yields () } bb15: { - _10 = move _2; + _10 = copy _2; _9 = std::future::get_context::<'_, '_>(move _10) -> [return: bb14, unwind: bb7]; } @@ -196,7 +197,7 @@ yields () } bb28: { - _20 = move _2; + _20 = copy _2; _19 = std::future::get_context::<'_, '_>(move _20) -> [return: bb27, unwind: bb21]; } @@ -261,7 +262,7 @@ yields () } bb41: { - _30 = move _2; + _30 = copy _2; _29 = std::future::get_context::<'_, '_>(move _30) -> [return: bb40, unwind: bb34]; } @@ -297,7 +298,7 @@ yields () } bb48: { - _37 = move _2; + _37 = copy _2; _36 = std::future::get_context::<'_, '_>(move _37) -> [return: bb47, unwind: bb34]; } @@ -366,7 +367,7 @@ yields () } bb62: { - _47 = move _2; + _47 = copy _2; _46 = std::future::get_context::<'_, '_>(move _47) -> [return: bb61, unwind: bb55]; } @@ -402,7 +403,7 @@ yields () } bb69: { - _54 = move _2; + _54 = copy _2; _53 = std::future::get_context::<'_, '_>(move _54) -> [return: bb68, unwind: bb55]; } diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncThenAsync.StateTransform.diff b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncThenAsync.StateTransform.diff index 6a7ad149f1e82..62cdf1cefc2a3 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncThenAsync.StateTransform.diff +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.SyncThenAsync.StateTransform.diff @@ -106,6 +106,8 @@ } bb1: { +- _0 = const (); ++ _60 = const (); + _0 = Poll::<()>::Ready(move _60); + discriminant((*_62)) = 1; return; @@ -200,7 +202,7 @@ } bb15: { -- _10 = move _2; +- _10 = copy _2; - _9 = std::future::get_context::<'_, '_>(move _10) -> [return: bb14, unwind: bb7]; + StorageLive(_32); + _0 = Poll::<()>::Pending; @@ -225,7 +227,7 @@ bb18: { - _13 = &mut ((*_3).3: AsyncInt); - _12 = Pin::<&mut AsyncInt>::new_unchecked(move _13) -> [return: bb17, unwind: bb2]; -+ _37 = move _70; ++ _37 = copy _70; + _36 = copy (_37.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb17; } @@ -302,7 +304,7 @@ } bb28: { -- _30 = move _2; +- _30 = copy _2; - _29 = std::future::get_context::<'_, '_>(move _30) -> [return: bb27, unwind: bb21]; + _51 = discriminant(_50); + switchInt(move _51) -> [0: bb22, 1: bb27, otherwise: bb9]; @@ -318,7 +320,7 @@ - _2 = move _32; - StorageDead(_32); - goto -> bb36; -+ _54 = move _70; ++ _54 = copy _70; + _53 = copy (_54.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb29; } @@ -354,7 +356,7 @@ } bb35: { -- _37 = move _2; +- _37 = copy _2; - _36 = std::future::get_context::<'_, '_>(move _37) -> [return: bb34, unwind: bb21]; + StorageLive(_5); + _5 = move _70; @@ -438,7 +440,7 @@ - } - - bb48: { -- _47 = move _2; +- _47 = copy _2; - _46 = std::future::get_context::<'_, '_>(move _47) -> [return: bb47, unwind: bb41]; - } - @@ -474,7 +476,7 @@ - } - - bb55: { -- _54 = move _2; +- _54 = copy _2; - _53 = std::future::get_context::<'_, '_>(move _54) -> [return: bb54, unwind: bb41]; - } - diff --git a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.[AsyncInt;2].StateTransform.diff b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.[AsyncInt;2].StateTransform.diff index 7718554895afd..ad795995c4892 100644 --- a/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.[AsyncInt;2].StateTransform.diff +++ b/tests/mir-opt/coroutine/async_drop.core.future-async_drop-async_drop_in_place-{closure#0}.[AsyncInt;2].StateTransform.diff @@ -84,6 +84,8 @@ } bb1: { +- _0 = const (); ++ _41 = const (); + _0 = Poll::<()>::Ready(move _41); + discriminant((*_43)) = 1; return; @@ -195,7 +197,7 @@ } bb16: { -- _18 = move _2; +- _18 = copy _2; - _17 = std::future::get_context::<'_, '_>(move _18) -> [return: bb15, unwind: bb8]; + StorageLive(_32); + _0 = Poll::<()>::Pending; @@ -222,7 +224,7 @@ - _7 = Add(move _7, const 1_usize); - _40 = &mut (*_22); - _39 = Pin::<&mut AsyncInt>::new_unchecked(move _40) -> [return: bb39, unwind: bb4]; -+ _37 = move _46; ++ _37 = copy _46; + _36 = copy (_37.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb18; } @@ -298,7 +300,7 @@ - } - - bb30: { -- _30 = move _2; +- _30 = copy _2; - _29 = std::future::get_context::<'_, '_>(move _30) -> [return: bb29, unwind: bb23]; - } - @@ -334,7 +336,7 @@ - } - - bb37: { -- _37 = move _2; +- _37 = copy _2; - _36 = std::future::get_context::<'_, '_>(move _37) -> [return: bb36, unwind: bb23]; - } - diff --git a/tests/mir-opt/coroutine/async_drop.double-{closure#0}.ElaborateDrops.diff b/tests/mir-opt/coroutine/async_drop.double-{closure#0}.ElaborateDrops.diff index b11d65b4da8b8..a89b00e268fb7 100644 --- a/tests/mir-opt/coroutine/async_drop.double-{closure#0}.ElaborateDrops.diff +++ b/tests/mir-opt/coroutine/async_drop.double-{closure#0}.ElaborateDrops.diff @@ -189,7 +189,7 @@ + } + + bb25: { -+ _12 = move _2; ++ _12 = copy _2; + _11 = std::future::get_context::<'_, '_>(move _12) -> [return: bb24, unwind: bb17]; + } + @@ -225,7 +225,7 @@ + } + + bb32: { -+ _19 = move _2; ++ _19 = copy _2; + _18 = std::future::get_context::<'_, '_>(move _19) -> [return: bb31, unwind: bb17]; + } + @@ -290,7 +290,7 @@ + } + + bb45: { -+ _29 = move _2; ++ _29 = copy _2; + _28 = std::future::get_context::<'_, '_>(move _29) -> [return: bb44, unwind: bb38]; + } + @@ -326,7 +326,7 @@ + } + + bb52: { -+ _36 = move _2; ++ _36 = copy _2; + _35 = std::future::get_context::<'_, '_>(move _36) -> [return: bb51, unwind: bb38]; + } + diff --git a/tests/mir-opt/coroutine/async_drop.double-{closure#0}.StateTransform.diff b/tests/mir-opt/coroutine/async_drop.double-{closure#0}.StateTransform.diff index b2f525bb0289e..03b6bebff6933 100644 --- a/tests/mir-opt/coroutine/async_drop.double-{closure#0}.StateTransform.diff +++ b/tests/mir-opt/coroutine/async_drop.double-{closure#0}.StateTransform.diff @@ -229,7 +229,7 @@ - _2 = move _7; - StorageDead(_7); - goto -> bb24; -+ _19 = move _43; ++ _19 = copy _43; + _18 = copy (_19.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb17; } @@ -261,7 +261,7 @@ } - bb23: { -- _12 = move _2; +- _12 = copy _2; - _11 = std::future::get_context::<'_, '_>(move _12) -> [return: bb22, unwind: bb15]; + bb23 (cleanup): { + nop; @@ -315,9 +315,9 @@ } bb30: { -- _19 = move _2; +- _19 = copy _2; - _18 = std::future::get_context::<'_, '_>(move _19) -> [return: bb29, unwind: bb15]; -+ _36 = move _43; ++ _36 = copy _43; + _35 = copy (_36.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb29; } @@ -407,7 +407,7 @@ - } - - bb43: { -- _29 = move _2; +- _29 = copy _2; - _28 = std::future::get_context::<'_, '_>(move _29) -> [return: bb42, unwind: bb36]; - } - @@ -443,7 +443,7 @@ - } - - bb50: { -- _36 = move _2; +- _36 = copy _2; - _35 = std::future::get_context::<'_, '_>(move _36) -> [return: bb49, unwind: bb36]; - } - diff --git a/tests/mir-opt/coroutine/async_drop.double-{closure#0}.coroutine_drop_async.0.mir b/tests/mir-opt/coroutine/async_drop.double-{closure#0}.coroutine_drop_async.0.mir index 0239f6a98cb05..eefb66e259021 100644 --- a/tests/mir-opt/coroutine/async_drop.double-{closure#0}.coroutine_drop_async.0.mir +++ b/tests/mir-opt/coroutine/async_drop.double-{closure#0}.coroutine_drop_async.0.mir @@ -140,7 +140,7 @@ fn double::{closure#0}(_1: Pin<&mut {async fn body of double()}>, _2: &mut Conte } bb16: { - _12 = move _43; + _12 = copy _43; _11 = copy (_12.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); goto -> bb15; } @@ -190,7 +190,7 @@ fn double::{closure#0}(_1: Pin<&mut {async fn body of double()}>, _2: &mut Conte } bb25: { - _29 = move _43; + _29 = copy _43; _28 = copy (_29.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); goto -> bb24; } diff --git a/tests/mir-opt/coroutine/async_drop.elaborate_drops-{closure#0}.ElaborateDrops.diff b/tests/mir-opt/coroutine/async_drop.elaborate_drops-{closure#0}.ElaborateDrops.diff index 4e0cbbfc4357c..142b8f7918e97 100644 --- a/tests/mir-opt/coroutine/async_drop.elaborate_drops-{closure#0}.ElaborateDrops.diff +++ b/tests/mir-opt/coroutine/async_drop.elaborate_drops-{closure#0}.ElaborateDrops.diff @@ -791,7 +791,7 @@ + } + + bb93: { -+ _33 = move _2; ++ _33 = copy _2; + _32 = std::future::get_context::<'_, '_>(move _33) -> [return: bb92, unwind: bb85]; + } + @@ -827,7 +827,7 @@ + } + + bb100: { -+ _40 = move _2; ++ _40 = copy _2; + _39 = std::future::get_context::<'_, '_>(move _40) -> [return: bb99, unwind: bb85]; + } + @@ -892,7 +892,7 @@ + } + + bb113: { -+ _50 = move _2; ++ _50 = copy _2; + _49 = std::future::get_context::<'_, '_>(move _50) -> [return: bb112, unwind: bb106]; + } + @@ -928,7 +928,7 @@ + } + + bb120: { -+ _57 = move _2; ++ _57 = copy _2; + _56 = std::future::get_context::<'_, '_>(move _57) -> [return: bb119, unwind: bb106]; + } + @@ -993,7 +993,7 @@ + } + + bb133: { -+ _67 = move _2; ++ _67 = copy _2; + _66 = std::future::get_context::<'_, '_>(move _67) -> [return: bb132, unwind: bb126]; + } + @@ -1029,7 +1029,7 @@ + } + + bb140: { -+ _74 = move _2; ++ _74 = copy _2; + _73 = std::future::get_context::<'_, '_>(move _74) -> [return: bb139, unwind: bb126]; + } + @@ -1094,7 +1094,7 @@ + } + + bb153: { -+ _84 = move _2; ++ _84 = copy _2; + _83 = std::future::get_context::<'_, '_>(move _84) -> [return: bb152, unwind: bb146]; + } + @@ -1130,7 +1130,7 @@ + } + + bb160: { -+ _91 = move _2; ++ _91 = copy _2; + _90 = std::future::get_context::<'_, '_>(move _91) -> [return: bb159, unwind: bb146]; + } + @@ -1195,7 +1195,7 @@ + } + + bb173: { -+ _101 = move _2; ++ _101 = copy _2; + _100 = std::future::get_context::<'_, '_>(move _101) -> [return: bb172, unwind: bb166]; + } + @@ -1231,7 +1231,7 @@ + } + + bb180: { -+ _108 = move _2; ++ _108 = copy _2; + _107 = std::future::get_context::<'_, '_>(move _108) -> [return: bb179, unwind: bb166]; + } + @@ -1296,7 +1296,7 @@ + } + + bb193: { -+ _118 = move _2; ++ _118 = copy _2; + _117 = std::future::get_context::<'_, '_>(move _118) -> [return: bb192, unwind: bb186]; + } + @@ -1332,7 +1332,7 @@ + } + + bb200: { -+ _125 = move _2; ++ _125 = copy _2; + _124 = std::future::get_context::<'_, '_>(move _125) -> [return: bb199, unwind: bb186]; + } + @@ -1397,7 +1397,7 @@ + } + + bb213: { -+ _135 = move _2; ++ _135 = copy _2; + _134 = std::future::get_context::<'_, '_>(move _135) -> [return: bb212, unwind: bb206]; + } + @@ -1433,7 +1433,7 @@ + } + + bb220: { -+ _142 = move _2; ++ _142 = copy _2; + _141 = std::future::get_context::<'_, '_>(move _142) -> [return: bb219, unwind: bb206]; + } + @@ -1498,7 +1498,7 @@ + } + + bb233: { -+ _152 = move _2; ++ _152 = copy _2; + _151 = std::future::get_context::<'_, '_>(move _152) -> [return: bb232, unwind: bb226]; + } + @@ -1534,7 +1534,7 @@ + } + + bb240: { -+ _159 = move _2; ++ _159 = copy _2; + _158 = std::future::get_context::<'_, '_>(move _159) -> [return: bb239, unwind: bb226]; + } + @@ -1599,7 +1599,7 @@ + } + + bb253: { -+ _169 = move _2; ++ _169 = copy _2; + _168 = std::future::get_context::<'_, '_>(move _169) -> [return: bb252, unwind: bb246]; + } + @@ -1635,7 +1635,7 @@ + } + + bb260: { -+ _176 = move _2; ++ _176 = copy _2; + _175 = std::future::get_context::<'_, '_>(move _176) -> [return: bb259, unwind: bb246]; + } + diff --git a/tests/mir-opt/coroutine/async_drop.elaborate_drops-{closure#0}.StateTransform.diff b/tests/mir-opt/coroutine/async_drop.elaborate_drops-{closure#0}.StateTransform.diff index 08e9ff5c2796f..ea603b2ec325b 100644 --- a/tests/mir-opt/coroutine/async_drop.elaborate_drops-{closure#0}.StateTransform.diff +++ b/tests/mir-opt/coroutine/async_drop.elaborate_drops-{closure#0}.StateTransform.diff @@ -702,7 +702,7 @@ - StorageDead(_5); - drop(_4) -> [return: bb49, unwind terminate(cleanup)]; + bb48: { -+ _40 = move _183; ++ _40 = copy _183; + _39 = copy (_40.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb47; } @@ -796,7 +796,7 @@ bb60: { - _30 = discriminant(_29); - switchInt(move _30) -> [0: bb53, 1: bb58, otherwise: bb59]; -+ _57 = move _183; ++ _57 = copy _183; + _56 = copy (_57.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb59; } @@ -808,7 +808,7 @@ } bb62: { -- _33 = move _2; +- _33 = copy _2; - _32 = std::future::get_context::<'_, '_>(move _33) -> [return: bb61, unwind: bb54]; + nop; + (((*_182) as variant#6).10: impl std::future::Future) = async_drop_in_place::<{closure@$DIR/async_drop.rs:78:25: 78:27}>(copy (_59.0: &mut {closure@$DIR/async_drop.rs:78:25: 78:27})) -> [return: bb61, unwind: bb53]; @@ -860,7 +860,7 @@ } bb69: { -- _40 = move _2; +- _40 = copy _2; - _39 = std::future::get_context::<'_, '_>(move _40) -> [return: bb68, unwind: bb54]; + StorageLive(_69); + _0 = Poll::<()>::Pending; @@ -886,7 +886,7 @@ bb72: { - _43 = &mut _26; - _42 = Pin::<&mut {async closure@$DIR/async_drop.rs:86:27: 86:35}>::new_unchecked(move _43) -> [return: bb71, unwind: bb36]; -+ _74 = move _183; ++ _74 = copy _183; + _73 = copy (_74.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb71; } @@ -962,7 +962,7 @@ } bb82: { -- _50 = move _2; +- _50 = copy _2; - _49 = std::future::get_context::<'_, '_>(move _50) -> [return: bb81, unwind: bb75]; + _88 = discriminant(_87); + switchInt(move _88) -> [0: bb76, 1: bb81, otherwise: bb43]; @@ -978,7 +978,7 @@ - _2 = move _52; - StorageDead(_52); - goto -> bb90; -+ _91 = move _183; ++ _91 = copy _183; + _90 = copy (_91.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb83; } @@ -1012,7 +1012,7 @@ } - bb89: { -- _57 = move _2; +- _57 = copy _2; - _56 = std::future::get_context::<'_, '_>(move _57) -> [return: bb88, unwind: bb75]; + bb89 (cleanup): { + nop; @@ -1067,7 +1067,7 @@ bb96: { - assert(const false, "`async fn` resumed after async drop") -> [success: bb96, unwind: bb95]; -+ _108 = move _183; ++ _108 = copy _183; + _107 = copy (_108.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb95; } @@ -1110,7 +1110,7 @@ } bb102: { -- _67 = move _2; +- _67 = copy _2; - _66 = std::future::get_context::<'_, '_>(move _67) -> [return: bb101, unwind: bb95]; + assert(const false, "`async fn` resumed after async drop") -> [success: bb102, unwind: bb101]; } @@ -1158,13 +1158,13 @@ bb108: { - _70 = as Future>::poll(move _72, move _73) -> [return: bb107, unwind: bb95]; -+ _125 = move _183; ++ _125 = copy _183; + _124 = copy (_125.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb107; } bb109: { -- _74 = move _2; +- _74 = copy _2; - _73 = std::future::get_context::<'_, '_>(move _74) -> [return: bb108, unwind: bb95]; + _126 = &mut (((*_182) as variant#14).6: impl std::future::Future); + _123 = Pin::<&mut impl Future>::new_unchecked(move _126) -> [return: bb108, unwind: bb101]; @@ -1249,7 +1249,7 @@ bb120: { - _81 = discriminant(_80); - switchInt(move _81) -> [0: bb114, 1: bb119, otherwise: bb59]; -+ _142 = move _183; ++ _142 = copy _183; + _141 = copy (_142.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb119; } @@ -1261,7 +1261,7 @@ } bb122: { -- _84 = move _2; +- _84 = copy _2; - _83 = std::future::get_context::<'_, '_>(move _84) -> [return: bb121, unwind: bb115]; + nop; + (((*_182) as variant#16).5: impl std::future::Future) = async_drop_in_place::(copy (_144.0: &mut AsyncStruct)) -> [return: bb121, unwind: bb113]; @@ -1313,7 +1313,7 @@ } bb129: { -- _91 = move _2; +- _91 = copy _2; - _90 = std::future::get_context::<'_, '_>(move _91) -> [return: bb128, unwind: bb115]; + StorageLive(_154); + _0 = Poll::<()>::Pending; @@ -1338,7 +1338,7 @@ bb132: { - _94 = &mut _19; - _93 = Pin::<&mut AsyncInt>::new_unchecked(move _94) -> [return: bb131, unwind: bb41]; -+ _159 = move _183; ++ _159 = copy _183; + _158 = copy (_159.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb131; } @@ -1413,7 +1413,7 @@ } bb142: { -- _101 = move _2; +- _101 = copy _2; - _100 = std::future::get_context::<'_, '_>(move _101) -> [return: bb141, unwind: bb135]; + _173 = discriminant(_172); + switchInt(move _173) -> [0: bb136, 1: bb141, otherwise: bb43]; @@ -1429,7 +1429,7 @@ - _2 = move _103; - StorageDead(_103); - goto -> bb150; -+ _176 = move _183; ++ _176 = copy _183; + _175 = copy (_176.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb143; } @@ -1464,7 +1464,7 @@ } bb149: { -- _108 = move _2; +- _108 = copy _2; - _107 = std::future::get_context::<'_, '_>(move _108) -> [return: bb148, unwind: bb135]; + StorageLive(_17); + StorageLive(_23); @@ -1582,7 +1582,7 @@ } bb162: { -- _118 = move _2; +- _118 = copy _2; - _117 = std::future::get_context::<'_, '_>(move _118) -> [return: bb161, unwind: bb155]; + StorageLive(_137); + _137 = move _183; @@ -1635,7 +1635,7 @@ } bb169: { -- _125 = move _2; +- _125 = copy _2; - _124 = std::future::get_context::<'_, '_>(move _125) -> [return: bb168, unwind: bb155]; - } - @@ -1700,7 +1700,7 @@ - } - - bb182: { -- _135 = move _2; +- _135 = copy _2; - _134 = std::future::get_context::<'_, '_>(move _135) -> [return: bb181, unwind: bb175]; - } - @@ -1736,7 +1736,7 @@ - } - - bb189: { -- _142 = move _2; +- _142 = copy _2; - _141 = std::future::get_context::<'_, '_>(move _142) -> [return: bb188, unwind: bb175]; - } - @@ -1801,7 +1801,7 @@ - } - - bb202: { -- _152 = move _2; +- _152 = copy _2; - _151 = std::future::get_context::<'_, '_>(move _152) -> [return: bb201, unwind: bb195]; - } - @@ -1837,7 +1837,7 @@ - } - - bb209: { -- _159 = move _2; +- _159 = copy _2; - _158 = std::future::get_context::<'_, '_>(move _159) -> [return: bb208, unwind: bb195]; - } - @@ -1902,7 +1902,7 @@ - } - - bb222: { -- _169 = move _2; +- _169 = copy _2; - _168 = std::future::get_context::<'_, '_>(move _169) -> [return: bb221, unwind: bb215]; - } - @@ -1938,7 +1938,7 @@ - } - - bb229: { -- _176 = move _2; +- _176 = copy _2; - _175 = std::future::get_context::<'_, '_>(move _176) -> [return: bb228, unwind: bb215]; - } - diff --git a/tests/mir-opt/coroutine/async_drop.simple-{closure#0}.ElaborateDrops.diff b/tests/mir-opt/coroutine/async_drop.simple-{closure#0}.ElaborateDrops.diff index 0e18ddf5ad2dd..1652deae72e1f 100644 --- a/tests/mir-opt/coroutine/async_drop.simple-{closure#0}.ElaborateDrops.diff +++ b/tests/mir-opt/coroutine/async_drop.simple-{closure#0}.ElaborateDrops.diff @@ -143,7 +143,7 @@ + } + + bb21: { -+ _11 = move _2; ++ _11 = copy _2; + _10 = std::future::get_context::<'_, '_>(move _11) -> [return: bb20, unwind: bb13]; + } + @@ -179,7 +179,7 @@ + } + + bb28: { -+ _18 = move _2; ++ _18 = copy _2; + _17 = std::future::get_context::<'_, '_>(move _18) -> [return: bb27, unwind: bb13]; + } + diff --git a/tests/mir-opt/coroutine/async_drop.simple-{closure#0}.StateTransform.diff b/tests/mir-opt/coroutine/async_drop.simple-{closure#0}.StateTransform.diff index 59ab86c82a193..faec9d5b2d28e 100644 --- a/tests/mir-opt/coroutine/async_drop.simple-{closure#0}.StateTransform.diff +++ b/tests/mir-opt/coroutine/async_drop.simple-{closure#0}.StateTransform.diff @@ -184,7 +184,7 @@ bb16: { - StorageLive(_6); - _6 = yield(const ()) -> [resume: bb14, drop: bb15]; -+ _18 = move _25; ++ _18 = copy _25; + _17 = copy (_18.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + goto -> bb15; } @@ -209,7 +209,7 @@ } - bb20: { -- _11 = move _2; +- _11 = copy _2; - _10 = std::future::get_context::<'_, '_>(move _11) -> [return: bb19, unwind: bb12]; + bb20 (cleanup): { + discriminant((*_24)) = 2; @@ -256,7 +256,7 @@ - } - - bb27: { -- _18 = move _2; +- _18 = copy _2; - _17 = std::future::get_context::<'_, '_>(move _18) -> [return: bb26, unwind: bb12]; - } - diff --git a/tests/mir-opt/coroutine/async_drop.simple-{closure#0}.coroutine_drop_async.0.mir b/tests/mir-opt/coroutine/async_drop.simple-{closure#0}.coroutine_drop_async.0.mir index b883f518564d9..021c3c4e4234f 100644 --- a/tests/mir-opt/coroutine/async_drop.simple-{closure#0}.coroutine_drop_async.0.mir +++ b/tests/mir-opt/coroutine/async_drop.simple-{closure#0}.coroutine_drop_async.0.mir @@ -109,7 +109,7 @@ fn simple::{closure#0}(_1: Pin<&mut {async fn body of simple()}>, _2: &mut Conte } bb14: { - _11 = move _25; + _11 = copy _25; _10 = copy (_11.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); goto -> bb13; } diff --git a/tests/mir-opt/coroutine/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir b/tests/mir-opt/coroutine/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir index 83617c6cd7874..e92ee3b0ae82d 100644 --- a/tests/mir-opt/coroutine/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir +++ b/tests/mir-opt/coroutine/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir @@ -82,7 +82,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) } bb9: { - _10 = move _24; + _10 = copy _24; _9 = copy (_10.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); goto -> bb8; } diff --git a/tests/mir-opt/coroutine/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir b/tests/mir-opt/coroutine/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir index b6fb5c325d767..5fb2d8fa7d0d5 100644 --- a/tests/mir-opt/coroutine/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir +++ b/tests/mir-opt/coroutine/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir @@ -96,7 +96,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) } bb12: { - _10 = move _24; + _10 = copy _24; _9 = copy (_10.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); goto -> bb11; } diff --git a/tests/mir-opt/coroutine/async_drop_mir_pin.core.future-async_drop-async_drop_in_place-{closure#0}.[Foo;1].MentionedItems.after.mir b/tests/mir-opt/coroutine/async_drop_mir_pin.core.future-async_drop-async_drop_in_place-{closure#0}.[Foo;1].MentionedItems.after.mir index e06189e941211..18018efd928cc 100644 --- a/tests/mir-opt/coroutine/async_drop_mir_pin.core.future-async_drop-async_drop_in_place-{closure#0}.[Foo;1].MentionedItems.after.mir +++ b/tests/mir-opt/coroutine/async_drop_mir_pin.core.future-async_drop-async_drop_in_place-{closure#0}.[Foo;1].MentionedItems.after.mir @@ -49,6 +49,7 @@ yields () } bb1: { + _0 = const (); return; } @@ -123,7 +124,7 @@ yields () } bb16: { - _18 = move _2; + _18 = copy _2; _17 = std::future::get_context::<'_, '_>(move _18) -> [return: bb15, unwind: bb8]; } @@ -199,7 +200,7 @@ yields () } bb31: { - _30 = move _2; + _30 = copy _2; _29 = std::future::get_context::<'_, '_>(move _30) -> [return: bb30, unwind: bb24]; } @@ -235,7 +236,7 @@ yields () } bb38: { - _37 = move _2; + _37 = copy _2; _36 = std::future::get_context::<'_, '_>(move _37) -> [return: bb37, unwind: bb24]; } diff --git a/tests/mir-opt/coroutine/async_fn.add-{closure#0}-{closure#0}.coroutine_drop.0.mir b/tests/mir-opt/coroutine/async_fn.add-{closure#0}-{closure#0}.coroutine_drop.0.mir index 4dec40924719b..ff2b0a4812478 100644 --- a/tests/mir-opt/coroutine/async_fn.add-{closure#0}-{closure#0}.coroutine_drop.0.mir +++ b/tests/mir-opt/coroutine/async_fn.add-{closure#0}-{closure#0}.coroutine_drop.0.mir @@ -15,6 +15,7 @@ fn add::{closure#0}::{closure#0}(_1: &mut {async block@$DIR/async_fn.rs:34:13: 3 bb0: { _8 = discriminant((*_1)); + _0 = const (); switchInt(move _8) -> [0: bb2, otherwise: bb3]; } diff --git a/tests/mir-opt/coroutine/async_fn.add-{closure#0}.coroutine_drop.0.mir b/tests/mir-opt/coroutine/async_fn.add-{closure#0}.coroutine_drop.0.mir index cd2092e97177a..7b7a2469360d2 100644 --- a/tests/mir-opt/coroutine/async_fn.add-{closure#0}.coroutine_drop.0.mir +++ b/tests/mir-opt/coroutine/async_fn.add-{closure#0}.coroutine_drop.0.mir @@ -48,6 +48,7 @@ fn add::{closure#0}(_1: &mut {async fn body of add()}) -> () { bb0: { _26 = discriminant((*_1)); + _0 = const (); switchInt(move _26) -> [0: bb11, 3: bb12, otherwise: bb13]; } diff --git a/tests/mir-opt/coroutine/async_fn.build_aggregate-{closure#0}.coroutine_drop.0.mir b/tests/mir-opt/coroutine/async_fn.build_aggregate-{closure#0}.coroutine_drop.0.mir index d107b3a2dbbbd..5ca617278ce67 100644 --- a/tests/mir-opt/coroutine/async_fn.build_aggregate-{closure#0}.coroutine_drop.0.mir +++ b/tests/mir-opt/coroutine/async_fn.build_aggregate-{closure#0}.coroutine_drop.0.mir @@ -86,6 +86,7 @@ fn build_aggregate::{closure#0}(_1: &mut {async fn body of build_aggregate()}) - bb0: { _50 = discriminant((*_1)); + _0 = const (); switchInt(move _50) -> [0: bb16, 3: bb17, 4: bb18, otherwise: bb19]; } diff --git a/tests/mir-opt/coroutine/async_fn.foo-{closure#0}-{closure#0}.coroutine_drop.0.mir b/tests/mir-opt/coroutine/async_fn.foo-{closure#0}-{closure#0}.coroutine_drop.0.mir index a098d65cbc084..156649c264646 100644 --- a/tests/mir-opt/coroutine/async_fn.foo-{closure#0}-{closure#0}.coroutine_drop.0.mir +++ b/tests/mir-opt/coroutine/async_fn.foo-{closure#0}-{closure#0}.coroutine_drop.0.mir @@ -15,6 +15,7 @@ fn foo::{closure#0}::{closure#0}(_1: &mut {async block@$DIR/async_fn.rs:21:13: 2 bb0: { _8 = discriminant((*_1)); + _0 = const (); switchInt(move _8) -> [0: bb2, otherwise: bb3]; } diff --git a/tests/mir-opt/coroutine/async_fn.foo-{closure#0}.coroutine_drop.0.mir b/tests/mir-opt/coroutine/async_fn.foo-{closure#0}.coroutine_drop.0.mir index d5d9182fb2071..ae051b6b144a0 100644 --- a/tests/mir-opt/coroutine/async_fn.foo-{closure#0}.coroutine_drop.0.mir +++ b/tests/mir-opt/coroutine/async_fn.foo-{closure#0}.coroutine_drop.0.mir @@ -72,6 +72,7 @@ fn foo::{closure#0}(_1: &mut {async fn body of foo()}) -> () { bb0: { _35 = discriminant((*_1)); + _0 = const (); switchInt(move _35) -> [0: bb9, 3: bb10, otherwise: bb11]; } diff --git a/tests/mir-opt/coroutine/async_fn.hello_world-{closure#0}.coroutine_drop.0.mir b/tests/mir-opt/coroutine/async_fn.hello_world-{closure#0}.coroutine_drop.0.mir index b32943d88c10f..8cceffbf2f5bb 100644 --- a/tests/mir-opt/coroutine/async_fn.hello_world-{closure#0}.coroutine_drop.0.mir +++ b/tests/mir-opt/coroutine/async_fn.hello_world-{closure#0}.coroutine_drop.0.mir @@ -54,6 +54,7 @@ fn hello_world::{closure#0}(_1: &mut {async fn body of hello_world()}) -> () { bb0: { _34 = discriminant((*_1)); + _0 = const (); switchInt(move _34) -> [0: bb9, 3: bb10, otherwise: bb11]; } diff --git a/tests/mir-opt/coroutine/async_fn.includes_never-{closure#0}-{closure#0}.coroutine_drop.0.mir b/tests/mir-opt/coroutine/async_fn.includes_never-{closure#0}-{closure#0}.coroutine_drop.0.mir index 686b4323e0185..a3c8802306e2e 100644 --- a/tests/mir-opt/coroutine/async_fn.includes_never-{closure#0}-{closure#0}.coroutine_drop.0.mir +++ b/tests/mir-opt/coroutine/async_fn.includes_never-{closure#0}-{closure#0}.coroutine_drop.0.mir @@ -14,6 +14,7 @@ fn includes_never::{closure#0}::{closure#0}(_1: &mut {async block@$DIR/async_fn. bb0: { _8 = discriminant((*_1)); + _0 = const (); switchInt(move _8) -> [0: bb2, otherwise: bb3]; } diff --git a/tests/mir-opt/coroutine/async_fn.includes_never-{closure#0}-{closure#1}.coroutine_drop.0.mir b/tests/mir-opt/coroutine/async_fn.includes_never-{closure#0}-{closure#1}.coroutine_drop.0.mir index 3ae5c1dc89643..260d18f89feab 100644 --- a/tests/mir-opt/coroutine/async_fn.includes_never-{closure#0}-{closure#1}.coroutine_drop.0.mir +++ b/tests/mir-opt/coroutine/async_fn.includes_never-{closure#0}-{closure#1}.coroutine_drop.0.mir @@ -14,6 +14,7 @@ fn includes_never::{closure#0}::{closure#1}(_1: &mut {async block@$DIR/async_fn. bb0: { _8 = discriminant((*_1)); + _0 = const (); switchInt(move _8) -> [0: bb2, otherwise: bb3]; } diff --git a/tests/mir-opt/coroutine/async_fn.includes_never-{closure#0}.coroutine_drop.0.mir b/tests/mir-opt/coroutine/async_fn.includes_never-{closure#0}.coroutine_drop.0.mir index 97fdec425898f..de0a2475f0458 100644 --- a/tests/mir-opt/coroutine/async_fn.includes_never-{closure#0}.coroutine_drop.0.mir +++ b/tests/mir-opt/coroutine/async_fn.includes_never-{closure#0}.coroutine_drop.0.mir @@ -82,6 +82,7 @@ fn includes_never::{closure#0}(_1: &mut {async fn body of includes_never()}) -> bb0: { _49 = discriminant((*_1)); + _0 = const (); switchInt(move _49) -> [0: bb9, 3: bb10, otherwise: bb11]; } diff --git a/tests/mir-opt/coroutine/async_fn.partial_init-{closure#0}-{closure#0}.coroutine_drop.0.mir b/tests/mir-opt/coroutine/async_fn.partial_init-{closure#0}-{closure#0}.coroutine_drop.0.mir index 7a21ec09f8a10..ae14335b22348 100644 --- a/tests/mir-opt/coroutine/async_fn.partial_init-{closure#0}-{closure#0}.coroutine_drop.0.mir +++ b/tests/mir-opt/coroutine/async_fn.partial_init-{closure#0}-{closure#0}.coroutine_drop.0.mir @@ -14,6 +14,7 @@ fn partial_init::{closure#0}::{closure#0}(_1: &mut {async block@$DIR/async_fn.rs bb0: { _8 = discriminant((*_1)); + _0 = const (); switchInt(move _8) -> [0: bb2, otherwise: bb3]; } diff --git a/tests/mir-opt/coroutine/async_fn.partial_init-{closure#0}.coroutine_drop.0.mir b/tests/mir-opt/coroutine/async_fn.partial_init-{closure#0}.coroutine_drop.0.mir index c76b3ca617f76..b808b5f37e6f1 100644 --- a/tests/mir-opt/coroutine/async_fn.partial_init-{closure#0}.coroutine_drop.0.mir +++ b/tests/mir-opt/coroutine/async_fn.partial_init-{closure#0}.coroutine_drop.0.mir @@ -47,6 +47,7 @@ fn partial_init::{closure#0}(_1: &mut {async fn body of partial_init()}) -> () { bb0: { _27 = discriminant((*_1)); + _0 = const (); switchInt(move _27) -> [0: bb11, 3: bb12, otherwise: bb13]; } diff --git a/tests/mir-opt/coroutine/async_fn.uninhabited_variant-{closure#0}.coroutine_drop.0.mir b/tests/mir-opt/coroutine/async_fn.uninhabited_variant-{closure#0}.coroutine_drop.0.mir index 19cde28ce35fb..e3a25192eb17f 100644 --- a/tests/mir-opt/coroutine/async_fn.uninhabited_variant-{closure#0}.coroutine_drop.0.mir +++ b/tests/mir-opt/coroutine/async_fn.uninhabited_variant-{closure#0}.coroutine_drop.0.mir @@ -68,6 +68,7 @@ fn uninhabited_variant::{closure#0}(_1: &mut {async fn body of uninhabited_varia bb0: { _45 = discriminant((*_1)); + _0 = const (); switchInt(move _45) -> [0: bb22, 3: bb23, 4: bb24, otherwise: bb25]; } diff --git a/tests/mir-opt/coroutine/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-abort.mir b/tests/mir-opt/coroutine/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-abort.mir index e8873e341cc77..ec34b8750cc0e 100644 --- a/tests/mir-opt/coroutine/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-abort.mir +++ b/tests/mir-opt/coroutine/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-abort.mir @@ -14,6 +14,7 @@ fn main::{closure#0}(_1: &mut {coroutine@$DIR/coroutine_drop_cleanup.rs:12:5: 12 bb0: { _7 = discriminant((*_1)); + _0 = const (); switchInt(move _7) -> [0: bb5, 3: bb6, otherwise: bb7]; } diff --git a/tests/mir-opt/coroutine/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-unwind.mir b/tests/mir-opt/coroutine/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-unwind.mir index 9e450d702cc38..7d2f1c309b416 100644 --- a/tests/mir-opt/coroutine/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-unwind.mir +++ b/tests/mir-opt/coroutine/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-unwind.mir @@ -14,6 +14,7 @@ fn main::{closure#0}(_1: &mut {coroutine@$DIR/coroutine_drop_cleanup.rs:12:5: 12 bb0: { _7 = discriminant((*_1)); + _0 = const (); switchInt(move _7) -> [0: bb7, 3: bb8, otherwise: bb9]; } diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-abort.diff index 436f0350700e3..1bd6e72d8300d 100644 --- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-abort.diff +++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-abort.diff @@ -31,6 +31,7 @@ bb0: { StorageLive(_1); StorageLive(_2); + StorageAlloc(_2); StorageLive(_3); StorageLive(_4); StorageLive(_5); diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-unwind.diff index 616f92cb81198..a16300138798d 100644 --- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-unwind.diff +++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.32bit.panic-unwind.diff @@ -31,6 +31,7 @@ bb0: { StorageLive(_1); StorageLive(_2); + StorageAlloc(_2); StorageLive(_3); StorageLive(_4); StorageLive(_5); diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-abort.diff index 70153ea3cb940..b0d9884c52bc8 100644 --- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-abort.diff +++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-abort.diff @@ -31,6 +31,7 @@ bb0: { StorageLive(_1); StorageLive(_2); + StorageAlloc(_2); StorageLive(_3); StorageLive(_4); StorageLive(_5); diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-unwind.diff index 158861f0d07ef..98165f07e1185 100644 --- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-unwind.diff +++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.DataflowConstProp.64bit.panic-unwind.diff @@ -31,6 +31,7 @@ bb0: { StorageLive(_1); StorageLive(_2); + StorageAlloc(_2); StorageLive(_3); StorageLive(_4); StorageLive(_5); diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-abort.diff index 042af7d87f1f7..4c553b560e091 100644 --- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-abort.diff +++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-abort.diff @@ -31,6 +31,7 @@ bb0: { StorageLive(_1); StorageLive(_2); + StorageAlloc(_2); StorageLive(_3); StorageLive(_4); StorageLive(_5); diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-unwind.diff index 73025eb4ce5c4..0ca345c38406c 100644 --- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-unwind.diff +++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.32bit.panic-unwind.diff @@ -31,6 +31,7 @@ bb0: { StorageLive(_1); StorageLive(_2); + StorageAlloc(_2); StorageLive(_3); StorageLive(_4); StorageLive(_5); diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-abort.diff index e2bc701f3dc9f..f0c04f7af3905 100644 --- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-abort.diff +++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-abort.diff @@ -31,6 +31,7 @@ bb0: { StorageLive(_1); StorageLive(_2); + StorageAlloc(_2); StorageLive(_3); StorageLive(_4); StorageLive(_5); diff --git a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-unwind.diff index 35d824ee17fc2..8868bb0a2cde1 100644 --- a/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-unwind.diff +++ b/tests/mir-opt/dataflow-const-prop/default_boxed_slice.main.GVN.64bit.panic-unwind.diff @@ -31,6 +31,7 @@ bb0: { StorageLive(_1); StorageLive(_2); + StorageAlloc(_2); StorageLive(_3); StorageLive(_4); StorageLive(_5); diff --git a/tests/mir-opt/dataflow-const-prop/inherit_overflow.main.DataflowConstProp.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/inherit_overflow.main.DataflowConstProp.panic-abort.diff index 1b695e4b53c3e..03d00b3b0852b 100644 --- a/tests/mir-opt/dataflow-const-prop/inherit_overflow.main.DataflowConstProp.panic-abort.diff +++ b/tests/mir-opt/dataflow-const-prop/inherit_overflow.main.DataflowConstProp.panic-abort.diff @@ -14,6 +14,7 @@ bb0: { StorageLive(_1); + StorageAlloc(_1); StorageLive(_2); _2 = const u8::MAX; StorageLive(_3); diff --git a/tests/mir-opt/dataflow-const-prop/inherit_overflow.main.DataflowConstProp.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/inherit_overflow.main.DataflowConstProp.panic-unwind.diff index a0bd8e17d8c4d..0f4c384ca2b7b 100644 --- a/tests/mir-opt/dataflow-const-prop/inherit_overflow.main.DataflowConstProp.panic-unwind.diff +++ b/tests/mir-opt/dataflow-const-prop/inherit_overflow.main.DataflowConstProp.panic-unwind.diff @@ -14,6 +14,7 @@ bb0: { StorageLive(_1); + StorageAlloc(_1); StorageLive(_2); _2 = const u8::MAX; StorageLive(_3); diff --git a/tests/mir-opt/dead-store-elimination/storage_alloc.dead.DeadStoreElimination-initial.diff b/tests/mir-opt/dead-store-elimination/storage_alloc.dead.DeadStoreElimination-initial.diff new file mode 100644 index 0000000000000..ef395cd6cc07a --- /dev/null +++ b/tests/mir-opt/dead-store-elimination/storage_alloc.dead.DeadStoreElimination-initial.diff @@ -0,0 +1,15 @@ +- // MIR for `dead` before DeadStoreElimination-initial ++ // MIR for `dead` after DeadStoreElimination-initial + + fn dead(_1: MaybeUninit) -> MaybeUninit { + let mut _0: std::mem::MaybeUninit; + let mut _2: std::mem::MaybeUninit; + + bb0: { +- StorageAlloc(_2); + _2 = copy _1; + _0 = copy _2; + return; + } + } + diff --git a/tests/mir-opt/dead-store-elimination/storage_alloc.live.DeadStoreElimination-initial.diff b/tests/mir-opt/dead-store-elimination/storage_alloc.live.DeadStoreElimination-initial.diff new file mode 100644 index 0000000000000..faf130c5012d5 --- /dev/null +++ b/tests/mir-opt/dead-store-elimination/storage_alloc.live.DeadStoreElimination-initial.diff @@ -0,0 +1,14 @@ +- // MIR for `live` before DeadStoreElimination-initial ++ // MIR for `live` after DeadStoreElimination-initial + + fn live() -> MaybeUninit { + let mut _0: std::mem::MaybeUninit; + let mut _1: std::mem::MaybeUninit; + + bb0: { + StorageAlloc(_1); + _0 = copy _1; + return; + } + } + diff --git a/tests/mir-opt/dead-store-elimination/storage_alloc.rs b/tests/mir-opt/dead-store-elimination/storage_alloc.rs new file mode 100644 index 0000000000000..13605d84a3605 --- /dev/null +++ b/tests/mir-opt/dead-store-elimination/storage_alloc.rs @@ -0,0 +1,64 @@ +//@ test-mir-pass: DeadStoreElimination-initial + +#![crate_type = "lib"] +#![feature(custom_mir, core_intrinsics)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; +use std::mem::MaybeUninit; + +// The assignment provides the allocation needed by the later read. +// EMIT_MIR storage_alloc.dead.DeadStoreElimination-initial.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn dead(value: MaybeUninit) -> MaybeUninit { + // CHECK-LABEL: fn dead( + // CHECK-NOT: StorageAlloc + // CHECK: return; + mir! { + let x: MaybeUninit; + { + StorageAlloc(x); + x = value; + RET = x; + Return() + } + } +} + +// Uninitialized bytes may be read as MaybeUninit, but the allocation must exist. +// EMIT_MIR storage_alloc.live.DeadStoreElimination-initial.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn live() -> MaybeUninit { + // CHECK-LABEL: fn live( + // CHECK: StorageAlloc([[LOCAL:_[0-9]+]]); + // CHECK-NEXT: _0 = copy [[LOCAL]]; + mir! { + let x: MaybeUninit; + { + StorageAlloc(x); + RET = x; + Return() + } + } +} + +// Remove all storage statements along with the dead store. +// EMIT_MIR storage_alloc.unused_storage.DeadStoreElimination-initial.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn unused_storage(value: u8) { + // CHECK-LABEL: fn unused_storage( + // CHECK-NOT: Storage + // CHECK: return; + mir! { + let x: &u8; + debug x => x; + { + StorageLive(x); + StorageAlloc(x); + x = &value; + StorageDead(x); + RET = (); + Return() + } + } +} diff --git a/tests/mir-opt/dead-store-elimination/storage_alloc.unused_storage.DeadStoreElimination-initial.diff b/tests/mir-opt/dead-store-elimination/storage_alloc.unused_storage.DeadStoreElimination-initial.diff new file mode 100644 index 0000000000000..a8e63fbd55323 --- /dev/null +++ b/tests/mir-opt/dead-store-elimination/storage_alloc.unused_storage.DeadStoreElimination-initial.diff @@ -0,0 +1,19 @@ +- // MIR for `unused_storage` before DeadStoreElimination-initial ++ // MIR for `unused_storage` after DeadStoreElimination-initial + + fn unused_storage(_1: u8) -> () { + debug x => _2; + let mut _0: (); + let mut _2: &u8; + + bb0: { +- StorageLive(_2); +- StorageAlloc(_2); +- _2 = &_1; +- StorageDead(_2); ++ // DBG: _2 = &_1; + _0 = (); + return; + } + } + diff --git a/tests/mir-opt/debuginfo/ref_prop.remap_debuginfo_locals.ReferencePropagation.diff b/tests/mir-opt/debuginfo/ref_prop.remap_debuginfo_locals.ReferencePropagation.diff index 2d93b1d842fa1..caa970d65378e 100644 --- a/tests/mir-opt/debuginfo/ref_prop.remap_debuginfo_locals.ReferencePropagation.diff +++ b/tests/mir-opt/debuginfo/ref_prop.remap_debuginfo_locals.ReferencePropagation.diff @@ -20,6 +20,7 @@ _5 = const remap_debuginfo_locals::promoted[0]; - _3 = &(*_5); - _2 = &raw const (*_3); +- StorageAlloc(_1); - // DBG: _1 = &(*_2); - _1 = &(*_2); - StorageDead(_2); diff --git a/tests/mir-opt/debuginfo/single_use_consts.invalid_debuginfo.SingleUseConsts.diff b/tests/mir-opt/debuginfo/single_use_consts.invalid_debuginfo.SingleUseConsts.diff index 09e9f26546a85..6ec54d0b434bc 100644 --- a/tests/mir-opt/debuginfo/single_use_consts.invalid_debuginfo.SingleUseConsts.diff +++ b/tests/mir-opt/debuginfo/single_use_consts.invalid_debuginfo.SingleUseConsts.diff @@ -30,6 +30,7 @@ bb0: { StorageLive(_1); + StorageAlloc(_1); StorageLive(_4); - _4 = const foo::promoted[0]; + nop; diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff index 451d639ca2aa4..bcd161c7e284c 100644 --- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff +++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff @@ -48,6 +48,7 @@ StorageLive(_4); - _4 = (); + _4 = const (); + StorageAlloc(_3); StorageLive(_12); StorageLive(_13); - _13 = boxed::box_new_uninit(const <() as std::mem::SizedTypeProperties>::LAYOUT) -> [return: bb2, unwind unreachable]; @@ -78,6 +79,7 @@ StorageLive(_6); StorageLive(_7); _7 = copy _5; + StorageAlloc(_6); StorageLive(_14); _14 = const 1_usize; - _6 = *const [()] from (copy _7, copy _14); @@ -87,6 +89,7 @@ StorageLive(_8); StorageLive(_9); _9 = copy _6; + StorageAlloc(_8); StorageLive(_15); - _15 = copy _9; - _8 = copy _9 as *mut () (PtrToPtr); diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff index 75b00c8885cd0..71898b5d13889 100644 --- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff +++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff @@ -58,6 +58,7 @@ StorageLive(_6); StorageLive(_7); _7 = copy _5; + StorageAlloc(_6); StorageLive(_12); _12 = const 1_usize; - _6 = *const [()] from (copy _7, copy _12); @@ -67,6 +68,7 @@ StorageLive(_8); StorageLive(_9); _9 = copy _6; + StorageAlloc(_8); StorageLive(_13); - _13 = copy _9; - _8 = copy _9 as *mut () (PtrToPtr); diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff index 3473ceb21a0ab..cf449c05f645b 100644 --- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff +++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff @@ -48,6 +48,7 @@ StorageLive(_4); - _4 = (); + _4 = const (); + StorageAlloc(_3); StorageLive(_12); StorageLive(_13); - _13 = boxed::box_new_uninit(const <() as std::mem::SizedTypeProperties>::LAYOUT) -> [return: bb2, unwind unreachable]; @@ -78,6 +79,7 @@ StorageLive(_6); StorageLive(_7); _7 = copy _5; + StorageAlloc(_6); StorageLive(_14); _14 = const 1_usize; - _6 = *const [()] from (copy _7, copy _14); @@ -87,6 +89,7 @@ StorageLive(_8); StorageLive(_9); _9 = copy _6; + StorageAlloc(_8); StorageLive(_15); - _15 = copy _9; - _8 = copy _9 as *mut () (PtrToPtr); diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff index 75b00c8885cd0..71898b5d13889 100644 --- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff +++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff @@ -58,6 +58,7 @@ StorageLive(_6); StorageLive(_7); _7 = copy _5; + StorageAlloc(_6); StorageLive(_12); _12 = const 1_usize; - _6 = *const [()] from (copy _7, copy _12); @@ -67,6 +68,7 @@ StorageLive(_8); StorageLive(_9); _9 = copy _6; + StorageAlloc(_8); StorageLive(_13); - _13 = copy _9; - _8 = copy _9 as *mut () (PtrToPtr); diff --git a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-abort.diff b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-abort.diff index 703811633f853..97f3da3268353 100644 --- a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-abort.diff +++ b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-abort.diff @@ -44,6 +44,7 @@ bb0: { StorageLive(_4); + StorageAlloc(_4); StorageLive(_20); StorageLive(_21); _21 = copy (((*_1).0: std::fmt::FormattingOptions).0: u32); diff --git a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-unwind.diff b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-unwind.diff index a7187876e663e..c1be68d34a8fc 100644 --- a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-unwind.diff +++ b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.32bit.panic-unwind.diff @@ -44,6 +44,7 @@ bb0: { StorageLive(_4); + StorageAlloc(_4); StorageLive(_20); StorageLive(_21); _21 = copy (((*_1).0: std::fmt::FormattingOptions).0: u32); diff --git a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-abort.diff b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-abort.diff index 1e2ee782b444a..bb61e04978496 100644 --- a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-abort.diff +++ b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-abort.diff @@ -44,6 +44,7 @@ bb0: { StorageLive(_4); + StorageAlloc(_4); StorageLive(_20); StorageLive(_21); _21 = copy (((*_1).0: std::fmt::FormattingOptions).0: u32); diff --git a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-unwind.diff b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-unwind.diff index 0640cb2d2b8dd..aa1926417c011 100644 --- a/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-unwind.diff +++ b/tests/mir-opt/funky_arms.float_to_exponential_common.GVN.64bit.panic-unwind.diff @@ -44,6 +44,7 @@ bb0: { StorageLive(_4); + StorageAlloc(_4); StorageLive(_20); StorageLive(_21); _21 = copy (((*_1).0: std::fmt::FormattingOptions).0: u32); diff --git a/tests/mir-opt/gvn_ptr_eq_with_constant.main.GVN.diff b/tests/mir-opt/gvn_ptr_eq_with_constant.main.GVN.diff index d72f62fc69afc..75c52213240ff 100644 --- a/tests/mir-opt/gvn_ptr_eq_with_constant.main.GVN.diff +++ b/tests/mir-opt/gvn_ptr_eq_with_constant.main.GVN.diff @@ -31,12 +31,14 @@ bb0: { StorageLive(_1); StorageLive(_2); + StorageAlloc(_2); StorageLive(_3); - _3 = const ::ALIGNMENT; - _2 = copy _3 as *mut u8 (Transmute); + _3 = const std::mem::Alignment {{ _inner_repr_trick: mem::alignment::AlignmentEnum::_Align1Shl0 }}; + _2 = const {0x1 as *mut u8}; StorageDead(_3); + StorageAlloc(_1); StorageLive(_4); StorageLive(_5); - _5 = copy _2; @@ -46,8 +48,9 @@ StorageDead(_5); StorageLive(_6); - _6 = const Foo::::SENTINEL as *const u8 (PtrToPtr); -- _1 = Eq(copy _4, copy _6); + _6 = const {0x1 as *const u8}; + StorageAlloc(_1); +- _1 = Eq(copy _4, copy _6); + _1 = const true; StorageDead(_6); StorageDead(_4); diff --git a/tests/mir-opt/inline/asm_unwind.main.Inline.panic-unwind.diff b/tests/mir-opt/inline/asm_unwind.main.Inline.panic-unwind.diff index dc0004105a7b7..fdcf8e419b32f 100644 --- a/tests/mir-opt/inline/asm_unwind.main.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/asm_unwind.main.Inline.panic-unwind.diff @@ -14,7 +14,9 @@ bb0: { StorageLive(_1); - _1 = foo() -> [return: bb1, unwind continue]; ++ StorageAlloc(_1); + StorageLive(_2); ++ _2 = const D; + asm!("", options(MAY_UNWIND)) -> [return: bb2, unwind: bb3]; } @@ -26,6 +28,7 @@ + } + + bb2: { ++ _1 = const (); + drop(_2) -> [return: bb1, unwind continue]; + } + diff --git a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff index 92060c211330e..9122b570f0dea 100644 --- a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff @@ -19,6 +19,7 @@ StorageLive(_4); _4 = move _2; - _0 = > as FnMut>::call_mut(move _3, move _4) -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_0); + StorageLive(_6); + StorageLive(_7); + StorageLive(_5); diff --git a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff index 085fa453ade13..c66ea93cff259 100644 --- a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff @@ -19,6 +19,7 @@ StorageLive(_4); _4 = move _2; - _0 = > as FnMut>::call_mut(move _3, move _4) -> [return: bb1, unwind: bb3]; ++ StorageAlloc(_0); + StorageLive(_6); + StorageLive(_7); + StorageLive(_5); diff --git a/tests/mir-opt/inline/dyn_trait.get_query.Inline.panic-abort.diff b/tests/mir-opt/inline/dyn_trait.get_query.Inline.panic-abort.diff index df79001ce75b6..4d3341818aa28 100644 --- a/tests/mir-opt/inline/dyn_trait.get_query.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/dyn_trait.get_query.Inline.panic-abort.diff @@ -30,6 +30,7 @@ StorageLive(_4); _4 = copy _2; - _0 = try_execute_query::<::C>(move _4) -> [return: bb2, unwind unreachable]; ++ StorageAlloc(_0); + StorageLive(_5); + _5 = copy _4 as &dyn Cache::V> (PointerCoercion(Unsize, Implicit)); + _0 = ::V> as Cache>::store_nocache(move _5) -> [return: bb2, unwind unreachable]; diff --git a/tests/mir-opt/inline/dyn_trait.get_query.Inline.panic-unwind.diff b/tests/mir-opt/inline/dyn_trait.get_query.Inline.panic-unwind.diff index 014f950940c5f..189f2fc9a822e 100644 --- a/tests/mir-opt/inline/dyn_trait.get_query.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/dyn_trait.get_query.Inline.panic-unwind.diff @@ -30,6 +30,7 @@ StorageLive(_4); _4 = copy _2; - _0 = try_execute_query::<::C>(move _4) -> [return: bb2, unwind continue]; ++ StorageAlloc(_0); + StorageLive(_5); + _5 = copy _4 as &dyn Cache::V> (PointerCoercion(Unsize, Implicit)); + _0 = ::V> as Cache>::store_nocache(move _5) -> [return: bb2, unwind continue]; diff --git a/tests/mir-opt/inline/dyn_trait.try_execute_query.Inline.panic-abort.diff b/tests/mir-opt/inline/dyn_trait.try_execute_query.Inline.panic-abort.diff index 64d12461e8de1..3454052e3cc11 100644 --- a/tests/mir-opt/inline/dyn_trait.try_execute_query.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/dyn_trait.try_execute_query.Inline.panic-abort.diff @@ -17,6 +17,7 @@ _2 = move _3 as &dyn Cache::V> (PointerCoercion(Unsize, Implicit)); StorageDead(_3); - _0 = mk_cycle::<::V>(move _2) -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_0); + _0 = ::V> as Cache>::store_nocache(move _2) -> [return: bb1, unwind unreachable]; } diff --git a/tests/mir-opt/inline/dyn_trait.try_execute_query.Inline.panic-unwind.diff b/tests/mir-opt/inline/dyn_trait.try_execute_query.Inline.panic-unwind.diff index 21791cb0d2a59..d3966fe007cd6 100644 --- a/tests/mir-opt/inline/dyn_trait.try_execute_query.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/dyn_trait.try_execute_query.Inline.panic-unwind.diff @@ -17,6 +17,7 @@ _2 = move _3 as &dyn Cache::V> (PointerCoercion(Unsize, Implicit)); StorageDead(_3); - _0 = mk_cycle::<::V>(move _2) -> [return: bb1, unwind continue]; ++ StorageAlloc(_0); + _0 = ::V> as Cache>::store_nocache(move _2) -> [return: bb1, unwind continue]; } diff --git a/tests/mir-opt/inline/exponential_runtime.main.Inline.panic-abort.diff b/tests/mir-opt/inline/exponential_runtime.main.Inline.panic-abort.diff index 3ffe2ee0c0b94..389618c4cbdba 100644 --- a/tests/mir-opt/inline/exponential_runtime.main.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/exponential_runtime.main.Inline.panic-abort.diff @@ -38,28 +38,39 @@ bb0: { StorageLive(_1); - _1 = <() as G>::call() -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_1); + StorageLive(_2); + StorageLive(_3); + StorageLive(_4); ++ StorageAlloc(_2); + StorageLive(_5); + StorageLive(_6); + StorageLive(_7); ++ StorageAlloc(_5); + StorageLive(_8); + StorageLive(_9); + StorageLive(_10); ++ StorageAlloc(_8); + StorageLive(_11); + StorageLive(_12); + StorageLive(_13); ++ StorageAlloc(_11); + StorageLive(_14); + StorageLive(_15); + StorageLive(_16); ++ StorageAlloc(_14); + StorageLive(_17); + StorageLive(_18); + StorageLive(_19); -+ _17 = <() as A>::call() -> [return: bb12, unwind unreachable]; ++ _17 = <() as A>::call() -> [return: bb11, unwind unreachable]; } bb1: { ++ _4 = <() as F>::call() -> [return: bb2, unwind unreachable]; ++ } ++ ++ bb2: { ++ _1 = const (); + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); @@ -68,67 +79,68 @@ return; + } + -+ bb2: { -+ _4 = <() as F>::call() -> [return: bb1, unwind unreachable]; ++ bb3: { ++ _7 = <() as E>::call() -> [return: bb4, unwind unreachable]; + } + -+ bb3: { ++ bb4: { ++ _2 = const (); + StorageDead(_7); + StorageDead(_6); + StorageDead(_5); -+ _3 = <() as F>::call() -> [return: bb2, unwind unreachable]; ++ _3 = <() as F>::call() -> [return: bb1, unwind unreachable]; + } + -+ bb4: { -+ _7 = <() as E>::call() -> [return: bb3, unwind unreachable]; ++ bb5: { ++ _10 = <() as D>::call() -> [return: bb6, unwind unreachable]; + } + -+ bb5: { ++ bb6: { ++ _5 = const (); + StorageDead(_10); + StorageDead(_9); + StorageDead(_8); -+ _6 = <() as E>::call() -> [return: bb4, unwind unreachable]; ++ _6 = <() as E>::call() -> [return: bb3, unwind unreachable]; + } + -+ bb6: { -+ _10 = <() as D>::call() -> [return: bb5, unwind unreachable]; ++ bb7: { ++ _13 = <() as C>::call() -> [return: bb8, unwind unreachable]; + } + -+ bb7: { ++ bb8: { ++ _8 = const (); + StorageDead(_13); + StorageDead(_12); + StorageDead(_11); -+ _9 = <() as D>::call() -> [return: bb6, unwind unreachable]; ++ _9 = <() as D>::call() -> [return: bb5, unwind unreachable]; + } + -+ bb8: { -+ _13 = <() as C>::call() -> [return: bb7, unwind unreachable]; ++ bb9: { ++ _16 = <() as B>::call() -> [return: bb10, unwind unreachable]; + } + -+ bb9: { ++ bb10: { ++ _11 = const (); + StorageDead(_16); + StorageDead(_15); + StorageDead(_14); -+ _12 = <() as C>::call() -> [return: bb8, unwind unreachable]; -+ } -+ -+ bb10: { -+ _16 = <() as B>::call() -> [return: bb9, unwind unreachable]; ++ _12 = <() as C>::call() -> [return: bb7, unwind unreachable]; + } + + bb11: { -+ StorageDead(_19); -+ StorageDead(_18); -+ StorageDead(_17); -+ _15 = <() as B>::call() -> [return: bb10, unwind unreachable]; ++ _18 = <() as A>::call() -> [return: bb12, unwind unreachable]; + } + + bb12: { -+ _18 = <() as A>::call() -> [return: bb13, unwind unreachable]; ++ _19 = <() as A>::call() -> [return: bb13, unwind unreachable]; + } + + bb13: { -+ _19 = <() as A>::call() -> [return: bb11, unwind unreachable]; ++ _14 = const (); ++ StorageDead(_19); ++ StorageDead(_18); ++ StorageDead(_17); ++ _15 = <() as B>::call() -> [return: bb9, unwind unreachable]; } } diff --git a/tests/mir-opt/inline/exponential_runtime.main.Inline.panic-unwind.diff b/tests/mir-opt/inline/exponential_runtime.main.Inline.panic-unwind.diff index 3f334779e18f8..1561637f93cc6 100644 --- a/tests/mir-opt/inline/exponential_runtime.main.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/exponential_runtime.main.Inline.panic-unwind.diff @@ -38,28 +38,39 @@ bb0: { StorageLive(_1); - _1 = <() as G>::call() -> [return: bb1, unwind continue]; ++ StorageAlloc(_1); + StorageLive(_2); + StorageLive(_3); + StorageLive(_4); ++ StorageAlloc(_2); + StorageLive(_5); + StorageLive(_6); + StorageLive(_7); ++ StorageAlloc(_5); + StorageLive(_8); + StorageLive(_9); + StorageLive(_10); ++ StorageAlloc(_8); + StorageLive(_11); + StorageLive(_12); + StorageLive(_13); ++ StorageAlloc(_11); + StorageLive(_14); + StorageLive(_15); + StorageLive(_16); ++ StorageAlloc(_14); + StorageLive(_17); + StorageLive(_18); + StorageLive(_19); -+ _17 = <() as A>::call() -> [return: bb12, unwind continue]; ++ _17 = <() as A>::call() -> [return: bb11, unwind continue]; } bb1: { ++ _4 = <() as F>::call() -> [return: bb2, unwind continue]; ++ } ++ ++ bb2: { ++ _1 = const (); + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); @@ -68,67 +79,68 @@ return; + } + -+ bb2: { -+ _4 = <() as F>::call() -> [return: bb1, unwind continue]; ++ bb3: { ++ _7 = <() as E>::call() -> [return: bb4, unwind continue]; + } + -+ bb3: { ++ bb4: { ++ _2 = const (); + StorageDead(_7); + StorageDead(_6); + StorageDead(_5); -+ _3 = <() as F>::call() -> [return: bb2, unwind continue]; ++ _3 = <() as F>::call() -> [return: bb1, unwind continue]; + } + -+ bb4: { -+ _7 = <() as E>::call() -> [return: bb3, unwind continue]; ++ bb5: { ++ _10 = <() as D>::call() -> [return: bb6, unwind continue]; + } + -+ bb5: { ++ bb6: { ++ _5 = const (); + StorageDead(_10); + StorageDead(_9); + StorageDead(_8); -+ _6 = <() as E>::call() -> [return: bb4, unwind continue]; ++ _6 = <() as E>::call() -> [return: bb3, unwind continue]; + } + -+ bb6: { -+ _10 = <() as D>::call() -> [return: bb5, unwind continue]; ++ bb7: { ++ _13 = <() as C>::call() -> [return: bb8, unwind continue]; + } + -+ bb7: { ++ bb8: { ++ _8 = const (); + StorageDead(_13); + StorageDead(_12); + StorageDead(_11); -+ _9 = <() as D>::call() -> [return: bb6, unwind continue]; ++ _9 = <() as D>::call() -> [return: bb5, unwind continue]; + } + -+ bb8: { -+ _13 = <() as C>::call() -> [return: bb7, unwind continue]; ++ bb9: { ++ _16 = <() as B>::call() -> [return: bb10, unwind continue]; + } + -+ bb9: { ++ bb10: { ++ _11 = const (); + StorageDead(_16); + StorageDead(_15); + StorageDead(_14); -+ _12 = <() as C>::call() -> [return: bb8, unwind continue]; -+ } -+ -+ bb10: { -+ _16 = <() as B>::call() -> [return: bb9, unwind continue]; ++ _12 = <() as C>::call() -> [return: bb7, unwind continue]; + } + + bb11: { -+ StorageDead(_19); -+ StorageDead(_18); -+ StorageDead(_17); -+ _15 = <() as B>::call() -> [return: bb10, unwind continue]; ++ _18 = <() as A>::call() -> [return: bb12, unwind continue]; + } + + bb12: { -+ _18 = <() as A>::call() -> [return: bb13, unwind continue]; ++ _19 = <() as A>::call() -> [return: bb13, unwind continue]; + } + + bb13: { -+ _19 = <() as A>::call() -> [return: bb11, unwind continue]; ++ _14 = const (); ++ StorageDead(_19); ++ StorageDead(_18); ++ StorageDead(_17); ++ _15 = <() as B>::call() -> [return: bb9, unwind continue]; } } diff --git a/tests/mir-opt/inline/forced.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced.caller.ForceInline.panic-abort.diff index f894f06e5a0ab..2bff13ca52a0d 100644 --- a/tests/mir-opt/inline/forced.caller.ForceInline.panic-abort.diff +++ b/tests/mir-opt/inline/forced.caller.ForceInline.panic-abort.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_1); ++ _1 = const (); StorageDead(_1); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced.caller.ForceInline.panic-unwind.diff index 7b70fd665666a..37fbe055d71bd 100644 --- a/tests/mir-opt/inline/forced.caller.ForceInline.panic-unwind.diff +++ b/tests/mir-opt/inline/forced.caller.ForceInline.panic-unwind.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_1); ++ _1 = const (); StorageDead(_1); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-abort.diff index def2375efd068..84c1aeba18e8e 100644 --- a/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-abort.diff +++ b/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-abort.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_2); ++ _2 = const (); StorageDead(_2); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-unwind.diff index 8f50c20587d70..cbac3166329e8 100644 --- a/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-unwind.diff +++ b/tests/mir-opt/inline/forced_closure.caller-{closure#0}.ForceInline.panic-unwind.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_2); ++ _2 = const (); StorageDead(_2); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-abort.diff index 8e03432c2af2d..9c2f6d50167cf 100644 --- a/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-abort.diff +++ b/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-abort.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_2); ++ _2 = const (); StorageDead(_2); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-unwind.diff index 0e41fd89dacf7..ac741cc8e9327 100644 --- a/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-unwind.diff +++ b/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-unwind.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_2); ++ _2 = const (); StorageDead(_2); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced_dead_code.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_dead_code.caller.ForceInline.panic-abort.diff index f894f06e5a0ab..2bff13ca52a0d 100644 --- a/tests/mir-opt/inline/forced_dead_code.caller.ForceInline.panic-abort.diff +++ b/tests/mir-opt/inline/forced_dead_code.caller.ForceInline.panic-abort.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_1); ++ _1 = const (); StorageDead(_1); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced_dead_code.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_dead_code.caller.ForceInline.panic-unwind.diff index 7b70fd665666a..37fbe055d71bd 100644 --- a/tests/mir-opt/inline/forced_dead_code.caller.ForceInline.panic-unwind.diff +++ b/tests/mir-opt/inline/forced_dead_code.caller.ForceInline.panic-unwind.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_1); ++ _1 = const (); StorageDead(_1); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-abort.diff index 6ea1894af9895..87fcb6dfa7625 100644 --- a/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-abort.diff +++ b/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-abort.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_1); ++ _1 = const (); StorageDead(_1); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-unwind.diff index dd91c3387723f..7110e69ffb3d4 100644 --- a/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-unwind.diff +++ b/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-unwind.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_1); ++ _1 = const (); StorageDead(_1); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-abort.diff index 6ea1894af9895..87fcb6dfa7625 100644 --- a/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-abort.diff +++ b/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-abort.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_1); ++ _1 = const (); StorageDead(_1); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-unwind.diff index dd91c3387723f..7110e69ffb3d4 100644 --- a/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-unwind.diff +++ b/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-unwind.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_1); ++ _1 = const (); StorageDead(_1); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-abort.diff index edaf2820d85c3..33ec6b0db32b5 100644 --- a/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-abort.diff +++ b/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-abort.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_1); ++ _1 = const (); StorageDead(_1); _0 = const (); return; diff --git a/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-unwind.diff index 22f8b14a724b8..3b9b2589b4e13 100644 --- a/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-unwind.diff +++ b/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-unwind.diff @@ -13,6 +13,8 @@ - } - - bb1: { ++ StorageAlloc(_1); ++ _1 = const (); StorageDead(_1); _0 = const (); return; diff --git a/tests/mir-opt/inline/inline_any_operand.bar.Inline.after.mir b/tests/mir-opt/inline/inline_any_operand.bar.Inline.after.mir index 0ee5968419a06..b7fb3b5896603 100644 --- a/tests/mir-opt/inline/inline_any_operand.bar.Inline.after.mir +++ b/tests/mir-opt/inline/inline_any_operand.bar.Inline.after.mir @@ -19,6 +19,7 @@ fn bar() -> bool { _1 = foo; StorageLive(_2); _2 = copy _1; + StorageAlloc(_0); StorageLive(_3); _3 = const 1_i32; StorageLive(_4); diff --git a/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff index d04dc8f5ff5b2..0876e2b3f650a 100644 --- a/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff @@ -20,6 +20,7 @@ StorageLive(_4); _4 = (const 1_i32,); - _2 = as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_2); + StorageLive(_6); + StorageLive(_7); + StorageLive(_5); diff --git a/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff index f2fc8c7388f7d..7ad7e37aa6d7c 100644 --- a/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff @@ -20,6 +20,7 @@ StorageLive(_4); _4 = (const 1_i32,); - _2 = as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind: bb3]; ++ StorageAlloc(_2); + StorageLive(_6); + StorageLive(_7); + StorageLive(_5); diff --git a/tests/mir-opt/inline/inline_closure.foo.Inline.after.mir b/tests/mir-opt/inline/inline_closure.foo.Inline.after.mir index 5bc227c87df17..c05cfd0a39062 100644 --- a/tests/mir-opt/inline/inline_closure.foo.Inline.after.mir +++ b/tests/mir-opt/inline/inline_closure.foo.Inline.after.mir @@ -30,6 +30,7 @@ fn foo(_1: T, _2: i32) -> i32 { StorageLive(_7); _7 = copy _2; _5 = (move _6, move _7); + StorageAlloc(_0); StorageLive(_8); _8 = move (_5.0: i32); StorageLive(_9); diff --git a/tests/mir-opt/inline/inline_closure_borrows_arg.foo.Inline.after.mir b/tests/mir-opt/inline/inline_closure_borrows_arg.foo.Inline.after.mir index 7930111cf2296..14bb035d4b79a 100644 --- a/tests/mir-opt/inline/inline_closure_borrows_arg.foo.Inline.after.mir +++ b/tests/mir-opt/inline/inline_closure_borrows_arg.foo.Inline.after.mir @@ -30,6 +30,7 @@ fn foo(_1: T, _2: &i32) -> i32 { StorageLive(_7); _7 = copy _2; _5 = (move _6, move _7); + StorageAlloc(_0); StorageLive(_8); _8 = move (_5.0: &i32); StorageLive(_9); diff --git a/tests/mir-opt/inline/inline_closure_captures.foo.Inline.after.mir b/tests/mir-opt/inline/inline_closure_captures.foo.Inline.after.mir index 38d9fc4cfd567..97fe0b0f4e503 100644 --- a/tests/mir-opt/inline/inline_closure_captures.foo.Inline.after.mir +++ b/tests/mir-opt/inline/inline_closure_captures.foo.Inline.after.mir @@ -36,6 +36,7 @@ fn foo(_1: T, _2: i32) -> (i32, T) { StorageLive(_8); _8 = copy _2; _7 = (move _8,); + StorageAlloc(_0); StorageLive(_9); _9 = move (_7.0: i32); StorageLive(_10); diff --git a/tests/mir-opt/inline/inline_compatibility.rs b/tests/mir-opt/inline/inline_compatibility.rs index 3f72584017abd..43875669be3c7 100644 --- a/tests/mir-opt/inline/inline_compatibility.rs +++ b/tests/mir-opt/inline/inline_compatibility.rs @@ -15,6 +15,7 @@ fn nop() {} // CHECK-LABEL: fn f0() // CHECK: bb0: { +// CHECK-NEXT: _0 = const (); // CHECK-NEXT: return; #[target_feature(enable = "sse2")] pub unsafe fn f0() { @@ -42,6 +43,7 @@ pub unsafe fn sanitize_off() {} // CHECK-LABEL: fn inlined_sanitize_off() // CHECK: bb0: { +// CHECK-NEXT: _0 = const (); // CHECK-NEXT: return; #[sanitize(address = "off")] pub unsafe fn inlined_sanitize_off() { diff --git a/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-abort.diff index 151580da19e09..bc646840c2574 100644 --- a/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-abort.diff @@ -31,10 +31,13 @@ StorageLive(_3); StorageLive(_4); - _4 = g() -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_4); + _4 = {coroutine@$DIR/inline_coroutine.rs:20:5: 20:8 (#0)}; + _3 = &mut _4; ++ StorageAlloc(_2); + _2 = Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:20:5: 20:8}> { pointer: copy _3 }; + StorageDead(_3); ++ StorageAlloc(_1); + StorageLive(_5); + _5 = const false; + StorageLive(_6); diff --git a/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-unwind.diff index 6196fc0d0c6bf..79a0ead84f299 100644 --- a/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-unwind.diff @@ -31,10 +31,13 @@ StorageLive(_3); StorageLive(_4); - _4 = g() -> [return: bb1, unwind continue]; ++ StorageAlloc(_4); + _4 = {coroutine@$DIR/inline_coroutine.rs:20:5: 20:8 (#0)}; + _3 = &mut _4; ++ StorageAlloc(_2); + _2 = Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:20:5: 20:8}> { pointer: copy _3 }; + StorageDead(_3); ++ StorageAlloc(_1); + StorageLive(_5); + _5 = const false; + StorageLive(_6); diff --git a/tests/mir-opt/inline/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff index 4b117a453c326..3259f9970211d 100644 --- a/tests/mir-opt/inline/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff @@ -103,6 +103,7 @@ - } - - bb1: { ++ StorageAlloc(_2); + _2 = {coroutine@$DIR/inline_coroutine_body.rs:25:28: 27:6 (#0)} { self: move _3 }; StorageDead(_3); StorageLive(_4); @@ -112,6 +113,7 @@ - } - - bb2: { ++ StorageAlloc(_4); + _4 = Pin::<&mut {async fn body of ActionPermit<'_, T>::perform()}> { pointer: copy _5 }; StorageDead(_5); StorageLive(_6); @@ -122,6 +124,7 @@ _10 = no_retag copy (_1.1: &mut std::task::Context<'_>); _9 = &mut (*_10); - _7 = <{async fn body of ActionPermit<'_, T>::perform()} as Future>::poll(move _8, move _9) -> [return: bb3, unwind unreachable]; ++ StorageAlloc(_7); + StorageLive(_16); + StorageLive(_25); + StorageLive(_30); @@ -169,6 +172,7 @@ + StorageLive(_21); + _21 = &mut (((*_32) as variant#3).1: std::future::Ready<()>); + _20 = &mut (*_21); ++ StorageAlloc(_19); + _19 = Pin::<&mut std::future::Ready<()>> { pointer: copy _20 }; + StorageDead(_20); + StorageLive(_22); @@ -178,6 +182,7 @@ + _23 = copy (_24.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + _22 = &mut (*_23); + StorageDead(_24); ++ StorageAlloc(_18); + StorageLive(_38); + StorageLive(_40); + StorageLive(_45); @@ -188,11 +193,13 @@ + _40 = copy _41 as *mut std::pin::helper::PinHelper<&mut std::future::Ready<()>> (PtrToPtr); + StorageDead(_41); + _38 = no_retag copy ((*_40).0: &mut std::future::Ready<()>); ++ StorageAlloc(_36); + StorageLive(_43); + _43 = Option::<()>::None; + _36 = copy ((*_38).0: std::option::Option<()>); + ((*_38).0: std::option::Option<()>) = move _43; + StorageDead(_43); ++ StorageAlloc(_35); + StorageLive(_44); + _44 = discriminant(_36); + switchInt(move _44) -> [0: bb11, 1: bb12, otherwise: bb4]; @@ -262,11 +269,13 @@ + StorageLive(_13); + StorageLive(_14); + _14 = (); ++ StorageAlloc(_13); + StorageLive(_46); + _46 = Option::<()>::Some(copy _14); + _13 = std::future::Ready::<()>(move _46); + StorageDead(_46); + StorageDead(_14); ++ StorageAlloc(_12); + _12 = move _13; + StorageDead(_13); + (((*_32) as variant#3).1: std::future::Ready<()>) = move _12; diff --git a/tests/mir-opt/inline/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff index c365aee05f4ec..1d48d8e003787 100644 --- a/tests/mir-opt/inline/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff @@ -103,6 +103,7 @@ - } - - bb1: { ++ StorageAlloc(_2); + _2 = {coroutine@$DIR/inline_coroutine_body.rs:25:28: 27:6 (#0)} { self: move _3 }; StorageDead(_3); StorageLive(_4); @@ -112,6 +113,7 @@ - } - - bb2: { ++ StorageAlloc(_4); + _4 = Pin::<&mut {async fn body of ActionPermit<'_, T>::perform()}> { pointer: copy _5 }; StorageDead(_5); StorageLive(_6); @@ -122,6 +124,7 @@ _10 = no_retag copy (_1.1: &mut std::task::Context<'_>); _9 = &mut (*_10); - _7 = <{async fn body of ActionPermit<'_, T>::perform()} as Future>::poll(move _8, move _9) -> [return: bb3, unwind: bb5]; ++ StorageAlloc(_7); + StorageLive(_16); + StorageLive(_25); + StorageLive(_30); @@ -180,6 +183,7 @@ + StorageLive(_21); + _21 = &mut (((*_32) as variant#3).1: std::future::Ready<()>); + _20 = &mut (*_21); ++ StorageAlloc(_19); + _19 = Pin::<&mut std::future::Ready<()>> { pointer: copy _20 }; + StorageDead(_20); + StorageLive(_22); @@ -189,6 +193,7 @@ + _23 = copy (_24.0: std::ptr::NonNull>) as &mut std::task::Context<'_> (Transmute); + _22 = &mut (*_23); + StorageDead(_24); ++ StorageAlloc(_18); + StorageLive(_38); + StorageLive(_40); + StorageLive(_45); @@ -199,11 +204,13 @@ + _40 = copy _41 as *mut std::pin::helper::PinHelper<&mut std::future::Ready<()>> (PtrToPtr); + StorageDead(_41); + _38 = no_retag copy ((*_40).0: &mut std::future::Ready<()>); ++ StorageAlloc(_36); + StorageLive(_43); + _43 = Option::<()>::None; + _36 = copy ((*_38).0: std::option::Option<()>); + ((*_38).0: std::option::Option<()>) = move _43; + StorageDead(_43); ++ StorageAlloc(_35); + StorageLive(_44); + _44 = discriminant(_36); + switchInt(move _44) -> [0: bb16, 1: bb17, otherwise: bb6]; @@ -295,11 +302,13 @@ + StorageLive(_13); + StorageLive(_14); + _14 = (); ++ StorageAlloc(_13); + StorageLive(_46); + _46 = Option::<()>::Some(copy _14); + _13 = std::future::Ready::<()>(move _46); + StorageDead(_46); + StorageDead(_14); ++ StorageAlloc(_12); + _12 = move _13; + StorageDead(_13); + (((*_32) as variant#3).1: std::future::Ready<()>) = move _12; diff --git a/tests/mir-opt/inline/inline_cycle.one.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_cycle.one.Inline.panic-abort.diff index c8805a4466a7b..77265afe8a3cc 100644 --- a/tests/mir-opt/inline/inline_cycle.one.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_cycle.one.Inline.panic-abort.diff @@ -13,6 +13,8 @@ bb0: { StorageLive(_1); ++ StorageAlloc(_1); ++ StorageAlloc(_1); _1 = ::call() -> [return: bb1, unwind unreachable]; } diff --git a/tests/mir-opt/inline/inline_cycle.one.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_cycle.one.Inline.panic-unwind.diff index 75ac40bea61fb..5c90a79082833 100644 --- a/tests/mir-opt/inline/inline_cycle.one.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_cycle.one.Inline.panic-unwind.diff @@ -13,6 +13,8 @@ bb0: { StorageLive(_1); ++ StorageAlloc(_1); ++ StorageAlloc(_1); _1 = ::call() -> [return: bb1, unwind continue]; } diff --git a/tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff index c878b8b632414..2258981e69acf 100644 --- a/tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff @@ -19,18 +19,23 @@ bb0: { StorageLive(_1); - _1 = call::(f) -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_1); + StorageLive(_2); + _2 = f; + StorageLive(_3); ++ StorageAlloc(_3); + StorageLive(_4); + _4 = const (); ++ StorageAlloc(_3); + StorageLive(_5); + _5 = call::(f) -> [return: bb1, unwind unreachable]; } bb1: { ++ _3 = const (); + StorageDead(_5); + StorageDead(_4); ++ _1 = const (); + StorageDead(_3); + StorageDead(_2); StorageDead(_1); diff --git a/tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff index 072b7ab3fa491..3919849c71f9d 100644 --- a/tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff @@ -19,18 +19,23 @@ bb0: { StorageLive(_1); - _1 = call::(f) -> [return: bb1, unwind continue]; ++ StorageAlloc(_1); + StorageLive(_2); + _2 = f; + StorageLive(_3); ++ StorageAlloc(_3); + StorageLive(_4); + _4 = const (); ++ StorageAlloc(_3); + StorageLive(_5); + _5 = call::(f) -> [return: bb1, unwind continue]; } bb1: { ++ _3 = const (); + StorageDead(_5); + StorageDead(_4); ++ _1 = const (); + StorageDead(_3); + StorageDead(_2); StorageDead(_1); diff --git a/tests/mir-opt/inline/inline_cycle_generic.main.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_cycle_generic.main.Inline.panic-abort.diff index d437dbf5763a2..7980350c5a457 100644 --- a/tests/mir-opt/inline/inline_cycle_generic.main.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_cycle_generic.main.Inline.panic-abort.diff @@ -14,6 +14,9 @@ bb0: { StorageLive(_1); - _1 = ::call() -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_1); ++ StorageAlloc(_1); ++ StorageAlloc(_1); + _1 = as Call>::call() -> [return: bb1, unwind unreachable]; } diff --git a/tests/mir-opt/inline/inline_cycle_generic.main.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_cycle_generic.main.Inline.panic-unwind.diff index 8314526ee0431..f20dbb8b34de9 100644 --- a/tests/mir-opt/inline/inline_cycle_generic.main.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_cycle_generic.main.Inline.panic-unwind.diff @@ -14,6 +14,9 @@ bb0: { StorageLive(_1); - _1 = ::call() -> [return: bb1, unwind continue]; ++ StorageAlloc(_1); ++ StorageAlloc(_1); ++ StorageAlloc(_1); + _1 = as Call>::call() -> [return: bb1, unwind continue]; } diff --git a/tests/mir-opt/inline/inline_diverging.f.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_diverging.f.Inline.panic-abort.diff index 3de72d03ced76..0c0e06cc6361a 100644 --- a/tests/mir-opt/inline/inline_diverging.f.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_diverging.f.Inline.panic-abort.diff @@ -11,6 +11,7 @@ bb0: { StorageLive(_2); - _2 = sleep() -> unwind unreachable; ++ StorageAlloc(_2); + goto -> bb1; + } + diff --git a/tests/mir-opt/inline/inline_diverging.f.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_diverging.f.Inline.panic-unwind.diff index b799189925347..cb4081200b67c 100644 --- a/tests/mir-opt/inline/inline_diverging.f.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_diverging.f.Inline.panic-unwind.diff @@ -11,6 +11,7 @@ bb0: { StorageLive(_2); - _2 = sleep() -> unwind continue; ++ StorageAlloc(_2); + goto -> bb1; + } + diff --git a/tests/mir-opt/inline/inline_diverging.g.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_diverging.g.Inline.panic-abort.diff index 8d7a32322ad93..5a3085f8ff06f 100644 --- a/tests/mir-opt/inline/inline_diverging.g.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_diverging.g.Inline.panic-abort.diff @@ -35,6 +35,7 @@ StorageDead(_3); StorageLive(_6); - _6 = panic() -> unwind unreachable; ++ StorageAlloc(_6); + StorageLive(_7); + _7 = std::rt::begin_panic::<&str>(const "explicit panic") -> unwind unreachable; } diff --git a/tests/mir-opt/inline/inline_diverging.g.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_diverging.g.Inline.panic-unwind.diff index 11a9ae601fb50..8b0bd084decb1 100644 --- a/tests/mir-opt/inline/inline_diverging.g.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_diverging.g.Inline.panic-unwind.diff @@ -35,6 +35,7 @@ StorageDead(_3); StorageLive(_6); - _6 = panic() -> unwind continue; ++ StorageAlloc(_6); + StorageLive(_7); + _7 = std::rt::begin_panic::<&str>(const "explicit panic") -> unwind continue; } diff --git a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff index a116086a0ce14..7dbfec7e97f54 100644 --- a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff @@ -32,13 +32,16 @@ bb0: { StorageLive(_1); - _1 = call_twice:: ! {sleep}>(sleep) -> unwind unreachable; ++ StorageAlloc(_1); + StorageLive(_2); + _2 = sleep; + StorageLive(_4); + StorageLive(_3); + _3 = &_2; ++ StorageAlloc(_4); + StorageLive(_7); + _7 = const (); ++ StorageAlloc(_4); + goto -> bb1; + } + diff --git a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff index 978de2884c081..f18043bfb1bf2 100644 --- a/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff @@ -33,13 +33,16 @@ bb0: { StorageLive(_1); - _1 = call_twice:: ! {sleep}>(sleep) -> unwind continue; ++ StorageAlloc(_1); + StorageLive(_2); + _2 = sleep; + StorageLive(_4); + StorageLive(_3); + _3 = &_2; ++ StorageAlloc(_4); + StorageLive(_8); + _8 = const (); ++ StorageAlloc(_4); + goto -> bb1; + } + diff --git a/tests/mir-opt/inline/inline_empty_drop_glue.rs b/tests/mir-opt/inline/inline_empty_drop_glue.rs index ad7dbb06da1eb..ce446b24e3237 100644 --- a/tests/mir-opt/inline/inline_empty_drop_glue.rs +++ b/tests/mir-opt/inline/inline_empty_drop_glue.rs @@ -6,7 +6,8 @@ unsafe fn slice_in_place(ptr: *mut [char]) { // CHECK-LABEL: fn slice_in_place(_1: *mut [char]) // CHECK: bb0: { - // CHECK-NEXT: return; + // CHECK-NOT: -> + // CHECK: return; // CHECK-NEXT: } std::ptr::drop_in_place(ptr) } diff --git a/tests/mir-opt/inline/inline_empty_drop_glue.slice_in_place.Inline.diff b/tests/mir-opt/inline/inline_empty_drop_glue.slice_in_place.Inline.diff index 2b5cbe6e44820..924a97ecb07f9 100644 --- a/tests/mir-opt/inline/inline_empty_drop_glue.slice_in_place.Inline.diff +++ b/tests/mir-opt/inline/inline_empty_drop_glue.slice_in_place.Inline.diff @@ -18,8 +18,11 @@ - } - - bb1: { ++ StorageAlloc(_0); + StorageLive(_3); + _3 = &mut (*_2); ++ StorageAlloc(_0); ++ _0 = const (); + StorageDead(_3); StorageDead(_2); return; diff --git a/tests/mir-opt/inline/inline_instruction_set.default.Inline.diff b/tests/mir-opt/inline/inline_instruction_set.default.Inline.diff index 158cc973779c8..a93596e224996 100644 --- a/tests/mir-opt/inline/inline_instruction_set.default.Inline.diff +++ b/tests/mir-opt/inline/inline_instruction_set.default.Inline.diff @@ -30,14 +30,18 @@ - } - - bb3: { ++ StorageAlloc(_3); ++ _3 = const (); StorageDead(_3); StorageLive(_4); - _4 = inline_always_and_using_inline_asm() -> [return: bb4, unwind unreachable]; ++ StorageAlloc(_4); + asm!("/* do nothing */", options()) -> [return: bb3, unwind unreachable]; } - bb4: { + bb3: { ++ _4 = const (); StorageDead(_4); _0 = const (); return; diff --git a/tests/mir-opt/inline/inline_instruction_set.t32.Inline.diff b/tests/mir-opt/inline/inline_instruction_set.t32.Inline.diff index 33dc116f622e1..1e2b16a2e6d5a 100644 --- a/tests/mir-opt/inline/inline_instruction_set.t32.Inline.diff +++ b/tests/mir-opt/inline/inline_instruction_set.t32.Inline.diff @@ -24,12 +24,16 @@ - } - - bb2: { ++ StorageAlloc(_2); ++ _2 = const (); StorageDead(_2); StorageLive(_3); - _3 = instruction_set_default() -> [return: bb3, unwind unreachable]; - } - - bb3: { ++ StorageAlloc(_3); ++ _3 = const (); StorageDead(_3); StorageLive(_4); - _4 = inline_always_and_using_inline_asm() -> [return: bb4, unwind unreachable]; diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-abort.mir b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-abort.mir index 374ed294d8b26..9fb1ff22dfcbb 100644 --- a/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-abort.mir +++ b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-abort.mir @@ -13,11 +13,13 @@ fn marked_inline_indirect(_1: i32) -> () { StorageLive(_2); StorageLive(_3); _3 = copy _1; + StorageAlloc(_2); StorageLive(_4); _4 = call_twice(move _3) -> [return: bb1, unwind unreachable]; } bb1: { + _2 = const (); StorageDead(_4); StorageDead(_3); StorageDead(_2); diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-unwind.mir b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-unwind.mir index 97bed8020bad7..ac666bebf1f83 100644 --- a/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-unwind.mir +++ b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-unwind.mir @@ -13,11 +13,13 @@ fn marked_inline_indirect(_1: i32) -> () { StorageLive(_2); StorageLive(_3); _3 = copy _1; + StorageAlloc(_2); StorageLive(_4); _4 = call_twice(move _3) -> [return: bb1, unwind continue]; } bb1: { + _2 = const (); StorageDead(_4); StorageDead(_3); StorageDead(_2); diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-abort.mir b/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-abort.mir index cffe0e96f5c94..93e9e420f508c 100644 --- a/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-abort.mir +++ b/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-abort.mir @@ -14,12 +14,18 @@ fn monomorphic_not_inline(_1: i32) -> () { StorageLive(_2); StorageLive(_3); _3 = copy _1; + StorageAlloc(_2); StorageLive(_4); StorageLive(_5); - _4 = other_thing(copy _3) -> [return: bb2, unwind unreachable]; + _4 = other_thing(copy _3) -> [return: bb1, unwind unreachable]; } bb1: { + _5 = other_thing(move _3) -> [return: bb2, unwind unreachable]; + } + + bb2: { + _2 = const (); StorageDead(_5); StorageDead(_4); StorageDead(_3); @@ -27,8 +33,4 @@ fn monomorphic_not_inline(_1: i32) -> () { _0 = const (); return; } - - bb2: { - _5 = other_thing(move _3) -> [return: bb1, unwind unreachable]; - } } diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-unwind.mir b/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-unwind.mir index 7f3a093012bcd..5d877423613e6 100644 --- a/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-unwind.mir +++ b/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-unwind.mir @@ -14,12 +14,18 @@ fn monomorphic_not_inline(_1: i32) -> () { StorageLive(_2); StorageLive(_3); _3 = copy _1; + StorageAlloc(_2); StorageLive(_4); StorageLive(_5); - _4 = other_thing(copy _3) -> [return: bb2, unwind continue]; + _4 = other_thing(copy _3) -> [return: bb1, unwind continue]; } bb1: { + _5 = other_thing(move _3) -> [return: bb2, unwind continue]; + } + + bb2: { + _2 = const (); StorageDead(_5); StorageDead(_4); StorageDead(_3); @@ -27,8 +33,4 @@ fn monomorphic_not_inline(_1: i32) -> () { _0 = const (); return; } - - bb2: { - _5 = other_thing(move _3) -> [return: bb1, unwind continue]; - } } diff --git a/tests/mir-opt/inline/inline_options.main.Inline.after.panic-abort.mir b/tests/mir-opt/inline/inline_options.main.Inline.after.panic-abort.mir index 3218c4e567378..3b42eb975fcd7 100644 --- a/tests/mir-opt/inline/inline_options.main.Inline.after.panic-abort.mir +++ b/tests/mir-opt/inline/inline_options.main.Inline.after.panic-abort.mir @@ -18,26 +18,28 @@ fn main() -> () { bb1: { StorageDead(_1); StorageLive(_2); + StorageAlloc(_2); StorageLive(_3); StorageLive(_4); StorageLive(_5); - _3 = g() -> [return: bb3, unwind unreachable]; + _3 = g() -> [return: bb2, unwind unreachable]; } bb2: { - StorageDead(_5); - StorageDead(_4); - StorageDead(_3); - StorageDead(_2); - _0 = const (); - return; + _4 = g() -> [return: bb3, unwind unreachable]; } bb3: { - _4 = g() -> [return: bb4, unwind unreachable]; + _5 = g() -> [return: bb4, unwind unreachable]; } bb4: { - _5 = g() -> [return: bb2, unwind unreachable]; + _2 = const (); + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; } } diff --git a/tests/mir-opt/inline/inline_options.main.Inline.after.panic-unwind.mir b/tests/mir-opt/inline/inline_options.main.Inline.after.panic-unwind.mir index df0cab513eca7..7e0b1b3f0fde1 100644 --- a/tests/mir-opt/inline/inline_options.main.Inline.after.panic-unwind.mir +++ b/tests/mir-opt/inline/inline_options.main.Inline.after.panic-unwind.mir @@ -18,26 +18,28 @@ fn main() -> () { bb1: { StorageDead(_1); StorageLive(_2); + StorageAlloc(_2); StorageLive(_3); StorageLive(_4); StorageLive(_5); - _3 = g() -> [return: bb3, unwind continue]; + _3 = g() -> [return: bb2, unwind continue]; } bb2: { - StorageDead(_5); - StorageDead(_4); - StorageDead(_3); - StorageDead(_2); - _0 = const (); - return; + _4 = g() -> [return: bb3, unwind continue]; } bb3: { - _4 = g() -> [return: bb4, unwind continue]; + _5 = g() -> [return: bb4, unwind continue]; } bb4: { - _5 = g() -> [return: bb2, unwind continue]; + _2 = const (); + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; } } diff --git a/tests/mir-opt/inline/inline_shims.clone.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_shims.clone.Inline.panic-abort.diff index 26b4dc357f337..dc785d935387c 100644 --- a/tests/mir-opt/inline/inline_shims.clone.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_shims.clone.Inline.panic-abort.diff @@ -15,6 +15,7 @@ - } - - bb1: { ++ StorageAlloc(_0); + _0 = copy (*_2); StorageDead(_2); return; diff --git a/tests/mir-opt/inline/inline_shims.clone.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_shims.clone.Inline.panic-unwind.diff index faa12d5c73605..0fc1f3dd38ef4 100644 --- a/tests/mir-opt/inline/inline_shims.clone.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_shims.clone.Inline.panic-unwind.diff @@ -15,6 +15,7 @@ - } - - bb1: { ++ StorageAlloc(_0); + _0 = copy (*_2); StorageDead(_2); return; diff --git a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff index a7cf6a7733083..395673b6658f5 100644 --- a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff @@ -13,49 +13,13 @@ + scope 2 (inlined std::ptr::drop_glue::> - shim(Some(Vec))) { + let mut _7: &mut std::vec::Vec; + let mut _8: (); -+ scope 3 (inlined as Drop>::drop) { -+ let mut _9: *mut [A]; -+ let mut _10: *mut A; -+ let mut _11: usize; -+ scope 4 (inlined Vec::::as_mut_ptr) { -+ scope 5 (inlined alloc::raw_vec::RawVec::::ptr) { -+ scope 6 (inlined alloc::raw_vec::RawVecInner::ptr::) { -+ scope 7 (inlined alloc::raw_vec::RawVecInner::non_null::) { -+ let mut _12: std::ptr::NonNull; -+ scope 8 (inlined NonNull::::cast::) { -+ scope 9 (inlined NonNull::::as_ptr) { -+ } -+ } -+ } -+ scope 10 (inlined NonNull::::as_ptr) { -+ } -+ } -+ } -+ } -+ scope 11 (inlined std::ptr::mut_ptr::::cast_slice) { -+ scope 12 (inlined slice_from_raw_parts_mut::) { -+ scope 13 (inlined std::ptr::from_raw_parts_mut::<[A], A>) { -+ } -+ } -+ } -+ scope 14 (inlined std::ptr::mut_ptr::::drop_in_place) { -+ scope 15 (inlined drop_in_place::<[A]>) { -+ let mut _13: &mut [A]; -+ scope 16 (inlined std::ptr::drop_glue::<[A]> - shim(Some([A]))) { -+ let mut _14: usize; -+ let mut _15: *mut A; -+ let mut _16: bool; -+ } -+ } -+ } -+ } + } + } -+ scope 17 (inlined drop_in_place::>) { -+ let mut _17: &mut std::option::Option; -+ scope 18 (inlined std::ptr::drop_glue::> - shim(Some(Option))) { -+ let mut _18: isize; -+ let mut _19: isize; ++ scope 3 (inlined drop_in_place::>) { ++ let mut _9: &mut std::option::Option; ++ scope 4 (inlined std::ptr::drop_glue::> - shim(Some(Option))) { ++ let mut _10: isize; ++ let mut _11: isize; + } + } @@ -64,31 +28,18 @@ StorageLive(_4); _4 = copy _1; - _3 = drop_in_place::>(move _4) -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_3); + StorageLive(_6); + _6 = &mut (*_4); ++ StorageAlloc(_3); + StorageLive(_7); + StorageLive(_8); + _7 = copy _6; -+ StorageLive(_11); -+ StorageLive(_9); -+ StorageLive(_10); -+ StorageLive(_12); -+ _12 = copy ((((*_7).0: alloc::raw_vec::RawVec).0: alloc::raw_vec::RawVecInner).0: std::ptr::NonNull); -+ _10 = copy _12 as *mut A (Transmute); -+ StorageDead(_12); -+ _11 = copy ((*_7).1: usize); -+ _9 = *mut [A] from (copy _10, copy _11); -+ StorageDead(_10); -+ StorageLive(_13); -+ _13 = &mut (*_9); -+ StorageLive(_14); -+ StorageLive(_15); -+ StorageLive(_16); -+ _14 = const 0_usize; -+ goto -> bb4; ++ _8 = as Drop>::drop(move _7) -> [return: bb2, unwind unreachable]; } bb1: { ++ _3 = const (); + StorageDead(_8); + StorageDead(_7); + StorageDead(_6); @@ -97,43 +48,29 @@ StorageLive(_5); _5 = copy _2; - _0 = drop_in_place::>(move _5) -> [return: bb2, unwind unreachable]; -+ StorageLive(_17); -+ _17 = &mut (*_5); -+ StorageLive(_18); -+ _18 = discriminant((*_17)); -+ switchInt(move _18) -> [0: bb5, otherwise: bb6]; ++ StorageAlloc(_0); ++ StorageLive(_9); ++ _9 = &mut (*_5); ++ StorageAlloc(_0); ++ StorageLive(_10); ++ _10 = discriminant((*_9)); ++ switchInt(move _10) -> [0: bb3, otherwise: bb4]; } bb2: { -+ StorageDead(_16); -+ StorageDead(_15); -+ StorageDead(_14); -+ StorageDead(_13); -+ StorageDead(_9); -+ StorageDead(_11); + drop(((*_6).0: alloc::raw_vec::RawVec)) -> [return: bb1, unwind unreachable]; + } + + bb3: { -+ _15 = &raw mut (*_13)[_14]; -+ _14 = Add(move _14, const 1_usize); -+ drop((*_15)) -> [return: bb4, unwind unreachable]; -+ } -+ -+ bb4: { -+ _16 = Eq(copy _14, copy _11); -+ switchInt(move _16) -> [0: bb3, otherwise: bb2]; -+ } -+ -+ bb5: { -+ StorageDead(_18); -+ StorageDead(_17); ++ _0 = const (); ++ StorageDead(_10); ++ StorageDead(_9); StorageDead(_5); return; + } + -+ bb6: { -+ drop((((*_17) as Some).0: B)) -> [return: bb5, unwind unreachable]; ++ bb4: { ++ drop((((*_9) as Some).0: B)) -> [return: bb3, unwind unreachable]; } } diff --git a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-unwind.diff index 2160cd19652e3..5923380968806 100644 --- a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-unwind.diff @@ -24,6 +24,7 @@ StorageLive(_4); _4 = copy _1; - _3 = drop_in_place::>(move _4) -> [return: bb1, unwind continue]; ++ StorageAlloc(_3); + StorageLive(_6); + _6 = &mut (*_4); + _3 = std::ptr::drop_glue::>(move _6) -> [return: bb1, unwind continue]; @@ -36,14 +37,17 @@ StorageLive(_5); _5 = copy _2; - _0 = drop_in_place::>(move _5) -> [return: bb2, unwind continue]; ++ StorageAlloc(_0); + StorageLive(_7); + _7 = &mut (*_5); ++ StorageAlloc(_0); + StorageLive(_8); + _8 = discriminant((*_7)); + switchInt(move _8) -> [0: bb2, otherwise: bb3]; } bb2: { ++ _0 = const (); + StorageDead(_8); + StorageDead(_7); StorageDead(_5); diff --git a/tests/mir-opt/inline/inline_specialization.main.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_specialization.main.Inline.panic-abort.diff index 22afcd2ee5655..f796c49058be2 100644 --- a/tests/mir-opt/inline/inline_specialization.main.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_specialization.main.Inline.panic-abort.diff @@ -16,6 +16,7 @@ - } - - bb1: { ++ StorageAlloc(_1); + _1 = const 123_u32; _0 = const (); StorageDead(_1); diff --git a/tests/mir-opt/inline/inline_specialization.main.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_specialization.main.Inline.panic-unwind.diff index bc841101df739..9e323107a8669 100644 --- a/tests/mir-opt/inline/inline_specialization.main.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/inline_specialization.main.Inline.panic-unwind.diff @@ -16,6 +16,7 @@ - } - - bb1: { ++ StorageAlloc(_1); + _1 = const 123_u32; _0 = const (); StorageDead(_1); diff --git a/tests/mir-opt/inline/inline_trait_method_2.test2.Inline.after.panic-abort.mir b/tests/mir-opt/inline/inline_trait_method_2.test2.Inline.after.panic-abort.mir index 1105dba06ac41..1eefa8259801b 100644 --- a/tests/mir-opt/inline/inline_trait_method_2.test2.Inline.after.panic-abort.mir +++ b/tests/mir-opt/inline/inline_trait_method_2.test2.Inline.after.panic-abort.mir @@ -15,6 +15,7 @@ fn test2(_1: &dyn X) -> bool { _3 = copy _1; _2 = move _3; StorageDead(_3); + StorageAlloc(_0); _0 = ::y(move _2) -> [return: bb1, unwind unreachable]; } diff --git a/tests/mir-opt/inline/inline_trait_method_2.test2.Inline.after.panic-unwind.mir b/tests/mir-opt/inline/inline_trait_method_2.test2.Inline.after.panic-unwind.mir index faa6ef47b18a4..bfc5bc939b5b0 100644 --- a/tests/mir-opt/inline/inline_trait_method_2.test2.Inline.after.panic-unwind.mir +++ b/tests/mir-opt/inline/inline_trait_method_2.test2.Inline.after.panic-unwind.mir @@ -15,6 +15,7 @@ fn test2(_1: &dyn X) -> bool { _3 = copy _1; _2 = move _3; StorageDead(_3); + StorageAlloc(_0); _0 = ::y(move _2) -> [return: bb1, unwind continue]; } diff --git a/tests/mir-opt/inline/issue_106141.outer.Inline.panic-abort.diff b/tests/mir-opt/inline/issue_106141.outer.Inline.panic-abort.diff index e87a565f0fc98..068d0285214f4 100644 --- a/tests/mir-opt/inline/issue_106141.outer.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/issue_106141.outer.Inline.panic-abort.diff @@ -17,6 +17,7 @@ bb0: { - _0 = inner() -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_0); + StorageLive(_1); + StorageLive(_2); + _1 = const inner::promoted[0]; diff --git a/tests/mir-opt/inline/issue_106141.outer.Inline.panic-unwind.diff b/tests/mir-opt/inline/issue_106141.outer.Inline.panic-unwind.diff index 834851b75adf1..4c0492b5db594 100644 --- a/tests/mir-opt/inline/issue_106141.outer.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/issue_106141.outer.Inline.panic-unwind.diff @@ -17,6 +17,7 @@ bb0: { - _0 = inner() -> [return: bb1, unwind continue]; ++ StorageAlloc(_0); + StorageLive(_1); + StorageLive(_2); + _1 = const inner::promoted[0]; diff --git a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.a.Inline.after.mir b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.a.Inline.after.mir index b28c2b851cc93..ecf79f82f903a 100644 --- a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.a.Inline.after.mir +++ b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.a.Inline.after.mir @@ -15,6 +15,7 @@ fn a(_1: &mut [T]) -> &mut [T] { StorageLive(_3); StorageLive(_4); _4 = no_retag copy _1; + StorageAlloc(_3); _3 = copy _4; _2 = copy _3; StorageDead(_4); diff --git a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.b.Inline.after.mir b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.b.Inline.after.mir index 1d56fa0860654..2f6dcf5d284dc 100644 --- a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.b.Inline.after.mir +++ b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.b.Inline.after.mir @@ -17,6 +17,7 @@ fn b(_1: &mut Box) -> &mut T { StorageLive(_3); StorageLive(_4); _4 = no_retag copy _1; + StorageAlloc(_3); StorageLive(_5); StorageLive(_6); _6 = no_retag copy ((*_4).0: std::ptr::Unique); diff --git a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.c.Inline.after.mir b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.c.Inline.after.mir index 2d88f9e55249f..94dd50957d904 100644 --- a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.c.Inline.after.mir +++ b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.c.Inline.after.mir @@ -13,6 +13,7 @@ fn c(_1: &[T]) -> &[T] { StorageLive(_2); StorageLive(_3); _3 = copy _1; + StorageAlloc(_2); _2 = copy _3; _0 = copy _2; StorageDead(_3); diff --git a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.d.Inline.after.mir b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.d.Inline.after.mir index a74065283737b..feb25be581cd5 100644 --- a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.d.Inline.after.mir +++ b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.d.Inline.after.mir @@ -15,6 +15,7 @@ fn d(_1: &Box) -> &T { StorageLive(_2); StorageLive(_3); _3 = copy _1; + StorageAlloc(_2); StorageLive(_4); StorageLive(_5); _5 = no_retag copy ((*_3).0: std::ptr::Unique); diff --git a/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.main.Inline.after.mir b/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.main.Inline.after.mir index 63d3ae2a86987..bd8e947f6c289 100644 --- a/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.main.Inline.after.mir +++ b/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.main.Inline.after.mir @@ -26,8 +26,10 @@ fn main() -> () { StorageLive(_4); _4 = (); _3 = (move _4,); + StorageAlloc(_0); StorageLive(_5); _5 = move (_3.0: ()); + _0 = const (); StorageDead(_5); StorageDead(_4); StorageDead(_3); diff --git a/tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff b/tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff index 14a0c639c3abd..726770e069523 100644 --- a/tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff @@ -23,6 +23,8 @@ - } - - bb1: { ++ StorageAlloc(_4); ++ _4 = foo; _3 = &_4; StorageLive(_5); _5 = (); @@ -30,6 +32,9 @@ - } - - bb2: { ++ StorageAlloc(_2); ++ StorageAlloc(_2); ++ _2 = const (); StorageDead(_5); StorageDead(_3); StorageDead(_4); diff --git a/tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff b/tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff index e0a273eb13f50..991bf4d512e27 100644 --- a/tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff @@ -23,6 +23,8 @@ - } - - bb1: { ++ StorageAlloc(_4); ++ _4 = foo; _3 = &_4; StorageLive(_5); _5 = (); @@ -30,6 +32,9 @@ - } - - bb2: { ++ StorageAlloc(_2); ++ StorageAlloc(_2); ++ _2 = const (); StorageDead(_5); StorageDead(_3); StorageDead(_4); diff --git a/tests/mir-opt/inline/rustc_no_mir_inline.caller.runtime-optimized.after.panic-abort.mir b/tests/mir-opt/inline/rustc_no_mir_inline.caller.runtime-optimized.after.panic-abort.mir index f6a6e86bff26a..2b49d78d8c9d4 100644 --- a/tests/mir-opt/inline/rustc_no_mir_inline.caller.runtime-optimized.after.panic-abort.mir +++ b/tests/mir-opt/inline/rustc_no_mir_inline.caller.runtime-optimized.after.panic-abort.mir @@ -9,6 +9,7 @@ fn caller() -> () { } bb1: { + _0 = const (); return; } } diff --git a/tests/mir-opt/inline/rustc_no_mir_inline.caller.runtime-optimized.after.panic-unwind.mir b/tests/mir-opt/inline/rustc_no_mir_inline.caller.runtime-optimized.after.panic-unwind.mir index c8703ce515989..c991c6a3737ea 100644 --- a/tests/mir-opt/inline/rustc_no_mir_inline.caller.runtime-optimized.after.panic-unwind.mir +++ b/tests/mir-opt/inline/rustc_no_mir_inline.caller.runtime-optimized.after.panic-unwind.mir @@ -9,6 +9,7 @@ fn caller() -> () { } bb1: { + _0 = const (); return; } } diff --git a/tests/mir-opt/inline/storage_alloc.field.Inline.diff b/tests/mir-opt/inline/storage_alloc.field.Inline.diff new file mode 100644 index 0000000000000..bca940a727b33 --- /dev/null +++ b/tests/mir-opt/inline/storage_alloc.field.Inline.diff @@ -0,0 +1,41 @@ +- // MIR for `field` before Inline ++ // MIR for `field` after Inline + + fn field() -> MaybeUninit { + let mut _0: std::mem::MaybeUninit; + let mut _1: (std::mem::MaybeUninit, std::mem::MaybeUninit); ++ let mut _2: std::mem::MaybeUninit; ++ scope 1 (inlined callee) { ++ let _3: (); ++ scope 2 (inlined MaybeUninit::::uninit) { ++ } ++ } + + bb0: { +- (_1.0: std::mem::MaybeUninit) = callee() -> [return: bb1, unwind: bb2]; ++ StorageAlloc(_1); ++ StorageLive(_2); ++ StorageLive(_3); ++ _3 = opaque() -> [return: bb2, unwind: bb1]; + } + +- bb1: { ++ bb1 (cleanup): { + _0 = copy (_1.1: std::mem::MaybeUninit); +- return; ++ resume; + } + +- bb2 (cleanup): { ++ bb2: { ++ StorageDead(_3); ++ StorageAlloc(_2); ++ _2 = MaybeUninit:: { uninit: const () }; ++ (_1.0: std::mem::MaybeUninit) = move _2; ++ StorageDead(_2); + _0 = copy (_1.1: std::mem::MaybeUninit); +- resume; ++ return; + } + } + diff --git a/tests/mir-opt/inline/storage_alloc.indexed.Inline.diff b/tests/mir-opt/inline/storage_alloc.indexed.Inline.diff new file mode 100644 index 0000000000000..f2ba704a506c1 --- /dev/null +++ b/tests/mir-opt/inline/storage_alloc.indexed.Inline.diff @@ -0,0 +1,36 @@ +- // MIR for `indexed` before Inline ++ // MIR for `indexed` after Inline + + fn indexed(_1: usize) -> MaybeUninit { + let mut _0: std::mem::MaybeUninit; + let mut _2: [std::mem::MaybeUninit; 2]; ++ let mut _3: *mut std::mem::MaybeUninit; ++ let mut _4: std::mem::MaybeUninit; ++ scope 1 (inlined callee) { ++ let _5: (); ++ scope 2 (inlined MaybeUninit::::uninit) { ++ } ++ } + + bb0: { +- _2[_1] = callee() -> [return: bb1, unwind continue]; ++ StorageAlloc(_2); ++ StorageLive(_3); ++ _3 = &raw mut _2[_1]; ++ StorageLive(_4); ++ StorageLive(_5); ++ _5 = opaque() -> [return: bb1, unwind continue]; + } + + bb1: { ++ StorageDead(_5); ++ StorageAlloc(_4); ++ _4 = MaybeUninit:: { uninit: const () }; ++ (*_3) = move _4; ++ StorageDead(_4); ++ StorageDead(_3); + _0 = copy _2[_1]; + return; + } + } + diff --git a/tests/mir-opt/inline/storage_alloc.local.Inline.diff b/tests/mir-opt/inline/storage_alloc.local.Inline.diff new file mode 100644 index 0000000000000..87ef8691b5ec5 --- /dev/null +++ b/tests/mir-opt/inline/storage_alloc.local.Inline.diff @@ -0,0 +1,37 @@ +- // MIR for `local` before Inline ++ // MIR for `local` after Inline + + fn local() -> MaybeUninit { + let mut _0: std::mem::MaybeUninit; + let mut _1: std::mem::MaybeUninit; ++ scope 1 (inlined callee) { ++ let _2: (); ++ scope 2 (inlined MaybeUninit::::uninit) { ++ } ++ } + + bb0: { +- _1 = callee() -> [return: bb1, unwind: bb2]; ++ StorageAlloc(_1); ++ StorageLive(_2); ++ _2 = opaque() -> [return: bb2, unwind: bb1]; + } + +- bb1: { ++ bb1 (cleanup): { + _0 = copy _1; +- return; ++ resume; + } + +- bb2 (cleanup): { ++ bb2: { ++ StorageDead(_2); ++ StorageAlloc(_1); ++ _1 = MaybeUninit:: { uninit: const () }; + _0 = copy _1; +- resume; ++ return; + } + } + diff --git a/tests/mir-opt/inline/storage_alloc.rs b/tests/mir-opt/inline/storage_alloc.rs new file mode 100644 index 0000000000000..954039e9d8d54 --- /dev/null +++ b/tests/mir-opt/inline/storage_alloc.rs @@ -0,0 +1,91 @@ +//@ test-mir-pass: Inline +//@ needs-unwind + +#![crate_type = "lib"] +#![feature(custom_mir, core_intrinsics)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; +use std::mem::MaybeUninit; + +unsafe extern "C-unwind" { + fn opaque(); +} + +#[inline(always)] +fn callee() -> MaybeUninit { + unsafe { opaque() }; + MaybeUninit::uninit() +} + +// The destination must be allocated before the callee can unwind. +// EMIT_MIR storage_alloc.local.Inline.diff +#[custom_mir(dialect = "runtime")] +pub fn local() -> MaybeUninit { + // CHECK-LABEL: fn local( + // CHECK: bb0: { + // CHECK-NEXT: StorageAlloc([[LOCAL:_[0-9]+]]); + // CHECK: () -> [return: + // CHECK: (cleanup): { + // CHECK-NEXT: _0 = copy [[LOCAL]]; + mir! { + let x: MaybeUninit; + { + Call(x = callee(), ReturnTo(done), UnwindCleanup(cleanup)) + } + done = { + RET = x; + Return() + } + cleanup(cleanup) = { + RET = x; + UnwindResume() + } + } +} + +// Allocating a field of the destination must also allocate its uninitialized sibling. +// EMIT_MIR storage_alloc.field.Inline.diff +#[custom_mir(dialect = "runtime")] +pub fn field() -> MaybeUninit { + // CHECK-LABEL: fn field( + // CHECK: bb0: { + // CHECK-NEXT: StorageAlloc([[TUPLE:_[0-9]+]]); + // CHECK: () -> [return: + // CHECK: (cleanup): { + // CHECK-NEXT: _0 = copy ([[TUPLE]].1: std::mem::MaybeUninit); + mir! { + let x: (MaybeUninit, MaybeUninit); + { + Call(x.0 = callee(), ReturnTo(done), UnwindCleanup(cleanup)) + } + done = { + RET = x.1; + Return() + } + cleanup(cleanup) = { + RET = x.1; + UnwindResume() + } + } +} + +// StorageAlloc is needed before capturing a pointer to the destination when inlining. +// EMIT_MIR storage_alloc.indexed.Inline.diff +#[custom_mir(dialect = "runtime", phase = "initial")] +pub fn indexed(index: usize) -> MaybeUninit { + // CHECK-LABEL: fn indexed( + // CHECK-SAME: [[INDEX:_[0-9]+]]: usize) + // CHECK: StorageAlloc([[VALUES:_[0-9]+]]); + // CHECK: = &raw mut [[VALUES]][[[INDEX]]]; + mir! { + let values: [MaybeUninit; 2]; + { + Call(values[index] = callee(), ReturnTo(done), UnwindContinue()) + } + done = { + RET = values[index]; + Return() + } + } +} diff --git a/tests/mir-opt/inline/type_id_eq.call.Inline.diff b/tests/mir-opt/inline/type_id_eq.call.Inline.diff index 85090e0a2738b..c435b8138de05 100644 --- a/tests/mir-opt/inline/type_id_eq.call.Inline.diff +++ b/tests/mir-opt/inline/type_id_eq.call.Inline.diff @@ -21,6 +21,7 @@ - } - - bb1: { ++ StorageAlloc(_0); + StorageLive(_5); + _5 = copy _3 as u128 (Transmute); + StorageLive(_6); diff --git a/tests/mir-opt/inline/unchecked_shifts.unchecked_shl_unsigned_smaller.Inline.panic-abort.diff b/tests/mir-opt/inline/unchecked_shifts.unchecked_shl_unsigned_smaller.Inline.panic-abort.diff index d66223b91bc44..47beac2033be1 100644 --- a/tests/mir-opt/inline/unchecked_shifts.unchecked_shl_unsigned_smaller.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/unchecked_shifts.unchecked_shl_unsigned_smaller.Inline.panic-abort.diff @@ -21,6 +21,7 @@ StorageLive(_4); _4 = copy _2; - _0 = core::num::::unchecked_shl(move _3, move _4) -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_0); + StorageLive(_5); + switchInt(UbChecks) -> [0: bb2, otherwise: bb1]; } diff --git a/tests/mir-opt/inline/unchecked_shifts.unchecked_shl_unsigned_smaller.Inline.panic-unwind.diff b/tests/mir-opt/inline/unchecked_shifts.unchecked_shl_unsigned_smaller.Inline.panic-unwind.diff index 54907cf809b4f..3e4fcc8be5106 100644 --- a/tests/mir-opt/inline/unchecked_shifts.unchecked_shl_unsigned_smaller.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/unchecked_shifts.unchecked_shl_unsigned_smaller.Inline.panic-unwind.diff @@ -21,6 +21,7 @@ StorageLive(_4); _4 = copy _2; - _0 = core::num::::unchecked_shl(move _3, move _4) -> [return: bb1, unwind continue]; ++ StorageAlloc(_0); + StorageLive(_5); + switchInt(UbChecks) -> [0: bb2, otherwise: bb1]; } diff --git a/tests/mir-opt/inline/unchecked_shifts.unchecked_shr_signed_bigger.Inline.panic-abort.diff b/tests/mir-opt/inline/unchecked_shifts.unchecked_shr_signed_bigger.Inline.panic-abort.diff index c118f5b0df3ff..57dbfd1f1ae04 100644 --- a/tests/mir-opt/inline/unchecked_shifts.unchecked_shr_signed_bigger.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/unchecked_shifts.unchecked_shr_signed_bigger.Inline.panic-abort.diff @@ -21,6 +21,7 @@ StorageLive(_4); _4 = copy _2; - _0 = core::num::::unchecked_shr(move _3, move _4) -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_0); + StorageLive(_5); + switchInt(UbChecks) -> [0: bb2, otherwise: bb1]; } diff --git a/tests/mir-opt/inline/unchecked_shifts.unchecked_shr_signed_bigger.Inline.panic-unwind.diff b/tests/mir-opt/inline/unchecked_shifts.unchecked_shr_signed_bigger.Inline.panic-unwind.diff index 6a1e6a0a2d26f..7830e0c98a72b 100644 --- a/tests/mir-opt/inline/unchecked_shifts.unchecked_shr_signed_bigger.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/unchecked_shifts.unchecked_shr_signed_bigger.Inline.panic-unwind.diff @@ -21,6 +21,7 @@ StorageLive(_4); _4 = copy _2; - _0 = core::num::::unchecked_shr(move _3, move _4) -> [return: bb1, unwind continue]; ++ StorageAlloc(_0); + StorageLive(_5); + switchInt(UbChecks) -> [0: bb2, otherwise: bb1]; } diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff index 041dd27cb5647..916593bbd8585 100644 --- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff @@ -22,6 +22,7 @@ StorageLive(_2); _2 = move _1; - _0 = Option::::unwrap_unchecked(move _2) -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_0); + StorageLive(_3); + StorageLive(_4); + _3 = discriminant(_2); diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff index be89063a8ac96..57ad98aa118c0 100644 --- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff @@ -22,6 +22,7 @@ StorageLive(_2); _2 = move _1; - _0 = Option::::unwrap_unchecked(move _2) -> [return: bb1, unwind: bb2]; ++ StorageAlloc(_0); + StorageLive(_3); + StorageLive(_4); + _3 = discriminant(_2); diff --git a/tests/mir-opt/inline_double_cycle.a.Inline.panic-abort.diff b/tests/mir-opt/inline_double_cycle.a.Inline.panic-abort.diff index 90a4a509ac124..d2931f15febf8 100644 --- a/tests/mir-opt/inline_double_cycle.a.Inline.panic-abort.diff +++ b/tests/mir-opt/inline_double_cycle.a.Inline.panic-abort.diff @@ -19,6 +19,7 @@ StorageLive(_2); _2 = (); - _1 = >::call_once(a, move _2) -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_1); + StorageLive(_5); + _5 = a; + _1 = move _5() -> [return: bb1, unwind unreachable]; @@ -32,6 +33,7 @@ StorageLive(_4); _4 = (); - _3 = >::call_once(b, move _4) -> [return: bb2, unwind unreachable]; ++ StorageAlloc(_3); + StorageLive(_6); + _6 = b; + _3 = move _6() -> [return: bb2, unwind unreachable]; diff --git a/tests/mir-opt/inline_double_cycle.a.Inline.panic-unwind.diff b/tests/mir-opt/inline_double_cycle.a.Inline.panic-unwind.diff index 55da685a3d430..42ed7d7cdc635 100644 --- a/tests/mir-opt/inline_double_cycle.a.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline_double_cycle.a.Inline.panic-unwind.diff @@ -19,6 +19,7 @@ StorageLive(_2); _2 = (); - _1 = >::call_once(a, move _2) -> [return: bb1, unwind continue]; ++ StorageAlloc(_1); + StorageLive(_5); + _5 = a; + _1 = move _5() -> [return: bb1, unwind continue]; @@ -32,6 +33,7 @@ StorageLive(_4); _4 = (); - _3 = >::call_once(b, move _4) -> [return: bb2, unwind continue]; ++ StorageAlloc(_3); + StorageLive(_6); + _6 = b; + _3 = move _6() -> [return: bb2, unwind continue]; diff --git a/tests/mir-opt/inline_double_cycle.b.Inline.panic-abort.diff b/tests/mir-opt/inline_double_cycle.b.Inline.panic-abort.diff index 2090411276cf9..4a838c2ce957f 100644 --- a/tests/mir-opt/inline_double_cycle.b.Inline.panic-abort.diff +++ b/tests/mir-opt/inline_double_cycle.b.Inline.panic-abort.diff @@ -19,6 +19,7 @@ StorageLive(_2); _2 = (); - _1 = >::call_once(b, move _2) -> [return: bb1, unwind unreachable]; ++ StorageAlloc(_1); + StorageLive(_5); + _5 = b; + _1 = move _5() -> [return: bb1, unwind unreachable]; @@ -32,6 +33,7 @@ StorageLive(_4); _4 = (); - _3 = >::call_once(a, move _4) -> [return: bb2, unwind unreachable]; ++ StorageAlloc(_3); + StorageLive(_6); + _6 = a; + _3 = move _6() -> [return: bb2, unwind unreachable]; diff --git a/tests/mir-opt/inline_double_cycle.b.Inline.panic-unwind.diff b/tests/mir-opt/inline_double_cycle.b.Inline.panic-unwind.diff index 9e6eef1fa3049..09be997dcabc3 100644 --- a/tests/mir-opt/inline_double_cycle.b.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline_double_cycle.b.Inline.panic-unwind.diff @@ -19,6 +19,7 @@ StorageLive(_2); _2 = (); - _1 = >::call_once(b, move _2) -> [return: bb1, unwind continue]; ++ StorageAlloc(_1); + StorageLive(_5); + _5 = b; + _1 = move _5() -> [return: bb1, unwind continue]; @@ -32,6 +33,7 @@ StorageLive(_4); _4 = (); - _3 = >::call_once(a, move _4) -> [return: bb2, unwind continue]; ++ StorageAlloc(_3); + StorageLive(_6); + _6 = a; + _3 = move _6() -> [return: bb2, unwind continue]; diff --git a/tests/mir-opt/inline_fn_call_for_fn_def.test.Inline.diff b/tests/mir-opt/inline_fn_call_for_fn_def.test.Inline.diff index aef7aa2397095..1e0faa2049dc3 100644 --- a/tests/mir-opt/inline_fn_call_for_fn_def.test.Inline.diff +++ b/tests/mir-opt/inline_fn_call_for_fn_def.test.Inline.diff @@ -18,12 +18,15 @@ - bb1: { + bb0: { ++ StorageAlloc(_0); + StorageLive(_1); + _1 = yield_number; + StorageLive(_2); + _2 = move _1; + StorageLive(_3); + _3 = (); ++ StorageAlloc(_0); ++ StorageAlloc(_0); + _0 = const 64_i32; + StorageDead(_3); + StorageDead(_2); diff --git a/tests/mir-opt/inline_generically_if_sized.call.Inline.diff b/tests/mir-opt/inline_generically_if_sized.call.Inline.diff index fc5833f54ec66..e572537ee1db2 100644 --- a/tests/mir-opt/inline_generically_if_sized.call.Inline.diff +++ b/tests/mir-opt/inline_generically_if_sized.call.Inline.diff @@ -15,6 +15,7 @@ - } - - bb1: { ++ StorageAlloc(_0); + _0 = const 0_i32; StorageDead(_2); return; diff --git a/tests/mir-opt/instsimplify/casts.redundant.InstSimplify-after-simplifycfg.diff b/tests/mir-opt/instsimplify/casts.redundant.InstSimplify-after-simplifycfg.diff index 75038e899b3ee..0363ac8e115ca 100644 --- a/tests/mir-opt/instsimplify/casts.redundant.InstSimplify-after-simplifycfg.diff +++ b/tests/mir-opt/instsimplify/casts.redundant.InstSimplify-after-simplifycfg.diff @@ -17,6 +17,7 @@ StorageLive(_3); StorageLive(_4); _4 = copy _1; + StorageAlloc(_3); StorageLive(_5); StorageLive(_6); _6 = copy _4; diff --git a/tests/mir-opt/instsimplify/intrinsic_asserts.removable.InstSimplify-after-simplifycfg.diff b/tests/mir-opt/instsimplify/intrinsic_asserts.removable.InstSimplify-after-simplifycfg.diff index 20e046d8e194b..ef9385b1c8457 100644 --- a/tests/mir-opt/instsimplify/intrinsic_asserts.removable.InstSimplify-after-simplifycfg.diff +++ b/tests/mir-opt/instsimplify/intrinsic_asserts.removable.InstSimplify-after-simplifycfg.diff @@ -10,6 +10,7 @@ bb0: { StorageLive(_1); - _1 = assert_inhabited::<()>() -> [return: bb1, unwind unreachable]; ++ _1 = const (); + goto -> bb1; } @@ -17,6 +18,7 @@ StorageDead(_1); StorageLive(_2); - _2 = assert_zero_valid::() -> [return: bb2, unwind unreachable]; ++ _2 = const (); + goto -> bb2; } @@ -24,6 +26,7 @@ StorageDead(_2); StorageLive(_3); - _3 = assert_mem_uninitialized_valid::() -> [return: bb3, unwind unreachable]; ++ _3 = const (); + goto -> bb3; } diff --git a/tests/mir-opt/instsimplify/ub_check.unwrap_unchecked.InstSimplify-after-simplifycfg.diff b/tests/mir-opt/instsimplify/ub_check.unwrap_unchecked.InstSimplify-after-simplifycfg.diff index c4fa526a7af99..f3ff47029f918 100644 --- a/tests/mir-opt/instsimplify/ub_check.unwrap_unchecked.InstSimplify-after-simplifycfg.diff +++ b/tests/mir-opt/instsimplify/ub_check.unwrap_unchecked.InstSimplify-after-simplifycfg.diff @@ -21,6 +21,7 @@ bb0: { StorageLive(_2); _2 = copy _1; + StorageAlloc(_0); StorageLive(_3); StorageLive(_4); _3 = discriminant(_2); diff --git a/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-abort.diff b/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-abort.diff index 95f9b5997d4f6..cf95976e764b5 100644 --- a/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-abort.diff +++ b/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-abort.diff @@ -38,6 +38,7 @@ // DBG: _5 = &(*_2)[0 of 3]; // DBG: _6 = &(*_2)[1 of 3]; // DBG: _7 = &(*_2)[2 of 3]; + _0 = const (); StorageDead(_4); return; } diff --git a/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-unwind.diff b/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-unwind.diff index 87449b043fd5e..886eab2a4ffde 100644 --- a/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-unwind.diff +++ b/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-unwind.diff @@ -38,6 +38,7 @@ // DBG: _5 = &(*_2)[0 of 3]; // DBG: _6 = &(*_2)[1 of 3]; // DBG: _7 = &(*_2)[2 of 3]; + _0 = const (); StorageDead(_4); return; } diff --git a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff index fb406b8dfefc9..ce11f2c078701 100644 --- a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff @@ -221,10 +221,12 @@ StorageLive(_10); _21 = const chained_conditions::promoted[1]; _10 = &(*_21); + StorageAlloc(_7); StorageLive(_27); StorageLive(_28); _27 = no_retag copy (*_8); _28 = no_retag copy (*_10); + StorageAlloc(_7); StorageLive(_29); StorageLive(_30); StorageLive(_31); @@ -244,7 +246,9 @@ StorageDead(_30); StorageLive(_36); StorageLive(_38); + StorageAlloc(_36); _36 = copy _29 as &[u8] (Transmute); + StorageAlloc(_38); _38 = copy _28 as &[u8] (Transmute); StorageLive(_39); _39 = PtrMetadata(copy _36); @@ -283,10 +287,12 @@ StorageLive(_17); _20 = const chained_conditions::promoted[0]; _17 = &(*_20); + StorageAlloc(_14); StorageLive(_47); StorageLive(_48); _47 = no_retag copy (*_15); _48 = no_retag copy (*_17); + StorageAlloc(_14); StorageLive(_49); StorageLive(_50); StorageLive(_51); @@ -306,7 +312,9 @@ StorageDead(_50); StorageLive(_56); StorageLive(_58); + StorageAlloc(_56); _56 = copy _49 as &[u8] (Transmute); + StorageAlloc(_58); _58 = copy _48 as &[u8] (Transmute); StorageLive(_59); _59 = PtrMetadata(copy _56); diff --git a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff index d291e3d1eb51e..bc4ce21ff20e3 100644 --- a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff @@ -221,10 +221,12 @@ StorageLive(_10); _21 = const chained_conditions::promoted[1]; _10 = &(*_21); + StorageAlloc(_7); StorageLive(_27); StorageLive(_28); _27 = no_retag copy (*_8); _28 = no_retag copy (*_10); + StorageAlloc(_7); StorageLive(_29); StorageLive(_30); StorageLive(_31); @@ -244,7 +246,9 @@ StorageDead(_30); StorageLive(_36); StorageLive(_38); + StorageAlloc(_36); _36 = copy _29 as &[u8] (Transmute); + StorageAlloc(_38); _38 = copy _28 as &[u8] (Transmute); StorageLive(_39); _39 = PtrMetadata(copy _36); @@ -283,10 +287,12 @@ StorageLive(_17); _20 = const chained_conditions::promoted[0]; _17 = &(*_20); + StorageAlloc(_14); StorageLive(_47); StorageLive(_48); _47 = no_retag copy (*_15); _48 = no_retag copy (*_17); + StorageAlloc(_14); StorageLive(_49); StorageLive(_50); StorageLive(_51); @@ -306,7 +312,9 @@ StorageDead(_50); StorageLive(_56); StorageLive(_58); + StorageAlloc(_56); _56 = copy _49 as &[u8] (Transmute); + StorageAlloc(_58); _58 = copy _48 as &[u8] (Transmute); StorageLive(_59); _59 = PtrMetadata(copy _56); diff --git a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff index a822ca20a8e80..a5fdba980d3ff 100644 --- a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff @@ -48,6 +48,7 @@ StorageLive(_3); StorageLive(_4); _4 = copy _1; + StorageAlloc(_3); StorageLive(_10); _10 = discriminant(_4); switchInt(move _10) -> [0: bb7, 1: bb6, otherwise: bb1]; @@ -73,6 +74,7 @@ _6 = copy ((_3 as Break).0: std::result::Result); StorageLive(_8); _8 = copy _6; + StorageAlloc(_0); StorageLive(_14); StorageLive(_17); _14 = discriminant(_8); @@ -81,6 +83,7 @@ StorageLive(_15); _15 = move ((_8 as Err).0: i32); StorageLive(_16); + StorageAlloc(_16); _16 = move _15; _0 = Result::::Err(move _16); StorageDead(_16); diff --git a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff index a822ca20a8e80..a5fdba980d3ff 100644 --- a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff @@ -48,6 +48,7 @@ StorageLive(_3); StorageLive(_4); _4 = copy _1; + StorageAlloc(_3); StorageLive(_10); _10 = discriminant(_4); switchInt(move _10) -> [0: bb7, 1: bb6, otherwise: bb1]; @@ -73,6 +74,7 @@ _6 = copy ((_3 as Break).0: std::result::Result); StorageLive(_8); _8 = copy _6; + StorageAlloc(_0); StorageLive(_14); StorageLive(_17); _14 = discriminant(_8); @@ -81,6 +83,7 @@ StorageLive(_15); _15 = move ((_8 as Err).0: i32); StorageLive(_16); + StorageAlloc(_16); _16 = move _15; _0 = Result::::Err(move _16); StorageDead(_16); diff --git a/tests/mir-opt/lower_intrinsics.assume.LowerIntrinsics.panic-abort.diff b/tests/mir-opt/lower_intrinsics.assume.LowerIntrinsics.panic-abort.diff index 6c1f457cb5f50..2c83ba9629c01 100644 --- a/tests/mir-opt/lower_intrinsics.assume.LowerIntrinsics.panic-abort.diff +++ b/tests/mir-opt/lower_intrinsics.assume.LowerIntrinsics.panic-abort.diff @@ -9,6 +9,7 @@ StorageLive(_1); - _1 = std::intrinsics::assume(const true) -> [return: bb1, unwind unreachable]; + assume(const true); ++ _1 = const (); + goto -> bb1; } diff --git a/tests/mir-opt/lower_intrinsics.assume.LowerIntrinsics.panic-unwind.diff b/tests/mir-opt/lower_intrinsics.assume.LowerIntrinsics.panic-unwind.diff index 6c1f457cb5f50..2c83ba9629c01 100644 --- a/tests/mir-opt/lower_intrinsics.assume.LowerIntrinsics.panic-unwind.diff +++ b/tests/mir-opt/lower_intrinsics.assume.LowerIntrinsics.panic-unwind.diff @@ -9,6 +9,7 @@ StorageLive(_1); - _1 = std::intrinsics::assume(const true) -> [return: bb1, unwind unreachable]; + assume(const true); ++ _1 = const (); + goto -> bb1; } diff --git a/tests/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.panic-abort.diff b/tests/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.panic-abort.diff index d73cc3d214e4c..5e36aaeed3e5f 100644 --- a/tests/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.panic-abort.diff +++ b/tests/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.panic-abort.diff @@ -47,6 +47,7 @@ StorageDead(_9); - _3 = std::intrinsics::copy_nonoverlapping::(move _4, move _8, const 0_usize) -> [return: bb1, unwind unreachable]; + copy_nonoverlapping(dst = move _8, src = move _4, count = const 0_usize); ++ _3 = const (); + goto -> bb1; } diff --git a/tests/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.panic-unwind.diff b/tests/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.panic-unwind.diff index d73cc3d214e4c..5e36aaeed3e5f 100644 --- a/tests/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.panic-unwind.diff +++ b/tests/mir-opt/lower_intrinsics.f_copy_nonoverlapping.LowerIntrinsics.panic-unwind.diff @@ -47,6 +47,7 @@ StorageDead(_9); - _3 = std::intrinsics::copy_nonoverlapping::(move _4, move _8, const 0_usize) -> [return: bb1, unwind unreachable]; + copy_nonoverlapping(dst = move _8, src = move _4, count = const 0_usize); ++ _3 = const (); + goto -> bb1; } diff --git a/tests/mir-opt/lower_intrinsics.rs b/tests/mir-opt/lower_intrinsics.rs index e8a7701477415..b50b95fd1e527 100644 --- a/tests/mir-opt/lower_intrinsics.rs +++ b/tests/mir-opt/lower_intrinsics.rs @@ -147,6 +147,7 @@ pub fn discriminant(t: T) { pub fn f_copy_nonoverlapping() { // CHECK-LABEL: fn f_copy_nonoverlapping( // CHECK: copy_nonoverlapping({{.*}}); + // CHECK-NEXT: {{_[0-9]+}} = const (); let src = (); let mut dst = (); @@ -159,6 +160,7 @@ pub fn f_copy_nonoverlapping() { pub fn assume() { // CHECK-LABEL: fn assume( // CHECK: assume({{.*}}); + // CHECK-NEXT: {{_[0-9]+}} = const (); unsafe { std::intrinsics::assume(true); diff --git a/tests/mir-opt/match_branch_simplification_zst.rs b/tests/mir-opt/match_branch_simplification_zst.rs new file mode 100644 index 0000000000000..36dae88ce7fef --- /dev/null +++ b/tests/mir-opt/match_branch_simplification_zst.rs @@ -0,0 +1,14 @@ +//@ test-mir-pass: MatchBranchSimplification + +#![crate_type = "lib"] + +// Both arms assign the same unit constant with different source spans. +// EMIT_MIR match_branch_simplification_zst.same_unit.MatchBranchSimplification.diff +pub fn same_unit(condition: bool) { + // CHECK-LABEL: fn same_unit( + // CHECK-NOT: switchInt + // CHECK: _0 = no_retag const (); + // CHECK-NOT: switchInt + // CHECK: return; + if condition {} +} diff --git a/tests/mir-opt/match_branch_simplification_zst.same_unit.MatchBranchSimplification.diff b/tests/mir-opt/match_branch_simplification_zst.same_unit.MatchBranchSimplification.diff new file mode 100644 index 0000000000000..ff35a2a25887f --- /dev/null +++ b/tests/mir-opt/match_branch_simplification_zst.same_unit.MatchBranchSimplification.diff @@ -0,0 +1,31 @@ +- // MIR for `same_unit` before MatchBranchSimplification ++ // MIR for `same_unit` after MatchBranchSimplification + + fn same_unit(_1: bool) -> () { + debug condition => _1; + let mut _0: (); + let mut _2: bool; + + bb0: { + StorageLive(_2); + _2 = copy _1; +- switchInt(move _2) -> [0: bb2, otherwise: bb1]; +- } +- +- bb1: { +- _0 = const (); +- goto -> bb3; +- } +- +- bb2: { +- _0 = const (); +- goto -> bb3; +- } +- +- bb3: { ++ _0 = no_retag const (); + StorageDead(_2); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff new file mode 100644 index 0000000000000..64d0fe11bfa9f --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff @@ -0,0 +1,29 @@ +- // MIR for `aggregate_indirect_source_alias` before MoveElimination ++ // MIR for `aggregate_indirect_source_alias` after MoveElimination + + fn aggregate_indirect_source_alias() -> (u8, u8) { + let mut _0: (u8, u8); + let mut _1: u8; + let mut _2: *const u8; + let mut _3: (u8, u8); ++ let mut _4: u8; + + bb0: { +- _1 = const 1_u8; +- _2 = &raw const _1; +- _3 = (copy (*_2), move _1); +- _0 = copy _3; ++ (_0.1: u8) = const 1_u8; ++ StorageLive(_2); ++ _2 = &raw const (_0.1: u8); ++ StorageLive(_4); ++ _4 = no_retag copy (*_2); ++ (_0.0: u8) = no_retag move _4; ++ nop; ++ StorageDead(_4); ++ StorageDead(_2); ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff new file mode 100644 index 0000000000000..481d3ef60feb8 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff @@ -0,0 +1,84 @@ +- // MIR for `aggregate_swap` before MoveElimination ++ // MIR for `aggregate_swap` after MoveElimination + + fn aggregate_swap(_1: u8, _2: u8) -> (u8, u8) { + debug x => _1; + debug y => _2; + let mut _0: (u8, u8); + let mut _3: (u8, u8); + let mut _4: u8; + let mut _5: u8; + let mut _8: u8; + let mut _9: u8; ++ let mut _10: u8; + scope 1 { +- debug pair => _3; ++ debug pair => _0; + let _6: u8; + scope 2 { +- debug a => _6; ++ debug a => _9; + let _7: u8; + scope 3 { +- debug b => _7; ++ debug b => (_0.1: u8); + } + } + } + + bb0: { +- StorageLive(_3); +- StorageLive(_4); +- _4 = copy _1; +- StorageLive(_5); +- _5 = copy _2; +- _3 = (move _4, move _5); +- StorageDead(_5); +- StorageDead(_4); +- StorageLive(_6); +- _6 = copy (_3.0: u8); +- StorageLive(_7); +- _7 = copy (_3.1: u8); +- StorageLive(_8); +- _8 = copy _7; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = (move _1, move _2); ++ nop; ++ nop; ++ nop; + StorageLive(_9); +- _9 = copy _6; +- _3 = (move _8, move _9); ++ _9 = copy (_0.0: u8); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_10); ++ _10 = no_retag move (_0.1: u8); ++ (_0.0: u8) = no_retag move _10; ++ (_0.1: u8) = no_retag move _9; ++ nop; ++ StorageDead(_10); + StorageDead(_9); +- StorageDead(_8); +- _0 = copy _3; +- StorageDead(_7); +- StorageDead(_6); +- StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff new file mode 100644 index 0000000000000..27f6c10549040 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff @@ -0,0 +1,90 @@ +- // MIR for `mixed_aggregate_aliasing` before MoveElimination ++ // MIR for `mixed_aggregate_aliasing` after MoveElimination + + fn mixed_aggregate_aliasing(_1: bool, _2: u8) -> Triple { + debug flag => _1; + debug z => _2; + let mut _0: Triple; + let _3: Triple; + let mut _5: bool; + let mut _6: u8; + let mut _7: u8; + let mut _8: u8; ++ let mut _9: u8; + scope 1 { +- debug input => _3; ++ debug input => _0; + let _4: Triple; + scope 2 { +- debug out => _4; ++ debug out => _0; + } + } + + bb0: { +- StorageLive(_3); +- _3 = opaque_triple() -> [return: bb1, unwind unreachable]; ++ nop; ++ _0 = opaque_triple() -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageLive(_4); +- StorageLive(_5); +- _5 = copy _1; +- switchInt(move _5) -> [0: bb3, otherwise: bb2]; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb3, otherwise: bb2]; + } + + bb2: { +- _4 = copy _3; ++ nop; + goto -> bb4; + } + + bb3: { ++ nop; + StorageLive(_6); +- _6 = copy (_3.1: u8); +- StorageLive(_7); +- _7 = copy (_3.0: u8); +- StorageLive(_8); +- _8 = copy _2; +- _4 = Triple(move _6, move _7, move _8); +- StorageDead(_8); +- StorageDead(_7); ++ _6 = copy (_0.1: u8); ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_9); ++ _9 = no_retag move (_0.0: u8); ++ (_0.0: u8) = no_retag move _6; ++ (_0.1: u8) = no_retag move _9; ++ (_0.2: u8) = no_retag move _2; ++ nop; ++ StorageDead(_9); + StorageDead(_6); ++ nop; ++ nop; ++ nop; + goto -> bb4; + } + + bb4: { +- StorageDead(_5); +- _0 = copy _4; +- StorageDead(_4); +- StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.rs b/tests/mir-opt/move-elimination/alias_fixup.rs new file mode 100644 index 0000000000000..29d20a61488fa --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.rs @@ -0,0 +1,95 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort + +#![feature(core_intrinsics, custom_mir)] +#![allow(dead_code)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +#[derive(Copy, Clone)] +pub struct Triple(u8, u8, u8); + +pub union U { + a: [u8; 4], + b: [u8; 4], +} + +unsafe extern "C" { + safe fn opaque_triple() -> Triple; +} + +// EMIT_MIR alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff +pub fn mixed_aggregate_aliasing(flag: bool, z: u8) -> Triple { + // This checks an aggregate assignment on one branch after the other branch + // remaps the input into the return place: overlapping field reads are + // hoisted through temporaries before writing back into the return place. + // CHECK-LABEL: fn mixed_aggregate_aliasing( + // CHECK: debug z => _2; + // CHECK: debug input => _0; + // CHECK: debug out => _0; + // CHECK: [[field1:_.*]] = copy (_0.1: u8); + // CHECK: [[field0:_.*]] = no_retag move (_0.0: u8); + // CHECK: (_0.0: u8) = no_retag move [[field1]]; + // CHECK: (_0.1: u8) = no_retag move [[field0]]; + // CHECK: (_0.2: u8) = no_retag move _2; + let input = opaque_triple(); + let out = if flag { input } else { Triple(input.1, input.0, z) }; + out +} + +// EMIT_MIR alias_fixup.aggregate_swap.MoveElimination.diff +pub fn aggregate_swap(x: u8, y: u8) -> (u8, u8) { + // This checks that an aggregate swap-like assignment is safe after any + // remapping that makes source fields share storage with destination fields. + // CHECK-LABEL: fn aggregate_swap( + // CHECK: debug pair => _0; + // CHECK: [[saved:_.*]] = copy (_0.0: u8); + // CHECK: [[tmp:_.*]] = no_retag move (_0.1: u8); + // CHECK: (_0.0: u8) = no_retag move [[tmp]]; + // CHECK: (_0.1: u8) = no_retag move [[saved]]; + let mut pair = (x, y); + let a = pair.0; + let b = pair.1; + pair = (b, a); + pair +} + +// EMIT_MIR alias_fixup.simple_partial_alias.MoveElimination.diff +pub fn simple_partial_alias(x: [u8; 4]) -> U { + // This checks a non-aggregate assignment involving two same-typed union + // fields, which are conservatively treated as aliasing. + // CHECK-LABEL: fn simple_partial_alias( + // CHECK: debug u => _0; + // CHECK: _0 = U { a: move _1 }; + // CHECK: [[tmp:_.*]] = copy (_0.0: [u8; 4]); + // CHECK: (_0.1: [u8; 4]) = move [[tmp]]; + let mut u = U { a: x }; + let tmp = unsafe { u.a }; + u.b = tmp; + u +} + +// EMIT_MIR alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn aggregate_indirect_source_alias() -> (u8, u8) { + // This checks that, when an aggregate has a direct operand aliasing the + // destination, indirect operands are also hoisted before field writes. + // CHECK-LABEL: fn aggregate_indirect_source_alias( + // CHECK: [[p:_.*]] = &raw const (_0.1: u8); + // CHECK: [[tmp:_.*]] = no_retag copy (*[[p]]); + // CHECK: (_0.0: u8) = no_retag move [[tmp]]; + mir! { + let a: u8; + let p: *const u8; + let out: (u8, u8); + + { + a = 1_u8; + p = &raw const a; + out = (*p, Move(a)); + RET = out; + Return() + } + } +} diff --git a/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff new file mode 100644 index 0000000000000..af47eb7347d2a --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff @@ -0,0 +1,52 @@ +- // MIR for `simple_partial_alias` before MoveElimination ++ // MIR for `simple_partial_alias` after MoveElimination + + fn simple_partial_alias(_1: [u8; 4]) -> U { + debug x => _1; + let mut _0: U; + let mut _2: U; + let mut _3: [u8; 4]; + let mut _5: [u8; 4]; + scope 1 { +- debug u => _2; ++ debug u => _0; + let _4: [u8; 4]; + scope 2 { +- debug tmp => _4; ++ debug tmp => _5; + } + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- _2 = U { a: move _3 }; +- StorageDead(_3); +- StorageLive(_4); +- _4 = copy (_2.0: [u8; 4]); ++ nop; ++ nop; ++ nop; ++ _0 = U { a: move _1 }; ++ nop; ++ nop; + StorageLive(_5); +- _5 = copy _4; +- (_2.1: [u8; 4]) = move _5; ++ _5 = copy (_0.0: [u8; 4]); ++ nop; ++ nop; ++ (_0.1: [u8; 4]) = move _5; + StorageDead(_5); +- _0 = move _2; +- StorageDead(_4); +- StorageDead(_2); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..01938d10da6c5 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff @@ -0,0 +1,67 @@ +- // MIR for `array_aggregate` before MoveElimination ++ // MIR for `array_aggregate` after MoveElimination + + fn array_aggregate() -> [[u8; 8]; 3] { + let mut _0: [[u8; 8]; 3]; + let _1: [u8; 8]; + let mut _4: [u8; 8]; + let mut _5: [u8; 8]; + let mut _6: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => _0[0 of 1]; + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => _0[1 of 2]; + let _3: [u8; 8]; + scope 3 { +- debug c => _3; ++ debug c => _0[2 of 3]; + } + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- _3 = [const 3_u8; 8]; +- StorageLive(_4); +- _4 = move _1; +- StorageLive(_5); +- _5 = move _2; +- StorageLive(_6); +- _6 = move _3; +- _0 = [move _4, move _5, move _6]; +- StorageDead(_6); +- StorageDead(_5); +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ _0[0 of 1] = [const 1_u8; 8]; ++ nop; ++ _0[1 of 2] = [const 2_u8; 8]; ++ nop; ++ _0[2 of 3] = [const 3_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..5f40d8a3c82fd --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff @@ -0,0 +1,57 @@ +- // MIR for `enum_aggregate` before MoveElimination ++ // MIR for `enum_aggregate` after MoveElimination + + fn enum_aggregate() -> Result<([u8; 8], [u8; 8]), ()> { + let mut _0: std::result::Result<([u8; 8], [u8; 8]), ()>; + let _1: [u8; 8]; + let mut _3: ([u8; 8], [u8; 8]); + let mut _4: [u8; 8]; + let mut _5: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]); + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]); + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- StorageLive(_4); +- _4 = move _1; +- StorageLive(_5); +- _5 = move _2; +- _3 = (move _4, move _5); +- StorageDead(_5); +- StorageDead(_4); +- _0 = Result::<([u8; 8], [u8; 8]), ()>::Ok(move _3); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]) = [const 1_u8; 8]; ++ nop; ++ (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]) = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ discriminant(_0) = 0; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff new file mode 100644 index 0000000000000..8a07af2edceaf --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff @@ -0,0 +1,50 @@ +- // MIR for `nrvo_borrowed` before MoveElimination ++ // MIR for `nrvo_borrowed` after MoveElimination + + fn nrvo_borrowed() -> [u8; 8] { + let mut _0: [u8; 8]; + let mut _1: [u8; 8]; + let _2: (); + let mut _3: &mut [u8; 8]; + let mut _4: &mut [u8; 8]; + scope 1 { +- debug buf => _1; ++ debug buf => _0; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- StorageLive(_3); ++ nop; ++ _0 = [const 1_u8; 8]; ++ nop; ++ nop; ++ nop; + StorageLive(_4); +- _4 = &mut _1; ++ _4 = &mut _0; ++ StorageLive(_3); + _3 = &mut (*_4); ++ StorageDead(_4); ++ StorageLive(_2); + _2 = init(move _3) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_3); +- StorageDead(_4); + StorageDead(_2); +- _0 = move _1; +- StorageDead(_1); ++ StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff new file mode 100644 index 0000000000000..7a44d4b598211 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff @@ -0,0 +1,24 @@ +- // MIR for `nrvo_unborrowed` before MoveElimination ++ // MIR for `nrvo_unborrowed` after MoveElimination + + fn nrvo_unborrowed() -> [u8; 8] { + let mut _0: [u8; 8]; + let _1: [u8; 8]; + scope 1 { +- debug buf => _1; ++ debug buf => _0; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- _0 = move _1; +- StorageDead(_1); ++ nop; ++ _0 = [const 1_u8; 8]; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.rs b/tests/mir-opt/move-elimination/basic.rs new file mode 100644 index 0000000000000..96f1776bebef1 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.rs @@ -0,0 +1,80 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort -Zmir-enable-passes=+TailCopyToMove + +struct Pair { + a: [u8; 8], + b: [u8; 8], +} + +fn init(_: &mut [u8; 8]) {} + +// EMIT_MIR basic.nrvo_unborrowed.MoveElimination.diff +pub fn nrvo_unborrowed() -> [u8; 8] { + // This checks the simplest NRVO-style case: the local should be merged with + // the return place even though it has `Copy` type. + // CHECK-LABEL: fn nrvo_unborrowed( + // CHECK: debug buf => _0; + // CHECK: _0 = [const 1_u8; 8] + let buf = [1; 8]; + buf +} + +// EMIT_MIR basic.nrvo_borrowed.MoveElimination.diff +pub fn nrvo_borrowed() -> [u8; 8] { + // This checks that taking a temporary mutable borrow does not prevent + // merging a `Copy` local once the borrow has ended. + // CHECK-LABEL: fn nrvo_borrowed( + // CHECK: debug buf => _0; + // CHECK: _0 = [const 1_u8; 8] + // CHECK: init(move {{_.*}}) + // CHECK-NOT: _0 = move + let mut buf = [1; 8]; + init(&mut buf); + buf +} + +// EMIT_MIR basic.struct_aggregate.MoveElimination.diff +pub fn struct_aggregate() -> Pair { + // This checks aggregate field remapping: the field locals can live directly + // in the return place's fields. + // CHECK-LABEL: fn struct_aggregate( + // CHECK: debug a => (_0.0: [u8; 8]); + // CHECK: debug b => (_0.1: [u8; 8]); + // CHECK: (_0.0: [u8; 8]) = [const 1_u8; 8]; + // CHECK: (_0.1: [u8; 8]) = [const 2_u8; 8]; + let a = [1; 8]; + let b = [2; 8]; + Pair { a, b } +} + +// EMIT_MIR basic.enum_aggregate.MoveElimination.diff +pub fn enum_aggregate() -> Result<([u8; 8], [u8; 8]), ()> { + // This checks aggregate field remapping for enums: the payload fields can + // be written directly and then the discriminant is set for the variant. + // CHECK-LABEL: fn enum_aggregate( + // CHECK: debug a => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]); + // CHECK: debug b => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]); + // CHECK: (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]) = [const 1_u8; 8]; + // CHECK: (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]) = [const 2_u8; 8]; + // CHECK: discriminant(_0) = 0; + let a = [1; 8]; + let b = [2; 8]; + Result::Ok((a, b)) +} + +// EMIT_MIR basic.array_aggregate.MoveElimination.diff +pub fn array_aggregate() -> [[u8; 8]; 3] { + // This checks aggregate remapping for arrays, which uses ConstantIndex + // projections rather than field projections. + // CHECK-LABEL: fn array_aggregate( + // CHECK: debug a => _0[0 of 1]; + // CHECK: debug b => _0[1 of 2]; + // CHECK: debug c => _0[2 of 3]; + // CHECK: _0[0 of 1] = [const 1_u8; 8]; + // CHECK: _0[1 of 2] = [const 2_u8; 8]; + // CHECK: _0[2 of 3] = [const 3_u8; 8]; + let a = [1; 8]; + let b = [2; 8]; + let c = [3; 8]; + [a, b, c] +} diff --git a/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..b03c4d2f3cff9 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff @@ -0,0 +1,49 @@ +- // MIR for `struct_aggregate` before MoveElimination ++ // MIR for `struct_aggregate` after MoveElimination + + fn struct_aggregate() -> Pair { + let mut _0: Pair; + let _1: [u8; 8]; + let mut _3: [u8; 8]; + let mut _4: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => (_0.0: [u8; 8]); + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => (_0.1: [u8; 8]); + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- _3 = move _1; +- StorageLive(_4); +- _4 = move _2; +- _0 = Pair { a: move _3, b: move _4 }; +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ (_0.0: [u8; 8]) = [const 1_u8; 8]; ++ nop; ++ (_0.1: [u8; 8]) = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff b/tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff new file mode 100644 index 0000000000000..895dab2802597 --- /dev/null +++ b/tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff @@ -0,0 +1,97 @@ +- // MIR for `dse_guard` before MoveElimination ++ // MIR for `dse_guard` after MoveElimination + + fn dse_guard() -> () { + let mut _0: (); + let mut _1: Fields; + let mut _3: Fields; + let mut _4: Fields; + let _5: (); + let mut _6: *const Fields; + let mut _7: Fields; + let _8: (); + let mut _9: *const Fields; + scope 1 { +- debug a => _1; ++ debug a => _7; + let mut _2: Fields; + scope 2 { + debug b => _2; + } + } + + bb0: { +- StorageLive(_1); ++ nop; ++ nop; ++ nop; + StorageLive(_2); +- StorageLive(_3); +- _3 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; ++ _2 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- _2 = move _3; +- StorageDead(_3); +- StorageLive(_4); +- _4 = make_fields(const 1_u8) -> [return: bb2, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ StorageLive(_7); ++ _7 = make_fields(const 1_u8) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- _1 = move _4; +- StorageDead(_4); +- StorageLive(_5); ++ nop; ++ nop; ++ nop; ++ nop; + StorageLive(_6); +- _6 = &raw const _1; ++ _6 = &raw const _7; ++ StorageLive(_5); + _5 = observe(move _6) -> [return: bb3, unwind unreachable]; + } + + bb3: { +- StorageDead(_6); + StorageDead(_5); +- StorageLive(_7); +- _7 = move _1; ++ StorageDead(_6); ++ nop; ++ nop; ++ nop; ++ nop; + _2 = move _7; + StorageDead(_7); +- StorageLive(_8); ++ nop; ++ nop; ++ nop; + StorageLive(_9); + _9 = &raw const _2; ++ StorageLive(_8); + _8 = observe(move _9) -> [return: bb4, unwind unreachable]; + } + + bb4: { +- StorageDead(_9); + StorageDead(_8); ++ StorageDead(_9); ++ nop; ++ nop; + _0 = const (); ++ nop; + StorageDead(_2); +- StorageDead(_1); ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/dse.rs b/tests/mir-opt/move-elimination/dse.rs new file mode 100644 index 0000000000000..268b54fa96ae6 --- /dev/null +++ b/tests/mir-opt/move-elimination/dse.rs @@ -0,0 +1,38 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort -Zmir-enable-passes=+DeadStoreElimination-initial + +pub struct Fields { + data: [u8; 8], + tag: u8, +} + +unsafe extern "C" { + safe fn observe(_: *const Fields); + safe fn make_fields(_: u8) -> Fields; +} + +// EMIT_MIR dse.dse_guard.MoveElimination.diff +pub fn dse_guard() { + // This guards the RFC soundness hazard: DSE must not remove the first write + // to `b`, because that write keeps `b`'s address-observed lifetime + // overlapping with `a` and prevents the later move from being eliminated. + // CHECK-LABEL: fn dse_guard( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: StorageLive([[b]]); + // CHECK: [[b]] = make_fields(const 0_u8) + // CHECK: StorageLive([[a]]); + // CHECK: [[a]] = make_fields(const 1_u8) + // CHECK: observe(move + // CHECK: [[b]] = move [[a]] + // CHECK: observe(move + let mut a; + let mut b; + + b = make_fields(0); + + a = make_fields(1); + observe(&raw const a); + b = a; + observe(&raw const b); +} diff --git a/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..6aa7c42a2ba08 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff @@ -0,0 +1,20 @@ +- // MIR for `index_local_not_projected` before MoveElimination ++ // MIR for `index_local_not_projected` after MoveElimination + + fn index_local_not_projected(_1: [usize; 4]) -> [usize; 1] { + let mut _0: [usize; 1]; + let mut _2: usize; + let mut _3: usize; + + bb0: { ++ StorageLive(_2); + _2 = const 2_usize; ++ StorageLive(_3); + _3 = copy _1[_2]; ++ StorageDead(_3); + _0 = [copy _2]; ++ StorageDead(_2); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff new file mode 100644 index 0000000000000..51db7a9e85374 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff @@ -0,0 +1,136 @@ +- // MIR for `overlapping_lifetimes` before MoveElimination ++ // MIR for `overlapping_lifetimes` after MoveElimination + + fn overlapping_lifetimes(_1: bool) -> Fields { + debug flag => _1; + let mut _0: Fields; + let _2: Fields; + let _4: (); + let mut _5: *const Fields; + let _6: (); + let mut _7: bool; + let mut _8: Fields; + let mut _9: Fields; + let mut _10: Fields; + let _11: (); + let mut _12: *const Fields; + scope 1 { +- debug src => _2; ++ debug src => _10; + let mut _3: Fields; + scope 2 { +- debug dst => _3; ++ debug dst => _0; + } + } + + bb0: { +- StorageLive(_2); +- _2 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; ++ nop; ++ StorageLive(_10); ++ _10 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageLive(_3); +- StorageLive(_4); ++ nop; ++ nop; ++ nop; + StorageLive(_5); +- _5 = &raw const _2; ++ _5 = &raw const _10; ++ StorageLive(_4); + _4 = observe(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_5); + StorageDead(_4); +- StorageLive(_6); +- StorageLive(_7); +- _7 = copy _1; +- switchInt(move _7) -> [0: bb5, otherwise: bb3]; ++ StorageDead(_5); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb5, otherwise: bb3]; + } + + bb3: { +- StorageLive(_8); +- _8 = make_fields(const 1_u8) -> [return: bb4, unwind unreachable]; ++ nop; ++ _0 = make_fields(const 1_u8) -> [return: bb4, unwind unreachable]; + } + + bb4: { +- _3 = move _8; +- StorageDead(_8); +- StorageLive(_9); +- _9 = move _2; +- _3 = move _9; +- StorageDead(_9); ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = move _10; ++ StorageDead(_10); ++ nop; ++ StorageLive(_6); + _6 = const (); ++ StorageDead(_6); + goto -> bb6; + } + + bb5: { +- StorageLive(_10); +- _10 = move _2; +- _3 = move _10; ++ nop; ++ nop; ++ _0 = move _10; + StorageDead(_10); ++ nop; ++ StorageLive(_6); + _6 = const (); ++ StorageDead(_6); + goto -> bb6; + } + + bb6: { +- StorageDead(_7); +- StorageDead(_6); +- StorageLive(_11); ++ nop; ++ nop; ++ nop; ++ nop; + StorageLive(_12); +- _12 = &raw const _3; ++ _12 = &raw const _0; ++ StorageLive(_11); + _11 = observe(move _12) -> [return: bb7, unwind unreachable]; + } + + bb7: { +- StorageDead(_12); + StorageDead(_11); +- _0 = move _3; +- StorageDead(_3); +- StorageDead(_2); ++ StorageDead(_12); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..af6d2b34a0439 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff @@ -0,0 +1,49 @@ +- // MIR for `packed_fields_not_projected` before MoveElimination ++ // MIR for `packed_fields_not_projected` after MoveElimination + + fn packed_fields_not_projected() -> Packed { + let mut _0: Packed; + let _1: [u8; 8]; + let mut _3: [u8; 8]; + let mut _4: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => _3; + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => _4; + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; ++ nop; + StorageLive(_3); +- _3 = copy _1; ++ _3 = [const 1_u8; 8]; ++ nop; + StorageLive(_4); +- _4 = copy _2; ++ _4 = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; + _0 = Packed { a: move _3, b: move _4 }; +- StorageDead(_4); + StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ StorageDead(_4); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.rs b/tests/mir-opt/move-elimination/exclusions.rs new file mode 100644 index 0000000000000..2e45946d87ef1 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.rs @@ -0,0 +1,118 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort + +#![feature(core_intrinsics, custom_mir, repr_simd)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +pub struct Fields { + data: [u8; 8], + tag: u8, +} + +#[repr(packed)] +struct Packed { + a: [u8; 8], + b: [u8; 8], +} + +#[repr(simd)] +struct U32x4([u32; 4]); + +unsafe extern "C" { + safe fn observe(_: *const Fields); + safe fn make_fields(_: u8) -> Fields; +} + +// EMIT_MIR exclusions.index_local_not_projected.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn index_local_not_projected(a: [usize; 4]) -> [usize; 1] { + // This checks that a local used as an array index is kept as a bare local, + // because it cannot later be rewritten to a projection like `_0[0]`. + // CHECK-LABEL: fn index_local_not_projected( + // CHECK: [[idx:_.*]] = const 2_usize; + // CHECK: {{.*}} = copy _1[{{.*}}[[idx]]{{.*}}]; + // CHECK: _0 = [copy [[idx]]]; + mir! { + let idx: usize; + let b: usize; + + { + idx = 2usize; + b = a[idx]; + RET = [idx]; + Return() + } + } +} + +// EMIT_MIR exclusions.packed_fields_not_projected.MoveElimination.diff +pub fn packed_fields_not_projected() -> Packed { + // This checks that aggregate fields are not remapped into packed struct + // fields, which could create unaligned projected places. + // CHECK-LABEL: fn packed_fields_not_projected( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: _0 = Packed { a: move [[a]], b: move [[b]] }; + let a = [1; 8]; + let b = [2; 8]; + Packed { a, b } +} + +// EMIT_MIR exclusions.simd_field_not_projected.MoveElimination.diff +pub fn simd_field_not_projected() -> U32x4 { + // This checks that aggregate fields are not remapped into repr(simd) ADTs, + // since optimized MIR must not project into SIMD vectors. + // CHECK-LABEL: fn simd_field_not_projected( + // CHECK: debug lanes => [[lanes:_.*]]; + // CHECK: _0 = U32x4(move [[lanes]]); + let lanes = [1, 2, 3, 4]; + U32x4(lanes) +} + +// EMIT_MIR exclusions.overlapping_lifetimes.MoveElimination.diff +pub fn overlapping_lifetimes(flag: bool) -> Fields { + // This checks the liveness-matrix overlap test for an address-observed + // move-only local: `src` and `dst` only overlap on one branch, but that is + // enough to reject merging them for the whole function. + // CHECK-LABEL: fn overlapping_lifetimes( + // CHECK: debug flag => _1; + // CHECK: debug src => [[src:_[1-9][0-9]*]]; + // CHECK: debug dst => _0; + // CHECK: &raw const [[src]]; + // CHECK: observe + // CHECK: switchInt(move _1) + // CHECK: _0 = make_fields(const 1_u8) + // CHECK: _0 = move [[src]]; + // CHECK: &raw const _0; + // CHECK: observe + let src = make_fields(0); + let mut dst; + observe(&raw const src); + if flag { + dst = make_fields(1); + dst = src; + } else { + dst = src; + } + observe(&raw const dst); + dst +} + +// EMIT_MIR exclusions.rust_call_tuple_not_projected.MoveElimination.diff +pub fn rust_call_tuple_not_projected(f: F) { + // This checks that locals are not remapped into the tuple argument passed + // to a rust-call ABI function. If the tuple itself is never borrowed, alias + // analysis can trivially see that accesses to one argument don't affect the + // others. Merging the arguments into tuple fields from the start can hide + // that independence. + // CHECK-LABEL: fn rust_call_tuple_not_projected( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: [[tuple:_.*]] = (move [[a]], move [[b]]); + // CHECK: >::call_once(move _1, move [[tuple]]) + let a = [1; 8]; + let b = [2; 8]; + f(a, b); +} diff --git a/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..fcaf35ddeb8b0 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff @@ -0,0 +1,77 @@ +- // MIR for `rust_call_tuple_not_projected` before MoveElimination ++ // MIR for `rust_call_tuple_not_projected` after MoveElimination + + fn rust_call_tuple_not_projected(_1: F) -> () { + debug f => _1; + let mut _0: (); + let _2: [u8; 8]; + let _4: (); + let mut _5: F; + let mut _6: ([u8; 8], [u8; 8]); + let mut _7: [u8; 8]; + let mut _8: [u8; 8]; + scope 1 { +- debug a => _2; ++ debug a => _7; + let _3: [u8; 8]; + scope 2 { +- debug b => _3; ++ debug b => _8; + } + } + + bb0: { +- StorageLive(_2); +- _2 = [const 1_u8; 8]; +- StorageLive(_3); +- _3 = [const 2_u8; 8]; +- StorageLive(_4); +- StorageLive(_5); +- _5 = move _1; +- StorageLive(_6); ++ nop; + StorageLive(_7); +- _7 = copy _2; ++ _7 = [const 1_u8; 8]; ++ nop; + StorageLive(_8); +- _8 = copy _3; ++ _8 = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_6); + _6 = (move _7, move _8); +- _4 = >::call_once(move _5, move _6) -> [return: bb1, unwind unreachable]; ++ StorageDead(_7); ++ StorageDead(_8); ++ StorageLive(_4); ++ _4 = >::call_once(move _1, move _6) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_8); +- StorageDead(_7); +- StorageDead(_6); +- StorageDead(_5); + StorageDead(_4); ++ StorageDead(_6); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + _0 = const (); +- StorageDead(_3); +- StorageDead(_2); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..4844207200f34 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff @@ -0,0 +1,30 @@ +- // MIR for `simd_field_not_projected` before MoveElimination ++ // MIR for `simd_field_not_projected` after MoveElimination + + fn simd_field_not_projected() -> U32x4 { + let mut _0: U32x4; + let _1: [u32; 4]; + let mut _2: [u32; 4]; + scope 1 { +- debug lanes => _1; ++ debug lanes => _2; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u32, const 2_u32, const 3_u32, const 4_u32]; ++ nop; + StorageLive(_2); +- _2 = copy _1; ++ _2 = [const 1_u32, const 2_u32, const 3_u32, const 4_u32]; ++ nop; ++ nop; + _0 = U32x4(move _2); + StorageDead(_2); +- StorageDead(_1); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.address_observed_storage_dead_at_end.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.address_observed_storage_dead_at_end.MoveElimination.diff new file mode 100644 index 0000000000000..dfc680dc01ff2 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.address_observed_storage_dead_at_end.MoveElimination.diff @@ -0,0 +1,61 @@ +- // MIR for `address_observed_storage_dead_at_end` before MoveElimination ++ // MIR for `address_observed_storage_dead_at_end` after MoveElimination + + fn address_observed_storage_dead_at_end(_1: bool) -> () { + debug flag => _1; + let mut _0: (); + let _2: u32; + let mut _3: bool; + let _4: *const u32; + let mut _5: *const u32; + scope 1 { + debug x => _2; + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- switchInt(move _3) -> [0: bb3, otherwise: bb1]; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb3, otherwise: bb1]; + } + + bb1: { ++ StorageLive(_2); + _2 = const 1_u32; +- StorageLive(_4); ++ nop; ++ nop; + StorageLive(_5); + _5 = &raw const _2; ++ StorageLive(_4); + _4 = opaque::<*const u32>(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_5); + StorageDead(_4); ++ StorageDead(_5); ++ nop; ++ nop; + _0 = const (); + goto -> bb4; + } + + bb3: { + _0 = const (); + goto -> bb4; + } + + bb4: { +- StorageDead(_3); ++ nop; ++ nop; + StorageDead(_2); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff new file mode 100644 index 0000000000000..c28ca81eb5b9c --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff @@ -0,0 +1,87 @@ +- // MIR for `borrowed_not_shortened_to_last_direct_use` before MoveElimination ++ // MIR for `borrowed_not_shortened_to_last_direct_use` after MoveElimination + + fn borrowed_not_shortened_to_last_direct_use(_1: u32) -> () { + debug x => _1; + let mut _0: (); + let _2: u32; + let mut _3: u32; + let _6: u32; + let mut _7: u32; + let mut _8: u32; + let mut _9: u32; + scope 1 { + debug a => _2; + let _4: &u32; + scope 2 { + debug r => _4; + let _5: u32; + scope 3 { +- debug out => _5; ++ debug out => _9; + } + } + } + + bb0: { ++ nop; ++ nop; ++ nop; + StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- _2 = opaque::(move _3) -> [return: bb1, unwind unreachable]; ++ _2 = opaque::(move _1) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_3); ++ nop; ++ nop; + StorageLive(_4); + _4 = &_2; +- StorageLive(_5); +- _5 = copy _2; +- StorageLive(_6); +- StorageLive(_7); ++ nop; ++ StorageLive(_9); ++ _9 = copy _2; ++ nop; ++ nop; ++ nop; + StorageLive(_8); + _8 = copy (*_4); +- StorageLive(_9); +- _9 = copy _5; ++ StorageDead(_4); ++ nop; ++ nop; ++ StorageLive(_7); + _7 = Add(move _8, move _9); +- StorageDead(_9); + StorageDead(_8); ++ StorageDead(_9); ++ nop; ++ nop; ++ StorageLive(_6); + _6 = opaque::(move _7) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_7); + StorageDead(_6); ++ StorageDead(_7); ++ nop; ++ nop; + _0 = const (); +- StorageDead(_5); +- StorageDead(_4); ++ nop; ++ nop; ++ nop; + StorageDead(_2); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.critical_edge_split_for_storage_live.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.critical_edge_split_for_storage_live.MoveElimination.diff new file mode 100644 index 0000000000000..20a5abf62e439 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.critical_edge_split_for_storage_live.MoveElimination.diff @@ -0,0 +1,35 @@ +- // MIR for `critical_edge_split_for_storage_live` before MoveElimination ++ // MIR for `critical_edge_split_for_storage_live` after MoveElimination + + fn critical_edge_split_for_storage_live(_1: bool) -> () { + debug x => _2; + let mut _0: (); + let mut _2: u32; + let mut _3: *const u32; + + bb0: { +- switchInt(copy _1) -> [1: bb1, otherwise: bb2]; ++ switchInt(copy _1) -> [1: bb1, otherwise: bb3]; + } + + bb1: { ++ StorageLive(_2); + _2 = const 1_u32; ++ StorageLive(_3); + _3 = &raw const _2; + _3 = opaque::<*const u32>(copy _3) -> [return: bb2, unwind unreachable]; + } + + bb2: { ++ StorageDead(_3); + _2 = const 2_u32; ++ StorageDead(_2); + return; ++ } ++ ++ bb3: { ++ StorageLive(_2); ++ goto -> bb2; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.rs b/tests/mir-opt/move-elimination/storage.rs new file mode 100644 index 0000000000000..c699e4591c22c --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.rs @@ -0,0 +1,236 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort -Zlint-mir=false + +#![feature(custom_mir, core_intrinsics)] + +use std::intrinsics::mir::*; + +fn opaque(x: T) -> T { + x +} + +// EMIT_MIR storage.shorten_non_borrowed.MoveElimination.diff +pub fn shorten_non_borrowed(x: u32) { + // This checks that reconstruction can shorten a non-borrowed local's + // storage to its last use instead of keeping lexical storage markers. + // CHECK-LABEL: fn shorten_non_borrowed( + // CHECK: debug a => [[short:_.*]]; + // CHECK: debug b => [[short]]; + // CHECK: StorageLive([[short]]); + // CHECK: [[short]] = opaque::( + // CHECK: opaque::(move [[short]]) -> [return: [[short_ret:bb.*]], + // CHECK: [[short_ret]]: { + // CHECK: StorageDead([[short]]); + // CHECK: opaque::( + let a = opaque(x); + let b = a; + opaque(b); + opaque(x); +} + +// EMIT_MIR storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff +pub fn borrowed_not_shortened_to_last_direct_use(x: u32) { + // This checks that a borrowed local is not shortened merely to its last + // direct use; the borrow keeps its storage live while the reference exists. + // CHECK-LABEL: fn borrowed_not_shortened_to_last_direct_use( + // CHECK: debug a => [[borrowed:_.*]]; + // CHECK: debug r => [[borrow_ref:_.*]]; + // CHECK: debug out => [[out:_.*]]; + // CHECK: StorageLive([[borrowed]]); + // CHECK: [[borrowed]] = opaque::(move _1) + // CHECK: [[borrow_ref]] = &[[borrowed]]; + // CHECK: [[out]] = copy [[borrowed]]; + // CHECK: copy (*[[borrow_ref]]); + // CHECK: StorageDead([[borrowed]]); + let a = opaque(x); + let r = &a; + let out = a; + opaque(*r + out); +} + +// EMIT_MIR storage.storage_live_moved_to_branch.MoveElimination.diff +pub fn storage_live_moved_to_branch(flag: bool) { + // This checks storage reconstruction can shrink a local declared before a + // branch so its storage is live only on the arm where it is initialized. + // CHECK-LABEL: fn storage_live_moved_to_branch( + // CHECK: debug x => [[branch_tmp:_.*]]; + // CHECK: switchInt(move _1) -> [0: bb3, otherwise: bb1]; + // CHECK: bb1: { + // CHECK: StorageLive([[branch_tmp]]); + // CHECK: [[branch_tmp]] = const 1_u32; + // CHECK: opaque::(move [[branch_tmp]]) -> [return: [[branch_ret:bb.*]], + // CHECK: [[branch_ret]]: { + // CHECK: StorageDead([[branch_tmp]]); + let x: u32; + if flag { + x = 1; + opaque(x); + } +} + +// EMIT_MIR storage.address_observed_storage_dead_at_end.MoveElimination.diff +pub fn address_observed_storage_dead_at_end(flag: bool) { + // This checks that an address-observed local declared before a branch still + // has StorageLive moved into the initialized arm without adding one to the + // uninitialized arm, but StorageDead remains at the end of the function + // instead of being shortened to the last direct use. + // CHECK-LABEL: fn address_observed_storage_dead_at_end( + // CHECK: debug x => [[addr_tmp:_.*]]; + // CHECK: switchInt(move _1) -> [0: [[skip:bb.*]], otherwise: [[init:bb.*]]]; + // CHECK: [[init]]: { + // CHECK: StorageLive([[addr_tmp]]); + // CHECK: [[addr_tmp]] = const 1_u32; + // CHECK: &raw const [[addr_tmp]]; + // CHECK: opaque::<*const u32> + // CHECK-NOT: StorageDead([[addr_tmp]]); + // CHECK: [[skip]]: { + // CHECK-NOT: StorageLive([[addr_tmp]]); + // CHECK: {{bb.*}}: { + // CHECK: StorageDead([[addr_tmp]]); + let x: u32; + if flag { + x = 1; + opaque(&raw const x); + } +} + +// EMIT_MIR storage.terminator_end_storage_dead_in_successor.MoveElimination.diff +pub fn terminator_end_storage_dead_in_successor(x: u32) -> u32 { + // This checks storage reconstruction when the last use of a local is as a + // call argument in a terminator. + // CHECK-LABEL: fn terminator_end_storage_dead_in_successor( + // CHECK: debug tmp => [[term_tmp:_.*]]; + // CHECK: StorageLive([[term_tmp]]); + // CHECK: [[term_tmp]] = opaque::(move _1) + // CHECK: opaque::(move [[term_tmp]]) -> [return: [[term_ret:bb.*]], + // CHECK: [[term_ret]]: { + // CHECK-NEXT: StorageDead([[term_tmp]]); + let tmp = opaque(x); + let out = opaque(tmp); + out +} + +// EMIT_MIR storage.storage_dead_before_return.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn storage_dead_before_return() -> *const u32 { + // This checks that a borrowed local without input storage statements has + // its reconstructed storage ended before returning. + // CHECK-LABEL: fn storage_dead_before_return( + // CHECK: debug x => [[x:_.*]]; + // CHECK: StorageLive([[x]]); + // CHECK: [[x]] = const 1_u32; + // CHECK: [[ret:_.*]] = &raw const [[x]]; + // CHECK: StorageDead([[x]]); + // CHECK-NEXT: return; + mir! { + let x: u32; + debug x => x; + + { + x = 1; + RET = &raw const x; + Return() + } + } +} + +// EMIT_MIR storage.critical_edge_split_for_storage_live.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn critical_edge_split_for_storage_live(flag: bool) { + // This checks storage reconstruction on a custom CFG where both branches + // reach a block which initializes a maybe-live local. The local is dead on + // the direct incoming edge, so inserting StorageLive requires splitting the + // critical edge from the entry switch. + // CHECK-LABEL: fn critical_edge_split_for_storage_live( + // CHECK: debug x => [[crit_tmp:_.*]]; + // CHECK: switchInt(copy _1) -> [1: [[init:bb.*]], otherwise: [[split:bb.*]]]; + // CHECK: [[init]]: { + // CHECK: StorageLive([[crit_tmp]]); + // CHECK: [[crit_tmp]] = const 1_u32; + // CHECK: &raw const [[crit_tmp]]; + // CHECK: opaque::<*const u32>{{.*}} -> [return: [[ret:bb.*]], + // CHECK: [[ret]]: { + // CHECK: [[crit_tmp]] = const 2_u32; + // CHECK: [[split]]: { + // CHECK-NEXT: StorageLive([[crit_tmp]]); + // CHECK-NEXT: goto -> [[ret]]; + mir! { + let x: u32; + let ptr: *const u32; + debug x => x; + + { + match flag { + true => init, + _ => ret, + } + } + + init = { + x = 1; + ptr = &raw const x; + Call(ptr = opaque::<*const u32>(ptr), ReturnTo(ret), UnwindUnreachable()) + } + + ret = { + // An initialization forces a StorageLive on both incoming branches, + // which in turn forces a critical edge split. + x = 2; + Return() + } + } +} + +// EMIT_MIR storage.storage_live_elided_on_join_fork.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn storage_live_elided_on_join_fork(flag: bool) { + // This checks a join-fork CFG where x is live in only one predecessor and + // one successor of the middle block. The flag correlation means the path + // which reads x is only reached after x has been initialized. On the direct + // edge to join, every continuation either reads x before initializing it or + // never accesses it, so no StorageLive or critical-edge split is needed. + // CHECK-LABEL: fn storage_live_elided_on_join_fork( + // CHECK: debug x => [[join_tmp:_.*]]; + // CHECK: switchInt(copy _1) -> [1: [[init:bb.*]], otherwise: [[join:bb.*]]]; + // CHECK: [[init]]: { + // CHECK: StorageLive([[join_tmp]]); + // CHECK: [[join_tmp]] = const 1_u32; + // CHECK: goto -> [[join]]; + // CHECK: [[join]]: { + // CHECK-NOT: StorageLive([[join_tmp]]); + // CHECK: switchInt(copy _1) -> [1: [[use_x:bb.*]], otherwise: [[done:bb.*]]]; + // CHECK: [[use_x]]: { + // CHECK: opaque::(move [[join_tmp]]) -> [return: [[done]], unwind unreachable]; + mir! { + let x: u32; + let out: u32; + debug x => x; + + { + match flag { + true => init, + _ => join, + } + } + + init = { + x = 1; + Goto(join) + } + + join = { + match flag { + true => use_x, + _ => done, + } + } + + use_x = { + Call(out = opaque::(Move(x)), ReturnTo(done), UnwindUnreachable()) + } + + done = { + Return() + } + } +} diff --git a/tests/mir-opt/move-elimination/storage.shorten_non_borrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.shorten_non_borrowed.MoveElimination.diff new file mode 100644 index 0000000000000..67a6b0ef87618 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.shorten_non_borrowed.MoveElimination.diff @@ -0,0 +1,79 @@ +- // MIR for `shorten_non_borrowed` before MoveElimination ++ // MIR for `shorten_non_borrowed` after MoveElimination + + fn shorten_non_borrowed(_1: u32) -> () { + debug x => _1; + let mut _0: (); + let _2: u32; + let mut _3: u32; + let _5: u32; + let mut _6: u32; + let _7: u32; + let mut _8: u32; + scope 1 { +- debug a => _2; ++ debug a => _6; + let _4: u32; + scope 2 { +- debug b => _4; ++ debug b => _6; + } + } + + bb0: { +- StorageLive(_2); ++ nop; ++ nop; + StorageLive(_3); + _3 = copy _1; +- _2 = opaque::(move _3) -> [return: bb1, unwind unreachable]; ++ StorageLive(_6); ++ _6 = opaque::(move _3) -> [return: bb1, unwind unreachable]; + } + + bb1: { + StorageDead(_3); +- StorageLive(_4); +- _4 = copy _2; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + StorageLive(_5); +- StorageLive(_6); +- _6 = copy _4; + _5 = opaque::(move _6) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_6); + StorageDead(_5); ++ StorageDead(_6); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + StorageLive(_7); +- StorageLive(_8); +- _8 = copy _1; +- _7 = opaque::(move _8) -> [return: bb3, unwind unreachable]; ++ _7 = opaque::(move _1) -> [return: bb3, unwind unreachable]; + } + + bb3: { +- StorageDead(_8); + StorageDead(_7); ++ nop; ++ nop; + _0 = const (); +- StorageDead(_4); +- StorageDead(_2); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.storage_dead_before_return.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.storage_dead_before_return.MoveElimination.diff new file mode 100644 index 0000000000000..bc6299c97b6be --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.storage_dead_before_return.MoveElimination.diff @@ -0,0 +1,17 @@ +- // MIR for `storage_dead_before_return` before MoveElimination ++ // MIR for `storage_dead_before_return` after MoveElimination + + fn storage_dead_before_return() -> *const u32 { + debug x => _1; + let mut _0: *const u32; + let mut _1: u32; + + bb0: { ++ StorageLive(_1); + _1 = const 1_u32; + _0 = &raw const _1; ++ StorageDead(_1); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.storage_live_elided_on_join_fork.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.storage_live_elided_on_join_fork.MoveElimination.diff new file mode 100644 index 0000000000000..d5b9552b4fdf9 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.storage_live_elided_on_join_fork.MoveElimination.diff @@ -0,0 +1,36 @@ +- // MIR for `storage_live_elided_on_join_fork` before MoveElimination ++ // MIR for `storage_live_elided_on_join_fork` after MoveElimination + + fn storage_live_elided_on_join_fork(_1: bool) -> () { + debug x => _2; + let mut _0: (); + let mut _2: u32; + let mut _3: u32; + + bb0: { + switchInt(copy _1) -> [1: bb1, otherwise: bb2]; + } + + bb1: { ++ StorageLive(_2); + _2 = const 1_u32; + goto -> bb2; + } + + bb2: { + switchInt(copy _1) -> [1: bb3, otherwise: bb4]; + } + + bb3: { ++ StorageLive(_3); + _3 = opaque::(move _2) -> [return: bb4, unwind unreachable]; + } + + bb4: { ++ StorageDead(_2); ++ StorageDead(_2); ++ StorageDead(_3); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.storage_live_moved_to_branch.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.storage_live_moved_to_branch.MoveElimination.diff new file mode 100644 index 0000000000000..9a7c3df9cb98c --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.storage_live_moved_to_branch.MoveElimination.diff @@ -0,0 +1,63 @@ +- // MIR for `storage_live_moved_to_branch` before MoveElimination ++ // MIR for `storage_live_moved_to_branch` after MoveElimination + + fn storage_live_moved_to_branch(_1: bool) -> () { + debug flag => _1; + let mut _0: (); + let _2: u32; + let mut _3: bool; + let _4: u32; + let mut _5: u32; + scope 1 { +- debug x => _2; ++ debug x => _5; + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- switchInt(move _3) -> [0: bb3, otherwise: bb1]; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb3, otherwise: bb1]; + } + + bb1: { +- _2 = const 1_u32; +- StorageLive(_4); + StorageLive(_5); +- _5 = copy _2; ++ _5 = const 1_u32; ++ nop; ++ nop; ++ nop; ++ StorageLive(_4); + _4 = opaque::(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_5); + StorageDead(_4); ++ StorageDead(_5); ++ nop; ++ nop; + _0 = const (); + goto -> bb4; + } + + bb3: { + _0 = const (); + goto -> bb4; + } + + bb4: { +- StorageDead(_3); +- StorageDead(_2); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.terminator_end_storage_dead_in_successor.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.terminator_end_storage_dead_in_successor.MoveElimination.diff new file mode 100644 index 0000000000000..ffc1f04c58097 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.terminator_end_storage_dead_in_successor.MoveElimination.diff @@ -0,0 +1,57 @@ +- // MIR for `terminator_end_storage_dead_in_successor` before MoveElimination ++ // MIR for `terminator_end_storage_dead_in_successor` after MoveElimination + + fn terminator_end_storage_dead_in_successor(_1: u32) -> u32 { + debug x => _1; + let mut _0: u32; + let _2: u32; + let mut _3: u32; + let mut _5: u32; + scope 1 { +- debug tmp => _2; ++ debug tmp => _5; + let _4: u32; + scope 2 { +- debug out => _4; ++ debug out => _0; + } + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- _2 = opaque::(move _3) -> [return: bb1, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ StorageLive(_5); ++ _5 = opaque::(move _1) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_3); +- StorageLive(_4); +- StorageLive(_5); +- _5 = copy _2; +- _4 = opaque::(move _5) -> [return: bb2, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = opaque::(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { + StorageDead(_5); +- _0 = copy _4; +- StorageDead(_4); +- StorageDead(_2); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage_alloc.projected.MoveElimination.diff b/tests/mir-opt/move-elimination/storage_alloc.projected.MoveElimination.diff new file mode 100644 index 0000000000000..4fbad4e0c053d --- /dev/null +++ b/tests/mir-opt/move-elimination/storage_alloc.projected.MoveElimination.diff @@ -0,0 +1,28 @@ +- // MIR for `projected` before MoveElimination ++ // MIR for `projected` after MoveElimination + + fn projected() -> ([u32; 4], u32) { + let mut _0: ([u32; 4], u32); + let mut _1: [u32; 4]; + let mut _2: *mut [u32; 4]; + + bb0: { +- StorageLive(_1); +- StorageAlloc(_1); +- _2 = &raw mut _1; +- _1 = [const 42_u32; 4]; +- _0 = (move _1, const 99_u32); +- StorageDead(_1); ++ nop; ++ StorageAlloc(_0); ++ StorageLive(_2); ++ _2 = &raw mut (_0.0: [u32; 4]); ++ StorageDead(_2); ++ (_0.0: [u32; 4]) = [const 42_u32; 4]; ++ (_0.1: u32) = no_retag const 99_u32; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage_alloc.rs b/tests/mir-opt/move-elimination/storage_alloc.rs new file mode 100644 index 0000000000000..f7631303a3e2e --- /dev/null +++ b/tests/mir-opt/move-elimination/storage_alloc.rs @@ -0,0 +1,34 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +// The allocation starts the lifetime before the address is taken, and follows +// the local when it is moved into a field of the return place. +// EMIT_MIR storage_alloc.projected.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn projected() -> ([u32; 4], u32) { + // CHECK-LABEL: fn projected( + // CHECK: StorageAlloc(_0); + // CHECK: &raw mut (_0.0: [u32; 4]); + mir! { + let value: [u32; 4]; + let ptr: *mut [u32; 4]; + { + StorageLive(value); + StorageAlloc(value); + ptr = &raw mut value; + value = [42_u32; 4]; + RET = (Move(value), 99_u32); + StorageDead(value); + Return() + } + } +} + +fn main() { + assert_eq!(projected(), ([42; 4], 99)); +} diff --git a/tests/mir-opt/move-elimination/unwind.call_destination.MoveElimination.diff b/tests/mir-opt/move-elimination/unwind.call_destination.MoveElimination.diff new file mode 100644 index 0000000000000..6eec7504a13b4 --- /dev/null +++ b/tests/mir-opt/move-elimination/unwind.call_destination.MoveElimination.diff @@ -0,0 +1,32 @@ +- // MIR for `call_destination` before MoveElimination ++ // MIR for `call_destination` after MoveElimination + + fn call_destination() -> () { + let mut _0: (); + let mut _1: u32; + let mut _2: u32; + let mut _3: *const u32; + + bb0: { ++ StorageLive(_1); + _1 = const 0_u32; ++ StorageLive(_3); + _3 = &raw const _1; ++ StorageDead(_3); + _1 = opaque() -> [return: bb1, unwind: bb2]; + } + + bb1: { + _0 = (); ++ StorageDead(_1); + return; + } + + bb2 (cleanup): { ++ StorageLive(_2); + _2 = const 2_u32; + _1 = move _2; + resume; + } + } + diff --git a/tests/mir-opt/move-elimination/unwind.rs b/tests/mir-opt/move-elimination/unwind.rs new file mode 100644 index 0000000000000..0015997f7b543 --- /dev/null +++ b/tests/mir-opt/move-elimination/unwind.rs @@ -0,0 +1,41 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=unwind + +#![crate_type = "lib"] +#![feature(custom_mir, core_intrinsics)] +use std::intrinsics::mir::*; + +#[inline(never)] +fn opaque() -> u32 { + 1 +} + +// EMIT_MIR unwind.call_destination.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn call_destination() { + // CHECK-LABEL: fn call_destination( + // CHECK: [[DEST:_[0-9]+]] = opaque() + // CHECK: [[SRC:_[0-9]+]] = const 2_u32; + // CHECK-NEXT: [[DEST]] = move [[SRC]]; + // Unwinding doesn't deallocate the call destination, it only leaves it + // uninitialized. This checks that dest is not merged with src. + mir! { + let dest: u32; + let src: u32; + let p: *const u32; + { + dest = 0; + p = &raw const dest; + Call(dest = opaque(), ReturnTo(done), UnwindCleanup(cleanup)) + } + done = { + RET = (); + Return() + } + cleanup (cleanup) = { + src = 2; + dest = Move(src); + UnwindResume() + } + } +} diff --git a/tests/mir-opt/multiple_return_terminators.test.MultipleReturnTerminators.diff b/tests/mir-opt/multiple_return_terminators.test.MultipleReturnTerminators.diff index e28809f38afa6..b9c2de4aaf5f8 100644 --- a/tests/mir-opt/multiple_return_terminators.test.MultipleReturnTerminators.diff +++ b/tests/mir-opt/multiple_return_terminators.test.MultipleReturnTerminators.diff @@ -6,6 +6,7 @@ let mut _0: (); bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pattern_types.main.runtime-optimized.after.mir b/tests/mir-opt/pattern_types.main.runtime-optimized.after.mir index 541472fc22957..9bc455d98ebbf 100644 --- a/tests/mir-opt/pattern_types.main.runtime-optimized.after.mir +++ b/tests/mir-opt/pattern_types.main.runtime-optimized.after.mir @@ -10,6 +10,7 @@ fn main() -> () { } bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/checked_ops.use_checked_sub.runtime-optimized.after.panic-abort.mir b/tests/mir-opt/pre-codegen/checked_ops.use_checked_sub.runtime-optimized.after.panic-abort.mir index f24a8e41b7db5..8e21c607b6fbd 100644 --- a/tests/mir-opt/pre-codegen/checked_ops.use_checked_sub.runtime-optimized.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/checked_ops.use_checked_sub.runtime-optimized.after.panic-abort.mir @@ -34,12 +34,14 @@ fn use_checked_sub(_1: u32, _2: u32) -> () { } bb2: { + _0 = const (); StorageDead(_6); goto -> bb4; } bb3: { StorageDead(_3); + _0 = const (); goto -> bb4; } diff --git a/tests/mir-opt/pre-codegen/checked_ops.use_checked_sub.runtime-optimized.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/checked_ops.use_checked_sub.runtime-optimized.after.panic-unwind.mir index 6bcb730be7f14..fef388f87a7ce 100644 --- a/tests/mir-opt/pre-codegen/checked_ops.use_checked_sub.runtime-optimized.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/checked_ops.use_checked_sub.runtime-optimized.after.panic-unwind.mir @@ -34,12 +34,14 @@ fn use_checked_sub(_1: u32, _2: u32) -> () { } bb2: { + _0 = const (); StorageDead(_6); goto -> bb4; } bb3: { StorageDead(_3); + _0 = const (); goto -> bb4; } diff --git a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.runtime-optimized.after.panic-abort.mir b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.runtime-optimized.after.panic-abort.mir index 8384d563e52d2..16e8f326aa6dc 100644 --- a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.runtime-optimized.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.runtime-optimized.after.panic-abort.mir @@ -78,6 +78,7 @@ fn drop_bytes(_1: *mut Box<[u8; 1024]>) -> () { bb1: { StorageDead(_2); StorageDead(_3); + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.runtime-optimized.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.runtime-optimized.after.panic-unwind.mir index 8384d563e52d2..16e8f326aa6dc 100644 --- a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.runtime-optimized.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_bytes.runtime-optimized.after.panic-unwind.mir @@ -78,6 +78,7 @@ fn drop_bytes(_1: *mut Box<[u8; 1024]>) -> () { bb1: { StorageDead(_2); StorageDead(_3); + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.runtime-optimized.after.panic-abort.mir b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.runtime-optimized.after.panic-abort.mir index 3d24aa8c349e7..405a806d9fabf 100644 --- a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.runtime-optimized.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.runtime-optimized.after.panic-abort.mir @@ -88,6 +88,7 @@ fn drop_generic(_1: *mut Box) -> () { bb3: { StorageDead(_2); StorageDead(_4); + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.runtime-optimized.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.runtime-optimized.after.panic-unwind.mir index 3d24aa8c349e7..405a806d9fabf 100644 --- a/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.runtime-optimized.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/drop_box_of_sized.drop_generic.runtime-optimized.after.panic-unwind.mir @@ -88,6 +88,7 @@ fn drop_generic(_1: *mut Box) -> () { bb3: { StorageDead(_2); StorageDead(_4); + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.32bit.panic-abort.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.32bit.panic-abort.mir index 7c0cede85a4ca..3378e272bb867 100644 --- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.32bit.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.32bit.panic-abort.mir @@ -103,6 +103,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { StorageDead(_8); StorageDead(_5); StorageDead(_3); + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.32bit.panic-unwind.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.32bit.panic-unwind.mir index 7c0cede85a4ca..3378e272bb867 100644 --- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.32bit.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.32bit.panic-unwind.mir @@ -103,6 +103,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { StorageDead(_8); StorageDead(_5); StorageDead(_3); + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.64bit.panic-abort.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.64bit.panic-abort.mir index 7c0cede85a4ca..3378e272bb867 100644 --- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.64bit.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.64bit.panic-abort.mir @@ -103,6 +103,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { StorageDead(_8); StorageDead(_5); StorageDead(_3); + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.64bit.panic-unwind.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.64bit.panic-unwind.mir index 7c0cede85a4ca..3378e272bb867 100644 --- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.64bit.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.runtime-optimized.after.64bit.panic-unwind.mir @@ -103,6 +103,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () { StorageDead(_8); StorageDead(_5); StorageDead(_3); + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/intrinsics.f_u64.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/intrinsics.f_u64.runtime-optimized.after.mir index 0a5fa6daa9285..07ee6eb87e301 100644 --- a/tests/mir-opt/pre-codegen/intrinsics.f_u64.runtime-optimized.after.mir +++ b/tests/mir-opt/pre-codegen/intrinsics.f_u64.runtime-optimized.after.mir @@ -13,6 +13,7 @@ fn f_u64() -> () { } bb1: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/intrinsics.f_unit.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/intrinsics.f_unit.runtime-optimized.after.mir index 7c35963169d7c..4359dcc520fc1 100644 --- a/tests/mir-opt/pre-codegen/intrinsics.f_unit.runtime-optimized.after.mir +++ b/tests/mir-opt/pre-codegen/intrinsics.f_unit.runtime-optimized.after.mir @@ -13,6 +13,7 @@ fn f_unit() -> () { } bb1: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-abort.diff b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-abort.diff index a0526d279a863..6a58a50624d9f 100644 --- a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-abort.diff +++ b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-abort.diff @@ -45,6 +45,7 @@ StorageLive(_2); - _2 = Option::::None; + _2 = const Option::::None; + StorageAlloc(_1); StorageLive(_8); - _8 = discriminant(_2); - switchInt(move _8) -> [0: bb2, 1: bb3, otherwise: bb1]; @@ -84,6 +85,7 @@ } bb5: { + _11 = const std::alloc::AllocError; StorageLive(_13); StorageLive(_14); _14 = &_11; @@ -100,6 +102,7 @@ _3 = copy _4 as *mut u8 (PtrToPtr); StorageDead(_4); StorageDead(_3); + _0 = const (); StorageDead(_1); return; } diff --git a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-unwind.diff b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-unwind.diff index 3b5ecd3d2b16c..5d245daf5dddd 100644 --- a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-unwind.diff +++ b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-unwind.diff @@ -34,6 +34,7 @@ StorageLive(_2); - _2 = Option::::None; + _2 = const Option::::None; + StorageAlloc(_1); StorageLive(_8); - _8 = discriminant(_2); - switchInt(move _8) -> [0: bb3, 1: bb4, otherwise: bb2]; @@ -48,6 +49,7 @@ _3 = copy _4 as *mut u8 (PtrToPtr); StorageDead(_4); StorageDead(_3); + _0 = const (); StorageDead(_1); return; } diff --git a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-abort.diff b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-abort.diff index 3a6b52b0f2496..e6b148339528a 100644 --- a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-abort.diff +++ b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-abort.diff @@ -45,6 +45,7 @@ StorageLive(_2); - _2 = Option::::None; + _2 = const Option::::None; + StorageAlloc(_1); StorageLive(_8); - _8 = discriminant(_2); - switchInt(move _8) -> [0: bb2, 1: bb3, otherwise: bb1]; @@ -84,6 +85,7 @@ } bb5: { + _11 = const std::alloc::AllocError; StorageLive(_13); StorageLive(_14); _14 = &_11; @@ -100,6 +102,7 @@ _3 = copy _4 as *mut u8 (PtrToPtr); StorageDead(_4); StorageDead(_3); + _0 = const (); StorageDead(_1); return; } diff --git a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff index 673c05d673fb2..72ea30656f6d7 100644 --- a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff +++ b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff @@ -34,6 +34,7 @@ StorageLive(_2); - _2 = Option::::None; + _2 = const Option::::None; + StorageAlloc(_1); StorageLive(_8); - _8 = discriminant(_2); - switchInt(move _8) -> [0: bb3, 1: bb4, otherwise: bb2]; @@ -48,6 +49,7 @@ _3 = copy _4 as *mut u8 (PtrToPtr); StorageDead(_4); StorageDead(_3); + _0 = const (); StorageDead(_1); return; } diff --git a/tests/mir-opt/pre-codegen/loops.filter_mapped.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/loops.filter_mapped.runtime-optimized.after.mir index b3d38472a6a72..6643d93afd2ec 100644 --- a/tests/mir-opt/pre-codegen/loops.filter_mapped.runtime-optimized.after.mir +++ b/tests/mir-opt/pre-codegen/loops.filter_mapped.runtime-optimized.after.mir @@ -54,6 +54,7 @@ fn filter_mapped(_1: impl Iterator, _2: impl Fn(T) -> Option) -> () } bb4: { + _0 = const (); StorageDead(_7); drop(_4) -> [return: bb5, unwind continue]; } diff --git a/tests/mir-opt/pre-codegen/loops.int_range.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/loops.int_range.runtime-optimized.after.mir index cd146079c6328..7de0f54f28861 100644 --- a/tests/mir-opt/pre-codegen/loops.int_range.runtime-optimized.after.mir +++ b/tests/mir-opt/pre-codegen/loops.int_range.runtime-optimized.after.mir @@ -74,6 +74,7 @@ fn int_range(_1: usize, _2: usize) -> () { bb2: { StorageDead(_6); + _0 = const (); StorageDead(_9); return; } diff --git a/tests/mir-opt/pre-codegen/loops.mapped.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/loops.mapped.runtime-optimized.after.mir index f7c294bc1feb2..256d933552d1f 100644 --- a/tests/mir-opt/pre-codegen/loops.mapped.runtime-optimized.after.mir +++ b/tests/mir-opt/pre-codegen/loops.mapped.runtime-optimized.after.mir @@ -73,6 +73,7 @@ fn mapped(_1: impl Iterator, _2: impl Fn(T) -> U) -> () { StorageDead(_8); StorageDead(_7); StorageDead(_6); + _0 = const (); StorageDead(_12); drop(_4) -> [return: bb5, unwind continue]; } diff --git a/tests/mir-opt/pre-codegen/loops.vec_move.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/loops.vec_move.runtime-optimized.after.mir index 6815c22fc9552..c45e1320f9329 100644 --- a/tests/mir-opt/pre-codegen/loops.vec_move.runtime-optimized.after.mir +++ b/tests/mir-opt/pre-codegen/loops.vec_move.runtime-optimized.after.mir @@ -168,6 +168,7 @@ fn vec_move(_1: Vec) -> () { StorageLive(_4); StorageLive(_16); StorageLive(_2); + StorageAlloc(_2); _2 = ManuallyDrop::> { value: copy _1 }; StorageLive(_3); // DBG: _29 = &_2; @@ -177,9 +178,11 @@ fn vec_move(_1: Vec) -> () { StorageLive(_5); // DBG: _31 = &_2; // DBG: _30 = &((_2.0: std::vec::Vec).0: alloc::raw_vec::RawVec); + StorageAlloc(_5); _4 = copy ((((_2.0: std::vec::Vec).0: alloc::raw_vec::RawVec).0: alloc::raw_vec::RawVecInner).0: std::ptr::NonNull); _5 = copy _4 as std::ptr::NonNull (Transmute); StorageLive(_6); + StorageAlloc(_6); _6 = copy _4 as *mut impl Sized (Transmute); switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; } @@ -189,6 +192,7 @@ fn vec_move(_1: Vec) -> () { StorageLive(_7); // DBG: _35 = &_2; // DBG: _34 = &(_2.0: std::vec::Vec); + StorageAlloc(_7); _7 = copy ((_2.0: std::vec::Vec).1: usize); StorageLive(_8); _8 = Le(copy _7, const ::MAX_SLICE_LEN); @@ -205,6 +209,7 @@ fn vec_move(_1: Vec) -> () { StorageLive(_11); // DBG: _33 = &_2; // DBG: _32 = &(_2.0: std::vec::Vec); + StorageAlloc(_11); _11 = copy ((_2.0: std::vec::Vec).1: usize); StorageLive(_12); _12 = Le(copy _11, const ::MAX_SLICE_LEN); @@ -212,7 +217,9 @@ fn vec_move(_1: Vec) -> () { StorageDead(_12); StorageLive(_17); StorageLive(_13); + StorageAlloc(_13); _13 = copy _4 as *mut u8 (Transmute); + StorageAlloc(_17); StorageLive(_14); _14 = copy _11 as isize (IntToInt); StorageLive(_15); @@ -237,6 +244,7 @@ fn vec_move(_1: Vec) -> () { bb4: { // DBG: _37 = &_2; // DBG: _36 = &((_2.0: std::vec::Vec).0: alloc::raw_vec::RawVec); + StorageAlloc(_19); // DBG: _38 = &(((_2.0: std::vec::Vec).0: alloc::raw_vec::RawVec).0: alloc::raw_vec::RawVecInner); switchInt(const ::SIZE) -> [0: bb5, otherwise: bb6]; } @@ -281,6 +289,7 @@ fn vec_move(_1: Vec) -> () { } bb10: { + _0 = const (); StorageDead(_23); StorageDead(_24); drop(_22) -> [return: bb11, unwind continue]; diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.32bit.panic-abort.diff b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.32bit.panic-abort.diff index 5b39e45806e33..8f68119c7266d 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.32bit.panic-abort.diff +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.32bit.panic-abort.diff @@ -54,6 +54,7 @@ - _7 = copy _8; + _7 = const 42_u32; StorageDead(_8); + _0 = const (); StorageDead(_7); StorageDead(_3); StorageDead(_1); diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.32bit.panic-unwind.diff b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.32bit.panic-unwind.diff index ea2742a6471a1..d3fa5ff864c14 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.32bit.panic-unwind.diff +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.32bit.panic-unwind.diff @@ -54,6 +54,7 @@ - _7 = copy _8; + _7 = const 42_u32; StorageDead(_8); + _0 = const (); StorageDead(_7); StorageDead(_3); StorageDead(_1); diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.64bit.panic-abort.diff b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.64bit.panic-abort.diff index 5b39e45806e33..8f68119c7266d 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.64bit.panic-abort.diff +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.64bit.panic-abort.diff @@ -54,6 +54,7 @@ - _7 = copy _8; + _7 = const 42_u32; StorageDead(_8); + _0 = const (); StorageDead(_7); StorageDead(_3); StorageDead(_1); diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.64bit.panic-unwind.diff b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.64bit.panic-unwind.diff index ea2742a6471a1..d3fa5ff864c14 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.64bit.panic-unwind.diff +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.GVN.64bit.panic-unwind.diff @@ -54,6 +54,7 @@ - _7 = copy _8; + _7 = const 42_u32; StorageDead(_8); + _0 = const (); StorageDead(_7); StorageDead(_3); StorageDead(_1); diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.32bit.panic-abort.diff b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.32bit.panic-abort.diff index f7fe08831b97d..59571d145c2ef 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.32bit.panic-abort.diff +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.32bit.panic-abort.diff @@ -59,6 +59,7 @@ + StorageDead(_9); + StorageDead(_10); + nop; + _0 = const (); StorageDead(_7); StorageDead(_3); StorageDead(_1); diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.32bit.panic-unwind.diff b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.32bit.panic-unwind.diff index 6e36386bea601..b877371d2723e 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.32bit.panic-unwind.diff +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.32bit.panic-unwind.diff @@ -59,6 +59,7 @@ + StorageDead(_9); + StorageDead(_10); + nop; + _0 = const (); StorageDead(_7); StorageDead(_3); StorageDead(_1); diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.64bit.panic-abort.diff b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.64bit.panic-abort.diff index f7fe08831b97d..59571d145c2ef 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.64bit.panic-abort.diff +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.64bit.panic-abort.diff @@ -59,6 +59,7 @@ + StorageDead(_9); + StorageDead(_10); + nop; + _0 = const (); StorageDead(_7); StorageDead(_3); StorageDead(_1); diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.64bit.panic-unwind.diff b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.64bit.panic-unwind.diff index 6e36386bea601..b877371d2723e 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.64bit.panic-unwind.diff +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.ScalarReplacementOfAggregates.64bit.panic-unwind.diff @@ -59,6 +59,7 @@ + StorageDead(_9); + StorageDead(_10); + nop; + _0 = const (); StorageDead(_7); StorageDead(_3); StorageDead(_1); diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.32bit.panic-abort.mir b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.32bit.panic-abort.mir index 425b95db3363a..cc7ed0843fef7 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.32bit.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.32bit.panic-abort.mir @@ -13,6 +13,7 @@ fn main() -> () { } bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.32bit.panic-unwind.mir b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.32bit.panic-unwind.mir index 425b95db3363a..cc7ed0843fef7 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.32bit.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.32bit.panic-unwind.mir @@ -13,6 +13,7 @@ fn main() -> () { } bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.64bit.panic-abort.mir b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.64bit.panic-abort.mir index 425b95db3363a..cc7ed0843fef7 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.64bit.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.64bit.panic-abort.mir @@ -13,6 +13,7 @@ fn main() -> () { } bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.64bit.panic-unwind.mir b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.64bit.panic-unwind.mir index 425b95db3363a..cc7ed0843fef7 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.64bit.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.SimplifyLocals-final.after.64bit.panic-unwind.mir @@ -13,6 +13,7 @@ fn main() -> () { } bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.32bit.panic-abort.mir b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.32bit.panic-abort.mir index 1be0cd30068eb..d682c76299850 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.32bit.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.32bit.panic-abort.mir @@ -13,6 +13,7 @@ fn main() -> () { } bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.32bit.panic-unwind.mir b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.32bit.panic-unwind.mir index 1be0cd30068eb..d682c76299850 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.32bit.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.32bit.panic-unwind.mir @@ -13,6 +13,7 @@ fn main() -> () { } bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.64bit.panic-abort.mir b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.64bit.panic-abort.mir index 1be0cd30068eb..d682c76299850 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.64bit.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.64bit.panic-abort.mir @@ -13,6 +13,7 @@ fn main() -> () { } bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.64bit.panic-unwind.mir b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.64bit.panic-unwind.mir index 1be0cd30068eb..d682c76299850 100644 --- a/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.64bit.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/optimizes_into_variable.main.runtime-optimized.after.64bit.panic-unwind.mir @@ -13,6 +13,7 @@ fn main() -> () { } bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/range_iter.forward_loop.runtime-optimized.after.panic-abort.mir b/tests/mir-opt/pre-codegen/range_iter.forward_loop.runtime-optimized.after.panic-abort.mir index 47c78d4e80138..abaa8d465d9c9 100644 --- a/tests/mir-opt/pre-codegen/range_iter.forward_loop.runtime-optimized.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/range_iter.forward_loop.runtime-optimized.after.panic-abort.mir @@ -55,6 +55,7 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { bb2: { StorageDead(_5); + _0 = const (); StorageDead(_7); drop(_3) -> [return: bb3, unwind unreachable]; } @@ -67,6 +68,7 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { StorageLive(_6); _6 = copy _1; _1 = AddUnchecked(copy _6, const 1_u32); + StorageAlloc(_2); _7 = Option::::Some(copy _6); StorageDead(_6); StorageDead(_5); diff --git a/tests/mir-opt/pre-codegen/range_iter.forward_loop.runtime-optimized.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/range_iter.forward_loop.runtime-optimized.after.panic-unwind.mir index 5dfcba18be02f..eaeadff572b8d 100644 --- a/tests/mir-opt/pre-codegen/range_iter.forward_loop.runtime-optimized.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/range_iter.forward_loop.runtime-optimized.after.panic-unwind.mir @@ -55,6 +55,7 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { bb2: { StorageDead(_5); + _0 = const (); StorageDead(_7); drop(_3) -> [return: bb3, unwind continue]; } @@ -67,6 +68,7 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { StorageLive(_6); _6 = copy _1; _1 = AddUnchecked(copy _6, const 1_u32); + StorageAlloc(_2); _7 = Option::::Some(copy _6); StorageDead(_6); StorageDead(_5); diff --git a/tests/mir-opt/pre-codegen/range_iter.inclusive_loop.runtime-optimized.after.panic-abort.mir b/tests/mir-opt/pre-codegen/range_iter.inclusive_loop.runtime-optimized.after.panic-abort.mir index 8e89ffefe8ee7..67bc7b5666e49 100644 --- a/tests/mir-opt/pre-codegen/range_iter.inclusive_loop.runtime-optimized.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/range_iter.inclusive_loop.runtime-optimized.after.panic-abort.mir @@ -45,6 +45,7 @@ fn inclusive_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { } bb3: { + _0 = const (); StorageDead(_6); StorageDead(_7); StorageDead(_5); diff --git a/tests/mir-opt/pre-codegen/range_iter.inclusive_loop.runtime-optimized.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/range_iter.inclusive_loop.runtime-optimized.after.panic-unwind.mir index 084708b53c933..338c1cfcae8f5 100644 --- a/tests/mir-opt/pre-codegen/range_iter.inclusive_loop.runtime-optimized.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/range_iter.inclusive_loop.runtime-optimized.after.panic-unwind.mir @@ -45,6 +45,7 @@ fn inclusive_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () { } bb3: { + _0 = const (); StorageDead(_6); StorageDead(_7); StorageDead(_5); diff --git a/tests/mir-opt/pre-codegen/simple_option_map.map_via_question_mark.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/simple_option_map.map_via_question_mark.runtime-optimized.after.mir index 3c559614e9863..5e380762f2514 100644 --- a/tests/mir-opt/pre-codegen/simple_option_map.map_via_question_mark.runtime-optimized.after.mir +++ b/tests/mir-opt/pre-codegen/simple_option_map.map_via_question_mark.runtime-optimized.after.mir @@ -3,34 +3,31 @@ fn map_via_question_mark(_1: Option) -> Option { debug x => _1; let mut _0: std::option::Option; - let mut _4: std::option::Option; - let mut _7: std::ops::ControlFlow, i32>; - let _8: i32; - let mut _9: i32; + let mut _4: std::ops::ControlFlow, i32>; + let _5: i32; + let mut _6: i32; scope 1 { debug residual => const Option::::None; scope 2 { scope 7 (inlined as FromResidual>>::from_residual) { - let mut _3: isize; - let mut _5: bool; } } } scope 3 { - debug val => _8; + debug val => _5; scope 4 { } } scope 5 (inlined as Try>::branch) { let mut _2: isize; - let _6: i32; + let _3: i32; scope 6 { } } bb0: { - StorageLive(_9); - StorageLive(_7); + StorageLive(_6); + StorageLive(_4); StorageLive(_2); _2 = discriminant(_1); switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb4]; @@ -38,30 +35,23 @@ fn map_via_question_mark(_1: Option) -> Option { bb1: { StorageDead(_2); - StorageLive(_3); - StorageLive(_5); - _3 = discriminant(_4); - _5 = Eq(copy _3, const 0_isize); - assume(move _5); _0 = const Option::::None; - StorageDead(_5); - StorageDead(_3); - StorageDead(_9); - StorageDead(_7); + StorageDead(_6); + StorageDead(_4); goto -> bb3; } bb2: { - StorageLive(_6); - _6 = copy ((_1 as Some).0: i32); - _7 = ControlFlow::, i32>::Continue(copy _6); - StorageDead(_6); + StorageLive(_3); + _3 = copy ((_1 as Some).0: i32); + _4 = ControlFlow::, i32>::Continue(copy _3); + StorageDead(_3); StorageDead(_2); - _8 = copy ((_7 as Continue).0: i32); - _9 = Add(copy _8, const 1_i32); - _0 = Option::::Some(move _9); - StorageDead(_9); - StorageDead(_7); + _5 = copy ((_4 as Continue).0: i32); + _6 = Add(copy _5, const 1_i32); + _0 = Option::::Some(move _6); + StorageDead(_6); + StorageDead(_4); goto -> bb3; } diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.runtime-optimized.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.runtime-optimized.after.panic-abort.mir index b42087b5c822d..1323447598b9e 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.runtime-optimized.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.runtime-optimized.after.panic-abort.mir @@ -5,36 +5,33 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { debug f => _2; let mut _0: (); let mut _12: usize; - let mut _32: std::option::Option<(usize, &T)>; - let mut _35: &impl Fn(usize, &T); - let mut _36: (usize, &T); - let _37: (); + let mut _29: std::option::Option<(usize, &T)>; + let mut _32: &impl Fn(usize, &T); + let mut _33: (usize, &T); + let _34: (); scope 1 { debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _8; debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).1: *const T) => _11; debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>; debug ((iter: Enumerate>).1: usize) => _12; - let _33: usize; - let _34: &T; + let _30: usize; + let _31: &T; scope 2 { - debug i => _33; - debug x => _34; + debug i => _30; + debug x => _31; } scope 18 (inlined > as Iterator>::next) { - let mut _22: std::option::Option; - let mut _27: std::option::Option<&T>; - let mut _30: (usize, bool); - let mut _31: (usize, &T); + let mut _24: std::option::Option<&T>; + let mut _27: (usize, bool); + let mut _28: (usize, &T); scope 19 { - let _29: usize; + let _26: usize; scope 24 { } } scope 20 { scope 21 { scope 27 (inlined as FromResidual>>::from_residual) { - let mut _21: isize; - let mut _23: bool; } } } @@ -43,7 +40,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } } scope 25 (inlined as Try>::branch) { - let _28: &T; + let _25: &T; scope 26 { } } @@ -52,8 +49,8 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { let _13: std::ptr::NonNull; let _15: std::ptr::NonNull; let mut _18: bool; - let mut _24: usize; - let _26: &T; + let mut _21: usize; + let _23: &T; scope 29 { let _14: *const T; scope 30 { @@ -86,7 +83,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } } scope 43 (inlined NonNull::::as_ref::<'_>) { - let _25: *const T; + let _22: *const T; scope 44 (inlined NonNull::::as_ptr) { } scope 45 (inlined std::ptr::mut_ptr::::cast_const) { @@ -179,10 +176,10 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb4: { - StorageLive(_32); - StorageLive(_30); + StorageLive(_29); StorageLive(_27); StorageLive(_24); + StorageLive(_21); StorageLive(_16); StorageLive(_13); _13 = copy _8; @@ -210,6 +207,8 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { _19 = Offset(copy _16, const 1_usize); _8 = copy _19 as std::ptr::NonNull (Transmute); StorageDead(_19); + StorageAlloc(_12); + StorageAlloc(_11); goto -> bb13; } @@ -234,17 +233,11 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { StorageDead(_14); StorageDead(_13); StorageDead(_16); + StorageDead(_21); StorageDead(_24); StorageDead(_27); - StorageLive(_21); - StorageLive(_23); - _21 = discriminant(_22); - _23 = Eq(copy _21, const 0_isize); - assume(move _23); - StorageDead(_23); - StorageDead(_21); - StorageDead(_30); - StorageDead(_32); + _0 = const (); + StorageDead(_29); StorageDead(_12); drop(_2) -> [return: bb11, unwind unreachable]; } @@ -254,57 +247,61 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb12: { - _24 = SubUnchecked(copy _20, const 1_usize); - _11 = copy _24 as *const T (Transmute); + _21 = SubUnchecked(copy _20, const 1_usize); + StorageAlloc(_12); + StorageAlloc(_8); + _11 = copy _21 as *const T (Transmute); StorageDead(_20); goto -> bb13; } bb13: { - StorageLive(_26); - StorageLive(_25); - _25 = copy _13 as *const T (Transmute); - _26 = &(*_25); - StorageDead(_25); - _27 = Option::<&T>::Some(copy _26); - StorageDead(_26); + StorageLive(_23); + StorageLive(_22); + _22 = copy _13 as *const T (Transmute); + _23 = &(*_22); + StorageDead(_22); + _24 = Option::<&T>::Some(copy _23); + StorageDead(_23); StorageDead(_14); StorageDead(_13); StorageDead(_16); + StorageDead(_21); + _25 = copy ((_24 as Some).0: &T); StorageDead(_24); - _28 = copy ((_27 as Some).0: &T); - StorageDead(_27); - StorageLive(_29); - _29 = copy _12; - _30 = AddWithOverflow(copy _12, const 1_usize); - assert(!move (_30.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _12, const 1_usize) -> [success: bb14, unwind unreachable]; + StorageLive(_26); + _26 = copy _12; + _27 = AddWithOverflow(copy _12, const 1_usize); + assert(!move (_27.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _12, const 1_usize) -> [success: bb14, unwind unreachable]; } bb14: { - _12 = move (_30.0: usize); + StorageAlloc(_8); + StorageAlloc(_11); + _12 = move (_27.0: usize); + StorageLive(_28); + _28 = (copy _26, copy _25); + _29 = Option::<(usize, &T)>::Some(move _28); + StorageDead(_28); + StorageDead(_26); + StorageDead(_27); + StorageLive(_30); + _30 = copy (((_29 as Some).0: (usize, &T)).0: usize); StorageLive(_31); - _31 = (copy _29, copy _28); - _32 = Option::<(usize, &T)>::Some(move _31); - StorageDead(_31); - StorageDead(_29); - StorageDead(_30); + _31 = copy (((_29 as Some).0: (usize, &T)).1: &T); + StorageLive(_32); + _32 = &_2; StorageLive(_33); - _33 = copy (((_32 as Some).0: (usize, &T)).0: usize); - StorageLive(_34); - _34 = copy (((_32 as Some).0: (usize, &T)).1: &T); - StorageLive(_35); - _35 = &_2; - StorageLive(_36); - _36 = (copy _33, copy _34); - _37 = >::call(move _35, move _36) -> [return: bb15, unwind unreachable]; + _33 = (copy _30, copy _31); + _34 = >::call(move _32, move _33) -> [return: bb15, unwind unreachable]; } bb15: { - StorageDead(_36); - StorageDead(_35); - StorageDead(_34); StorageDead(_33); StorageDead(_32); + StorageDead(_31); + StorageDead(_30); + StorageDead(_29); goto -> bb4; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.runtime-optimized.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.runtime-optimized.after.panic-unwind.mir index bdb72abeac7e4..9cf28f01c2342 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.runtime-optimized.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.runtime-optimized.after.panic-unwind.mir @@ -125,6 +125,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb6: { + _0 = const (); StorageDead(_15); StorageDead(_16); StorageDead(_14); diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.runtime-optimized.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.runtime-optimized.after.panic-abort.mir index 2e662fc36ac67..5b7dd1c41ff19 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.runtime-optimized.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.runtime-optimized.after.panic-abort.mir @@ -170,6 +170,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { _18 = Offset(copy _15, const 1_usize); _8 = copy _18 as std::ptr::NonNull (Transmute); StorageDead(_18); + StorageAlloc(_11); goto -> bb13; } @@ -195,6 +196,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { StorageDead(_12); StorageDead(_15); StorageDead(_20); + _0 = const (); StorageDead(_23); drop(_2) -> [return: bb11, unwind unreachable]; } @@ -205,6 +207,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { bb12: { _20 = SubUnchecked(copy _19, const 1_usize); + StorageAlloc(_8); _11 = copy _20 as *const T (Transmute); StorageDead(_19); goto -> bb13; diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.runtime-optimized.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.runtime-optimized.after.panic-unwind.mir index dc5feaa1344b0..6ef6b1911acbd 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.runtime-optimized.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.runtime-optimized.after.panic-unwind.mir @@ -170,6 +170,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { _18 = Offset(copy _15, const 1_usize); _8 = copy _18 as std::ptr::NonNull (Transmute); StorageDead(_18); + StorageAlloc(_11); goto -> bb13; } @@ -195,6 +196,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { StorageDead(_12); StorageDead(_15); StorageDead(_20); + _0 = const (); StorageDead(_23); drop(_2) -> [return: bb11, unwind continue]; } @@ -205,6 +207,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { bb12: { _20 = SubUnchecked(copy _19, const 1_usize); + StorageAlloc(_8); _11 = copy _20 as *const T (Transmute); StorageDead(_19); goto -> bb13; diff --git a/tests/mir-opt/pre-codegen/slice_iter.range_loop.runtime-optimized.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.range_loop.runtime-optimized.after.panic-abort.mir index e817d9cd87289..a069bdcdeb5a6 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.range_loop.runtime-optimized.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.range_loop.runtime-optimized.after.panic-abort.mir @@ -63,6 +63,7 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb2: { StorageDead(_6); + _0 = const (); StorageDead(_8); drop(_2) -> [return: bb3, unwind unreachable]; } @@ -75,6 +76,7 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { StorageLive(_7); _7 = copy _4; _4 = AddUnchecked(copy _7, const 1_usize); + StorageAlloc(_3); _8 = Option::::Some(copy _7); StorageDead(_7); StorageDead(_6); diff --git a/tests/mir-opt/pre-codegen/slice_iter.range_loop.runtime-optimized.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.range_loop.runtime-optimized.after.panic-unwind.mir index b780f686570f8..586f93f728ae7 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.range_loop.runtime-optimized.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.range_loop.runtime-optimized.after.panic-unwind.mir @@ -63,6 +63,7 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb2: { StorageDead(_6); + _0 = const (); StorageDead(_8); drop(_2) -> [return: bb3, unwind continue]; } @@ -75,6 +76,7 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { StorageLive(_7); _7 = copy _4; _4 = AddUnchecked(copy _7, const 1_usize); + StorageAlloc(_3); _8 = Option::::Some(copy _7); StorageDead(_7); StorageDead(_6); diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.runtime-optimized.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.runtime-optimized.after.panic-abort.mir index c2c521729b75c..e4c82f9276034 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.runtime-optimized.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.runtime-optimized.after.panic-abort.mir @@ -303,6 +303,7 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { bb15: { StorageDead(_20); + _0 = const (); StorageDead(_35); StorageDead(_14); drop(_2) -> [return: bb16, unwind unreachable]; diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.runtime-optimized.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.runtime-optimized.after.panic-unwind.mir index c11eab18521dd..75782845f14d1 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.runtime-optimized.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.runtime-optimized.after.panic-unwind.mir @@ -311,6 +311,7 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { bb17: { StorageDead(_20); + _0 = const (); StorageDead(_35); StorageDead(_14); drop(_2) -> [return: bb18, unwind continue]; diff --git a/tests/mir-opt/pre-codegen/transmutes.forget_at_home.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/transmutes.forget_at_home.runtime-optimized.after.mir index 80ad68c8a48bb..6bd43650734c0 100644 --- a/tests/mir-opt/pre-codegen/transmutes.forget_at_home.runtime-optimized.after.mir +++ b/tests/mir-opt/pre-codegen/transmutes.forget_at_home.runtime-optimized.after.mir @@ -15,6 +15,7 @@ fn forget_at_home(_1: String) -> () { } bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/pre-codegen/transmutes.rs b/tests/mir-opt/pre-codegen/transmutes.rs index d8369e839b6ba..8c46968dba5c5 100644 --- a/tests/mir-opt/pre-codegen/transmutes.rs +++ b/tests/mir-opt/pre-codegen/transmutes.rs @@ -45,6 +45,7 @@ pub fn pad_for_alignment(x: u32) -> Align64 { pub fn forget_at_home(x: String) { // CHECK-LABEL: fn forget_at_home // CHECK: bb0: - // CHECK-NEXT: return; + // CHECK-NOT: -> + // CHECK: return; unsafe { transmute_prefix(x) } } diff --git a/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.GVN.diff b/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.GVN.diff index 78d58c6f60fae..637cd98c1ad81 100644 --- a/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.GVN.diff +++ b/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.GVN.diff @@ -44,6 +44,7 @@ - StorageLive(_3); + nop; _3 = copy (*_1); + StorageAlloc(_2); - StorageLive(_8); + nop; _8 = discriminant(_3); @@ -70,6 +71,7 @@ StorageLive(_5); - _5 = copy (*_1); + _5 = copy _3; + StorageAlloc(_4); StorageLive(_9); - _9 = discriminant(_5); - switchInt(move _9) -> [0: bb4, 1: bb5, otherwise: bb1]; diff --git a/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.runtime-optimized.after.mir index 9853713df439b..34a81231cc988 100644 --- a/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.runtime-optimized.after.mir +++ b/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.runtime-optimized.after.mir @@ -37,10 +37,12 @@ fn two_unwrap_unchecked(_1: &Option) -> i32 { bb0: { StorageLive(_5); _2 = copy (*_1); + StorageAlloc(_5); _3 = discriminant(_2); _4 = Eq(copy _3, const 1_isize); assume(move _4); _5 = copy ((_2 as Some).0: i32); + StorageAlloc(_5); _0 = Add(copy _5, copy _5); StorageDead(_5); return; diff --git a/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_generic.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_generic.runtime-optimized.after.mir index ffb1fe334b201..8c46d09c4639e 100644 --- a/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_generic.runtime-optimized.after.mir +++ b/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_generic.runtime-optimized.after.mir @@ -37,6 +37,7 @@ fn unaligned_copy_generic(_1: *const T, _2: *mut T) -> () { bb0: { StorageLive(_5); + StorageAlloc(_5); StorageLive(_3); _3 = copy _1 as *const std::ptr::Unaligned (PtrToPtr); StorageLive(_4); @@ -51,6 +52,7 @@ fn unaligned_copy_generic(_1: *const T, _2: *mut T) -> () { (*_6) = copy _7; StorageDead(_7); StorageDead(_6); + _0 = const (); StorageDead(_5); return; } diff --git a/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_manual.runtime-optimized.after.mir b/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_manual.runtime-optimized.after.mir index 450df935a0fb3..1908732a0e01d 100644 --- a/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_manual.runtime-optimized.after.mir +++ b/tests/mir-opt/pre-codegen/unaligned.unaligned_copy_manual.runtime-optimized.after.mir @@ -20,13 +20,16 @@ fn unaligned_copy_manual(_1: *const u128, _2: *mut u128) -> () { bb0: { StorageLive(_3); + StorageAlloc(_3); _3 = copy _1 as *const unaligned_copy_manual::Packed (PtrToPtr); StorageLive(_4); + StorageAlloc(_4); _4 = copy _2 as *mut unaligned_copy_manual::Packed (PtrToPtr); StorageLive(_5); _5 = copy ((*_3).0: u128); ((*_4).0: u128) = move _5; StorageDead(_5); + _0 = const (); StorageDead(_4); StorageDead(_3); return; diff --git a/tests/mir-opt/range/ssa_range.on_assert.SsaRangePropagation.diff b/tests/mir-opt/range/ssa_range.on_assert.SsaRangePropagation.diff index ae3f49a8847b0..af1fa882faaef 100644 --- a/tests/mir-opt/range/ssa_range.on_assert.SsaRangePropagation.diff +++ b/tests/mir-opt/range/ssa_range.on_assert.SsaRangePropagation.diff @@ -27,6 +27,7 @@ nop; StorageLive(_7); _7 = &(*_2); + StorageAlloc(_6); _6 = PtrMetadata(copy _2); StorageDead(_7); _4 = Lt(copy _1, copy _6); diff --git a/tests/mir-opt/range/ssa_range.on_assume.SsaRangePropagation.diff b/tests/mir-opt/range/ssa_range.on_assume.SsaRangePropagation.diff index 3be6f083604d3..311208d4ba05d 100644 --- a/tests/mir-opt/range/ssa_range.on_assume.SsaRangePropagation.diff +++ b/tests/mir-opt/range/ssa_range.on_assume.SsaRangePropagation.diff @@ -28,12 +28,14 @@ nop; StorageLive(_8); _8 = &(*_2); + StorageAlloc(_7); _7 = PtrMetadata(copy _2); StorageDead(_8); _5 = Lt(copy _1, copy _7); nop; StorageDead(_6); assume(copy _5); + _4 = const (); nop; StorageDead(_4); _3 = const (); diff --git a/tests/mir-opt/range/ssa_range.on_if.SsaRangePropagation.diff b/tests/mir-opt/range/ssa_range.on_if.SsaRangePropagation.diff index d097ddf619f0a..696b5f1c269b8 100644 --- a/tests/mir-opt/range/ssa_range.on_if.SsaRangePropagation.diff +++ b/tests/mir-opt/range/ssa_range.on_if.SsaRangePropagation.diff @@ -24,6 +24,7 @@ nop; StorageLive(_6); _6 = &(*_2); + StorageAlloc(_5); _5 = PtrMetadata(copy _2); StorageDead(_6); _3 = Lt(copy _1, copy _5); diff --git a/tests/mir-opt/remove_zsts.get_union.RemoveZsts.diff b/tests/mir-opt/remove_zsts.get_union.RemoveZsts.diff index 748c18ba7da97..ed338d69906c3 100644 --- a/tests/mir-opt/remove_zsts.get_union.RemoveZsts.diff +++ b/tests/mir-opt/remove_zsts.get_union.RemoveZsts.diff @@ -7,11 +7,10 @@ bb0: { - StorageLive(_1); -- _1 = (); ++ nop; + _1 = (); - _0 = Foo { x: move _1 }; - StorageDead(_1); -+ nop; -+ nop; + _0 = Foo { x: const () }; + nop; return; diff --git a/tests/mir-opt/remove_zsts.remove_generic_array.RemoveZsts.diff b/tests/mir-opt/remove_zsts.remove_generic_array.RemoveZsts.diff index 2ac944a6c6bc5..3241d40363753 100644 --- a/tests/mir-opt/remove_zsts.remove_generic_array.RemoveZsts.diff +++ b/tests/mir-opt/remove_zsts.remove_generic_array.RemoveZsts.diff @@ -1,12 +1,14 @@ - // MIR for `remove_generic_array` before RemoveZsts + // MIR for `remove_generic_array` after RemoveZsts - fn remove_generic_array(_1: T) -> () { + fn remove_generic_array(_1: T) -> ([T; 0], [T; 0]) { debug x => _1; - let mut _0: (); + let mut _0: ([T; 0], [T; 0]); let _2: [T; 0]; let mut _3: T; let mut _5: T; + let mut _6: [T; 0]; + let mut _7: [T; 0]; scope 1 { - debug a => _2; + debug a => const ZeroSized: [T; 0]; @@ -22,20 +24,30 @@ + nop; StorageLive(_3); _3 = copy _1; -- _2 = []; -+ nop; + _2 = []; StorageDead(_3); - StorageLive(_4); + nop; StorageLive(_5); _5 = copy _1; -- _4 = []; -+ nop; + _4 = []; StorageDead(_5); -- _0 = const (); +- StorageLive(_6); +- _6 = copy _2; +- StorageLive(_7); +- _7 = copy _4; +- _0 = (move _6, move _7); +- StorageDead(_7); +- StorageDead(_6); - StorageDead(_4); - StorageDead(_2); + nop; ++ _6 = const ZeroSized: [T; 0]; ++ nop; ++ _7 = const ZeroSized: [T; 0]; ++ _0 = (const ZeroSized: [T; 0], const ZeroSized: [T; 0]); ++ nop; ++ nop; + nop; + nop; return; diff --git a/tests/mir-opt/remove_zsts.rs b/tests/mir-opt/remove_zsts.rs index baf9d8ece266d..ff80edda68e77 100644 --- a/tests/mir-opt/remove_zsts.rs +++ b/tests/mir-opt/remove_zsts.rs @@ -15,14 +15,14 @@ fn get_union() -> Foo { const MYSTERY: usize = 280_usize.isqrt() - 260_usize.isqrt(); // EMIT_MIR remove_zsts.remove_generic_array.RemoveZsts.diff -fn remove_generic_array(x: T) { +fn remove_generic_array(x: T) -> ([T; 0], [T; MYSTERY]) { // CHECK-LABEL: fn remove_generic_array // CHECK: debug a => const ZeroSized: [T; 0]; // CHECK: debug b => const ZeroSized: [T; 0]; - // CHECK-NOT: = []; - // CHECK-NOT: ; 1] + // CHECK: _0 = (const ZeroSized: [T; 0], const ZeroSized: [T; 0]); let a = [x; 0]; let b = [x; MYSTERY]; + (a, b) } fn main() { diff --git a/tests/mir-opt/remove_zsts_move_elimination.discriminant.RemoveZsts.diff b/tests/mir-opt/remove_zsts_move_elimination.discriminant.RemoveZsts.diff new file mode 100644 index 0000000000000..7c2b40b6cc33e --- /dev/null +++ b/tests/mir-opt/remove_zsts_move_elimination.discriminant.RemoveZsts.diff @@ -0,0 +1,12 @@ +- // MIR for `discriminant` before RemoveZsts ++ // MIR for `discriminant` after RemoveZsts + + fn discriminant() -> Unit { + let mut _0: Unit; + + bb0: { + discriminant(_0) = 0; + return; + } + } + diff --git a/tests/mir-opt/remove_zsts_move_elimination.indirect.RemoveZsts.diff b/tests/mir-opt/remove_zsts_move_elimination.indirect.RemoveZsts.diff new file mode 100644 index 0000000000000..2d6fc87bbd906 --- /dev/null +++ b/tests/mir-opt/remove_zsts_move_elimination.indirect.RemoveZsts.diff @@ -0,0 +1,16 @@ +- // MIR for `indirect` before RemoveZsts ++ // MIR for `indirect` after RemoveZsts + + fn indirect(_1: *mut (), _2: *mut Unit) -> () { + let mut _0: (); + + bb0: { +- (*_1) = (); +- discriminant((*_2)) = 0; ++ nop; ++ nop; + _0 = (); + return; + } + } + diff --git a/tests/mir-opt/remove_zsts_move_elimination.local.RemoveZsts.diff b/tests/mir-opt/remove_zsts_move_elimination.local.RemoveZsts.diff new file mode 100644 index 0000000000000..4212876542ec5 --- /dev/null +++ b/tests/mir-opt/remove_zsts_move_elimination.local.RemoveZsts.diff @@ -0,0 +1,19 @@ +- // MIR for `local` before RemoveZsts ++ // MIR for `local` after RemoveZsts + + fn local() -> () { + let mut _0: (); + let mut _1: (); + + bb0: { +- StorageLive(_1); ++ nop; + _1 = (); +- _0 = move _1; +- StorageDead(_1); ++ _0 = const (); ++ nop; + return; + } + } + diff --git a/tests/mir-opt/remove_zsts_move_elimination.rs b/tests/mir-opt/remove_zsts_move_elimination.rs new file mode 100644 index 0000000000000..2065c1f7c58ba --- /dev/null +++ b/tests/mir-opt/remove_zsts_move_elimination.rs @@ -0,0 +1,68 @@ +//@ test-mir-pass: RemoveZsts + +#![crate_type = "lib"] +#![feature(custom_mir, core_intrinsics)] +use std::intrinsics::mir::*; + +// Under move elimination semantics we can't remove direct assignments since they have +// the side effect of creating an allocation for a local. If they are truly +// dead then they will be eliminated later optimization passes. + +// Preserve ZST writes while removing storage markers and replacing operands with constants. +// EMIT_MIR remove_zsts_move_elimination.local.RemoveZsts.diff +#[custom_mir(dialect = "runtime", phase = "initial")] +pub fn local() { + // CHECK-LABEL: fn local( + // CHECK-NOT: StorageLive + // CHECK: {{_[0-9]+}} = (); + // CHECK-NEXT: _0 = const (); + // CHECK-NOT: StorageDead + // CHECK: return; + mir! { + let value: (); + { + StorageLive(value); + value = (); + RET = Move(value); + StorageDead(value); + Return() + } + } +} + +pub enum Unit { + Value, +} + +// Preserve ZST SetDiscriminant since it allocates the destination local. +// EMIT_MIR remove_zsts_move_elimination.discriminant.RemoveZsts.diff +#[custom_mir(dialect = "runtime", phase = "initial")] +pub fn discriminant() -> Unit { + // CHECK-LABEL: fn discriminant( + // CHECK: discriminant(_0) = 0; + // CHECK-NEXT: return; + mir! { + { + SetDiscriminant(RET, 0); + Return() + } + } +} + +// Indirect ZST writes cannot allocate a local and can still be removed. +// EMIT_MIR remove_zsts_move_elimination.indirect.RemoveZsts.diff +#[custom_mir(dialect = "runtime", phase = "initial")] +pub unsafe fn indirect(value: *mut (), discriminant: *mut Unit) { + // CHECK-LABEL: fn indirect( + // CHECK-NOT: (* + // CHECK: _0 = (); + // CHECK-NEXT: return; + mir! { + { + *value = (); + SetDiscriminant(*discriminant, 0); + RET = (); + Return() + } + } +} diff --git a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-abort.diff b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-abort.diff index c67fd69235b47..60eb2add6cb5c 100644 --- a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-abort.diff +++ b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-abort.diff @@ -9,8 +9,8 @@ bb0: { StorageLive(_1); _1 = const false; -- switchInt(move _1) -> [0: bb2, otherwise: bb1]; -+ goto -> bb2; +- switchInt(move _1) -> [0: bb3, otherwise: bb1]; ++ goto -> bb3; } bb1: { @@ -18,6 +18,16 @@ } bb2: { + _0 = const (); + goto -> bb4; + } + + bb3: { + _0 = const (); + goto -> bb4; + } + + bb4: { StorageDead(_1); return; } diff --git a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-unwind.diff b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-unwind.diff index 07f179dfdd454..ff6f4f5d397c1 100644 --- a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-unwind.diff +++ b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-unwind.diff @@ -9,8 +9,8 @@ bb0: { StorageLive(_1); _1 = const false; -- switchInt(move _1) -> [0: bb2, otherwise: bb1]; -+ goto -> bb2; +- switchInt(move _1) -> [0: bb3, otherwise: bb1]; ++ goto -> bb3; } bb1: { @@ -18,6 +18,16 @@ } bb2: { + _0 = const (); + goto -> bb4; + } + + bb3: { + _0 = const (); + goto -> bb4; + } + + bb4: { StorageDead(_1); return; } diff --git a/tests/mir-opt/simplify_if.rs b/tests/mir-opt/simplify_if.rs index 6aca233655544..e86f19a7823a8 100644 --- a/tests/mir-opt/simplify_if.rs +++ b/tests/mir-opt/simplify_if.rs @@ -7,6 +7,7 @@ fn main() { // CHECK-LABEL: fn main( // CHECK: bb0: { + // CHECK-NEXT: _0 = const (); // CHECK-NEXT: return; if false { noop(); diff --git a/tests/mir-opt/simplify_locals_fixedpoint.foo.SimplifyLocals-final.panic-abort.diff b/tests/mir-opt/simplify_locals_fixedpoint.foo.SimplifyLocals-final.panic-abort.diff index 98727b80ca12e..2a2ce9ddb81fd 100644 --- a/tests/mir-opt/simplify_locals_fixedpoint.foo.SimplifyLocals-final.panic-abort.diff +++ b/tests/mir-opt/simplify_locals_fixedpoint.foo.SimplifyLocals-final.panic-abort.diff @@ -25,31 +25,37 @@ StorageDead(_3); StorageDead(_2); _5 = discriminant((_1.0: std::option::Option)); - switchInt(move _5) -> [1: bb1, 0: bb3, otherwise: bb5]; + switchInt(move _5) -> [1: bb1, 0: bb3, otherwise: bb6]; } bb1: { _4 = discriminant((_1.1: std::option::Option)); - switchInt(move _4) -> [0: bb2, 1: bb3, otherwise: bb5]; + switchInt(move _4) -> [0: bb2, 1: bb3, otherwise: bb6]; } bb2: { StorageLive(_6); _6 = copy (((_1.0: std::option::Option) as Some).0: u8); + _0 = no_retag const (); StorageDead(_6); - goto -> bb3; + goto -> bb4; } bb3: { - drop(_1) -> [return: bb4, unwind unreachable]; + _0 = const (); + goto -> bb4; } bb4: { + drop(_1) -> [return: bb5, unwind unreachable]; + } + + bb5: { StorageDead(_1); return; } - bb5: { + bb6: { unreachable; } } diff --git a/tests/mir-opt/simplify_locals_fixedpoint.foo.SimplifyLocals-final.panic-unwind.diff b/tests/mir-opt/simplify_locals_fixedpoint.foo.SimplifyLocals-final.panic-unwind.diff index 12b0a7f4ba535..1517268b7df5a 100644 --- a/tests/mir-opt/simplify_locals_fixedpoint.foo.SimplifyLocals-final.panic-unwind.diff +++ b/tests/mir-opt/simplify_locals_fixedpoint.foo.SimplifyLocals-final.panic-unwind.diff @@ -25,31 +25,37 @@ StorageDead(_3); StorageDead(_2); _5 = discriminant((_1.0: std::option::Option)); - switchInt(move _5) -> [1: bb1, 0: bb3, otherwise: bb5]; + switchInt(move _5) -> [1: bb1, 0: bb3, otherwise: bb6]; } bb1: { _4 = discriminant((_1.1: std::option::Option)); - switchInt(move _4) -> [0: bb2, 1: bb3, otherwise: bb5]; + switchInt(move _4) -> [0: bb2, 1: bb3, otherwise: bb6]; } bb2: { StorageLive(_6); _6 = copy (((_1.0: std::option::Option) as Some).0: u8); + _0 = no_retag const (); StorageDead(_6); - goto -> bb3; + goto -> bb4; } bb3: { - drop(_1) -> [return: bb4, unwind continue]; + _0 = const (); + goto -> bb4; } bb4: { + drop(_1) -> [return: bb5, unwind continue]; + } + + bb5: { StorageDead(_1); return; } - bb5: { + bb6: { unreachable; } } diff --git a/tests/mir-opt/simplify_match.main.GVN.panic-abort.diff b/tests/mir-opt/simplify_match.main.GVN.panic-abort.diff index 30c3b0bcfa1d7..6674b425bd492 100644 --- a/tests/mir-opt/simplify_match.main.GVN.panic-abort.diff +++ b/tests/mir-opt/simplify_match.main.GVN.panic-abort.diff @@ -16,15 +16,20 @@ - _1 = copy _2; + _1 = const false; StorageDead(_2); -- switchInt(copy _1) -> [0: bb2, otherwise: bb1]; -+ switchInt(const false) -> [0: bb2, otherwise: bb1]; +- switchInt(copy _1) -> [0: bb1, otherwise: bb2]; ++ switchInt(const false) -> [0: bb1, otherwise: bb2]; } bb1: { - _0 = noop() -> [return: bb2, unwind unreachable]; + _0 = const (); + goto -> bb3; } bb2: { + _0 = noop() -> [return: bb3, unwind unreachable]; + } + + bb3: { StorageDead(_1); return; } diff --git a/tests/mir-opt/simplify_match.main.GVN.panic-unwind.diff b/tests/mir-opt/simplify_match.main.GVN.panic-unwind.diff index 7923d3210d83f..2cef773f0d1c4 100644 --- a/tests/mir-opt/simplify_match.main.GVN.panic-unwind.diff +++ b/tests/mir-opt/simplify_match.main.GVN.panic-unwind.diff @@ -16,15 +16,20 @@ - _1 = copy _2; + _1 = const false; StorageDead(_2); -- switchInt(copy _1) -> [0: bb2, otherwise: bb1]; -+ switchInt(const false) -> [0: bb2, otherwise: bb1]; +- switchInt(copy _1) -> [0: bb1, otherwise: bb2]; ++ switchInt(const false) -> [0: bb1, otherwise: bb2]; } bb1: { - _0 = noop() -> [return: bb2, unwind continue]; + _0 = const (); + goto -> bb3; } bb2: { + _0 = noop() -> [return: bb3, unwind continue]; + } + + bb3: { StorageDead(_1); return; } diff --git a/tests/mir-opt/simplify_match.rs b/tests/mir-opt/simplify_match.rs index ca9dac40aa917..1776379086cab 100644 --- a/tests/mir-opt/simplify_match.rs +++ b/tests/mir-opt/simplify_match.rs @@ -9,6 +9,7 @@ fn noop() {} // CHECK: debug x => const false; // CHECK-NOT: switchInt // CHECK: bb0: { +// CHECK-NEXT: _0 = const (); // CHECK-NEXT: return; fn main() { match { diff --git a/tests/mir-opt/slice_drop_shim.core.ptr-drop_glue.[String;42].AddMovesForPackedDrops.before.mir b/tests/mir-opt/slice_drop_shim.core.ptr-drop_glue.[String;42].AddMovesForPackedDrops.before.mir index 1d4b0f033e0be..b7daabb24ebd0 100644 --- a/tests/mir-opt/slice_drop_shim.core.ptr-drop_glue.[String;42].AddMovesForPackedDrops.before.mir +++ b/tests/mir-opt/slice_drop_shim.core.ptr-drop_glue.[String;42].AddMovesForPackedDrops.before.mir @@ -10,6 +10,7 @@ fn std::ptr::drop_glue(_1: &mut [String; 42]) -> () { } bb1: { + _0 = const (); return; } } diff --git a/tests/mir-opt/slice_drop_shim.core.ptr-drop_glue.[String].AddMovesForPackedDrops.before.mir b/tests/mir-opt/slice_drop_shim.core.ptr-drop_glue.[String].AddMovesForPackedDrops.before.mir index 227f02d4c0e25..0b29f234e1b42 100644 --- a/tests/mir-opt/slice_drop_shim.core.ptr-drop_glue.[String].AddMovesForPackedDrops.before.mir +++ b/tests/mir-opt/slice_drop_shim.core.ptr-drop_glue.[String].AddMovesForPackedDrops.before.mir @@ -14,6 +14,7 @@ fn std::ptr::drop_glue(_1: &mut [String]) -> () { } bb1: { + _0 = const (); return; } diff --git a/tests/mir-opt/sroa/partial_write.assignment.ScalarReplacementOfAggregates.diff b/tests/mir-opt/sroa/partial_write.assignment.ScalarReplacementOfAggregates.diff new file mode 100644 index 0000000000000..8bfece06e0015 --- /dev/null +++ b/tests/mir-opt/sroa/partial_write.assignment.ScalarReplacementOfAggregates.diff @@ -0,0 +1,19 @@ +- // MIR for `assignment` before ScalarReplacementOfAggregates ++ // MIR for `assignment` after ScalarReplacementOfAggregates + + fn assignment() -> MaybeUninit { + let mut _0: std::mem::MaybeUninit; + let mut _1: (usize, std::mem::MaybeUninit); ++ let mut _2: usize; ++ let mut _3: std::mem::MaybeUninit; + + bb0: { +- (_1.0: usize) = const 0_usize; +- _0 = copy (_1.1: std::mem::MaybeUninit); ++ StorageAlloc(_3); ++ _2 = const 0_usize; ++ _0 = copy _3; + return; + } + } + diff --git a/tests/mir-opt/sroa/partial_write.call.ScalarReplacementOfAggregates.diff b/tests/mir-opt/sroa/partial_write.call.ScalarReplacementOfAggregates.diff new file mode 100644 index 0000000000000..d2b094be16738 --- /dev/null +++ b/tests/mir-opt/sroa/partial_write.call.ScalarReplacementOfAggregates.diff @@ -0,0 +1,28 @@ +- // MIR for `call` before ScalarReplacementOfAggregates ++ // MIR for `call` after ScalarReplacementOfAggregates + + fn call(_1: fn() -> usize) -> MaybeUninit { + let mut _0: std::mem::MaybeUninit; + let mut _2: (usize, std::mem::MaybeUninit); ++ let mut _3: usize; ++ let mut _4: std::mem::MaybeUninit; + + bb0: { +- (_2.0: usize) = copy _1() -> [return: bb1, unwind: bb2]; ++ StorageAlloc(_4); ++ _3 = copy _1() -> [return: bb1, unwind: bb2]; + } + + bb1: { +- _0 = copy (_2.1: std::mem::MaybeUninit); ++ _0 = copy _4; + return; + } + + bb2 (cleanup): { +- _0 = copy (_2.1: std::mem::MaybeUninit); ++ _0 = copy _4; + resume; + } + } + diff --git a/tests/mir-opt/sroa/partial_write.rs b/tests/mir-opt/sroa/partial_write.rs new file mode 100644 index 0000000000000..a2e132f150ce7 --- /dev/null +++ b/tests/mir-opt/sroa/partial_write.rs @@ -0,0 +1,56 @@ +//@ test-mir-pass: ScalarReplacementOfAggregates +//@ needs-unwind + +#![feature(custom_mir, core_intrinsics)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; +use std::mem::MaybeUninit; + +// A field write must also allocate the uninitialized sibling field after SROA. +// EMIT_MIR partial_write.assignment.ScalarReplacementOfAggregates.diff +#[custom_mir(dialect = "runtime")] +pub fn assignment() -> MaybeUninit { + // CHECK-LABEL: fn assignment( + // CHECK: bb0: { + // CHECK-NEXT: StorageAlloc([[SIBLING:_[0-9]+]]); + // CHECK-NEXT: {{_[0-9]+}} = const 0_usize; + // CHECK-NEXT: _0 = copy [[SIBLING]]; + mir! { + let x: (usize, MaybeUninit); + { + x.0 = 0; + RET = x.1; + Return() + } + } +} + +// Call destinations must allocate sibling fields before the call. +// EMIT_MIR partial_write.call.ScalarReplacementOfAggregates.diff +#[custom_mir(dialect = "runtime")] +pub fn call(f: fn() -> usize) -> MaybeUninit { + // CHECK-LABEL: fn call( + // CHECK: bb0: { + // CHECK-NEXT: StorageAlloc([[CALL_SIBLING:_[0-9]+]]); + // CHECK-NEXT: {{_[0-9]+}} = copy _1() -> + // CHECK: _0 = copy [[CALL_SIBLING]]; + // CHECK: (cleanup): { + // CHECK-NEXT: _0 = copy [[CALL_SIBLING]]; + mir! { + let x: (usize, MaybeUninit); + { + Call(x.0 = f(), ReturnTo(done), UnwindCleanup(cleanup)) + } + done = { + RET = x.1; + Return() + } + cleanup(cleanup) = { + RET = x.1; + UnwindResume() + } + } +} + +fn main() {} diff --git a/tests/mir-opt/sroa/scalable_sroa.bar.ScalarReplacementOfAggregates.diff b/tests/mir-opt/sroa/scalable_sroa.bar.ScalarReplacementOfAggregates.diff index fa4b14620b10e..5dc6a85ab5725 100644 --- a/tests/mir-opt/sroa/scalable_sroa.bar.ScalarReplacementOfAggregates.diff +++ b/tests/mir-opt/sroa/scalable_sroa.bar.ScalarReplacementOfAggregates.diff @@ -25,6 +25,7 @@ bb1: { _3 = copy (*_1)[_4]; StorageDead(_4); + _0 = const (); StorageDead(_3); return; } diff --git a/tests/mir-opt/sroa/scalable_sroa.foo.ScalarReplacementOfAggregates.diff b/tests/mir-opt/sroa/scalable_sroa.foo.ScalarReplacementOfAggregates.diff index 0aceea57e17a6..ee8f64fe33327 100644 --- a/tests/mir-opt/sroa/scalable_sroa.foo.ScalarReplacementOfAggregates.diff +++ b/tests/mir-opt/sroa/scalable_sroa.foo.ScalarReplacementOfAggregates.diff @@ -25,6 +25,7 @@ bb1: { _3 = copy (*_1)[_4]; StorageDead(_4); + _0 = const (); StorageDead(_3); return; } diff --git a/tests/mir-opt/sroa/simd_sroa.foo.ScalarReplacementOfAggregates.diff b/tests/mir-opt/sroa/simd_sroa.foo.ScalarReplacementOfAggregates.diff index 744032471089d..477d05b90702a 100644 --- a/tests/mir-opt/sroa/simd_sroa.foo.ScalarReplacementOfAggregates.diff +++ b/tests/mir-opt/sroa/simd_sroa.foo.ScalarReplacementOfAggregates.diff @@ -25,6 +25,7 @@ bb1: { _3 = copy (*_1)[_4]; StorageDead(_4); + _0 = const (); StorageDead(_3); return; } diff --git a/tests/mir-opt/sroa/storage_alloc.fields.ScalarReplacementOfAggregates.diff b/tests/mir-opt/sroa/storage_alloc.fields.ScalarReplacementOfAggregates.diff new file mode 100644 index 0000000000000..6d0dc1f7b85e8 --- /dev/null +++ b/tests/mir-opt/sroa/storage_alloc.fields.ScalarReplacementOfAggregates.diff @@ -0,0 +1,20 @@ +- // MIR for `fields` before ScalarReplacementOfAggregates ++ // MIR for `fields` after ScalarReplacementOfAggregates + + fn fields() -> (MaybeUninit, MaybeUninit) { + let mut _0: (std::mem::MaybeUninit, std::mem::MaybeUninit); + let mut _1: (std::mem::MaybeUninit, std::mem::MaybeUninit); ++ let mut _2: std::mem::MaybeUninit; ++ let mut _3: std::mem::MaybeUninit; + + bb0: { +- StorageAlloc(_1); +- _0 = (copy (_1.0: std::mem::MaybeUninit), copy (_1.1: std::mem::MaybeUninit)); ++ StorageAlloc(_2); ++ StorageAlloc(_3); ++ nop; ++ _0 = (copy _2, copy _3); + return; + } + } + diff --git a/tests/mir-opt/sroa/storage_alloc.rs b/tests/mir-opt/sroa/storage_alloc.rs new file mode 100644 index 0000000000000..7b5757bb1c623 --- /dev/null +++ b/tests/mir-opt/sroa/storage_alloc.rs @@ -0,0 +1,27 @@ +//@ test-mir-pass: ScalarReplacementOfAggregates + +#![crate_type = "lib"] +#![feature(custom_mir, core_intrinsics)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; +use std::mem::MaybeUninit; + +// Explicit StorageAlloc of an aggregate must allocate every replacement field. +// EMIT_MIR storage_alloc.fields.ScalarReplacementOfAggregates.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn fields() -> (MaybeUninit, MaybeUninit) { + // CHECK-LABEL: fn fields( + // CHECK: bb0: { + // CHECK-NEXT: StorageAlloc([[FIRST:_[0-9]+]]); + // CHECK-NEXT: StorageAlloc([[SECOND:_[0-9]+]]); + // CHECK: _0 = (copy [[FIRST]], copy [[SECOND]]); + mir! { + let x: (MaybeUninit, MaybeUninit); + { + StorageAlloc(x); + RET = (x.0, x.1); + Return() + } + } +} diff --git a/tests/mir-opt/uninhabited_enum.process_void.SimplifyLocals-final.after.mir b/tests/mir-opt/uninhabited_enum.process_void.SimplifyLocals-final.after.mir index 64711755f7394..585c6f3fdb06a 100644 --- a/tests/mir-opt/uninhabited_enum.process_void.SimplifyLocals-final.after.mir +++ b/tests/mir-opt/uninhabited_enum.process_void.SimplifyLocals-final.after.mir @@ -8,6 +8,7 @@ fn process_void(_1: *const Void) -> () { } bb0: { + _0 = const (); return; } } diff --git a/tests/mir-opt/uninhabited_not_read.main.SimplifyLocals-final.after.mir b/tests/mir-opt/uninhabited_not_read.main.SimplifyLocals-final.after.mir index 89f4cbc2edea5..ecea1ced574f9 100644 --- a/tests/mir-opt/uninhabited_not_read.main.SimplifyLocals-final.after.mir +++ b/tests/mir-opt/uninhabited_not_read.main.SimplifyLocals-final.after.mir @@ -43,6 +43,7 @@ fn main() -> () { _5 = copy _6 as *const ! (PtrToPtr); StorageDead(_6); StorageDead(_5); + _0 = const (); StorageDead(_4); return; } diff --git a/tests/mir-opt/unusual_item_types.core.ptr-drop_glue.Vec_i32_.AddMovesForPackedDrops.before.mir b/tests/mir-opt/unusual_item_types.core.ptr-drop_glue.Vec_i32_.AddMovesForPackedDrops.before.mir index 4f124c666bfb0..e4dda75280d14 100644 --- a/tests/mir-opt/unusual_item_types.core.ptr-drop_glue.Vec_i32_.AddMovesForPackedDrops.before.mir +++ b/tests/mir-opt/unusual_item_types.core.ptr-drop_glue.Vec_i32_.AddMovesForPackedDrops.before.mir @@ -10,6 +10,7 @@ fn std::ptr::drop_glue(_1: &mut Vec) -> () { } bb1: { + _0 = const (); return; } diff --git a/tests/run-make/const_fn_mir/dump.mir b/tests/run-make/const_fn_mir/dump.mir index 2b5c684bbef00..f758dea568e8c 100644 --- a/tests/run-make/const_fn_mir/dump.mir +++ b/tests/run-make/const_fn_mir/dump.mir @@ -41,6 +41,7 @@ fn main() -> () { } bb1: { + _0 = const (); return; } } diff --git a/tests/ui/inline-const/const-expr-generic-err.stderr b/tests/ui/inline-const/const-expr-generic-err.stderr index 26039ba6d4449..364d1338808f6 100644 --- a/tests/ui/inline-const/const-expr-generic-err.stderr +++ b/tests/ui/inline-const/const-expr-generic-err.stderr @@ -10,6 +10,14 @@ note: erroneous constant encountered LL | const { assert!(std::mem::size_of::() == 0); } | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +note: erroneous constant encountered + --> $DIR/const-expr-generic-err.rs:4:5 + | +LL | const { assert!(std::mem::size_of::() == 0); } + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no` + note: the above error was encountered while instantiating `fn foo::` --> $DIR/const-expr-generic-err.rs:12:5 | diff --git a/tests/ui/mir/lint/call-overlap.rs b/tests/ui/mir/lint/call-overlap.rs index f43edfb873826..b604ce6152a20 100644 --- a/tests/ui/mir/lint/call-overlap.rs +++ b/tests/ui/mir/lint/call-overlap.rs @@ -10,9 +10,10 @@ use core::intrinsics::mir::*; #[custom_mir(dialect = "runtime", phase = "optimized")] pub fn main() { mir! { - let a: [u8; 1024]; + let a: ([u8; 1024],); { - Call(a = f(Move(a)), ReturnTo(bb1), UnwindUnreachable()) //~ ERROR broken MIR + // Use a projection otherwise this would be considered malformed MIR. + Call(a.0 = f(Move(a.0)), ReturnTo(bb1), UnwindUnreachable()) //~ ERROR broken MIR //~^ ERROR encountered overlapping memory in `Move` arguments to `Call` } bb1 = { @@ -21,4 +22,6 @@ pub fn main() { } } -pub fn f(a: T) -> T { a } +pub fn f(a: T) -> T { + a +} diff --git a/tests/ui/mir/validate/call-move-destination.rs b/tests/ui/mir/validate/call-move-destination.rs new file mode 100644 index 0000000000000..887bf4a092fc4 --- /dev/null +++ b/tests/ui/mir/validate/call-move-destination.rs @@ -0,0 +1,112 @@ +//@ revisions: bare field zst generic built indirect +//@[built] compile-flags: -Zvalidate-mir +//@[bare] build-fail +//@[field] build-fail +//@[zst] build-fail +//@[generic] build-fail +//@[built] check-fail +//@[bare] failure-status: 101 +//@[bare] dont-check-compiler-stderr +//@[field] failure-status: 101 +//@[field] dont-check-compiler-stderr +//@[zst] failure-status: 101 +//@[zst] dont-check-compiler-stderr +//@[generic] failure-status: 101 +//@[generic] dont-check-compiler-stderr +//@[built] failure-status: 101 +//@[built] dont-check-compiler-stderr +//@[indirect] run-pass +//@[indirect] compile-flags: -Zvalidate-mir -Zmir-opt-level=0 + +// Direct call destinations cannot be rooted in a whole-local move argument. +// This is a structural invariant, including ZSTs and types with unknown layout. + +#![feature(custom_mir, core_intrinsics)] +use std::intrinsics::mir::*; + +#[cfg(any(bare, built))] +#[cfg_attr(built, custom_mir(dialect = "built"))] +#[cfg_attr(bare, custom_mir(dialect = "runtime", phase = "optimized"))] +fn main() { + mir! { + { + let value = 42u32; + Call(value = identity(Move(value)), ReturnTo(done), UnwindContinue()) + //[bare,built]~^ ERROR broken MIR + //[bare,built]~| ERROR return destination refers to a moved local + } + done = { Return() } + } +} + +#[cfg(field)] +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + { + let value = (1u32, 2u32); + Call(value.0 = first(Move(value)), ReturnTo(done), UnwindContinue()) + //[field]~^ ERROR broken MIR + //[field]~| ERROR return destination refers to a moved local + } + done = { Return() } + } +} +#[cfg(field)] +fn first(value: (u32, u32)) -> u32 { value.0 } + +#[cfg(zst)] +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + { + let value = (); + Call(value = identity(Move(value)), ReturnTo(done), UnwindContinue()) + //[zst]~^ ERROR broken MIR + //[zst]~| ERROR return destination refers to a moved local + } + done = { Return() } + } +} + +#[cfg(generic)] +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn generic(value: T) { + mir! { + { + Call(value = identity(Move(value)), ReturnTo(done), UnwindContinue()) + //[generic]~^ ERROR broken MIR + //[generic]~| ERROR return destination refers to a moved local + } + done = { Return() } + } +} +#[cfg(generic)] +fn main() { generic(()); } + +#[cfg(any(bare, built, zst, generic))] +fn identity(value: T) -> T { value } + +// Before analysis cleanup, a dereference need not be the first projection. +// Moving the pointer-containing local does not overlap its pointee destination. + +#[cfg(indirect)] +#[custom_mir(dialect = "built")] +fn call() -> u32 { + mir! { + let raw: *mut u32; + let pointer: (*mut u32,); + { + RET = 0; + raw = &raw mut RET; + pointer = (raw,); + Call(*pointer.0 = consume(Move(pointer)), ReturnTo(done), UnwindContinue()) + } + done = { Return() } + } +} +#[cfg(indirect)] +fn consume(_: (*mut u32,)) -> u32 { 42 } + +#[cfg(indirect)] +fn main() { assert_eq!(call(), 42); }