From 73e478e504b252df580cbd903457ef0100d67315 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Wed, 23 Sep 2026 04:02:26 +0100 Subject: [PATCH 1/5] Re-introduce ResultsVisitor::visit_block_exit --- compiler/rustc_mir_dataflow/src/framework/direction.rs | 4 ++++ compiler/rustc_mir_dataflow/src/framework/visitor.rs | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/compiler/rustc_mir_dataflow/src/framework/direction.rs b/compiler/rustc_mir_dataflow/src/framework/direction.rs index 39bd2be9409a3..bb0685cb7ff08 100644 --- a/compiler/rustc_mir_dataflow/src/framework/direction.rs +++ b/compiler/rustc_mir_dataflow/src/framework/direction.rs @@ -127,6 +127,8 @@ impl Direction for Backward { analysis.apply_primary_statement_effect(state, stmt, loc); vis.visit_after_primary_statement_effect(state, stmt, loc); } + + vis.visit_block_exit(state, block); } } @@ -242,5 +244,7 @@ impl Direction for Forward { vis.visit_after_early_terminator_effect(state, term, loc); analysis.apply_primary_terminator_effect(state, term, loc); vis.visit_after_primary_terminator_effect(state, term, loc); + + vis.visit_block_exit(state, block); } } diff --git a/compiler/rustc_mir_dataflow/src/framework/visitor.rs b/compiler/rustc_mir_dataflow/src/framework/visitor.rs index e4b840a73e502..5b8a3374e04d9 100644 --- a/compiler/rustc_mir_dataflow/src/framework/visitor.rs +++ b/compiler/rustc_mir_dataflow/src/framework/visitor.rs @@ -34,6 +34,13 @@ pub trait ResultsVisitor<'tcx, A> where A: Analysis<'tcx>, { + /// Called after all effects in a block have been applied in the direction + /// of the analysis. + /// + /// In a forwards analysis, `state` is from the block's end. In a backwards + /// analysis, `state` is from the block's start. + fn visit_block_exit(&mut self, _state: &A::Domain, _block: BasicBlock) {} + /// Called after the "early" effect of the given statement is applied to `state`. fn visit_after_early_statement_effect( &mut self, From d8b7d62c8829a171a6f7305778ec2e4e26d5cf52 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Wed, 23 Sep 2026 04:02:26 +0100 Subject: [PATCH 2/5] Add precise MIR liveness analysis --- compiler/rustc_mir_dataflow/src/impls/mod.rs | 5 + .../src/impls/precise_liveness.rs | 596 ++++++++++++++++++ 2 files changed, 601 insertions(+) create mode 100644 compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs diff --git a/compiler/rustc_mir_dataflow/src/impls/mod.rs b/compiler/rustc_mir_dataflow/src/impls/mod.rs index 1e12e41ce1fb4..faa4b0dd1694b 100644 --- a/compiler/rustc_mir_dataflow/src/impls/mod.rs +++ b/compiler/rustc_mir_dataflow/src/impls/mod.rs @@ -1,6 +1,7 @@ mod borrowed_locals; mod initialized; mod liveness; +mod precise_liveness; mod storage_liveness; pub use self::borrowed_locals::{MaybeBorrowedLocals, borrowed_locals}; @@ -11,6 +12,10 @@ pub use self::initialized::{ pub use self::liveness::{ DefUse, LivenessTransferFunction, MaybeLiveLocals, MaybeTransitiveLiveLocals, }; +pub use self::precise_liveness::{ + SplitPointEffect, SplitPointIndex, VisitAllocatedLocalsWith, VisitDeallocatedLocalsWith, + dump_liveness_matrix, liveness_matrix, +}; pub use self::storage_liveness::{ MaybeRequiresStorage, MaybeStorageDead, MaybeStorageLive, always_storage_live_locals, }; diff --git a/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs new file mode 100644 index 0000000000000..0234c240ef304 --- /dev/null +++ b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs @@ -0,0 +1,596 @@ +//! Computes the points where each local must have a distinct allocation. +//! +//! The result is a [`SparseIntervalMatrix`] with one row per local. Two locals +//! may share the same address only if their rows are disjoint. To model MIR +//! statements where a source operand and destination place may share an +//! address, each statement and terminator is split into an early point, where +//! operands are read, and a late point, where destinations are written. +//! +//! A local live range starts at the late point of any statement or terminator +//! that writes to it without a `Deref` projection. It ends at `StorageLive` or +//! `StorageDead`, or at the early point of a whole-local move operand or the +//! last use of that local on a control-flow path (only for locals whose address +//! is never observed). This happens at `StorageLive` because it resets a local +//! to an uninitialized state. +//! +//! `Call` terminators are handled specially: move operands are kept live +//! through the late point of the terminator so they conflict with each other +//! and with the destination place. This matches the runtime behavior where the +//! place is donated to the callee for the duration of the call. +//! +//! The analysis consists of three passes: +//! +//! 1. A backward value-liveness analysis which identifies the last point at +//! which each local is directly used in each control flow path. This is used +//! to shorten the lifetime of unborrowed locals to their last use. +//! 2. A forward allocation-liveness analysis which uses that information +//! together with writes, moves, and storage markers to compute which +//! allocations may be live on entry to each block. +//! 3. A final walk using those entry states which constructs the matrix and +//! distinguishes whether each effect happens at the early or late point of a +//! statement or terminator. + +use rustc_index::IndexVec; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::interval::SparseIntervalMatrix; +use rustc_middle::mir::visit::{ + MutatingUseContext, NonMutatingUseContext, NonUseContext, PlaceContext, VisitPlacesWith, + Visitor, +}; +use rustc_middle::mir::{self, BasicBlock, Local, Location, MirDumper, PassWhere, Place}; +use rustc_middle::ty::TyCtxt; +use tracing::trace; + +use crate::impls::{DefUse, MaybeLiveLocals, borrowed_locals}; +use crate::points::{DenseLocationMap, PointIndex}; +use crate::{Analysis, GenKill, ResultsVisitor, visit_results}; + +/// Visits locals that are implicitly allocated by a statement or terminator as +/// part of destination place evaluation. +pub struct VisitAllocatedLocalsWith(pub F); + +impl<'tcx, F> Visitor<'tcx> for VisitAllocatedLocalsWith +where + F: FnMut(Local, PlaceContext), +{ + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _: Location) { + match DefUse::for_place(*place, context) { + DefUse::Def | DefUse::PartialWrite => (self.0)(place.local, context), + DefUse::Use | DefUse::NonUse => {} + } + } +} + +/// Visits locals that are implicitly freed by being evaluated as a bare-local +/// move operand or through a storage statement. +pub struct VisitDeallocatedLocalsWith(pub F); + +impl<'tcx, F> Visitor<'tcx> for VisitDeallocatedLocalsWith +where + F: FnMut(Local, PlaceContext), +{ + fn visit_statement(&mut self, statement: &mir::Statement<'tcx>, location: Location) { + match statement.kind { + // StorageLive resets a local to the live-but-unallocated state, + // which also has the effect of freeing its allocation. + mir::StatementKind::StorageLive(local) => { + (self.0)(local, PlaceContext::NonUse(NonUseContext::StorageLive)); + } + mir::StatementKind::StorageDead(local) => { + (self.0)(local, PlaceContext::NonUse(NonUseContext::StorageDead)); + } + _ => self.super_statement(statement, location), + } + } + + fn visit_operand(&mut self, operand: &mir::Operand<'tcx>, _: Location) { + if let mir::Operand::Move(place) = operand + && let Some(local) = place.as_local() + { + (self.0)(local, PlaceContext::NonMutatingUse(NonMutatingUseContext::Move)); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Backward dataflow pass +// +// This pass computes "kill points" for each local, indicating the location of +// their last use in a particular control flow branch. These are later used in +// the forward pass to end the live range of locals that are never borrowed at +// their last direct use. +// +// This pass ignores borrowed locals: we can't determine their last use from +// syntactic uses of the base local, so we conservatively treat them as always +// live in the backwards pass and don't emit kill points for them. The main +// forward pass still kills at `StorageDead` and whole-local moves, which is +// when their allocation is freed. +// +// This pass has 2 outputs: a set of kill points that mark the last use +// locations of locals and a per-block bitset indicating which locals are live +// on entry to that block. The latter acts as a set of kill points at block +// entry for all locals not in the bit set. + +struct KillPoints<'a> { + live_on_entry: IndexVec>, + kill_points_map: IndexVec, +} + +impl<'a> KillPoints<'a> { + fn compute<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + pass_name: Option<&'static str>, + points: &DenseLocationMap, + kill_points: &'a mut Vec<(Local, Location)>, + ) -> Self { + let maybe_live_locals = MaybeLiveLocals.iterate_to_fixpoint(tcx, body, pass_name); + let borrowed_locals = borrowed_locals(body); + + // Initialize all borrowed locals as live on entry. We never try to kill + // those. + let mut live_on_entry = + IndexVec::from_elem_n(borrowed_locals.clone(), body.basic_blocks.len()); + + // Collect kill points and live-on-entry states from the results of + // MaybeLiveLocals. + kill_points.clear(); + let mut visitor = KillPointsVisitor { + kill_points, + live_on_entry: &mut live_on_entry, + borrowed_locals: &borrowed_locals, + }; + visit_results( + body, + mir::traversal::reachable(body).map(|(block, _)| block), + &maybe_live_locals, + &mut visitor, + ); + trace!(?kill_points); + trace!(?live_on_entry); + + // Create a mapping of `PointIndex` to the set of killed locals at that + // location. + let mut kill_points_map = IndexVec::from_elem_n(&[][..], points.num_points()); + for chunk in kill_points.chunk_by(|a, b| a.1 == b.1) { + let point = points.point_from_location(chunk[0].1); + trace!("Kill points at {:?}: {:?}", chunk[0].1, chunk); + debug_assert!(kill_points_map[point].is_empty()); + kill_points_map[point] = chunk; + } + + Self { live_on_entry, kill_points_map } + } +} + +struct KillPointsVisitor<'a> { + kill_points: &'a mut Vec<(Local, Location)>, + live_on_entry: &'a mut IndexVec>, + borrowed_locals: &'a DenseBitSet, +} + +impl<'tcx> ResultsVisitor<'tcx, MaybeLiveLocals> for KillPointsVisitor<'_> { + fn visit_block_exit(&mut self, state: &DenseBitSet, block: BasicBlock) { + // Borrowed locals are already marked as live when live_on_entry was + // initialized. This adds the non-borrowed locals that we have + // determined are live on entry to this block. + self.live_on_entry[block].union(state); + } + + fn visit_after_early_statement_effect( + &mut self, + state: &DenseBitSet, + statement: &mir::Statement<'tcx>, + location: Location, + ) { + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // Ignore non-uses. + match ctxt { + PlaceContext::NonMutatingUse(_) | PlaceContext::MutatingUse(_) => {} + PlaceContext::NonUse(_) => return, + } + + // If a local is used in a statement but is dead after it then this + // location is a kill point. Don't emit a kill point for borrowed + // locals. + if !state.contains(place.local) && !self.borrowed_locals.contains(place.local) { + self.kill_points.push((place.local, location)); + } + }) + .visit_statement(statement, location); + } + + fn visit_after_early_terminator_effect( + &mut self, + state: &DenseBitSet, + terminator: &mir::Terminator<'tcx>, + location: Location, + ) { + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // Ignore non-uses (they don't do anything) and edge uses + // (implicitly killed though live_on_entry at the start of the + // corresponding successor). + match ctxt { + PlaceContext::MutatingUse( + MutatingUseContext::AsmOutput + | MutatingUseContext::Call + | MutatingUseContext::Yield, + ) + | PlaceContext::NonUse(_) => return, + PlaceContext::NonMutatingUse(_) | PlaceContext::MutatingUse(_) => {} + } + + // If a local is used in a terminator but is dead after it then this + // location is a kill point. Don't emit a kill point for borrowed + // locals. + if !state.contains(place.local) && !self.borrowed_locals.contains(place.local) { + self.kill_points.push((place.local, location)); + } + }) + .visit_terminator(terminator, location); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Forward dataflow pass +// +// This is a simpler version of the main pass in `liveness_matrix`. It only +// needs to determine local liveness at block edges and doesn't distinguish +// between "early" and "late" effects. However it must still preserve the +// ordering of effects: moved operands are killed before destinations are +// generated. + +struct PreciseLiveness<'a> { + kill_points: &'a KillPoints<'a>, + points: &'a DenseLocationMap, +} + +impl PreciseLiveness<'_> { + fn apply_block_start_effect(&self, state: &mut DenseBitSet, block: BasicBlock) { + // Kill any unborrowed locals that are dead at the start of this block. + // Notably this kills any dead results produced by a predecessor's + // terminator. Borrowed locals are always included in live_on_entry. + state.intersect(&self.kill_points.live_on_entry[block]); + } +} + +impl<'tcx> Analysis<'tcx> for PreciseLiveness<'_> { + type Domain = DenseBitSet; + + const NAME: &'static str = "precise_liveness"; + + fn bottom_value(&self, body: &mir::Body<'tcx>) -> DenseBitSet { + DenseBitSet::new_empty(body.local_decls.len()) + } + + fn initialize_start_block(&self, body: &mir::Body<'tcx>, state: &mut DenseBitSet) { + // Function arguments start out as live. + for arg in body.args_iter() { + state.gen_(arg); + } + } + + fn apply_primary_statement_effect( + &self, + state: &mut DenseBitSet, + statement: &mir::Statement<'tcx>, + location: Location, + ) { + if location.statement_index == 0 { + self.apply_block_start_effect(state, location.block); + } + + VisitDeallocatedLocalsWith(|local, _| state.kill(local)) + .visit_statement(statement, location); + VisitAllocatedLocalsWith(|local, _| state.gen_(local)).visit_statement(statement, location); + + // Apply kill points at this statement: if a variable is dead then it + // doesn't need storage. + let point = self.points.point_from_location(location); + for &(local, _) in self.kill_points.kill_points_map[point] { + state.kill(local); + } + } + + fn apply_primary_terminator_effect( + &self, + state: &mut DenseBitSet, + terminator: &mir::Terminator<'tcx>, + location: Location, + ) { + if location.statement_index == 0 { + self.apply_block_start_effect(state, location.block); + } + + VisitDeallocatedLocalsWith(|local, _| state.kill(local)) + .visit_terminator(terminator, location); + + // Gen destination places. + VisitAllocatedLocalsWith(|local, ctxt| { + // The resume place of a Yield is only evaluated after the coroutine + // is resumed. This is in contrast to call destinations and inline + // asm outputs which are evaluated before the call and therefore + // remain allocated on both normal and unwind edges. + if ctxt != PlaceContext::MutatingUse(MutatingUseContext::Yield) { + state.gen_(local); + } + }) + .visit_terminator(terminator, location); + } + + fn apply_call_return_effect( + &self, + state: &mut DenseBitSet, + _block: BasicBlock, + return_places: mir::CallReturnPlaces<'_, 'tcx>, + ) { + if let mir::CallReturnPlaces::Yield(place) = return_places + && !place.is_indirect() + { + state.gen_(place.local); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Matrix construction + +/// Different "phases" of a single MIR statement, used to describe how +/// overlapping operands are handled. +/// +/// As a general rule, source operands are read in the `Early` phase and +/// destination places are written in the `Late` phase. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum SplitPointEffect { + Early = 0, + Late = 1, +} + +rustc_index::newtype_index! { + /// A `PointIndex` with the lower bit encoding early/late inside a + /// statement. + /// + /// This is used to model overlap constraints within a MIR statement: if a + /// source/destination are allowed to overlap then the source is read in + /// `SplitPointEffect::Early` and the write is done in + /// `SplitPointEffect::Late`. + #[orderable] + #[debug_format = "SplitPointIndex({})"] + pub struct SplitPointIndex {} +} + +impl SplitPointIndex { + pub fn new(point: PointIndex, effect: SplitPointEffect) -> SplitPointIndex { + let index = (point.as_u32() << 1) | (effect as u32); + SplitPointIndex::from_u32(index) + } + + pub fn point(self) -> PointIndex { + PointIndex::from_u32(self.as_u32() >> 1) + } + + pub fn effect(self) -> SplitPointEffect { + match self.as_u32() & 1 { + 0 => SplitPointEffect::Early, + 1 => SplitPointEffect::Late, + _ => unreachable!(), + } + } +} + +/// Helper type to construct a `SparseIntervalMatrix`. +struct MatrixBuilder { + matrix: SparseIntervalMatrix, + range_start: IndexVec>, + + // Track locals that have been live at any point in a block so that at the + // end of a block we don't need to iterate over all locals. This + // significantly speeds up matrix building. + maybe_live_locals: Vec, +} + +impl MatrixBuilder { + fn gen_(&mut self, local: Local, point: PointIndex, effect: SplitPointEffect) { + let split_point = SplitPointIndex::new(point, effect); + + // No-op if the local is already live. + if self.range_start[local].is_none() { + self.range_start[local] = Some(split_point); + self.maybe_live_locals.push(local); + } + } + + fn kill(&mut self, local: Local, point: PointIndex, effect: SplitPointEffect) { + let end = SplitPointIndex::new(point, effect); + + // No-op if the local is already dead. + if let Some(start) = self.range_start[local].take() { + debug_assert!(end >= start); + self.matrix.append_range(local, start..=end); + } + } + + fn kill_all(&mut self, point: PointIndex, effect: SplitPointEffect) { + while let Some(local) = self.maybe_live_locals.pop() { + self.kill(local, point, effect); + } + } + + fn kill_all_except(&mut self, except: Local, point: PointIndex, effect: SplitPointEffect) { + while let Some(local) = self.maybe_live_locals.pop() { + if local != except { + self.kill(local, point, effect); + } + } + self.maybe_live_locals.push(except); + } +} + +pub fn liveness_matrix<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + points: &DenseLocationMap, + pass_name: Option<&'static str>, +) -> SparseIntervalMatrix { + let mut kill_points_vec = vec![]; + let kill_points = KillPoints::compute(tcx, body, pass_name, points, &mut kill_points_vec); + let mut results = PreciseLiveness { kill_points: &kill_points, points } + .iterate_to_fixpoint(tcx, body, pass_name); + + let mut builder = MatrixBuilder { + matrix: SparseIntervalMatrix::new(points.num_points() * 2), + range_start: IndexVec::from_elem_n(None, body.local_decls.len()), + maybe_live_locals: Vec::new(), + }; + for (block, block_data) in body.basic_blocks.iter_enumerated() { + // We can mutate the state in-place since we're not using it any more + // after this point. + let state = &mut results.entry_states[block]; + + // Notably this kills any dead results produced by a predecessor's + // terminator. + state.intersect(&kill_points.live_on_entry[block]); + + // Gen any locals that are live at the start of the block. If this block + // only consists of a return terminator then only gen the return place + // instead. This ensures that StorageDead for all other locals are + // inserted before the return terminator. + let terminator = block_data.terminator(); + if let mir::TerminatorKind::Return = terminator.kind + && block_data.statements.is_empty() + { + if state.contains(mir::RETURN_PLACE) { + builder.gen_(mir::RETURN_PLACE, points.entry_point(block), SplitPointEffect::Early); + } + } else { + for local in state.iter() { + builder.gen_(local, points.entry_point(block), SplitPointEffect::Early); + } + } + + for (statement_index, statement) in block_data.statements.iter().enumerate() { + let location = Location { block, statement_index }; + let point = points.point_from_location(location); + + // Kill moved locals at the early point. + VisitDeallocatedLocalsWith(|local, _| { + builder.kill(local, point, SplitPointEffect::Early); + }) + .visit_statement(statement, location); + + // Kill any locals which are no longer used after this statement. + for &(local, _) in kill_points.kill_points_map[point] { + builder.kill(local, point, SplitPointEffect::Early); + } + + // Gen destination places at the late point. + VisitAllocatedLocalsWith(|local, _| { + builder.gen_(local, point, SplitPointEffect::Late); + }) + .visit_statement(statement, location); + + // Kill any dead destination places: they will only appear at the + // late point of the statement they are generated in, which is + // sufficient for determining overlap. + for &(local, _) in kill_points.kill_points_map[point] { + builder.kill(local, point, SplitPointEffect::Late); + } + } + + // If this block ends in a return terminator, end all live ranges before + // the terminator so that StorageDead statements are inserted before it. + // + // This is useful after inlining so that the lifetime of locals in the + // inlined callee don't extend past the call in the caller. + if let mir::TerminatorKind::Return = terminator.kind + && !block_data.statements.is_empty() + { + // Blocks with only a return terminator are handled above. + let location = Location { block, statement_index: block_data.statements.len() - 1 }; + let point = points.point_from_location(location); + builder.kill_all_except(mir::RETURN_PLACE, point, SplitPointEffect::Late); + } + + let location = Location { block, statement_index: block_data.statements.len() }; + let point = points.point_from_location(location); + + // Kill moved locals at the early point. + VisitDeallocatedLocalsWith(|local, _| { + builder.kill(local, point, SplitPointEffect::Early); + }) + .visit_terminator(terminator, location); + + // Kill any locals which are no longer used after this terminator. + for &(local, _) in kill_points.kill_points_map[point] { + builder.kill(local, point, SplitPointEffect::Early); + } + + // Gen destination places at the late point. + VisitAllocatedLocalsWith(|local, _| { + builder.gen_(local, point, SplitPointEffect::Late); + }) + .visit_terminator(terminator, location); + + // Move arguments to a call are treated specially: the place that they + // represent is passed directly to the callee, which means that they are + // not allowed to alias any other move operand or the destination place. + // This is represented here by extending their live range to the late + // part, making it overlap with that of the destination place. + // + // Notably, this *doesn't* apply to TailCall. + if let mir::TerminatorKind::Call { + func: _, + args, + destination: _, + target: _, + unwind: _, + call_source: _, + fn_span: _, + } = &terminator.kind + { + for arg in args { + if let mir::Operand::Move(place) = arg.node + && !place.is_indirect() + { + builder.gen_(place.local, point, SplitPointEffect::Late); + } + } + } + + // End the lifetimes of all locals at the end of the block. Successor + // blocks (which may not be contiguous in the index space!) will + // initialize the lifetimes again from their entry state. + builder.kill_all(point, SplitPointEffect::Late); + } + + builder.matrix +} + +pub fn dump_liveness_matrix<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + pass_name: &'static str, + points: &DenseLocationMap, + matrix: &SparseIntervalMatrix, +) { + let locals_live_at = |split_point| { + matrix.rows().filter(|&r| matrix.contains(r, split_point)).collect::>() + }; + + if let Some(dumper) = MirDumper::new(tcx, pass_name, body) { + let extra_data = &|pass_where, w: &mut dyn std::io::Write| { + if let PassWhere::BeforeLocation(loc) = pass_where { + let point = points.point_from_location(loc); + let split_point = SplitPointIndex::new(point, SplitPointEffect::Early); + let live = locals_live_at(split_point); + writeln!(w, " // {loc:?}-early => {live:?}")?; + let split_point = SplitPointIndex::new(point, SplitPointEffect::Late); + let live = locals_live_at(split_point); + writeln!(w, " // {loc:?}-late => {live:?}")?; + } + Ok(()) + }; + + dumper.set_extra_data(extra_data).dump_mir(body) + } +} From e637d1a0a12739e5ebb66504a62813b91aa81084 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 5 Oct 2026 20:59:53 +0100 Subject: [PATCH 3/5] Add dataflow tests for PreciseLiveness --- compiler/rustc_attr_ir/src/data_structures.rs | 1 + .../src/attributes/rustc_internal.rs | 4 + compiler/rustc_mir_dataflow/src/pretty.rs | 29 +- compiler/rustc_span/src/symbol.rs | 1 + .../precise_liveness.borrowed.dataflow.0.mir | 30 ++ .../precise_liveness.branch.dataflow.0.mir | 33 ++ .../precise_liveness.call.dataflow.0.mir | 39 ++ ...iveness.call_borrowed_field.dataflow.0.mir | 35 ++ ...precise_liveness.call_field.dataflow.0.mir | 21 ++ ...cise_liveness.call_indirect.dataflow.0.mir | 17 + ..._liveness.dead_destinations.dataflow.0.mir | 31 ++ ...recise_liveness.dereference.dataflow.0.mir | 14 + .../precise_liveness.fields.dataflow.0.mir | 25 ++ .../precise_liveness.indexed.dataflow.0.mir | 15 + tests/mir-opt/precise_liveness.rs | 357 ++++++++++++++++++ .../precise_liveness.storage.dataflow.0.mir | 37 ++ ...precise_liveness.unborrowed.dataflow.0.mir | 22 ++ 17 files changed, 710 insertions(+), 1 deletion(-) create mode 100644 tests/mir-opt/precise_liveness.borrowed.dataflow.0.mir create mode 100644 tests/mir-opt/precise_liveness.branch.dataflow.0.mir create mode 100644 tests/mir-opt/precise_liveness.call.dataflow.0.mir create mode 100644 tests/mir-opt/precise_liveness.call_borrowed_field.dataflow.0.mir create mode 100644 tests/mir-opt/precise_liveness.call_field.dataflow.0.mir create mode 100644 tests/mir-opt/precise_liveness.call_indirect.dataflow.0.mir create mode 100644 tests/mir-opt/precise_liveness.dead_destinations.dataflow.0.mir create mode 100644 tests/mir-opt/precise_liveness.dereference.dataflow.0.mir create mode 100644 tests/mir-opt/precise_liveness.fields.dataflow.0.mir create mode 100644 tests/mir-opt/precise_liveness.indexed.dataflow.0.mir create mode 100644 tests/mir-opt/precise_liveness.rs create mode 100644 tests/mir-opt/precise_liveness.storage.dataflow.0.mir create mode 100644 tests/mir-opt/precise_liveness.unborrowed.dataflow.0.mir diff --git a/compiler/rustc_attr_ir/src/data_structures.rs b/compiler/rustc_attr_ir/src/data_structures.rs index bca4d8fa7b5ff..04fee06e2434c 100644 --- a/compiler/rustc_attr_ir/src/data_structures.rs +++ b/compiler/rustc_attr_ir/src/data_structures.rs @@ -589,6 +589,7 @@ pub enum RustcDumpLayoutKind { #[derive(Clone, Debug, StableHash, Encodable, Decodable, PrintAttribute, PartialEq, Eq)] pub enum RustcMirKind { PrettyLiveLocals, + PrettyPreciseLiveness, PrettyTransitiveLiveLocals, PeekMaybeInit, PeekMaybeUninit, diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs index b9c03883d7c0b..70e5e4803c362 100644 --- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs +++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs @@ -672,6 +672,10 @@ impl CombineAttributeParser for RustcMirParser { cx.expect_no_args(mi.args())?; Some(RustcMirKind::PrettyLiveLocals) } + sym::rustc_pretty_precise_liveness => { + cx.expect_no_args(mi.args())?; + Some(RustcMirKind::PrettyPreciseLiveness) + } sym::rustc_pretty_transitive_live_locals => { cx.expect_no_args(mi.args())?; Some(RustcMirKind::PrettyTransitiveLiveLocals) diff --git a/compiler/rustc_mir_dataflow/src/pretty.rs b/compiler/rustc_mir_dataflow/src/pretty.rs index b37ad708e20e0..5a159ae40bd3c 100644 --- a/compiler/rustc_mir_dataflow/src/pretty.rs +++ b/compiler/rustc_mir_dataflow/src/pretty.rs @@ -7,7 +7,11 @@ use rustc_middle::ty::TyCtxt; use crate::debuginfo::debuginfo_locals; use crate::framework::Analysis; -use crate::impls::{MaybeLiveLocals, MaybeTransitiveLiveLocals, borrowed_locals}; +use crate::impls::{ + MaybeLiveLocals, MaybeTransitiveLiveLocals, SplitPointEffect, SplitPointIndex, borrowed_locals, + liveness_matrix, +}; +use crate::points::DenseLocationMap; use crate::{ResultsVisitor, visit_results}; type ExtraDataFn = dyn Fn(PassWhere, &mut dyn io::Write) -> io::Result<()>; @@ -33,6 +37,29 @@ pub(crate) fn mir_pretty_extra_data<'tcx>( &mut Annotator { annotations: &mut annotations }, ); } + RustcMirKind::PrettyPreciseLiveness => { + let points = DenseLocationMap::new(body); + let matrix = liveness_matrix(tcx, body, &points, None); + for (block, data) in body.basic_blocks.iter_enumerated() { + for statement_index in 0..=data.statements.len() { + let location = mir::Location { block, statement_index }; + let point = points.point_from_location(location); + let locals_live_at = |effect| { + let split_point = SplitPointIndex::new(point, effect); + matrix + .rows() + .filter(|&r| matrix.contains(r, split_point)) + .collect::>() + }; + let early = locals_live_at(SplitPointEffect::Early); + annotations + .add(PassWhere::BeforeLocation(location), format!("early: {early:?}")); + let late = locals_live_at(SplitPointEffect::Late); + annotations + .add(PassWhere::BeforeLocation(location), format!("late: {late:?}")); + } + } + } RustcMirKind::PrettyTransitiveLiveLocals => { let borrowed_locals = borrowed_locals(body); let debuginfo_locals = debuginfo_locals(body); diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index b9e7a9f5f94cb..a9275f489308a 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1903,6 +1903,7 @@ symbols! { rustc_peek_maybe_uninit, rustc_preserve_ub_checks, rustc_pretty_live_locals, + rustc_pretty_precise_liveness, rustc_pretty_transitive_live_locals, rustc_private, rustc_proc_macro_decls, diff --git a/tests/mir-opt/precise_liveness.borrowed.dataflow.0.mir b/tests/mir-opt/precise_liveness.borrowed.dataflow.0.mir new file mode 100644 index 0000000000000..64f9513b8757a --- /dev/null +++ b/tests/mir-opt/precise_liveness.borrowed.dataflow.0.mir @@ -0,0 +1,30 @@ +// MIR for `borrowed` 0 dataflow + +fn borrowed() -> u32 { + let mut _0: u32; + let mut _1: u32; + let mut _2: u32; + let mut _3: *const u32; + let mut _4: *const u32; + + bb0: { + // early: [] + // late: [_1] + _1 = const 1_u32; + // early: [_1] + // late: [_1, _2] + _2 = const 2_u32; + // early: [_1, _2] + // late: [_1, _2, _3] + _3 = &raw const _1; + // early: [_1, _2] + // late: [_1, _2, _4] + _4 = &raw const _2; + // early: [_1, _2] + // late: [_0, _1] + _0 = Add(copy _1, move _2); + // early: [_0] + // late: [] + return; + } +} diff --git a/tests/mir-opt/precise_liveness.branch.dataflow.0.mir b/tests/mir-opt/precise_liveness.branch.dataflow.0.mir new file mode 100644 index 0000000000000..d92d560b0dab3 --- /dev/null +++ b/tests/mir-opt/precise_liveness.branch.dataflow.0.mir @@ -0,0 +1,33 @@ +// MIR for `branch` 0 dataflow + +fn branch(_1: bool) -> u32 { + let mut _0: u32; + let mut _2: u32; + + bb0: { + // early: [_1] + // late: [_1, _2] + _2 = const 1_u32; + // early: [_1, _2] + // late: [_2] + switchInt(copy _1) -> [1: bb1, otherwise: bb2]; + } + + bb1: { + // early: [_2] + // late: [_0] + _0 = move _2; + // early: [_0] + // late: [] + return; + } + + bb2: { + // early: [] + // late: [_0] + _0 = const 0_u32; + // early: [_0] + // late: [] + return; + } +} diff --git a/tests/mir-opt/precise_liveness.call.dataflow.0.mir b/tests/mir-opt/precise_liveness.call.dataflow.0.mir new file mode 100644 index 0000000000000..f5e6d06e672d4 --- /dev/null +++ b/tests/mir-opt/precise_liveness.call.dataflow.0.mir @@ -0,0 +1,39 @@ +// MIR for `call` 0 dataflow + +fn call() -> u32 { + let mut _0: u32; + let mut _1: u32; + let mut _2: u32; + let mut _3: u32; + let mut _4: *const u32; + + bb0: { + // early: [] + // late: [_1] + _1 = const 1_u32; + // early: [_1] + // late: [_1, _2] + _2 = const 2_u32; + // early: [_1, _2] + // late: [_1, _3] + _3 = callee(move _1, copy _2) -> [return: bb1, unwind: bb2]; + } + + bb1: { + // early: [_3] + // late: [_3, _4] + _4 = &raw const _3; + // early: [_3] + // late: [_0] + _0 = move _3; + // early: [_0] + // late: [] + return; + } + + bb2 (cleanup): { + // early: [_3] + // late: [_3] + resume; + } +} diff --git a/tests/mir-opt/precise_liveness.call_borrowed_field.dataflow.0.mir b/tests/mir-opt/precise_liveness.call_borrowed_field.dataflow.0.mir new file mode 100644 index 0000000000000..96ecf63a27f53 --- /dev/null +++ b/tests/mir-opt/precise_liveness.call_borrowed_field.dataflow.0.mir @@ -0,0 +1,35 @@ +// MIR for `call_borrowed_field` 0 dataflow + +fn call_borrowed_field() -> u32 { + let mut _0: u32; + let mut _1: (u32,); + let mut _2: u32; + let mut _3: *const (u32,); + + bb0: { + // early: [] + // late: [_1] + (_1.0: u32) = const 1_u32; + // early: [_1] + // late: [_1, _3] + _3 = &raw const _1; + // early: [_1] + // late: [_1, _2] + _2 = callee(move (_1.0: u32), const 0_u32) -> [return: bb1, unwind: bb2]; + } + + bb1: { + // early: [_1, _2] + // late: [_0, _1] + _0 = move _2; + // early: [_0] + // late: [] + return; + } + + bb2 (cleanup): { + // early: [_1] + // late: [_1] + resume; + } +} diff --git a/tests/mir-opt/precise_liveness.call_field.dataflow.0.mir b/tests/mir-opt/precise_liveness.call_field.dataflow.0.mir new file mode 100644 index 0000000000000..2bcd1434eeafd --- /dev/null +++ b/tests/mir-opt/precise_liveness.call_field.dataflow.0.mir @@ -0,0 +1,21 @@ +// MIR for `call_field` 0 dataflow + +fn call_field() -> u32 { + let mut _0: u32; + let mut _1: (u32,); + + bb0: { + // early: [] + // late: [_1] + (_1.0: u32) = const 1_u32; + // early: [_1] + // late: [_0, _1] + _0 = callee(move (_1.0: u32), const 0_u32) -> [return: bb1, unwind unreachable]; + } + + bb1: { + // early: [_0] + // late: [] + return; + } +} diff --git a/tests/mir-opt/precise_liveness.call_indirect.dataflow.0.mir b/tests/mir-opt/precise_liveness.call_indirect.dataflow.0.mir new file mode 100644 index 0000000000000..fff5e24358bda --- /dev/null +++ b/tests/mir-opt/precise_liveness.call_indirect.dataflow.0.mir @@ -0,0 +1,17 @@ +// MIR for `call_indirect` 0 dataflow + +fn call_indirect(_1: *const u32) -> u32 { + let mut _0: u32; + + bb0: { + // early: [_1] + // late: [_0] + _0 = callee(move (*_1), const 0_u32) -> [return: bb1, unwind unreachable]; + } + + bb1: { + // early: [_0] + // late: [] + return; + } +} diff --git a/tests/mir-opt/precise_liveness.dead_destinations.dataflow.0.mir b/tests/mir-opt/precise_liveness.dead_destinations.dataflow.0.mir new file mode 100644 index 0000000000000..9b6a304bc7184 --- /dev/null +++ b/tests/mir-opt/precise_liveness.dead_destinations.dataflow.0.mir @@ -0,0 +1,31 @@ +// MIR for `dead_destinations` 0 dataflow + +fn dead_destinations() -> u32 { + let mut _0: u32; + let mut _1: u32; + let mut _2: u32; + + bb0: { + // early: [] + // late: [_1] + _1 = const 1_u32; + // early: [] + // late: [_2] + _2 = callee(const 1_u32, const 2_u32) -> [return: bb1, unwind: bb2]; + } + + bb1: { + // early: [] + // late: [_0] + _0 = const 0_u32; + // early: [_0] + // late: [] + return; + } + + bb2 (cleanup): { + // early: [] + // late: [] + resume; + } +} diff --git a/tests/mir-opt/precise_liveness.dereference.dataflow.0.mir b/tests/mir-opt/precise_liveness.dereference.dataflow.0.mir new file mode 100644 index 0000000000000..bb00f3aaf5247 --- /dev/null +++ b/tests/mir-opt/precise_liveness.dereference.dataflow.0.mir @@ -0,0 +1,14 @@ +// MIR for `dereference` 0 dataflow + +fn dereference(_1: *mut u32) -> () { + let mut _0: (); + + bb0: { + // early: [_1] + // late: [] + (*_1) = const 1_u32; + // early: [] + // late: [] + return; + } +} diff --git a/tests/mir-opt/precise_liveness.fields.dataflow.0.mir b/tests/mir-opt/precise_liveness.fields.dataflow.0.mir new file mode 100644 index 0000000000000..2734ebac6301f --- /dev/null +++ b/tests/mir-opt/precise_liveness.fields.dataflow.0.mir @@ -0,0 +1,25 @@ +// MIR for `fields` 0 dataflow + +fn fields() -> u32 { + let mut _0: u32; + let mut _1: (u32, u32); + let mut _2: *const (u32, u32); + + bb0: { + // early: [] + // late: [_1] + (_1.0: u32) = const 1_u32; + // early: [_1] + // late: [_1] + (_1.1: u32) = const 2_u32; + // early: [_1] + // late: [_1, _2] + _2 = &raw const _1; + // early: [_1] + // late: [_0, _1] + _0 = Add(move (_1.0: u32), move (_1.1: u32)); + // early: [_0] + // late: [] + return; + } +} diff --git a/tests/mir-opt/precise_liveness.indexed.dataflow.0.mir b/tests/mir-opt/precise_liveness.indexed.dataflow.0.mir new file mode 100644 index 0000000000000..e36103a330aab --- /dev/null +++ b/tests/mir-opt/precise_liveness.indexed.dataflow.0.mir @@ -0,0 +1,15 @@ +// MIR for `indexed` 0 dataflow + +fn indexed(_1: usize) -> () { + let mut _0: (); + let mut _2: [u32; 2]; + + bb0: { + // early: [_1] + // late: [_2] + _2[_1] = const 1_u32; + // early: [] + // late: [] + return; + } +} diff --git a/tests/mir-opt/precise_liveness.rs b/tests/mir-opt/precise_liveness.rs new file mode 100644 index 0000000000000..783da35d037d0 --- /dev/null +++ b/tests/mir-opt/precise_liveness.rs @@ -0,0 +1,357 @@ +//@ needs-unwind + +#![feature(core_intrinsics, custom_mir, rustc_attrs)] +#![crate_type = "lib"] + +use std::intrinsics::mir::*; + +// A non-borrowed local dies on its last use. +// EMIT_MIR precise_liveness.unborrowed.dataflow.0.mir +#[custom_mir(dialect = "runtime", phase = "optimized")] +#[rustc_mir(rustc_pretty_precise_liveness)] +pub fn unborrowed() -> u32 { + // CHECK-LABEL: fn unborrowed( + // CHECK: [[A:_[0-9]+]] = const 1_u32; + // CHECK: [[B:_[0-9]+]] = const 2_u32; + // CHECK: // early: [[[A]], [[B]]] + // CHECK-NEXT: // late: [_0] + // CHECK-NEXT: _0 = Add(copy [[A]], move [[B]]); + mir! { + let a: u32; + let b: u32; + { + a = 1; + b = 2; + RET = a + Move(b); + Return() + } + } +} + +// A borrowed local only dies on move. +// EMIT_MIR precise_liveness.borrowed.dataflow.0.mir +#[custom_mir(dialect = "runtime", phase = "optimized")] +#[rustc_mir(rustc_pretty_precise_liveness)] +pub fn borrowed() -> u32 { + // CHECK-LABEL: fn borrowed( + // CHECK: [[A:_[0-9]+]] = const 1_u32; + // CHECK: [[B:_[0-9]+]] = const 2_u32; + // CHECK: = &raw const [[B]]; + // CHECK-NEXT: // early: [[[A]], [[B]]] + // CHECK-NEXT: // late: [_0, [[A]]] + // CHECK-NEXT: _0 = Add(copy [[A]], move [[B]]); + mir! { + let a: u32; + let b: u32; + let a_ptr: *const u32; + let b_ptr: *const u32; + { + a = 1; + b = 2; + a_ptr = &raw const a; + b_ptr = &raw const b; + RET = a + Move(b); + Return() + } + } +} + +// StorageDead kills a lifetime, but it can be re-initialized later. +// EMIT_MIR precise_liveness.storage.dataflow.0.mir +#[custom_mir(dialect = "runtime", phase = "optimized")] +#[rustc_mir(rustc_pretty_precise_liveness)] +pub fn storage() -> u32 { + // CHECK-LABEL: fn storage( + // CHECK: // early: [_0, [[BORROWED:_[0-9]+]]] + // CHECK-NEXT: // late: [_0] + // CHECK-NEXT: StorageDead([[BORROWED]]); + // CHECK-NEXT: // early: [_0] + // CHECK-NEXT: // late: [_0] + // CHECK-NEXT: StorageLive([[BORROWED]]); + // CHECK-NEXT: // early: [_0] + // CHECK-NEXT: // late: [_0, [[BORROWED]]] + // CHECK-NEXT: [[BORROWED]] = const 3_u32; + // CHECK-NEXT: // early: [_0, [[BORROWED]]] + // CHECK-NEXT: // late: [_0] + // CHECK-NEXT: StorageDead([[BORROWED]]); + // CHECK-NEXT: // early: [_0] + // CHECK-NEXT: // late: [] + // CHECK-NEXT: return; + mir! { + let borrowed: u32; + let pointer: *const u32; + { + StorageLive(borrowed); + borrowed = 2; + pointer = &raw const borrowed; + RET = borrowed; + StorageDead(borrowed); + StorageLive(borrowed); + borrowed = 3; + StorageDead(borrowed); + Return() + } + } +} + +// Partial initialization starts a lifetime, but only a bare local move ends it. +// EMIT_MIR precise_liveness.fields.dataflow.0.mir +#[custom_mir(dialect = "runtime", phase = "optimized")] +#[rustc_mir(rustc_pretty_precise_liveness)] +pub fn fields() -> u32 { + // CHECK-LABEL: fn fields( + // CHECK: // early: [] + // CHECK-NEXT: // late: [[[AGGREGATE:_[0-9]+]]] + // CHECK-NEXT: ([[AGGREGATE]].0: u32) = const 1_u32; + // CHECK: = &raw const [[AGGREGATE]]; + // CHECK-NEXT: // early: [[[AGGREGATE]]] + // CHECK-NEXT: // late: [_0, [[AGGREGATE]]] + // CHECK-NEXT: _0 = Add(move ([[AGGREGATE]].0: u32), move ([[AGGREGATE]].1: u32)); + mir! { + let aggregate: (u32, u32); + let pointer: *const (u32, u32); + { + aggregate.0 = 1; + aggregate.1 = 2; + pointer = &raw const aggregate; + RET = Move(aggregate.0) + Move(aggregate.1); + Return() + } + } +} + +unsafe extern "C-unwind" { + fn callee(a: u32, b: u32) -> u32; +} + +// Check the behavior around calls: +// - Destination is initialized at the late point. +// - Move arguments have their live range extended to the late point. +// - Copy arguments die at the early point when this is their last use. +// - Destination remains live on unwind edges. +// EMIT_MIR precise_liveness.call.dataflow.0.mir +#[custom_mir(dialect = "runtime", phase = "optimized")] +#[rustc_mir(rustc_pretty_precise_liveness)] +pub fn call() -> u32 { + // CHECK-LABEL: fn call( + // CHECK: [[A:_[0-9]+]] = const 1_u32; + // CHECK: [[B:_[0-9]+]] = const 2_u32; + // CHECK-NEXT: // early: [[[A]], [[B]]] + // CHECK-NEXT: // late: [[[A]], [[DESTINATION:_[0-9]+]]] + // CHECK-NEXT: [[DESTINATION]] = callee(move [[A]], copy [[B]]) + // CHECK: = &raw const [[DESTINATION]]; + // CHECK-NEXT: // early: [[[DESTINATION]]] + // CHECK-NEXT: // late: [_0] + // CHECK-NEXT: _0 = move [[DESTINATION]]; + // CHECK: (cleanup): { + // CHECK-NEXT: // early: [[[DESTINATION]]] + // CHECK-NEXT: // late: [[[DESTINATION]]] + // CHECK-NEXT: resume; + mir! { + let a: u32; + let b: u32; + let destination: u32; + let pointer: *const u32; + { + a = 1; + b = 2; + Call(destination = callee(Move(a), b), ReturnTo(done), UnwindCleanup(cleanup)) + } + done = { + pointer = &raw const destination; + RET = Move(destination); + Return() + } + cleanup(cleanup) = { + UnwindResume() + } + } +} + +// A value used on only one branch is not live on the other branch. +// EMIT_MIR precise_liveness.branch.dataflow.0.mir +#[custom_mir(dialect = "runtime", phase = "optimized")] +#[rustc_mir(rustc_pretty_precise_liveness)] +pub fn branch(condition: bool) -> u32 { + // CHECK-LABEL: fn branch( + // CHECK: [[VALUE:_[0-9]+]] = const 1_u32; + // CHECK-NEXT: // early: [[[CONDITION:_[0-9]+]], [[VALUE]]] + // CHECK-NEXT: // late: [[[VALUE]]] + // CHECK-NEXT: switchInt(copy [[CONDITION]]) -> [1: [[USE:bb[0-9]+]], otherwise: [[SKIP:bb[0-9]+]]]; + // CHECK: [[USE]]: { + // CHECK-NEXT: // early: [[[VALUE]]] + // CHECK-NEXT: // late: [_0] + // CHECK-NEXT: _0 = move [[VALUE]]; + // CHECK: [[SKIP]]: { + // CHECK-NEXT: // early: [] + // CHECK-NEXT: // late: [_0] + // CHECK-NEXT: _0 = const 0_u32; + mir! { + let value: u32; + { + value = 1; + match condition { true => use_value, _ => skip_value } + } + use_value = { + RET = Move(value); + Return() + } + skip_value = { + RET = 0; + Return() + } + } +} + +// Dead destinations are live only at the late point of the assignment or call. +// EMIT_MIR precise_liveness.dead_destinations.dataflow.0.mir +#[custom_mir(dialect = "runtime", phase = "optimized")] +#[rustc_mir(rustc_pretty_precise_liveness)] +pub fn dead_destinations() -> u32 { + // CHECK-LABEL: fn dead_destinations( + // CHECK: // early: [] + // CHECK-NEXT: // late: [[[ASSIGNMENT:_[0-9]+]]] + // CHECK-NEXT: [[ASSIGNMENT]] = const 1_u32; + // CHECK-NEXT: // early: [] + // CHECK-NEXT: // late: [[[DESTINATION:_[0-9]+]]] + // CHECK-NEXT: [[DESTINATION]] = callee(const 1_u32, const 2_u32) -> [return: [[DONE:bb[0-9]+]], unwind: [[CLEANUP:bb[0-9]+]]]; + // CHECK: [[DONE]]: { + // CHECK-NEXT: // early: [] + // CHECK-NEXT: // late: [_0] + // CHECK-NEXT: _0 = const 0_u32; + // CHECK: [[CLEANUP]] (cleanup): { + // CHECK-NEXT: // early: [] + // CHECK-NEXT: // late: [] + // CHECK-NEXT: resume; + mir! { + let assignment: u32; + let destination: u32; + { + assignment = 1; + Call(destination = callee(1, 2), ReturnTo(done), UnwindCleanup(cleanup)) + } + done = { + RET = 0; + Return() + } + cleanup(cleanup) = { + UnwindResume() + } + } +} + +// An indexed write allocates the array. The index is a use, not a destination. +// EMIT_MIR precise_liveness.indexed.dataflow.0.mir +#[custom_mir(dialect = "runtime", phase = "optimized")] +#[rustc_mir(rustc_pretty_precise_liveness)] +pub fn indexed(index: usize) { + // CHECK-LABEL: fn indexed( + // CHECK: // early: [[[INDEX:_[0-9]+]]] + // CHECK-NEXT: // late: [[[ARRAY:_[0-9]+]]] + // CHECK-NEXT: [[ARRAY]][[[INDEX]]] = const 1_u32; + mir! { + let array: [u32; 2]; + { + array[index] = 1; + Return() + } + } +} + +// A write through a pointer is a use of the pointer, not a def. +// EMIT_MIR precise_liveness.dereference.dataflow.0.mir +#[custom_mir(dialect = "runtime", phase = "optimized")] +#[rustc_mir(rustc_pretty_precise_liveness)] +pub fn dereference(pointer: *mut u32) { + // CHECK-LABEL: fn dereference( + // CHECK: // early: [[[POINTER:_[0-9]+]]] + // CHECK-NEXT: // late: [] + // CHECK-NEXT: (*[[POINTER]]) = const 1_u32; + mir! { + { + *pointer = 1; + Return() + } + } +} + +// A field move keeps the base allocation live through the call's late point. +// EMIT_MIR precise_liveness.call_field.dataflow.0.mir +#[custom_mir(dialect = "runtime", phase = "optimized")] +#[rustc_mir(rustc_pretty_precise_liveness)] +pub fn call_field() -> u32 { + // CHECK-LABEL: fn call_field( + // CHECK: ([[AGGREGATE:_[0-9]+]].0: u32) = const 1_u32; + // CHECK-NEXT: // early: [[[AGGREGATE]]] + // CHECK-NEXT: // late: [_0, [[AGGREGATE]]] + // CHECK-NEXT: _0 = callee(move ([[AGGREGATE]].0: u32), const 0_u32) + // CHECK: // early: [_0] + // CHECK-NEXT: // late: [] + // CHECK-NEXT: return; + mir! { + let aggregate: (u32,); + { + aggregate.0 = 1; + Call(RET = callee(Move(aggregate.0), 0), ReturnTo(done), UnwindUnreachable()) + } + done = { + Return() + } + } +} + +// A borrowed field's base remains live in both successors after a field move. +// EMIT_MIR precise_liveness.call_borrowed_field.dataflow.0.mir +#[custom_mir(dialect = "runtime", phase = "optimized")] +#[rustc_mir(rustc_pretty_precise_liveness)] +pub fn call_borrowed_field() -> u32 { + // CHECK-LABEL: fn call_borrowed_field( + // CHECK: = &raw const [[AGGREGATE:_[0-9]+]]; + // CHECK-NEXT: // early: [[[AGGREGATE]]] + // CHECK-NEXT: // late: [[[AGGREGATE]], [[DESTINATION:_[0-9]+]]] + // CHECK-NEXT: [[DESTINATION]] = callee(move ([[AGGREGATE]].0: u32), const 0_u32) -> [return: [[DONE:bb[0-9]+]], unwind: [[CLEANUP:bb[0-9]+]]]; + // CHECK: [[DONE]]: { + // CHECK-NEXT: // early: [[[AGGREGATE]], [[DESTINATION]]] + // CHECK-NEXT: // late: [_0, [[AGGREGATE]]] + // CHECK-NEXT: _0 = move [[DESTINATION]]; + // CHECK: [[CLEANUP]] (cleanup): { + // CHECK-NEXT: // early: [[[AGGREGATE]]] + // CHECK-NEXT: // late: [[[AGGREGATE]]] + // CHECK-NEXT: resume; + mir! { + let aggregate: (u32,); + let destination: u32; + let borrow: *const (u32,); + { + aggregate.0 = 1; + borrow = &raw const aggregate; + Call(destination = callee(Move(aggregate.0), 0), ReturnTo(done), UnwindCleanup(cleanup)) + } + done = { + RET = Move(destination); + Return() + } + cleanup(cleanup) = { + UnwindResume() + } + } +} + +// An indirect move does not extend the pointer local's lifetime to the late point. +// EMIT_MIR precise_liveness.call_indirect.dataflow.0.mir +#[custom_mir(dialect = "runtime", phase = "optimized")] +#[rustc_mir(rustc_pretty_precise_liveness)] +pub fn call_indirect(pointer: *const u32) -> u32 { + // CHECK-LABEL: fn call_indirect( + // CHECK: // early: [[[POINTER:_[0-9]+]]] + // CHECK-NEXT: // late: [_0] + // CHECK-NEXT: _0 = callee(move (*[[POINTER]]), const 0_u32) + mir! { + { + Call(RET = callee(Move(*pointer), 0), ReturnTo(done), UnwindUnreachable()) + } + done = { + Return() + } + } +} diff --git a/tests/mir-opt/precise_liveness.storage.dataflow.0.mir b/tests/mir-opt/precise_liveness.storage.dataflow.0.mir new file mode 100644 index 0000000000000..56ae734839ee9 --- /dev/null +++ b/tests/mir-opt/precise_liveness.storage.dataflow.0.mir @@ -0,0 +1,37 @@ +// MIR for `storage` 0 dataflow + +fn storage() -> u32 { + let mut _0: u32; + let mut _1: u32; + let mut _2: *const u32; + + bb0: { + // early: [] + // late: [] + StorageLive(_1); + // early: [] + // late: [_1] + _1 = const 2_u32; + // early: [_1] + // late: [_1, _2] + _2 = &raw const _1; + // early: [_1] + // late: [_0, _1] + _0 = copy _1; + // early: [_0, _1] + // late: [_0] + StorageDead(_1); + // early: [_0] + // late: [_0] + StorageLive(_1); + // early: [_0] + // late: [_0, _1] + _1 = const 3_u32; + // early: [_0, _1] + // late: [_0] + StorageDead(_1); + // early: [_0] + // late: [] + return; + } +} diff --git a/tests/mir-opt/precise_liveness.unborrowed.dataflow.0.mir b/tests/mir-opt/precise_liveness.unborrowed.dataflow.0.mir new file mode 100644 index 0000000000000..4a27e7b0f9d5a --- /dev/null +++ b/tests/mir-opt/precise_liveness.unborrowed.dataflow.0.mir @@ -0,0 +1,22 @@ +// MIR for `unborrowed` 0 dataflow + +fn unborrowed() -> u32 { + let mut _0: u32; + let mut _1: u32; + let mut _2: u32; + + bb0: { + // early: [] + // late: [_1] + _1 = const 1_u32; + // early: [_1] + // late: [_1, _2] + _2 = const 2_u32; + // early: [_1, _2] + // late: [_0] + _0 = Add(copy _1, move _2); + // early: [_0] + // late: [] + return; + } +} From b41e9fc10ea0d663a018c4c2f6821edd63c56e0c Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Wed, 23 Sep 2026 04:02:28 +0100 Subject: [PATCH 4/5] Remove incorrect comments about Rvalue::Aggregate not being allowed --- compiler/rustc_middle/src/mir/syntax.rs | 4 ---- compiler/rustc_public/src/mir/body.rs | 3 --- 2 files changed, 7 deletions(-) diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 487c2c43da7d5..b0462a8c2bd28 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -127,7 +127,6 @@ pub enum RuntimePhase { /// disallowed: /// * [`TerminatorKind::Yield`] /// * [`TerminatorKind::CoroutineDrop`] - /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array` /// * [`Rvalue::CopyForDeref`] /// * [`PlaceElem::OpaqueCast`] /// * [`LocalInfo::DerefTemp`](super::LocalInfo::DerefTemp) @@ -1449,9 +1448,6 @@ pub enum Rvalue<'tcx> { /// This is needed because dataflow analysis needs to distinguish /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo` /// has a destructor. - /// - /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After - /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too. Aggregate(Box>, IndexVec>), /// A CopyForDeref is equivalent to a read from a place at the diff --git a/compiler/rustc_public/src/mir/body.rs b/compiler/rustc_public/src/mir/body.rs index a65798aff1a00..f4e93bd05a20c 100644 --- a/compiler/rustc_public/src/mir/body.rs +++ b/compiler/rustc_public/src/mir/body.rs @@ -551,9 +551,6 @@ pub enum Rvalue { /// This is needed because dataflow analysis needs to distinguish /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo` /// has a destructor. - /// - /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After - /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too. Aggregate(AggregateKind, Vec), /// * `Offset` has the same semantics as `<*const T>::offset`, except that the second From 785792f0ceceab913ef78a7084da6fdc6b38b128 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Wed, 23 Sep 2026 04:02:30 +0100 Subject: [PATCH 5/5] Add MIR move elimination pass --- compiler/rustc_index/src/interval.rs | 16 + compiler/rustc_mir_transform/src/dest_prop.rs | 4 +- compiler/rustc_mir_transform/src/lib.rs | 2 + .../src/move_elimination.rs | 898 ++++++++++++++++++ ...indirect_source_alias.MoveElimination.diff | 27 + ..._fixup.aggregate_swap.MoveElimination.diff | 84 ++ ...ed_aggregate_aliasing.MoveElimination.diff | 90 ++ tests/mir-opt/move-elimination/alias_fixup.rs | 95 ++ ....simple_partial_alias.MoveElimination.diff | 52 + ...basic.array_aggregate.MoveElimination.diff | 67 ++ .../basic.enum_aggregate.MoveElimination.diff | 57 ++ .../basic.nrvo_borrowed.MoveElimination.diff | 46 + ...basic.nrvo_unborrowed.MoveElimination.diff | 24 + tests/mir-opt/move-elimination/basic.rs | 80 ++ ...asic.struct_aggregate.MoveElimination.diff | 49 + .../dse.dse_guard.MoveElimination.diff | 92 ++ tests/mir-opt/move-elimination/dse.rs | 36 + ...x_local_not_projected.MoveElimination.diff | 16 + ...overlapping_lifetimes.MoveElimination.diff | 126 +++ ..._fields_not_projected.MoveElimination.diff | 48 + tests/mir-opt/move-elimination/exclusions.rs | 118 +++ ...l_tuple_not_projected.MoveElimination.diff | 72 ++ ...d_field_not_projected.MoveElimination.diff | 30 + ...wind.call_destination.MoveElimination.diff | 27 + tests/mir-opt/move-elimination/unwind.rs | 41 + 25 files changed, 2196 insertions(+), 1 deletion(-) create mode 100644 compiler/rustc_mir_transform/src/move_elimination.rs create mode 100644 tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/alias_fixup.rs create mode 100644 tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.rs create mode 100644 tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/dse.rs create mode 100644 tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.rs create mode 100644 tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/unwind.call_destination.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/unwind.rs diff --git a/compiler/rustc_index/src/interval.rs b/compiler/rustc_index/src/interval.rs index b7b1531e50857..d0dcb947c6d1c 100644 --- a/compiler/rustc_index/src/interval.rs +++ b/compiler/rustc_index/src/interval.rs @@ -374,6 +374,12 @@ impl SparseIntervalMatrix { self.rows.iter_enumerated() } + pub fn clear_row(&mut self, row: R) { + if let Some(row) = self.rows.get_mut(row) { + row.clear(); + } + } + fn ensure_row(&mut self, row: R) -> &mut IntervalSet { self.rows.ensure_contains_elem(row, || IntervalSet::new(self.column_size)) } @@ -397,6 +403,16 @@ impl SparseIntervalMatrix { write_row.union(read_row) } + pub fn disjoint_rows(&self, a: R, b: R) -> bool + where + C: Step, + { + let (Some(a), Some(b)) = (self.rows.get(a), self.rows.get(b)) else { + return true; + }; + a.disjoint(b) + } + pub fn insert_all_into_row(&mut self, row: R) { self.ensure_row(row).insert_all(); } diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 7f9bba03ff345..5ea98a8fb92f5 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -155,7 +155,9 @@ pub(super) struct DestinationPropagation; impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { fn policy(&self, ctx: &crate::PassCtx<'_>) -> PassPolicy { - PassPolicy::optional(ctx.mir_opt_level() >= 2) + PassPolicy::optional( + ctx.mir_opt_level() >= 2 && !ctx.opts.unstable_opts.mir_move_elimination, + ) } #[tracing::instrument(level = "trace", skip(self, tcx, body))] diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index d0b9c686c3037..697071a31429f 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -164,6 +164,7 @@ declare_passes! { mod lower_slice_len : LowerSliceLenCalls; mod match_branches : MatchBranchSimplification; mod mentioned_items : MentionedItems; + mod move_elimination : MoveElimination; mod multiple_return_terminators : MultipleReturnTerminators; mod post_drop_elaboration : CheckLiveDrops; mod prettify : ReorderBasicBlocks, ReorderLocals; @@ -763,6 +764,7 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' &dead_store_elimination::DeadStoreElimination::Final, &dest_prop::DestinationPropagation, &tail_copy_to_move::TailCopyToMove, + &move_elimination::MoveElimination, &simplify::SimplifyLocals::Final, &multiple_return_terminators::MultipleReturnTerminators, // Some cleanup necessary at least for LLVM and potentially other codegen backends. diff --git a/compiler/rustc_mir_transform/src/move_elimination.rs b/compiler/rustc_mir_transform/src/move_elimination.rs new file mode 100644 index 0000000000000..1b9af52e642b4 --- /dev/null +++ b/compiler/rustc_mir_transform/src/move_elimination.rs @@ -0,0 +1,898 @@ +//! Eliminates copies and moves by unifying MIR places whose allocation ranges +//! are disjoint. +//! +//! See RFC 3943 for the local-lifetime semantics that make this optimization +//! possible. +//! +//! # Motivation +//! +//! MIR building can insert a lot of redundant copies, and Rust code in general +//! often tends to move values around a lot. The result is a lot of assignments +//! of the form `dest = {move} src;` in MIR. MIR building for constants in +//! particular tends to create additional locals that are only used inside a +//! single block to shuffle a value around unnecessarily. +//! +//! Additionally, Rust constructs nested aggregates by repeatedly moving values +//! into fields. For example, a function may build an inner value in a local, +//! move it into an outer aggregate, then move that aggregate into the caller's +//! destination. If these intermediate source and destination places have +//! different addresses, each layer needs an actual copy or move of the bytes. +//! +//! LLVM cannot remove these copies when both the source and destination +//! addresses are observed because merging the allocations would be an +//! observable change: the program could see that two addresses which were +//! previously distinct have become the same. This pass removes the copies +//! earlier, while MIR still has the information needed to prove that the two +//! allocation ranges do not overlap. +//! +//! # Optimization +//! +//! The basis of this optimization is place unification. If the source and +//! destination of an assignment have the same address, then the assignment is a +//! no-op. The same idea applies to aggregate construction: if a field operand +//! is already located at the corresponding field of the destination, then the +//! aggregate assignment does not need to copy those fields. +//! +//! The pass represents each unification as a mapping from a local to the place +//! that should replace it. Mappings are transitive, so `_3` can be resolved +//! through `_2.1` to `_1.0.1` if earlier mappings established those +//! relationships. +//! +//! The mapping is built by scanning the MIR for assignment statements. For +//! simple `Use` assignments, it tries to unify the source and destination +//! places. For `Aggregate` assignments, it tries to map each field operand to +//! the corresponding field in the assignment destination. Once all mappings +//! have been chosen, they are applied with one rewrite pass over the body. +//! +//! # Constraints +//! +//! Adding a mapping must preserve these conditions: +//! +//! * At least one side of the candidate pair must be a bare local. The pass can +//! map a local to a place with projections, but it cannot map between two +//! places that both already have projections. +//! +//! * Any projections in the mapped place must be stable everywhere the local is +//! used. `Deref` and `Index` projections are rejected because they may refer +//! to different memory at different points in the function. +//! +//! * The allocation ranges of the source and destination places must not +//! overlap. This is checked using `PreciseLiveness`, which computes the +//! points where each local must have a distinct allocation. The non-overlap +//! proof is required so that the operational semantics can allow both places +//! to have the same address. +//! +//! * Special-use locals such as arguments and the return place must keep their +//! roles. Temps may be mapped into an argument or return place, but two +//! special-use locals are not mapped into each other. +//! +//! * Some locals are used in contexts where projections cannot be added, such +//! as `Index` projections. These locals may only be replaced by another bare +//! local. +//! +//! # Storage lifetimes +//! +//! The original storage markers no longer describe the merged locals, so they +//! are removed. Locals without storage markers have storage for the full function. +//! +//! # Aliasing fixup +//! +//! MIR assignments currently require source and destination places not to +//! overlap for types that are not treated as scalars in codegen. After local +//! unification, some assignments may violate that invariant, so a final phase +//! rewrites them into a form codegen can handle. For each assignment: +//! +//! * Self-assignments, such as `_1 = _1`, are deleted. +//! +//! * Simple `Use` assignments whose source and destination overlap but are not +//! identical are routed through a temporary: the source is read into the +//! temporary first, and the temporary is then moved into the destination. +//! +//! * Aggregate assignments with any operand that aliases the destination are +//! decomposed into per-field assignments. Field self-assignments are dropped. +//! Other aliasing fields are read into temporaries first, then all +//! destination fields are written. For enum aggregates the discriminant is +//! set after fields are written. +//! +//! * Other rvalues, such as `Repeat` and `Cast`, are hoisted into a temporary +//! if any place they access aliases the destination. +//! +//! * Rvalues that operate only on scalar types, such as binary and unary ops, +//! `Discriminant`, `Ref`, and `RawPtr`, are left untouched because their +//! codegen does not rely on the no-aliasing assumption. + +use rustc_abi::{ExternAbi, FieldIdx, VariantIdx}; +use rustc_const_eval::util::most_packed_projection; +use rustc_index::IndexVec; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::interval::SparseIntervalMatrix; +use rustc_middle::mir::visit::{MutVisitor, NonUseContext, PlaceContext, VisitPlacesWith, Visitor}; +use rustc_middle::mir::*; +use rustc_middle::ty::{Ty, TyCtxt}; +use rustc_mir_dataflow::impls::{SplitPointIndex, dump_liveness_matrix, liveness_matrix}; +use rustc_mir_dataflow::points::DenseLocationMap; +use tracing::{debug, trace}; + +use crate::PassPolicy; +use crate::patch::MirPatch; + +pub(super) struct MoveElimination; + +impl<'tcx> crate::MirPass<'tcx> for MoveElimination { + fn policy(&self, ctx: &crate::PassCtx<'_>) -> PassPolicy { + PassPolicy::optional( + ctx.mir_opt_level() >= 2 && ctx.opts.unstable_opts.mir_move_elimination, + ) + } + + #[tracing::instrument(level = "trace", skip(self, tcx, body))] + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let def_id = body.source.def_id(); + trace!(?def_id); + + let points = DenseLocationMap::new(body); + let mut liveness_matrix = + liveness_matrix(tcx, body, &points, Some("MoveElimination.liveness")); + + dump_liveness_matrix(tcx, body, "MoveElimination.pre-liveness", &points, &liveness_matrix); + + let unprojectable_locals = UnprojectableLocals::find(body); + trace!(?unprojectable_locals); + + let rust_call_tuples = find_rust_call_tuples(tcx, body); + trace!(?rust_call_tuples); + + let remapped_locals = PlaceUnification::run( + tcx, + body, + &mut liveness_matrix, + unprojectable_locals, + rust_call_tuples, + ); + + apply_mappings(tcx, body, &remapped_locals); + + dump_liveness_matrix(tcx, body, "MoveElimination.post-liveness", &points, &liveness_matrix); + + apply_alias_fixup(tcx, body); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Unprojectable locals + +/// Set of locals which can only be replaced with another local, instead of +/// an arbitrary place. This is usually because it is used directly as a +/// `Local` outside of a place (e.g. `Index` projections). +#[derive(Debug)] +struct UnprojectableLocals { + locals: DenseBitSet, +} + +impl UnprojectableLocals { + fn find(body: &Body<'_>) -> DenseBitSet { + let mut out = Self { locals: DenseBitSet::new_empty(body.local_decls.len()) }; + + // Arguments and return places have fixed roles and cannot be replaced + // with projected locals. + out.locals.insert(RETURN_PLACE); + for arg in body.args_iter() { + out.locals.insert(arg); + } + + out.visit_body(body); + out.locals + } +} + +impl<'tcx> Visitor<'tcx> for UnprojectableLocals { + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) { + // We can't add more projections before a first position Deref projection. + if place.is_indirect() { + trace!( + "unprojectable local {:?} due to use as deref base at {location:?}", + place.local + ); + self.locals.insert(place.local); + } + + // Only call visit_local for projections, not the base local. + self.visit_projection(place.as_ref(), context, location); + } + + fn visit_local(&mut self, local: Local, context: PlaceContext, location: Location) { + // Storage markers are removed. + if let PlaceContext::NonUse(NonUseContext::StorageLive | NonUseContext::StorageDead) = + context + { + return; + } + + // If this is reached, it means that this is a bare local used outside + // of a place, which means it cannot be replaced with a projection of + // another local. + trace!("unprojectable local {local:?} at {location:?} ({context:?})"); + self.locals.insert(local); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// "rust-call" tuple handling + +/// Search for tuple locals passed to calls using the "rust-call" ABI. +/// +/// For rust-call ABI calls, caller-side MIR passes the logical arguments as a +/// tuple operand. We want to avoid remapping other locals into fields of that +/// tuple, especially if one of those locals is borrowed. +/// +/// Since the tuple itself is never borrowed, it is trivial for LLVM alias +/// analysis to see that accesses to one argument do not affect the others, but +/// merging the arguments into tuple fields from the start can hide that +/// independence. +fn find_rust_call_tuples<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> DenseBitSet { + let mut rust_call_tuples = DenseBitSet::new_empty(body.local_decls.len()); + + for block in body.basic_blocks.iter() { + let terminator = block.terminator(); + let (func, args) = match &terminator.kind { + TerminatorKind::Call { func, args, .. } + | TerminatorKind::TailCall { func, args, .. } => (func, args), + _ => continue, + }; + + let sig = func.ty(&body.local_decls, tcx).fn_sig(tcx); + if sig.abi() != ExternAbi::RustCall { + continue; + } + + let arg_tuple = args.last().expect("rust-call ABI requires a tuple argument"); + let (Operand::Copy(place) | Operand::Move(place)) = arg_tuple.node else { + continue; + }; + if let Some(local) = place.as_local() { + rust_call_tuples.insert(local); + } + } + + rust_call_tuples +} + +//////////////////////////////////////////////////////////////////////////////// +// Local unification + +struct PlaceUnification<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + body: &'a Body<'tcx>, + liveness_matrix: &'a mut SparseIntervalMatrix, + unprojectable_locals: DenseBitSet, + rust_call_tuples: DenseBitSet, + remapped_locals: IndexVec>>, +} + +impl<'tcx> PlaceUnification<'_, 'tcx> { + fn run( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + liveness_matrix: &mut SparseIntervalMatrix, + unprojectable_locals: DenseBitSet, + rust_call_tuples: DenseBitSet, + ) -> IndexVec>> { + let mut visitor = PlaceUnification { + tcx, + body, + liveness_matrix, + unprojectable_locals, + rust_call_tuples, + remapped_locals: IndexVec::from_elem_n(None, body.local_decls.len()), + }; + visitor.visit_body(body); + + // Finalize the mappings by transitively resolving all locals to their + // new final place. + for local in visitor.remapped_locals.indices() { + if let Some(place) = visitor.remapped_locals[local] { + let place = visitor.resolve_place(place); + visitor.remapped_locals[local] = Some(place); + trace!("Remapped {local:?} to {place:?}"); + } + } + + visitor.remapped_locals + } + + #[tracing::instrument(ret, level = "trace", skip(self))] + fn resolve_place(&self, mut place: Place<'tcx>) -> Place<'tcx> { + while let Some(new_place) = self.remapped_locals[place.local] { + place = new_place.project_deeper(place.projection, self.tcx); + } + place + } + + #[tracing::instrument(ret, level = "trace", skip(self))] + fn can_unify_places(&self, a: Place<'tcx>, b: Place<'tcx>) -> Option<(Local, Place<'tcx>)> { + let a = self.resolve_place(a); + let b = self.resolve_place(b); + + if a.local == b.local { + if a.projection != b.projection { + trace!("cannot unify same local with different projections"); + } + return None; + } + + if self.rust_call_tuples.contains(a.local) || self.rust_call_tuples.contains(b.local) { + trace!("cannot unify {a:?} and {b:?} involving a rust-call tuple argument"); + return None; + } + + let (local, place) = match (a.as_local(), b.as_local()) { + (None, None) => { + trace!("cannot unify 2 places that both have projections"); + return None; + } + (None, Some(b)) => { + if self.unprojectable_locals.contains(b) { + trace!("cannot unify {b:?} which cannot be projected"); + return None; + } + (b, a) + } + (Some(a), None) => { + if self.unprojectable_locals.contains(a) { + trace!("cannot unify {a:?} which cannot be projected"); + return None; + } + (a, b) + } + (Some(a), Some(b)) => match (self.body.local_kind(a), self.body.local_kind(b)) { + ( + LocalKind::Arg | LocalKind::ReturnPointer, + LocalKind::Arg | LocalKind::ReturnPointer, + ) => { + trace!("cannot unify {a:?} and {b:?} which are both arguments or return place"); + return None; + } + (LocalKind::Arg | LocalKind::ReturnPointer, LocalKind::Temp) => (b, a.into()), + (LocalKind::Temp, _) => (a, b.into()), + }, + }; + + if most_packed_projection(self.tcx, &self.body.local_decls, place).is_some() { + trace!("cannot unify {place:?} which has packed field projections"); + return None; + } + + if !self.liveness_matrix.disjoint_rows(local, place.local) { + trace!("cannot unify {a:?} and {b:?} which have overlapping live ranges"); + return None; + } + + // FIXME(#112651): This can be removed afterwards. + let local_ty = self.body.local_decls[local].ty; + let place_ty = place.ty(&self.body.local_decls, self.tcx).ty; + if local_ty != place_ty { + trace!( + "cannot unify {a:?} and {b:?} which have different types due to subtyping ({local_ty:?} vs {place_ty:?})" + ); + return None; + } + + Some((local, place)) + } + + #[tracing::instrument(level = "trace", skip(self))] + fn remap_local(&mut self, local: Local, place: Place<'tcx>) { + self.remapped_locals[local] = Some(place); + + self.liveness_matrix.union_rows(local, place.local); + self.liveness_matrix.clear_row(local); + + // If the original local was unprojectable then this now also applies to + // the mapped local. + if self.unprojectable_locals.contains(local) { + debug_assert!(place.projection.is_empty()); + self.unprojectable_locals.insert(place.local); + } + } + + fn visit_aggregate_assign( + &mut self, + dest: Place<'tcx>, + project_field: impl Fn(TyCtxt<'tcx>, Place<'tcx>, FieldIdx, Ty<'tcx>) -> Place<'tcx>, + operands: &IndexVec>, + location: Location, + ) { + // Attempt to unify each field operand with the corresponding field in + // the destination place. + let mut candidates = vec![]; + for (idx, operand) in operands.iter_enumerated() { + let (Operand::Copy(src) | Operand::Move(src)) = *operand else { + continue; + }; + let Some(src) = src.as_local() else { + continue; + }; + let dest = project_field(self.tcx, dest, idx, self.body.local_decls[src].ty); + trace!("Attempting to unify {dest:?} and {src:?} at {location:?}"); + if let Some((local, place)) = self.can_unify_places(dest, src.into()) { + candidates.push((local, place)); + } + } + + // Do the actual remapping *after* checking for live range overlaps. + // This is necessary because the input operands necessarily have + // overlapping live ranges. + for (local, place) in candidates { + self.remap_local(local, place); + } + } +} + +/// Since we are replacing all uses of a local with another place, we need to +/// ensure that the projections on that place are stable no matter where it is +/// used in the body. Additional this local may be used in debuginfo, so ensure +/// that the projections are compatible with usage in debuginfo. +fn check_projections(place: Place<'_>) -> bool { + place.projection.iter().all(|elem| elem.is_stable_offset() && elem.can_use_in_debuginfo()) +} + +impl<'tcx> Visitor<'tcx> for PlaceUnification<'_, 'tcx> { + fn visit_assign(&mut self, dest: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) { + if !check_projections(*dest) { + return; + } + match rvalue { + Rvalue::Use(Operand::Copy(src) | Operand::Move(src), _) => { + if !check_projections(*src) { + return; + } + + trace!("Attempting to unify {dest:?} and {src:?} at {location:?}"); + if let Some((local, place)) = self.can_unify_places(*src, *dest) { + self.remap_local(local, place); + } + } + Rvalue::Aggregate(aggregate_kind, operands) => match *aggregate_kind { + AggregateKind::Array(_) => self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, _field_ty| { + place.project_deeper( + &[PlaceElem::ConstantIndex { + offset: field_idx.as_u32().into(), + min_length: field_idx.as_u32() as u64 + 1, + from_end: false, + }], + tcx, + ) + }, + operands, + location, + ), + AggregateKind::Tuple => self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, field_ty| { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + }, + operands, + location, + ), + AggregateKind::Adt(_, _, _, _, Some(union_field_idx)) => { + debug_assert_eq!(operands.len(), 1); + self.visit_aggregate_assign( + *dest, + |tcx, place, _, field_ty| { + place + .project_deeper(&[PlaceElem::Field(union_field_idx, field_ty)], tcx) + }, + operands, + location, + ) + } + AggregateKind::Adt(adt_did, var_idx, _, _, None) => { + let def = self.tcx.adt_def(adt_did); + if def.repr().simd() { + // MCP#838 banned projections into SIMD types. + return; + } + self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, field_ty| { + if def.is_enum() { + place.project_deeper( + &[ + PlaceElem::Downcast(None, var_idx), + PlaceElem::Field(field_idx, field_ty), + ], + tcx, + ) + } else { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + } + }, + operands, + location, + ) + } + _ => {} + }, + _ => {} + }; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Apply place mappings to the MIR body. + +fn apply_mappings<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + remapped_locals: &IndexVec>>, +) { + let mut rewriter = PlaceUpdater { tcx, remapped_locals }; + rewriter.visit_body_preserves_cfg(body); +} + +struct PlaceUpdater<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + remapped_locals: &'a IndexVec>>, +} + +impl<'tcx> MutVisitor<'tcx> for PlaceUpdater<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_local(&mut self, local: &mut Local, context: PlaceContext, location: Location) { + if let Some(new_place) = self.remapped_locals[*local] { + trace!("replacing {local:?} with {new_place:?} at {location:?} ({context:?})"); + *local = new_place.as_local().expect("mapped place shouldn't have projections"); + } + } + + fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { + if let Some(new_place) = self.remapped_locals[place.local] { + trace!("replacing {place:?} with {new_place:?} at {location:?} ({context:?})"); + *place = new_place.project_deeper(place.projection, self.tcx) + } + + // Only call visit_local for projections, not the base local. + if let Some(new_projection) = self.process_projection(&place.projection, location) { + place.projection = self.tcx().mk_place_elems(&new_projection); + } + } + + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + match statement.kind { + // Remove storage lifetime markers because they no longer describe + // the merged locals. + StatementKind::StorageDead(_) | StatementKind::StorageLive(_) => { + statement.make_nop(true); + return; + } + _ => {} + } + + self.super_statement(statement, location); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Aliasing assignment fixup +// +// MIR assignments currently do not allow source and destination to alias, so +// fix this in post-processing. + +fn apply_alias_fixup<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let mut patcher = MirPatch::new(body); + let mut fixup = AliasFixup { tcx, local_decls: &body.local_decls, patcher: &mut patcher }; + for (block, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { + fixup.visit_basic_block_data(block, data); + } + patcher.apply(body); +} + +/// Returns whether 2 places alias, ignoring indirect places. +fn places_directly_alias<'tcx>( + tcx: TyCtxt<'tcx>, + local_decls: &IndexVec>, + a: Place<'tcx>, + b: Place<'tcx>, +) -> bool { + // This function doesn't handle indirect aliasing. + if a.local != b.local || a.is_indirect_first_projection() || b.is_indirect_first_projection() { + return false; + } + + for ((prefix, elem_a), (_, elem_b)) in a.iter_projections().zip(b.iter_projections()) { + // Continue until we find the first mismatching projection. + if elem_a == elem_b { + continue; + } + + match (elem_a, elem_b) { + // Disjoint fields don't alias except if they are union fields. + (PlaceElem::Field(_, _), PlaceElem::Field(_, _)) => { + let ty = prefix.ty(local_decls, tcx).ty; + return ty.is_union(); + } + + // Disjoint slice elements don't alias. + ( + PlaceElem::ConstantIndex { offset: offset_a, min_length: _, from_end: from_end_a }, + PlaceElem::ConstantIndex { offset: offset_b, min_length: _, from_end: from_end_b }, + ) if from_end_a == from_end_b && offset_a != offset_b => { + return false; + } + + // Conservatively assume the places may alias. + _ => return true, + } + } + + // If the projections are identical *or* one is a prefix of the other then + // the places alias. + true +} + +struct AliasFixup<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + local_decls: &'a IndexVec>, + patcher: &'a mut MirPatch<'tcx>, +} + +impl<'tcx> AliasFixup<'_, 'tcx> { + fn isolate_rvalue_to_local( + &mut self, + rvalue: Rvalue<'tcx>, + source_info: SourceInfo, + location: Location, + ) -> Place<'tcx> { + let ty = rvalue.ty(self.local_decls, self.tcx); + let temp = Place::from(self.patcher.new_temp(ty, source_info.span)); + trace!("isolating {rvalue:?} to {temp:?} due to conflict"); + self.patcher.add_statement(location, StatementKind::StorageLive(temp.local)); + self.patcher.add_assign(location, Place::from(temp), rvalue); + self.patcher.add_statement( + location.successor_within_block(), + StatementKind::StorageDead(temp.local), + ); + temp + } + + fn visit_aggregate_assign( + &mut self, + dest: Place<'tcx>, + enum_variant: Option, + project_field: impl Fn(TyCtxt<'tcx>, Place<'tcx>, FieldIdx, Ty<'tcx>) -> Place<'tcx>, + operands: &IndexVec>, + source_info: SourceInfo, + location: Location, + ) { + // Fast path: if no direct operand aliases the destination, we're done. + // + // We only look for direct aliases here, which is sufficient because we + // know the input MIR did not have any aliasing and we didn't introduce + // any indirect aliasing in this pass. + // + // If the destination place is indirect then it cannot be the start of + // a lifetime as per the RFC 3943 MIR semantics. This means that the + // lifetime of the underlying allocation must have started earlier, + // which overlaps the early point of the assignment statement. Therefore + // we couldn't have unified any source operand with this destination + // place. + // + // If the source place is indirect then a similar reasoning applies. The + // only exception is if there are multiple source places (e.g. + // aggregates). In that situation it's possible for an indirect source + // to overlap the destination if and only if there is also a direct + // source that overlaps it: + // + // _2 = &_1 + // _3 = (copy *_2, move _1) // _1 becomes _3.1 after unification + // + // We handle this here in 2 ways: if there is no direct alias, then + // we're fine. Otherwise, treat all indirect sources as potentially + // aliasing with the destination operand. + let has_direct_alias = operands.iter().any(|op| match op { + Operand::Copy(src) | Operand::Move(src) => { + places_directly_alias(self.tcx, self.local_decls, dest, *src) + } + Operand::Constant(_) | Operand::RuntimeChecks(_) => false, + }); + if !has_direct_alias { + return; + } + + debug!("splitting aggregate assignment at {location:?}"); + + // Split into per-field assignments. + let mut assignments = vec![]; + for (idx, op) in operands.iter_enumerated() { + let field_ty = op.ty(self.local_decls, self.tcx); + let dest_field = project_field(self.tcx, dest, idx, field_ty); + + let emit_op = match op { + Operand::Copy(src) | Operand::Move(src) => { + if *src == dest_field { + // Skip identity assignments. + continue; + } else if src.is_indirect_first_projection() + || places_directly_alias(self.tcx, self.local_decls, dest, *src) + { + // Partial alias: hoist the source to a temp first so the + // per-field write no longer overlaps the dest. Indirect + // sources also need hoisting here because they may point + // at one of the direct aliasing operands. + Operand::Move(self.isolate_rvalue_to_local( + Rvalue::Use(op.clone(), WithRetag::No), + source_info, + location, + )) + } else { + op.clone() + } + } + Operand::Constant(_) | Operand::RuntimeChecks(_) => op.clone(), + }; + assignments.push((dest_field, emit_op)); + } + + // Perform assignments *after* all aliasing fields have been read into + // temporary locals. + for (dest_field, emit_op) in assignments { + self.patcher.add_assign(location, dest_field, Rvalue::Use(emit_op, WithRetag::No)); + } + + // Delete the original aggregate assignment. + self.patcher.nop_statement(location); + + // For enum variants, set the discriminant after all field writes. + if let Some(variant_index) = enum_variant { + self.patcher.add_statement( + location, + StatementKind::SetDiscriminant { place: Box::new(dest), variant_index }, + ); + } + } +} + +impl<'tcx> MutVisitor<'tcx> for AliasFixup<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + // Fixup the MIR to remove aliasing assignments. + if let StatementKind::Assign((dest, rvalue)) = &mut statement.kind { + match *rvalue { + Rvalue::Use(Operand::Copy(src) | Operand::Move(src), with_retag) => { + if places_directly_alias(self.tcx, self.local_decls, *dest, src) { + if src == *dest { + debug!("{:?} turned into self-assignment, deleting", location); + statement.make_nop(true); + } else { + let temp = self.isolate_rvalue_to_local( + rvalue.clone(), + statement.source_info, + location, + ); + *rvalue = Rvalue::Use(Operand::Move(temp), with_retag); + } + } + } + Rvalue::Aggregate(AggregateKind::Array(_), ref mut operands) => self + .visit_aggregate_assign( + *dest, + None, + |tcx, place, field_idx, _field_ty| { + place.project_deeper( + &[PlaceElem::ConstantIndex { + offset: field_idx.as_u32().into(), + min_length: field_idx.as_u32() as u64 + 1, + from_end: false, + }], + tcx, + ) + }, + operands, + statement.source_info, + location, + ), + Rvalue::Aggregate(AggregateKind::Tuple, ref mut operands) => self + .visit_aggregate_assign( + *dest, + None, + |tcx, place, field_idx, field_ty| { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + }, + operands, + statement.source_info, + location, + ), + Rvalue::Aggregate( + AggregateKind::Adt(_, _, _, _, Some(union_field_idx)), + ref mut operands, + ) => { + debug_assert_eq!(operands.len(), 1); + self.visit_aggregate_assign( + *dest, + None, + |tcx, place, _, field_ty| { + place + .project_deeper(&[PlaceElem::Field(union_field_idx, field_ty)], tcx) + }, + operands, + statement.source_info, + location, + ) + } + Rvalue::Aggregate( + AggregateKind::Adt(adt_did, var_idx, _, _, None), + ref mut operands, + ) => { + let def = self.tcx.adt_def(adt_did); + if def.repr().simd() { + // MCP#838 banned projections into SIMD types. + return; + } + self.visit_aggregate_assign( + *dest, + def.is_enum().then_some(var_idx), + |tcx, place, field_idx, field_ty| { + if def.is_enum() { + place.project_deeper( + &[ + PlaceElem::Downcast(None, var_idx), + PlaceElem::Field(field_idx, field_ty), + ], + tcx, + ) + } else { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + } + }, + operands, + statement.source_info, + location, + ) + } + + // For other rvalues, don't try to split them into components + // and instead just introduce a temporary if there is any + // aliasing + Rvalue::Aggregate(..) + | Rvalue::Repeat(..) + | Rvalue::Cast(..) + | Rvalue::CopyForDeref(..) + | Rvalue::WrapUnsafeBinder(..) => { + let mut overlaps_dest = false; + VisitPlacesWith(|place, _ctxt| { + if places_directly_alias(self.tcx, self.local_decls, *dest, place) { + overlaps_dest = true; + } + }) + .visit_rvalue(rvalue, location); + if overlaps_dest { + let temp = self.isolate_rvalue_to_local( + rvalue.clone(), + statement.source_info, + location, + ); + *rvalue = Rvalue::Use(Operand::Move(temp), WithRetag::No); + } + } + + // These either cannot have aliasing, or allow it because they + // only operate on scalar backend types. + Rvalue::Use(Operand::Constant(..) | Operand::RuntimeChecks(..), _) + | Rvalue::Ref(..) + | Rvalue::ThreadLocalRef(..) + | Rvalue::BinaryOp(..) + | Rvalue::UnaryOp(..) + | Rvalue::Discriminant(..) + | Rvalue::RawPtr(..) + | Rvalue::Reborrow(..) => {} + } + } + } +} diff --git a/tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff new file mode 100644 index 0000000000000..1b4e6dbba1825 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff @@ -0,0 +1,27 @@ +- // MIR for `aggregate_indirect_source_alias` before MoveElimination ++ // MIR for `aggregate_indirect_source_alias` after MoveElimination + + fn aggregate_indirect_source_alias() -> (u8, u8) { + let mut _0: (u8, u8); + let mut _1: u8; + let mut _2: *const u8; + let mut _3: (u8, u8); ++ let mut _4: u8; + + bb0: { +- _1 = const 1_u8; +- _2 = &raw const _1; +- _3 = (copy (*_2), move _1); +- _0 = copy _3; ++ (_0.1: u8) = const 1_u8; ++ _2 = &raw const (_0.1: u8); ++ StorageLive(_4); ++ _4 = no_retag copy (*_2); ++ (_0.0: u8) = no_retag move _4; ++ nop; ++ StorageDead(_4); ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff new file mode 100644 index 0000000000000..d958b6d346d35 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff @@ -0,0 +1,84 @@ +- // MIR for `aggregate_swap` before MoveElimination ++ // MIR for `aggregate_swap` after MoveElimination + + fn aggregate_swap(_1: u8, _2: u8) -> (u8, u8) { + debug x => _1; + debug y => _2; + let mut _0: (u8, u8); + let mut _3: (u8, u8); + let mut _4: u8; + let mut _5: u8; + let mut _8: u8; + let mut _9: u8; ++ let mut _10: u8; + scope 1 { +- debug pair => _3; ++ debug pair => _0; + let _6: u8; + scope 2 { +- debug a => _6; ++ debug a => _9; + let _7: u8; + scope 3 { +- debug b => _7; ++ debug b => (_0.1: u8); + } + } + } + + bb0: { +- StorageLive(_3); +- StorageLive(_4); +- _4 = copy _1; +- StorageLive(_5); +- _5 = copy _2; +- _3 = (move _4, move _5); +- StorageDead(_5); +- StorageDead(_4); +- StorageLive(_6); +- _6 = copy (_3.0: u8); +- StorageLive(_7); +- _7 = copy (_3.1: u8); +- StorageLive(_8); +- _8 = copy _7; +- StorageLive(_9); +- _9 = copy _6; +- _3 = (move _8, move _9); +- StorageDead(_9); +- StorageDead(_8); +- _0 = copy _3; +- StorageDead(_7); +- StorageDead(_6); +- StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = (move _1, move _2); ++ nop; ++ nop; ++ nop; ++ _9 = copy (_0.0: u8); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_10); ++ _10 = no_retag move (_0.1: u8); ++ (_0.0: u8) = no_retag move _10; ++ (_0.1: u8) = no_retag move _9; ++ nop; ++ StorageDead(_10); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff new file mode 100644 index 0000000000000..0b3f73be637a0 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff @@ -0,0 +1,90 @@ +- // MIR for `mixed_aggregate_aliasing` before MoveElimination ++ // MIR for `mixed_aggregate_aliasing` after MoveElimination + + fn mixed_aggregate_aliasing(_1: bool, _2: u8) -> Triple { + debug flag => _1; + debug z => _2; + let mut _0: Triple; + let _3: Triple; + let mut _5: bool; + let mut _6: u8; + let mut _7: u8; + let mut _8: u8; ++ let mut _9: u8; + scope 1 { +- debug input => _3; ++ debug input => _0; + let _4: Triple; + scope 2 { +- debug out => _4; ++ debug out => _0; + } + } + + bb0: { +- StorageLive(_3); +- _3 = opaque_triple() -> [return: bb1, unwind unreachable]; ++ nop; ++ _0 = opaque_triple() -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageLive(_4); +- StorageLive(_5); +- _5 = copy _1; +- switchInt(move _5) -> [0: bb3, otherwise: bb2]; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb3, otherwise: bb2]; + } + + bb2: { +- _4 = copy _3; ++ nop; + goto -> bb4; + } + + bb3: { +- StorageLive(_6); +- _6 = copy (_3.1: u8); +- StorageLive(_7); +- _7 = copy (_3.0: u8); +- StorageLive(_8); +- _8 = copy _2; +- _4 = Triple(move _6, move _7, move _8); +- StorageDead(_8); +- StorageDead(_7); +- StorageDead(_6); ++ nop; ++ _6 = copy (_0.1: u8); ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_9); ++ _9 = no_retag move (_0.0: u8); ++ (_0.0: u8) = no_retag move _6; ++ (_0.1: u8) = no_retag move _9; ++ (_0.2: u8) = no_retag move _2; ++ nop; ++ StorageDead(_9); ++ nop; ++ nop; ++ nop; + goto -> bb4; + } + + bb4: { +- StorageDead(_5); +- _0 = copy _4; +- StorageDead(_4); +- StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.rs b/tests/mir-opt/move-elimination/alias_fixup.rs new file mode 100644 index 0000000000000..29d20a61488fa --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.rs @@ -0,0 +1,95 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort + +#![feature(core_intrinsics, custom_mir)] +#![allow(dead_code)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +#[derive(Copy, Clone)] +pub struct Triple(u8, u8, u8); + +pub union U { + a: [u8; 4], + b: [u8; 4], +} + +unsafe extern "C" { + safe fn opaque_triple() -> Triple; +} + +// EMIT_MIR alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff +pub fn mixed_aggregate_aliasing(flag: bool, z: u8) -> Triple { + // This checks an aggregate assignment on one branch after the other branch + // remaps the input into the return place: overlapping field reads are + // hoisted through temporaries before writing back into the return place. + // CHECK-LABEL: fn mixed_aggregate_aliasing( + // CHECK: debug z => _2; + // CHECK: debug input => _0; + // CHECK: debug out => _0; + // CHECK: [[field1:_.*]] = copy (_0.1: u8); + // CHECK: [[field0:_.*]] = no_retag move (_0.0: u8); + // CHECK: (_0.0: u8) = no_retag move [[field1]]; + // CHECK: (_0.1: u8) = no_retag move [[field0]]; + // CHECK: (_0.2: u8) = no_retag move _2; + let input = opaque_triple(); + let out = if flag { input } else { Triple(input.1, input.0, z) }; + out +} + +// EMIT_MIR alias_fixup.aggregate_swap.MoveElimination.diff +pub fn aggregate_swap(x: u8, y: u8) -> (u8, u8) { + // This checks that an aggregate swap-like assignment is safe after any + // remapping that makes source fields share storage with destination fields. + // CHECK-LABEL: fn aggregate_swap( + // CHECK: debug pair => _0; + // CHECK: [[saved:_.*]] = copy (_0.0: u8); + // CHECK: [[tmp:_.*]] = no_retag move (_0.1: u8); + // CHECK: (_0.0: u8) = no_retag move [[tmp]]; + // CHECK: (_0.1: u8) = no_retag move [[saved]]; + let mut pair = (x, y); + let a = pair.0; + let b = pair.1; + pair = (b, a); + pair +} + +// EMIT_MIR alias_fixup.simple_partial_alias.MoveElimination.diff +pub fn simple_partial_alias(x: [u8; 4]) -> U { + // This checks a non-aggregate assignment involving two same-typed union + // fields, which are conservatively treated as aliasing. + // CHECK-LABEL: fn simple_partial_alias( + // CHECK: debug u => _0; + // CHECK: _0 = U { a: move _1 }; + // CHECK: [[tmp:_.*]] = copy (_0.0: [u8; 4]); + // CHECK: (_0.1: [u8; 4]) = move [[tmp]]; + let mut u = U { a: x }; + let tmp = unsafe { u.a }; + u.b = tmp; + u +} + +// EMIT_MIR alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn aggregate_indirect_source_alias() -> (u8, u8) { + // This checks that, when an aggregate has a direct operand aliasing the + // destination, indirect operands are also hoisted before field writes. + // CHECK-LABEL: fn aggregate_indirect_source_alias( + // CHECK: [[p:_.*]] = &raw const (_0.1: u8); + // CHECK: [[tmp:_.*]] = no_retag copy (*[[p]]); + // CHECK: (_0.0: u8) = no_retag move [[tmp]]; + mir! { + let a: u8; + let p: *const u8; + let out: (u8, u8); + + { + a = 1_u8; + p = &raw const a; + out = (*p, Move(a)); + RET = out; + Return() + } + } +} diff --git a/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff new file mode 100644 index 0000000000000..e548e921f3142 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff @@ -0,0 +1,52 @@ +- // MIR for `simple_partial_alias` before MoveElimination ++ // MIR for `simple_partial_alias` after MoveElimination + + fn simple_partial_alias(_1: [u8; 4]) -> U { + debug x => _1; + let mut _0: U; + let mut _2: U; + let mut _3: [u8; 4]; + let mut _5: [u8; 4]; + scope 1 { +- debug u => _2; ++ debug u => _0; + let _4: [u8; 4]; + scope 2 { +- debug tmp => _4; ++ debug tmp => _5; + } + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- _2 = U { a: move _3 }; +- StorageDead(_3); +- StorageLive(_4); +- _4 = copy (_2.0: [u8; 4]); +- StorageLive(_5); +- _5 = copy _4; +- (_2.1: [u8; 4]) = move _5; +- StorageDead(_5); +- _0 = move _2; +- StorageDead(_4); +- StorageDead(_2); ++ nop; ++ nop; ++ nop; ++ _0 = U { a: move _1 }; ++ nop; ++ nop; ++ _5 = copy (_0.0: [u8; 4]); ++ nop; ++ nop; ++ (_0.1: [u8; 4]) = move _5; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..01938d10da6c5 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff @@ -0,0 +1,67 @@ +- // MIR for `array_aggregate` before MoveElimination ++ // MIR for `array_aggregate` after MoveElimination + + fn array_aggregate() -> [[u8; 8]; 3] { + let mut _0: [[u8; 8]; 3]; + let _1: [u8; 8]; + let mut _4: [u8; 8]; + let mut _5: [u8; 8]; + let mut _6: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => _0[0 of 1]; + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => _0[1 of 2]; + let _3: [u8; 8]; + scope 3 { +- debug c => _3; ++ debug c => _0[2 of 3]; + } + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- _3 = [const 3_u8; 8]; +- StorageLive(_4); +- _4 = move _1; +- StorageLive(_5); +- _5 = move _2; +- StorageLive(_6); +- _6 = move _3; +- _0 = [move _4, move _5, move _6]; +- StorageDead(_6); +- StorageDead(_5); +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ _0[0 of 1] = [const 1_u8; 8]; ++ nop; ++ _0[1 of 2] = [const 2_u8; 8]; ++ nop; ++ _0[2 of 3] = [const 3_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..5f40d8a3c82fd --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff @@ -0,0 +1,57 @@ +- // MIR for `enum_aggregate` before MoveElimination ++ // MIR for `enum_aggregate` after MoveElimination + + fn enum_aggregate() -> Result<([u8; 8], [u8; 8]), ()> { + let mut _0: std::result::Result<([u8; 8], [u8; 8]), ()>; + let _1: [u8; 8]; + let mut _3: ([u8; 8], [u8; 8]); + let mut _4: [u8; 8]; + let mut _5: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]); + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]); + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- StorageLive(_4); +- _4 = move _1; +- StorageLive(_5); +- _5 = move _2; +- _3 = (move _4, move _5); +- StorageDead(_5); +- StorageDead(_4); +- _0 = Result::<([u8; 8], [u8; 8]), ()>::Ok(move _3); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]) = [const 1_u8; 8]; ++ nop; ++ (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]) = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ discriminant(_0) = 0; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff new file mode 100644 index 0000000000000..6762e06029772 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff @@ -0,0 +1,46 @@ +- // MIR for `nrvo_borrowed` before MoveElimination ++ // MIR for `nrvo_borrowed` after MoveElimination + + fn nrvo_borrowed() -> [u8; 8] { + let mut _0: [u8; 8]; + let mut _1: [u8; 8]; + let _2: (); + let mut _3: &mut [u8; 8]; + let mut _4: &mut [u8; 8]; + scope 1 { +- debug buf => _1; ++ debug buf => _0; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- StorageLive(_3); +- StorageLive(_4); +- _4 = &mut _1; ++ nop; ++ _0 = [const 1_u8; 8]; ++ nop; ++ nop; ++ nop; ++ _4 = &mut _0; + _3 = &mut (*_4); + _2 = init(move _3) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_3); +- StorageDead(_4); +- StorageDead(_2); +- _0 = move _1; +- StorageDead(_1); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff new file mode 100644 index 0000000000000..7a44d4b598211 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff @@ -0,0 +1,24 @@ +- // MIR for `nrvo_unborrowed` before MoveElimination ++ // MIR for `nrvo_unborrowed` after MoveElimination + + fn nrvo_unborrowed() -> [u8; 8] { + let mut _0: [u8; 8]; + let _1: [u8; 8]; + scope 1 { +- debug buf => _1; ++ debug buf => _0; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- _0 = move _1; +- StorageDead(_1); ++ nop; ++ _0 = [const 1_u8; 8]; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.rs b/tests/mir-opt/move-elimination/basic.rs new file mode 100644 index 0000000000000..96f1776bebef1 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.rs @@ -0,0 +1,80 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort -Zmir-enable-passes=+TailCopyToMove + +struct Pair { + a: [u8; 8], + b: [u8; 8], +} + +fn init(_: &mut [u8; 8]) {} + +// EMIT_MIR basic.nrvo_unborrowed.MoveElimination.diff +pub fn nrvo_unborrowed() -> [u8; 8] { + // This checks the simplest NRVO-style case: the local should be merged with + // the return place even though it has `Copy` type. + // CHECK-LABEL: fn nrvo_unborrowed( + // CHECK: debug buf => _0; + // CHECK: _0 = [const 1_u8; 8] + let buf = [1; 8]; + buf +} + +// EMIT_MIR basic.nrvo_borrowed.MoveElimination.diff +pub fn nrvo_borrowed() -> [u8; 8] { + // This checks that taking a temporary mutable borrow does not prevent + // merging a `Copy` local once the borrow has ended. + // CHECK-LABEL: fn nrvo_borrowed( + // CHECK: debug buf => _0; + // CHECK: _0 = [const 1_u8; 8] + // CHECK: init(move {{_.*}}) + // CHECK-NOT: _0 = move + let mut buf = [1; 8]; + init(&mut buf); + buf +} + +// EMIT_MIR basic.struct_aggregate.MoveElimination.diff +pub fn struct_aggregate() -> Pair { + // This checks aggregate field remapping: the field locals can live directly + // in the return place's fields. + // CHECK-LABEL: fn struct_aggregate( + // CHECK: debug a => (_0.0: [u8; 8]); + // CHECK: debug b => (_0.1: [u8; 8]); + // CHECK: (_0.0: [u8; 8]) = [const 1_u8; 8]; + // CHECK: (_0.1: [u8; 8]) = [const 2_u8; 8]; + let a = [1; 8]; + let b = [2; 8]; + Pair { a, b } +} + +// EMIT_MIR basic.enum_aggregate.MoveElimination.diff +pub fn enum_aggregate() -> Result<([u8; 8], [u8; 8]), ()> { + // This checks aggregate field remapping for enums: the payload fields can + // be written directly and then the discriminant is set for the variant. + // CHECK-LABEL: fn enum_aggregate( + // CHECK: debug a => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]); + // CHECK: debug b => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]); + // CHECK: (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]) = [const 1_u8; 8]; + // CHECK: (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]) = [const 2_u8; 8]; + // CHECK: discriminant(_0) = 0; + let a = [1; 8]; + let b = [2; 8]; + Result::Ok((a, b)) +} + +// EMIT_MIR basic.array_aggregate.MoveElimination.diff +pub fn array_aggregate() -> [[u8; 8]; 3] { + // This checks aggregate remapping for arrays, which uses ConstantIndex + // projections rather than field projections. + // CHECK-LABEL: fn array_aggregate( + // CHECK: debug a => _0[0 of 1]; + // CHECK: debug b => _0[1 of 2]; + // CHECK: debug c => _0[2 of 3]; + // CHECK: _0[0 of 1] = [const 1_u8; 8]; + // CHECK: _0[1 of 2] = [const 2_u8; 8]; + // CHECK: _0[2 of 3] = [const 3_u8; 8]; + let a = [1; 8]; + let b = [2; 8]; + let c = [3; 8]; + [a, b, c] +} diff --git a/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..b03c4d2f3cff9 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff @@ -0,0 +1,49 @@ +- // MIR for `struct_aggregate` before MoveElimination ++ // MIR for `struct_aggregate` after MoveElimination + + fn struct_aggregate() -> Pair { + let mut _0: Pair; + let _1: [u8; 8]; + let mut _3: [u8; 8]; + let mut _4: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => (_0.0: [u8; 8]); + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => (_0.1: [u8; 8]); + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- _3 = move _1; +- StorageLive(_4); +- _4 = move _2; +- _0 = Pair { a: move _3, b: move _4 }; +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ (_0.0: [u8; 8]) = [const 1_u8; 8]; ++ nop; ++ (_0.1: [u8; 8]) = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff b/tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff new file mode 100644 index 0000000000000..2f3019364f343 --- /dev/null +++ b/tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff @@ -0,0 +1,92 @@ +- // MIR for `dse_guard` before MoveElimination ++ // MIR for `dse_guard` after MoveElimination + + fn dse_guard() -> () { + let mut _0: (); + let mut _1: Fields; + let mut _3: Fields; + let mut _4: Fields; + let _5: (); + let mut _6: *const Fields; + let mut _7: Fields; + let _8: (); + let mut _9: *const Fields; + scope 1 { +- debug a => _1; ++ debug a => _7; + let mut _2: Fields; + scope 2 { + debug b => _2; + } + } + + bb0: { +- StorageLive(_1); +- StorageLive(_2); +- StorageLive(_3); +- _3 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ _2 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- _2 = move _3; +- StorageDead(_3); +- StorageLive(_4); +- _4 = make_fields(const 1_u8) -> [return: bb2, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ _7 = make_fields(const 1_u8) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- _1 = move _4; +- StorageDead(_4); +- StorageLive(_5); +- StorageLive(_6); +- _6 = &raw const _1; ++ nop; ++ nop; ++ nop; ++ nop; ++ _6 = &raw const _7; + _5 = observe(move _6) -> [return: bb3, unwind unreachable]; + } + + bb3: { +- StorageDead(_6); +- StorageDead(_5); +- StorageLive(_7); +- _7 = move _1; ++ nop; ++ nop; ++ nop; ++ nop; + _2 = move _7; +- StorageDead(_7); +- StorageLive(_8); +- StorageLive(_9); ++ nop; ++ nop; ++ nop; + _9 = &raw const _2; + _8 = observe(move _9) -> [return: bb4, unwind unreachable]; + } + + bb4: { +- StorageDead(_9); +- StorageDead(_8); ++ nop; ++ nop; + _0 = const (); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/dse.rs b/tests/mir-opt/move-elimination/dse.rs new file mode 100644 index 0000000000000..77685fb533fdd --- /dev/null +++ b/tests/mir-opt/move-elimination/dse.rs @@ -0,0 +1,36 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort -Zmir-enable-passes=+DeadStoreElimination-initial + +pub struct Fields { + data: [u8; 8], + tag: u8, +} + +unsafe extern "C" { + safe fn observe(_: *const Fields); + safe fn make_fields(_: u8) -> Fields; +} + +// EMIT_MIR dse.dse_guard.MoveElimination.diff +pub fn dse_guard() { + // This guards the RFC soundness hazard: DSE must not remove the first write + // to `b`, because that write keeps `b`'s address-observed lifetime + // overlapping with `a` and prevents the later move from being eliminated. + // CHECK-LABEL: fn dse_guard( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: [[b]] = make_fields(const 0_u8) + // CHECK: [[a]] = make_fields(const 1_u8) + // CHECK: observe(move + // CHECK: [[b]] = move [[a]] + // CHECK: observe(move + let mut a; + let mut b; + + b = make_fields(0); + + a = make_fields(1); + observe(&raw const a); + b = a; + observe(&raw const b); +} diff --git a/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..4cdc1d24ee427 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff @@ -0,0 +1,16 @@ +- // MIR for `index_local_not_projected` before MoveElimination ++ // MIR for `index_local_not_projected` after MoveElimination + + fn index_local_not_projected(_1: [usize; 4]) -> [usize; 1] { + let mut _0: [usize; 1]; + let mut _2: usize; + let mut _3: usize; + + bb0: { + _2 = const 2_usize; + _3 = copy _1[_2]; + _0 = [copy _2]; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff new file mode 100644 index 0000000000000..ddf8630091e4b --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff @@ -0,0 +1,126 @@ +- // MIR for `overlapping_lifetimes` before MoveElimination ++ // MIR for `overlapping_lifetimes` after MoveElimination + + fn overlapping_lifetimes(_1: bool) -> Fields { + debug flag => _1; + let mut _0: Fields; + let _2: Fields; + let _4: (); + let mut _5: *const Fields; + let _6: (); + let mut _7: bool; + let mut _8: Fields; + let mut _9: Fields; + let mut _10: Fields; + let _11: (); + let mut _12: *const Fields; + scope 1 { +- debug src => _2; ++ debug src => _10; + let mut _3: Fields; + scope 2 { +- debug dst => _3; ++ debug dst => _0; + } + } + + bb0: { +- StorageLive(_2); +- _2 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; ++ nop; ++ _10 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageLive(_3); +- StorageLive(_4); +- StorageLive(_5); +- _5 = &raw const _2; ++ nop; ++ nop; ++ nop; ++ _5 = &raw const _10; + _4 = observe(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_5); +- StorageDead(_4); +- StorageLive(_6); +- StorageLive(_7); +- _7 = copy _1; +- switchInt(move _7) -> [0: bb5, otherwise: bb3]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb5, otherwise: bb3]; + } + + bb3: { +- StorageLive(_8); +- _8 = make_fields(const 1_u8) -> [return: bb4, unwind unreachable]; ++ nop; ++ _0 = make_fields(const 1_u8) -> [return: bb4, unwind unreachable]; + } + + bb4: { +- _3 = move _8; +- StorageDead(_8); +- StorageLive(_9); +- _9 = move _2; +- _3 = move _9; +- StorageDead(_9); ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = move _10; ++ nop; + _6 = const (); + goto -> bb6; + } + + bb5: { +- StorageLive(_10); +- _10 = move _2; +- _3 = move _10; +- StorageDead(_10); ++ nop; ++ nop; ++ _0 = move _10; ++ nop; + _6 = const (); + goto -> bb6; + } + + bb6: { +- StorageDead(_7); +- StorageDead(_6); +- StorageLive(_11); +- StorageLive(_12); +- _12 = &raw const _3; ++ nop; ++ nop; ++ nop; ++ nop; ++ _12 = &raw const _0; + _11 = observe(move _12) -> [return: bb7, unwind unreachable]; + } + + bb7: { +- StorageDead(_12); +- StorageDead(_11); +- _0 = move _3; +- StorageDead(_3); +- StorageDead(_2); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..6c0bb304d3171 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff @@ -0,0 +1,48 @@ +- // MIR for `packed_fields_not_projected` before MoveElimination ++ // MIR for `packed_fields_not_projected` after MoveElimination + + fn packed_fields_not_projected() -> Packed { + let mut _0: Packed; + let _1: [u8; 8]; + let mut _3: [u8; 8]; + let mut _4: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => _3; + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => _4; + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- _3 = copy _1; +- StorageLive(_4); +- _4 = copy _2; ++ nop; ++ _3 = [const 1_u8; 8]; ++ nop; ++ _4 = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; + _0 = Packed { a: move _3, b: move _4 }; +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.rs b/tests/mir-opt/move-elimination/exclusions.rs new file mode 100644 index 0000000000000..2e45946d87ef1 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.rs @@ -0,0 +1,118 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort + +#![feature(core_intrinsics, custom_mir, repr_simd)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +pub struct Fields { + data: [u8; 8], + tag: u8, +} + +#[repr(packed)] +struct Packed { + a: [u8; 8], + b: [u8; 8], +} + +#[repr(simd)] +struct U32x4([u32; 4]); + +unsafe extern "C" { + safe fn observe(_: *const Fields); + safe fn make_fields(_: u8) -> Fields; +} + +// EMIT_MIR exclusions.index_local_not_projected.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn index_local_not_projected(a: [usize; 4]) -> [usize; 1] { + // This checks that a local used as an array index is kept as a bare local, + // because it cannot later be rewritten to a projection like `_0[0]`. + // CHECK-LABEL: fn index_local_not_projected( + // CHECK: [[idx:_.*]] = const 2_usize; + // CHECK: {{.*}} = copy _1[{{.*}}[[idx]]{{.*}}]; + // CHECK: _0 = [copy [[idx]]]; + mir! { + let idx: usize; + let b: usize; + + { + idx = 2usize; + b = a[idx]; + RET = [idx]; + Return() + } + } +} + +// EMIT_MIR exclusions.packed_fields_not_projected.MoveElimination.diff +pub fn packed_fields_not_projected() -> Packed { + // This checks that aggregate fields are not remapped into packed struct + // fields, which could create unaligned projected places. + // CHECK-LABEL: fn packed_fields_not_projected( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: _0 = Packed { a: move [[a]], b: move [[b]] }; + let a = [1; 8]; + let b = [2; 8]; + Packed { a, b } +} + +// EMIT_MIR exclusions.simd_field_not_projected.MoveElimination.diff +pub fn simd_field_not_projected() -> U32x4 { + // This checks that aggregate fields are not remapped into repr(simd) ADTs, + // since optimized MIR must not project into SIMD vectors. + // CHECK-LABEL: fn simd_field_not_projected( + // CHECK: debug lanes => [[lanes:_.*]]; + // CHECK: _0 = U32x4(move [[lanes]]); + let lanes = [1, 2, 3, 4]; + U32x4(lanes) +} + +// EMIT_MIR exclusions.overlapping_lifetimes.MoveElimination.diff +pub fn overlapping_lifetimes(flag: bool) -> Fields { + // This checks the liveness-matrix overlap test for an address-observed + // move-only local: `src` and `dst` only overlap on one branch, but that is + // enough to reject merging them for the whole function. + // CHECK-LABEL: fn overlapping_lifetimes( + // CHECK: debug flag => _1; + // CHECK: debug src => [[src:_[1-9][0-9]*]]; + // CHECK: debug dst => _0; + // CHECK: &raw const [[src]]; + // CHECK: observe + // CHECK: switchInt(move _1) + // CHECK: _0 = make_fields(const 1_u8) + // CHECK: _0 = move [[src]]; + // CHECK: &raw const _0; + // CHECK: observe + let src = make_fields(0); + let mut dst; + observe(&raw const src); + if flag { + dst = make_fields(1); + dst = src; + } else { + dst = src; + } + observe(&raw const dst); + dst +} + +// EMIT_MIR exclusions.rust_call_tuple_not_projected.MoveElimination.diff +pub fn rust_call_tuple_not_projected(f: F) { + // This checks that locals are not remapped into the tuple argument passed + // to a rust-call ABI function. If the tuple itself is never borrowed, alias + // analysis can trivially see that accesses to one argument don't affect the + // others. Merging the arguments into tuple fields from the start can hide + // that independence. + // CHECK-LABEL: fn rust_call_tuple_not_projected( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: [[tuple:_.*]] = (move [[a]], move [[b]]); + // CHECK: >::call_once(move _1, move [[tuple]]) + let a = [1; 8]; + let b = [2; 8]; + f(a, b); +} diff --git a/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..bbde5a905a9e7 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff @@ -0,0 +1,72 @@ +- // MIR for `rust_call_tuple_not_projected` before MoveElimination ++ // MIR for `rust_call_tuple_not_projected` after MoveElimination + + fn rust_call_tuple_not_projected(_1: F) -> () { + debug f => _1; + let mut _0: (); + let _2: [u8; 8]; + let _4: (); + let mut _5: F; + let mut _6: ([u8; 8], [u8; 8]); + let mut _7: [u8; 8]; + let mut _8: [u8; 8]; + scope 1 { +- debug a => _2; ++ debug a => _7; + let _3: [u8; 8]; + scope 2 { +- debug b => _3; ++ debug b => _8; + } + } + + bb0: { +- StorageLive(_2); +- _2 = [const 1_u8; 8]; +- StorageLive(_3); +- _3 = [const 2_u8; 8]; +- StorageLive(_4); +- StorageLive(_5); +- _5 = move _1; +- StorageLive(_6); +- StorageLive(_7); +- _7 = copy _2; +- StorageLive(_8); +- _8 = copy _3; ++ nop; ++ _7 = [const 1_u8; 8]; ++ nop; ++ _8 = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + _6 = (move _7, move _8); +- _4 = >::call_once(move _5, move _6) -> [return: bb1, unwind unreachable]; ++ _4 = >::call_once(move _1, move _6) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_8); +- StorageDead(_7); +- StorageDead(_6); +- StorageDead(_5); +- StorageDead(_4); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + _0 = const (); +- StorageDead(_3); +- StorageDead(_2); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..8d63e07039695 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff @@ -0,0 +1,30 @@ +- // MIR for `simd_field_not_projected` before MoveElimination ++ // MIR for `simd_field_not_projected` after MoveElimination + + fn simd_field_not_projected() -> U32x4 { + let mut _0: U32x4; + let _1: [u32; 4]; + let mut _2: [u32; 4]; + scope 1 { +- debug lanes => _1; ++ debug lanes => _2; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u32, const 2_u32, const 3_u32, const 4_u32]; +- StorageLive(_2); +- _2 = copy _1; ++ nop; ++ _2 = [const 1_u32, const 2_u32, const 3_u32, const 4_u32]; ++ nop; ++ nop; + _0 = U32x4(move _2); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/unwind.call_destination.MoveElimination.diff b/tests/mir-opt/move-elimination/unwind.call_destination.MoveElimination.diff new file mode 100644 index 0000000000000..7507f7bc81e7e --- /dev/null +++ b/tests/mir-opt/move-elimination/unwind.call_destination.MoveElimination.diff @@ -0,0 +1,27 @@ +- // MIR for `call_destination` before MoveElimination ++ // MIR for `call_destination` after MoveElimination + + fn call_destination() -> () { + let mut _0: (); + let mut _1: u32; + let mut _2: u32; + let mut _3: *const u32; + + bb0: { + _1 = const 0_u32; + _3 = &raw const _1; + _1 = opaque() -> [return: bb1, unwind: bb2]; + } + + bb1: { + _0 = (); + return; + } + + bb2 (cleanup): { + _2 = const 2_u32; + _1 = move _2; + resume; + } + } + diff --git a/tests/mir-opt/move-elimination/unwind.rs b/tests/mir-opt/move-elimination/unwind.rs new file mode 100644 index 0000000000000..0015997f7b543 --- /dev/null +++ b/tests/mir-opt/move-elimination/unwind.rs @@ -0,0 +1,41 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=unwind + +#![crate_type = "lib"] +#![feature(custom_mir, core_intrinsics)] +use std::intrinsics::mir::*; + +#[inline(never)] +fn opaque() -> u32 { + 1 +} + +// EMIT_MIR unwind.call_destination.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn call_destination() { + // CHECK-LABEL: fn call_destination( + // CHECK: [[DEST:_[0-9]+]] = opaque() + // CHECK: [[SRC:_[0-9]+]] = const 2_u32; + // CHECK-NEXT: [[DEST]] = move [[SRC]]; + // Unwinding doesn't deallocate the call destination, it only leaves it + // uninitialized. This checks that dest is not merged with src. + mir! { + let dest: u32; + let src: u32; + let p: *const u32; + { + dest = 0; + p = &raw const dest; + Call(dest = opaque(), ReturnTo(done), UnwindCleanup(cleanup)) + } + done = { + RET = (); + Return() + } + cleanup (cleanup) = { + src = 2; + dest = Move(src); + UnwindResume() + } + } +}