MIR move elimination [4/6]: MoveElimination

compiler/rustc_attr_ir/src/data_structures.rs

@@ -589,6 +589,7 @@ pub enum RustcDumpLayoutKind {
 #[derive(Clone, Debug, StableHash, Encodable, Decodable, PrintAttribute, PartialEq, Eq)]
 pub enum RustcMirKind {
     PrettyLiveLocals,
+    PrettyPreciseLiveness,
     PrettyTransitiveLiveLocals,
     PeekMaybeInit,
     PeekMaybeUninit,

compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs

@@ -672,6 +672,10 @@ impl CombineAttributeParser for RustcMirParser {
                             cx.expect_no_args(mi.args())?;
                             Some(RustcMirKind::PrettyLiveLocals)
                         }
+                        sym::rustc_pretty_precise_liveness => {
+                            cx.expect_no_args(mi.args())?;
+                            Some(RustcMirKind::PrettyPreciseLiveness)
+                        }
                         sym::rustc_pretty_transitive_live_locals => {
                             cx.expect_no_args(mi.args())?;
                             Some(RustcMirKind::PrettyTransitiveLiveLocals)

compiler/rustc_index/src/interval.rs

@@ -374,6 +374,12 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         self.rows.iter_enumerated()
     }
 
+    pub fn clear_row(&mut self, row: R) {
+        if let Some(row) = self.rows.get_mut(row) {
+            row.clear();
+        }
+    }
+
     fn ensure_row(&mut self, row: R) -> &mut IntervalSet<C> {
         self.rows.ensure_contains_elem(row, || IntervalSet::new(self.column_size))
     }
@@ -397,6 +403,16 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         write_row.union(read_row)
     }
 
+    pub fn disjoint_rows(&self, a: R, b: R) -> bool
+    where
+        C: Step,
+    {
+        let (Some(a), Some(b)) = (self.rows.get(a), self.rows.get(b)) else {
+            return true;
+        };
+        a.disjoint(b)
+    }
+
     pub fn insert_all_into_row(&mut self, row: R) {
         self.ensure_row(row).insert_all();
     }

compiler/rustc_middle/src/mir/syntax.rs

@@ -127,7 +127,6 @@ pub enum RuntimePhase {
     /// disallowed:
     /// * [`TerminatorKind::Yield`]
     /// * [`TerminatorKind::CoroutineDrop`]
-    /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array`
     /// * [`Rvalue::CopyForDeref`]
     /// * [`PlaceElem::OpaqueCast`]
     /// * [`LocalInfo::DerefTemp`](super::LocalInfo::DerefTemp)
@@ -1449,9 +1448,6 @@ pub enum Rvalue<'tcx> {
     /// This is needed because dataflow analysis needs to distinguish
     /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo`
     /// has a destructor.
-    ///
-    /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After
-    /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too.
     Aggregate(Box<AggregateKind<'tcx>>, IndexVec<FieldIdx, Operand<'tcx>>),
 
     /// A CopyForDeref is equivalent to a read from a place at the

compiler/rustc_mir_dataflow/src/framework/direction.rs

@@ -127,6 +127,8 @@ impl Direction for Backward {
             analysis.apply_primary_statement_effect(state, stmt, loc);
             vis.visit_after_primary_statement_effect(state, stmt, loc);
         }
+
+        vis.visit_block_exit(state, block);
     }
 }
 
@@ -242,5 +244,7 @@ impl Direction for Forward {
         vis.visit_after_early_terminator_effect(state, term, loc);
         analysis.apply_primary_terminator_effect(state, term, loc);
         vis.visit_after_primary_terminator_effect(state, term, loc);
+
+        vis.visit_block_exit(state, block);
     }
 }

compiler/rustc_mir_dataflow/src/framework/visitor.rs

@@ -34,6 +34,13 @@ pub trait ResultsVisitor<'tcx, A>
 where
     A: Analysis<'tcx>,
 {
+    /// Called after all effects in a block have been applied in the direction
+    /// of the analysis.
+    ///
+    /// In a forwards analysis, `state` is from the block's end. In a backwards
+    /// analysis, `state` is from the block's start.
+    fn visit_block_exit(&mut self, _state: &A::Domain, _block: BasicBlock) {}
+
     /// Called after the "early" effect of the given statement is applied to `state`.
     fn visit_after_early_statement_effect(
         &mut self,

compiler/rustc_mir_dataflow/src/impls/mod.rs

@@ -1,6 +1,7 @@
 mod borrowed_locals;
 mod initialized;
 mod liveness;
+mod precise_liveness;
 mod storage_liveness;
 
 pub use self::borrowed_locals::{MaybeBorrowedLocals, borrowed_locals};
@@ -11,6 +12,10 @@ pub use self::initialized::{
 pub use self::liveness::{
     DefUse, LivenessTransferFunction, MaybeLiveLocals, MaybeTransitiveLiveLocals,
 };
+pub use self::precise_liveness::{
+    SplitPointEffect, SplitPointIndex, VisitAllocatedLocalsWith, VisitDeallocatedLocalsWith,
+    dump_liveness_matrix, liveness_matrix,
+};
 pub use self::storage_liveness::{
     MaybeRequiresStorage, MaybeStorageDead, MaybeStorageLive, always_storage_live_locals,
 };