diff --git a/compiler/rustc_hir_id/src/definitions.rs b/compiler/rustc_hir_id/src/definitions.rs index 426a4e0109153..54f5561d7e901 100644 --- a/compiler/rustc_hir_id/src/definitions.rs +++ b/compiler/rustc_hir_id/src/definitions.rs @@ -6,6 +6,7 @@ use std::fmt::{self, Write}; use std::hash::Hash; +use std::sync::OnceLock; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sorted_map::SortedMap; @@ -95,6 +96,7 @@ pub struct Definitions { def_id_to_key: IndexVec, // We do only store the local hash, as all the definitions are from the current crate. def_path_hashes: IndexVec, + last_deterministic_index: OnceLock, def_path_hash_to_index: DefPathToIndexMap, } @@ -292,13 +294,21 @@ pub enum DefPathData { } impl Definitions { - /// This function indicates that the order of def id allocations - /// may be non-deterministic after it was called. pub fn commit_end_of_determinism(&mut self) { assert!( self.def_path_hash_to_index.after_parallel_alloc.replace(Default::default()).is_none(), "this function should be called only once" - ) + ); + + self.last_deterministic_index + .set( + if self.def_id_to_key.is_empty() { 0 } else { self.def_id_to_key.len() - 1 }.into(), + ) + .expect("must be called once"); + } + + pub fn last_deterministic_index(&self) -> DefIndex { + self.last_deterministic_index.get().copied().expect("must contain index") } #[inline(always)] @@ -352,6 +362,7 @@ impl Definitions { def_path_hashes: Default::default(), def_id_to_key: Default::default(), def_path_hash_to_index: Default::default(), + last_deterministic_index: Default::default(), }; // Create the root definition. diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 5348ee0ccaac1..9c159fb3477a8 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -1967,8 +1967,12 @@ impl CrateMetadata { let trait_impls = root .impls .decode(&blob) - .map(|trait_impls| (trait_impls.trait_id, trait_impls.impls)) + .map(|trait_impls| { + let (krate, index) = trait_impls.trait_id; + ((krate, DefIndex::from_u32(index)), trait_impls.impls) + }) .collect(); + let alloc_decoding_state = AllocDecodingState::new(root.interpret_alloc_index.decode(&blob).collect()); diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 03493d0e1b6a9..9e185ff97744c 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -21,7 +21,7 @@ use rustc_hir as hir; use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE, LocalDefId, LocalDefIdSet}; use rustc_hir::definitions::DefPathData; use rustc_hir_pretty::id_to_string; -use rustc_index::IndexVec; +use rustc_index::{Idx, IndexVec}; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::hir::map::compute_hir_hash; use rustc_middle::middle::dependency_format::Linkage; @@ -78,6 +78,8 @@ pub(super) struct EncodeContext<'a, 'tcx> { hygiene_ctxt: Rc>, // Used for both `Symbol`s and `ByteSymbol`s. symbol_index_table: FxHashMap, + pub(super) def_indexes_remapping: FxHashMap, + last_deterministic_index: u32, } /// If the current crate is a proc-macro, returns early with `LazyArray::default()`. @@ -156,12 +158,14 @@ impl<'a, 'tcx> SpanEncoder for EncodeContext<'a, 'tcx> { } fn encode_def_index(&mut self, def_index: DefIndex) { - self.emit_u32(def_index.as_u32()); + self.emit_u32(self.map_index(def_index).as_u32()); } fn encode_def_id(&mut self, def_id: DefId) { + let def_id = self.map_def_id(def_id); + def_id.krate.encode(self); - def_id.index.encode(self); + self.emit_u32(def_id.index.as_u32()); } fn encode_syntax_context(&mut self, syntax_context: SyntaxContext) { @@ -409,20 +413,23 @@ macro_rules! record_some_lazy { ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ let value = $value; let lazy = $self.lazy(value); - $self.$tables.$table.set_some($def_id.index, lazy); + let def_id = $self.map_def_id($def_id); + $self.$tables.$table.set_some(def_id.index, lazy); }}; } macro_rules! record_some { - ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => { - $self.$tables.$table.set_some($def_id.index, $value) - }; + ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ + let def_id = $self.map_def_id($def_id); + $self.$tables.$table.set_some(def_id.index, $value) + }}; } macro_rules! record_value { - ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => { - $self.$tables.$table.set($def_id.index, $value) - }; + ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ + let def_id = $self.map_def_id($def_id); + $self.$tables.$table.set(def_id.index, $value) + }}; } // Shorthand for `$self.$tables.$table.set_some($def_id.index, $self.lazy_array($value))`, which would @@ -431,7 +438,8 @@ macro_rules! record_array { ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ let value = $value; let lazy = $self.lazy_array(value); - $self.$tables.$table.set_some($def_id.index, lazy); + let def_id = $self.map_def_id($def_id); + $self.$tables.$table.set_some(def_id.index, lazy); }}; } @@ -439,7 +447,8 @@ macro_rules! record_defaulted_array { ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{ let value = $value; let lazy = $self.lazy_array(value); - $self.$tables.$table.set($def_id.index, lazy); + let def_id = $self.map_def_id($def_id); + $self.$tables.$table.set(def_id.index, lazy); }}; } @@ -528,7 +537,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { } } - fn encode_def_path_table(&mut self) { + fn encode_def_path_table(&mut self, sorted_ids: &[LocalDefId]) { let defs = self.tcx.definitions(); if self.is_proc_macro { for def_id in std::iter::once(CRATE_DEF_ID) @@ -542,10 +551,14 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { record_value!(self.tables.def_path_hashes[def_id] <- def_path_hash.local_hash().as_u64()) } } else { - for (def_index, def_key, def_path_hash) in defs.enumerated_keys_and_path_hashes() { - let def_id = LocalDefId { local_def_index: def_index }.to_def_id(); + for &def_id in sorted_ids { + let def_key = defs.def_key(def_id); + let hash = defs.def_path_hash(def_id).local_hash().as_u64(); + + let def_id = def_id.to_def_id(); + record_some_lazy!(self.tables.def_keys[def_id] <- def_key); - record_value!(self.tables.def_path_hashes[def_id] <- def_path_hash.local_hash().as_u64()) + record_value!(self.tables.def_path_hashes[def_id] <- hash); } } } @@ -624,7 +637,13 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { adapted.encode(&mut self.opaque) } - fn encode_crate_root(&mut self) -> (LazyValue, LazyValue) { + fn encode_crate_root( + &mut self, + remapping: FxHashMap, + sorted_ids: Vec, + ) -> (LazyValue, LazyValue) { + self.def_indexes_remapping = remapping; + let tcx = self.tcx; let mut stats: Vec<(&'static str, usize)> = Vec::with_capacity(32); @@ -667,7 +686,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { let foreign_modules = stat!("foreign-modules", || self.encode_foreign_modules()); - _ = stat!("def-path-table", || self.encode_def_path_table()); + _ = stat!("def-path-table", || self.encode_def_path_table(&sorted_ids)); // Encode the def IDs of traits, for rustdoc and diagnostics. let traits = stat!("traits", || self.encode_traits()); @@ -679,7 +698,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { _ = stat!("mir", || self.encode_mir()); - _ = stat!("def-ids", || self.encode_def_ids()); + _ = stat!("def-ids", || self.encode_def_ids(&sorted_ids)); let interpret_alloc_index = stat!("interpret-alloc-index", || { let mut interpret_alloc_index = Vec::new(); @@ -1455,7 +1474,26 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { record_value!(self.tables.attr_flags[def_id.to_def_id()] <- attr_flags) } - fn encode_def_ids(&mut self) { + fn map_def_id(&self, def_id: DefId) -> DefId { + if def_id.is_local() { + DefId { krate: LOCAL_CRATE, index: self.map_index(def_id.index) } + } else { + def_id + } + } + + #[inline] + fn map_index(&self, def_index: DefIndex) -> DefIndex { + let index = def_index.as_u32(); + + if index <= self.last_deterministic_index { + def_index + } else { + self.def_indexes_remapping.get(&def_index).copied().unwrap_or(def_index) + } + } + + fn encode_def_ids(&mut self, sorted_ids: &[LocalDefId]) { self.encode_info_for_mod(CRATE_DEF_ID); // Proc-macro crates only export proc-macro items, which are looked @@ -1465,8 +1503,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { } let tcx = self.tcx; - - for local_id in tcx.iter_local_def_id() { + for &local_id in sorted_ids { let def_id = local_id.to_def_id(); let def_kind = tcx.def_kind(local_id); record_some!(self.tables.def_kind[def_id] <- def_kind); @@ -2302,7 +2339,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { if let Ok(mut an) = trait_def.ancestors(tcx, def_id) && let Some(specialization_graph::Node::Impl(parent)) = an.nth(1) { - record_some!(self.tables.impl_parent[def_id] <- parent.into()); + record_some!(self.tables.impl_parent[def_id] <- self.map_def_id(parent).into()); } // if this is an impl of `CoerceUnsized`, create its @@ -2316,9 +2353,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { let trait_impls: Vec<_> = trait_impls .into_iter() - .map(|(trait_def_id, impls)| TraitImpls { - trait_id: (trait_def_id.krate.as_u32(), trait_def_id.index), - impls: self.lazy_array(&impls), + .map(|(trait_id, impls)| { + let trait_id = self.map_def_id(trait_id); + TraitImpls { + trait_id: (trait_id.krate.as_u32(), trait_id.index.as_u32()), + impls: self.lazy_array(&impls), + } }) .collect(); @@ -2601,7 +2641,9 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { with_encode_metadata_header(tcx, path, |ecx| { // Encode all the entries and extra information in the crate, // culminating in the `CrateRoot` which points to all of it. - let (root, unhashed) = ecx.encode_crate_root(); + let (remapping, sequence) = create_def_index_remapping(tcx); + + let (root, unhashed) = ecx.encode_crate_root(remapping, sequence); // Flush buffer to ensure backing file has the correct size. ecx.opaque.flush(); @@ -2634,6 +2676,40 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { } } +fn create_def_index_remapping(tcx: TyCtxt<'_>) -> (FxHashMap, Vec) { + let defs = tcx.untracked().definitions.read(); + + let mut to_remap = vec![]; + let mut def_ids = vec![]; + + let start = tcx.definitions().last_deterministic_index().as_usize() + 1; + for idx in start..defs.num_definitions() { + let def_id = LocalDefId { local_def_index: idx.into() }; + + to_remap.push((def_id, defs.def_path_hash(def_id).local_hash())); + def_ids.push(def_id); + } + + to_remap.sort_by_key(|(_, hash)| *hash); + + let mut remapping = FxHashMap::default(); + for ((orig_id, _), remapped_id) in to_remap.into_iter().zip(def_ids) { + remapping.insert(orig_id.local_def_index, remapped_id.local_def_index); + } + + let mut sorted_def_ids = (0..defs.num_definitions()) + .into_iter() + .map(|idx| LocalDefId { local_def_index: DefIndex::new(idx) }) + .collect::>(); + + sorted_def_ids.sort_by_key(|id| { + let def_index = id.local_def_index; + remapping.get(&def_index).copied().unwrap_or(def_index) + }); + + (remapping, sorted_def_ids) +} + fn with_encode_metadata_header( tcx: TyCtxt<'_>, path: &Path, @@ -2707,6 +2783,8 @@ fn with_encode_metadata_header( is_proc_macro: tcx.crate_types().contains(&CrateType::ProcMacro), hygiene_ctxt: Default::default(), symbol_index_table: Default::default(), + def_indexes_remapping: Default::default(), + last_deterministic_index: tcx.definitions().last_deterministic_index().as_u32(), }; // Encode the rustc version string in a predictable location. diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index f7a437043dade..3848bb3eda88e 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -378,7 +378,7 @@ pub(crate) struct CrateDep { #[derive(MetadataEncodable, LazyDecodable)] pub(crate) struct TraitImpls { - trait_id: (u32, DefIndex), + trait_id: (u32 /* crate */, u32 /* def index */), impls: LazyArray<(DefIndex, Option)>, } diff --git a/tests/run-make/parallel-reproducible-build/async-fns-issue-162202.rs b/tests/run-make/parallel-reproducible-build/async-fns-issue-162202.rs new file mode 100644 index 0000000000000..1e959cfd62543 --- /dev/null +++ b/tests/run-make/parallel-reproducible-build/async-fns-issue-162202.rs @@ -0,0 +1,3 @@ +async fn test1(_: &'_ u8) {} +async fn test2<'s>(_: &'s u8) {} +fn main() {} diff --git a/tests/run-make/parallel-reproducible-build/rmake.rs b/tests/run-make/parallel-reproducible-build/rmake.rs index caaee1b47c583..01de08e0551f3 100644 --- a/tests/run-make/parallel-reproducible-build/rmake.rs +++ b/tests/run-make/parallel-reproducible-build/rmake.rs @@ -14,6 +14,39 @@ fn main() { const TESTS: &[(&str, &[&str])] = &[ ("static-muts-issue-140413", &["-Zthreads=50"]), ("derives-issue-129094", &["-Zthreads=16", "-Copt-level=3"]), + ( + "async-fns-issue-162202", + &[ + "-Zthreads=30", + "--crate-type=lib", + "--crate-name=test", + "-Ccodegen-units=1", + "-Zremap-cwd-prefix=reproducible_dir", + "--edition=2024", + ], + ), + ( + "rpit-issue-162202", + &[ + "-Zthreads=30", + "--crate-type=lib", + "--crate-name=test", + "-Ccodegen-units=1", + "-Zremap-cwd-prefix=reproducible_dir", + "--edition=2024", + ], + ), + ( + "static-muts-lib-issue-162203", + &[ + "-Zthreads=30", + "--edition=2024", + "-Zremap-cwd-prefix=reproducible_dir", + "--cap-lints=warn", + "-Ccodegen-units=1", + "-Clink-dead-code=true", + ], + ), ]; for (file, args) in TESTS { diff --git a/tests/run-make/parallel-reproducible-build/rpit-issue-162202.rs b/tests/run-make/parallel-reproducible-build/rpit-issue-162202.rs new file mode 100644 index 0000000000000..1147f4d450d7d --- /dev/null +++ b/tests/run-make/parallel-reproducible-build/rpit-issue-162202.rs @@ -0,0 +1,19 @@ +trait Foo { + fn test() -> impl IntoIterator + Send; +} + +struct A; +impl Foo for A { + fn test() -> impl IntoIterator + Send { + [] + } +} + +struct B; +impl Foo for B { + fn test() -> impl IntoIterator + Send { + [] + } +} + +fn main() {} diff --git a/tests/run-make/parallel-reproducible-build/static-muts-lib-issue-162203.rs b/tests/run-make/parallel-reproducible-build/static-muts-lib-issue-162203.rs new file mode 100644 index 0000000000000..adda309c8cd39 --- /dev/null +++ b/tests/run-make/parallel-reproducible-build/static-muts-lib-issue-162203.rs @@ -0,0 +1,4 @@ +#![crate_type = "lib"] +static mut TEST: &'static mut [isize] = &mut [1]; +static mut TEST_RAW: *mut [isize] = &mut [1isize] as *mut _; +pub fn main() {}