diff --git a/Cargo.lock b/Cargo.lock index 2164bb7fa4323..13a130a1b49c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4126,6 +4126,7 @@ dependencies = [ name = "rustc_query_system" version = "0.0.0" dependencies = [ + "arena", "log", "parking_lot 0.10.2", "rustc-rayon-core", diff --git a/src/librustc_codegen_llvm/attributes.rs b/src/librustc_codegen_llvm/attributes.rs index 6ef303c848dc3..fc357ebb05d62 100644 --- a/src/librustc_codegen_llvm/attributes.rs +++ b/src/librustc_codegen_llvm/attributes.rs @@ -253,7 +253,7 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty:: inline(cx, llfn, attributes::InlineAttr::Hint); } - inline(cx, llfn, codegen_fn_attrs.inline); + inline(cx, llfn, codegen_fn_attrs.inline.clone()); // The `uwtable` attribute according to LLVM is: // @@ -350,15 +350,12 @@ pub fn provide(providers: &mut Providers<'_>) { if tcx.sess.opts.actually_rustdoc { // rustdoc needs to be able to document functions that use all the features, so // whitelist them all - tcx.arena - .alloc(llvm_util::all_known_features().map(|(a, b)| (a.to_string(), b)).collect()) + llvm_util::all_known_features().map(|(a, b)| (a.to_string(), b)).collect() } else { - tcx.arena.alloc( - llvm_util::target_feature_whitelist(tcx.sess) - .iter() - .map(|&(a, b)| (a.to_string(), b)) - .collect(), - ) + llvm_util::target_feature_whitelist(tcx.sess) + .iter() + .map(|&(a, b)| (a.to_string(), b)) + .collect() } }; @@ -390,7 +387,7 @@ pub fn provide_extern(providers: &mut Providers<'_>) { })); } - tcx.arena.alloc(ret) + ret }; } diff --git a/src/librustc_codegen_ssa/back/symbol_export.rs b/src/librustc_codegen_ssa/back/symbol_export.rs index 9faddf254d7e5..338211ca11f3c 100644 --- a/src/librustc_codegen_ssa/back/symbol_export.rs +++ b/src/librustc_codegen_ssa/back/symbol_export.rs @@ -42,14 +42,11 @@ pub fn crates_export_threshold(crate_types: &[config::CrateType]) -> SymbolExpor } } -fn reachable_non_generics_provider( - tcx: TyCtxt<'_>, - cnum: CrateNum, -) -> &DefIdMap { +fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap { assert_eq!(cnum, LOCAL_CRATE); if !tcx.sess.opts.output_types.should_codegen() { - return tcx.arena.alloc(Default::default()); + return Default::default(); } // Check to see if this crate is a "special runtime crate". These @@ -145,7 +142,7 @@ fn reachable_non_generics_provider( reachable_non_generics.insert(id, SymbolExportLevel::C); } - tcx.arena.alloc(reachable_non_generics) + reachable_non_generics } fn is_reachable_non_generic_provider_local(tcx: TyCtxt<'_>, def_id: DefId) -> bool { @@ -281,7 +278,7 @@ fn exported_symbols_provider_local( fn upstream_monomorphizations_provider( tcx: TyCtxt<'_>, cnum: CrateNum, -) -> &DefIdMap, CrateNum>> { +) -> DefIdMap, CrateNum>> { debug_assert!(cnum == LOCAL_CRATE); let cnums = tcx.all_crate_nums(LOCAL_CRATE); @@ -338,7 +335,7 @@ fn upstream_monomorphizations_provider( } } - tcx.arena.alloc(instances) + instances } fn upstream_monomorphizations_for_provider( diff --git a/src/librustc_codegen_ssa/base.rs b/src/librustc_codegen_ssa/base.rs index cd43ca8257e92..fc6b547c5e70c 100644 --- a/src/librustc_codegen_ssa/base.rs +++ b/src/librustc_codegen_ssa/base.rs @@ -908,7 +908,7 @@ pub fn provide_both(providers: &mut Providers<'_>) { .map(|id| &module_map[&id]) .flat_map(|module| module.foreign_items.iter().cloned()) .collect(); - tcx.arena.alloc(dllimports) + dllimports }; providers.is_dllimport_foreign_item = diff --git a/src/librustc_lint/levels.rs b/src/librustc_lint/levels.rs index c78b088fb1261..7e8e4f59707c1 100644 --- a/src/librustc_lint/levels.rs +++ b/src/librustc_lint/levels.rs @@ -22,7 +22,7 @@ use rustc_span::symbol::{sym, Symbol}; use std::cmp; -fn lint_levels(tcx: TyCtxt<'_>, cnum: CrateNum) -> &LintLevelMap { +fn lint_levels(tcx: TyCtxt<'_>, cnum: CrateNum) -> LintLevelMap { assert_eq!(cnum, LOCAL_CRATE); let store = unerased_lint_store(tcx); let levels = LintLevelsBuilder::new(tcx.sess, false, &store); @@ -37,7 +37,7 @@ fn lint_levels(tcx: TyCtxt<'_>, cnum: CrateNum) -> &LintLevelMap { intravisit::walk_crate(&mut builder, krate); builder.levels.pop(push); - tcx.arena.alloc(builder.levels.build_map()) + builder.levels.build_map() } pub struct LintLevelsBuilder<'s> { diff --git a/src/librustc_macros/src/query.rs b/src/librustc_macros/src/query.rs index 7ed77304a90e1..229ccfe08f7af 100644 --- a/src/librustc_macros/src/query.rs +++ b/src/librustc_macros/src/query.rs @@ -107,7 +107,9 @@ impl Parse for QueryModifier { let block = input.parse()?; Ok(QueryModifier::LoadCached(tcx, id, block)) } else if modifier == "storage" { - let ty = input.parse()?; + let args; + parenthesized!(args in input); + let ty = args.parse()?; Ok(QueryModifier::Storage(ty)) } else if modifier == "fatal_cycle" { Ok(QueryModifier::FatalCycle) diff --git a/src/librustc_metadata/rmeta/decoder.rs b/src/librustc_metadata/rmeta/decoder.rs index 39d8213f2629b..6a4b35ed3d7fa 100644 --- a/src/librustc_metadata/rmeta/decoder.rs +++ b/src/librustc_metadata/rmeta/decoder.rs @@ -939,8 +939,8 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { } /// Iterates over the diagnostic items in the given crate. - fn get_diagnostic_items(&self, tcx: TyCtxt<'tcx>) -> &'tcx FxHashMap { - tcx.arena.alloc(if self.root.is_proc_macro_crate() { + fn get_diagnostic_items(&self) -> FxHashMap { + if self.root.is_proc_macro_crate() { // Proc macro crates do not export any diagnostic-items to the target. Default::default() } else { @@ -949,7 +949,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { .decode(self) .map(|(name, def_index)| (name, self.local_def_id(def_index))) .collect() - }) + } } /// Iterates over each child of the given item. diff --git a/src/librustc_metadata/rmeta/decoder/cstore_impl.rs b/src/librustc_metadata/rmeta/decoder/cstore_impl.rs index 7d1639cbcf7a1..c6c06c98ee5c2 100644 --- a/src/librustc_metadata/rmeta/decoder/cstore_impl.rs +++ b/src/librustc_metadata/rmeta/decoder/cstore_impl.rs @@ -88,15 +88,11 @@ impl IntoArgs for (CrateNum, DefId) { provide! { <'tcx> tcx, def_id, other, cdata, type_of => { cdata.get_type(def_id.index, tcx) } - generics_of => { - tcx.arena.alloc(cdata.get_generics(def_id.index, tcx.sess)) - } + generics_of => { cdata.get_generics(def_id.index, tcx.sess) } explicit_predicates_of => { cdata.get_explicit_predicates(def_id.index, tcx) } inferred_outlives_of => { cdata.get_inferred_outlives(def_id.index, tcx) } super_predicates_of => { cdata.get_super_predicates(def_id.index, tcx) } - trait_def => { - tcx.arena.alloc(cdata.get_trait_def(def_id.index, tcx.sess)) - } + trait_def => { cdata.get_trait_def(def_id.index, tcx.sess) } adt_def => { cdata.get_adt_def(def_id.index, tcx) } adt_destructor => { let _ = cdata; @@ -117,8 +113,8 @@ provide! { <'tcx> tcx, def_id, other, cdata, bug!("coerce_unsized_info: `{:?}` is missing its info", def_id); }) } - optimized_mir => { tcx.arena.alloc(cdata.get_optimized_mir(tcx, def_id.index)) } - promoted_mir => { tcx.arena.alloc(cdata.get_promoted_mir(tcx, def_id.index)) } + optimized_mir => { cdata.get_optimized_mir(tcx, def_id.index) } + promoted_mir => { cdata.get_promoted_mir(tcx, def_id.index) } mir_const_qualif => { cdata.mir_const_qualif(def_id.index) } fn_sig => { cdata.fn_sig(def_id.index, tcx) } inherent_impls => { cdata.get_inherent_implementations_for_type(tcx, def_id.index) } @@ -178,7 +174,7 @@ provide! { <'tcx> tcx, def_id, other, cdata, }) .collect(); - tcx.arena.alloc(reachable_non_generics) + reachable_non_generics } native_libraries => { Lrc::new(cdata.get_native_libraries(tcx.sess)) } foreign_modules => { cdata.get_foreign_modules(tcx) } @@ -220,7 +216,7 @@ provide! { <'tcx> tcx, def_id, other, cdata, } defined_lib_features => { cdata.get_lib_features(tcx) } defined_lang_items => { cdata.get_lang_items(tcx) } - diagnostic_items => { cdata.get_diagnostic_items(tcx) } + diagnostic_items => { cdata.get_diagnostic_items() } missing_lang_items => { cdata.get_missing_lang_items(tcx) } missing_extern_crate_item => { @@ -363,7 +359,7 @@ pub fn provide(providers: &mut Providers<'_>) { } } - tcx.arena.alloc(visible_parent_map) + visible_parent_map }, dependency_formats: |tcx, cnum| { diff --git a/src/librustc_middle/arena.rs b/src/librustc_middle/arena.rs index d0f7002994770..bbeacbfc5382e 100644 --- a/src/librustc_middle/arena.rs +++ b/src/librustc_middle/arena.rs @@ -12,36 +12,13 @@ macro_rules! arena_types { ($macro:path, $args:tt, $tcx:lifetime) => ( $macro!($args, [ [] layouts: rustc_target::abi::Layout, - [] generics: rustc_middle::ty::Generics, - [] trait_def: rustc_middle::ty::TraitDef, + // AdtDef are interned and compared by address [] adt_def: rustc_middle::ty::AdtDef, - [] steal_mir: rustc_middle::ty::steal::Steal>, - [] mir: rustc_middle::mir::Body<$tcx>, - [] steal_promoted: rustc_middle::ty::steal::Steal< - rustc_index::vec::IndexVec< - rustc_middle::mir::Promoted, - rustc_middle::mir::Body<$tcx> - > - >, - [] promoted: rustc_index::vec::IndexVec< - rustc_middle::mir::Promoted, - rustc_middle::mir::Body<$tcx> - >, [decode] tables: rustc_middle::ty::TypeckTables<$tcx>, - [decode] borrowck_result: rustc_middle::mir::BorrowCheckResult<$tcx>, [] const_allocs: rustc_middle::mir::interpret::Allocation, - [] vtable_method: Option<( - rustc_hir::def_id::DefId, - rustc_middle::ty::subst::SubstsRef<$tcx> - )>, - [few, decode] collect_and_partition_mono_items: rustc_hir::def_id::DefIdSet, - [few, decode] mir_keys: rustc_data_structures::fx::FxHashSet, - [decode] specialization_graph: rustc_middle::traits::specialization_graph::Graph, + // Required for the incremental on-disk cache + [few, decode] mir_keys: rustc_hir::def_id::DefIdSet, [] region_scope_tree: rustc_middle::middle::region::ScopeTree, - [] item_local_set: rustc_hir::ItemLocalSet, - [decode] mir_const_qualif: rustc_index::bit_set::BitSet, - [] trait_impls_of: rustc_middle::ty::trait_def::TraitImpls, - [] associated_items: rustc_middle::ty::AssociatedItems, [] dropck_outlives: rustc_middle::infer::canonical::Canonical<'tcx, rustc_middle::infer::canonical::QueryResponse<'tcx, @@ -80,42 +57,10 @@ macro_rules! arena_types { rustc_middle::infer::canonical::Canonical<'tcx, rustc_middle::infer::canonical::QueryResponse<'tcx, rustc_middle::ty::Ty<'tcx>> >, - [few] crate_inherent_impls: rustc_middle::ty::CrateInherentImpls, - [few] upstream_monomorphizations: - rustc_hir::def_id::DefIdMap< - rustc_data_structures::fx::FxHashMap< - rustc_middle::ty::subst::SubstsRef<'tcx>, - rustc_hir::def_id::CrateNum - > - >, - [few] diagnostic_items: rustc_data_structures::fx::FxHashMap< - rustc_span::symbol::Symbol, - rustc_hir::def_id::DefId, - >, - [few] resolve_lifetimes: rustc_middle::middle::resolve_lifetime::ResolveLifetimes, - [few] lint_levels: rustc_middle::lint::LintLevelMap, - [few] stability_index: rustc_middle::middle::stability::Index<'tcx>, - [few] features: rustc_feature::Features, [few] all_traits: Vec, [few] privacy_access_levels: rustc_middle::middle::privacy::AccessLevels, - [few] target_features_whitelist: rustc_data_structures::fx::FxHashMap< - String, - Option - >, - [few] wasm_import_module_map: rustc_data_structures::fx::FxHashMap< - rustc_hir::def_id::DefId, - String - >, - [few] get_lib_features: rustc_middle::middle::lib_features::LibFeatures, - [few] defined_lib_features: rustc_hir::lang_items::LanguageItems, - [few] visible_parent_map: rustc_hir::def_id::DefIdMap, [few] foreign_module: rustc_middle::middle::cstore::ForeignModule, [few] foreign_modules: Vec, - [few] reachable_non_generics: rustc_hir::def_id::DefIdMap< - rustc_middle::middle::exported_symbols::SymbolExportLevel - >, - [few] crate_variances: rustc_middle::ty::CrateVariancesMap<'tcx>, - [few] inferred_outlives_crate: rustc_middle::ty::CratePredicatesMap<'tcx>, [] upvars: rustc_data_structures::fx::FxIndexMap, [] object_safety_violations: rustc_middle::traits::ObjectSafetyViolation, [] codegen_unit: rustc_middle::mir::mono::CodegenUnit<$tcx>, diff --git a/src/librustc_middle/query/mod.rs b/src/librustc_middle/query/mod.rs index c3e023f40299e..e57c51b9eefbe 100644 --- a/src/librustc_middle/query/mod.rs +++ b/src/librustc_middle/query/mod.rs @@ -102,12 +102,13 @@ rustc_queries! { /// Maps from the `DefId` of an item (trait/struct/enum/fn) to its /// associated generics. - query generics_of(key: DefId) -> &'tcx ty::Generics { + query generics_of(key: DefId) -> ty::Generics { + storage(ArenaCacheSelector<'tcx>) cache_on_disk_if { key.is_local() } load_cached(tcx, id) { let generics: Option = tcx.queries.on_disk_cache .try_load_query_result(tcx, id); - generics.map(|x| &*tcx.arena.alloc(x)) + generics } } @@ -134,7 +135,8 @@ rustc_queries! { desc { "looking up the native libraries of a linked crate" } } - query lint_levels(_: CrateNum) -> &'tcx LintLevelMap { + query lint_levels(_: CrateNum) -> LintLevelMap { + storage(ArenaCacheSelector<'tcx>) eval_always desc { "computing the lint levels for items in this crate" } } @@ -156,7 +158,8 @@ rustc_queries! { /// Set of all the `DefId`s in this crate that have MIR associated with /// them. This includes all the body owners, but also things like struct /// constructors. - query mir_keys(_: CrateNum) -> &'tcx FxHashSet { + query mir_keys(_: CrateNum) -> FxHashSet { + storage(ArenaCacheSelector<'tcx>) desc { "getting a list of all mir_keys" } } @@ -170,7 +173,8 @@ rustc_queries! { /// Fetch the MIR for a given `DefId` right after it's built - this includes /// unreachable code. - query mir_built(_: LocalDefId) -> &'tcx Steal> { + query mir_built(_: LocalDefId) -> Steal> { + storage(ArenaCacheSelector<'tcx>) desc { "building MIR for" } } @@ -178,40 +182,31 @@ rustc_queries! { /// ready for const evaluation. /// /// See the README for the `mir` module for details. - query mir_const(_: DefId) -> &'tcx Steal> { + query mir_const(_: DefId) -> Steal> { + storage(ArenaCacheSelector<'tcx>) no_hash } query mir_validated(key: LocalDefId) -> ( - &'tcx Steal>, - &'tcx Steal>> + Steal>, + Steal>> ) { + storage(ArenaCacheSelector<'tcx>) no_hash desc { |tcx| "processing `{}`", tcx.def_path_str(key.to_def_id()) } } /// MIR after our optimization passes have run. This is MIR that is ready /// for codegen. This is also the only query that can fetch non-local MIR, at present. - query optimized_mir(key: DefId) -> &'tcx mir::Body<'tcx> { + query optimized_mir(key: DefId) -> mir::Body<'tcx> { + storage(ArenaCacheSelector<'tcx>) cache_on_disk_if { key.is_local() } - load_cached(tcx, id) { - let mir: Option> - = tcx.queries.on_disk_cache.try_load_query_result(tcx, id); - mir.map(|x| &*tcx.arena.alloc(x)) - } } - query promoted_mir(key: DefId) -> &'tcx IndexVec> { + query promoted_mir(key: DefId) -> IndexVec> { + storage(ArenaCacheSelector<'tcx>) cache_on_disk_if { key.is_local() } - load_cached(tcx, id) { - let promoted: Option< - rustc_index::vec::IndexVec< - crate::mir::Promoted, - crate::mir::Body<'tcx> - >> = tcx.queries.on_disk_cache.try_load_query_result(tcx, id); - promoted.map(|p| &*tcx.arena.alloc(p)) - } } } @@ -245,7 +240,8 @@ rustc_queries! { } Linking { - query wasm_import_module_map(_: CrateNum) -> &'tcx FxHashMap { + query wasm_import_module_map(_: CrateNum) -> FxHashMap { + storage(ArenaCacheSelector<'tcx>) desc { "wasm import module map" } } } @@ -283,8 +279,11 @@ rustc_queries! { }} } - query trait_def(_: DefId) -> &'tcx ty::TraitDef {} - query adt_def(_: DefId) -> &'tcx ty::AdtDef {} + query trait_def(_: DefId) -> ty::TraitDef { + storage(ArenaCacheSelector<'tcx>) + } + query adt_def(_: DefId) -> &'tcx ty::AdtDef { + } query adt_destructor(_: DefId) -> Option {} // The cycle error here should be reported as an error by `check_representable`. @@ -345,7 +344,8 @@ rustc_queries! { query generator_kind(_: DefId) -> Option {} /// Gets a map with the variance of every item; use `item_variance` instead. - query crate_variances(_: CrateNum) -> &'tcx ty::CrateVariancesMap<'tcx> { + query crate_variances(_: CrateNum) -> ty::CrateVariancesMap<'tcx> { + storage(ArenaCacheSelector<'tcx>) desc { "computing the variances for items in this crate" } } @@ -356,7 +356,8 @@ rustc_queries! { TypeChecking { /// Maps from thee `DefId` of a type to its (inferred) outlives. query inferred_outlives_crate(_: CrateNum) - -> &'tcx ty::CratePredicatesMap<'tcx> { + -> ty::CratePredicatesMap<'tcx> { + storage(ArenaCacheSelector<'tcx>) desc { "computing the inferred outlives predicates for items in this crate" } } } @@ -366,10 +367,13 @@ rustc_queries! { query associated_item_def_ids(_: DefId) -> &'tcx [DefId] {} /// Maps from a trait item to the trait item "descriptor". - query associated_item(_: DefId) -> ty::AssocItem {} + query associated_item(_: DefId) -> ty::AssocItem { + storage(ArenaCacheSelector<'tcx>) + } /// Collects the associated items defined on a trait or impl. - query associated_items(key: DefId) -> &'tcx ty::AssociatedItems { + query associated_items(key: DefId) -> ty::AssociatedItems<'tcx> { + storage(ArenaCacheSelector<'tcx>) desc { |tcx| "collecting associated items of {}", tcx.def_path_str(key) } } @@ -393,6 +397,7 @@ rustc_queries! { query unsafety_check_result(key: LocalDefId) -> mir::UnsafetyCheckResult { desc { |tcx| "unsafety-checking `{}`", tcx.def_path_str(key.to_def_id()) } cache_on_disk_if { true } + storage(ArenaCacheSelector<'tcx>) } /// HACK: when evaluated, this reports a "unsafe derive on repr(packed)" error @@ -472,7 +477,7 @@ rustc_queries! { .queries.on_disk_cache .try_load_query_result(tcx, id); - typeck_tables.map(|tables| &*tcx.arena.alloc(tables)) + typeck_tables.map(|x| &*tcx.arena.alloc(x)) } } } @@ -495,7 +500,8 @@ rustc_queries! { BorrowChecking { /// Borrow-checks the function body. If this is a closure, returns /// additional requirements that the closure's creator must verify. - query mir_borrowck(key: LocalDefId) -> &'tcx mir::BorrowCheckResult<'tcx> { + query mir_borrowck(key: LocalDefId) -> mir::BorrowCheckResult<'tcx> { + storage(ArenaCacheSelector<'tcx>) desc { |tcx| "borrow-checking `{}`", tcx.def_path_str(key.to_def_id()) } cache_on_disk_if(tcx, opt_result) { tcx.is_closure(key.to_def_id()) @@ -509,7 +515,8 @@ rustc_queries! { /// Not meant to be used directly outside of coherence. /// (Defined only for `LOCAL_CRATE`.) query crate_inherent_impls(k: CrateNum) - -> &'tcx CrateInherentImpls { + -> CrateInherentImpls { + storage(ArenaCacheSelector<'tcx>) eval_always desc { "all inherent impls defined in crate `{:?}`", k } } @@ -610,7 +617,8 @@ rustc_queries! { /// in the case of closures, this will be redirected to the enclosing function. query region_scope_tree(_: DefId) -> &'tcx region::ScopeTree {} - query mir_shims(key: ty::InstanceDef<'tcx>) -> &'tcx mir::Body<'tcx> { + query mir_shims(key: ty::InstanceDef<'tcx>) -> mir::Body<'tcx> { + storage(ArenaCacheSelector<'tcx>) desc { |tcx| "generating MIR shim for `{}`", tcx.def_path_str(key.def_id()) } } @@ -639,6 +647,7 @@ rustc_queries! { Codegen { query codegen_fn_attrs(_: DefId) -> CodegenFnAttrs { + storage(ArenaCacheSelector<'tcx>) cache_on_disk_if { true } } } @@ -684,10 +693,12 @@ rustc_queries! { query all_local_trait_impls(key: CrateNum) -> &'tcx BTreeMap> { desc { "local trait impls" } } - query trait_impls_of(key: DefId) -> &'tcx ty::trait_def::TraitImpls { + query trait_impls_of(key: DefId) -> ty::trait_def::TraitImpls { + storage(ArenaCacheSelector<'tcx>) desc { |tcx| "trait impls of `{}`", tcx.def_path_str(key) } } - query specialization_graph_of(key: DefId) -> &'tcx specialization_graph::Graph { + query specialization_graph_of(key: DefId) -> specialization_graph::Graph { + storage(ArenaCacheSelector<'tcx>) desc { |tcx| "building specialization graph of trait `{}`", tcx.def_path_str(key) } cache_on_disk_if { true } } @@ -829,7 +840,8 @@ rustc_queries! { // Does not include external symbols that don't have a corresponding DefId, // like the compiler-generated `main` function and so on. query reachable_non_generics(_: CrateNum) - -> &'tcx DefIdMap { + -> DefIdMap { + storage(ArenaCacheSelector<'tcx>) desc { "looking up the exported symbols of a crate" } } query is_reachable_non_generic(_: DefId) -> bool {} @@ -845,7 +857,8 @@ rustc_queries! { /// better, `Instance::upstream_monomorphization()`. query upstream_monomorphizations( k: CrateNum - ) -> &'tcx DefIdMap, CrateNum>> { + ) -> DefIdMap, CrateNum>> { + storage(ArenaCacheSelector<'tcx>) desc { "collecting available upstream monomorphizations `{:?}`", k } } @@ -930,7 +943,8 @@ rustc_queries! { Other { query dllimport_foreign_items(_: CrateNum) - -> &'tcx FxHashSet { + -> FxHashSet { + storage(ArenaCacheSelector<'tcx>) desc { "dllimport_foreign_items" } } query is_dllimport_foreign_item(_: DefId) -> bool {} @@ -948,7 +962,8 @@ rustc_queries! { BorrowChecking { /// Lifetime resolution. See `middle::resolve_lifetimes`. - query resolve_lifetimes(_: CrateNum) -> &'tcx ResolveLifetimes { + query resolve_lifetimes(_: CrateNum) -> ResolveLifetimes { + storage(ArenaCacheSelector<'tcx>) desc { "resolving lifetimes" } } query named_region_map(_: LocalDefId) -> @@ -981,7 +996,8 @@ rustc_queries! { query item_children(_: DefId) -> &'tcx [Export] {} query extern_mod_stmt_cnum(_: DefId) -> Option {} - query get_lib_features(_: CrateNum) -> &'tcx LibFeatures { + query get_lib_features(_: CrateNum) -> LibFeatures { + storage(ArenaCacheSelector<'tcx>) eval_always desc { "calculating the lib features map" } } @@ -992,13 +1008,15 @@ rustc_queries! { /// Returns the lang items defined in another crate by loading it from metadata. // FIXME: It is illegal to pass a `CrateNum` other than `LOCAL_CRATE` here, just get rid // of that argument? - query get_lang_items(_: CrateNum) -> &'tcx LanguageItems { + query get_lang_items(_: CrateNum) -> LanguageItems { + storage(ArenaCacheSelector<'tcx>) eval_always desc { "calculating the lang items map" } } /// Returns all diagnostic items defined in all crates. - query all_diagnostic_items(_: CrateNum) -> &'tcx FxHashMap { + query all_diagnostic_items(_: CrateNum) -> FxHashMap { + storage(ArenaCacheSelector<'tcx>) eval_always desc { "calculating the diagnostic items map" } } @@ -1009,7 +1027,8 @@ rustc_queries! { } /// Returns the diagnostic items defined in a crate. - query diagnostic_items(_: CrateNum) -> &'tcx FxHashMap { + query diagnostic_items(_: CrateNum) -> FxHashMap { + storage(ArenaCacheSelector<'tcx>) desc { "calculating the diagnostic items map in a crate" } } @@ -1017,7 +1036,8 @@ rustc_queries! { desc { "calculating the missing lang items in a crate" } } query visible_parent_map(_: CrateNum) - -> &'tcx DefIdMap { + -> DefIdMap { + storage(ArenaCacheSelector<'tcx>) desc { "calculating the visible parent map" } } query missing_extern_crate_item(_: CrateNum) -> bool { @@ -1051,7 +1071,8 @@ rustc_queries! { desc { |tcx| "names_imported_by_glob_use for `{}`", tcx.def_path_str(def_id.to_def_id()) } } - query stability_index(_: CrateNum) -> &'tcx stability::Index<'tcx> { + query stability_index(_: CrateNum) -> stability::Index<'tcx> { + storage(ArenaCacheSelector<'tcx>) eval_always desc { "calculating the stability index for the local crate" } } @@ -1242,7 +1263,8 @@ rustc_queries! { } Other { - query target_features_whitelist(_: CrateNum) -> &'tcx FxHashMap> { + query target_features_whitelist(_: CrateNum) -> FxHashMap> { + storage(ArenaCacheSelector<'tcx>) eval_always desc { "looking up the whitelist of target features" } } diff --git a/src/librustc_middle/ty/context.rs b/src/librustc_middle/ty/context.rs index c5813ae57a653..ae06008d0f9f6 100644 --- a/src/librustc_middle/ty/context.rs +++ b/src/librustc_middle/ty/context.rs @@ -991,22 +991,15 @@ pub struct GlobalCtxt<'tcx> { } impl<'tcx> TyCtxt<'tcx> { - pub fn alloc_steal_mir(self, mir: Body<'tcx>) -> &'tcx Steal> { - self.arena.alloc(Steal::new(mir)) + pub fn alloc_steal_mir(self, mir: Body<'tcx>) -> Steal> { + Steal::new(mir) } pub fn alloc_steal_promoted( self, promoted: IndexVec>, - ) -> &'tcx Steal>> { - self.arena.alloc(Steal::new(promoted)) - } - - pub fn intern_promoted( - self, - promoted: IndexVec>, - ) -> &'tcx IndexVec> { - self.arena.alloc(promoted) + ) -> Steal>> { + Steal::new(promoted) } pub fn alloc_adt_def( @@ -1016,8 +1009,7 @@ impl<'tcx> TyCtxt<'tcx> { variants: IndexVec, repr: ReprOptions, ) -> &'tcx ty::AdtDef { - let def = ty::AdtDef::new(self, did, kind, variants, repr); - self.arena.alloc(def) + self.arena.alloc(ty::AdtDef::new(self, did, kind, variants, repr)) } pub fn intern_const_alloc(self, alloc: Allocation) -> &'tcx Allocation { @@ -2745,7 +2737,7 @@ pub fn provide(providers: &mut ty::query::Providers<'_>) { }; providers.features_query = |tcx, cnum| { assert_eq!(cnum, LOCAL_CRATE); - tcx.arena.alloc(tcx.sess.features_untracked().clone()) + tcx.sess.features_untracked() }; providers.is_panic_runtime = |tcx, cnum| { assert_eq!(cnum, LOCAL_CRATE); diff --git a/src/librustc_middle/ty/mod.rs b/src/librustc_middle/ty/mod.rs index af278f9000ec0..bef74289fd850 100644 --- a/src/librustc_middle/ty/mod.rs +++ b/src/librustc_middle/ty/mod.rs @@ -256,13 +256,13 @@ impl AssocItem { /// it is relatively expensive. Instead, items are indexed by `Symbol` and hygienic comparison is /// done only on items with the same name. #[derive(Debug, Clone, PartialEq, HashStable)] -pub struct AssociatedItems { - items: SortedIndexMultiMap, +pub struct AssociatedItems<'tcx> { + items: SortedIndexMultiMap, } -impl AssociatedItems { +impl<'tcx> AssociatedItems<'tcx> { /// Constructs an `AssociatedItems` map from a series of `ty::AssocItem`s in definition order. - pub fn new(items_in_def_order: impl IntoIterator) -> Self { + pub fn new(items_in_def_order: impl IntoIterator) -> Self { let items = items_in_def_order.into_iter().map(|item| (item.ident.name, item)).collect(); AssociatedItems { items } } @@ -272,7 +272,7 @@ impl AssociatedItems { /// New code should avoid relying on definition order. If you need a particular associated item /// for a known trait, make that trait a lang item instead of indexing this array. pub fn in_definition_order(&self) -> impl '_ + Iterator { - self.items.iter().map(|(_, v)| v) + self.items.iter().map(|(_, v)| *v) } /// Returns an iterator over all associated items with the given name, ignoring hygiene. @@ -280,7 +280,7 @@ impl AssociatedItems { &self, name: Symbol, ) -> impl '_ + Iterator { - self.items.get_by_key(&name) + self.items.get_by_key(&name).map(|v| *v) } /// Returns an iterator over all associated items with the given name. @@ -2671,7 +2671,7 @@ impl<'tcx> TyCtxt<'tcx> { .and_then(|def_id| self.hir().get(self.hir().as_local_hir_id(def_id)).ident()) } - pub fn opt_associated_item(self, def_id: DefId) -> Option { + pub fn opt_associated_item(self, def_id: DefId) -> Option<&'tcx AssocItem> { let is_associated_item = if let Some(def_id) = def_id.as_local() { match self.hir().get(self.hir().as_local_hir_id(def_id)) { Node::TraitItem(_) | Node::ImplItem(_) => true, diff --git a/src/librustc_middle/ty/query/plumbing.rs b/src/librustc_middle/ty/query/plumbing.rs index 068322b08b7a3..e4237df592394 100644 --- a/src/librustc_middle/ty/query/plumbing.rs +++ b/src/librustc_middle/ty/query/plumbing.rs @@ -213,7 +213,7 @@ macro_rules! query_storage { <<$K as Key>::CacheSelector as CacheSelector<$K, $V>>::Cache }; ([storage($ty:ty) $($rest:tt)*][$K:ty, $V:ty]) => { - $ty + <$ty as CacheSelector<$K, $V>>::Cache }; ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => { query_storage!([$($($modifiers)*)*][$($args)*]) @@ -328,6 +328,10 @@ macro_rules! define_queries_inner { $(impl<$tcx> QueryConfig> for queries::$name<$tcx> { type Key = $($K)*; type Value = $V; + type Stored = < + query_storage!([$($modifiers)*][$($K)*, $V]) + as QueryStorage + >::Stored; const NAME: &'static str = stringify!($name); const CATEGORY: ProfileCategory = $category; } @@ -426,8 +430,10 @@ macro_rules! define_queries_inner { $($(#[$attr])* #[inline(always)] - pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V { - self.at(DUMMY_SP).$name(key) + pub fn $name(self, key: query_helper_param_ty!($($K)*)) + -> as QueryConfig>>::Stored + { + self.at(DUMMY_SP).$name(key.into_query_param()) })* /// All self-profiling events generated by the query engine use @@ -463,7 +469,9 @@ macro_rules! define_queries_inner { impl TyCtxtAt<$tcx> { $($(#[$attr])* #[inline(always)] - pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V { + pub fn $name(self, key: query_helper_param_ty!($($K)*)) + -> as QueryConfig>>::Stored + { get_query::, _>(self.tcx, self.span, key.into_query_param()) })* } diff --git a/src/librustc_middle/ty/trait_def.rs b/src/librustc_middle/ty/trait_def.rs index 72dfebd8ccd44..89bcb240d9013 100644 --- a/src/librustc_middle/ty/trait_def.rs +++ b/src/librustc_middle/ty/trait_def.rs @@ -187,7 +187,7 @@ pub(super) fn all_local_trait_impls<'tcx>( } // Query provider for `trait_impls_of`. -pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> &TraitImpls { +pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> TraitImpls { let mut impls = TraitImpls::default(); { @@ -219,7 +219,7 @@ pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> &Trai } } - tcx.arena.alloc(impls) + impls } impl<'a> HashStable> for TraitImpls { diff --git a/src/librustc_mir/borrow_check/mod.rs b/src/librustc_mir/borrow_check/mod.rs index af5c392aa986e..b95a1043d92cb 100644 --- a/src/librustc_mir/borrow_check/mod.rs +++ b/src/librustc_mir/borrow_check/mod.rs @@ -92,7 +92,7 @@ pub fn provide(providers: &mut Providers<'_>) { *providers = Providers { mir_borrowck, ..*providers }; } -fn mir_borrowck(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &BorrowCheckResult<'_> { +fn mir_borrowck(tcx: TyCtxt<'_>, def_id: LocalDefId) -> BorrowCheckResult<'_> { let (input_body, promoted) = tcx.mir_validated(def_id); debug!("run query mir_borrowck: {}", tcx.def_path_str(def_id.to_def_id())); @@ -103,7 +103,7 @@ fn mir_borrowck(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &BorrowCheckResult<'_> { }); debug!("mir_borrowck done"); - tcx.arena.alloc(opt_closure_req) + opt_closure_req } fn do_mir_borrowck<'a, 'tcx>( diff --git a/src/librustc_mir/shim.rs b/src/librustc_mir/shim.rs index 4bef2bc711ba9..dfa1bb764e47a 100644 --- a/src/librustc_mir/shim.rs +++ b/src/librustc_mir/shim.rs @@ -26,7 +26,7 @@ pub fn provide(providers: &mut Providers<'_>) { providers.mir_shims = make_shim; } -fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> &'tcx Body<'tcx> { +fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<'tcx> { debug!("make_shim({:?})", instance); let mut result = match instance { @@ -128,7 +128,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> &'tcx debug!("make_shim({:?}) = {:?}", instance, result); - tcx.arena.alloc(result) + result } #[derive(Copy, Clone, Debug, PartialEq)] @@ -828,7 +828,7 @@ fn build_call_shim<'tcx>( body } -pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> &Body<'_> { +pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> { debug_assert!(tcx.is_constructor(ctor_id)); let span = @@ -895,5 +895,5 @@ pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> &Body<'_> { |_, _| Ok(()), ); - tcx.arena.alloc(body) + body } diff --git a/src/librustc_mir/transform/mod.rs b/src/librustc_mir/transform/mod.rs index 9f63340065fbf..1d982d18eeb0d 100644 --- a/src/librustc_mir/transform/mod.rs +++ b/src/librustc_mir/transform/mod.rs @@ -60,7 +60,7 @@ fn is_mir_available(tcx: TyCtxt<'_>, def_id: DefId) -> bool { /// Finds the full set of `DefId`s within the current crate that have /// MIR associated with them. -fn mir_keys(tcx: TyCtxt<'_>, krate: CrateNum) -> &FxHashSet { +fn mir_keys(tcx: TyCtxt<'_>, krate: CrateNum) -> FxHashSet { assert_eq!(krate, LOCAL_CRATE); let mut set = FxHashSet::default(); @@ -97,7 +97,7 @@ fn mir_keys(tcx: TyCtxt<'_>, krate: CrateNum) -> &FxHashSet { .krate() .visit_all_item_likes(&mut GatherCtors { tcx, set: &mut set }.as_deep_visitor()); - tcx.arena.alloc(set) + set } /// Where a specific `mir::Body` comes from. @@ -211,7 +211,7 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def_id: DefId) -> ConstQualifs { validator.qualifs_in_return_place() } -fn mir_const(tcx: TyCtxt<'_>, def_id: DefId) -> &Steal> { +fn mir_const(tcx: TyCtxt<'_>, def_id: DefId) -> Steal> { let def_id = def_id.expect_local(); // Unsafety check uses the raw mir, so make sure it is run @@ -241,7 +241,7 @@ fn mir_const(tcx: TyCtxt<'_>, def_id: DefId) -> &Steal> { fn mir_validated( tcx: TyCtxt<'tcx>, def_id: LocalDefId, -) -> (&'tcx Steal>, &'tcx Steal>>) { +) -> (Steal>, Steal>>) { // Ensure that we compute the `mir_const_qualif` for constants at // this point, before we steal the mir-const result. let _ = tcx.mir_const_qualif(def_id.to_def_id()); @@ -360,7 +360,7 @@ fn run_optimization_passes<'tcx>( ); } -fn optimized_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &Body<'_> { +fn optimized_mir(tcx: TyCtxt<'_>, def_id: DefId) -> Body<'_> { if tcx.is_constructor(def_id) { // There's no reason to run all of the MIR passes on constructors when // we can just output the MIR we want directly. This also saves const @@ -381,12 +381,12 @@ fn optimized_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &Body<'_> { debug_assert!(!body.has_free_regions(), "Free regions in optimized MIR"); - tcx.arena.alloc(body) + body } -fn promoted_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &IndexVec> { +fn promoted_mir(tcx: TyCtxt<'_>, def_id: DefId) -> IndexVec> { if tcx.is_constructor(def_id) { - return tcx.intern_promoted(IndexVec::new()); + return IndexVec::new(); } let def_id = def_id.expect_local(); @@ -401,5 +401,5 @@ fn promoted_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &IndexVec> debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR"); - tcx.intern_promoted(promoted) + promoted } diff --git a/src/librustc_mir_build/build/mod.rs b/src/librustc_mir_build/build/mod.rs index e2f75e6f227f1..2ce2627987a01 100644 --- a/src/librustc_mir_build/build/mod.rs +++ b/src/librustc_mir_build/build/mod.rs @@ -21,7 +21,7 @@ use rustc_target::spec::PanicStrategy; use super::lints; -crate fn mir_built(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &ty::steal::Steal> { +crate fn mir_built(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::steal::Steal> { tcx.alloc_steal_mir(mir_build(tcx, def_id)) } diff --git a/src/librustc_mir_build/lints.rs b/src/librustc_mir_build/lints.rs index 990f55f6d422f..69bfad6b13999 100644 --- a/src/librustc_mir_build/lints.rs +++ b/src/librustc_mir_build/lints.rs @@ -24,7 +24,7 @@ crate fn check<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, def_id: LocalDefId) { Some(AssocItem { container: AssocItemContainer::TraitContainer(trait_def_id), .. }) => { - let trait_substs_count = tcx.generics_of(trait_def_id).count(); + let trait_substs_count = tcx.generics_of(*trait_def_id).count(); &InternalSubsts::identity_for_item(tcx, def_id.to_def_id())[..trait_substs_count] } _ => &[], diff --git a/src/librustc_passes/diagnostic_items.rs b/src/librustc_passes/diagnostic_items.rs index bed2221fe6076..d91f49554ff48 100644 --- a/src/librustc_passes/diagnostic_items.rs +++ b/src/librustc_passes/diagnostic_items.rs @@ -93,18 +93,18 @@ fn extract(attrs: &[ast::Attribute]) -> Option { } /// Traverse and collect the diagnostic items in the current -fn collect<'tcx>(tcx: TyCtxt<'tcx>) -> &'tcx FxHashMap { +fn collect<'tcx>(tcx: TyCtxt<'tcx>) -> FxHashMap { // Initialize the collector. let mut collector = DiagnosticItemCollector::new(tcx); // Collect diagnostic items in this crate. tcx.hir().krate().visit_all_item_likes(&mut collector); - tcx.arena.alloc(collector.items) + collector.items } /// Traverse and collect all the diagnostic items in all crates. -fn collect_all<'tcx>(tcx: TyCtxt<'tcx>) -> &'tcx FxHashMap { +fn collect_all<'tcx>(tcx: TyCtxt<'tcx>) -> FxHashMap { // Initialize the collector. let mut collector = FxHashMap::default(); @@ -115,7 +115,7 @@ fn collect_all<'tcx>(tcx: TyCtxt<'tcx>) -> &'tcx FxHashMap { } } - tcx.arena.alloc(collector) + collector } pub fn provide(providers: &mut Providers<'_>) { diff --git a/src/librustc_passes/lang_items.rs b/src/librustc_passes/lang_items.rs index c48e6f9133bcd..779fb8039d157 100644 --- a/src/librustc_passes/lang_items.rs +++ b/src/librustc_passes/lang_items.rs @@ -169,6 +169,6 @@ fn collect(tcx: TyCtxt<'_>) -> LanguageItems { pub fn provide(providers: &mut Providers<'_>) { providers.get_lang_items = |tcx, id| { assert_eq!(id, LOCAL_CRATE); - tcx.arena.alloc(collect(tcx)) + collect(tcx) }; } diff --git a/src/librustc_passes/lib_features.rs b/src/librustc_passes/lib_features.rs index dd972f37569f6..31c7ba2a4b205 100644 --- a/src/librustc_passes/lib_features.rs +++ b/src/librustc_passes/lib_features.rs @@ -138,6 +138,6 @@ fn collect(tcx: TyCtxt<'_>) -> LibFeatures { pub fn provide(providers: &mut Providers<'_>) { providers.get_lib_features = |tcx, id| { assert_eq!(id, LOCAL_CRATE); - tcx.arena.alloc(collect(tcx)) + collect(tcx) }; } diff --git a/src/librustc_passes/reachable.rs b/src/librustc_passes/reachable.rs index d913486cb0e18..1a86babbc46cf 100644 --- a/src/librustc_passes/reachable.rs +++ b/src/librustc_passes/reachable.rs @@ -23,7 +23,7 @@ use rustc_target::spec::abi::Abi; // Returns true if the given item must be inlined because it may be // monomorphized or it was marked with `#[inline]`. This will only return // true for functions. -fn item_might_be_inlined(tcx: TyCtxt<'tcx>, item: &hir::Item<'_>, attrs: CodegenFnAttrs) -> bool { +fn item_might_be_inlined(tcx: TyCtxt<'tcx>, item: &hir::Item<'_>, attrs: &CodegenFnAttrs) -> bool { if attrs.requests_inline() { return true; } diff --git a/src/librustc_passes/stability.rs b/src/librustc_passes/stability.rs index ad9934d379ac5..054748c09fc44 100644 --- a/src/librustc_passes/stability.rs +++ b/src/librustc_passes/stability.rs @@ -480,7 +480,7 @@ pub(crate) fn provide(providers: &mut Providers<'_>) { *providers = Providers { check_mod_unstable_api_usage, ..*providers }; providers.stability_index = |tcx, cnum| { assert_eq!(cnum, LOCAL_CRATE); - tcx.arena.alloc(new_index(tcx)) + new_index(tcx) }; } diff --git a/src/librustc_query_system/Cargo.toml b/src/librustc_query_system/Cargo.toml index 6629d841fc8bb..392e19e1f4471 100644 --- a/src/librustc_query_system/Cargo.toml +++ b/src/librustc_query_system/Cargo.toml @@ -10,6 +10,7 @@ path = "lib.rs" doctest = false [dependencies] +arena = { path = "../libarena" } log = { version = "0.4", features = ["release_max_level_info", "std"] } rustc-rayon-core = "0.3.0" rustc_data_structures = { path = "../librustc_data_structures" } diff --git a/src/librustc_query_system/query/caches.rs b/src/librustc_query_system/query/caches.rs index 0c0335ba04f9a..6a47abc5b4612 100644 --- a/src/librustc_query_system/query/caches.rs +++ b/src/librustc_query_system/query/caches.rs @@ -2,19 +2,29 @@ use crate::dep_graph::DepNodeIndex; use crate::query::plumbing::{QueryLookup, QueryState}; use crate::query::QueryContext; +use arena::TypedArena; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sharded::Sharded; +use rustc_data_structures::sync::WorkerLocal; use std::default::Default; use std::hash::Hash; use std::marker::PhantomData; -pub trait CacheSelector { - type Cache: QueryCache; +pub trait CacheSelector { + type Cache; } -pub trait QueryCache: Default { - type Key: Hash; +pub trait QueryStorage: Default { type Value; + type Stored: Clone; + + /// Store a value without putting it in the cache. + /// This is meant to be used with cycle errors. + fn store_nocache(&self, value: Self::Value) -> Self::Stored; +} + +pub trait QueryCache: QueryStorage { + type Key: Hash; type Sharded: Default; /// Checks if the query is already computed and in the cache. @@ -30,7 +40,7 @@ pub trait QueryCache: Default { on_miss: OnMiss, ) -> R where - OnHit: FnOnce(&Self::Value, DepNodeIndex) -> R, + OnHit: FnOnce(&Self::Stored, DepNodeIndex) -> R, OnMiss: FnOnce(Self::Key, QueryLookup<'_, CTX, Self::Key, Self::Sharded>) -> R; fn complete( @@ -40,7 +50,7 @@ pub trait QueryCache: Default { key: Self::Key, value: Self::Value, index: DepNodeIndex, - ); + ) -> Self::Stored; fn iter( &self, @@ -66,9 +76,19 @@ impl Default for DefaultCache { } } +impl QueryStorage for DefaultCache { + type Value = V; + type Stored = V; + + #[inline] + fn store_nocache(&self, value: Self::Value) -> Self::Stored { + // We have no dedicated storage + value + } +} + impl QueryCache for DefaultCache { type Key = K; - type Value = V; type Sharded = FxHashMap; #[inline(always)] @@ -99,8 +119,94 @@ impl QueryCache for DefaultCache { key: K, value: V, index: DepNodeIndex, - ) { - lock_sharded_storage.insert(key, (value, index)); + ) -> Self::Stored { + lock_sharded_storage.insert(key, (value.clone(), index)); + value + } + + fn iter( + &self, + shards: &Sharded, + get_shard: impl Fn(&mut L) -> &mut Self::Sharded, + f: impl for<'a> FnOnce(Box + 'a>) -> R, + ) -> R { + let mut shards = shards.lock_shards(); + let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect(); + let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1)); + f(Box::new(results)) + } +} + +pub struct ArenaCacheSelector<'tcx>(PhantomData<&'tcx ()>); + +impl<'tcx, K: Eq + Hash, V: 'tcx> CacheSelector for ArenaCacheSelector<'tcx> { + type Cache = ArenaCache<'tcx, K, V>; +} + +pub struct ArenaCache<'tcx, K, V> { + arena: WorkerLocal>, + phantom: PhantomData<(K, &'tcx V)>, +} + +impl<'tcx, K, V> Default for ArenaCache<'tcx, K, V> { + fn default() -> Self { + ArenaCache { arena: WorkerLocal::new(|_| TypedArena::default()), phantom: PhantomData } + } +} + +impl<'tcx, K: Eq + Hash, V: 'tcx> QueryStorage for ArenaCache<'tcx, K, V> { + type Value = V; + type Stored = &'tcx V; + + #[inline] + fn store_nocache(&self, value: Self::Value) -> Self::Stored { + let value = self.arena.alloc((value, DepNodeIndex::INVALID)); + let value = unsafe { &*(&value.0 as *const _) }; + &value + } +} + +impl<'tcx, K: Eq + Hash, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V> { + type Key = K; + type Sharded = FxHashMap; + + #[inline(always)] + fn lookup( + &self, + state: &QueryState, + key: K, + on_hit: OnHit, + on_miss: OnMiss, + ) -> R + where + OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R, + OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R, + { + let mut lookup = state.get_lookup(&key); + let lock = &mut *lookup.lock; + + let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key); + + if let Some((_, value)) = result { + on_hit(&&value.0, value.1) + } else { + on_miss(key, lookup) + } + } + + #[inline] + fn complete( + &self, + _: CTX, + lock_sharded_storage: &mut Self::Sharded, + key: K, + value: V, + index: DepNodeIndex, + ) -> Self::Stored { + let value = self.arena.alloc((value, index)); + let value = unsafe { &*(value as *const _) }; + lock_sharded_storage.insert(key, value); + &value.0 } fn iter( diff --git a/src/librustc_query_system/query/config.rs b/src/librustc_query_system/query/config.rs index 06e3302b26360..710ec3bfb0b3c 100644 --- a/src/librustc_query_system/query/config.rs +++ b/src/librustc_query_system/query/config.rs @@ -20,7 +20,8 @@ pub trait QueryConfig { const CATEGORY: ProfileCategory; type Key: Eq + Hash + Clone + Debug; - type Value: Clone; + type Value; + type Stored: Clone; } pub trait QueryAccessors: QueryConfig { @@ -28,7 +29,7 @@ pub trait QueryAccessors: QueryConfig { const EVAL_ALWAYS: bool; const DEP_KIND: CTX::DepKind; - type Cache: QueryCache; + type Cache: QueryCache; // Don't use this method to access query results, instead use the methods on TyCtxt fn query_state<'a>(tcx: CTX) -> &'a QueryState; diff --git a/src/librustc_query_system/query/mod.rs b/src/librustc_query_system/query/mod.rs index b1677c5c93da9..49097725bc9b9 100644 --- a/src/librustc_query_system/query/mod.rs +++ b/src/librustc_query_system/query/mod.rs @@ -7,7 +7,9 @@ pub use self::job::deadlock; pub use self::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo}; mod caches; -pub use self::caches::{CacheSelector, DefaultCacheSelector, QueryCache}; +pub use self::caches::{ + ArenaCacheSelector, CacheSelector, DefaultCacheSelector, QueryCache, QueryStorage, +}; mod config; pub use self::config::{QueryAccessors, QueryConfig, QueryDescription}; diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index 9da13f236643b..0aeec269e617a 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -148,7 +148,6 @@ struct JobOwner<'tcx, CTX: QueryContext, C> where C: QueryCache, C::Key: Eq + Hash + Clone + Debug, - C::Value: Clone, { state: &'tcx QueryState, key: C::Key, @@ -159,7 +158,6 @@ impl<'tcx, CTX: QueryContext, C> JobOwner<'tcx, CTX, C> where C: QueryCache, C::Key: Eq + Hash + Clone + Debug, - C::Value: Clone, { /// Either gets a `JobOwner` corresponding the query, allowing us to /// start executing the query, or returns with the result of the query. @@ -177,7 +175,7 @@ where mut lookup: QueryLookup<'a, CTX, C::Key, C::Sharded>, ) -> TryGetJob<'b, CTX, C> where - Q: QueryDescription, + Q: QueryDescription, CTX: QueryContext, { let lock = &mut *lookup.lock; @@ -229,7 +227,8 @@ where // so we just return the error. #[cfg(not(parallel_compiler))] return TryGetJob::Cycle(cold_path(|| { - Q::handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span)) + let value = Q::handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span)); + Q::query_state(tcx).cache.store_nocache(value) })); // With parallel queries we might just have to wait on some other @@ -239,7 +238,9 @@ where let result = latch.wait_on(tcx, span); if let Err(cycle) = result { - return TryGetJob::Cycle(Q::handle_cycle_error(tcx, cycle)); + let value = Q::handle_cycle_error(tcx, cycle); + let value = Q::query_state(tcx).cache.store_nocache(value); + return TryGetJob::Cycle(value); } let cached = try_get_cached( @@ -261,7 +262,7 @@ where /// Completes the query by updating the query cache with the `result`, /// signals the waiter and forgets the JobOwner, so it won't poison the query #[inline(always)] - fn complete(self, tcx: CTX, result: &C::Value, dep_node_index: DepNodeIndex) { + fn complete(self, tcx: CTX, result: C::Value, dep_node_index: DepNodeIndex) -> C::Stored { // We can move out of `self` here because we `mem::forget` it below let key = unsafe { ptr::read(&self.key) }; let state = self.state; @@ -269,18 +270,18 @@ where // Forget ourself so our destructor won't poison the query mem::forget(self); - let job = { - let result = result.clone(); + let (job, result) = { let mut lock = state.shards.get_shard_by_value(&key).lock(); let job = match lock.active.remove(&key).unwrap() { QueryResult::Started(job) => job, QueryResult::Poisoned => panic!(), }; - state.cache.complete(tcx, &mut lock.cache, key, result, dep_node_index); - job + let result = state.cache.complete(tcx, &mut lock.cache, key, result, dep_node_index); + (job, result) }; job.signal_complete(); + result } } @@ -297,7 +298,6 @@ where impl<'tcx, CTX: QueryContext, C: QueryCache> Drop for JobOwner<'tcx, CTX, C> where C::Key: Eq + Hash + Clone + Debug, - C::Value: Clone, { #[inline(never)] #[cold] @@ -331,7 +331,6 @@ pub struct CycleError { enum TryGetJob<'tcx, CTX: QueryContext, C: QueryCache> where C::Key: Eq + Hash + Clone + Debug, - C::Value: Clone, { /// The query is not yet started. Contains a guard to the cache eventually used to start it. NotYetStarted(JobOwner<'tcx, CTX, C>), @@ -340,10 +339,10 @@ where /// Returns the result of the query and its dep-node index /// if it succeeded or a cycle error if it failed. #[cfg(parallel_compiler)] - JobCompleted((C::Value, DepNodeIndex)), + JobCompleted((C::Stored, DepNodeIndex)), /// Trying to execute the query resulted in a cycle. - Cycle(C::Value), + Cycle(C::Stored), } /// Checks if the query is already computed and in the cache. @@ -362,7 +361,7 @@ fn try_get_cached( where C: QueryCache, CTX: QueryContext, - OnHit: FnOnce(&C::Value, DepNodeIndex) -> R, + OnHit: FnOnce(&C::Stored, DepNodeIndex) -> R, OnMiss: FnOnce(C::Key, QueryLookup<'_, CTX, C::Key, C::Sharded>) -> R, { state.cache.lookup( @@ -388,7 +387,7 @@ fn try_execute_query( span: Span, key: Q::Key, lookup: QueryLookup<'_, CTX, Q::Key, ::Sharded>, -) -> Q::Value +) -> Q::Stored where Q: QueryDescription, CTX: QueryContext, @@ -427,9 +426,7 @@ where tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics); } - job.complete(tcx, &result, dep_node_index); - - return result; + return job.complete(tcx, result, dep_node_index); } let dep_node = Q::to_dep_node(tcx, &key); @@ -454,8 +451,7 @@ where }) }); if let Some((result, dep_node_index)) = loaded { - job.complete(tcx, &result, dep_node_index); - return result; + return job.complete(tcx, result, dep_node_index); } } @@ -558,7 +554,7 @@ fn force_query_with_job( key: Q::Key, job: JobOwner<'_, CTX, Q::Cache>, dep_node: DepNode, -) -> (Q::Value, DepNodeIndex) +) -> (Q::Stored, DepNodeIndex) where Q: QueryDescription, CTX: QueryContext, @@ -603,13 +599,13 @@ where } } - job.complete(tcx, &result, dep_node_index); + let result = job.complete(tcx, result, dep_node_index); (result, dep_node_index) } #[inline(never)] -pub fn get_query(tcx: CTX, span: Span, key: Q::Key) -> Q::Value +pub fn get_query(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored where Q: QueryDescription, CTX: QueryContext, diff --git a/src/librustc_resolve/late/lifetimes.rs b/src/librustc_resolve/late/lifetimes.rs index d95481b959037..a062330ba4be6 100644 --- a/src/librustc_resolve/late/lifetimes.rs +++ b/src/librustc_resolve/late/lifetimes.rs @@ -294,7 +294,7 @@ pub fn provide(providers: &mut ty::query::Providers<'_>) { /// entire crate. You should not read the result of this query /// directly, but rather use `named_region_map`, `is_late_bound_map`, /// etc. -fn resolve_lifetimes(tcx: TyCtxt<'_>, for_krate: CrateNum) -> &ResolveLifetimes { +fn resolve_lifetimes(tcx: TyCtxt<'_>, for_krate: CrateNum) -> ResolveLifetimes { assert_eq!(for_krate, LOCAL_CRATE); let named_region_map = krate(tcx); @@ -314,7 +314,7 @@ fn resolve_lifetimes(tcx: TyCtxt<'_>, for_krate: CrateNum) -> &ResolveLifetimes map.insert(hir_id.local_id, v); } - tcx.arena.alloc(rl) + rl } fn krate(tcx: TyCtxt<'_>) -> NamedRegionMap { diff --git a/src/librustc_trait_selection/traits/specialize/mod.rs b/src/librustc_trait_selection/traits/specialize/mod.rs index 897ac0d736e08..f2b43754acaea 100644 --- a/src/librustc_trait_selection/traits/specialize/mod.rs +++ b/src/librustc_trait_selection/traits/specialize/mod.rs @@ -254,7 +254,7 @@ fn fulfill_implication<'a, 'tcx>( pub(super) fn specialization_graph_provider( tcx: TyCtxt<'_>, trait_id: DefId, -) -> &specialization_graph::Graph { +) -> specialization_graph::Graph { let mut sg = specialization_graph::Graph::new(); let mut trait_impls: Vec<_> = tcx.all_impls(trait_id).collect(); @@ -286,7 +286,7 @@ pub(super) fn specialization_graph_provider( } } - tcx.arena.alloc(sg) + sg } fn report_overlap_conflict( diff --git a/src/librustc_ty/ty.rs b/src/librustc_ty/ty.rs index 16f395a4fec8d..b65f1cd6fac4e 100644 --- a/src/librustc_ty/ty.rs +++ b/src/librustc_ty/ty.rs @@ -220,9 +220,9 @@ fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: DefId) -> &[DefId] { } } -fn associated_items(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::AssociatedItems { +fn associated_items(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AssociatedItems<'_> { let items = tcx.associated_item_def_ids(def_id).iter().map(|did| tcx.associated_item(*did)); - tcx.arena.alloc(ty::AssociatedItems::new(items)) + ty::AssociatedItems::new(items) } fn def_span(tcx: TyCtxt<'_>, def_id: DefId) -> Span { diff --git a/src/librustc_typeck/coherence/inherent_impls.rs b/src/librustc_typeck/coherence/inherent_impls.rs index 6817312e303ae..653b7b8f2a5ba 100644 --- a/src/librustc_typeck/coherence/inherent_impls.rs +++ b/src/librustc_typeck/coherence/inherent_impls.rs @@ -17,13 +17,13 @@ use rustc_ast::ast; use rustc_span::Span; /// On-demand query: yields a map containing all types mapped to their inherent impls. -pub fn crate_inherent_impls(tcx: TyCtxt<'_>, crate_num: CrateNum) -> &CrateInherentImpls { +pub fn crate_inherent_impls(tcx: TyCtxt<'_>, crate_num: CrateNum) -> CrateInherentImpls { assert_eq!(crate_num, LOCAL_CRATE); let krate = tcx.hir().krate(); let mut collect = InherentCollect { tcx, impls_map: Default::default() }; krate.visit_all_item_likes(&mut collect); - tcx.arena.alloc(collect.impls_map) + collect.impls_map } /// On-demand query: yields a vector of the inherent impls for a specific type. diff --git a/src/librustc_typeck/collect.rs b/src/librustc_typeck/collect.rs index 9f70a0fbc2462..e6aa53dd4f3b7 100644 --- a/src/librustc_typeck/collect.rs +++ b/src/librustc_typeck/collect.rs @@ -1002,7 +1002,7 @@ fn super_predicates_of(tcx: TyCtxt<'_>, trait_def_id: DefId) -> ty::GenericPredi ty::GenericPredicates { parent: None, predicates: superbounds } } -fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::TraitDef { +fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef { let hir_id = tcx.hir().as_local_hir_id(def_id.expect_local()); let item = tcx.hir().expect_item(hir_id); @@ -1033,16 +1033,7 @@ fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::TraitDef { ty::trait_def::TraitSpecializationKind::None }; let def_path_hash = tcx.def_path_hash(def_id); - let def = ty::TraitDef::new( - def_id, - unsafety, - paren_sugar, - is_auto, - is_marker, - spec_kind, - def_path_hash, - ); - tcx.arena.alloc(def) + ty::TraitDef::new(def_id, unsafety, paren_sugar, is_auto, is_marker, spec_kind, def_path_hash) } fn has_late_bound_regions<'tcx>(tcx: TyCtxt<'tcx>, node: Node<'tcx>) -> Option { @@ -1158,7 +1149,7 @@ fn has_late_bound_regions<'tcx>(tcx: TyCtxt<'tcx>, node: Node<'tcx>) -> Option, def_id: DefId) -> &ty::Generics { +fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::Generics { use rustc_hir::*; let hir_id = tcx.hir().as_local_hir_id(def_id.expect_local()); @@ -1403,14 +1394,14 @@ fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::Generics { let param_def_id_to_index = params.iter().map(|param| (param.def_id, param.index)).collect(); - tcx.arena.alloc(ty::Generics { + ty::Generics { parent: parent_def_id, parent_count, params, param_def_id_to_index, has_self: has_self || parent_has_self, has_late_bound_regions: has_late_bound_regions(tcx, node), - }) + } } fn are_suggestable_generic_args(generic_args: &[hir::GenericArg<'_>]) -> bool { diff --git a/src/librustc_typeck/outlives/mod.rs b/src/librustc_typeck/outlives/mod.rs index 15e0473bbf596..a49d8e5ed0f0a 100644 --- a/src/librustc_typeck/outlives/mod.rs +++ b/src/librustc_typeck/outlives/mod.rs @@ -58,7 +58,7 @@ fn inferred_outlives_of(tcx: TyCtxt<'_>, item_def_id: DefId) -> &[(ty::Predicate } } -fn inferred_outlives_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> &CratePredicatesMap<'_> { +fn inferred_outlives_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> CratePredicatesMap<'_> { assert_eq!(crate_num, LOCAL_CRATE); // Compute a map from each struct/enum/union S to the **explicit** @@ -105,5 +105,5 @@ fn inferred_outlives_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> &CratePredic }) .collect(); - tcx.arena.alloc(ty::CratePredicatesMap { predicates }) + ty::CratePredicatesMap { predicates } } diff --git a/src/librustc_typeck/variance/mod.rs b/src/librustc_typeck/variance/mod.rs index 2d78ac4b3c58f..631f551ade4fe 100644 --- a/src/librustc_typeck/variance/mod.rs +++ b/src/librustc_typeck/variance/mod.rs @@ -29,12 +29,12 @@ pub fn provide(providers: &mut Providers<'_>) { *providers = Providers { variances_of, crate_variances, ..*providers }; } -fn crate_variances(tcx: TyCtxt<'_>, crate_num: CrateNum) -> &CrateVariancesMap<'_> { +fn crate_variances(tcx: TyCtxt<'_>, crate_num: CrateNum) -> CrateVariancesMap<'_> { assert_eq!(crate_num, LOCAL_CRATE); let mut arena = arena::TypedArena::default(); let terms_cx = terms::determine_parameters_to_be_inferred(tcx, &mut arena); let constraints_cx = constraints::add_constraints_from_crate(terms_cx); - tcx.arena.alloc(solve::solve_constraints(constraints_cx)) + solve::solve_constraints(constraints_cx) } fn variances_of(tcx: TyCtxt<'_>, item_def_id: DefId) -> &[ty::Variance] { diff --git a/src/test/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs b/src/test/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs index d271b8645746e..efc62361694a5 100644 --- a/src/test/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs +++ b/src/test/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs @@ -53,7 +53,7 @@ impl CodegenBackend for TheBackend { rustc_symbol_mangling::provide(providers); providers.target_features_whitelist = |tcx, _cnum| { - tcx.arena.alloc(Default::default()) // Just a dummy + Default::default() // Just a dummy }; providers.is_reachable_non_generic = |_tcx, _defid| true; providers.exported_symbols = |_tcx, _crate| &[];