Auto merge of #70674 - cjgillot:query-arena-all, r=matthewjasper

Have the per-query caches store the results on arenas

This PR leverages the cache for each query to serve as storage area for the query results.

It introduces a new cache `ArenaCache`, which moves the result to an arena,
and only stores the reference in the hash map.
This allows to remove a sizeable part of the usage of the global `TyCtxt` arena.

I only migrated queries that already used arenas before.

Author: bors

Cargo.lock

@@ -4126,6 +4126,7 @@ dependencies = [
 name = "rustc_query_system"
 version = "0.0.0"
 dependencies = [
+ "arena",
  "log",
  "parking_lot 0.10.2",
  "rustc-rayon-core",

src/librustc_codegen_llvm/attributes.rs

@@ -253,7 +253,7 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
         inline(cx, llfn, attributes::InlineAttr::Hint);
     }
 
-    inline(cx, llfn, codegen_fn_attrs.inline);
+    inline(cx, llfn, codegen_fn_attrs.inline.clone());
 
     // The `uwtable` attribute according to LLVM is:
     //
@@ -350,15 +350,12 @@ pub fn provide(providers: &mut Providers<'_>) {
         if tcx.sess.opts.actually_rustdoc {
             // rustdoc needs to be able to document functions that use all the features, so
             // whitelist them all
-            tcx.arena
-                .alloc(llvm_util::all_known_features().map(|(a, b)| (a.to_string(), b)).collect())
+            llvm_util::all_known_features().map(|(a, b)| (a.to_string(), b)).collect()
         } else {
-            tcx.arena.alloc(
-                llvm_util::target_feature_whitelist(tcx.sess)
-                    .iter()
-                    .map(|&(a, b)| (a.to_string(), b))
-                    .collect(),
-            )
+            llvm_util::target_feature_whitelist(tcx.sess)
+                .iter()
+                .map(|&(a, b)| (a.to_string(), b))
+                .collect()
         }
     };
 
@@ -390,7 +387,7 @@ pub fn provide_extern(providers: &mut Providers<'_>) {
             }));
         }
 
-        tcx.arena.alloc(ret)
+        ret
     };
 }
 

src/librustc_codegen_ssa/back/symbol_export.rs

@@ -42,14 +42,11 @@ pub fn crates_export_threshold(crate_types: &[config::CrateType]) -> SymbolExpor
     }
 }
 
-fn reachable_non_generics_provider(
-    tcx: TyCtxt<'_>,
-    cnum: CrateNum,
-) -> &DefIdMap<SymbolExportLevel> {
+fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<SymbolExportLevel> {
     assert_eq!(cnum, LOCAL_CRATE);
 
     if !tcx.sess.opts.output_types.should_codegen() {
-        return tcx.arena.alloc(Default::default());
+        return Default::default();
     }
 
     // Check to see if this crate is a "special runtime crate". These
@@ -145,7 +142,7 @@ fn reachable_non_generics_provider(
         reachable_non_generics.insert(id, SymbolExportLevel::C);
     }
 
-    tcx.arena.alloc(reachable_non_generics)
+    reachable_non_generics
 }
 
 fn is_reachable_non_generic_provider_local(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
@@ -281,7 +278,7 @@ fn exported_symbols_provider_local(
 fn upstream_monomorphizations_provider(
     tcx: TyCtxt<'_>,
     cnum: CrateNum,
-) -> &DefIdMap<FxHashMap<SubstsRef<'_>, CrateNum>> {
+) -> DefIdMap<FxHashMap<SubstsRef<'_>, CrateNum>> {
     debug_assert!(cnum == LOCAL_CRATE);
 
     let cnums = tcx.all_crate_nums(LOCAL_CRATE);
@@ -338,7 +335,7 @@ fn upstream_monomorphizations_provider(
         }
     }
 
-    tcx.arena.alloc(instances)
+    instances
 }
 
 fn upstream_monomorphizations_for_provider(

src/librustc_codegen_ssa/base.rs

@@ -908,7 +908,7 @@ pub fn provide_both(providers: &mut Providers<'_>) {
             .map(|id| &module_map[&id])
             .flat_map(|module| module.foreign_items.iter().cloned())
             .collect();
-        tcx.arena.alloc(dllimports)
+        dllimports
     };
 
     providers.is_dllimport_foreign_item =

src/librustc_lint/levels.rs

@@ -22,7 +22,7 @@ use rustc_span::symbol::{sym, Symbol};
 
 use std::cmp;
 
-fn lint_levels(tcx: TyCtxt<'_>, cnum: CrateNum) -> &LintLevelMap {
+fn lint_levels(tcx: TyCtxt<'_>, cnum: CrateNum) -> LintLevelMap {
     assert_eq!(cnum, LOCAL_CRATE);
     let store = unerased_lint_store(tcx);
     let levels = LintLevelsBuilder::new(tcx.sess, false, &store);
@@ -37,7 +37,7 @@ fn lint_levels(tcx: TyCtxt<'_>, cnum: CrateNum) -> &LintLevelMap {
     intravisit::walk_crate(&mut builder, krate);
     builder.levels.pop(push);
 
-    tcx.arena.alloc(builder.levels.build_map())
+    builder.levels.build_map()
 }
 
 pub struct LintLevelsBuilder<'s> {

src/librustc_macros/src/query.rs

@@ -107,7 +107,9 @@ impl Parse for QueryModifier {
             let block = input.parse()?;
             Ok(QueryModifier::LoadCached(tcx, id, block))
         } else if modifier == "storage" {
-            let ty = input.parse()?;
+            let args;
+            parenthesized!(args in input);
+            let ty = args.parse()?;
             Ok(QueryModifier::Storage(ty))
         } else if modifier == "fatal_cycle" {
             Ok(QueryModifier::FatalCycle)

src/librustc_metadata/rmeta/decoder.rs

@@ -939,8 +939,8 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     }
 
     /// Iterates over the diagnostic items in the given crate.
-    fn get_diagnostic_items(&self, tcx: TyCtxt<'tcx>) -> &'tcx FxHashMap<Symbol, DefId> {
-        tcx.arena.alloc(if self.root.is_proc_macro_crate() {
+    fn get_diagnostic_items(&self) -> FxHashMap<Symbol, DefId> {
+        if self.root.is_proc_macro_crate() {
             // Proc macro crates do not export any diagnostic-items to the target.
             Default::default()
         } else {
@@ -949,7 +949,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
                 .decode(self)
                 .map(|(name, def_index)| (name, self.local_def_id(def_index)))
                 .collect()
-        })
+        }
     }
 
     /// Iterates over each child of the given item.

src/librustc_metadata/rmeta/decoder/cstore_impl.rs

@@ -88,15 +88,11 @@ impl IntoArgs for (CrateNum, DefId) {
 
 provide! { <'tcx> tcx, def_id, other, cdata,
     type_of => { cdata.get_type(def_id.index, tcx) }
-    generics_of => {
-        tcx.arena.alloc(cdata.get_generics(def_id.index, tcx.sess))
-    }
+    generics_of => { cdata.get_generics(def_id.index, tcx.sess) }
     explicit_predicates_of => { cdata.get_explicit_predicates(def_id.index, tcx) }
     inferred_outlives_of => { cdata.get_inferred_outlives(def_id.index, tcx) }
     super_predicates_of => { cdata.get_super_predicates(def_id.index, tcx) }
-    trait_def => {
-        tcx.arena.alloc(cdata.get_trait_def(def_id.index, tcx.sess))
-    }
+    trait_def => { cdata.get_trait_def(def_id.index, tcx.sess) }
     adt_def => { cdata.get_adt_def(def_id.index, tcx) }
     adt_destructor => {
         let _ = cdata;
@@ -117,8 +113,8 @@ provide! { <'tcx> tcx, def_id, other, cdata,
             bug!("coerce_unsized_info: `{:?}` is missing its info", def_id);
         })
     }
-    optimized_mir => { tcx.arena.alloc(cdata.get_optimized_mir(tcx, def_id.index)) }
-    promoted_mir => { tcx.arena.alloc(cdata.get_promoted_mir(tcx, def_id.index)) }
+    optimized_mir => { cdata.get_optimized_mir(tcx, def_id.index) }
+    promoted_mir => { cdata.get_promoted_mir(tcx, def_id.index) }
     mir_const_qualif => { cdata.mir_const_qualif(def_id.index) }
     fn_sig => { cdata.fn_sig(def_id.index, tcx) }
     inherent_impls => { cdata.get_inherent_implementations_for_type(tcx, def_id.index) }
@@ -178,7 +174,7 @@ provide! { <'tcx> tcx, def_id, other, cdata,
             })
             .collect();
 
-        tcx.arena.alloc(reachable_non_generics)
+        reachable_non_generics
     }
     native_libraries => { Lrc::new(cdata.get_native_libraries(tcx.sess)) }
     foreign_modules => { cdata.get_foreign_modules(tcx) }
@@ -220,7 +216,7 @@ provide! { <'tcx> tcx, def_id, other, cdata,
     }
     defined_lib_features => { cdata.get_lib_features(tcx) }
     defined_lang_items => { cdata.get_lang_items(tcx) }
-    diagnostic_items => { cdata.get_diagnostic_items(tcx) }
+    diagnostic_items => { cdata.get_diagnostic_items() }
     missing_lang_items => { cdata.get_missing_lang_items(tcx) }
 
     missing_extern_crate_item => {
@@ -363,7 +359,7 @@ pub fn provide(providers: &mut Providers<'_>) {
                 }
             }
 
-            tcx.arena.alloc(visible_parent_map)
+            visible_parent_map
         },
 
         dependency_formats: |tcx, cnum| {

src/librustc_middle/arena.rs

@@ -12,36 +12,13 @@ macro_rules! arena_types {
     ($macro:path, $args:tt, $tcx:lifetime) => (
         $macro!($args, [
             [] layouts: rustc_target::abi::Layout,
-            [] generics: rustc_middle::ty::Generics,
-            [] trait_def: rustc_middle::ty::TraitDef,
+            // AdtDef are interned and compared by address
             [] adt_def: rustc_middle::ty::AdtDef,
-            [] steal_mir: rustc_middle::ty::steal::Steal<rustc_middle::mir::Body<$tcx>>,
-            [] mir: rustc_middle::mir::Body<$tcx>,
-            [] steal_promoted: rustc_middle::ty::steal::Steal<
-                rustc_index::vec::IndexVec<
-                    rustc_middle::mir::Promoted,
-                    rustc_middle::mir::Body<$tcx>
-                >
-            >,
-            [] promoted: rustc_index::vec::IndexVec<
-                rustc_middle::mir::Promoted,
-                rustc_middle::mir::Body<$tcx>
-            >,
             [decode] tables: rustc_middle::ty::TypeckTables<$tcx>,
-            [decode] borrowck_result: rustc_middle::mir::BorrowCheckResult<$tcx>,
             [] const_allocs: rustc_middle::mir::interpret::Allocation,
-            [] vtable_method: Option<(
-                rustc_hir::def_id::DefId,
-                rustc_middle::ty::subst::SubstsRef<$tcx>
-            )>,
-            [few, decode] collect_and_partition_mono_items: rustc_hir::def_id::DefIdSet,
-            [few, decode] mir_keys: rustc_data_structures::fx::FxHashSet<rustc_hir::def_id::LocalDefId>,
-            [decode] specialization_graph: rustc_middle::traits::specialization_graph::Graph,
+            // Required for the incremental on-disk cache
+            [few, decode] mir_keys: rustc_hir::def_id::DefIdSet,
             [] region_scope_tree: rustc_middle::middle::region::ScopeTree,
-            [] item_local_set: rustc_hir::ItemLocalSet,
-            [decode] mir_const_qualif: rustc_index::bit_set::BitSet<rustc_middle::mir::Local>,
-            [] trait_impls_of: rustc_middle::ty::trait_def::TraitImpls,
-            [] associated_items: rustc_middle::ty::AssociatedItems,
             [] dropck_outlives:
                 rustc_middle::infer::canonical::Canonical<'tcx,
                     rustc_middle::infer::canonical::QueryResponse<'tcx,
@@ -80,42 +57,10 @@ macro_rules! arena_types {
                 rustc_middle::infer::canonical::Canonical<'tcx,
                     rustc_middle::infer::canonical::QueryResponse<'tcx, rustc_middle::ty::Ty<'tcx>>
                 >,
-            [few] crate_inherent_impls: rustc_middle::ty::CrateInherentImpls,
-            [few] upstream_monomorphizations:
-                rustc_hir::def_id::DefIdMap<
-                    rustc_data_structures::fx::FxHashMap<
-                        rustc_middle::ty::subst::SubstsRef<'tcx>,
-                        rustc_hir::def_id::CrateNum
-                    >
-                >,
-            [few] diagnostic_items: rustc_data_structures::fx::FxHashMap<
-                rustc_span::symbol::Symbol,
-                rustc_hir::def_id::DefId,
-            >,
-            [few] resolve_lifetimes: rustc_middle::middle::resolve_lifetime::ResolveLifetimes,
-            [few] lint_levels: rustc_middle::lint::LintLevelMap,
-            [few] stability_index: rustc_middle::middle::stability::Index<'tcx>,
-            [few] features: rustc_feature::Features,
             [few] all_traits: Vec<rustc_hir::def_id::DefId>,
             [few] privacy_access_levels: rustc_middle::middle::privacy::AccessLevels,
-            [few] target_features_whitelist: rustc_data_structures::fx::FxHashMap<
-                String,
-                Option<rustc_span::symbol::Symbol>
-            >,
-            [few] wasm_import_module_map: rustc_data_structures::fx::FxHashMap<
-                rustc_hir::def_id::DefId,
-                String
-            >,
-            [few] get_lib_features: rustc_middle::middle::lib_features::LibFeatures,
-            [few] defined_lib_features: rustc_hir::lang_items::LanguageItems,
-            [few] visible_parent_map: rustc_hir::def_id::DefIdMap<rustc_hir::def_id::DefId>,
             [few] foreign_module: rustc_middle::middle::cstore::ForeignModule,
             [few] foreign_modules: Vec<rustc_middle::middle::cstore::ForeignModule>,
-            [few] reachable_non_generics: rustc_hir::def_id::DefIdMap<
-                rustc_middle::middle::exported_symbols::SymbolExportLevel
-            >,
-            [few] crate_variances: rustc_middle::ty::CrateVariancesMap<'tcx>,
-            [few] inferred_outlives_crate: rustc_middle::ty::CratePredicatesMap<'tcx>,
             [] upvars: rustc_data_structures::fx::FxIndexMap<rustc_hir::HirId, rustc_hir::Upvar>,
             [] object_safety_violations: rustc_middle::traits::ObjectSafetyViolation,
             [] codegen_unit: rustc_middle::mir::mono::CodegenUnit<$tcx>,