Add an abstraction for custom query caches

src/librustc/dep_graph/dep_node.rs

@@ -76,10 +76,6 @@ macro_rules! erase {
     ($x:tt) => {{}};
 }
 
-macro_rules! replace {
-    ($x:tt with $($y:tt)*) => ($($y)*)
-}
-
 macro_rules! is_anon_attr {
     (anon) => {
         true
@@ -99,19 +95,18 @@ macro_rules! is_eval_always_attr {
 }
 
 macro_rules! contains_anon_attr {
-    ($($attr:ident),*) => ({$(is_anon_attr!($attr) | )* false});
+    ($($attr:ident $(($($attr_args:tt)*))* ),*) => ({$(is_anon_attr!($attr) | )* false});
 }
 
 macro_rules! contains_eval_always_attr {
-    ($($attr:ident),*) => ({$(is_eval_always_attr!($attr) | )* false});
+    ($($attr:ident $(($($attr_args:tt)*))* ),*) => ({$(is_eval_always_attr!($attr) | )* false});
 }
 
 macro_rules! define_dep_nodes {
     (<$tcx:tt>
     $(
-        [$($attr:ident),* ]
+        [$($attrs:tt)*]
         $variant:ident $(( $tuple_arg_ty:ty $(,)? ))*
-                       $({ $($struct_arg_name:ident : $struct_arg_ty:ty),* })*
       ,)*
     ) => (
         #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash,
@@ -126,7 +121,7 @@ macro_rules! define_dep_nodes {
                 match *self {
                     $(
                         DepKind :: $variant => {
-                            if contains_anon_attr!($($attr),*) {
+                            if contains_anon_attr!($($attrs)*) {
                                 return false;
                             }
 
@@ -136,13 +131,6 @@ macro_rules! define_dep_nodes {
                                     ::CAN_RECONSTRUCT_QUERY_KEY;
                             })*
 
-                            // struct args
-                            $({
-
-                                return <( $($struct_arg_ty,)* ) as DepNodeParams>
-                                    ::CAN_RECONSTRUCT_QUERY_KEY;
-                            })*
-
                             true
                         }
                     )*
@@ -152,15 +140,15 @@ macro_rules! define_dep_nodes {
             pub fn is_anon(&self) -> bool {
                 match *self {
                     $(
-                        DepKind :: $variant => { contains_anon_attr!($($attr),*) }
+                        DepKind :: $variant => { contains_anon_attr!($($attrs)*) }
                     )*
                 }
             }
 
             pub fn is_eval_always(&self) -> bool {
                 match *self {
                     $(
-                        DepKind :: $variant => { contains_eval_always_attr!($($attr), *) }
+                        DepKind :: $variant => { contains_eval_always_attr!($($attrs)*) }
                     )*
                 }
             }
@@ -176,24 +164,50 @@ macro_rules! define_dep_nodes {
                                 return true;
                             })*
 
-                            // struct args
-                            $({
-                                $(erase!($struct_arg_name);)*
-                                return true;
-                            })*
-
                             false
                         }
                     )*
                 }
             }
         }
 
-        pub enum DepConstructor<$tcx> {
+        pub struct DepConstructor;
+
+        impl DepConstructor {
             $(
-                $variant $(( $tuple_arg_ty ))*
-                         $({ $($struct_arg_name : $struct_arg_ty),* })*
-            ),*
+                #[inline(always)]
+                #[allow(unreachable_code, non_snake_case)]
+                pub fn $variant<'tcx>(_tcx: TyCtxt<'tcx>, $(arg: $tuple_arg_ty)*) -> DepNode {
+                    // tuple args
+                    $({
+                        erase!($tuple_arg_ty);
+                        let hash = DepNodeParams::to_fingerprint(&arg, _tcx);
+                        let dep_node = DepNode {
+                            kind: DepKind::$variant,
+                            hash
+                        };
+
+                        #[cfg(debug_assertions)]
+                        {
+                            if !dep_node.kind.can_reconstruct_query_key() &&
+                            (_tcx.sess.opts.debugging_opts.incremental_info ||
+                                _tcx.sess.opts.debugging_opts.query_dep_graph)
+                            {
+                                _tcx.dep_graph.register_dep_node_debug_str(dep_node, || {
+                                    arg.to_debug_str(_tcx)
+                                });
+                            }
+                        }
+
+                        return dep_node;
+                    })*
+
+                    DepNode {
+                        kind: DepKind::$variant,
+                        hash: Fingerprint::ZERO,
+                    }
+                }
+            )*
         }
 
         #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash,
@@ -204,75 +218,6 @@ macro_rules! define_dep_nodes {
         }
 
         impl DepNode {
-            #[allow(unreachable_code, non_snake_case)]
-            pub fn new<'tcx>(tcx: TyCtxt<'tcx>,
-                                       dep: DepConstructor<'tcx>)
-                                       -> DepNode
-            {
-                match dep {
-                    $(
-                        DepConstructor :: $variant $(( replace!(($tuple_arg_ty) with arg) ))*
-                                                   $({ $($struct_arg_name),* })*
-                            =>
-                        {
-                            // tuple args
-                            $({
-                                erase!($tuple_arg_ty);
-                                let hash = DepNodeParams::to_fingerprint(&arg, tcx);
-                                let dep_node = DepNode {
-                                    kind: DepKind::$variant,
-                                    hash
-                                };
-
-                                #[cfg(debug_assertions)]
-                                {
-                                    if !dep_node.kind.can_reconstruct_query_key() &&
-                                    (tcx.sess.opts.debugging_opts.incremental_info ||
-                                        tcx.sess.opts.debugging_opts.query_dep_graph)
-                                    {
-                                        tcx.dep_graph.register_dep_node_debug_str(dep_node, || {
-                                            arg.to_debug_str(tcx)
-                                        });
-                                    }
-                                }
-
-                                return dep_node;
-                            })*
-
-                            // struct args
-                            $({
-                                let tupled_args = ( $($struct_arg_name,)* );
-                                let hash = DepNodeParams::to_fingerprint(&tupled_args,
-                                                                         tcx);
-                                let dep_node = DepNode {
-                                    kind: DepKind::$variant,
-                                    hash
-                                };
-
-                                #[cfg(debug_assertions)]
-                                {
-                                    if !dep_node.kind.can_reconstruct_query_key() &&
-                                    (tcx.sess.opts.debugging_opts.incremental_info ||
-                                        tcx.sess.opts.debugging_opts.query_dep_graph)
-                                    {
-                                        tcx.dep_graph.register_dep_node_debug_str(dep_node, || {
-                                            tupled_args.to_debug_str(tcx)
-                                        });
-                                    }
-                                }
-
-                                return dep_node;
-                            })*
-
-                            DepNode {
-                                kind: DepKind::$variant,
-                                hash: Fingerprint::ZERO,
-                            }
-                        }
-                    )*
-                }
-            }
-
             /// Construct a DepNode from the given DepKind and DefPathHash. This
             /// method will assert that the given DepKind actually requires a
             /// single DefId/DefPathHash parameter.

src/librustc/dep_graph/graph.rs

@@ -1122,6 +1122,7 @@ impl CurrentDepGraph {
 }
 
 impl DepGraphData {
+    #[inline(never)]
     fn read_index(&self, source: DepNodeIndex) {
         ty::tls::with_context_opt(|icx| {
             let icx = if let Some(icx) = icx { icx } else { return };

src/librustc/mir/mono.rs

@@ -362,7 +362,7 @@ impl<'tcx> CodegenUnit<'tcx> {
     }
 
     pub fn codegen_dep_node(&self, tcx: TyCtxt<'tcx>) -> DepNode {
-        DepNode::new(tcx, DepConstructor::CompileCodegenUnit(self.name()))
+        DepConstructor::CompileCodegenUnit(tcx, self.name())
     }
 }
 

src/librustc/ty/context.rs

@@ -2,7 +2,7 @@
 
 use crate::arena::Arena;
 use crate::dep_graph::DepGraph;
-use crate::dep_graph::{self, DepConstructor, DepNode};
+use crate::dep_graph::{self, DepConstructor};
 use crate::hir::exports::Export;
 use crate::hir::map as hir_map;
 use crate::hir::map::DefPathHash;
@@ -1347,7 +1347,7 @@ impl<'tcx> TyCtxt<'tcx> {
         // We cannot use the query versions of crates() and crate_hash(), since
         // those would need the DepNodes that we are allocating here.
         for cnum in self.cstore.crates_untracked() {
-            let dep_node = DepNode::new(self, DepConstructor::CrateMetadata(cnum));
+            let dep_node = DepConstructor::CrateMetadata(self, cnum);
             let crate_hash = self.cstore.crate_hash_untracked(cnum);
             self.dep_graph.with_task(
                 dep_node,
@@ -1688,6 +1688,7 @@ pub mod tls {
 
     /// Gets the pointer to the current `ImplicitCtxt`.
     #[cfg(not(parallel_compiler))]
+    #[inline]
     fn get_tlv() -> usize {
         TLV.with(|tlv| tlv.get())
     }

src/librustc/ty/query/caches.rs

@@ -0,0 +1,112 @@
+use crate::dep_graph::DepNodeIndex;
+use crate::ty::query::config::QueryAccessors;
+use crate::ty::query::plumbing::{QueryLookup, QueryState, QueryStateShard};
+use crate::ty::TyCtxt;
+
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sharded::Sharded;
+use std::default::Default;
+use std::hash::Hash;
+
+pub(crate) trait CacheSelector<K, V> {
+    type Cache: QueryCache<K, V>;
+}
+
+pub(crate) trait QueryCache<K, V>: Default {
+    type Sharded: Default;
+
+    /// Checks if the query is already computed and in the cache.
+    /// It returns the shard index and a lock guard to the shard,
+    /// which will be used if the query is not in the cache and we need
+    /// to compute it.
+    fn lookup<'tcx, R, GetCache, OnHit, OnMiss, Q>(
+        &self,
+        state: &'tcx QueryState<'tcx, Q>,
+        get_cache: GetCache,
+        key: K,
+        // `on_hit` can be called while holding a lock to the query state shard.
+        on_hit: OnHit,
+        on_miss: OnMiss,
+    ) -> R
+    where
+        Q: QueryAccessors<'tcx>,
+        GetCache: for<'a> Fn(&'a mut QueryStateShard<'tcx, Q>) -> &'a mut Self::Sharded,
+        OnHit: FnOnce(&V, DepNodeIndex) -> R,
+        OnMiss: FnOnce(K, QueryLookup<'tcx, Q>) -> R;
+
+    fn complete(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        lock_sharded_storage: &mut Self::Sharded,
+        key: K,
+        value: V,
+        index: DepNodeIndex,
+    );
+
+    fn iter<R, L>(
+        &self,
+        shards: &Sharded<L>,
+        get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
+        f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
+    ) -> R;
+}
+
+pub struct DefaultCacheSelector;
+
+impl<K: Eq + Hash, V: Clone> CacheSelector<K, V> for DefaultCacheSelector {
+    type Cache = DefaultCache;
+}
+
+#[derive(Default)]
+pub struct DefaultCache;
+
+impl<K: Eq + Hash, V: Clone> QueryCache<K, V> for DefaultCache {
+    type Sharded = FxHashMap<K, (V, DepNodeIndex)>;
+
+    #[inline(always)]
+    fn lookup<'tcx, R, GetCache, OnHit, OnMiss, Q>(
+        &self,
+        state: &'tcx QueryState<'tcx, Q>,
+        get_cache: GetCache,
+        key: K,
+        on_hit: OnHit,
+        on_miss: OnMiss,
+    ) -> R
+    where
+        Q: QueryAccessors<'tcx>,
+        GetCache: for<'a> Fn(&'a mut QueryStateShard<'tcx, Q>) -> &'a mut Self::Sharded,
+        OnHit: FnOnce(&V, DepNodeIndex) -> R,
+        OnMiss: FnOnce(K, QueryLookup<'tcx, Q>) -> R,
+    {
+        let mut lookup = state.get_lookup(&key);
+        let lock = &mut *lookup.lock;
+
+        let result = get_cache(lock).raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key);
+
+        if let Some((_, value)) = result { on_hit(&value.0, value.1) } else { on_miss(key, lookup) }
+    }
+
+    #[inline]
+    fn complete(
+        &self,
+        _: TyCtxt<'tcx>,
+        lock_sharded_storage: &mut Self::Sharded,
+        key: K,
+        value: V,
+        index: DepNodeIndex,
+    ) {
+        lock_sharded_storage.insert(key, (value, index));
+    }
+
+    fn iter<R, L>(
+        &self,
+        shards: &Sharded<L>,
+        get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
+        f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
+    ) -> R {
+        let mut shards = shards.lock_shards();
+        let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect();
+        let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
+        f(Box::new(results))
+    }
+}