Auto merge of #90210 - cjgillot:qarray2, r=Mark-Simulacrum

Build the query vtable directly.

Continuation of #89978.

This shrinks the query interface and attempts to reduce the amount of function pointer calls.

Author: bors

compiler/rustc_macros/src/query.rs

@@ -349,24 +349,14 @@ fn add_query_description_impl(
         let try_load_from_disk = if let Some((tcx, id, block)) = modifiers.load_cached.as_ref() {
             // Use custom code to load the query from disk
             quote! {
-                #[inline]
-                fn try_load_from_disk(
-                    #tcx: QueryCtxt<'tcx>,
-                    #id: SerializedDepNodeIndex
-                ) -> Option<Self::Value> {
-                    #block
-                }
+                const TRY_LOAD_FROM_DISK: Option<fn(QueryCtxt<$tcx>, SerializedDepNodeIndex) -> Option<Self::Value>>
+                    = Some(|#tcx, #id| { #block });
             }
         } else {
             // Use the default code to load the query from disk
             quote! {
-                #[inline]
-                fn try_load_from_disk(
-                    tcx: QueryCtxt<'tcx>,
-                    id: SerializedDepNodeIndex
-                ) -> Option<Self::Value> {
-                    tcx.on_disk_cache().as_ref()?.try_load_query_result(*tcx, id)
-                }
+                const TRY_LOAD_FROM_DISK: Option<fn(QueryCtxt<$tcx>, SerializedDepNodeIndex) -> Option<Self::Value>>
+                    = Some(|tcx, id| tcx.on_disk_cache().as_ref()?.try_load_query_result(*tcx, id));
             }
         };
 
@@ -380,12 +370,9 @@ fn add_query_description_impl(
         // expr is a `Block`, meaning that `{ #expr }` gets expanded
         // to `{ { stmts... } }`, which triggers the `unused_braces` lint.
         quote! {
-            #[inline]
             #[allow(unused_variables, unused_braces)]
-            fn cache_on_disk(
-                #tcx: QueryCtxt<'tcx>,
-                #key: &Self::Key,
-            ) -> bool {
+            #[inline]
+            fn cache_on_disk(#tcx: TyCtxt<'tcx>, #key: &Self::Key) -> bool {
                 #expr
             }
 
@@ -395,25 +382,32 @@ fn add_query_description_impl(
         if modifiers.load_cached.is_some() {
             panic!("load_cached modifier on query `{}` without a cache modifier", name);
         }
-        quote! {}
+        quote! {
+            #[inline]
+            fn cache_on_disk(_: TyCtxt<'tcx>, _: &Self::Key) -> bool {
+                false
+            }
+
+            const TRY_LOAD_FROM_DISK: Option<fn(QueryCtxt<$tcx>, SerializedDepNodeIndex) -> Option<Self::Value>> = None;
+        }
     };
 
     let (tcx, desc) = modifiers.desc;
     let tcx = tcx.as_ref().map_or_else(|| quote! { _ }, |t| quote! { #t });
 
     let desc = quote! {
         #[allow(unused_variables)]
-        fn describe(tcx: QueryCtxt<'tcx>, key: Self::Key) -> String {
+        fn describe(tcx: QueryCtxt<$tcx>, key: Self::Key) -> String {
             let (#tcx, #key) = (*tcx, key);
             ::rustc_middle::ty::print::with_no_trimmed_paths(|| format!(#desc).into())
         }
     };
 
     impls.extend(quote! {
-        impl<'tcx> QueryDescription<QueryCtxt<'tcx>> for queries::#name<'tcx> {
+        (#name<$tcx:tt>) => {
             #desc
             #cache
-        }
+        };
     });
 }
 
@@ -521,7 +515,7 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream {
         }
         #[macro_export]
         macro_rules! rustc_query_description {
-            () => { #query_description_stream }
+            #query_description_stream
         }
     })
 }

compiler/rustc_query_impl/src/lib.rs

@@ -15,15 +15,13 @@ extern crate rustc_macros;
 #[macro_use]
 extern crate rustc_middle;
 
-use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_errors::DiagnosticBuilder;
 use rustc_middle::arena::Arena;
-use rustc_middle::dep_graph::{self, DepKindStruct};
+use rustc_middle::dep_graph::{self, DepKindStruct, SerializedDepNodeIndex};
 use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
 use rustc_middle::ty::query::{Providers, QueryEngine};
 use rustc_middle::ty::{self, TyCtxt};
-use rustc_query_system::ich::StableHashingContext;
+use rustc_span::def_id::LocalDefId;
 use rustc_span::Span;
 
 #[macro_use]
@@ -40,9 +38,8 @@ use keys::Key;
 mod values;
 use self::values::Value;
 
-use rustc_query_system::query::QueryAccessors;
 pub use rustc_query_system::query::QueryConfig;
-pub(crate) use rustc_query_system::query::QueryDescription;
+pub(crate) use rustc_query_system::query::{QueryDescription, QueryVtable};
 
 mod on_disk_cache;
 pub use on_disk_cache::OnDiskCache;
@@ -52,6 +49,14 @@ pub use self::profiling_support::alloc_self_profile_query_strings;
 
 mod util;
 
+fn describe_as_module(def_id: LocalDefId, tcx: TyCtxt<'_>) -> String {
+    if def_id.is_top_level_module() {
+        "top-level module".to_string()
+    } else {
+        format!("module `{}`", tcx.def_path_str(def_id.to_def_id()))
+    }
+}
+
 rustc_query_append! { [define_queries!][<'tcx>] }
 
 impl<'tcx> Queries<'tcx> {

compiler/rustc_query_impl/src/on_disk_cache.rs

@@ -1018,7 +1018,7 @@ pub fn encode_query_results<'a, 'tcx, CTX, Q>(
 ) -> FileEncodeResult
 where
     CTX: QueryContext + 'tcx,
-    Q: super::QueryDescription<CTX> + super::QueryAccessors<CTX>,
+    Q: super::QueryDescription<CTX>,
     Q::Value: Encodable<CacheEncoder<'a, 'tcx, FileEncoder>>,
 {
     let _timer = tcx
@@ -1033,7 +1033,7 @@ where
         if res.is_err() {
             return;
         }
-        if Q::cache_on_disk(tcx, &key) {
+        if Q::cache_on_disk(*tcx.dep_context(), &key) {
             let dep_node = SerializedDepNodeIndex::new(dep_node.index());
 
             // Record position of the cache entry.

compiler/rustc_query_impl/src/plumbing.rs

@@ -2,20 +2,17 @@
 //! generate the actual methods on tcx which find and execute the provider,
 //! manage the caches, and so forth.
 
-use crate::{on_disk_cache, queries, Queries};
+use crate::{on_disk_cache, Queries};
 use rustc_middle::dep_graph::{DepKind, DepNodeIndex, SerializedDepNodeIndex};
 use rustc_middle::ty::tls::{self, ImplicitCtxt};
-use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::ty::TyCtxt;
 use rustc_query_system::dep_graph::HasDepContext;
-use rustc_query_system::query::{
-    QueryContext, QueryDescription, QueryJobId, QueryMap, QuerySideEffects,
-};
+use rustc_query_system::query::{QueryContext, QueryJobId, QueryMap, QuerySideEffects};
 
 use rustc_data_structures::sync::Lock;
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_errors::{Diagnostic, Handler};
 use rustc_serialize::opaque;
-use rustc_span::def_id::LocalDefId;
 
 use std::any::Any;
 
@@ -290,11 +287,8 @@ macro_rules! define_queries {
             const NAME: &'static str = stringify!($name);
         }
 
-        impl<$tcx> QueryAccessors<QueryCtxt<$tcx>> for queries::$name<$tcx> {
-            const ANON: bool = is_anon!([$($modifiers)*]);
-            const EVAL_ALWAYS: bool = is_eval_always!([$($modifiers)*]);
-            const DEP_KIND: dep_graph::DepKind = dep_graph::DepKind::$name;
-            const HASH_RESULT: Option<fn(&mut StableHashingContext<'_>, &Self::Value) -> Fingerprint> = hash_result!([$($modifiers)*]);
+        impl<$tcx> QueryDescription<QueryCtxt<$tcx>> for queries::$name<$tcx> {
+            rustc_query_description! { $name<$tcx> }
 
             type Cache = query_storage::$name<$tcx>;
 
@@ -313,22 +307,26 @@ macro_rules! define_queries {
             }
 
             #[inline]
-            fn compute_fn(tcx: QueryCtxt<'tcx>, key: &Self::Key) ->
-                fn(TyCtxt<'tcx>, Self::Key) -> Self::Value
+            fn make_vtable(tcx: QueryCtxt<'tcx>, key: &Self::Key) ->
+                QueryVtable<QueryCtxt<$tcx>, Self::Key, Self::Value>
             {
-                if key.query_crate_is_local() {
+                let compute = if key.query_crate_is_local() {
                     tcx.queries.local_providers.$name
                 } else {
                     tcx.queries.extern_providers.$name
+                };
+                let cache_on_disk = Self::cache_on_disk(tcx.tcx, key);
+                QueryVtable {
+                    anon: is_anon!([$($modifiers)*]),
+                    eval_always: is_eval_always!([$($modifiers)*]),
+                    dep_kind: dep_graph::DepKind::$name,
+                    hash_result: hash_result!([$($modifiers)*]),
+                    handle_cycle_error: |tcx, mut error| handle_cycle_error!([$($modifiers)*][tcx, error]),
+                    compute,
+                    cache_on_disk,
+                    try_load_from_disk: Self::TRY_LOAD_FROM_DISK,
                 }
             }
-
-            fn handle_cycle_error(
-                tcx: QueryCtxt<'tcx>,
-                mut error: DiagnosticBuilder<'_>,
-            ) -> Self::Value {
-                handle_cycle_error!([$($modifiers)*][tcx, error])
-            }
         })*
 
         #[allow(nonstandard_style)]
@@ -417,7 +415,6 @@ macro_rules! define_queries {
                     debug_assert!(tcx.dep_graph.is_green(&dep_node));
 
                     let key = recover(tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
-                    let tcx = QueryCtxt::from_tcx(tcx);
                     if queries::$name::cache_on_disk(tcx, &key) {
                         let _ = tcx.$name(key);
                     }
@@ -518,13 +515,3 @@ macro_rules! define_queries_struct {
         }
     };
 }
-
-fn describe_as_module(def_id: LocalDefId, tcx: TyCtxt<'_>) -> String {
-    if def_id.is_top_level_module() {
-        "top-level module".to_string()
-    } else {
-        format!("module `{}`", tcx.def_path_str(def_id.to_def_id()))
-    }
-}
-
-rustc_query_description! {}

compiler/rustc_query_system/src/query/config.rs

@@ -19,16 +19,16 @@ pub trait QueryConfig {
     type Stored: Clone;
 }
 
-pub(crate) struct QueryVtable<CTX: QueryContext, K, V> {
+pub struct QueryVtable<CTX: QueryContext, K, V> {
     pub anon: bool,
     pub dep_kind: CTX::DepKind,
     pub eval_always: bool,
+    pub cache_on_disk: bool,
 
     pub compute: fn(CTX::DepContext, K) -> V,
     pub hash_result: Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>,
     pub handle_cycle_error: fn(CTX, DiagnosticBuilder<'_>) -> V,
-    pub cache_on_disk: fn(CTX, &K) -> bool,
-    pub try_load_from_disk: fn(CTX, SerializedDepNodeIndex) -> Option<V>,
+    pub try_load_from_disk: Option<fn(CTX, SerializedDepNodeIndex) -> Option<V>>,
 }
 
 impl<CTX: QueryContext, K, V> QueryVtable<CTX, K, V> {
@@ -43,25 +43,21 @@ impl<CTX: QueryContext, K, V> QueryVtable<CTX, K, V> {
         (self.compute)(tcx, key)
     }
 
-    pub(crate) fn cache_on_disk(&self, tcx: CTX, key: &K) -> bool {
-        (self.cache_on_disk)(tcx, key)
-    }
-
     pub(crate) fn try_load_from_disk(&self, tcx: CTX, index: SerializedDepNodeIndex) -> Option<V> {
-        (self.try_load_from_disk)(tcx, index)
+        self.try_load_from_disk
+            .expect("QueryDescription::load_from_disk() called for an unsupported query.")(
+            tcx, index,
+        )
     }
 }
 
-pub trait QueryAccessors<CTX: QueryContext>: QueryConfig {
-    const ANON: bool;
-    const EVAL_ALWAYS: bool;
-    const DEP_KIND: CTX::DepKind;
-    const HASH_RESULT: Option<
-        fn(hcx: &mut StableHashingContext<'_>, result: &Self::Value) -> Fingerprint,
-    >;
+pub trait QueryDescription<CTX: QueryContext>: QueryConfig {
+    const TRY_LOAD_FROM_DISK: Option<fn(CTX, SerializedDepNodeIndex) -> Option<Self::Value>>;
 
     type Cache: QueryCache<Key = Self::Key, Stored = Self::Stored, Value = Self::Value>;
 
+    fn describe(tcx: CTX, key: Self::Key) -> String;
+
     // Don't use this method to access query results, instead use the methods on TyCtxt
     fn query_state<'a>(tcx: CTX) -> &'a QueryState<CTX::DepKind, Self::Key>
     where
@@ -73,43 +69,7 @@ pub trait QueryAccessors<CTX: QueryContext>: QueryConfig {
         CTX: 'a;
 
     // Don't use this method to compute query results, instead use the methods on TyCtxt
-    fn compute_fn(tcx: CTX, key: &Self::Key) -> fn(CTX::DepContext, Self::Key) -> Self::Value;
-
-    fn handle_cycle_error(tcx: CTX, diag: DiagnosticBuilder<'_>) -> Self::Value;
-}
-
-pub trait QueryDescription<CTX: QueryContext>: QueryAccessors<CTX> {
-    fn describe(tcx: CTX, key: Self::Key) -> String;
+    fn make_vtable(tcx: CTX, key: &Self::Key) -> QueryVtable<CTX, Self::Key, Self::Value>;
 
-    #[inline]
-    fn cache_on_disk(_: CTX, _: &Self::Key) -> bool {
-        false
-    }
-
-    fn try_load_from_disk(_: CTX, _: SerializedDepNodeIndex) -> Option<Self::Value> {
-        panic!("QueryDescription::load_from_disk() called for an unsupported query.")
-    }
-}
-
-pub(crate) trait QueryVtableExt<CTX: QueryContext, K, V> {
-    fn make_vtable(tcx: CTX, key: &K) -> QueryVtable<CTX, K, V>;
-}
-
-impl<CTX, Q> QueryVtableExt<CTX, Q::Key, Q::Value> for Q
-where
-    CTX: QueryContext,
-    Q: QueryDescription<CTX>,
-{
-    fn make_vtable(tcx: CTX, key: &Q::Key) -> QueryVtable<CTX, Q::Key, Q::Value> {
-        QueryVtable {
-            anon: Q::ANON,
-            dep_kind: Q::DEP_KIND,
-            eval_always: Q::EVAL_ALWAYS,
-            hash_result: Q::HASH_RESULT,
-            compute: Q::compute_fn(tcx, key),
-            handle_cycle_error: Q::handle_cycle_error,
-            cache_on_disk: Q::cache_on_disk,
-            try_load_from_disk: Q::try_load_from_disk,
-        }
-    }
+    fn cache_on_disk(tcx: CTX::DepContext, key: &Self::Key) -> bool;
 }

compiler/rustc_query_system/src/query/mod.rs

@@ -12,7 +12,7 @@ pub use self::caches::{
 };
 
 mod config;
-pub use self::config::{QueryAccessors, QueryConfig, QueryDescription};
+pub use self::config::{QueryConfig, QueryDescription, QueryVtable};
 
 use crate::dep_graph::{DepNodeIndex, HasDepContext, SerializedDepNodeIndex};
 

compiler/rustc_query_system/src/query/plumbing.rs

@@ -4,7 +4,7 @@
 
 use crate::dep_graph::{DepContext, DepNode, DepNodeIndex, DepNodeParams};
 use crate::query::caches::QueryCache;
-use crate::query::config::{QueryDescription, QueryVtable, QueryVtableExt};
+use crate::query::config::{QueryDescription, QueryVtable};
 use crate::query::job::{
     report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId,
 };
@@ -512,7 +512,7 @@ where
 
     // First we try to load the result from the on-disk cache.
     // Some things are never cached on disk.
-    if query.cache_on_disk(tcx, key) {
+    if query.cache_on_disk {
         let prof_timer = tcx.dep_context().profiler().incr_cache_loading();
         let result = query.try_load_from_disk(tcx, prev_dep_node_index);
         prof_timer.finish_with_query_invocation_id(dep_node_index.into());
@@ -713,8 +713,6 @@ where
     Q::Key: DepNodeParams<CTX::DepContext>,
     CTX: QueryContext,
 {
-    assert!(!Q::ANON);
-
     // We may be concurrently trying both execute and force a query.
     // Ensure that only one of them runs the query.
     let cache = Q::query_cache(tcx);
@@ -731,5 +729,7 @@ where
 
     let query = Q::make_vtable(tcx, &key);
     let state = Q::query_state(tcx);
+    debug_assert!(!query.anon);
+
     try_execute_query(tcx, state, cache, DUMMY_SP, key, lookup, Some(dep_node), &query);
 }