Skip to content

Commit a51a205

Browse files
ZoxcZoxc
committed
Factor query arena allocation out from query caches
1 parent 4b34c7b commit a51a205

File tree

9 files changed

+159
-253
lines changed

9 files changed

+159
-253
lines changed

‎compiler/rustc_interface/src/passes.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -676,9 +676,7 @@ pub fn create_global_ctxt<'tcx>(
676676
callback(sess, &mut local_providers, &mut extern_providers);
677677
}
678678

679-
let queries = queries.get_or_init(|| {
680-
TcxQueries::new(local_providers, extern_providers, query_result_on_disk_cache)
681-
});
679+
let queries = queries.get_or_init(|| TcxQueries::new(query_result_on_disk_cache));
682680

683681
sess.time("setup_global_ctxt", || {
684682
gcx_cell.get_or_init(move || {
@@ -690,6 +688,8 @@ pub fn create_global_ctxt<'tcx>(
690688
untracked,
691689
dep_graph,
692690
queries.on_disk_cache.as_ref().map(OnDiskCache::as_dyn),
691+
local_providers,
692+
extern_providers,
693693
queries.as_dyn(),
694694
rustc_query_impl::query_callbacks(arena),
695695
)

‎compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ macro_rules! provide_one {
114114
fn $name<'tcx>(
115115
$tcx: TyCtxt<'tcx>,
116116
def_id_arg: ty::query::query_keys::$name<'tcx>,
117-
) -> ty::query::query_values::$name<'tcx> {
117+
) -> ty::query::query_provided::$name<'tcx> {
118118
let _prof_timer =
119119
$tcx.prof.generic_activity(concat!("metadata_decode_entry_", stringify!($name)));
120120

‎compiler/rustc_middle/src/ty/context.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ use crate::mir::{
1818
use crate::thir::Thir;
1919
use crate::traits;
2020
use crate::traits::solve::{ExternalConstraints, ExternalConstraintsData};
21+
use crate::ty::query::ExternProviders;
22+
use crate::ty::query::Providers;
2123
use crate::ty::query::{self, TyCtxtAt};
2224
use crate::ty::{
2325
self, AdtDef, AdtDefData, AdtKind, Binder, Const, ConstData, DefIdTree, FloatTy, FloatVar,
@@ -479,7 +481,7 @@ pub struct GlobalCtxt<'tcx> {
479481
pub on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
480482

481483
pub queries: &'tcx dyn query::QueryEngine<'tcx>,
482-
pub query_caches: query::QueryCaches<'tcx>,
484+
pub query_system: query::QuerySystem<'tcx>,
483485
pub(crate) query_kinds: &'tcx [DepKindStruct<'tcx>],
484486

485487
// Internal caches for metadata decoding. No need to track deps on this.
@@ -639,6 +641,8 @@ impl<'tcx> TyCtxt<'tcx> {
639641
untracked: Untracked,
640642
dep_graph: DepGraph,
641643
on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
644+
local_providers: Providers,
645+
extern_providers: ExternProviders,
642646
queries: &'tcx dyn query::QueryEngine<'tcx>,
643647
query_kinds: &'tcx [DepKindStruct<'tcx>],
644648
) -> GlobalCtxt<'tcx> {
@@ -664,7 +668,7 @@ impl<'tcx> TyCtxt<'tcx> {
664668
untracked,
665669
on_disk_cache,
666670
queries,
667-
query_caches: query::QueryCaches::default(),
671+
query_system: query::QuerySystem::new(local_providers, extern_providers),
668672
query_kinds,
669673
ty_rcache: Default::default(),
670674
pred_rcache: Default::default(),

‎compiler/rustc_middle/src/ty/query.rs‎

Lines changed: 76 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#![allow(unused_parens)]
2+
13
use crate::dep_graph;
24
use crate::infer::canonical::{self, Canonical};
35
use crate::lint::LintExpectation;
@@ -34,13 +36,15 @@ use crate::ty::subst::{GenericArg, SubstsRef};
3436
use crate::ty::util::AlwaysRequiresDrop;
3537
use crate::ty::GeneratorDiagnosticData;
3638
use crate::ty::{self, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt, UnusedGenericParams};
39+
use rustc_arena::TypedArena;
3740
use rustc_ast as ast;
3841
use rustc_ast::expand::allocator::AllocatorKind;
3942
use rustc_attr as attr;
4043
use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
4144
use rustc_data_structures::steal::Steal;
4245
use rustc_data_structures::svh::Svh;
4346
use rustc_data_structures::sync::Lrc;
47+
use rustc_data_structures::sync::WorkerLocal;
4448
use rustc_data_structures::unord::UnordSet;
4549
use rustc_errors::ErrorGuaranteed;
4650
use rustc_hir as hir;
@@ -66,6 +70,24 @@ use std::sync::Arc;
6670
pub(crate) use rustc_query_system::query::QueryJobId;
6771
use rustc_query_system::query::*;
6872

73+
pub struct QuerySystem<'tcx> {
74+
pub local_providers: Box<Providers>,
75+
pub extern_providers: Box<ExternProviders>,
76+
pub arenas: QueryArenas<'tcx>,
77+
pub caches: QueryCaches<'tcx>,
78+
}
79+
80+
impl<'tcx> QuerySystem<'tcx> {
81+
pub fn new(local_providers: Providers, extern_providers: ExternProviders) -> Self {
82+
QuerySystem {
83+
local_providers: Box::new(local_providers),
84+
extern_providers: Box::new(extern_providers),
85+
arenas: Default::default(),
86+
caches: Default::default(),
87+
}
88+
}
89+
}
90+
6991
#[derive(Copy, Clone)]
7092
pub struct TyCtxtAt<'tcx> {
7193
pub tcx: TyCtxt<'tcx>,
@@ -112,10 +134,10 @@ macro_rules! query_helper_param_ty {
112134
}
113135

114136
macro_rules! query_if_arena {
115-
([] $arena:ty, $no_arena:ty) => {
137+
([] $arena:tt $no_arena:tt) => {
116138
$no_arena
117139
};
118-
([(arena_cache) $($rest:tt)*] $arena:ty, $no_arena:ty) => {
140+
([(arena_cache) $($rest:tt)*] $arena:tt $no_arena:tt) => {
119141
$arena
120142
};
121143
([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
@@ -131,7 +153,7 @@ macro_rules! separate_provide_extern_decl {
131153
for<'tcx> fn(
132154
TyCtxt<'tcx>,
133155
query_keys::$name<'tcx>,
134-
) -> query_values::$name<'tcx>
156+
) -> query_provided::$name<'tcx>
135157
};
136158
([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
137159
separate_provide_extern_decl!([$($modifiers)*][$($args)*])
@@ -183,30 +205,62 @@ macro_rules! define_callbacks {
183205

184206
$(pub type $name<'tcx> = $($K)*;)*
185207
}
186-
#[allow(nonstandard_style, unused_lifetimes, unused_parens)]
208+
#[allow(nonstandard_style, unused_lifetimes)]
187209
pub mod query_values {
188210
use super::*;
189211

190-
$(pub type $name<'tcx> = query_if_arena!([$($modifiers)*] <$V as Deref>::Target, $V);)*
212+
$(pub type $name<'tcx> = $V;)*
191213
}
192-
#[allow(nonstandard_style, unused_lifetimes, unused_parens)]
193-
pub mod query_storage {
214+
#[allow(nonstandard_style, unused_lifetimes)]
215+
pub mod query_provided {
194216
use super::*;
195217

196218
$(
197-
pub type $name<'tcx> = query_if_arena!([$($modifiers)*]
198-
<<$($K)* as Key>::CacheSelector
199-
as CacheSelector<'tcx, <$V as Deref>::Target>>::ArenaCache,
200-
<<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache
201-
);
219+
pub type $name<'tcx> = query_if_arena!([$($modifiers)*] (<$V as Deref>::Target) ($V));
202220
)*
203221
}
222+
#[allow(nonstandard_style, unused_lifetimes)]
223+
pub mod query_provided_to_value {
224+
use super::*;
204225

226+
$(
227+
#[inline]
228+
pub fn $name<'tcx>(
229+
_tcx: TyCtxt<'tcx>,
230+
value: query_provided::$name<'tcx>,
231+
) -> query_values::$name<'tcx> {
232+
query_if_arena!([$($modifiers)*]
233+
(&*_tcx.query_system.arenas.$name.alloc(value))
234+
(value)
235+
)
236+
}
237+
)*
238+
}
205239
#[allow(nonstandard_style, unused_lifetimes)]
206-
pub mod query_stored {
240+
pub mod query_storage {
207241
use super::*;
208242

209-
$(pub type $name<'tcx> = $V;)*
243+
$(
244+
pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache;
245+
)*
246+
}
247+
248+
pub struct QueryArenas<'tcx> {
249+
$($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*]
250+
(WorkerLocal<TypedArena<<$V as Deref>::Target>>)
251+
()
252+
),)*
253+
}
254+
255+
impl Default for QueryArenas<'_> {
256+
fn default() -> Self {
257+
Self {
258+
$($name: query_if_arena!([$($modifiers)*]
259+
(WorkerLocal::new(|_| Default::default()))
260+
()
261+
),)*
262+
}
263+
}
210264
}
211265

212266
#[derive(Default)]
@@ -221,7 +275,7 @@ macro_rules! define_callbacks {
221275
let key = key.into_query_param();
222276
opt_remap_env_constness!([$($modifiers)*][key]);
223277

224-
match try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key) {
278+
match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
225279
Some(_) => return,
226280
None => self.tcx.queries.$name(self.tcx, DUMMY_SP, key, QueryMode::Ensure),
227281
};
@@ -246,7 +300,7 @@ macro_rules! define_callbacks {
246300
let key = key.into_query_param();
247301
opt_remap_env_constness!([$($modifiers)*][key]);
248302

249-
match try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key) {
303+
match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
250304
Some(value) => value,
251305
None => self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap(),
252306
}
@@ -257,7 +311,7 @@ macro_rules! define_callbacks {
257311
$(pub $name: for<'tcx> fn(
258312
TyCtxt<'tcx>,
259313
query_keys::$name<'tcx>,
260-
) -> query_values::$name<'tcx>,)*
314+
) -> query_provided::$name<'tcx>,)*
261315
}
262316

263317
pub struct ExternProviders {
@@ -334,12 +388,13 @@ macro_rules! define_feedable {
334388
$(impl<'tcx, K: IntoQueryParam<$($K)*> + Copy> TyCtxtFeed<'tcx, K> {
335389
$(#[$attr])*
336390
#[inline(always)]
337-
pub fn $name(self, value: query_values::$name<'tcx>) -> $V {
391+
pub fn $name(self, value: query_provided::$name<'tcx>) -> $V {
338392
let key = self.key().into_query_param();
339393
opt_remap_env_constness!([$($modifiers)*][key]);
340394

341395
let tcx = self.tcx;
342-
let cache = &tcx.query_caches.$name;
396+
let value = query_provided_to_value::$name(tcx, value);
397+
let cache = &tcx.query_system.caches.$name;
343398

344399
match try_get_cached(tcx, cache, &key) {
345400
Some(old) => {
@@ -357,7 +412,8 @@ macro_rules! define_feedable {
357412
&value,
358413
hash_result!([$($modifiers)*]),
359414
);
360-
cache.complete(key, value, dep_node_index)
415+
cache.complete(key, value, dep_node_index);
416+
value
361417
}
362418
}
363419
}

‎compiler/rustc_query_impl/src/lib.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@ use rustc_data_structures::sync::AtomicU64;
2121
use rustc_middle::arena::Arena;
2222
use rustc_middle::dep_graph::{self, DepKindStruct};
2323
use rustc_middle::query::Key;
24-
use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
25-
use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
24+
use rustc_middle::ty::query::QueryEngine;
25+
use rustc_middle::ty::query::{
26+
query_keys, query_provided, query_provided_to_value, query_storage, query_values,
27+
};
2628
use rustc_middle::ty::TyCtxt;
2729
use rustc_span::Span;
2830

0 commit comments

Comments
 (0)