Skip to content

Commit 65f0ce0

Browse files
ZoxcZoxc
committed
Factor query arena allocation out from query caches
1 parent 97befd4 commit 65f0ce0

File tree

10 files changed

+160
-255
lines changed

10 files changed

+160
-255
lines changed

‎compiler/rustc_data_structures/src/sync/worker_local.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,12 @@ impl<T> WorkerLocal<Vec<T>> {
164164
}
165165
}
166166

167+
impl<T: Default> Default for WorkerLocal<T> {
168+
fn default() -> Self {
169+
WorkerLocal::new(|_| T::default())
170+
}
171+
}
172+
167173
impl<T> Deref for WorkerLocal<T> {
168174
type Target = T;
169175

‎compiler/rustc_interface/src/passes.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -781,9 +781,7 @@ pub fn create_global_ctxt<'tcx>(
781781
callback(sess, &mut local_providers, &mut extern_providers);
782782
}
783783

784-
let queries = queries.get_or_init(|| {
785-
TcxQueries::new(local_providers, extern_providers, query_result_on_disk_cache)
786-
});
784+
let queries = queries.get_or_init(|| TcxQueries::new(query_result_on_disk_cache));
787785

788786
let gcx = sess.time("setup_global_ctxt", || {
789787
global_ctxt.get_or_init(move || {
@@ -795,6 +793,8 @@ pub fn create_global_ctxt<'tcx>(
795793
untracked,
796794
dep_graph,
797795
queries.on_disk_cache.as_ref().map(OnDiskCache::as_dyn),
796+
local_providers,
797+
extern_providers,
798798
queries.as_dyn(),
799799
rustc_query_impl::query_callbacks(arena),
800800
)

‎compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ macro_rules! provide_one {
114114
fn $name<'tcx>(
115115
$tcx: TyCtxt<'tcx>,
116116
def_id_arg: ty::query::query_keys::$name<'tcx>,
117-
) -> ty::query::query_values::$name<'tcx> {
117+
) -> ty::query::query_provided::$name<'tcx> {
118118
let _prof_timer =
119119
$tcx.prof.generic_activity(concat!("metadata_decode_entry_", stringify!($name)));
120120

‎compiler/rustc_middle/src/ty/context.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ use crate::mir::{
1818
use crate::thir::Thir;
1919
use crate::traits;
2020
use crate::traits::solve::{ExternalConstraints, ExternalConstraintsData};
21+
use crate::ty::query::ExternProviders;
22+
use crate::ty::query::Providers;
2123
use crate::ty::query::{self, TyCtxtAt};
2224
use crate::ty::{
2325
self, AdtDef, AdtDefData, AdtKind, Binder, Const, ConstData, DefIdTree, FloatTy, FloatVar,
@@ -445,7 +447,7 @@ pub struct GlobalCtxt<'tcx> {
445447
pub on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
446448

447449
pub queries: &'tcx dyn query::QueryEngine<'tcx>,
448-
pub query_caches: query::QueryCaches<'tcx>,
450+
pub query_system: query::QuerySystem<'tcx>,
449451
pub(crate) query_kinds: &'tcx [DepKindStruct<'tcx>],
450452

451453
// Internal caches for metadata decoding. No need to track deps on this.
@@ -593,6 +595,8 @@ impl<'tcx> TyCtxt<'tcx> {
593595
untracked: Untracked,
594596
dep_graph: DepGraph,
595597
on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
598+
local_providers: Providers,
599+
extern_providers: ExternProviders,
596600
queries: &'tcx dyn query::QueryEngine<'tcx>,
597601
query_kinds: &'tcx [DepKindStruct<'tcx>],
598602
) -> GlobalCtxt<'tcx> {
@@ -618,7 +622,7 @@ impl<'tcx> TyCtxt<'tcx> {
618622
untracked,
619623
on_disk_cache,
620624
queries,
621-
query_caches: query::QueryCaches::default(),
625+
query_system: query::QuerySystem::new(local_providers, extern_providers),
622626
query_kinds,
623627
ty_rcache: Default::default(),
624628
pred_rcache: Default::default(),

‎compiler/rustc_middle/src/ty/query.rs‎

Lines changed: 72 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#![allow(unused_parens)]
2+
13
use crate::dep_graph;
24
use crate::infer::canonical::{self, Canonical};
35
use crate::lint::LintExpectation;
@@ -35,13 +37,15 @@ use crate::ty::subst::{GenericArg, SubstsRef};
3537
use crate::ty::util::AlwaysRequiresDrop;
3638
use crate::ty::GeneratorDiagnosticData;
3739
use crate::ty::{self, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt, UnusedGenericParams};
40+
use rustc_arena::TypedArena;
3841
use rustc_ast as ast;
3942
use rustc_ast::expand::allocator::AllocatorKind;
4043
use rustc_attr as attr;
4144
use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
4245
use rustc_data_structures::steal::Steal;
4346
use rustc_data_structures::svh::Svh;
4447
use rustc_data_structures::sync::Lrc;
48+
use rustc_data_structures::sync::WorkerLocal;
4549
use rustc_data_structures::unord::UnordSet;
4650
use rustc_errors::ErrorGuaranteed;
4751
use rustc_hir as hir;
@@ -67,6 +71,24 @@ use std::sync::Arc;
6771
pub(crate) use rustc_query_system::query::QueryJobId;
6872
use rustc_query_system::query::*;
6973

74+
pub struct QuerySystem<'tcx> {
75+
pub local_providers: Box<Providers>,
76+
pub extern_providers: Box<ExternProviders>,
77+
pub arenas: QueryArenas<'tcx>,
78+
pub caches: QueryCaches<'tcx>,
79+
}
80+
81+
impl<'tcx> QuerySystem<'tcx> {
82+
pub fn new(local_providers: Providers, extern_providers: ExternProviders) -> Self {
83+
QuerySystem {
84+
local_providers: Box::new(local_providers),
85+
extern_providers: Box::new(extern_providers),
86+
arenas: Default::default(),
87+
caches: Default::default(),
88+
}
89+
}
90+
}
91+
7092
#[derive(Copy, Clone)]
7193
pub struct TyCtxtAt<'tcx> {
7294
pub tcx: TyCtxt<'tcx>,
@@ -115,14 +137,14 @@ fn query_get_at<'tcx, Cache, K>(
115137
Span,
116138
Cache::Key,
117139
QueryMode,
118-
) -> Option<Cache::Stored>,
140+
) -> Option<Cache::Value>,
119141
query_cache: &Cache,
120142
span: Span,
121143
key: K,
122-
) -> Cache::Stored
144+
) -> Cache::Value
123145
where
124146
K: IntoQueryParam<Cache::Key>,
125-
Cache::Stored: Copy,
147+
Cache::Value: Copy,
126148
Cache: QueryCache,
127149
{
128150
let key = key.into_query_param();
@@ -141,12 +163,12 @@ fn query_ensure<'tcx, Cache, K>(
141163
Span,
142164
Cache::Key,
143165
QueryMode,
144-
) -> Option<Cache::Stored>,
166+
) -> Option<Cache::Value>,
145167
query_cache: &Cache,
146168
key: K,
147169
) where
148170
K: IntoQueryParam<Cache::Key>,
149-
Cache::Stored: Copy,
171+
Cache::Value: Copy,
150172
Cache: QueryCache,
151173
{
152174
let key = key.into_query_param();
@@ -162,10 +184,10 @@ macro_rules! query_helper_param_ty {
162184
}
163185

164186
macro_rules! query_if_arena {
165-
([] $arena:ty, $no_arena:ty) => {
187+
([] $arena:tt $no_arena:tt) => {
166188
$no_arena
167189
};
168-
([(arena_cache) $($rest:tt)*] $arena:ty, $no_arena:ty) => {
190+
([(arena_cache) $($rest:tt)*] $arena:tt $no_arena:tt) => {
169191
$arena
170192
};
171193
([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
@@ -181,7 +203,7 @@ macro_rules! separate_provide_extern_decl {
181203
for<'tcx> fn(
182204
TyCtxt<'tcx>,
183205
query_keys::$name<'tcx>,
184-
) -> query_values::$name<'tcx>
206+
) -> query_provided::$name<'tcx>
185207
};
186208
([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
187209
separate_provide_extern_decl!([$($modifiers)*][$($args)*])
@@ -233,30 +255,52 @@ macro_rules! define_callbacks {
233255

234256
$(pub type $name<'tcx> = $($K)*;)*
235257
}
236-
#[allow(nonstandard_style, unused_lifetimes, unused_parens)]
258+
#[allow(nonstandard_style, unused_lifetimes)]
237259
pub mod query_values {
238260
use super::*;
239261

240-
$(pub type $name<'tcx> = query_if_arena!([$($modifiers)*] <$V as Deref>::Target, $V);)*
262+
$(pub type $name<'tcx> = $V;)*
241263
}
242-
#[allow(nonstandard_style, unused_lifetimes, unused_parens)]
243-
pub mod query_storage {
264+
#[allow(nonstandard_style, unused_lifetimes)]
265+
pub mod query_provided {
244266
use super::*;
245267

246268
$(
247-
pub type $name<'tcx> = query_if_arena!([$($modifiers)*]
248-
<<$($K)* as Key>::CacheSelector
249-
as CacheSelector<'tcx, <$V as Deref>::Target>>::ArenaCache,
250-
<<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache
251-
);
269+
pub type $name<'tcx> = query_if_arena!([$($modifiers)*] (<$V as Deref>::Target) ($V));
252270
)*
253271
}
272+
#[allow(nonstandard_style, unused_lifetimes)]
273+
pub mod query_provided_to_value {
274+
use super::*;
254275

276+
$(
277+
#[inline]
278+
pub fn $name<'tcx>(
279+
_tcx: TyCtxt<'tcx>,
280+
value: query_provided::$name<'tcx>,
281+
) -> query_values::$name<'tcx> {
282+
query_if_arena!([$($modifiers)*]
283+
(&*_tcx.query_system.arenas.$name.alloc(value))
284+
(value)
285+
)
286+
}
287+
)*
288+
}
255289
#[allow(nonstandard_style, unused_lifetimes)]
256-
pub mod query_stored {
290+
pub mod query_storage {
257291
use super::*;
258292

259-
$(pub type $name<'tcx> = $V;)*
293+
$(
294+
pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache;
295+
)*
296+
}
297+
298+
#[derive(Default)]
299+
pub struct QueryArenas<'tcx> {
300+
$($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*]
301+
(WorkerLocal<TypedArena<<$V as Deref>::Target>>)
302+
()
303+
),)*
260304
}
261305

262306
#[derive(Default)]
@@ -271,7 +315,7 @@ macro_rules! define_callbacks {
271315
query_ensure(
272316
self.tcx,
273317
QueryEngine::$name,
274-
&self.tcx.query_caches.$name,
318+
&self.tcx.query_system.caches.$name,
275319
opt_remap_env_constness!([$($modifiers)*][key]),
276320
);
277321
})*
@@ -286,7 +330,7 @@ macro_rules! define_callbacks {
286330
query_get_at(
287331
self,
288332
QueryEngine::$name,
289-
&self.query_caches.$name,
333+
&self.query_system.caches.$name,
290334
DUMMY_SP,
291335
opt_remap_env_constness!([$($modifiers)*][key]),
292336
)
@@ -301,7 +345,7 @@ macro_rules! define_callbacks {
301345
query_get_at(
302346
self.tcx,
303347
QueryEngine::$name,
304-
&self.tcx.query_caches.$name,
348+
&self.tcx.query_system.caches.$name,
305349
self.span,
306350
opt_remap_env_constness!([$($modifiers)*][key]),
307351
)
@@ -312,7 +356,7 @@ macro_rules! define_callbacks {
312356
$(pub $name: for<'tcx> fn(
313357
TyCtxt<'tcx>,
314358
query_keys::$name<'tcx>,
315-
) -> query_values::$name<'tcx>,)*
359+
) -> query_provided::$name<'tcx>,)*
316360
}
317361

318362
pub struct ExternProviders {
@@ -389,12 +433,13 @@ macro_rules! define_feedable {
389433
$(impl<'tcx, K: IntoQueryParam<$($K)*> + Copy> TyCtxtFeed<'tcx, K> {
390434
$(#[$attr])*
391435
#[inline(always)]
392-
pub fn $name(self, value: query_values::$name<'tcx>) -> $V {
436+
pub fn $name(self, value: query_provided::$name<'tcx>) -> $V {
393437
let key = self.key().into_query_param();
394438
let key = opt_remap_env_constness!([$($modifiers)*][key]);
395439

396440
let tcx = self.tcx;
397-
let cache = &tcx.query_caches.$name;
441+
let value = query_provided_to_value::$name(tcx, value);
442+
let cache = &tcx.query_system.caches.$name;
398443

399444
match try_get_cached(tcx, cache, &key) {
400445
Some(old) => {
@@ -412,7 +457,8 @@ macro_rules! define_feedable {
412457
&value,
413458
hash_result!([$($modifiers)*]),
414459
);
415-
cache.complete(key, value, dep_node_index)
460+
cache.complete(key, value, dep_node_index);
461+
value
416462
}
417463
}
418464
}

‎compiler/rustc_query_impl/src/lib.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@ use rustc_data_structures::sync::AtomicU64;
2121
use rustc_middle::arena::Arena;
2222
use rustc_middle::dep_graph::{self, DepKindStruct};
2323
use rustc_middle::query::Key;
24-
use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
25-
use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
24+
use rustc_middle::ty::query::QueryEngine;
25+
use rustc_middle::ty::query::{
26+
query_keys, query_provided, query_provided_to_value, query_storage, query_values,
27+
};
2628
use rustc_middle::ty::TyCtxt;
2729
use rustc_span::Span;
2830

0 commit comments

Comments
 (0)