Skip to content

Commit b93a98e

Browse files
aliemjayaliemjay
committed
canonicalize param env only once
1 parent 492e57c commit b93a98e

File tree

7 files changed

+129
-17
lines changed

7 files changed

+129
-17
lines changed

‎compiler/rustc_borrowck/src/type_check/canonical.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
3838
let old_universe = self.infcx.universe();
3939

4040
let TypeOpOutput { output, constraints, error_info } =
41-
op.fully_perform(self.infcx, locations.span(self.body))?;
41+
op.fully_perform(self.infcx, locations.span(self.body), &self.canon_cache)?;
4242

4343
debug!(?output, ?constraints);
4444

‎compiler/rustc_borrowck/src/type_check/free_region_relations.rs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use rustc_middle::ty::{self, RegionVid, Ty};
1111
use rustc_span::{ErrorGuaranteed, Span, DUMMY_SP};
1212
use rustc_trait_selection::traits::query::type_op::{self, TypeOp};
1313
use std::rc::Rc;
14-
use type_op::TypeOpOutput;
14+
use type_op::{CanonCache, TypeOpOutput};
1515

1616
use crate::{
1717
type_check::constraint_conversion,
@@ -50,13 +50,15 @@ pub(crate) struct CreateResult<'tcx> {
5050
pub(crate) fn create<'tcx>(
5151
infcx: &InferCtxt<'tcx>,
5252
param_env: ty::ParamEnv<'tcx>,
53+
canon_cache: &CanonCache<'tcx>,
5354
implicit_region_bound: ty::Region<'tcx>,
5455
universal_regions: &Rc<UniversalRegions<'tcx>>,
5556
constraints: &mut MirTypeckRegionConstraints<'tcx>,
5657
) -> CreateResult<'tcx> {
5758
UniversalRegionRelationsBuilder {
5859
infcx,
5960
param_env,
61+
canon_cache,
6062
implicit_region_bound,
6163
constraints,
6264
universal_regions: universal_regions.clone(),
@@ -174,6 +176,7 @@ impl UniversalRegionRelations<'_> {
174176
struct UniversalRegionRelationsBuilder<'this, 'tcx> {
175177
infcx: &'this InferCtxt<'tcx>,
176178
param_env: ty::ParamEnv<'tcx>,
179+
canon_cache: &'this CanonCache<'tcx>,
177180
universal_regions: Rc<UniversalRegions<'tcx>>,
178181
implicit_region_bound: ty::Region<'tcx>,
179182
constraints: &'this mut MirTypeckRegionConstraints<'tcx>,
@@ -243,7 +246,7 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> {
243246
let TypeOpOutput { output: norm_ty, constraints: constraints_normalize, .. } = self
244247
.param_env
245248
.and(type_op::normalize::Normalize::new(ty))
246-
.fully_perform(self.infcx, span)
249+
.fully_perform(self.infcx, span, self.canon_cache)
247250
.unwrap_or_else(|guar| TypeOpOutput {
248251
output: Ty::new_error(self.infcx.tcx, guar),
249252
constraints: None,
@@ -317,7 +320,7 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> {
317320
let TypeOpOutput { output: bounds, constraints, .. } = self
318321
.param_env
319322
.and(type_op::implied_outlives_bounds::ImpliedOutlivesBounds { ty })
320-
.fully_perform(self.infcx, DUMMY_SP)
323+
.fully_perform(self.infcx, DUMMY_SP, self.canon_cache)
321324
.map_err(|_: ErrorGuaranteed| debug!("failed to compute implied bounds {:?}", ty))
322325
.ok()?;
323326
debug!(?bounds, ?constraints);

‎compiler/rustc_borrowck/src/type_check/liveness/trace.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -642,11 +642,11 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
642642
) -> DropData<'tcx> {
643643
debug!("compute_drop_data(dropped_ty={:?})", dropped_ty,);
644644

645-
match typeck
646-
.param_env
647-
.and(DropckOutlives::new(dropped_ty))
648-
.fully_perform(typeck.infcx, DUMMY_SP)
649-
{
645+
match typeck.param_env.and(DropckOutlives::new(dropped_ty)).fully_perform(
646+
typeck.infcx,
647+
DUMMY_SP,
648+
&typeck.canon_cache,
649+
) {
650650
Ok(TypeOpOutput { output, constraints, .. }) => {
651651
DropData { dropck_result: output, region_constraint_data: constraints }
652652
}

‎compiler/rustc_borrowck/src/type_check/mod.rs‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ use rustc_hir::def_id::LocalDefId;
1616
use rustc_hir::lang_items::LangItem;
1717
use rustc_index::bit_set::SparseBitMatrix;
1818
use rustc_index::{IndexSlice, IndexVec};
19-
use rustc_infer::infer::canonical::QueryRegionConstraints;
19+
use rustc_infer::infer::canonical::{OriginalQueryValues, QueryRegionConstraints};
2020
use rustc_infer::infer::outlives::env::RegionBoundPairs;
2121
use rustc_infer::infer::region_constraints::RegionConstraintData;
2222
use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
@@ -43,7 +43,7 @@ use rustc_span::{Span, DUMMY_SP};
4343
use rustc_target::abi::{FieldIdx, FIRST_VARIANT};
4444
use rustc_trait_selection::traits::query::type_op::custom::scrape_region_constraints;
4545
use rustc_trait_selection::traits::query::type_op::custom::CustomTypeOp;
46-
use rustc_trait_selection::traits::query::type_op::{TypeOp, TypeOpOutput};
46+
use rustc_trait_selection::traits::query::type_op::{CanonCache, TypeOp, TypeOpOutput};
4747

4848
use rustc_trait_selection::traits::PredicateObligation;
4949

@@ -152,13 +152,19 @@ pub(crate) fn type_check<'mir, 'tcx>(
152152
universe_causes: FxIndexMap::default(),
153153
};
154154

155+
let canon_cache = {
156+
let mut var_values = OriginalQueryValues::default();
157+
Some((infcx.canonicalize_query_keep_static(param_env, &mut var_values), var_values))
158+
};
159+
155160
let CreateResult {
156161
universal_region_relations,
157162
region_bound_pairs,
158163
normalized_inputs_and_output,
159164
} = free_region_relations::create(
160165
infcx,
161166
param_env,
167+
&canon_cache,
162168
implicit_region_bound,
163169
universal_regions,
164170
&mut constraints,
@@ -183,6 +189,7 @@ pub(crate) fn type_check<'mir, 'tcx>(
183189
infcx,
184190
body,
185191
param_env,
192+
canon_cache,
186193
&region_bound_pairs,
187194
implicit_region_bound,
188195
&mut borrowck_context,
@@ -840,6 +847,7 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
840847
struct TypeChecker<'a, 'tcx> {
841848
infcx: &'a BorrowckInferCtxt<'a, 'tcx>,
842849
param_env: ty::ParamEnv<'tcx>,
850+
canon_cache: CanonCache<'tcx>,
843851
last_span: Span,
844852
body: &'a Body<'tcx>,
845853
/// User type annotations are shared between the main MIR and the MIR of
@@ -1002,6 +1010,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
10021010
infcx: &'a BorrowckInferCtxt<'a, 'tcx>,
10031011
body: &'a Body<'tcx>,
10041012
param_env: ty::ParamEnv<'tcx>,
1013+
canon_cache: CanonCache<'tcx>,
10051014
region_bound_pairs: &'a RegionBoundPairs<'tcx>,
10061015
implicit_region_bound: ty::Region<'tcx>,
10071016
borrowck_context: &'a mut BorrowCheckContext<'a, 'tcx>,
@@ -1012,6 +1021,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
10121021
body,
10131022
user_type_annotations: &body.user_type_annotations,
10141023
param_env,
1024+
canon_cache,
10151025
region_bound_pairs,
10161026
implicit_region_bound,
10171027
borrowck_context,
@@ -2764,6 +2774,7 @@ impl<'tcx> TypeOp<'tcx> for InstantiateOpaqueType<'tcx> {
27642774
mut self,
27652775
infcx: &InferCtxt<'tcx>,
27662776
span: Span,
2777+
_: &CanonCache<'tcx>,
27672778
) -> Result<TypeOpOutput<'tcx, Self>, ErrorGuaranteed> {
27682779
let (mut output, region_constraints) = scrape_region_constraints(
27692780
infcx,

‎compiler/rustc_infer/src/infer/canonical/canonicalizer.rs‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use rustc_middle::ty::flags::FlagComputation;
1313
use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
1414
use rustc_middle::ty::GenericArg;
1515
use rustc_middle::ty::{self, BoundVar, InferConst, List, Ty, TyCtxt, TypeFlags, TypeVisitableExt};
16+
use std::borrow::Cow;
1617
use std::sync::atomic::Ordering;
1718

1819
use rustc_data_structures::fx::FxHashMap;
@@ -148,6 +149,27 @@ impl<'tcx> InferCtxt<'tcx> {
148149
query_state,
149150
)
150151
}
152+
153+
pub fn canonicalize_query_keep_static_continue<U, V>(
154+
&self,
155+
base: Canonical<'tcx, U>,
156+
value: V,
157+
query_state: &mut Cow<'_, OriginalQueryValues<'tcx>>,
158+
) -> Canonical<'tcx, (U, V)>
159+
where
160+
V: TypeFoldable<TyCtxt<'tcx>>,
161+
{
162+
self.tcx.sess.perf_stats.queries_canonicalized.fetch_add(1, Ordering::Relaxed);
163+
164+
Canonicalizer::canonicalize_continue(
165+
base,
166+
value,
167+
self,
168+
self.tcx,
169+
&CanonicalizeFreeRegionsOtherThanStatic,
170+
query_state,
171+
)
172+
}
151173
}
152174

153175
/// Controls how we canonicalize "free regions" that are not inference
@@ -616,6 +638,66 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
616638
Canonical { max_universe, variables: canonical_variables, value: out_value }
617639
}
618640

641+
fn canonicalize_continue<U, V>(
642+
base: Canonical<'tcx, U>,
643+
value: V,
644+
infcx: &InferCtxt<'tcx>,
645+
tcx: TyCtxt<'tcx>,
646+
canonicalize_region_mode: &dyn CanonicalizeMode,
647+
query_state: &mut Cow<'_, OriginalQueryValues<'tcx>>,
648+
) -> Canonical<'tcx, (U, V)>
649+
where
650+
V: TypeFoldable<TyCtxt<'tcx>>,
651+
{
652+
let needs_canonical_flags = if canonicalize_region_mode.any() {
653+
TypeFlags::HAS_INFER | TypeFlags::HAS_PLACEHOLDER | TypeFlags::HAS_FREE_REGIONS
654+
} else {
655+
TypeFlags::HAS_INFER | TypeFlags::HAS_PLACEHOLDER
656+
};
657+
658+
// Fast path: nothing that needs to be canonicalized.
659+
if !value.has_type_flags(needs_canonical_flags) {
660+
return base.unchecked_map(|b| (b, value));
661+
}
662+
663+
let mut canonicalizer = Canonicalizer {
664+
infcx,
665+
tcx,
666+
canonicalize_mode: canonicalize_region_mode,
667+
needs_canonical_flags,
668+
variables: SmallVec::from_slice(base.variables),
669+
query_state: query_state.to_mut(),
670+
indices: FxHashMap::default(),
671+
binder_index: ty::INNERMOST,
672+
};
673+
if canonicalizer.query_state.var_values.spilled() {
674+
canonicalizer.indices = canonicalizer
675+
.query_state
676+
.var_values
677+
.iter()
678+
.enumerate()
679+
.map(|(i, &kind)| (kind, BoundVar::new(i)))
680+
.collect();
681+
}
682+
let out_value = value.fold_with(&mut canonicalizer);
683+
684+
// Once we have canonicalized `out_value`, it should not
685+
// contain anything that ties it to this inference context
686+
// anymore.
687+
debug_assert!(!out_value.has_infer() && !out_value.has_placeholders());
688+
689+
let canonical_variables =
690+
tcx.mk_canonical_var_infos(&canonicalizer.universe_canonicalized_variables());
691+
692+
let max_universe = canonical_variables
693+
.iter()
694+
.map(|cvar| cvar.universe())
695+
.max()
696+
.unwrap_or(ty::UniverseIndex::ROOT);
697+
698+
Canonical { max_universe, variables: canonical_variables, value: (base.value, out_value) }
699+
}
700+
619701
/// Creates a canonical variable replacing `kind` from the input,
620702
/// or returns an existing variable if `kind` has already been
621703
/// seen. `kind` is expected to be an unbound variable (or

‎compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use crate::infer::canonical::query_response;
22
use crate::infer::InferCtxt;
3-
use crate::traits::query::type_op::TypeOpOutput;
3+
use crate::traits::query::type_op::{CanonCache, TypeOpOutput};
44
use crate::traits::ObligationCtxt;
55
use rustc_errors::ErrorGuaranteed;
66
use rustc_infer::infer::region_constraints::RegionConstraintData;
@@ -41,6 +41,7 @@ where
4141
self,
4242
infcx: &InferCtxt<'tcx>,
4343
span: Span,
44+
_: &CanonCache<'tcx>,
4445
) -> Result<TypeOpOutput<'tcx, Self>, ErrorGuaranteed> {
4546
if cfg!(debug_assertions) {
4647
info!("fully_perform({:?})", self);

‎compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@ use rustc_infer::infer::canonical::Certainty;
88
use rustc_infer::traits::PredicateObligations;
99
use rustc_middle::traits::query::NoSolution;
1010
use rustc_middle::ty::fold::TypeFoldable;
11-
use rustc_middle::ty::{ParamEnvAnd, TyCtxt};
11+
use rustc_middle::ty::{ParamEnv, ParamEnvAnd, TyCtxt};
1212
use rustc_span::Span;
13+
use std::borrow::Cow;
1314
use std::fmt;
1415

1516
pub mod ascribe_user_type;
@@ -25,6 +26,8 @@ pub use rustc_middle::traits::query::type_op::*;
2526

2627
use self::custom::scrape_region_constraints;
2728

29+
pub type CanonCache<'tcx> = Option<(Canonical<'tcx, ParamEnv<'tcx>>, OriginalQueryValues<'tcx>)>;
30+
2831
/// "Type ops" are used in NLL to perform some particular action and
2932
/// extract out the resulting region constraints (or an error if it
3033
/// cannot be completed).
@@ -39,6 +42,7 @@ pub trait TypeOp<'tcx>: Sized + fmt::Debug {
3942
self,
4043
infcx: &InferCtxt<'tcx>,
4144
span: Span,
45+
cache: &CanonCache<'tcx>,
4246
) -> Result<TypeOpOutput<'tcx, Self>, ErrorGuaranteed>;
4347
}
4448

@@ -98,6 +102,7 @@ pub trait QueryTypeOp<'tcx>: fmt::Debug + Copy + TypeFoldable<TyCtxt<'tcx>> + 't
98102
query_key: ParamEnvAnd<'tcx, Self>,
99103
infcx: &InferCtxt<'tcx>,
100104
output_query_region_constraints: &mut QueryRegionConstraints<'tcx>,
105+
cache: &CanonCache<'tcx>,
101106
) -> Result<
102107
(
103108
Self::QueryResponse,
@@ -115,10 +120,18 @@ pub trait QueryTypeOp<'tcx>: fmt::Debug + Copy + TypeFoldable<TyCtxt<'tcx>> + 't
115120
// `canonicalize_query_keep_static` here because of things
116121
// like the subtype query, which go awry around
117122
// `'static` otherwise.
118-
let mut canonical_var_values = OriginalQueryValues::default();
123+
let mut canonical_var_values;
119124
let old_param_env = query_key.param_env;
120-
let canonical_self =
121-
infcx.canonicalize_query_keep_static(query_key, &mut canonical_var_values);
125+
let canonical_self = if let &Some((base, ref cache_var_values)) = cache {
126+
canonical_var_values = Cow::Borrowed(cache_var_values);
127+
let ParamEnvAnd { param_env: _, value } = query_key;
128+
infcx
129+
.canonicalize_query_keep_static_continue(base, value, &mut canonical_var_values)
130+
.unchecked_map(|(param_env, value)| param_env.and(value))
131+
} else {
132+
canonical_var_values = Cow::Owned(OriginalQueryValues::default());
133+
infcx.canonicalize_query_keep_static(query_key, canonical_var_values.to_mut())
134+
};
122135
let canonical_result = Self::perform_query(infcx.tcx, canonical_self)?;
123136

124137
let InferOk { value, obligations } = infcx
@@ -145,6 +158,7 @@ where
145158
self,
146159
infcx: &InferCtxt<'tcx>,
147160
span: Span,
161+
cache: &CanonCache<'tcx>,
148162
) -> Result<TypeOpOutput<'tcx, Self>, ErrorGuaranteed> {
149163
if infcx.next_trait_solver() {
150164
return Ok(scrape_region_constraints(
@@ -158,7 +172,7 @@ where
158172

159173
let mut region_constraints = QueryRegionConstraints::default();
160174
let (output, error_info, mut obligations) =
161-
Q::fully_perform_into(self, infcx, &mut region_constraints)
175+
Q::fully_perform_into(self, infcx, &mut region_constraints, cache)
162176
.map_err(|_| {
163177
infcx.tcx.sess.delay_span_bug(span, format!("error performing {self:?}"))
164178
})
@@ -184,6 +198,7 @@ where
184198
obligation.param_env.and(ProvePredicate::new(obligation.predicate)),
185199
infcx,
186200
&mut region_constraints,
201+
cache,
187202
) {
188203
Ok(((), _, new, certainty)) => {
189204
obligations.extend(new);

0 commit comments

Comments
 (0)