Skip to content

Commit 14ea63a

Browse files
borsbors
committed
Auto merge of #107627 - nnethercote:optimize-fold_ty, r=compiler-errors
Optimize `fold_ty` Micro-optimizing the heck out of the important `fold_ty` methods. r? `@oli-obk`
2 parents 75a0be9 + 4aec134 commit 14ea63a

File tree

3 files changed

+104
-99
lines changed

3 files changed

+104
-99
lines changed

‎compiler/rustc_infer/src/infer/freshen.rs‎

Lines changed: 61 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -140,79 +140,21 @@ impl<'a, 'tcx> TypeFolder<'tcx> for TypeFreshener<'a, 'tcx> {
140140
}
141141
}
142142

143+
#[inline]
143144
fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
144145
if !t.needs_infer() && !t.has_erasable_regions() {
145-
return t;
146-
}
147-
148-
let tcx = self.infcx.tcx;
149-
150-
match *t.kind() {
151-
ty::Infer(ty::TyVar(v)) => {
152-
let opt_ty = self.infcx.inner.borrow_mut().type_variables().probe(v).known();
153-
self.freshen_ty(opt_ty, ty::TyVar(v), ty::FreshTy)
154-
}
146+
t
147+
} else {
148+
match *t.kind() {
149+
ty::Infer(v) => self.fold_infer_ty(v).unwrap_or(t),
155150

156-
ty::Infer(ty::IntVar(v)) => self.freshen_ty(
157-
self.infcx
158-
.inner
159-
.borrow_mut()
160-
.int_unification_table()
161-
.probe_value(v)
162-
.map(|v| v.to_type(tcx)),
163-
ty::IntVar(v),
164-
ty::FreshIntTy,
165-
),
151+
// This code is hot enough that a non-debug assertion here makes a noticeable
152+
// difference on benchmarks like `wg-grammar`.
153+
#[cfg(debug_assertions)]
154+
ty::Placeholder(..) | ty::Bound(..) => bug!("unexpected type {:?}", t),
166155

167-
ty::Infer(ty::FloatVar(v)) => self.freshen_ty(
168-
self.infcx
169-
.inner
170-
.borrow_mut()
171-
.float_unification_table()
172-
.probe_value(v)
173-
.map(|v| v.to_type(tcx)),
174-
ty::FloatVar(v),
175-
ty::FreshFloatTy,
176-
),
177-
178-
ty::Infer(ty::FreshTy(ct) | ty::FreshIntTy(ct) | ty::FreshFloatTy(ct)) => {
179-
if ct >= self.ty_freshen_count {
180-
bug!(
181-
"Encountered a freshend type with id {} \
182-
but our counter is only at {}",
183-
ct,
184-
self.ty_freshen_count
185-
);
186-
}
187-
t
156+
_ => t.super_fold_with(self),
188157
}
189-
190-
ty::Generator(..)
191-
| ty::Bool
192-
| ty::Char
193-
| ty::Int(..)
194-
| ty::Uint(..)
195-
| ty::Float(..)
196-
| ty::Adt(..)
197-
| ty::Str
198-
| ty::Error(_)
199-
| ty::Array(..)
200-
| ty::Slice(..)
201-
| ty::RawPtr(..)
202-
| ty::Ref(..)
203-
| ty::FnDef(..)
204-
| ty::FnPtr(_)
205-
| ty::Dynamic(..)
206-
| ty::Never
207-
| ty::Tuple(..)
208-
| ty::Alias(..)
209-
| ty::Foreign(..)
210-
| ty::Param(..)
211-
| ty::Closure(..)
212-
| ty::GeneratorWitnessMIR(..)
213-
| ty::GeneratorWitness(..) => t.super_fold_with(self),
214-
215-
ty::Placeholder(..) | ty::Bound(..) => bug!("unexpected type {:?}", t),
216158
}
217159
}
218160

@@ -253,3 +195,54 @@ impl<'a, 'tcx> TypeFolder<'tcx> for TypeFreshener<'a, 'tcx> {
253195
}
254196
}
255197
}
198+
199+
impl<'a, 'tcx> TypeFreshener<'a, 'tcx> {
200+
// This is separate from `fold_ty` to keep that method small and inlinable.
201+
#[inline(never)]
202+
fn fold_infer_ty(&mut self, v: ty::InferTy) -> Option<Ty<'tcx>> {
203+
match v {
204+
ty::TyVar(v) => {
205+
let opt_ty = self.infcx.inner.borrow_mut().type_variables().probe(v).known();
206+
Some(self.freshen_ty(opt_ty, ty::TyVar(v), ty::FreshTy))
207+
}
208+
209+
ty::IntVar(v) => Some(
210+
self.freshen_ty(
211+
self.infcx
212+
.inner
213+
.borrow_mut()
214+
.int_unification_table()
215+
.probe_value(v)
216+
.map(|v| v.to_type(self.infcx.tcx)),
217+
ty::IntVar(v),
218+
ty::FreshIntTy,
219+
),
220+
),
221+
222+
ty::FloatVar(v) => Some(
223+
self.freshen_ty(
224+
self.infcx
225+
.inner
226+
.borrow_mut()
227+
.float_unification_table()
228+
.probe_value(v)
229+
.map(|v| v.to_type(self.infcx.tcx)),
230+
ty::FloatVar(v),
231+
ty::FreshFloatTy,
232+
),
233+
),
234+
235+
ty::FreshTy(ct) | ty::FreshIntTy(ct) | ty::FreshFloatTy(ct) => {
236+
if ct >= self.ty_freshen_count {
237+
bug!(
238+
"Encountered a freshend type with id {} \
239+
but our counter is only at {}",
240+
ct,
241+
self.ty_freshen_count
242+
);
243+
}
244+
None
245+
}
246+
}
247+
}
248+
}

‎compiler/rustc_infer/src/infer/mod.rs‎

Lines changed: 35 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ use rustc_middle::ty::relate::RelateResult;
3030
use rustc_middle::ty::subst::{GenericArg, GenericArgKind, InternalSubsts, SubstsRef};
3131
use rustc_middle::ty::visit::TypeVisitable;
3232
pub use rustc_middle::ty::IntVarValue;
33-
use rustc_middle::ty::{self, GenericParamDefKind, InferConst, Ty, TyCtxt};
33+
use rustc_middle::ty::{self, GenericParamDefKind, InferConst, InferTy, Ty, TyCtxt};
3434
use rustc_middle::ty::{ConstVid, FloatVid, IntVid, TyVid};
3535
use rustc_span::symbol::Symbol;
3636
use rustc_span::Span;
@@ -1389,8 +1389,8 @@ impl<'tcx> InferCtxt<'tcx> {
13891389
where
13901390
T: TypeFoldable<'tcx>,
13911391
{
1392-
if !value.needs_infer() {
1393-
return value; // Avoid duplicated subst-folding.
1392+
if !value.has_non_region_infer() {
1393+
return value;
13941394
}
13951395
let mut r = resolve::OpportunisticVarResolver::new(self);
13961396
value.fold_with(&mut r)
@@ -1870,9 +1870,33 @@ impl<'a, 'tcx> TypeFolder<'tcx> for ShallowResolver<'a, 'tcx> {
18701870
/// If `ty` is a type variable of some kind, resolve it one level
18711871
/// (but do not resolve types found in the result). If `typ` is
18721872
/// not a type variable, just return it unmodified.
1873+
#[inline]
18731874
fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
1874-
match *ty.kind() {
1875-
ty::Infer(ty::TyVar(v)) => {
1875+
if let ty::Infer(v) = ty.kind() { self.fold_infer_ty(*v).unwrap_or(ty) } else { ty }
1876+
}
1877+
1878+
fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> {
1879+
if let ty::ConstKind::Infer(InferConst::Var(vid)) = ct.kind() {
1880+
self.infcx
1881+
.inner
1882+
.borrow_mut()
1883+
.const_unification_table()
1884+
.probe_value(vid)
1885+
.val
1886+
.known()
1887+
.unwrap_or(ct)
1888+
} else {
1889+
ct
1890+
}
1891+
}
1892+
}
1893+
1894+
impl<'a, 'tcx> ShallowResolver<'a, 'tcx> {
1895+
// This is separate from `fold_ty` to keep that method small and inlinable.
1896+
#[inline(never)]
1897+
fn fold_infer_ty(&mut self, v: InferTy) -> Option<Ty<'tcx>> {
1898+
match v {
1899+
ty::TyVar(v) => {
18761900
// Not entirely obvious: if `typ` is a type variable,
18771901
// it can be resolved to an int/float variable, which
18781902
// can then be recursively resolved, hence the
@@ -1886,41 +1910,26 @@ impl<'a, 'tcx> TypeFolder<'tcx> for ShallowResolver<'a, 'tcx> {
18861910
// Note: if these two lines are combined into one we get
18871911
// dynamic borrow errors on `self.inner`.
18881912
let known = self.infcx.inner.borrow_mut().type_variables().probe(v).known();
1889-
known.map_or(ty, |t| self.fold_ty(t))
1913+
known.map(|t| self.fold_ty(t))
18901914
}
18911915

1892-
ty::Infer(ty::IntVar(v)) => self
1916+
ty::IntVar(v) => self
18931917
.infcx
18941918
.inner
18951919
.borrow_mut()
18961920
.int_unification_table()
18971921
.probe_value(v)
1898-
.map_or(ty, |v| v.to_type(self.infcx.tcx)),
1922+
.map(|v| v.to_type(self.infcx.tcx)),
18991923

1900-
ty::Infer(ty::FloatVar(v)) => self
1924+
ty::FloatVar(v) => self
19011925
.infcx
19021926
.inner
19031927
.borrow_mut()
19041928
.float_unification_table()
19051929
.probe_value(v)
1906-
.map_or(ty, |v| v.to_type(self.infcx.tcx)),
1930+
.map(|v| v.to_type(self.infcx.tcx)),
19071931

1908-
_ => ty,
1909-
}
1910-
}
1911-
1912-
fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> {
1913-
if let ty::ConstKind::Infer(InferConst::Var(vid)) = ct.kind() {
1914-
self.infcx
1915-
.inner
1916-
.borrow_mut()
1917-
.const_unification_table()
1918-
.probe_value(vid)
1919-
.val
1920-
.known()
1921-
.unwrap_or(ct)
1922-
} else {
1923-
ct
1932+
ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_) => None,
19241933
}
19251934
}
19261935
}

‎compiler/rustc_infer/src/infer/resolve.rs‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,26 +16,29 @@ use std::ops::ControlFlow;
1616
/// useful for printing messages etc but also required at various
1717
/// points for correctness.
1818
pub struct OpportunisticVarResolver<'a, 'tcx> {
19-
infcx: &'a InferCtxt<'tcx>,
19+
// The shallow resolver is used to resolve inference variables at every
20+
// level of the type.
21+
shallow_resolver: crate::infer::ShallowResolver<'a, 'tcx>,
2022
}
2123

2224
impl<'a, 'tcx> OpportunisticVarResolver<'a, 'tcx> {
2325
#[inline]
2426
pub fn new(infcx: &'a InferCtxt<'tcx>) -> Self {
25-
OpportunisticVarResolver { infcx }
27+
OpportunisticVarResolver { shallow_resolver: crate::infer::ShallowResolver { infcx } }
2628
}
2729
}
2830

2931
impl<'a, 'tcx> TypeFolder<'tcx> for OpportunisticVarResolver<'a, 'tcx> {
3032
fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
31-
self.infcx.tcx
33+
TypeFolder::tcx(&self.shallow_resolver)
3234
}
3335

36+
#[inline]
3437
fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
3538
if !t.has_non_region_infer() {
3639
t // micro-optimize -- if there is nothing in this type that this fold affects...
3740
} else {
38-
let t = self.infcx.shallow_resolve(t);
41+
let t = self.shallow_resolver.fold_ty(t);
3942
t.super_fold_with(self)
4043
}
4144
}
@@ -44,7 +47,7 @@ impl<'a, 'tcx> TypeFolder<'tcx> for OpportunisticVarResolver<'a, 'tcx> {
4447
if !ct.has_non_region_infer() {
4548
ct // micro-optimize -- if there is nothing in this const that this fold affects...
4649
} else {
47-
let ct = self.infcx.shallow_resolve(ct);
50+
let ct = self.shallow_resolver.fold_const(ct);
4851
ct.super_fold_with(self)
4952
}
5053
}

0 commit comments

Comments
 (0)