Skip to content

Commit faf5684

Browse files
davidtwcoJamieCunliffe
davidtwco
and
JamieCunliffe
committed
codegen: implement repr(scalable)
Introduces `BackendRepr::ScalableVector` corresponding to scalable vector types annotated with `repr(scalable)` which lowers to a scalable vector type in LLVM. Co-authored-by: Jamie Cunliffe <Jamie.Cunliffe@arm.com>
1 parent 16d90b2 commit faf5684

File tree

40 files changed

+496
-47
lines changed

40 files changed

+496
-47
lines changed

‎compiler/rustc_abi/src/callconv.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
8282
}))
8383
}
8484

85+
BackendRepr::ScalableVector { .. } => Err(Heterogeneous),
86+
8587
BackendRepr::ScalarPair(..) | BackendRepr::Memory { sized: true } => {
8688
// Helper for computing `homogeneous_aggregate`, allowing a custom
8789
// starting offset (used below for handling variants).

‎compiler/rustc_abi/src/layout.rs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
151151
element: F,
152152
count: u64,
153153
repr_packed: bool,
154+
scalable: Option<u32>,
154155
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
155156
let elt = element.as_ref();
156157
if count == 0 {
@@ -169,7 +170,12 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
169170
let dl = self.cx.data_layout();
170171
let size =
171172
elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
172-
let (repr, align) = if repr_packed && !count.is_power_of_two() {
173+
let (repr, align) = if let Some(elt) = scalable {
174+
(
175+
BackendRepr::ScalableVector { element: e_repr, count: elt as u64 },
176+
dl.llvmlike_vector_align(size),
177+
)
178+
} else if repr_packed && !count.is_power_of_two() {
173179
// Non-power-of-two vectors have padding up to the next power-of-two.
174180
// If we're a packed repr, remove the padding while keeping the alignment as close
175181
// to a vector as possible.
@@ -461,6 +467,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
461467
BackendRepr::Scalar(..)
462468
| BackendRepr::ScalarPair(..)
463469
| BackendRepr::SimdVector { .. }
470+
| BackendRepr::ScalableVector { .. }
464471
| BackendRepr::Memory { .. } => repr,
465472
},
466473
};
@@ -532,7 +539,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
532539
hide_niches(a);
533540
hide_niches(b);
534541
}
535-
BackendRepr::SimdVector { element, count: _ } => hide_niches(element),
542+
BackendRepr::SimdVector { element, .. }
543+
| BackendRepr::ScalableVector { element, .. } => hide_niches(element),
536544
BackendRepr::Memory { sized: _ } => {}
537545
}
538546
st.largest_niche = None;

‎compiler/rustc_abi/src/lib.rs‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1671,6 +1671,10 @@ impl AddressSpace {
16711671
pub enum BackendRepr {
16721672
Scalar(Scalar),
16731673
ScalarPair(Scalar, Scalar),
1674+
ScalableVector {
1675+
element: Scalar,
1676+
count: u64,
1677+
},
16741678
SimdVector {
16751679
element: Scalar,
16761680
count: u64,
@@ -1689,6 +1693,9 @@ impl BackendRepr {
16891693
match *self {
16901694
BackendRepr::Scalar(_)
16911695
| BackendRepr::ScalarPair(..)
1696+
// FIXME(repr_scalable): Scalable vectors are forced to be `Sized` while the
1697+
// `sized_hierarchy` feature is not yet fully implemented
1698+
| BackendRepr::ScalableVector { .. }
16921699
| BackendRepr::SimdVector { .. } => false,
16931700
BackendRepr::Memory { sized } => !sized,
16941701
}
@@ -1729,7 +1736,9 @@ impl BackendRepr {
17291736
BackendRepr::Scalar(s) => Some(s.align(cx).abi),
17301737
BackendRepr::ScalarPair(s1, s2) => Some(s1.align(cx).max(s2.align(cx)).abi),
17311738
// The align of a Vector can vary in surprising ways
1732-
BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
1739+
BackendRepr::SimdVector { .. }
1740+
| BackendRepr::Memory { .. }
1741+
| BackendRepr::ScalableVector { .. } => None,
17331742
}
17341743
}
17351744

@@ -1751,7 +1760,9 @@ impl BackendRepr {
17511760
Some(size)
17521761
}
17531762
// The size of a Vector can vary in surprising ways
1754-
BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
1763+
BackendRepr::SimdVector { .. }
1764+
| BackendRepr::Memory { .. }
1765+
| BackendRepr::ScalableVector { .. } => None,
17551766
}
17561767
}
17571768

@@ -1766,6 +1777,9 @@ impl BackendRepr {
17661777
BackendRepr::SimdVector { element: element.to_union(), count }
17671778
}
17681779
BackendRepr::Memory { .. } => BackendRepr::Memory { sized: true },
1780+
BackendRepr::ScalableVector { element, count } => {
1781+
BackendRepr::ScalableVector { element: element.to_union(), count }
1782+
}
17691783
}
17701784
}
17711785

@@ -2006,7 +2020,9 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
20062020
/// Returns `true` if this is an aggregate type (including a ScalarPair!)
20072021
pub fn is_aggregate(&self) -> bool {
20082022
match self.backend_repr {
2009-
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => false,
2023+
BackendRepr::Scalar(_)
2024+
| BackendRepr::SimdVector { .. }
2025+
| BackendRepr::ScalableVector { .. } => false,
20102026
BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => true,
20112027
}
20122028
}
@@ -2100,6 +2116,19 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
21002116
self.is_sized() && self.size.bytes() == 0 && self.align.abi.bytes() == 1
21012117
}
21022118

2119+
/// Returns `true` if the size of the type is only known at runtime.
2120+
pub fn is_runtime_sized(&self) -> bool {
2121+
matches!(self.backend_repr, BackendRepr::ScalableVector { .. })
2122+
}
2123+
2124+
/// Returns the elements count of a scalable vector.
2125+
pub fn scalable_vector_element_count(&self) -> Option<u64> {
2126+
match self.backend_repr {
2127+
BackendRepr::ScalableVector { count, .. } => Some(count),
2128+
_ => None,
2129+
}
2130+
}
2131+
21032132
/// Returns `true` if the type is a ZST and not unsized.
21042133
///
21052134
/// Note that this does *not* imply that the type is irrelevant for layout! It can still have
@@ -2108,6 +2137,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
21082137
match self.backend_repr {
21092138
BackendRepr::Scalar(_)
21102139
| BackendRepr::ScalarPair(..)
2140+
| BackendRepr::ScalableVector { .. }
21112141
| BackendRepr::SimdVector { .. } => false,
21122142
BackendRepr::Memory { sized } => sized && self.size.bytes() == 0,
21132143
}

‎compiler/rustc_codegen_gcc/src/builder.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -926,6 +926,10 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
926926
.get_address(self.location)
927927
}
928928

929+
fn scalable_alloca(&mut self, _elt: u64, _align: Align, _element_ty: Ty<'_>) -> RValue<'gcc> {
930+
todo!()
931+
}
932+
929933
fn load(&mut self, pointee_ty: Type<'gcc>, ptr: RValue<'gcc>, align: Align) -> RValue<'gcc> {
930934
let block = self.llbb();
931935
let function = block.get_function();

‎compiler/rustc_codegen_gcc/src/intrinsic/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -536,7 +536,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
536536
let layout = self.layout_of(tp_ty).layout;
537537
let _use_integer_compare = match layout.backend_repr() {
538538
Scalar(_) | ScalarPair(_, _) => true,
539-
SimdVector { .. } => false,
539+
SimdVector { .. } | ScalableVector { .. } => false,
540540
Memory { .. } => {
541541
// For rusty ABIs, small aggregates are actually passed
542542
// as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),

‎compiler/rustc_codegen_gcc/src/type_of.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ fn uncached_gcc_type<'gcc, 'tcx>(
8585
);
8686
}
8787
BackendRepr::Memory { .. } => {}
88+
BackendRepr::ScalableVector { .. } => todo!(),
8889
}
8990

9091
let name = match *layout.ty.kind() {
@@ -178,7 +179,9 @@ pub trait LayoutGccExt<'tcx> {
178179
impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
179180
fn is_gcc_immediate(&self) -> bool {
180181
match self.backend_repr {
181-
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => true,
182+
BackendRepr::Scalar(_)
183+
| BackendRepr::SimdVector { .. }
184+
| BackendRepr::ScalableVector { .. } => true,
182185
BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => false,
183186
}
184187
}
@@ -188,6 +191,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
188191
BackendRepr::ScalarPair(..) => true,
189192
BackendRepr::Scalar(_)
190193
| BackendRepr::SimdVector { .. }
194+
| BackendRepr::ScalableVector { .. }
191195
| BackendRepr::Memory { .. } => false,
192196
}
193197
}

‎compiler/rustc_codegen_llvm/src/builder.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -538,6 +538,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
538538
}
539539
}
540540

541+
fn scalable_alloca(&mut self, elt: u64, align: Align, element_ty: Ty<'_>) -> Self::Value {
542+
let mut bx = Builder::with_cx(self.cx);
543+
bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
544+
let llvm_ty = match element_ty.kind() {
545+
ty::Bool => bx.type_i1(),
546+
ty::Int(int_ty) => self.cx.type_int_from_ty(*int_ty),
547+
ty::Uint(uint_ty) => self.cx.type_uint_from_ty(*uint_ty),
548+
ty::Float(float_ty) => self.cx.type_float_from_ty(*float_ty),
549+
_ => unreachable!("scalable vectors can only contain a bool, int, uint or float"),
550+
};
551+
552+
unsafe {
553+
let ty = llvm::LLVMScalableVectorType(llvm_ty, elt.try_into().unwrap());
554+
let alloca = llvm::LLVMBuildAlloca(&bx.llbuilder, ty, UNNAMED);
555+
llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
556+
alloca
557+
}
558+
}
559+
541560
fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
542561
unsafe {
543562
let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);

‎compiler/rustc_codegen_llvm/src/intrinsic.rs‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -454,6 +454,14 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
454454
let use_integer_compare = match layout.backend_repr() {
455455
Scalar(_) | ScalarPair(_, _) => true,
456456
SimdVector { .. } => false,
457+
ScalableVector { .. } => {
458+
tcx.dcx().emit_err(InvalidMonomorphization::NonScalableType {
459+
span,
460+
name: sym::raw_eq,
461+
ty: tp_ty,
462+
});
463+
return Ok(());
464+
}
457465
Memory { .. } => {
458466
// For rusty ABIs, small aggregates are actually passed
459467
// as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),
@@ -1254,7 +1262,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
12541262
bx.bitcast(val, llret_ty)
12551263
}
12561264
OperandValue::ZeroSized => bx.const_undef(llret_ty),
1257-
OperandValue::Pair(_, _) => todo!(),
1265+
OperandValue::Pair(_, _) => {
1266+
return_error!(InvalidMonomorphization::NonScalableType { span, name, ty: ret_ty })
1267+
}
12581268
});
12591269
}
12601270

@@ -1453,11 +1463,27 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
14531463
m_len == v_len,
14541464
InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
14551465
);
1456-
let in_elem_bitwidth = require_int_or_uint_ty!(
1457-
m_elem_ty.kind(),
1458-
InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
1459-
);
1460-
let m_i1s = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len);
1466+
1467+
let m_i1s = if args[1].layout.ty.is_scalable_simd() {
1468+
match m_elem_ty.kind() {
1469+
ty::Bool => {}
1470+
_ => return_error!(InvalidMonomorphization::MaskWrongElementType {
1471+
span,
1472+
name,
1473+
ty: m_elem_ty
1474+
}),
1475+
};
1476+
let i1 = bx.type_i1();
1477+
let i1xn = bx.type_scalable_vector(i1, m_len as u64);
1478+
bx.trunc(args[0].immediate(), i1xn)
1479+
} else {
1480+
let in_elem_bitwidth = require_int_or_uint_ty!(
1481+
m_elem_ty.kind(),
1482+
InvalidMonomorphization::MaskWrongElementType { span, name, ty: m_elem_ty }
1483+
);
1484+
vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len)
1485+
};
1486+
14611487
return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
14621488
}
14631489

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1072,6 +1072,7 @@ unsafe extern "C" {
10721072
// Operations on array, pointer, and vector types (sequence types)
10731073
pub(crate) fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type;
10741074
pub(crate) fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type;
1075+
pub(crate) fn LLVMScalableVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type;
10751076

10761077
pub(crate) fn LLVMGetElementType(Ty: &Type) -> &Type;
10771078
pub(crate) fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint;

‎compiler/rustc_codegen_llvm/src/type_.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
6868
unsafe { llvm::LLVMVectorType(ty, len as c_uint) }
6969
}
7070

71+
pub(crate) fn type_scalable_vector(&self, ty: &'ll Type, count: u64) -> &'ll Type {
72+
unsafe { llvm::LLVMScalableVectorType(ty, count as c_uint) }
73+
}
74+
7175
pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> {
7276
unsafe {
7377
let n_args = llvm::LLVMCountParamTypes(ty) as usize;

0 commit comments

Comments
 (0)