Skip to content

Commit dace4f8

Browse files
davidtwcoJamieCunliffe
davidtwco
and
JamieCunliffe
committed
codegen: implement repr(scalable)
Introduces `BackendRepr::ScalableVector` corresponding to scalable vector types annotated with `repr(scalable)` which lowers to a scalable vector type in LLVM. Co-authored-by: Jamie Cunliffe <Jamie.Cunliffe@arm.com>
1 parent 12342ef commit dace4f8

File tree

41 files changed

+553
-75
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+553
-75
lines changed

‎compiler/rustc_abi/src/callconv.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
8282
}))
8383
}
8484

85+
BackendRepr::ScalableVector { .. } => Err(Heterogeneous),
86+
8587
BackendRepr::ScalarPair(..) | BackendRepr::Memory { sized: true } => {
8688
// Helper for computing `homogeneous_aggregate`, allowing a custom
8789
// starting offset (used below for handling variants).

‎compiler/rustc_abi/src/layout.rs‎

Lines changed: 87 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use tracing::{debug, trace};
1111
use crate::{
1212
AbiAlign, Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer,
1313
LayoutData, Niche, NonZeroUsize, Primitive, ReprOptions, Scalar, Size, StructKind, TagEncoding,
14-
Variants, WrappingRange,
14+
TargetDataLayout, Variants, WrappingRange,
1515
};
1616

1717
mod coroutine;
@@ -143,58 +143,32 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
143143
})
144144
}
145145

146-
pub fn simd_type<
146+
pub fn scalable_vector_type<FieldIdx, VariantIdx, F>(
147+
&self,
148+
element: F,
149+
count: u64,
150+
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
151+
where
147152
FieldIdx: Idx,
148153
VariantIdx: Idx,
149154
F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
150-
>(
155+
{
156+
vector_type_layout(VectorKind::Scalable, self.cx.data_layout(), element, count)
157+
}
158+
159+
pub fn simd_type<FieldIdx, VariantIdx, F>(
151160
&self,
152161
element: F,
153162
count: u64,
154163
repr_packed: bool,
155-
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
156-
let elt = element.as_ref();
157-
if count == 0 {
158-
return Err(LayoutCalculatorError::ZeroLengthSimdType);
159-
} else if count > crate::MAX_SIMD_LANES {
160-
return Err(LayoutCalculatorError::OversizedSimdType {
161-
max_lanes: crate::MAX_SIMD_LANES,
162-
});
163-
}
164-
165-
let BackendRepr::Scalar(e_repr) = elt.backend_repr else {
166-
return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));
167-
};
168-
169-
// Compute the size and alignment of the vector
170-
let dl = self.cx.data_layout();
171-
let size =
172-
elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
173-
let (repr, align) = if repr_packed && !count.is_power_of_two() {
174-
// Non-power-of-two vectors have padding up to the next power-of-two.
175-
// If we're a packed repr, remove the padding while keeping the alignment as close
176-
// to a vector as possible.
177-
(BackendRepr::Memory { sized: true }, AbiAlign { abi: Align::max_aligned_factor(size) })
178-
} else {
179-
(BackendRepr::SimdVector { element: e_repr, count }, dl.llvmlike_vector_align(size))
180-
};
181-
let size = size.align_to(align.abi);
182-
183-
Ok(LayoutData {
184-
variants: Variants::Single { index: VariantIdx::new(0) },
185-
fields: FieldsShape::Arbitrary {
186-
offsets: [Size::ZERO].into(),
187-
memory_index: [0].into(),
188-
},
189-
backend_repr: repr,
190-
largest_niche: elt.largest_niche,
191-
uninhabited: false,
192-
size,
193-
align,
194-
max_repr_align: None,
195-
unadjusted_abi_align: elt.align.abi,
196-
randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)),
197-
})
164+
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
165+
where
166+
FieldIdx: Idx,
167+
VariantIdx: Idx,
168+
F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
169+
{
170+
let kind = if repr_packed { VectorKind::PackedFixed } else { VectorKind::Fixed };
171+
vector_type_layout(kind, self.cx.data_layout(), element, count)
198172
}
199173

200174
/// Compute the layout for a coroutine.
@@ -455,6 +429,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
455429
BackendRepr::Scalar(..)
456430
| BackendRepr::ScalarPair(..)
457431
| BackendRepr::SimdVector { .. }
432+
| BackendRepr::ScalableVector { .. }
458433
| BackendRepr::Memory { .. } => repr,
459434
},
460435
};
@@ -526,7 +501,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
526501
hide_niches(a);
527502
hide_niches(b);
528503
}
529-
BackendRepr::SimdVector { element, count: _ } => hide_niches(element),
504+
BackendRepr::SimdVector { element, .. }
505+
| BackendRepr::ScalableVector { element, .. } => hide_niches(element),
530506
BackendRepr::Memory { sized: _ } => {}
531507
}
532508
st.largest_niche = None;
@@ -1525,3 +1501,67 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
15251501
s
15261502
}
15271503
}
1504+
1505+
enum VectorKind {
1506+
/// `#[rustc_scalable_vector]`
1507+
Scalable,
1508+
/// `#[repr(simd, packed)]`
1509+
PackedFixed,
1510+
/// `#[repr(simd)]`
1511+
Fixed,
1512+
}
1513+
1514+
fn vector_type_layout<FieldIdx, VariantIdx, F>(
1515+
kind: VectorKind,
1516+
dl: &TargetDataLayout,
1517+
element: F,
1518+
count: u64,
1519+
) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
1520+
where
1521+
FieldIdx: Idx,
1522+
VariantIdx: Idx,
1523+
F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
1524+
{
1525+
let elt = element.as_ref();
1526+
if count == 0 {
1527+
return Err(LayoutCalculatorError::ZeroLengthSimdType);
1528+
} else if count > crate::MAX_SIMD_LANES {
1529+
return Err(LayoutCalculatorError::OversizedSimdType { max_lanes: crate::MAX_SIMD_LANES });
1530+
}
1531+
1532+
let BackendRepr::Scalar(element) = elt.backend_repr else {
1533+
return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));
1534+
};
1535+
1536+
// Compute the size and alignment of the vector
1537+
let size =
1538+
elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
1539+
let (repr, align) = match kind {
1540+
VectorKind::Scalable => {
1541+
(BackendRepr::ScalableVector { element, count }, dl.llvmlike_vector_align(size))
1542+
}
1543+
// Non-power-of-two vectors have padding up to the next power-of-two.
1544+
// If we're a packed repr, remove the padding while keeping the alignment as close
1545+
// to a vector as possible.
1546+
VectorKind::PackedFixed if !count.is_power_of_two() => {
1547+
(BackendRepr::Memory { sized: true }, AbiAlign { abi: Align::max_aligned_factor(size) })
1548+
}
1549+
VectorKind::PackedFixed | VectorKind::Fixed => {
1550+
(BackendRepr::SimdVector { element, count }, dl.llvmlike_vector_align(size))
1551+
}
1552+
};
1553+
let size = size.align_to(align.abi);
1554+
1555+
Ok(LayoutData {
1556+
variants: Variants::Single { index: VariantIdx::new(0) },
1557+
fields: FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), memory_index: [0].into() },
1558+
backend_repr: repr,
1559+
largest_niche: elt.largest_niche,
1560+
uninhabited: false,
1561+
size,
1562+
align,
1563+
max_repr_align: None,
1564+
unadjusted_abi_align: elt.align.abi,
1565+
randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)),
1566+
})
1567+
}

‎compiler/rustc_abi/src/lib.rs‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1753,6 +1753,10 @@ impl AddressSpace {
17531753
pub enum BackendRepr {
17541754
Scalar(Scalar),
17551755
ScalarPair(Scalar, Scalar),
1756+
ScalableVector {
1757+
element: Scalar,
1758+
count: u64,
1759+
},
17561760
SimdVector {
17571761
element: Scalar,
17581762
count: u64,
@@ -1771,6 +1775,9 @@ impl BackendRepr {
17711775
match *self {
17721776
BackendRepr::Scalar(_)
17731777
| BackendRepr::ScalarPair(..)
1778+
// FIXME(repr_scalable): Scalable vectors are `Sized` while the `sized_hierarchy`
1779+
// feature is not yet fully implemented
1780+
| BackendRepr::ScalableVector { .. }
17741781
| BackendRepr::SimdVector { .. } => false,
17751782
BackendRepr::Memory { sized } => !sized,
17761783
}
@@ -1811,7 +1818,9 @@ impl BackendRepr {
18111818
BackendRepr::Scalar(s) => Some(s.align(cx).abi),
18121819
BackendRepr::ScalarPair(s1, s2) => Some(s1.align(cx).max(s2.align(cx)).abi),
18131820
// The align of a Vector can vary in surprising ways
1814-
BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
1821+
BackendRepr::SimdVector { .. }
1822+
| BackendRepr::Memory { .. }
1823+
| BackendRepr::ScalableVector { .. } => None,
18151824
}
18161825
}
18171826

@@ -1833,7 +1842,9 @@ impl BackendRepr {
18331842
Some(size)
18341843
}
18351844
// The size of a Vector can vary in surprising ways
1836-
BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
1845+
BackendRepr::SimdVector { .. }
1846+
| BackendRepr::Memory { .. }
1847+
| BackendRepr::ScalableVector { .. } => None,
18371848
}
18381849
}
18391850

@@ -1848,6 +1859,9 @@ impl BackendRepr {
18481859
BackendRepr::SimdVector { element: element.to_union(), count }
18491860
}
18501861
BackendRepr::Memory { .. } => BackendRepr::Memory { sized: true },
1862+
BackendRepr::ScalableVector { element, count } => {
1863+
BackendRepr::ScalableVector { element: element.to_union(), count }
1864+
}
18511865
}
18521866
}
18531867

@@ -2088,7 +2102,9 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
20882102
/// Returns `true` if this is an aggregate type (including a ScalarPair!)
20892103
pub fn is_aggregate(&self) -> bool {
20902104
match self.backend_repr {
2091-
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => false,
2105+
BackendRepr::Scalar(_)
2106+
| BackendRepr::SimdVector { .. }
2107+
| BackendRepr::ScalableVector { .. } => false,
20922108
BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => true,
20932109
}
20942110
}
@@ -2182,6 +2198,19 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
21822198
self.is_sized() && self.size.bytes() == 0 && self.align.abi.bytes() == 1
21832199
}
21842200

2201+
/// Returns `true` if the size of the type is only known at runtime.
2202+
pub fn is_runtime_sized(&self) -> bool {
2203+
matches!(self.backend_repr, BackendRepr::ScalableVector { .. })
2204+
}
2205+
2206+
/// Returns the elements count of a scalable vector.
2207+
pub fn scalable_vector_element_count(&self) -> Option<u64> {
2208+
match self.backend_repr {
2209+
BackendRepr::ScalableVector { count, .. } => Some(count),
2210+
_ => None,
2211+
}
2212+
}
2213+
21852214
/// Returns `true` if the type is a ZST and not unsized.
21862215
///
21872216
/// Note that this does *not* imply that the type is irrelevant for layout! It can still have
@@ -2190,6 +2219,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
21902219
match self.backend_repr {
21912220
BackendRepr::Scalar(_)
21922221
| BackendRepr::ScalarPair(..)
2222+
| BackendRepr::ScalableVector { .. }
21932223
| BackendRepr::SimdVector { .. } => false,
21942224
BackendRepr::Memory { sized } => sized && self.size.bytes() == 0,
21952225
}

‎compiler/rustc_codegen_gcc/src/builder.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -933,6 +933,10 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
933933
.get_address(self.location)
934934
}
935935

936+
fn scalable_alloca(&mut self, _elt: u64, _align: Align, _element_ty: Ty<'_>) -> RValue<'gcc> {
937+
todo!()
938+
}
939+
936940
fn load(&mut self, pointee_ty: Type<'gcc>, ptr: RValue<'gcc>, align: Align) -> RValue<'gcc> {
937941
let block = self.llbb();
938942
let function = block.get_function();

‎compiler/rustc_codegen_gcc/src/intrinsic/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -536,7 +536,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
536536
let layout = self.layout_of(tp_ty).layout;
537537
let _use_integer_compare = match layout.backend_repr() {
538538
Scalar(_) | ScalarPair(_, _) => true,
539-
SimdVector { .. } => false,
539+
SimdVector { .. } | ScalableVector { .. } => false,
540540
Memory { .. } => {
541541
// For rusty ABIs, small aggregates are actually passed
542542
// as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),

‎compiler/rustc_codegen_gcc/src/type_of.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ fn uncached_gcc_type<'gcc, 'tcx>(
8585
);
8686
}
8787
BackendRepr::Memory { .. } => {}
88+
BackendRepr::ScalableVector { .. } => todo!(),
8889
}
8990

9091
let name = match *layout.ty.kind() {
@@ -178,7 +179,9 @@ pub trait LayoutGccExt<'tcx> {
178179
impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
179180
fn is_gcc_immediate(&self) -> bool {
180181
match self.backend_repr {
181-
BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => true,
182+
BackendRepr::Scalar(_)
183+
| BackendRepr::SimdVector { .. }
184+
| BackendRepr::ScalableVector { .. } => true,
182185
BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => false,
183186
}
184187
}
@@ -188,6 +191,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
188191
BackendRepr::ScalarPair(..) => true,
189192
BackendRepr::Scalar(_)
190193
| BackendRepr::SimdVector { .. }
194+
| BackendRepr::ScalableVector { .. }
191195
| BackendRepr::Memory { .. } => false,
192196
}
193197
}

‎compiler/rustc_codegen_llvm/src/builder.rs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -617,6 +617,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
617617
}
618618
}
619619

620+
fn scalable_alloca(&mut self, elt: u64, align: Align, element_ty: Ty<'_>) -> Self::Value {
621+
let mut bx = Builder::with_cx(self.cx);
622+
bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
623+
let llvm_ty = match element_ty.kind() {
624+
ty::Bool => bx.type_i1(),
625+
ty::Int(int_ty) => self.cx.type_int_from_ty(*int_ty),
626+
ty::Uint(uint_ty) => self.cx.type_uint_from_ty(*uint_ty),
627+
ty::Float(float_ty) => self.cx.type_float_from_ty(*float_ty),
628+
_ => unreachable!("scalable vectors can only contain a bool, int, uint or float"),
629+
};
630+
631+
unsafe {
632+
let ty = llvm::LLVMScalableVectorType(llvm_ty, elt.try_into().unwrap());
633+
let alloca = llvm::LLVMBuildAlloca(&bx.llbuilder, ty, UNNAMED);
634+
llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
635+
alloca
636+
}
637+
}
638+
620639
fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
621640
unsafe {
622641
let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);

0 commit comments

Comments
 (0)