Skip to content

Commit e062c37

Browse files
antoyoantoyo
authored
Merge pull request rust-lang#172 from rust-lang/feature/more-simd
Implement more SIMD intrinsics
2 parents 852735d + 4a97440 commit e062c37

File tree

9 files changed

+580
-35
lines changed

9 files changed

+580
-35
lines changed

‎src/asm.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -595,7 +595,7 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
595595
InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
596596
| InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x",
597597
InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v",
598-
InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => unimplemented!(),
598+
InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "Yk",
599599
InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => unimplemented!(),
600600
InlineAsmRegClass::X86(
601601
X86InlineAsmRegClass::x87_reg | X86InlineAsmRegClass::mmx_reg,

‎src/builder.rs‎

Lines changed: 108 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use gccjit::{
77
BinaryOp,
88
Block,
99
ComparisonOp,
10+
Context,
1011
Function,
1112
LValue,
1213
RValue,
@@ -47,6 +48,7 @@ use rustc_target::spec::{HasTargetSpec, Target};
4748

4849
use crate::common::{SignType, TypeReflection, type_is_pointer};
4950
use crate::context::CodegenCx;
51+
use crate::intrinsic::llvm;
5052
use crate::type_of::LayoutGccExt;
5153

5254
// TODO(antoyo)
@@ -216,11 +218,17 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
216218
return Cow::Borrowed(args);
217219
}
218220

221+
let func_name = format!("{:?}", func_ptr);
222+
219223
let casted_args: Vec<_> = param_types
220224
.into_iter()
221225
.zip(args.iter())
222226
.enumerate()
223227
.map(|(index, (expected_ty, &actual_val))| {
228+
if llvm::ignore_arg_cast(&func_name, index, args.len()) {
229+
return actual_val;
230+
}
231+
224232
let actual_ty = actual_val.get_type();
225233
if expected_ty != actual_ty {
226234
if !actual_ty.is_vector() && !expected_ty.is_vector() && actual_ty.is_integral() && expected_ty.is_integral() && actual_ty.get_size() != expected_ty.get_size() {
@@ -297,6 +305,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
297305
if return_type != void_type {
298306
unsafe { RETURN_VALUE_COUNT += 1 };
299307
let result = current_func.new_local(None, return_type, &format!("ptrReturnValue{}", unsafe { RETURN_VALUE_COUNT }));
308+
let func_name = format!("{:?}", func_ptr);
309+
let args = llvm::adjust_intrinsic_arguments(&self, gcc_func, args, &func_name);
300310
self.block.add_assignment(None, result, self.cx.context.new_call_through_ptr(None, func_ptr, &args));
301311
result.to_rvalue()
302312
}
@@ -1316,6 +1326,11 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
13161326
element_type
13171327
}
13181328
else {
1329+
#[cfg(feature="master")]
1330+
{
1331+
self.cx.type_ix(element_type.get_size() as u64 * 8)
1332+
}
1333+
#[cfg(not(feature="master"))]
13191334
self.int_type
13201335
};
13211336
for i in 0..mask_num_units {
@@ -1343,7 +1358,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
13431358
}
13441359
// TODO(antoyo): switch to using new_vector_access.
13451360
let array = self.context.new_bitcast(None, v2, array_type);
1346-
for i in 0..vec_num_units {
1361+
for i in 0..(mask_num_units - vec_num_units) {
13471362
elements.push(self.context.new_array_access(None, array, self.context.new_rvalue_from_int(self.int_type, i as i32)).to_rvalue());
13481363
}
13491364
let v1 = self.context.new_rvalue_from_vector(None, result_type, &elements);
@@ -1380,6 +1395,98 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
13801395
pub fn shuffle_vector(&mut self, _v1: RValue<'gcc>, _v2: RValue<'gcc>, _mask: RValue<'gcc>) -> RValue<'gcc> {
13811396
unimplemented!();
13821397
}
1398+
1399+
#[cfg(feature="master")]
1400+
pub fn vector_reduce<F>(&mut self, src: RValue<'gcc>, op: F) -> RValue<'gcc>
1401+
where F: Fn(RValue<'gcc>, RValue<'gcc>, &'gcc Context<'gcc>) -> RValue<'gcc>
1402+
{
1403+
let vector_type = src.get_type().unqualified().dyncast_vector().expect("vector type");
1404+
let element_count = vector_type.get_num_units();
1405+
let mut vector_elements = vec![];
1406+
for i in 0..element_count {
1407+
vector_elements.push(i);
1408+
}
1409+
let mask_type = self.context.new_vector_type(self.int_type, element_count as u64);
1410+
let mut shift = 1;
1411+
let mut res = src;
1412+
while shift < element_count {
1413+
let vector_elements: Vec<_> =
1414+
vector_elements.iter()
1415+
.map(|i| self.context.new_rvalue_from_int(self.int_type, ((i + shift) % element_count) as i32))
1416+
.collect();
1417+
let mask = self.context.new_rvalue_from_vector(None, mask_type, &vector_elements);
1418+
let shifted = self.context.new_rvalue_vector_perm(None, res, res, mask);
1419+
shift *= 2;
1420+
res = op(res, shifted, &self.context);
1421+
}
1422+
self.context.new_vector_access(None, res, self.context.new_rvalue_zero(self.int_type))
1423+
.to_rvalue()
1424+
}
1425+
1426+
#[cfg(not(feature="master"))]
1427+
pub fn vector_reduce<F>(&mut self, src: RValue<'gcc>, op: F) -> RValue<'gcc>
1428+
where F: Fn(RValue<'gcc>, RValue<'gcc>, &'gcc Context<'gcc>) -> RValue<'gcc>
1429+
{
1430+
unimplemented!();
1431+
}
1432+
1433+
pub fn vector_reduce_op(&mut self, src: RValue<'gcc>, op: BinaryOp) -> RValue<'gcc> {
1434+
self.vector_reduce(src, |a, b, context| context.new_binary_op(None, op, a.get_type(), a, b))
1435+
}
1436+
1437+
pub fn vector_reduce_fadd_fast(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
1438+
unimplemented!();
1439+
}
1440+
1441+
pub fn vector_reduce_fmul_fast(&mut self, _acc: RValue<'gcc>, _src: RValue<'gcc>) -> RValue<'gcc> {
1442+
unimplemented!();
1443+
}
1444+
1445+
// Inspired by Hacker's Delight min implementation.
1446+
pub fn vector_reduce_min(&mut self, src: RValue<'gcc>) -> RValue<'gcc> {
1447+
self.vector_reduce(src, |a, b, context| {
1448+
let differences_or_zeros = difference_or_zero(a, b, context);
1449+
context.new_binary_op(None, BinaryOp::Minus, a.get_type(), a, differences_or_zeros)
1450+
})
1451+
}
1452+
1453+
// Inspired by Hacker's Delight max implementation.
1454+
pub fn vector_reduce_max(&mut self, src: RValue<'gcc>) -> RValue<'gcc> {
1455+
self.vector_reduce(src, |a, b, context| {
1456+
let differences_or_zeros = difference_or_zero(a, b, context);
1457+
context.new_binary_op(None, BinaryOp::Plus, b.get_type(), b, differences_or_zeros)
1458+
})
1459+
}
1460+
1461+
pub fn vector_select(&mut self, cond: RValue<'gcc>, then_val: RValue<'gcc>, else_val: RValue<'gcc>) -> RValue<'gcc> {
1462+
// cond is a vector of integers, not of bools.
1463+
let cond_type = cond.get_type();
1464+
let vector_type = cond_type.unqualified().dyncast_vector().expect("vector type");
1465+
let num_units = vector_type.get_num_units();
1466+
let element_type = vector_type.get_element_type();
1467+
let zeros = vec![self.context.new_rvalue_zero(element_type); num_units];
1468+
let zeros = self.context.new_rvalue_from_vector(None, cond_type, &zeros);
1469+
1470+
let masks = self.context.new_comparison(None, ComparisonOp::NotEquals, cond, zeros);
1471+
let then_vals = masks & then_val;
1472+
1473+
let ones = vec![self.context.new_rvalue_one(element_type); num_units];
1474+
let ones = self.context.new_rvalue_from_vector(None, cond_type, &ones);
1475+
let inverted_masks = masks + ones;
1476+
// NOTE: sometimes, the type of else_val can be different than the type of then_val in
1477+
// libgccjit (vector of int vs vector of int32_t), but they should be the same for the AND
1478+
// operation to work.
1479+
let else_val = self.context.new_bitcast(None, else_val, then_val.get_type());
1480+
let else_vals = inverted_masks & else_val;
1481+
1482+
then_vals | else_vals
1483+
}
1484+
}
1485+
1486+
fn difference_or_zero<'gcc>(a: RValue<'gcc>, b: RValue<'gcc>, context: &'gcc Context<'gcc>) -> RValue<'gcc> {
1487+
let difference = a - b;
1488+
let masks = context.new_comparison(None, ComparisonOp::GreaterThanEquals, b, a);
1489+
difference & masks
13831490
}
13841491

13851492
impl<'a, 'gcc, 'tcx> StaticBuilderMethods for Builder<'a, 'gcc, 'tcx> {

‎src/common.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,8 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
117117
unimplemented!();
118118
}
119119

120-
fn const_real(&self, _t: Type<'gcc>, _val: f64) -> RValue<'gcc> {
121-
unimplemented!();
120+
fn const_real(&self, typ: Type<'gcc>, val: f64) -> RValue<'gcc> {
121+
self.context.new_rvalue_from_double(typ, val)
122122
}
123123

124124
fn const_str(&self, s: Symbol) -> (RValue<'gcc>, RValue<'gcc>) {

‎src/consts.rs‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
2727
}
2828
// NOTE: since bitcast makes a value non-constant, don't bitcast if not necessary as some
2929
// SIMD builtins require a constant value.
30-
if value.get_type() != typ {
31-
self.context.new_bitcast(None, value, typ)
32-
}
33-
else {
34-
value
35-
}
30+
self.bitcast_if_needed(value, typ)
3631
}
3732
}
3833

@@ -86,13 +81,7 @@ impl<'gcc, 'tcx> StaticMethods for CodegenCx<'gcc, 'tcx> {
8681

8782
// TODO(antoyo): set alignment.
8883

89-
let value =
90-
if value.get_type() != gcc_type {
91-
self.context.new_bitcast(None, value, gcc_type)
92-
}
93-
else {
94-
value
95-
};
84+
let value = self.bitcast_if_needed(value, gcc_type);
9685
global.global_set_initializer_rvalue(value);
9786

9887
// As an optimization, all shared statics which do not have interior

‎src/context.rs‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ pub struct CodegenCx<'gcc, 'tcx> {
3535
pub normal_function_addresses: RefCell<FxHashSet<RValue<'gcc>>>,
3636

3737
pub functions: RefCell<FxHashMap<String, Function<'gcc>>>,
38+
pub intrinsics: RefCell<FxHashMap<String, Function<'gcc>>>,
3839

3940
pub tls_model: gccjit::TlsModel,
4041

@@ -184,6 +185,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
184185
current_func: RefCell::new(None),
185186
normal_function_addresses: Default::default(),
186187
functions: RefCell::new(functions),
188+
intrinsics: RefCell::new(FxHashMap::default()),
187189

188190
tls_model,
189191

@@ -279,6 +281,15 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
279281
pub fn sess(&self) -> &Session {
280282
&self.tcx.sess
281283
}
284+
285+
pub fn bitcast_if_needed(&self, value: RValue<'gcc>, expected_type: Type<'gcc>) -> RValue<'gcc> {
286+
if value.get_type() != expected_type {
287+
self.context.new_bitcast(None, value, expected_type)
288+
}
289+
else {
290+
value
291+
}
292+
}
282293
}
283294

284295
impl<'gcc, 'tcx> BackendTypes for CodegenCx<'gcc, 'tcx> {
@@ -306,8 +317,16 @@ impl<'gcc, 'tcx> MiscMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
306317
}
307318

308319
fn get_fn_addr(&self, instance: Instance<'tcx>) -> RValue<'gcc> {
309-
let func = get_fn(self, instance);
310-
let func = self.rvalue_as_function(func);
320+
let func_name = self.tcx.symbol_name(instance).name;
321+
322+
let func =
323+
if self.intrinsics.borrow().contains_key(func_name) {
324+
self.intrinsics.borrow()[func_name].clone()
325+
}
326+
else {
327+
let func = get_fn(self, instance);
328+
self.rvalue_as_function(func)
329+
};
311330
let ptr = func.get_address(None);
312331

313332
// TODO(antoyo): don't do this twice: i.e. in declare_fn and here.

‎src/declare.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use crate::intrinsic::llvm;
1111
impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
1212
pub fn get_or_insert_global(&self, name: &str, ty: Type<'gcc>, is_tls: bool, link_section: Option<Symbol>) -> LValue<'gcc> {
1313
if self.globals.borrow().contains_key(name) {
14-
let typ = self.globals.borrow().get(name).expect("global").get_type();
14+
let typ = self.globals.borrow()[name].get_type();
1515
let global = self.context.new_global(None, GlobalKind::Imported, typ, name);
1616
if is_tls {
1717
global.set_tls_model(self.tls_model);
@@ -103,11 +103,13 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
103103
/// update the declaration and return existing Value instead.
104104
fn declare_raw_fn<'gcc>(cx: &CodegenCx<'gcc, '_>, name: &str, _callconv: () /*llvm::CallConv*/, return_type: Type<'gcc>, param_types: &[Type<'gcc>], variadic: bool) -> Function<'gcc> {
105105
if name.starts_with("llvm.") {
106-
return llvm::intrinsic(name, cx);
106+
let intrinsic = llvm::intrinsic(name, cx);
107+
cx.intrinsics.borrow_mut().insert(name.to_string(), intrinsic);
108+
return intrinsic;
107109
}
108110
let func =
109111
if cx.functions.borrow().contains_key(name) {
110-
*cx.functions.borrow().get(name).expect("function")
112+
cx.functions.borrow()[name]
111113
}
112114
else {
113115
let params: Vec<_> = param_types.into_iter().enumerate()

0 commit comments

Comments
 (0)