Skip to content

Commit a460e23

Browse files
sayantnsayantn
committed
Codegen **non-overloaded** LLVM intrinsics using their name
1 parent cdb45c8 commit a460e23

File tree

12 files changed

+214
-46
lines changed

12 files changed

+214
-46
lines changed

‎compiler/rustc_codegen_gcc/src/type_of.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use std::fmt::Write;
22

3-
use gccjit::{Struct, Type};
3+
use gccjit::{RValue, Struct, Type};
44
use rustc_abi as abi;
55
use rustc_abi::Primitive::*;
66
use rustc_abi::{
@@ -373,7 +373,11 @@ impl<'gcc, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
373373
unimplemented!();
374374
}
375375

376-
fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> {
376+
fn fn_decl_backend_type(
377+
&self,
378+
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
379+
_fn_ptr: RValue<'gcc>,
380+
) -> Type<'gcc> {
377381
// FIXME(antoyo): Should we do something with `FnAbiGcc::fn_attributes`?
378382
let FnAbiGcc { return_type, arguments_type, is_c_variadic, .. } = fn_abi.gcc_type(self);
379383
self.context.new_function_pointer_type(None, return_type, &arguments_type, is_c_variadic)

‎compiler/rustc_codegen_llvm/src/abi.rs‎

Lines changed: 140 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use std::borrow::Borrow;
2-
use std::cmp;
2+
use std::{cmp, iter};
33

44
use libc::c_uint;
55
use rustc_abi::{
@@ -306,8 +306,39 @@ impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
306306
}
307307
}
308308

309+
pub(crate) enum FunctionSignature<'ll> {
310+
/// The signature is obtained directly from LLVM, and **may not match the Rust signature**
311+
Intrinsic(&'ll Type),
312+
/// The name starts with `llvm.`, but can't obtain the intrinsic ID. May be invalid or upgradable
313+
MaybeInvalidIntrinsic(&'ll Type),
314+
/// Just the Rust signature
315+
Rust(&'ll Type),
316+
}
317+
318+
impl<'ll> FunctionSignature<'ll> {
319+
pub(crate) fn fn_ty(&self) -> &'ll Type {
320+
match self {
321+
FunctionSignature::Intrinsic(fn_ty)
322+
| FunctionSignature::MaybeInvalidIntrinsic(fn_ty)
323+
| FunctionSignature::Rust(fn_ty) => fn_ty,
324+
}
325+
}
326+
}
327+
309328
pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
310-
fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
329+
fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
330+
fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type>;
331+
/// When `do_verify` is set, this function performs checks for the signature of LLVM intrinsics
332+
/// and emits a fatal error if it doesn't match. These checks are important,but somewhat expensive
333+
/// So they are only used at function definitions, not at callsites
334+
fn llvm_type(
335+
&self,
336+
cx: &CodegenCx<'ll, 'tcx>,
337+
name: &[u8],
338+
do_verify: bool,
339+
) -> FunctionSignature<'ll>;
340+
/// **If this function is an LLVM intrinsic** checks if the LLVM signature provided matches with this
341+
fn verify_intrinsic_signature(&self, cx: &CodegenCx<'ll, 'tcx>, llvm_ty: &'ll Type) -> bool;
311342
fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
312343
fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv;
313344

@@ -320,30 +351,38 @@ pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
320351
);
321352

322353
/// Apply attributes to a function call.
323-
fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value);
354+
fn apply_attrs_callsite(
355+
&self,
356+
bx: &mut Builder<'_, 'll, 'tcx>,
357+
callsite: &'ll Value,
358+
llfn: &'ll Value,
359+
);
324360
}
325361

326362
impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
327-
fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
363+
fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
364+
match &self.ret.mode {
365+
PassMode::Ignore => cx.type_void(),
366+
PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
367+
PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx),
368+
PassMode::Indirect { .. } => cx.type_void(),
369+
}
370+
}
371+
372+
fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type> {
373+
let indirect_return = matches!(self.ret.mode, PassMode::Indirect { .. });
374+
328375
// Ignore "extra" args from the call site for C variadic functions.
329376
// Only the "fixed" args are part of the LLVM function signature.
330377
let args =
331378
if self.c_variadic { &self.args[..self.fixed_count as usize] } else { &self.args };
332379

333-
// This capacity calculation is approximate.
334-
let mut llargument_tys = Vec::with_capacity(
335-
self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 },
336-
);
380+
let mut llargument_tys =
381+
Vec::with_capacity(args.len() + if indirect_return { 1 } else { 0 });
337382

338-
let llreturn_ty = match &self.ret.mode {
339-
PassMode::Ignore => cx.type_void(),
340-
PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
341-
PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx),
342-
PassMode::Indirect { .. } => {
343-
llargument_tys.push(cx.type_ptr());
344-
cx.type_void()
345-
}
346-
};
383+
if indirect_return {
384+
llargument_tys.push(cx.type_ptr());
385+
}
347386

348387
for arg in args {
349388
// Note that the exact number of arguments pushed here is carefully synchronized with
@@ -390,10 +429,74 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
390429
llargument_tys.push(llarg_ty);
391430
}
392431

393-
if self.c_variadic {
394-
cx.type_variadic_func(&llargument_tys, llreturn_ty)
432+
llargument_tys
433+
}
434+
435+
fn verify_intrinsic_signature(&self, cx: &CodegenCx<'ll, 'tcx>, llvm_fn_ty: &'ll Type) -> bool {
436+
let rust_return_ty = self.llvm_return_type(cx);
437+
let rust_argument_tys = self.llvm_argument_types(cx);
438+
439+
let llvm_return_ty = cx.get_return_type(llvm_fn_ty);
440+
let llvm_argument_tys = cx.func_params_types(llvm_fn_ty);
441+
let llvm_is_variadic = cx.func_is_variadic(llvm_fn_ty);
442+
443+
if self.c_variadic != llvm_is_variadic || rust_argument_tys.len() != llvm_argument_tys.len()
444+
{
445+
return false;
446+
}
447+
448+
// todo: add bypasses for types not accessible from Rust here
449+
iter::once((rust_return_ty, llvm_return_ty))
450+
.chain(iter::zip(rust_argument_tys, llvm_argument_tys))
451+
.all(|(rust_ty, llvm_ty)| rust_ty == llvm_ty)
452+
}
453+
454+
fn llvm_type(
455+
&self,
456+
cx: &CodegenCx<'ll, 'tcx>,
457+
name: &[u8],
458+
do_verify: bool,
459+
) -> FunctionSignature<'ll> {
460+
let mut maybe_invalid = false;
461+
462+
if name.starts_with(b"llvm.") {
463+
if let Some(intrinsic) = llvm::Intrinsic::lookup(name) {
464+
if !intrinsic.is_overloaded() {
465+
// FIXME: also do this for overloaded intrinsics
466+
let llvm_fn_ty = intrinsic.get_type(cx.llcx, &[]);
467+
if do_verify {
468+
if !self.verify_intrinsic_signature(cx, llvm_fn_ty) {
469+
cx.tcx.dcx().fatal(format!(
470+
"Intrinsic signature mismatch for `{}`: expected signature `{llvm_fn_ty:?}`",
471+
str::from_utf8(name).unwrap()
472+
));
473+
}
474+
}
475+
return FunctionSignature::Intrinsic(llvm_fn_ty);
476+
}
477+
} else {
478+
// it's one of 2 cases,
479+
// - either the base name is invalid
480+
// - it has been superseded by something else, so the intrinsic was removed entirely
481+
// to check for upgrades, we need the `llfn`, so we defer it for now
482+
483+
maybe_invalid = true;
484+
}
485+
}
486+
487+
let return_ty = self.llvm_return_type(cx);
488+
let argument_tys = self.llvm_argument_types(cx);
489+
490+
let fn_ty = if self.c_variadic {
491+
cx.type_variadic_func(&argument_tys, return_ty)
395492
} else {
396-
cx.type_func(&llargument_tys, llreturn_ty)
493+
cx.type_func(&argument_tys, return_ty)
494+
};
495+
496+
if maybe_invalid {
497+
FunctionSignature::MaybeInvalidIntrinsic(fn_ty)
498+
} else {
499+
FunctionSignature::Rust(fn_ty)
397500
}
398501
}
399502

@@ -545,7 +648,23 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
545648
}
546649
}
547650

548-
fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) {
651+
fn apply_attrs_callsite(
652+
&self,
653+
bx: &mut Builder<'_, 'll, 'tcx>,
654+
callsite: &'ll Value,
655+
llfn: &'ll Value,
656+
) {
657+
// if we are using the LLVM signature, use the LLVM attributes otherwise it might be problematic
658+
let name = llvm::get_value_name(llfn);
659+
if name.starts_with(b"llvm.")
660+
&& let Some(intrinsic) = llvm::Intrinsic::lookup(name)
661+
{
662+
// FIXME: also do this for overloaded intrinsics
663+
if !intrinsic.is_overloaded() {
664+
return;
665+
}
666+
}
667+
549668
let mut func_attrs = SmallVec::<[_; 2]>::new();
550669
if self.ret.layout.is_uninhabited() {
551670
func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx));

‎compiler/rustc_codegen_llvm/src/builder.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -446,7 +446,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
446446
)
447447
};
448448
if let Some(fn_abi) = fn_abi {
449-
fn_abi.apply_attrs_callsite(self, invoke);
449+
fn_abi.apply_attrs_callsite(self, invoke, llfn);
450450
}
451451
invoke
452452
}
@@ -1430,7 +1430,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
14301430
)
14311431
};
14321432
if let Some(fn_abi) = fn_abi {
1433-
fn_abi.apply_attrs_callsite(self, call);
1433+
fn_abi.apply_attrs_callsite(self, call, llfn);
14341434
}
14351435
call
14361436
}
@@ -1779,7 +1779,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
17791779
)
17801780
};
17811781
if let Some(fn_abi) = fn_abi {
1782-
fn_abi.apply_attrs_callsite(self, callbr);
1782+
fn_abi.apply_attrs_callsite(self, callbr, llfn);
17831783
}
17841784
callbr
17851785
}

‎compiler/rustc_codegen_llvm/src/declare.rs‎

Lines changed: 29 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use rustc_target::callconv::FnAbi;
2222
use smallvec::SmallVec;
2323
use tracing::debug;
2424

25-
use crate::abi::FnAbiLlvmExt;
25+
use crate::abi::{FnAbiLlvmExt, FunctionSignature};
2626
use crate::common::AsCCharPtr;
2727
use crate::context::{CodegenCx, GenericCx, SCx, SimpleCx};
2828
use crate::llvm::AttributePlace::Function;
@@ -150,17 +150,34 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
150150
) -> &'ll Value {
151151
debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi);
152152

153-
// Function addresses in Rust are never significant, allowing functions to
154-
// be merged.
155-
let llfn = declare_raw_fn(
156-
self,
157-
name,
158-
fn_abi.llvm_cconv(self),
159-
llvm::UnnamedAddr::Global,
160-
llvm::Visibility::Default,
161-
fn_abi.llvm_type(self),
162-
);
163-
fn_abi.apply_attrs_llfn(self, llfn, instance);
153+
let signature = fn_abi.llvm_type(self, name.as_bytes(), true);
154+
let llfn;
155+
156+
if let FunctionSignature::Intrinsic(fn_ty) = signature {
157+
// intrinsics have a specified set of attributes, so we don't use the `FnAbi` set for them
158+
llfn = declare_simple_fn(
159+
self,
160+
name,
161+
fn_abi.llvm_cconv(self),
162+
llvm::UnnamedAddr::Global,
163+
llvm::Visibility::Default,
164+
fn_ty,
165+
);
166+
} else {
167+
// Function addresses in Rust are never significant, allowing functions to
168+
// be merged.
169+
llfn = declare_raw_fn(
170+
self,
171+
name,
172+
fn_abi.llvm_cconv(self),
173+
llvm::UnnamedAddr::Global,
174+
llvm::Visibility::Default,
175+
signature.fn_ty(),
176+
);
177+
fn_abi.apply_attrs_llfn(self, llfn, instance);
178+
}
179+
180+
// todo: check for upgrades, and emit error if not upgradable
164181

165182
if self.tcx.sess.is_sanitizer_cfi_enabled() {
166183
if let Some(instance) = instance {

‎compiler/rustc_codegen_llvm/src/intrinsic.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1066,7 +1066,7 @@ fn gen_fn<'a, 'll, 'tcx>(
10661066
codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
10671067
) -> (&'ll Type, &'ll Value) {
10681068
let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
1069-
let llty = fn_abi.llvm_type(cx);
1069+
let llty = fn_abi.llvm_type(cx, name.as_bytes(), true).fn_ty();
10701070
let llfn = cx.declare_fn(name, fn_abi, None);
10711071
cx.set_frame_pointer_type(llfn);
10721072
cx.apply_target_cpu_attr(llfn);

‎compiler/rustc_codegen_llvm/src/llvm/ffi.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1110,6 +1110,7 @@ unsafe extern "C" {
11101110
) -> &'a Type;
11111111
pub(crate) fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint;
11121112
pub(crate) fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type);
1113+
pub(crate) fn LLVMIsFunctionVarArg(FunctionTy: &Type) -> Bool;
11131114

11141115
// Operations on struct types
11151116
pub(crate) fn LLVMStructTypeInContext<'a>(
@@ -1252,6 +1253,13 @@ unsafe extern "C" {
12521253

12531254
// Operations about llvm intrinsics
12541255
pub(crate) fn LLVMLookupIntrinsicID(Name: *const c_char, NameLen: size_t) -> c_uint;
1256+
pub(crate) fn LLVMIntrinsicIsOverloaded(ID: NonZero<c_uint>) -> Bool;
1257+
pub(crate) fn LLVMIntrinsicGetType<'a>(
1258+
C: &'a Context,
1259+
ID: NonZero<c_uint>,
1260+
ParamTypes: *const &'a Type,
1261+
ParamCount: size_t,
1262+
) -> &'a Type;
12551263
pub(crate) fn LLVMGetIntrinsicDeclaration<'a>(
12561264
Mod: &'a Module,
12571265
ID: NonZero<c_uint>,

‎compiler/rustc_codegen_llvm/src/llvm/mod.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,14 @@ impl Intrinsic {
304304
NonZero::new(id).map(|id| Self { id })
305305
}
306306

307+
pub(crate) fn is_overloaded(self) -> bool {
308+
unsafe { LLVMIntrinsicIsOverloaded(self.id) == True }
309+
}
310+
311+
pub(crate) fn get_type<'ll>(self, llcx: &'ll Context, type_params: &[&'ll Type]) -> &'ll Type {
312+
unsafe { LLVMIntrinsicGetType(llcx, self.id, type_params.as_ptr(), type_params.len()) }
313+
}
314+
307315
pub(crate) fn get_declaration<'ll>(
308316
self,
309317
llmod: &'ll Module,

‎compiler/rustc_codegen_llvm/src/type_.rs‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
7979
args
8080
}
8181
}
82+
83+
pub(crate) fn func_is_variadic(&self, ty: &'ll Type) -> bool {
84+
unsafe { llvm::LLVMIsFunctionVarArg(ty) == True }
85+
}
8286
}
8387
impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
8488
pub(crate) fn type_bool(&self) -> &'ll Type {
@@ -288,8 +292,12 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
288292
fn cast_backend_type(&self, ty: &CastTarget) -> &'ll Type {
289293
ty.llvm_type(self)
290294
}
291-
fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type {
292-
fn_abi.llvm_type(self)
295+
fn fn_decl_backend_type(
296+
&self,
297+
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
298+
fn_ptr: &'ll Value,
299+
) -> &'ll Type {
300+
fn_abi.llvm_type(self, llvm::get_value_name(fn_ptr), false).fn_ty()
293301
}
294302
fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type {
295303
fn_abi.ptr_to_llvm_type(self)

‎compiler/rustc_codegen_ssa/src/mir/block.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
197197

198198
// If there is a cleanup block and the function we're calling can unwind, then
199199
// do an invoke, otherwise do a call.
200-
let fn_ty = bx.fn_decl_backend_type(fn_abi);
200+
let fn_ty = bx.fn_decl_backend_type(fn_abi, fn_ptr);
201201

202202
let fn_attrs = if bx.tcx().def_kind(fx.instance.def_id()).has_codegen_attrs() {
203203
Some(bx.tcx().codegen_fn_attrs(fx.instance.def_id()))
@@ -1841,7 +1841,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
18411841
if is_call_from_compiler_builtins_to_upstream_monomorphization(bx.tcx(), instance) {
18421842
bx.abort();
18431843
} else {
1844-
let fn_ty = bx.fn_decl_backend_type(fn_abi);
1844+
let fn_ty = bx.fn_decl_backend_type(fn_abi, fn_ptr);
18451845

18461846
let llret = bx.call(fn_ty, None, Some(fn_abi), fn_ptr, &[], funclet.as_ref(), None);
18471847
bx.apply_attrs_to_cleanup_callsite(llret);

0 commit comments

Comments
 (0)