error on calls to ABIs that cannot be called

Author: folkertdev

compiler/rustc_abi/src/extern_abi.rs

@@ -249,6 +249,47 @@ impl ExternAbi {
             _ => false,
         }
     }
+
+    /// Can a function with this ABI be called with a rust call expression?
+    ///
+    /// Some ABIs cannot be called from rust, either because rust does not know how to
+    /// generate code for the call, or because a call does not semantically make sense.
+    pub fn can_be_called_with_call_expr(self) -> bool {
+        match self {
+            // Rust doesn't know how to call functions with this ABI.
+            ExternAbi::Custom => false,
+
+            // These define entry points for the host, and cannot be called on the GPU.
+            ExternAbi::PtxKernel | ExternAbi::GpuKernel => false,
+
+            // The interrupt ABIs should only be called by the CPU.
+            // They have complex pre- and postconditions.
+            ExternAbi::AvrInterrupt
+            | ExternAbi::AvrNonBlockingInterrupt
+            | ExternAbi::Msp430Interrupt
+            | ExternAbi::RiscvInterruptM
+            | ExternAbi::RiscvInterruptS
+            | ExternAbi::X86Interrupt => false,
+
+            ExternAbi::C { .. }
+            | ExternAbi::System { .. }
+            | ExternAbi::Rust
+            | ExternAbi::RustCall
+            | ExternAbi::RustCold
+            | ExternAbi::Unadjusted
+            | ExternAbi::EfiApi
+            | ExternAbi::Aapcs { .. }
+            | ExternAbi::CCmseNonSecureCall
+            | ExternAbi::CCmseNonSecureEntry
+            | ExternAbi::Cdecl { .. }
+            | ExternAbi::Stdcall { .. }
+            | ExternAbi::Fastcall { .. }
+            | ExternAbi::Thiscall { .. }
+            | ExternAbi::Vectorcall { .. }
+            | ExternAbi::SysV64 { .. }
+            | ExternAbi::Win64 { .. } => true,
+        }
+    }
 }
 
 pub fn all_names() -> Vec<&'static str> {

compiler/rustc_hir_typeck/messages.ftl

@@ -1,6 +1,6 @@
-hir_typeck_abi_custom_call =
-    functions with the `"custom"` ABI cannot be called
-    .note = an `extern "custom"` function can only be called from within inline assembly
+hir_typeck_abi_cannot_be_called =
+    functions with the `"{$abi}"` ABI cannot be called
+    .note = an `extern "{$abi}"` function can only be called using inline assembly
 
 hir_typeck_add_missing_parentheses_in_range = you must surround the range in parentheses to call its `{$func_name}` function
 

compiler/rustc_hir_typeck/src/callee.rs

@@ -1,6 +1,5 @@
 use std::iter;
 
-use rustc_abi::ExternAbi;
 use rustc_ast::util::parser::ExprPrecedence;
 use rustc_errors::{Applicability, Diag, ErrorGuaranteed, StashKey};
 use rustc_hir::def::{self, CtorKind, Namespace, Res};
@@ -84,7 +83,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         while result.is_none() && autoderef.next().is_some() {
             result = self.try_overloaded_call_step(call_expr, callee_expr, arg_exprs, &autoderef);
         }
-        self.check_call_custom_abi(autoderef.final_ty(false), call_expr.span);
+        self.check_call_abi(autoderef.final_ty(false), call_expr.span);
         self.register_predicates(autoderef.into_obligations());
 
         let output = match result {
@@ -137,19 +136,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         output
     }
 
-    /// Functions of type `extern "custom" fn(/* ... */)` cannot be called using `ExprKind::Call`.
-    ///
-    /// These functions have a calling convention that is unknown to rust, hence it cannot generate
-    /// code for the call. The only way to execute such a function is via inline assembly.
-    fn check_call_custom_abi(&self, callee_ty: Ty<'tcx>, span: Span) {
+    fn check_call_abi(&self, callee_ty: Ty<'tcx>, span: Span) {
         let abi = match callee_ty.kind() {
             ty::FnDef(def_id, _) => self.tcx.fn_sig(def_id).skip_binder().skip_binder().abi,
             ty::FnPtr(_, header) => header.abi,
             _ => return,
         };
 
-        if let ExternAbi::Custom = abi {
-            self.tcx.dcx().emit_err(errors::AbiCustomCall { span });
+        // Some ABIs cannot be called from rust, either because rust does not know how to generate
+        // code for the call, or because a call does not semantically make sense.
+        if !abi.can_be_called_with_call_expr() {
+            self.tcx.dcx().emit_err(crate::errors::AbiCannotBeCalled { span, abi: abi.as_str() });
         }
     }
 

compiler/rustc_hir_typeck/src/errors.rs

@@ -1165,8 +1165,10 @@ pub(crate) struct NakedFunctionsMustNakedAsm {
 }
 
 #[derive(Diagnostic)]
-#[diag(hir_typeck_abi_custom_call)]
-pub(crate) struct AbiCustomCall {
+#[diag(hir_typeck_abi_cannot_be_called)]
+pub(crate) struct AbiCannotBeCalled {
     #[primary_span]
+    #[note]
     pub span: Span,
+    pub abi: &'static str,
 }

compiler/rustc_hir_typeck/src/expr.rs

@@ -5,7 +5,7 @@
 //!
 //! See [`rustc_hir_analysis::check`] for more context on type checking in general.
 
-use rustc_abi::{ExternAbi, FIRST_VARIANT, FieldIdx};
+use rustc_abi::{FIRST_VARIANT, FieldIdx};
 use rustc_ast::util::parser::ExprPrecedence;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::stack::ensure_sufficient_stack;
@@ -1651,12 +1651,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     Some(method.def_id),
                 );
 
-                // Functions of type `extern "custom" fn(/* ... */)` cannot be called using
-                // `ExprKind::MethodCall`. These functions have a calling convention that is
-                // unknown to rust, hence it cannot generate code for the call. The only way
-                // to execute such a function is via inline assembly.
-                if let ExternAbi::Custom = method.sig.abi {
-                    self.tcx.dcx().emit_err(crate::errors::AbiCustomCall { span: expr.span });
+                // Some ABIs cannot be called from rust, either because rust does not know how to generate
+                // code for the call, or because a call does not semantically make sense.
+                if !method.sig.abi.can_be_called_with_call_expr() {
+                    self.tcx.dcx().emit_err(crate::errors::AbiCannotBeCalled {
+                        span: expr.span,
+                        abi: method.sig.abi.as_str(),
+                    });
                 }
 
                 method.sig.output()

tests/ui/abi/bad-custom.stderr

@@ -248,42 +248,84 @@ LL |     extern "custom" fn negate(a: i64) -> i64 {
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:75:14
    |
+LL |     unsafe { f(x) }
+   |              ^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:75:14
+   |
 LL |     unsafe { f(x) }
    |              ^^^^
 
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:80:14
    |
+LL |     unsafe { f(x) }
+   |              ^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:80:14
+   |
 LL |     unsafe { f(x) }
    |              ^^^^
 
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:87:14
    |
+LL |     unsafe { f(x) }
+   |              ^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:87:14
+   |
 LL |     unsafe { f(x) }
    |              ^^^^
 
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:109:20
    |
+LL |         assert_eq!(double(21), 42);
+   |                    ^^^^^^^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:109:20
+   |
 LL |         assert_eq!(double(21), 42);
    |                    ^^^^^^^^^^
 
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:112:29
    |
+LL |         assert_eq!(unsafe { increment(41) }, 42);
+   |                             ^^^^^^^^^^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:112:29
+   |
 LL |         assert_eq!(unsafe { increment(41) }, 42);
    |                             ^^^^^^^^^^^^^
 
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:115:17
    |
+LL |         assert!(Thing(41).is_even());
+   |                 ^^^^^^^^^^^^^^^^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:115:17
+   |
 LL |         assert!(Thing(41).is_even());
    |                 ^^^^^^^^^^^^^^^^^^^
 
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:118:20
    |
+LL |         assert_eq!(Thing::bitwise_not(42), !42);
+   |                    ^^^^^^^^^^^^^^^^^^^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:118:20
+   |
 LL |         assert_eq!(Thing::bitwise_not(42), !42);
    |                    ^^^^^^^^^^^^^^^^^^^^^^