error on calls to ABIs that cannot be called

Author: folkertdev

compiler/rustc_hir_typeck/messages.ftl

@@ -1,6 +1,6 @@
-hir_typeck_abi_custom_call =
-    functions with the `"custom"` ABI cannot be called
-    .note = an `extern "custom"` function can only be called from within inline assembly
+hir_typeck_abi_cannot_be_called =
+    functions with the `{$abi}` ABI cannot be called
+    .note = an `extern {$abi}` function can only be called using inline assembly
 
 hir_typeck_add_missing_parentheses_in_range = you must surround the range in parentheses to call its `{$func_name}` function
 

compiler/rustc_hir_typeck/src/callee.rs

@@ -1,6 +1,6 @@
 use std::iter;
 
-use rustc_abi::ExternAbi;
+use rustc_abi::CanonAbi;
 use rustc_ast::util::parser::ExprPrecedence;
 use rustc_errors::{Applicability, Diag, ErrorGuaranteed, StashKey};
 use rustc_hir::def::{self, CtorKind, Namespace, Res};
@@ -16,6 +16,7 @@ use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::{bug, span_bug};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{Span, sym};
+use rustc_target::spec::{AbiMap, AbiMapping};
 use rustc_trait_selection::error_reporting::traits::DefIdOrName;
 use rustc_trait_selection::infer::InferCtxtExt as _;
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt as _;
@@ -84,7 +85,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         while result.is_none() && autoderef.next().is_some() {
             result = self.try_overloaded_call_step(call_expr, callee_expr, arg_exprs, &autoderef);
         }
-        self.check_call_custom_abi(autoderef.final_ty(false), call_expr.span);
+        self.check_call_abi(autoderef.final_ty(false), call_expr.span);
         self.register_predicates(autoderef.into_obligations());
 
         let output = match result {
@@ -137,19 +138,47 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         output
     }
 
-    /// Functions of type `extern "custom" fn(/* ... */)` cannot be called using `ExprKind::Call`.
-    ///
-    /// These functions have a calling convention that is unknown to rust, hence it cannot generate
-    /// code for the call. The only way to execute such a function is via inline assembly.
-    fn check_call_custom_abi(&self, callee_ty: Ty<'tcx>, span: Span) {
+    fn check_call_abi(&self, callee_ty: Ty<'tcx>, span: Span) {
         let abi = match callee_ty.kind() {
             ty::FnDef(def_id, _) => self.tcx.fn_sig(def_id).skip_binder().skip_binder().abi,
             ty::FnPtr(_, header) => header.abi,
             _ => return,
         };
 
-        if let ExternAbi::Custom = abi {
-            self.tcx.dcx().emit_err(errors::AbiCustomCall { span });
+        match AbiMap::from_target(&self.sess().target).canonize_abi(abi, false) {
+            AbiMapping::Direct(canon_abi) | AbiMapping::Deprecated(canon_abi) => {
+                if !Self::can_be_called_with_call_expr(canon_abi) {
+                    let err = crate::errors::AbiCannotBeCalled { span, abi };
+                    self.tcx.dcx().emit_err(err);
+                }
+            }
+            AbiMapping::Invalid => {
+                // The ABI is invalid for the target: ignore.
+            }
+        }
+    }
+
+    /// Can a function with this ABI be called with a rust call expression?
+    ///
+    /// Some ABIs cannot be called from rust, either because rust does not know how to generate
+    /// code for the call, or because a call does not semantically make sense.
+    pub(crate) fn can_be_called_with_call_expr(abi: CanonAbi) -> bool {
+        match abi {
+            // Rust doesn't know how to call functions with this ABI.
+            CanonAbi::Custom => false,
+
+            // These is an entry point for the host, and cannot be called on the GPU.
+            CanonAbi::GpuKernel => false,
+
+            // The interrupt ABIs should only be called by the CPU. They have complex
+            // pre- and postconditions, and can use non-standard instructions like `iret` on x86.
+            CanonAbi::Interrupt(_) => false,
+
+            CanonAbi::C
+            | CanonAbi::Rust
+            | CanonAbi::RustCold
+            | CanonAbi::Arm(_)
+            | CanonAbi::X86(_) => true,
         }
     }
 

compiler/rustc_hir_typeck/src/errors.rs

@@ -2,6 +2,7 @@
 
 use std::borrow::Cow;
 
+use rustc_abi::ExternAbi;
 use rustc_ast::Label;
 use rustc_errors::codes::*;
 use rustc_errors::{
@@ -1165,8 +1166,10 @@ pub(crate) struct NakedFunctionsMustNakedAsm {
 }
 
 #[derive(Diagnostic)]
-#[diag(hir_typeck_abi_custom_call)]
-pub(crate) struct AbiCustomCall {
+#[diag(hir_typeck_abi_cannot_be_called)]
+pub(crate) struct AbiCannotBeCalled {
     #[primary_span]
+    #[note]
     pub span: Span,
+    pub abi: ExternAbi,
 }

compiler/rustc_hir_typeck/src/expr.rs

@@ -5,7 +5,7 @@
 //!
 //! See [`rustc_hir_analysis::check`] for more context on type checking in general.
 
-use rustc_abi::{ExternAbi, FIRST_VARIANT, FieldIdx};
+use rustc_abi::{FIRST_VARIANT, FieldIdx};
 use rustc_ast::util::parser::ExprPrecedence;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::stack::ensure_sufficient_stack;
@@ -34,6 +34,7 @@ use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::hygiene::DesugaringKind;
 use rustc_span::source_map::Spanned;
 use rustc_span::{Ident, Span, Symbol, kw, sym};
+use rustc_target::spec::{AbiMap, AbiMapping};
 use rustc_trait_selection::infer::InferCtxtExt;
 use rustc_trait_selection::traits::{self, ObligationCauseCode, ObligationCtxt};
 use tracing::{debug, instrument, trace};
@@ -1651,12 +1652,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     Some(method.def_id),
                 );
 
-                // Functions of type `extern "custom" fn(/* ... */)` cannot be called using
-                // `ExprKind::MethodCall`. These functions have a calling convention that is
-                // unknown to rust, hence it cannot generate code for the call. The only way
-                // to execute such a function is via inline assembly.
-                if let ExternAbi::Custom = method.sig.abi {
-                    self.tcx.dcx().emit_err(crate::errors::AbiCustomCall { span: expr.span });
+                match AbiMap::from_target(&self.sess().target).canonize_abi(method.sig.abi, false) {
+                    AbiMapping::Direct(canon_abi) | AbiMapping::Deprecated(canon_abi) => {
+                        if !Self::can_be_called_with_call_expr(canon_abi) {
+                            self.tcx.dcx().emit_err(crate::errors::AbiCannotBeCalled {
+                                span: expr.span,
+                                abi: method.sig.abi,
+                            });
+                        }
+                    }
+                    AbiMapping::Invalid => { /* ignore */ }
                 }
 
                 method.sig.output()

tests/ui/abi/bad-custom.stderr

@@ -248,42 +248,84 @@ LL |     extern "custom" fn negate(a: i64) -> i64 {
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:75:14
    |
+LL |     unsafe { f(x) }
+   |              ^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:75:14
+   |
 LL |     unsafe { f(x) }
    |              ^^^^
 
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:80:14
    |
+LL |     unsafe { f(x) }
+   |              ^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:80:14
+   |
 LL |     unsafe { f(x) }
    |              ^^^^
 
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:87:14
    |
+LL |     unsafe { f(x) }
+   |              ^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:87:14
+   |
 LL |     unsafe { f(x) }
    |              ^^^^
 
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:109:20
    |
+LL |         assert_eq!(double(21), 42);
+   |                    ^^^^^^^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:109:20
+   |
 LL |         assert_eq!(double(21), 42);
    |                    ^^^^^^^^^^
 
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:112:29
    |
+LL |         assert_eq!(unsafe { increment(41) }, 42);
+   |                             ^^^^^^^^^^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:112:29
+   |
 LL |         assert_eq!(unsafe { increment(41) }, 42);
    |                             ^^^^^^^^^^^^^
 
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:115:17
    |
+LL |         assert!(Thing(41).is_even());
+   |                 ^^^^^^^^^^^^^^^^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:115:17
+   |
 LL |         assert!(Thing(41).is_even());
    |                 ^^^^^^^^^^^^^^^^^^^
 
 error: functions with the `"custom"` ABI cannot be called
   --> $DIR/bad-custom.rs:118:20
    |
+LL |         assert_eq!(Thing::bitwise_not(42), !42);
+   |                    ^^^^^^^^^^^^^^^^^^^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
+  --> $DIR/bad-custom.rs:118:20
+   |
 LL |         assert_eq!(Thing::bitwise_not(42), !42);
    |                    ^^^^^^^^^^^^^^^^^^^^^^