miri function ABI check: specifically look for repr(transparent)

Author: RalfJung

tests/fail/function_pointers/abi_mismatch_array_vs_struct.rs

@@ -0,0 +1,16 @@
+#![feature(portable_simd)]
+
+// Some targets treat arrays and structs very differently. We would probably catch that on those
+// targets since we check the `PassMode`; here we ensure that we catch it on *all* targets
+// (in particular, on x86-64 the pass mode is `Indirect` for both of these).
+struct S(i32, i32, i32, i32);
+type A = [i32; 4];
+
+fn main() {
+    fn f(_: S) {}
+
+    // These two types have the same size but are still not compatible.
+    let g = unsafe { std::mem::transmute::<fn(S), fn(A)>(f) };
+
+    g(Default::default()) //~ ERROR: calling a function with argument of type S passing data of type [i32; 4]
+}

tests/fail/function_pointers/abi_mismatch_array_vs_struct.stderr

@@ -0,0 +1,15 @@
+error: Undefined Behavior: calling a function with argument of type S passing data of type [i32; 4]
+  --> $DIR/abi_mismatch_array_vs_struct.rs:LL:CC
+   |
+LL |     g(Default::default())
+   |     ^^^^^^^^^^^^^^^^^^^^^ calling a function with argument of type S passing data of type [i32; 4]
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+   = note: BACKTRACE:
+   = note: inside `main` at $DIR/abi_mismatch_array_vs_struct.rs:LL:CC
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to previous error
+

tests/pass/function_calls/abi_compat.rs

@@ -24,10 +24,13 @@ fn test_abi_newtype<T: Copy>(t: T) {
     #[repr(transparent)]
     struct Wrapper2<T>(T, ());
     #[repr(transparent)]
+    struct Wrapper2a<T>((), T);
+    #[repr(transparent)]
     struct Wrapper3<T>(T, [u8; 0]);
 
     test_abi_compat(t, Wrapper1(t));
     test_abi_compat(t, Wrapper2(t, ()));
+    test_abi_compat(t, Wrapper2a((), t));
     test_abi_compat(t, Wrapper3(t, []));
 }
 
@@ -46,4 +49,5 @@ fn main() {
     test_abi_newtype(0f32);
     test_abi_newtype((0u32, 1u32, 2u32));
     test_abi_newtype([0u32, 1u32, 2u32]);
+    test_abi_newtype([0i32; 0]);
 }