miri: implement more restrictive trivial-ABI checks

Author: RalfJung

compiler/rustc_const_eval/src/interpret/call.rs

@@ -70,27 +70,75 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         })
     }
 
-    /// Find the wrapped inner type of a transparent wrapper.
-    /// Must not be called on 1-ZST (as they don't have a uniquely defined "wrapped field").
+    /// Returns whether the given type has trivial ABI.
+    fn has_trivial_abi(&self, layout: TyAndLayout<'tcx>) -> InterpResult<'tcx, bool> {
+        if !layout.is_1zst() {
+            return interp_ok(false);
+        }
+        match *layout.ty.kind() {
+            ty::Array(elem, _len) => self.has_trivial_abi(self.layout_of(elem)?),
+            ty::Tuple(..)
+            | ty::Never
+            | ty::FnDef(..)
+            | ty::Closure(..)
+            | ty::Coroutine(..)
+            | ty::CoroutineClosure(..) => interp_ok(true),
+            ty::Adt(adt_def, _args) => {
+                if adt_def.repr().transparent() {
+                    // All fields must have trivial ABI.
+                    (0..layout.fields.count()).try_fold(true, |acc, idx| {
+                        interp_ok(acc && self.has_trivial_abi(layout.field(self, idx))?)
+                    })
+                } else if adt_def.repr().c() {
+                    interp_ok(false)
+                } else {
+                    // Must be repr(Rust).
+                    interp_ok(true)
+                }
+            }
+            ty::Alias(..) => panic!("non-normalized type"),
+            _ => interp_ok(false),
+        }
+    }
+
+    /// Find the wrapped inner type of a transparent wrapper by going for the unique
+    /// non-trivial-ABI field.
     ///
     /// We work with `TyAndLayout` here since that makes it much easier to iterate over all fields.
     fn unfold_transparent(
         &self,
         layout: TyAndLayout<'tcx>,
         may_unfold: impl Fn(AdtDef<'tcx>) -> bool,
-    ) -> TyAndLayout<'tcx> {
+    ) -> InterpResult<'tcx, TyAndLayout<'tcx>> {
         match layout.ty.kind() {
             ty::Adt(adt_def, _) if adt_def.repr().transparent() && may_unfold(*adt_def) => {
                 assert_matches!(layout.variants, rustc_abi::Variants::Single { .. });
-                // Find the non-1-ZST field, and recurse.
-                let (_, field) = layout.non_1zst_field(self).unwrap();
+                // Look for non-trivial-ABI field(s).
+                let mut found = None;
+                for idx in 0..layout.fields.count() {
+                    let field = layout.field(self, idx);
+                    if self.has_trivial_abi(field)? {
+                        continue;
+                    }
+                    // Found a non-trivial ABI field!
+                    if found.is_some() {
+                        // There is more than one such field.
+                        // FIXME: we should just panic here. But currently such repr(transparent)
+                        // types are still accepted. We just don't treat them as transparent.
+                        return interp_ok(layout);
+                    }
+                    found = Some(field);
+                }
+                let Some(field) = found else {
+                    // All fields have trivial ABI. That means this type is effectively `()`.
+                    return interp_ok(self.layout_of(self.tcx.types.unit)?);
+                };
+                // Recurse.
                 self.unfold_transparent(field, may_unfold)
             }
-            ty::Pat(base, _) => self.layout_of(*base).expect(
-                "if the layout of a pattern type could be computed, so can the layout of its base",
-            ),
+            ty::Pat(base, _) => interp_ok(self.layout_of(*base)?),
             // Not a transparent type, no further unfolding.
-            _ => layout,
+            _ => interp_ok(layout),
         }
     }
 
@@ -145,7 +193,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         let inner = self.unfold_transparent(inner, /* may_unfold */ |def| {
             // Stop at NPO types so that we don't miss that attribute in the check below!
             def.is_struct() && !is_npo(def)
-        });
+        })?;
         interp_ok(match inner.ty.kind() {
             ty::Ref(..) | ty::FnPtr(..) => {
                 // Option<&T> behaves like &T, and same for fn()
@@ -154,7 +202,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             ty::Adt(def, _) if is_npo(*def) => {
                 // Once we found a `nonnull_optimization_guaranteed` type, further strip off
                 // newtype structs from it to find the underlying ABI type.
-                self.unfold_transparent(inner, /* may_unfold */ |def| def.is_struct())
+                self.unfold_transparent(inner, /* may_unfold */ |def| def.is_struct())?
             }
             _ => {
                 // Everything else we do not unfold.
@@ -175,16 +223,21 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         if caller.ty == callee.ty {
             return interp_ok(true);
         }
-        // 1-ZST are compatible with all 1-ZST (and with nothing else).
-        if caller.is_1zst() || callee.is_1zst() {
-            return interp_ok(caller.is_1zst() && callee.is_1zst());
+        // Handle trivial-ABI types.
+        if self.has_trivial_abi(caller)? && self.has_trivial_abi(callee)? {
+            return interp_ok(true);
         }
         // Unfold newtypes and NPO optimizations.
         let unfold = |layout: TyAndLayout<'tcx>| {
-            self.unfold_npo(self.unfold_transparent(layout, /* may_unfold */ |_def| true))
+            self.unfold_transparent(layout, /* may_unfold */ |_def| true)
+                .and_then(|f| self.unfold_npo(f))
         };
         let caller = unfold(caller)?;
         let callee = unfold(callee)?;
+        // Not-quite-so-fast path: if the types are equal now, they are compatible.
+        if caller.ty == callee.ty {
+            return interp_ok(true);
+        }
         // Now see if these inner types are compatible.
 
         // Compatible pointer types. For thin pointers, we have to accept even non-`repr(transparent)`
@@ -240,8 +293,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             return interp_ok(caller == callee);
         }
 
-        // Fall back to exact equality.
-        interp_ok(caller == callee)
+        // The rest is incompatible.
+        interp_ok(false)
     }
 
     /// Returns a `bool` saying whether the two arguments are ABI-compatible.

src/tools/miri/tests/fail/function_pointers/abi_mismatch_zst_array.rs

@@ -0,0 +1,8 @@
+fn callee(_s: [u8; 0]) {}
+//~^ ERROR: type [u8; 0] passing argument of type ()
+
+fn main() {
+    let fnptr: fn([u8; 0]) = callee;
+    let fnptr: fn(()) = unsafe { std::mem::transmute(fnptr) };
+    fnptr(());
+}

src/tools/miri/tests/fail/function_pointers/abi_mismatch_zst_array.stderr

@@ -0,0 +1,20 @@
+error: Undefined Behavior: calling a function whose parameter #1 has type [u8; 0] passing argument of type ()
+  --> tests/fail/function_pointers/abi_mismatch_zst_array.rs:LL:CC
+   |
+LL | fn callee(_s: [u8; 0]) {}
+   |           ^^ Undefined Behavior occurred here
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+   = help: this means these two types are not *guaranteed* to be ABI-compatible across all targets
+   = help: if you think this code should be accepted anyway, please report an issue with Miri
+   = note: stack backtrace:
+           0: callee
+               at tests/fail/function_pointers/abi_mismatch_zst_array.rs:LL:CC
+           1: main
+               at tests/fail/function_pointers/abi_mismatch_zst_array.rs:LL:CC
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+

src/tools/miri/tests/fail/function_pointers/abi_mismatch_zst_repr_C.rs

@@ -0,0 +1,11 @@
+#[repr(C)]
+struct C;
+
+fn callee() {}
+//~^ ERROR: return type () passing return place of type C
+
+fn main() {
+    let fnptr: fn() -> () = callee;
+    let fnptr: fn() -> C = unsafe { std::mem::transmute(fnptr) };
+    fnptr();
+}

src/tools/miri/tests/fail/function_pointers/abi_mismatch_zst_repr_C.stderr

@@ -0,0 +1,20 @@
+error: Undefined Behavior: calling a function with return type () passing return place of type C
+  --> tests/fail/function_pointers/abi_mismatch_zst_repr_C.rs:LL:CC
+   |
+LL | fn callee() {}
+   |            ^ Undefined Behavior occurred here
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+   = help: this means these two types are not *guaranteed* to be ABI-compatible across all targets
+   = help: if you think this code should be accepted anyway, please report an issue with Miri
+   = note: stack backtrace:
+           0: callee
+               at tests/fail/function_pointers/abi_mismatch_zst_repr_C.rs:LL:CC
+           1: main
+               at tests/fail/function_pointers/abi_mismatch_zst_repr_C.rs:LL:CC
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+

src/tools/miri/tests/fail/function_pointers/abi_mismatch_zst_transparent_array.rs

@@ -0,0 +1,11 @@
+#[repr(transparent)]
+struct Wrap([u8; 0]);
+
+fn callee(_s: Wrap) {}
+//~^ ERROR: type Wrap passing argument of type ()
+
+fn main() {
+    let fnptr: fn(Wrap) = callee;
+    let fnptr: fn(()) = unsafe { std::mem::transmute(fnptr) };
+    fnptr(());
+}

src/tools/miri/tests/fail/function_pointers/abi_mismatch_zst_transparent_array.stderr

@@ -0,0 +1,20 @@
+error: Undefined Behavior: calling a function whose parameter #1 has type Wrap passing argument of type ()
+  --> tests/fail/function_pointers/abi_mismatch_zst_transparent_array.rs:LL:CC
+   |
+LL | fn callee(_s: Wrap) {}
+   |           ^^ Undefined Behavior occurred here
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+   = help: this means these two types are not *guaranteed* to be ABI-compatible across all targets
+   = help: if you think this code should be accepted anyway, please report an issue with Miri
+   = note: stack backtrace:
+           0: callee
+               at tests/fail/function_pointers/abi_mismatch_zst_transparent_array.rs:LL:CC
+           1: main
+               at tests/fail/function_pointers/abi_mismatch_zst_transparent_array.rs:LL:CC
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+

src/tools/miri/tests/pass/function_calls/abi_compat.rs

@@ -62,19 +62,19 @@ fn test_abi_newtype<T: Copy + Default>() {
     struct Wrapper2a<T>((), T);
     #[repr(transparent)]
     #[derive(Copy, Clone)]
-    struct Wrapper3<T>(Zst, T, [u8; 0]);
+    struct Wrapper3<T>(Zst, T, [(); 0]);
     #[repr(transparent)]
     #[derive(Copy, Clone)]
     enum Wrapper4<T> {
-        V(Zst, T, [u8; 0]),
+        V(Zst, T, [(); 10]),
     }
 
     let t = T::default();
     test_abi_compat(t, Wrapper(t));
     test_abi_compat(t, Wrapper2(t, ()));
     test_abi_compat(t, Wrapper2a((), t));
     test_abi_compat(t, Wrapper3(Zst, t, []));
-    test_abi_compat(t, Wrapper4::V(Zst, t, []));
+    test_abi_compat(t, Wrapper4::V(Zst, t, [(); _]));
     // MaybeUninit is `repr(transparent)`; that covers the `union` case.
     test_abi_compat(t, mem::MaybeUninit::new(t));
 }
@@ -100,8 +100,8 @@ fn main() {
     test_abi_compat(&0u32, &([true; 4], [0u32; 0]));
     // - `fn` types
     test_abi_compat(main as fn(), id::<i32> as fn(i32) -> i32);
-    // - 1-ZST
-    test_abi_compat((), [0u8; 0]);
+    // - trivial-ABI types
+    test_abi_compat((), [(); 0]);
 
     // Guaranteed null-pointer-layout optimizations:
     // - Guaranteed Option<X> null-pointer-optimizations (RFC 3391).