Rollup merge of rust-lang#157397 - folkertdev:cmse-clear-padding, r=davidtwco,RalfJung

cmse: clear padding when crossing the secure boundary

tracking issue: rust-lang#81391
tracking issue: rust-lang#75835
RFC: rust-lang/rfcs#3884
related: rust-lang#147697

quick context: cmse creates a distinction between code running in secure mode and non-secure mode (think kernel space versus user space). Secure mode has access to data (e.g. encryption keys) that must not leak to non-secure mode. They use a special calling convention that clears unused registers, but padding in arguments/return values can contain stale secure data.

This PR clears the padding bytes (and similar, e.g. space not used in any variant of a union/enum) when values are passed over the secure boundary.

Separately we'll have a lint to warn on enums and unions being passed across the boundary: for them we can't statically know whether the variant that is passed contains padding.

This is conceptually modeled after a similar feature in `clang` ([implementation](https://github.com/llvm/llvm-project/blob/065a39b9f7f06fca0926394096ee1c1fac41d446/clang/lib/CodeGen/CGCall.cpp#L4041-L4087)).

cc @Jules-Bertholet
r? @davidtwco

Author: JonathanBrouwer

compiler/rustc_abi/src/layout/ty.rs

@@ -1,7 +1,8 @@
 use std::fmt;
-use std::ops::Deref;
+use std::ops::{Deref, Range};
 
 use rustc_data_structures::intern::Interned;
+use rustc_data_structures::range_set::RangeSet;
 use rustc_macros::StableHash;
 
 use crate::layout::{FieldIdx, VariantIdx};
@@ -282,4 +283,89 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
         }
         found
     }
+
+    /// The ranges of bytes that are always ignored by the representation relation of this type.
+    ///
+    /// In other words, for any sequence of bytes, if we reset the these padding bytes to uninit,
+    /// then these two sequences of bytes represent the same value (or they are both invalid).
+    /// This is the "guaranteed" padding. There may be more bytes that are padding for some
+    /// but not all variants of this type; those are not included.
+    /// (E.g. `Option<i8>` has no guaranteed padding so the empty range set is returned, but its `None` value still has padding).
+    pub fn padding_ranges<C>(&self, cx: &C) -> Vec<Range<Size>>
+    where
+        Ty: TyAbiInterface<'a, C> + Copy,
+    {
+        let mut data = RangeSet::new();
+        self.add_data_ranges(cx, Size::ZERO, &mut data);
+
+        // Find gaps between the data ranges.
+        let mut uninit_ranges = Vec::new();
+        let mut covered_until = Size::ZERO;
+        for &(offset, size) in data.0.iter() {
+            if offset > covered_until {
+                uninit_ranges.push(covered_until..offset);
+            }
+            covered_until = Ord::max(covered_until, offset + size);
+        }
+
+        // Add trailing padding.
+        if self.size > covered_until {
+            uninit_ranges.push(covered_until..self.size);
+        }
+
+        uninit_ranges
+    }
+
+    /// Extend `out` with all ranges of bytes that *may* carry relevant data for values of this type.
+    /// For enums and unions there are offsets that are initialized for some
+    /// variants but not for others; those offset *will* get added to `out`.
+    fn add_data_ranges<C>(self, cx: &C, base_offset: Size, out: &mut RangeSet<Size>)
+    where
+        Ty: TyAbiInterface<'a, C> + Copy,
+    {
+        if self.is_zst() {
+            return;
+        }
+
+        match &self.variants {
+            Variants::Empty => { /* done */ }
+            Variants::Single { index: _ } => match &self.fields {
+                FieldsShape::Primitive => {
+                    out.add_range(base_offset, self.size);
+                }
+                &FieldsShape::Union(field_count) => {
+                    for field in 0..field_count.get() {
+                        let field = self.field(cx, field);
+                        field.add_data_ranges(cx, base_offset, out);
+                    }
+                }
+                &FieldsShape::Array { stride, count } => {
+                    let elem = self.field(cx, 0);
+
+                    // For scalars we know there is no padding between the elements,
+                    // so the entire array is a single big data range.
+                    if elem.backend_repr.is_scalar() {
+                        out.add_range(base_offset, elem.size * count);
+                    } else {
+                        // FIXME: this is really inefficient for large arrays.
+                        for idx in 0..count {
+                            elem.add_data_ranges(cx, base_offset + idx * stride, out);
+                        }
+                    }
+                }
+                FieldsShape::Arbitrary { offsets, in_memory_order: _ } => {
+                    for (field, &offset) in offsets.iter_enumerated() {
+                        let field = self.field(cx, field.as_usize());
+                        field.add_data_ranges(cx, base_offset + offset, out);
+                    }
+                }
+            },
+            Variants::Multiple { variants, .. } => {
+                for variant in variants.indices() {
+                    let variant = self.for_variant(cx, variant);
+                    variant.add_data_ranges(cx, base_offset, out);
+                }
+            }
+        }
+    }
 }

compiler/rustc_abi/src/lib.rs

@@ -792,7 +792,7 @@ impl FromStr for Endian {
 }
 
 /// Size of a type in bytes.
-#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
 #[cfg_attr(feature = "nightly", derive(Encodable_NoContext, Decodable_NoContext, StableHash))]
 pub struct Size {
     raw: u64,

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1,6 +1,9 @@
 use std::cmp;
+use std::ops::Range;
 
-use rustc_abi::{Align, BackendRepr, ExternAbi, HasDataLayout, Reg, Size, WrappingRange};
+use rustc_abi::{
+    Align, ArmCall, BackendRepr, CanonAbi, ExternAbi, HasDataLayout, Reg, Size, WrappingRange,
+};
 use rustc_ast as ast;
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_data_structures::packed::Pu128;
@@ -597,6 +600,20 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     }
                     ZeroSized => bug!("ZST return value shouldn't be in PassMode::Cast"),
                 };
+
+                if self.fn_abi.conv == CanonAbi::Arm(ArmCall::CCmseNonSecureEntry) {
+                    // The return value of an `extern "cmse-nonsecure-entry"` function crosses the
+                    // secure boundary. Zero padding bytes so information does not leak.
+                    //
+                    // This only zeroes "guaranteed" padding. There may be more bytes that are
+                    // padding for some but not all variants of this type; those are not zeroed.
+                    //
+                    // Returning a value with value-dependent padding will instead trigger a lint.
+                    let ret_layout = self.fn_abi.ret.layout;
+                    let uninit_ranges = ret_layout.padding_ranges(bx.cx());
+                    self.zero_byte_ranges(bx, llslot, ret_layout.size, &uninit_ranges);
+                }
+
                 load_cast(bx, cast_ty, llslot, self.fn_abi.ret.layout.align.abi)
             }
         };
@@ -1341,6 +1358,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
             self.codegen_argument(
                 bx,
+                fn_abi.conv,
                 op,
                 by_move,
                 &mut llargs,
@@ -1351,6 +1369,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let num_untupled = untuple.map(|tup| {
             self.codegen_arguments_untupled(
                 bx,
+                fn_abi.conv,
                 &tup.node,
                 &mut llargs,
                 &fn_abi.args[first_args.len()..],
@@ -1380,6 +1399,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             let last_arg = fn_abi.args.last().unwrap();
             self.codegen_argument(
                 bx,
+                fn_abi.conv,
                 location,
                 /* by_move */ false,
                 &mut llargs,
@@ -1696,9 +1716,31 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         }
     }
 
+    fn zero_byte_ranges(
+        &mut self,
+        bx: &mut Bx,
+        ptr: Bx::Value,
+        limit: Size,
+        ranges: &[Range<Size>],
+    ) {
+        let zero = bx.const_u8(0);
+
+        for range in ranges {
+            let end = cmp::min(range.end, limit);
+            if range.start >= end {
+                continue;
+            }
+            let offset = bx.const_usize(range.start.bytes());
+            let len = bx.const_usize((end - range.start).bytes());
+            let ptr = bx.inbounds_ptradd(ptr, offset);
+            bx.memset(ptr, zero, len, Align::ONE, MemFlags::empty());
+        }
+    }
+
     fn codegen_argument(
         &mut self,
         bx: &mut Bx,
+        conv: CanonAbi,
         op: OperandRef<'tcx, Bx::Value>,
         by_move: bool,
         llargs: &mut Vec<Bx::Value>,
@@ -1822,6 +1864,23 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     MemFlags::empty(),
                     None,
                 );
+
+                // The arguments of an `extern "cmse-nonsecure-call"` function cross the secure
+                // boundary. Zero padding bytes so information does not leak.
+                //
+                // This only zeroes "guaranteed" padding. There may be more bytes that are
+                // padding for some but not all variants of this type; those are not zeroed.
+                //
+                // Passing an argument with value-dependent padding will instead trigger a lint.
+                if conv == CanonAbi::Arm(ArmCall::CCmseNonSecureCall) {
+                    self.zero_byte_ranges(
+                        bx,
+                        llscratch,
+                        Size::from_bytes(copy_bytes),
+                        &arg.layout.padding_ranges(bx.cx()),
+                    );
+                }
+
                 // ...and then load it with the ABI type.
                 llval = load_cast(bx, cast, llscratch, scratch_align);
                 bx.lifetime_end(llscratch, scratch_size);
@@ -1848,6 +1907,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     fn codegen_arguments_untupled(
         &mut self,
         bx: &mut Bx,
+        conv: CanonAbi,
         operand: &mir::Operand<'tcx>,
         llargs: &mut Vec<Bx::Value>,
         args: &[ArgAbi<'tcx, Ty<'tcx>>],
@@ -1867,6 +1927,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let field = bx.load_operand(field_ptr);
                 self.codegen_argument(
                     bx,
+                    conv,
                     field,
                     by_move,
                     llargs,
@@ -1878,7 +1939,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             // If the tuple is immediate, the elements are as well.
             for i in 0..tuple.layout.fields.count() {
                 let op = tuple.extract_field(self, bx, i);
-                self.codegen_argument(bx, op, by_move, llargs, &args[i], lifetime_ends_after_call);
+                self.codegen_argument(
+                    bx,
+                    conv,
+                    op,
+                    by_move,
+                    llargs,
+                    &args[i],
+                    lifetime_ends_after_call,
+                );
             }
         }
         tuple.layout.fields.count()

compiler/rustc_const_eval/src/interpret/validity.rs

@@ -345,55 +345,7 @@ fn write_path(out: &mut String, path: &[PathElem<'_>]) {
     }
 }
 
-/// Represents a set of `Size` values as a sorted list of ranges.
-// These are (offset, length) pairs, and they are sorted and mutually disjoint,
-// and never adjacent (i.e. there's always a gap between two of them).
-#[derive(Debug, Clone)]
-pub struct RangeSet(Vec<(Size, Size)>);
-
-impl RangeSet {
-    fn add_range(&mut self, offset: Size, size: Size) {
-        if size.bytes() == 0 {
-            // No need to track empty ranges.
-            return;
-        }
-        let v = &mut self.0;
-        // We scan for a partition point where the left partition is all the elements that end
-        // strictly before we start. Those are elements that are too "low" to merge with us.
-        let idx =
-            v.partition_point(|&(other_offset, other_size)| other_offset + other_size < offset);
-        // Now we want to either merge with the first element of the second partition, or insert ourselves before that.
-        if let Some(&(other_offset, other_size)) = v.get(idx)
-            && offset + size >= other_offset
-        {
-            // Their end is >= our start (otherwise it would not be in the 2nd partition) and
-            // our end is >= their start. This means we can merge the ranges.
-            let new_start = other_offset.min(offset);
-            let mut new_end = (other_offset + other_size).max(offset + size);
-            // We grew to the right, so merge with overlapping/adjacent elements.
-            // (We also may have grown to the left, but that can never make us adjacent with
-            // anything there since we selected the first such candidate via `partition_point`.)
-            let mut scan_right = 1;
-            while let Some(&(next_offset, next_size)) = v.get(idx + scan_right)
-                && new_end >= next_offset
-            {
-                // Increase our size to absorb the next element.
-                new_end = new_end.max(next_offset + next_size);
-                // Look at the next element.
-                scan_right += 1;
-            }
-            // Update the element we grew.
-            v[idx] = (new_start, new_end - new_start);
-            // Remove the elements we absorbed (if any).
-            if scan_right > 1 {
-                drop(v.drain((idx + 1)..(idx + scan_right)));
-            }
-        } else {
-            // Insert new element.
-            v.insert(idx, (offset, size));
-        }
-    }
-}
+pub type RangeSet = rustc_data_structures::range_set::RangeSet<Size>;
 
 struct ValidityVisitor<'rt, 'tcx, M: Machine<'tcx>> {
     /// The `path` may be pushed to, but the part that is present when a function
@@ -1194,7 +1146,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
         assert!(layout.is_sized(), "there are no unsized unions");
         let layout_cx = LayoutCx::new(*ecx.tcx, ecx.typing_env);
         return M::cached_union_data_range(ecx, layout.ty, || {
-            let mut out = RangeSet(Vec::new());
+            let mut out = RangeSet::new();
             union_data_range_uncached(&layout_cx, layout, Size::ZERO, &mut out);
             out
         });
@@ -1644,7 +1596,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 ctfe_mode,
                 ecx,
                 reset_provenance_and_padding,
-                data_bytes: reset_padding.then_some(RangeSet(Vec::new())),
+                data_bytes: reset_padding.then_some(RangeSet::new()),
                 may_dangle: start_in_may_dangle,
             };
             v.visit_value(val)?;

compiler/rustc_data_structures/src/lib.rs

@@ -68,6 +68,7 @@ pub mod obligation_forest;
 pub mod owned_slice;
 pub mod packed;
 pub mod profiling;
+pub mod range_set;
 pub mod sharded;
 pub mod small_c_str;
 pub mod snapshot_map;

compiler/rustc_data_structures/src/range_set.rs

@@ -0,0 +1,59 @@
+/// Represents a set of `Size` values as a sorted list of ranges.
+///
+/// These are (offset, length) pairs, and they are sorted and mutually disjoint,
+/// and never adjacent (i.e. there's always a gap between two of them).
+#[derive(Debug, Clone)]
+pub struct RangeSet<T>(pub Vec<(T, T)>);
+
+impl<T> RangeSet<T>
+where
+    T: Copy + Ord + Default,
+    T: core::ops::Add<Output = T>,
+    T: core::ops::Sub<Output = T>,
+{
+    pub fn new() -> Self {
+        Self(Vec::new())
+    }
+
+    pub fn add_range(&mut self, offset: T, size: T) {
+        if size == T::default() {
+            // No need to track empty ranges.
+            return;
+        }
+        let v = &mut self.0;
+        // We scan for a partition point where the left partition is all the elements that end
+        // strictly before we start. Those are elements that are too "low" to merge with us.
+        let idx =
+            v.partition_point(|&(other_offset, other_size)| other_offset + other_size < offset);
+        // Now we want to either merge with the first element of the second partition, or insert ourselves before that.
+        if let Some(&(other_offset, other_size)) = v.get(idx)
+            && offset + size >= other_offset
+        {
+            // Their end is >= our start (otherwise it would not be in the 2nd partition) and
+            // our end is >= their start. This means we can merge the ranges.
+            let new_start = other_offset.min(offset);
+            let mut new_end = (other_offset + other_size).max(offset + size);
+            // We grew to the right, so merge with overlapping/adjacent elements.
+            // (We also may have grown to the left, but that can never make us adjacent with
+            // anything there since we selected the first such candidate via `partition_point`.)
+            let mut scan_right = 1;
+            while let Some(&(next_offset, next_size)) = v.get(idx + scan_right)
+                && new_end >= next_offset
+            {
+                // Increase our size to absorb the next element.
+                new_end = new_end.max(next_offset + next_size);
+                // Look at the next element.
+                scan_right += 1;
+            }
+            // Update the element we grew.
+            v[idx] = (new_start, new_end - new_start);
+            // Remove the elements we absorbed (if any).
+            if scan_right > 1 {
+                drop(v.drain((idx + 1)..(idx + scan_right)));
+            }
+        } else {
+            // Insert new element.
+            v.insert(idx, (offset, size));
+        }
+    }
+}