zircon/kernel/object/message_packet.rs

// Copyright 2016 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT

//! Message packets are stored in lists of fixed-size buffers (`BufferChain`) rather than contiguous
//! blocks of memory to reduce heap fragmentation.

use super::buffer_chain::{BufferChain, CONTIGUOUS_SIZE};
use super::handle::HandleOwner;
use crate::user_copy::{UserInPtr, UserOutPtr};
use core::ffi::c_void;
use core::mem::{self, MaybeUninit, align_of, size_of};
use core::ops::{Deref, DerefMut};
use core::pin::Pin;
use core::ptr::{self, NonNull};
use core::{cmp, slice};
use fbl::{DoublyLinkedListContainable, DoublyLinkedListNode, ManagedPtr, MutPtr, PtrTraits};
use zerocopy::{FromBytes, Immutable, IntoBytes};
use zr::{static_assert, static_assert_size_and_align};
use zx_status::Status;
use zx_types::{
    ZX_CHANNEL_MAX_MSG_BYTES, ZX_CHANNEL_MAX_MSG_HANDLES, ZX_CHANNEL_MAX_MSG_IOVEC,
    zx_channel_iovec_t, zx_txid_t,
};

// The number of iovecs to read and process at a time. If number of iovecs <= IOVEC_CHUNK_SIZE,
// then the message buf size will be computed and the minimum required number of pages will be
// allocated. Otherwise, a large enough buffer for the largest possible message will be allocated.
const IOVEC_CHUNK_SIZE: usize = 16;

// Handles are stored just after the MessagePacket.
const HANDLES_OFFSET: usize = size_of::<MessagePacket>();

// The MessagePacket object, its handles and zx_txid_t must all fit in the first buffer.
const MIN_CONTIGUOUS_SIZE: usize = HANDLES_OFFSET
    + (ZX_CHANNEL_MAX_MSG_HANDLES as usize * size_of::<*mut c_void>())
    + size_of::<zx_txid_t>();
static_assert!(MIN_CONTIGUOUS_SIZE <= CONTIGUOUS_SIZE);

#[repr(C)]
#[derive(Copy, Clone, Default, FromBytes, IntoBytes, Immutable)]
struct RawChannelIovec {
    buffer: usize,
    capacity: u32,
    reserved: u32,
}

static_assert_size_and_align!(
    RawChannelIovec,
    size_of::<zx_channel_iovec_t>(),
    align_of::<zx_channel_iovec_t>()
);

// MessagePackets have special allocation requirements because they can contain a variable number of
// handles and a variable size payload.
//
// To reduce heap fragmentation, MessagePackets are stored in a lists of fixed size buffers
// (BufferChains) rather than a contiguous blocks of memory. These lists and buffers are allocated
// from the PMM.
//
// The first buffer in a MessagePacket's BufferChain contains the MessagePacket object, followed by
// its handles (if any), and finally its payload data (if any).
#[derive(DoublyLinkedListContainable)]
#[repr(C)]
pub struct MessagePacket {
    #[dll_node]
    node: DoublyLinkedListNode<MessagePacket>,
    buffer_chain: NonNull<BufferChain>,
    data_size: u32,
    num_handles: u16,
    owns_handles: bool,
}

// SAFETY: `MessagePacket` owns its handles array and backing `BufferChain`. It does not use
// thread-local storage or thread-bound resources and can be transferred across threads.
unsafe impl Send for MessagePacket {}

// SAFETY: All concurrent operations on `&MessagePacket` (such as `data_size`, `handles`,
// `start_of_payload`) access immutable or internally synchronized state.
unsafe impl Sync for MessagePacket {}

impl MessagePacket {
    /// Creates a message packet containing the provided data copied from userspace and space for
    /// `num_handles` handles. The handles array is uninitialized and must be completely overwritten
    /// by clients.
    #[inline]
    pub fn create_from_user(
        data: UserInPtr<u8>,
        data_size: usize,
        num_handles: usize,
    ) -> Result<MessagePacketPtr, Status> {
        let mut new_msg = Self::create_common(data_size, num_handles)?;
        if data_size == 0 {
            return Ok(new_msg);
        }
        let dst = new_msg.start_of_payload_uninit_mut();
        if dst.len() == data_size {
            data.copy_slice_from_user(dst)?;
        } else {
            new_msg.buffer_chain_mut().append_user(data, data_size)?;
        }
        Ok(new_msg)
    }

    /// Creates a message packet containing data gathered from userspace iovecs and space for
    /// `num_handles` handles.
    pub fn create_from_iovecs(
        user_iovecs: UserInPtr<zx_channel_iovec_t>,
        mut num_iovecs: usize,
        num_handles: usize,
    ) -> Result<MessagePacketPtr, Status> {
        if num_iovecs > ZX_CHANNEL_MAX_MSG_IOVEC as usize {
            return Err(Status::OUT_OF_RANGE);
        }

        let mut user_iovecs = user_iovecs.reinterpret::<RawChannelIovec>();
        let mut iovecs = [MaybeUninit::<RawChannelIovec>::uninit(); IOVEC_CHUNK_SIZE];

        if num_iovecs <= IOVEC_CHUNK_SIZE {
            let iovecs = if num_iovecs > 0 {
                user_iovecs.copy_slice_from_user(&mut iovecs[..num_iovecs])?
            } else {
                &mut []
            };
            let mut message_size = 0;
            for iovec in &*iovecs {
                if iovec.reserved != 0 {
                    return Err(Status::INVALID_ARGS);
                }
                message_size += iovec.capacity as usize;
            }
            let mut msg = Self::create_common(message_size, num_handles)?;
            let mut dst = msg.start_of_payload_uninit_mut();
            if dst.len() == message_size {
                for iovec in &*iovecs {
                    let len = iovec.capacity as usize;
                    if len > 0 {
                        let (chunk_dst, rest) = dst.split_at_mut(len);
                        dst = rest;
                        let src = UserInPtr::new(ptr::with_exposed_provenance(iovec.buffer));
                        src.copy_slice_from_user(chunk_dst)?;
                    }
                }
            } else {
                for iovec in &*iovecs {
                    let src = UserInPtr::new(ptr::with_exposed_provenance(iovec.buffer));
                    msg.buffer_chain_mut().append_user(src, iovec.capacity as usize)?;
                }
            }
            return Ok(msg);
        }

        let mut msg = Self::create_common(ZX_CHANNEL_MAX_MSG_BYTES as usize, num_handles)?;
        let mut message_size = 0;
        while num_iovecs > 0 {
            let chunk = cmp::min(num_iovecs, IOVEC_CHUNK_SIZE);
            let chunk_slice = user_iovecs.copy_slice_from_user(&mut iovecs[..chunk])?;
            for iovec in chunk_slice {
                if iovec.reserved != 0 {
                    return Err(Status::INVALID_ARGS);
                }
                message_size += iovec.capacity as usize;
                let src = UserInPtr::new(ptr::with_exposed_provenance(iovec.buffer));
                msg.buffer_chain_mut().append_user(src, iovec.capacity as usize)?;
            }
            num_iovecs -= chunk;
            user_iovecs = user_iovecs.element_offset(chunk);
        }

        msg.buffer_chain_mut().free_unused_buffers();
        msg.data_size = message_size as u32;
        Ok(msg)
    }

    /// Creates a message packet containing the provided data copied from kernel space and space
    /// for `num_handles` handles.
    pub fn create_from_kernel(data: &[u8], num_handles: usize) -> Result<MessagePacketPtr, Status> {
        let mut new_msg = Self::create_common(data.len(), num_handles)?;
        let dst = new_msg.start_of_payload_uninit_mut();
        if dst.len() == data.len() {
            // SAFETY: `dst` and `data` have equal length (`dst.len() == data.len()`) and do not
            // overlap (`dst` is inside the freshly allocated `BufferChain`).
            unsafe {
                ptr::copy_nonoverlapping(data.as_ptr(), dst.as_mut_ptr().cast::<u8>(), data.len());
            }
        } else {
            new_msg.buffer_chain_mut().append_kernel(data)?;
        }
        Ok(new_msg)
    }

    /// Returns payload data size in bytes.
    #[inline]
    pub fn data_size(&self) -> usize {
        self.data_size as usize
    }

    /// Copies the packet's `data_size` bytes to `buf`.
    ///
    /// Returns an error if `buf` points to a bad user address.
    #[inline]
    pub fn copy_data_to(&self, buf: UserOutPtr<u8>) -> Result<(), Status> {
        let data_size = self.data_size as usize;
        if data_size == 0 {
            return Ok(());
        }
        let first_chunk = self.start_of_payload();
        if first_chunk.len() == data_size {
            return buf.copy_slice_to_user(first_chunk);
        }
        // SAFETY: `self.buffer_chain` is non-null and points to the BufferChain inside the first
        // buffer.
        unsafe { self.buffer_chain.as_ref() }.copy_out(buf, self.payload_offset(), data_size)
    }

    /// Returns the number of handles attached to this message packet.
    #[inline]
    pub fn num_handles(&self) -> usize {
        self.num_handles as usize
    }

    /// Returns the transaction ID stored in the payload header (`zx_channel_call` treats the
    /// leading bytes of the payload as a transaction ID of type `zx_txid_t`).
    #[inline]
    pub fn get_txid(&self) -> zx_txid_t {
        if (self.data_size as usize) < size_of::<zx_txid_t>() {
            return 0;
        }
        // The first few bytes of the payload are a zx_txid_t.
        // SAFETY: `MIN_CONTIGUOUS_SIZE <= CONTIGUOUS_SIZE` guarantees that the `zx_txid_t` at
        // `payload_offset()` fits within the first buffer, and `payload_offset()` is
        // pointer-aligned (`>= align_of::<zx_txid_t>()`).
        unsafe {
            *ptr::from_ref(self)
                .cast::<u8>()
                .wrapping_add(self.payload_offset())
                .cast::<zx_txid_t>()
        }
    }

    /// Sets the transaction ID in the payload header.
    #[inline]
    pub fn set_txid(&mut self, txid: zx_txid_t) {
        if (self.data_size as usize) >= size_of::<zx_txid_t>() {
            // SAFETY: `MIN_CONTIGUOUS_SIZE <= CONTIGUOUS_SIZE` guarantees that the `zx_txid_t` at
            // `payload_offset()` fits within the first buffer, and `payload_offset()` is
            // pointer-aligned (`>= align_of::<zx_txid_t>()`).
            unsafe {
                *ptr::from_mut(self)
                    .cast::<u8>()
                    .wrapping_add(self.payload_offset())
                    .cast::<zx_txid_t>() = txid;
            }
        }
    }

    // A private destructor helps to make sure that only our custom deleter is ever used to destroy
    // this object which, in turn, makes it very difficult to not properly recycle the object.
    /// Recycles a `MessagePacket` by running its destructor in place and freeing its backing
    /// `BufferChain`.
    ///
    /// # Safety
    ///
    /// - `packet` must be a valid, uniquely owned pointer to a `MessagePacket` allocated inside
    ///   its backing `BufferChain`.
    /// - The packet must not be currently linked in any intrusive container
    ///   (`!packet.in_container()`).
    /// - If `packet.owns_handles` is true, every non-null entry in `packet.handles` must be a
    ///   valid, owned `Handle*`.
    #[inline]
    pub unsafe fn recycle(packet: *mut MessagePacket) {
        // SAFETY:
        // - `packet` is a valid, uniquely owned, aligned pointer to an allocated `MessagePacket`
        //   that is not linked in any container.
        // - We read `buffer_chain` before dropping `packet` in place so the backing memory can be
        //   freed.
        // - `chain` is a valid pointer originally returned from `BufferChain::alloc`.
        unsafe {
            // Grab the buffer chain for this packet before dropping.
            let chain = (*packet).buffer_chain;

            // Manually destruct the packet. Do not delete it; its memory did not come from new, it
            // is contained as part of the buffer chain.
            ptr::drop_in_place(packet);

            // Now return the buffer chain to where it came from.
            BufferChain::free(chain);
        }
    }

    // A private constructor ensures that users must use the static factory create methods to
    // create a MessagePacket. This, in turn, guarantees that when a user creates a MessagePacket,
    // they end up with the proper MessagePacketPtr type for managing the message packet's life
    // cycle.
    //
    // Creates a MessagePacket sufficient to hold `data_size` bytes and `num_handles`.
    //
    // Note: This method does not write the payload into the MessagePacket.
    //
    // Returns Ok(MessagePacketPtr) on success.
    #[inline]
    fn create_common(data_size: usize, num_handles: usize) -> Result<MessagePacketPtr, Status> {
        if data_size > ZX_CHANNEL_MAX_MSG_BYTES as usize
            || num_handles > ZX_CHANNEL_MAX_MSG_HANDLES as usize
        {
            return Err(Status::OUT_OF_RANGE);
        }

        // The payload comes after the handles.
        let payload_offset = Self::calc_payload_offset(num_handles);

        // MessagePackets lives *inside* a list of buffers. The first buffer holds the
        // MessagePacket object, followed by its handles (if any), and finally the payload data.
        let mut chain = BufferChain::alloc(payload_offset + data_size)?;
        // SAFETY: `chain` is valid and pinned inside the first buffer.
        let mut chain_pin = unsafe { Pin::new_unchecked(chain.as_mut()) };
        debug_assert!(!chain_pin.is_empty());
        chain_pin.as_mut().skip(payload_offset);

        // Construct the MessagePacket into the first buffer immediately after `BufferChain`.
        // SAFETY: `BufferChain` occupies the first `BUFFER_CHAIN_SIZE` bytes of the first buffer's
        // `raw_data`, so `chain.as_ptr().add(1)` points directly to `first_buffer_data_mut()`,
        // which is 8-byte aligned and has `CONTIGUOUS_SIZE` usable bytes.
        let packet = unsafe { chain.as_ptr().add(1).cast::<MessagePacket>() };
        debug_assert_eq!(packet.cast::<u8>(), chain_pin.first_buffer_data_mut());

        // The MessagePacket now owns the BufferChain and msg owns the MessagePacket.
        // SAFETY: `packet` points into the first buffer and is properly aligned and sized.
        unsafe {
            packet.write(MessagePacket {
                node: DoublyLinkedListNode::new(),
                buffer_chain: chain,
                data_size: data_size as u32,
                num_handles: num_handles as u16,
                owns_handles: false,
            });
            Ok(MessagePacketPtr::from_raw(packet))
        }
    }

    #[inline]
    const fn calc_payload_offset(num_handles: usize) -> usize {
        HANDLES_OFFSET + num_handles * size_of::<*mut c_void>()
    }

    #[inline]
    fn payload_offset(&self) -> usize {
        Self::calc_payload_offset(self.num_handles as usize)
    }

    #[inline]
    fn buffer_chain_mut(&mut self) -> Pin<&mut BufferChain> {
        // SAFETY: `self.buffer_chain` is uniquely owned and points to the pinned BufferChain
        // inside the first buffer.
        unsafe { Pin::new_unchecked(self.buffer_chain.as_mut()) }
    }

    #[inline]
    fn contiguous_payload_size(&self) -> usize {
        // The first Buffer of a BufferChain will contain the handles (if any are present) and at
        // least some of the message's payload. How much of message payload? Up to CONTIGUOUS_SIZE
        // minus the payload's offset.
        cmp::min(CONTIGUOUS_SIZE - self.payload_offset(), self.data_size as usize)
    }

    /// Returns a const pointer to the array of handle pointers attached to this message packet.
    #[inline]
    pub fn handles(&self) -> *const *mut c_void {
        // Handles are stored immediately after `MessagePacket` at `HANDLES_OFFSET` inside the
        // first buffer's contiguous data region.
        ptr::from_ref(self).cast::<u8>().wrapping_add(HANDLES_OFFSET).cast()
    }

    /// Returns a mutable pointer to the array of handle pointers attached to this message packet.
    #[inline]
    pub fn handles_mut(&mut self) -> *mut *mut c_void {
        // Handles are stored immediately after `MessagePacket` at `HANDLES_OFFSET` inside the
        // first buffer's contiguous data region.
        ptr::from_mut(self).cast::<u8>().wrapping_add(HANDLES_OFFSET).cast()
    }

    /// Sets whether this packet owns its attached handles and should delete them on recycle.
    #[inline]
    pub fn set_owns_handles(&mut self, owns_handles: bool) {
        self.owns_handles = owns_handles;
    }

    /// Returns a slice referencing the first chunk of payload stored contiguously in the first
    /// buffer backing the message packet.
    #[inline]
    pub fn start_of_payload(&self) -> &[u8] {
        // The first chunk of payload. Eventually we'd want to actually get the whole message out.
        // SAFETY: `self` is at the start of the first buffer's data, which contains at least
        // `payload_offset() + contiguous_payload_size()` contiguous bytes.
        unsafe {
            let ptr = ptr::from_ref(self).cast::<u8>().wrapping_add(self.payload_offset());
            slice::from_raw_parts(ptr, self.contiguous_payload_size())
        }
    }

    #[inline]
    fn start_of_payload_uninit_mut(&mut self) -> &mut [MaybeUninit<u8>] {
        // SAFETY: `self` is at the start of the first buffer's data, which contains at least
        // `payload_offset() + contiguous_payload_size()` contiguous bytes.
        unsafe {
            let ptr =
                ptr::from_mut(self).cast::<MaybeUninit<u8>>().wrapping_add(self.payload_offset());
            slice::from_raw_parts_mut(ptr, self.contiguous_payload_size())
        }
    }

    #[cold]
    fn drop_handles(&mut self) {
        // SAFETY: `handles_mut()` points to `num_handles` handle pointers owned by `self`.
        let handles = unsafe { slice::from_raw_parts_mut(self.handles_mut(), self.num_handles()) };
        for &mut raw in handles {
            // Delete the handle via HandleOwner dtor.
            // SAFETY: Each entry is a valid handle pointer (or null) owned by `self`.
            unsafe {
                drop(HandleOwner::from_raw(raw));
            }
        }
    }
}

// A private destructor helps to make sure that only our custom deleter is ever used to destroy this
// object which, in turn, makes it very difficult to not properly recycle the object.
impl Drop for MessagePacket {
    #[inline]
    fn drop(&mut self) {
        if self.owns_handles {
            self.drop_handles();
        }
    }
}

// Definition of a MessagePacket's specific pointer type. Message packets must be managed using this
// specific type of pointer, because MessagePackets have a specific custom deletion requirement.
pub struct MessagePacketPtr {
    ptr: NonNull<MessagePacket>,
}

// SAFETY: `MessagePacket` is safe to send between threads via `MessagePacketPtr`.
unsafe impl Send for MessagePacketPtr {}
unsafe impl Sync for MessagePacketPtr {}

impl MessagePacketPtr {
    /// Constructs a `MessagePacketPtr` from a non-null raw pointer.
    ///
    /// # Safety
    ///
    /// `raw` must be a valid, uniquely owned `MessagePacket*`.
    #[inline]
    pub unsafe fn from_raw(raw: *mut MessagePacket) -> Self {
        debug_assert!(!raw.is_null());
        // SAFETY: `raw` is non-null.
        Self { ptr: unsafe { NonNull::new_unchecked(raw) } }
    }

    /// Consumes the pointer and returns the raw pointer without running destructor.
    #[inline]
    pub fn into_raw(self) -> *mut MessagePacket {
        let ptr = self.ptr.as_ptr();
        mem::forget(self);
        ptr
    }
}

impl Deref for MessagePacketPtr {
    type Target = MessagePacket;

    #[inline]
    fn deref(&self) -> &Self::Target {
        // SAFETY: `self.ptr` is non-null and points to an initialized MessagePacket.
        unsafe { self.ptr.as_ref() }
    }
}

impl DerefMut for MessagePacketPtr {
    #[inline]
    fn deref_mut(&mut self) -> &mut Self::Target {
        // SAFETY: `self.ptr` is non-null and valid.
        unsafe { self.ptr.as_mut() }
    }
}

unsafe impl PtrTraits for MessagePacketPtr {
    type Target = MessagePacket;
    const IS_MANAGED: bool = true;

    #[inline]
    fn into_raw(self) -> *mut MessagePacket {
        Self::into_raw(self)
    }

    #[inline]
    unsafe fn from_raw(raw: *mut MessagePacket) -> Self {
        // SAFETY: Caller guarantees `raw` is a valid MessagePacket pointer.
        unsafe { Self::from_raw(raw) }
    }

    #[inline]
    fn get_ref(&self) -> &MessagePacket {
        self
    }
}

unsafe impl ManagedPtr for MessagePacketPtr {}

// SAFETY: `MessagePacketPtr` uniquely owns its `MessagePacket`, so exclusive access to a container
// holding it implies exclusive access to the packet.
unsafe impl MutPtr for MessagePacketPtr {}

impl Drop for MessagePacketPtr {
    #[inline]
    fn drop(&mut self) {
        // SAFETY: `self.ptr` is a valid, uniquely owned MessagePacket pointer.
        unsafe {
            MessagePacket::recycle(self.ptr.as_ptr());
        }
    }
}

/// Creates a `MessagePacket` with kernel payload data and space for `num_handles` handles.
///
/// # Safety
///
/// - If `data` is non-null and `data_size > 0`, `data` must point to `data_size` valid bytes.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_message_packet_create_kernel(
    data: *const u8,
    data_size: usize,
    num_handles: usize,
    out: &mut *mut MessagePacket,
) -> zx_types::zx_status_t {
    let payload = if data.is_null() || data_size == 0 {
        &[]
    } else {
        // SAFETY: Caller guarantees `data` points to `data_size` valid bytes in kernel memory.
        unsafe { slice::from_raw_parts(data, data_size) }
    };
    match MessagePacket::create_from_kernel(payload, num_handles) {
        Ok(packet) => {
            *out = packet.into_raw();
            zx_types::ZX_OK
        }
        Err(status) => status.into_raw(),
    }
}

/// Destroys and frees a `MessagePacket`.
///
/// # Safety
///
/// `packet` must be null or a valid, uniquely owned pointer returned by
/// `rust_message_packet_create_kernel`.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn rust_message_packet_delete(packet: *mut MessagePacket) {
    if !packet.is_null() {
        // SAFETY: Caller guarantees `packet` is a valid, uniquely owned MessagePacket pointer.
        unsafe {
            drop(MessagePacketPtr::from_raw(packet));
        }
    }
}

/// Returns the number of handles attached to the packet.
#[unsafe(no_mangle)]
pub extern "C" fn rust_message_packet_get_num_handles(packet: &MessagePacket) -> usize {
    packet.num_handles()
}

/// Returns a mutable pointer to the attached handle pointers.
#[unsafe(no_mangle)]
pub extern "C" fn rust_message_packet_get_mutable_handles(
    packet: &mut MessagePacket,
) -> *mut *mut c_void {
    packet.handles_mut()
}

/// Sets whether this packet owns its attached handles.
#[unsafe(no_mangle)]
pub extern "C" fn rust_message_packet_set_owns_handles(
    packet: &mut MessagePacket,
    owns_handles: bool,
) {
    packet.set_owns_handles(owns_handles);
}

/// In-tree kernel unit tests for `MessagePacket`.
#[cfg(ktest)]
#[unittest::suite(name = "message_packet_rust")]
mod tests {
    use super::{
        CONTIGUOUS_SIZE, IOVEC_CHUNK_SIZE, MessagePacket, MessagePacketPtr, RawChannelIovec,
    };
    use crate::user_copy::{UserInPtr, UserOutPtr};
    use crate::user_memory::UserMemory;
    use core::mem::{MaybeUninit, size_of};
    use core::{array, cmp, ptr, slice};
    use fbl::DoublyLinkedList;
    use pin_init::stack_pin_init;
    use unittest::{expect_eq, expect_false, expect_ok, expect_true, unwrap_ok};
    use zerocopy::IntoBytes;
    use zx_status::Status;
    use zx_types::{ZX_CHANNEL_MAX_MSG_BYTES, ZX_CHANNEL_MAX_MSG_HANDLES, zx_channel_iovec_t};

    fn create_user_memory(size: usize) -> Result<UserMemory, Status> {
        let mem = UserMemory::create(size).ok_or(Status::NO_MEMORY)?;
        mem.commit_and_map(0..size)?;
        Ok(mem)
    }

    macro_rules! create_user_memory {
        ($size:expr) => {
            unwrap_ok!(create_user_memory($size))
        };
    }

    fn fill_user_memory(mem: &UserMemory, byte: u8, offset: usize, size: usize) -> bool {
        let chunk = [byte; 256];
        let mut curr = offset;
        let end = offset + size;
        while curr < end {
            let to_write = cmp::min(chunk.len(), end - curr);
            if mem.vmo_write(&chunk[..to_write], curr as u64).is_err() {
                return false;
            }
            curr += to_write;
        }
        true
    }

    fn verify_user_memory_byte(mem: &UserMemory, offset: usize, size: usize, expected: u8) -> bool {
        let mut chunk = [MaybeUninit::<u8>::uninit(); 256];
        let mut curr = offset;
        let end = offset + size;
        while curr < end {
            let to_read = cmp::min(chunk.len(), end - curr);
            let Ok(read_bytes) = mem.vmo_read(&mut chunk[..to_read], curr as u64) else {
                return false;
            };
            for &mut b in read_bytes {
                if b != expected {
                    return false;
                }
            }
            curr += to_read;
        }
        true
    }

    /// Tests creating a MessagePacket from kernel data, matching C++ create_void_star.
    #[test]
    fn test_create_from_kernel() {
        const SIZE: usize = 32;
        let data = [b'B'; SIZE];
        let mut packet =
            unwrap_ok!(MessagePacket::create_from_kernel(&data, 0), "failed to create packet");
        expect_eq!(packet.data_size(), data.len());
        expect_eq!(packet.num_handles(), 0);
        expect_true!(packet.get_txid() != 0);

        let mem_out = create_user_memory!(SIZE);
        let user_out = UserOutPtr::new(ptr::with_exposed_provenance_mut::<u8>(mem_out.base()));
        expect_ok!(packet.copy_data_to(user_out));

        let mut read_buf = [MaybeUninit::<u8>::uninit(); SIZE];
        let read_bytes = unwrap_ok!(mem_out.vmo_read(&mut read_buf, 0));
        expect_true!(read_bytes == data);

        packet.set_txid(999);
        expect_eq!(packet.get_txid(), 999);
    }

    /// Tests creating a zero-length packet from user memory, matching C++ create_zero.
    #[test]
    fn test_create_zero() {
        let mem = create_user_memory!(1);
        let user_in = UserInPtr::new(ptr::with_exposed_provenance::<u8>(mem.base()));
        let user_out = UserOutPtr::new(ptr::with_exposed_provenance_mut::<u8>(mem.base()));

        let packet = unwrap_ok!(MessagePacket::create_from_user(user_in, 0, 0));
        expect_eq!(packet.data_size(), 0);
        expect_eq!(packet.num_handles(), 0);
        expect_eq!(packet.get_txid(), 0);

        expect_ok!(packet.copy_data_to(user_out));
    }

    /// Tests creating a MessagePacket with zero size payload.
    #[test]
    fn test_create_from_kernel_zero_size() {
        let packet =
            unwrap_ok!(MessagePacket::create_from_kernel(&[], 0), "failed to create packet");
        expect_eq!(packet.data_size(), 0);
        expect_eq!(packet.num_handles(), 0);
        expect_eq!(packet.get_txid(), 0);
    }

    /// Tests that creating a MessagePacket with too many handles fails with OUT_OF_RANGE.
    #[test]
    fn test_create_too_many_handles() {
        let mem = create_user_memory!(1);
        let user_in = UserInPtr::new(ptr::with_exposed_provenance::<u8>(mem.base()));
        let res_user =
            MessagePacket::create_from_user(user_in, 1, ZX_CHANNEL_MAX_MSG_HANDLES as usize + 1);
        expect_true!(matches!(res_user, Err(Status::OUT_OF_RANGE)));

        let res_kernel =
            MessagePacket::create_from_kernel(&[], ZX_CHANNEL_MAX_MSG_HANDLES as usize + 1);
        expect_true!(matches!(res_kernel, Err(Status::OUT_OF_RANGE)));
    }

    /// Tests set_owns_handles(true) and set_owns_handles(false).
    #[test]
    fn test_set_owns_handles() {
        let mut packet = unwrap_ok!(MessagePacket::create_from_kernel(&[], 2));
        expect_eq!(packet.handles(), packet.handles_mut().cast_const());
        // Initialize handles to null pointers so recycle doesn't attempt to drop invalid pointers.
        unsafe {
            slice::from_raw_parts_mut(packet.handles_mut(), 2).fill(ptr::null_mut());
        }
        packet.set_owns_handles(true);
        packet.set_owns_handles(false);

        let mut packet2 = unwrap_ok!(MessagePacket::create_from_kernel(&[], 2));
        unsafe {
            slice::from_raw_parts_mut(packet2.handles_mut(), 2).fill(ptr::null_mut());
        }
        packet2.set_owns_handles(true);
    }

    /// Tests creating a MessagePacket from user memory and copying data out.
    #[test]
    fn test_create_from_user_and_copy_out() {
        let size = 128;
        let mem_in = create_user_memory!(size);
        let payload: [u8; 128] = array::from_fn(|i| (i as u8).wrapping_mul(3));
        unwrap_ok!(mem_in.vmo_write(&payload, 0), "failed to write payload");

        let user_in = UserInPtr::new(ptr::with_exposed_provenance::<u8>(mem_in.base()));
        let packet = unwrap_ok!(
            MessagePacket::create_from_user(user_in, size, 0),
            "failed to create packet"
        );
        expect_eq!(packet.data_size(), size);
        expect_eq!(packet.num_handles(), 0);

        let mem_out = create_user_memory!(size);
        let user_out = UserOutPtr::new(ptr::with_exposed_provenance_mut::<u8>(mem_out.base()));
        expect_ok!(packet.copy_data_to(user_out));

        let mut read_buf = [MaybeUninit::<u8>::uninit(); 128];
        let read_bytes =
            unwrap_ok!(mem_out.vmo_read(&mut read_buf, 0), "failed to read back payload");
        expect_true!(read_bytes == payload);
    }

    /// Tests creating a large MessagePacket from user memory matching C++ create().
    #[test]
    fn test_create_large_user_message() {
        const SIZE: usize = 62234;
        const NUM_HANDLES: usize = 64;

        let mem_in = create_user_memory!(SIZE);
        expect_true!(fill_user_memory(&mem_in, b'A', 0, SIZE));

        let user_in = UserInPtr::new(ptr::with_exposed_provenance::<u8>(mem_in.base()));
        let packet = unwrap_ok!(MessagePacket::create_from_user(user_in, SIZE, NUM_HANDLES));
        expect_eq!(packet.data_size(), SIZE);
        expect_eq!(packet.num_handles(), NUM_HANDLES);
        expect_true!(packet.get_txid() != 0);

        let mem_out = create_user_memory!(SIZE);
        let user_out = UserOutPtr::new(ptr::with_exposed_provenance_mut::<u8>(mem_out.base()));
        expect_ok!(packet.copy_data_to(user_out));

        expect_true!(verify_user_memory_byte(&mem_out, 0, SIZE, b'A'));
    }

    /// Tests creating a MessagePacket from iovecs and copying data out.
    #[test]
    fn test_create_from_iovecs() {
        let chunk1 = b"chunk one ";
        let chunk2 = b"chunk two";
        let total_size = chunk1.len() + chunk2.len();

        let mem = create_user_memory!(4096);

        let offset_iovecs = 0;
        let offset_chunk1 = 128;
        let offset_chunk2 = 256;

        unwrap_ok!(mem.vmo_write(chunk1, offset_chunk1), "failed to write chunk1");
        unwrap_ok!(mem.vmo_write(chunk2, offset_chunk2), "failed to write chunk2");

        let base = mem.base();
        let iovecs = [
            RawChannelIovec {
                buffer: base + offset_chunk1 as usize,
                capacity: chunk1.len() as u32,
                reserved: 0,
            },
            RawChannelIovec {
                buffer: base + offset_chunk2 as usize,
                capacity: chunk2.len() as u32,
                reserved: 0,
            },
        ];
        unwrap_ok!(mem.vmo_write(iovecs.as_bytes(), offset_iovecs), "failed to write iovecs");

        let user_iovecs = UserInPtr::new(ptr::with_exposed_provenance::<zx_channel_iovec_t>(
            base + offset_iovecs as usize,
        ));
        let packet = unwrap_ok!(
            MessagePacket::create_from_iovecs(user_iovecs, 2, 0),
            "failed to create packet from iovecs"
        );
        expect_eq!(packet.data_size(), total_size);

        let mem_out = create_user_memory!(total_size);
        let user_out = UserOutPtr::new(ptr::with_exposed_provenance_mut::<u8>(mem_out.base()));
        expect_ok!(packet.copy_data_to(user_out));

        let mut read_buf = [MaybeUninit::<u8>::uninit(); 19];
        let read_bytes =
            unwrap_ok!(mem_out.vmo_read(&mut read_buf, 0), "failed to read back payload");
        expect_true!(&read_bytes[..10] == chunk1);
        expect_true!(&read_bytes[10..] == chunk2);
    }

    macro_rules! check_create_iovec {
        ($num_iovecs:expr, $num_handles:expr) => {{
            let n_iovecs = $num_iovecs;
            let n_handles = $num_handles;
            let num_bytes: usize = n_iovecs * (n_iovecs - 1) / 2;
            let bytes_mem = create_user_memory!(num_bytes);

            // Populate bytes with incrementing values: bytes[i] = i as u8.
            let mut chunk = [0u8; 256];
            let mut written = 0;
            while written < num_bytes {
                let to_write = cmp::min(chunk.len(), num_bytes - written);
                for j in 0..to_write {
                    chunk[j] = (written + j) as u8;
                }
                unwrap_ok!(bytes_mem.vmo_write(&chunk[..to_write], written as u64));
                written += to_write;
            }

            // Create iovecs where iovec[i] has capacity i.
            let iovec_mem_size = n_iovecs * size_of::<zx_channel_iovec_t>();
            let iovec_mem = create_user_memory!(iovec_mem_size);

            let mut byte_offset: usize = 0;
            for i in 0..n_iovecs {
                let iovec = RawChannelIovec {
                    buffer: bytes_mem.base() + byte_offset,
                    capacity: i as u32,
                    reserved: 0,
                };
                byte_offset += i;
                unwrap_ok!(
                    iovec_mem
                        .vmo_write(iovec.as_bytes(), (i * size_of::<zx_channel_iovec_t>()) as u64,)
                );
            }

            let user_iovecs = UserInPtr::new(ptr::with_exposed_provenance::<zx_channel_iovec_t>(
                iovec_mem.base(),
            ));
            let packet =
                unwrap_ok!(MessagePacket::create_from_iovecs(user_iovecs, n_iovecs, n_handles));

            expect_eq!(packet.num_handles(), n_handles);
            expect_eq!(packet.data_size(), num_bytes);

            let result_mem = create_user_memory!(num_bytes);
            let user_out =
                UserOutPtr::new(ptr::with_exposed_provenance_mut::<u8>(result_mem.base()));
            expect_ok!(packet.copy_data_to(user_out));

            // Verify result matches expected bytes.
            let mut read_buf = [MaybeUninit::<u8>::uninit(); 256];
            let mut verified = 0;
            while verified < num_bytes {
                let to_verify = cmp::min(read_buf.len(), num_bytes - verified);
                let read_bytes =
                    unwrap_ok!(result_mem.vmo_read(&mut read_buf[..to_verify], verified as u64));
                for (j, &b) in read_bytes.iter().enumerate() {
                    expect_eq!(b, (verified + j) as u8);
                }
                verified += to_verify;
            }
        }};
    }

    /// Tests bounded iovecs (16 iovecs) with 0 handles, matching C++ create_iovec_bounded.
    #[test]
    fn test_create_iovec_bounded() {
        check_create_iovec!(IOVEC_CHUNK_SIZE, 0);
    }

    /// Tests bounded iovecs (16 iovecs) with 3 handles, matching C++ create_iovec_bounded_handles.
    #[test]
    fn test_create_iovec_bounded_handles() {
        check_create_iovec!(IOVEC_CHUNK_SIZE, 3);
    }

    /// Tests unbounded iovecs (32 iovecs) with 0 handles, matching C++ create_iovec_unbounded.
    #[test]
    fn test_create_iovec_unbounded() {
        check_create_iovec!(2 * IOVEC_CHUNK_SIZE, 0);
    }

    /// Tests unbounded iovecs (32 iovecs, 3 handles) matching C++ create_iovec_unbounded_handles.
    #[test]
    fn test_create_iovec_unbounded_handles() {
        check_create_iovec!(2 * IOVEC_CHUNK_SIZE, 3);
    }

    /// Tests that passing an iovec with non-zero reserved returns Status::INVALID_ARGS.
    #[test]
    fn test_iovec_non_zero_reserved() {
        // Test bounded path (1 iovec).
        let mem = create_user_memory!(4096);
        let base = mem.base();

        let iovec = RawChannelIovec { buffer: base + 1024, capacity: 16, reserved: 1 };
        unwrap_ok!(mem.vmo_write(iovec.as_bytes(), 0));

        let user_iovecs = UserInPtr::new(ptr::with_exposed_provenance::<zx_channel_iovec_t>(base));
        let res_bounded = MessagePacket::create_from_iovecs(user_iovecs, 1, 0);
        expect_true!(matches!(res_bounded, Err(Status::INVALID_ARGS)));

        // Test unbounded path (> IOVEC_CHUNK_SIZE iovecs).
        let num_iovecs = IOVEC_CHUNK_SIZE + 1;
        let mem_unbounded = create_user_memory!(num_iovecs * size_of::<zx_channel_iovec_t>());
        unwrap_ok!(mem_unbounded.vmo_write(iovec.as_bytes(), 0));
        let user_iovecs_unbounded = UserInPtr::new(ptr::with_exposed_provenance::<
            zx_channel_iovec_t,
        >(mem_unbounded.base()));
        let res_unbounded = MessagePacket::create_from_iovecs(user_iovecs_unbounded, num_iovecs, 0);
        expect_true!(matches!(res_unbounded, Err(Status::INVALID_ARGS)));
    }

    /// Tests bad user memory handling, verifying both null and kernel addresses are rejected.
    #[test]
    fn test_bad_user_memory() {
        let null_in = UserInPtr::<u8>::new(ptr::null());
        let res = MessagePacket::create_from_user(null_in, 64, 0);
        expect_true!(matches!(res, Err(Status::INVALID_ARGS)));

        let kernel_buf = [0u8; 64];
        let kernel_in = UserInPtr::<u8>::new(kernel_buf.as_ptr());
        let res = MessagePacket::create_from_user(kernel_in, 64, 0);
        expect_true!(matches!(res, Err(Status::INVALID_ARGS)));

        let packet = unwrap_ok!(MessagePacket::create_from_kernel(b"test", 0));
        let null_out = UserOutPtr::<u8>::new(ptr::null_mut());
        let res = packet.copy_data_to(null_out);
        expect_true!(matches!(res, Err(Status::INVALID_ARGS)));

        let mut kernel_out_buf = [0u8; 4];
        let kernel_out = UserOutPtr::<u8>::new(kernel_out_buf.as_mut_ptr());
        let res = packet.copy_data_to(kernel_out);
        expect_true!(matches!(res, Err(Status::INVALID_ARGS)));
    }

    /// Verifies that start_of_payload returns a contiguous prefix for a maximum sized packet.
    #[test]
    fn test_start_of_payload_max_message() {
        let max_msg_bytes = ZX_CHANNEL_MAX_MSG_BYTES as usize;
        let mem = create_user_memory!(max_msg_bytes);
        expect_true!(fill_user_memory(&mem, b'A', 0, max_msg_bytes));

        let user_in = UserInPtr::new(ptr::with_exposed_provenance::<u8>(mem.base()));
        let packet = unwrap_ok!(MessagePacket::create_from_user(user_in, max_msg_bytes, 0));
        expect_eq!(packet.data_size(), max_msg_bytes);

        let expected_len = CONTIGUOUS_SIZE - packet.payload_offset();
        let start = packet.start_of_payload();
        expect_eq!(start.len(), expected_len);
        for &b in start {
            expect_eq!(b, b'A');
        }
    }

    /// Tests start_of_payload.
    #[test]
    fn test_start_of_payload() {
        let payload = [0x55u8; 64];
        let packet = unwrap_ok!(MessagePacket::create_from_kernel(&payload, 0));
        let start = packet.start_of_payload();
        expect_eq!(start.len(), 64);
        expect_true!(start == payload);
    }

    /// Tests MessagePacket with DoublyLinkedList and in_container check.
    #[test]
    fn test_message_packet_doubly_linked_list() {
        let packet1 = unwrap_ok!(MessagePacket::create_from_kernel(b"first", 0));
        let packet2 = unwrap_ok!(MessagePacket::create_from_kernel(b"second", 0));

        expect_false!(packet1.node.in_container());
        expect_false!(packet2.node.in_container());

        stack_pin_init!(let list = DoublyLinkedList::<MessagePacketPtr>::new());
        expect_true!(list.is_empty());

        list.as_mut().push_back(packet1);
        list.as_mut().push_back(packet2);
        expect_false!(list.is_empty());

        let popped1 = list.as_mut().pop_front().expect("expected packet1");
        expect_eq!(popped1.data_size(), 5);
        expect_false!(popped1.node.in_container());

        let popped2 = list.as_mut().pop_front().expect("expected packet2");
        expect_eq!(popped2.data_size(), 6);
        expect_false!(popped2.node.in_container());

        expect_true!(list.is_empty());
    }
}