| // Copyright 2026 The Fuchsia Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| use delivery_blob::compression::{ChunkedArchiveError, DataBuffer}; | |
| use mapping::{ | |
| DeliveryCommand, DeliveryHandler, PENDING_DELIVERY_COMMANDS_CAPACITY, PagePrecommit, | |
| PageRequest, RawDeliveryCommand, | |
| }; | |
| use std::ops::Range; | |
| use std::sync::Arc; | |
| use storage_ptr_slice::MutPtrByteSlice; | |
| use vmo_fifo::{OwnedPayloadBuffer, SyncSender}; | |
| // Leave 512 KiB of payload headroom on full-sized delivery VMOs (>= 1 MiB payload capacity, i.e. | |
| // the 2 MiB production delivery VMO) so metadata registrations (`register_blob`) running on block | |
| // completion threads never block behind in-flight page-in reservations. | |
| const REGISTER_BLOB_HEADROOM_BYTES: usize = 512 * 1024; | |
| const MIN_CAPACITY_FOR_HEADROOM_BYTES: usize = 1024 * 1024; | |
| // Flush decompressed data to the verifier queue every 32 KiB (rounded up to a multiple of | |
| // `read_alignment`) so Merkle verification overlaps with decompressing the remainder of a 128 KiB | |
| // readahead window, while keeping per-chunk FIFO overhead low and allowing the verifier to coalesce | |
| // contiguous chunks into a single `zx_pager_supply_pages` syscall. | |
| const VERIFICATION_FLUSH_THRESHOLD: usize = 32 * 1024; | |
| /// Manages data verification and page supply via the delivery queue VMO. | |
| #[derive(Debug)] | |
| pub struct Verifier { | |
| sender: Arc<SyncSender<RawDeliveryCommand>>, | |
| headroom: usize, | |
| } | |
| impl Verifier { | |
| pub fn new(delivery_queue: zx::Vmo) -> Result<Self, zx::Status> { | |
| // Payloads in the delivery queue are transferred to the kernel pager via | |
| // `zx_pager_supply_pages`, which requires the source offset to be page-aligned. Set | |
| // the alignment of payload offsets to be page aligned. | |
| let sender = Arc::new(SyncSender::<RawDeliveryCommand>::new( | |
| delivery_queue, | |
| zx::system_get_page_size() as usize, | |
| PENDING_DELIVERY_COMMANDS_CAPACITY, | |
| )?); | |
| let headroom = if sender.payload_capacity() >= MIN_CAPACITY_FOR_HEADROOM_BYTES { | |
| REGISTER_BLOB_HEADROOM_BYTES | |
| } else { | |
| 0 | |
| }; | |
| Ok(Self { sender, headroom }) | |
| } | |
| fn reserve_payload( | |
| &self, | |
| len: usize, | |
| ) -> Result<OwnedPayloadBuffer<RawDeliveryCommand>, ChunkedArchiveError> { | |
| self.sender.reserve_owned_payload(len, self.headroom).map_err(|error| { | |
| log::error!(error:?; "Verifier::reserve_payload failed"); | |
| ChunkedArchiveError::IntegrityError | |
| }) | |
| } | |
| } | |
| impl DeliveryHandler for Verifier { | |
| type Request = Buffer; | |
| /// Returns a [`PageRequest`] implementation for delivering page-in data. | |
| fn get_page_request(self: &Arc<Self>, key: u64, original_range: Range<u64>) -> Buffer { | |
| Buffer { | |
| verifier: Arc::clone(self), | |
| key, | |
| read_range: original_range, | |
| total_len: 0, | |
| read_alignment: 0, | |
| flush_threshold: usize::MAX, | |
| committed_len: 0, | |
| delivered_len: 0, | |
| payload: None, | |
| child_target: None, | |
| trace_flow_id: 0, | |
| } | |
| } | |
| /// Returns a [`Buffer`] configured to deliver parent blob chunks for a child file extent via | |
| /// [`DeliveryCommand::ChildData`]. | |
| fn get_child_page_request( | |
| self: &Arc<Self>, | |
| child_key: u64, | |
| child_target_offset: u64, | |
| parent_byte_range: Range<u64>, | |
| ) -> Buffer { | |
| Buffer { | |
| verifier: Arc::clone(self), | |
| key: child_key, | |
| read_range: parent_byte_range.clone(), | |
| total_len: 0, | |
| read_alignment: 0, | |
| flush_threshold: usize::MAX, | |
| committed_len: 0, | |
| delivered_len: 0, | |
| payload: None, | |
| child_target: Some(ChildTarget { child_target_offset, parent_byte_range }), | |
| trace_flow_id: 0, | |
| } | |
| } | |
| /// Pushes a header-only [`DeliveryCommand::ChildInlineTail`] command onto the delivery queue | |
| /// so the verifier can supply the child tail page from the verified parent VMO. | |
| fn deliver_child_inline_tail( | |
| self: &Arc<Self>, | |
| parent_byte_offset: u64, | |
| child_key: u64, | |
| child_target_offset: u64, | |
| tail_length: u32, | |
| trace_flow_id: u64, | |
| ) { | |
| let cmd = DeliveryCommand::ChildInlineTail { | |
| parent_byte_offset, | |
| child_key, | |
| child_target_offset, | |
| tail_length, | |
| }; | |
| let _ = self.sender.push(RawDeliveryCommand::from(cmd).with_trace_flow_id(trace_flow_id)); | |
| } | |
| /// Pushes a [`DeliveryCommand::ChildData`] command with `length == 0` onto the delivery queue | |
| /// to supply a zero-filled sparse range to `child_key`. | |
| fn deliver_child_sparse( | |
| &self, | |
| child_key: u64, | |
| child_target_offset: u64, | |
| child_length: u64, | |
| trace_flow_id: u64, | |
| ) -> Result<(), anyhow::Error> { | |
| let cmd = DeliveryCommand::ChildData { | |
| parent_offset: 0, | |
| offset: 0, | |
| length: 0, | |
| child_key, | |
| child_target_offset, | |
| in_chunk_offset: 0, | |
| child_length: child_length as u32, | |
| }; | |
| self.sender.push(RawDeliveryCommand::from(cmd).with_trace_flow_id(trace_flow_id))?; | |
| Ok(()) | |
| } | |
| /// Registers a blob's Merkle tree leaf hashes with the verifier via the delivery queue. | |
| fn register_blob( | |
| &self, | |
| key: u64, | |
| merkle_leaves: &[[u8; 32]], | |
| read_alignment: usize, | |
| ) -> Result<(), anyhow::Error> { | |
| let leaf_bytes: &[u8] = merkle_leaves.as_flattened(); | |
| let mut payload = self.sender.reserve_owned_payload(leaf_bytes.len(), 0)?; | |
| let payload_offset = payload.offset(); | |
| payload.data_mut().subslice_mut(0..leaf_bytes.len()).copy_from_slice(leaf_bytes); | |
| let cmd = DeliveryCommand::RegisterBlob { | |
| key, | |
| offset: payload_offset, | |
| length: leaf_bytes.len() as u32, | |
| read_alignment: read_alignment as u32, | |
| }; | |
| payload.commit(cmd.into())?; | |
| Ok(()) | |
| } | |
| } | |
| /// Target metadata for a [`Buffer`] servicing a page request on a child file mapped over a | |
| /// parent blob. | |
| #[derive(Debug)] | |
| struct ChildTarget { | |
| /// Starting page-aligned logical byte offset in the child VMO for this request. | |
| child_target_offset: u64, | |
| /// Byte range in the parent blob corresponding to this child extent slice. | |
| parent_byte_range: Range<u64>, | |
| } | |
| /// Implementation of [`PageRequest`] for verifier deliveries that writes directly into reserved | |
| /// `delivery_vmo` pages and forwards chunks via the delivery queue without bounce-buffering. | |
| #[derive(Debug)] | |
| pub struct Buffer { | |
| verifier: Arc<Verifier>, | |
| key: u64, | |
| read_range: Range<u64>, | |
| total_len: usize, | |
| // Alignment boundary (matching the blob's `read_alignment`) at which data is flushed to the | |
| // verifier queue. | |
| read_alignment: usize, | |
| // Precomputed byte threshold at which intermediate `read_alignment` chunks are flushed to the | |
| // verifier queue (`usize::MAX` if `read_alignment` is not a multiple of `page_size`). | |
| flush_threshold: usize, | |
| // Bytes committed into `self.payload` so far via calls to [`DataBuffer::commit`]. | |
| committed_len: usize, | |
| // Bytes delivered to the verifier queue so far. | |
| delivered_len: usize, | |
| payload: Option<OwnedPayloadBuffer<RawDeliveryCommand>>, | |
| child_target: Option<ChildTarget>, | |
| trace_flow_id: u64, | |
| } | |
| impl Buffer { | |
| fn make_delivery_command( | |
| child_target: Option<&ChildTarget>, | |
| key: u64, | |
| chunk_offset: u64, | |
| aligned_size: usize, | |
| payload_offset: u32, | |
| ) -> DeliveryCommand { | |
| if let Some(target) = child_target { | |
| let chunk_end = chunk_offset + aligned_size as u64; | |
| let slice_start = std::cmp::max(chunk_offset, target.parent_byte_range.start); | |
| let slice_end = std::cmp::min(chunk_end, target.parent_byte_range.end); | |
| let in_chunk_offset = (slice_start - chunk_offset) as u32; | |
| let child_target_offset = | |
| target.child_target_offset + (slice_start - target.parent_byte_range.start); | |
| let child_length = (slice_end.saturating_sub(slice_start)) as u32; | |
| DeliveryCommand::ChildData { | |
| parent_offset: chunk_offset, | |
| offset: payload_offset, | |
| length: aligned_size as u32, | |
| child_key: key, | |
| child_target_offset, | |
| in_chunk_offset, | |
| child_length, | |
| } | |
| } else { | |
| DeliveryCommand::Data { | |
| key, | |
| target_offset: chunk_offset, | |
| length: aligned_size as u32, | |
| offset: payload_offset, | |
| } | |
| } | |
| } | |
| fn deliver_chunk( | |
| &mut self, | |
| size: usize, | |
| trace_flow_id: u64, | |
| ) -> Result<(), ChunkedArchiveError> { | |
| let offset = self.read_range.start + self.delivered_len as u64; | |
| let page_size = zx::system_get_page_size() as usize; | |
| let aligned_size = size.next_multiple_of(page_size); | |
| let payload = self.payload.as_mut().expect("prepare must be called before commit"); | |
| if aligned_size > size { | |
| payload.data_mut().subslice_mut(size..aligned_size).fill(0); | |
| } | |
| let payload_offset = payload.offset(); | |
| let cmd = Self::make_delivery_command( | |
| self.child_target.as_ref(), | |
| self.key, | |
| offset, | |
| aligned_size, | |
| payload_offset, | |
| ); | |
| payload | |
| .commit_prefix( | |
| aligned_size, | |
| RawDeliveryCommand::from(cmd).with_trace_flow_id(trace_flow_id), | |
| ) | |
| .map_err(|_| ChunkedArchiveError::IntegrityError)?; | |
| self.delivered_len += size; | |
| Ok(()) | |
| } | |
| } | |
| impl DataBuffer for Buffer { | |
| fn range(&self) -> Range<u64> { | |
| self.read_range.clone() | |
| } | |
| /// Returns a mutable slice into the mapped transfer buffer. | |
| /// | |
| /// # Panics | |
| /// | |
| /// Panics if `prepare()` has not been called prior to accessing this method. | |
| fn mut_ptr_slice(&mut self) -> MutPtrByteSlice<'_> { | |
| let payload = | |
| self.payload.as_mut().expect("prepare must be called before accessing mut_ptr_slice"); | |
| // `payload.commit_prefix()` in `deliver_chunk()` already advances `payload` past | |
| // `self.delivered_len`, so slice relative to the remaining uncommitted payload. | |
| let start = self.committed_len - self.delivered_len; | |
| let remaining = self.total_len - self.committed_len; | |
| payload.data_mut().subslice_mut(start..start + remaining) | |
| } | |
| fn commit(&mut self, size: usize) -> Result<(), ChunkedArchiveError> { | |
| debug_assert!(self.committed_len + size <= self.total_len); | |
| self.committed_len += size; | |
| let undelivered = self.committed_len - self.delivered_len; | |
| if self.committed_len < self.total_len { | |
| // Once at least `self.flush_threshold` (32 KiB, rounded to a multiple of | |
| // `read_alignment`) has been committed, deliver complete `read_alignment` chunks so | |
| // verification overlaps with decompressing the remainder of the readahead window. | |
| if undelivered >= self.flush_threshold { | |
| let to_deliver = (undelivered / self.read_alignment) * self.read_alignment; | |
| self.deliver_chunk(to_deliver, 0)?; | |
| } | |
| } else if undelivered > 0 { | |
| // Deliver any remaining bytes once the entire prepared range has been committed, | |
| // attaching the page-fault flow ID to the final chunk. | |
| let trace_flow_id = std::mem::take(&mut self.trace_flow_id); | |
| self.deliver_chunk(undelivered, trace_flow_id)?; | |
| } | |
| Ok(()) | |
| } | |
| } | |
| impl PageRequest for Buffer { | |
| fn prepare( | |
| &mut self, | |
| read_range: Range<u64>, | |
| read_alignment: usize, | |
| ) -> Result<(), ChunkedArchiveError> { | |
| assert!(self.payload.is_none(), "prepare must only be called once"); | |
| assert!(read_alignment > 0, "read_alignment must be non-zero"); | |
| self.read_range = read_range; | |
| self.read_alignment = read_alignment; | |
| let total_len = (self.read_range.end - self.read_range.start) as usize; | |
| self.total_len = total_len; | |
| let page_size = zx::system_get_page_size() as usize; | |
| // Intermediate prefix flushes via `OwnedPayloadBuffer::commit_prefix` and | |
| // `DeliveryCommand::Data` must be page-aligned; only the final chunk of a request may have | |
| // an unaligned tail that gets zero-padded to a page boundary. | |
| self.flush_threshold = if read_alignment.is_multiple_of(page_size) { | |
| VERIFICATION_FLUSH_THRESHOLD.next_multiple_of(read_alignment) | |
| } else { | |
| usize::MAX | |
| }; | |
| self.payload = Some(self.verifier.reserve_payload(total_len)?); | |
| Ok(()) | |
| } | |
| fn precommit(&self) -> PagePrecommit { | |
| match &self.payload { | |
| Some(payload) => PagePrecommit::new( | |
| payload.vaddr(), | |
| payload.remaining_len(), | |
| Arc::clone(payload.sender()) as Arc<dyn Send + Sync>, | |
| ), | |
| None => PagePrecommit::default(), | |
| } | |
| } | |
| fn with_trace_flow_id(mut self, trace_flow_id: u64) -> Self { | |
| self.trace_flow_id = trace_flow_id; | |
| self | |
| } | |
| fn trace_flow_id(&self) -> u64 { | |
| self.trace_flow_id | |
| } | |
| fn forwards_trace_flow(&self) -> bool { | |
| true | |
| } | |
| fn split_at(&mut self, mid: u64) -> Self { | |
| assert!(self.payload.is_none(), "split_at must be called before prepare"); | |
| assert!(mid > self.read_range.start && mid < self.read_range.end); | |
| let head_range = self.read_range.start..mid; | |
| let head_child_target = self.child_target.as_mut().map(|target| { | |
| let head_target = ChildTarget { | |
| child_target_offset: target.child_target_offset, | |
| parent_byte_range: head_range.clone(), | |
| }; | |
| target.child_target_offset += mid - head_range.start; | |
| target.parent_byte_range.start = mid; | |
| head_target | |
| }); | |
| self.read_range.start = mid; | |
| Buffer { | |
| verifier: Arc::clone(&self.verifier), | |
| key: self.key, | |
| read_range: head_range, | |
| read_alignment: 0, | |
| flush_threshold: 0, | |
| total_len: 0, | |
| committed_len: 0, | |
| delivered_len: 0, | |
| payload: None, | |
| child_target: head_child_target, | |
| trace_flow_id: 0, | |
| } | |
| } | |
| } | |
| #[cfg(test)] | |
| mod tests { | |
| use super::*; | |
| use mapping::{DELIVERY_DATA_COMMAND, TARGET_READ_AHEAD_SIZE}; | |
| use test_case::test_case; | |
| #[fuchsia::test] | |
| fn test_verifier_buffer_incremental_commit() { | |
| let delivery_queue = zx::Vmo::create(65536).unwrap(); | |
| let mut receiver = vmo_fifo::Receiver::<RawDeliveryCommand>::new( | |
| delivery_queue.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(), | |
| PENDING_DELIVERY_COMMANDS_CAPACITY, | |
| ) | |
| .unwrap(); | |
| let verifier = Arc::new(Verifier::new(delivery_queue).unwrap()); | |
| let key = 42u64; | |
| let mut request = verifier.get_page_request(key, 0..8192); | |
| request.prepare(0..8192, TARGET_READ_AHEAD_SIZE as usize).expect("prepare"); | |
| // Fill first page (4096 bytes) with 0xAA | |
| let page1_data = vec![0xAAu8; 4096]; | |
| request.mut_ptr_slice().subslice_mut(0..4096).copy_from_slice(&page1_data); | |
| request.commit(4096).expect("commit page 1"); | |
| // Verify mut_ptr_slice advanced to second page | |
| assert_eq!(request.mut_ptr_slice().len(), 4096); | |
| // Not all data has been committed yet, so no command should be delivered yet. | |
| assert!(receiver.is_empty()); | |
| // Fill second page (4096 bytes) with 0xBB | |
| let page2_data = vec![0xBBu8; 4096]; | |
| request.mut_ptr_slice().subslice_mut(0..4096).copy_from_slice(&page2_data); | |
| request.commit(4096).expect("commit page 2"); | |
| // Verify remaining space is 0 | |
| assert_eq!(request.mut_ptr_slice().len(), 0); | |
| // Check single consolidated message delivered on queue | |
| let msg = receiver.peek().expect("msg"); | |
| assert_eq!(msg.opcode, DELIVERY_DATA_COMMAND); | |
| assert_eq!(msg.key, key); | |
| assert_eq!(msg.length, 8192); | |
| assert_eq!(msg.target_offset, 0); | |
| let mut buf = vec![0u8; 8192]; | |
| msg.payload_slice(msg.offset, msg.length).copy_to_slice(&mut buf); | |
| assert_eq!(&buf[..4096], &page1_data[..]); | |
| assert_eq!(&buf[4096..], &page2_data[..]); | |
| msg.pop().expect("pop msg"); | |
| } | |
| #[fuchsia::test] | |
| fn test_verifier_buffer_page_aligned_commits() { | |
| let delivery_queue = zx::Vmo::create(65536).unwrap(); | |
| let mut receiver = vmo_fifo::Receiver::<RawDeliveryCommand>::new( | |
| delivery_queue.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(), | |
| PENDING_DELIVERY_COMMANDS_CAPACITY, | |
| ) | |
| .unwrap(); | |
| let verifier = Arc::new(Verifier::new(delivery_queue).unwrap()); | |
| let key = 43u64; | |
| let mut request = verifier.get_page_request(key, 0..5000); | |
| request.prepare(0..5000, TARGET_READ_AHEAD_SIZE as usize).expect("prepare"); | |
| let data = vec![0xCCu8; 5000]; | |
| // Commit first 4096 bytes | |
| request.mut_ptr_slice().subslice_mut(0..4096).copy_from_slice(&data[..4096]); | |
| request.commit(4096).expect("commit 4096 bytes"); | |
| assert!(receiver.is_empty()); | |
| // Commit remaining 904 bytes (completes the prepared range) | |
| request.mut_ptr_slice().subslice_mut(0..904).copy_from_slice(&data[4096..5000]); | |
| request.commit(904).expect("commit 904 bytes"); | |
| // Check single message delivered, page-aligned to 8192 | |
| let msg = receiver.peek().expect("msg"); | |
| assert_eq!(msg.target_offset, 0); | |
| assert_eq!(msg.length, 8192); | |
| let mut buf = vec![0u8; 8192]; | |
| msg.payload_slice(msg.offset, msg.length).copy_to_slice(&mut buf); | |
| assert_eq!(&buf[..5000], &data[..]); | |
| assert_eq!(&buf[5000..], &[0u8; 3192]); | |
| msg.pop().expect("pop msg"); | |
| } | |
| #[test_case(32 * 1024, 128 * 1024; "zstd_32k_read_128k")] | |
| #[test_case(48 * 1024, 96 * 1024; "zstd_48k_read_96k")] | |
| #[test_case(152 * 1024, 152 * 1024; "zstd_152k_read_152k")] | |
| #[fuchsia::test] | |
| fn test_verifier_buffer_delivery_data_size_chunking( | |
| zstd_chunk_size: usize, | |
| read_alignment: usize, | |
| ) { | |
| let total_size = read_alignment * 2; | |
| let delivery_queue = | |
| zx::Vmo::create(total_size.next_power_of_two() as u64 + 65536).unwrap(); | |
| let mut receiver = vmo_fifo::Receiver::<RawDeliveryCommand>::new( | |
| delivery_queue.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(), | |
| PENDING_DELIVERY_COMMANDS_CAPACITY, | |
| ) | |
| .unwrap(); | |
| let verifier = Arc::new(Verifier::new(delivery_queue).unwrap()); | |
| let key = 44u64; | |
| let mut request = verifier.get_page_request(key, 0..total_size as u64); | |
| request.prepare(0..total_size as u64, read_alignment).expect("prepare"); | |
| let chunk = vec![0x55u8; zstd_chunk_size]; | |
| let chunks_per_read = read_alignment / zstd_chunk_size; | |
| // Push `chunks_per_read - 1` chunks - less than `read_alignment` | |
| for _ in 0..(chunks_per_read - 1) { | |
| request.mut_ptr_slice().subslice_mut(0..zstd_chunk_size).copy_from_slice(&chunk); | |
| request.commit(zstd_chunk_size).expect("commit"); | |
| } | |
| assert!(receiver.is_empty()); | |
| // Push the final chunk of the first `read_alignment` window | |
| request.mut_ptr_slice().subslice_mut(0..zstd_chunk_size).copy_from_slice(&chunk); | |
| request.commit(zstd_chunk_size).expect("commit"); | |
| // 1st `read_alignment` message delivered | |
| let msg1 = receiver.peek().expect("msg1"); | |
| assert_eq!(msg1.target_offset, 0); | |
| assert_eq!(msg1.length, read_alignment as u32); | |
| msg1.pop().expect("pop msg1"); | |
| // Push remaining `chunks_per_read` chunks (second `read_alignment` window) | |
| for _ in 0..chunks_per_read { | |
| request.mut_ptr_slice().subslice_mut(0..zstd_chunk_size).copy_from_slice(&chunk); | |
| request.commit(zstd_chunk_size).expect("commit"); | |
| } | |
| // 2nd `read_alignment` message delivered | |
| let msg2 = receiver.peek().expect("msg2"); | |
| assert_eq!(msg2.target_offset, read_alignment as u64); | |
| assert_eq!(msg2.length, read_alignment as u32); | |
| msg2.pop().expect("pop msg2"); | |
| } | |
| #[fuchsia::test] | |
| #[should_panic(expected = "prepare must only be called once")] | |
| fn test_verifier_buffer_prepare_multiple_times_panics() { | |
| let delivery_queue = zx::Vmo::create(65536).unwrap(); | |
| let verifier = Arc::new(Verifier::new(delivery_queue).unwrap()); | |
| let key = 46u64; | |
| let mut request = verifier.get_page_request(key, 0..8192); | |
| request.prepare(0..4096, TARGET_READ_AHEAD_SIZE as usize).expect("first prepare"); | |
| let _ = request.prepare(0..8192, TARGET_READ_AHEAD_SIZE as usize); | |
| } | |
| #[fuchsia::test] | |
| fn test_verifier_buffer_supplies_pages_via_delivery_receiver() { | |
| let pager = Arc::new(zx::Pager::create(zx::PagerOptions::empty()).unwrap()); | |
| let port = zx::Port::create(); | |
| let key = 44u64; | |
| let paged_vmo = pager.create_vmo(zx::VmoOptions::empty(), &port, key, 8192).unwrap(); | |
| let delivery_queue = zx::Vmo::create(65536).unwrap(); | |
| let vmo_provider = Arc::new(blob_pager_and_verifier::TestVmoProvider::new( | |
| pager.clone(), | |
| delivery_queue.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(), | |
| )); | |
| vmo_provider | |
| .register_vmo(key, paged_vmo.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap()); | |
| let receiver = vmo_fifo::Receiver::<RawDeliveryCommand>::new( | |
| delivery_queue.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(), | |
| PENDING_DELIVERY_COMMANDS_CAPACITY, | |
| ) | |
| .unwrap(); | |
| let _processor = blob_pager_and_verifier::DeliveryQueueProcessor::spawn( | |
| receiver, | |
| vmo_provider, | |
| delivery_queue.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(), | |
| ) | |
| .unwrap(); | |
| let verifier = Arc::new(Verifier::new(delivery_queue).unwrap()); | |
| let mut request = verifier.get_page_request(key, 0..8192); | |
| request.prepare(0..8192, TARGET_READ_AHEAD_SIZE as usize).expect("prepare"); | |
| let page1_data = vec![0xAAu8; 4096]; | |
| request.mut_ptr_slice().subslice_mut(0..4096).copy_from_slice(&page1_data); | |
| request.commit(4096).expect("commit page 1"); | |
| let page2_data = vec![0xBBu8; 4096]; | |
| request.mut_ptr_slice().subslice_mut(0..4096).copy_from_slice(&page2_data); | |
| request.commit(4096).expect("commit page 2"); | |
| let mut read_buf = vec![0u8; 8192]; | |
| paged_vmo.read(&mut read_buf, 0).expect("read paged_vmo"); | |
| assert_eq!(&read_buf[..4096], &page1_data[..]); | |
| assert_eq!(&read_buf[4096..8192], &page2_data[..]); | |
| } | |
| #[fuchsia::test] | |
| fn test_verifier_register_blob() { | |
| let delivery_queue = zx::Vmo::create(65536).unwrap(); | |
| let mut receiver = vmo_fifo::Receiver::<RawDeliveryCommand>::new( | |
| delivery_queue.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(), | |
| PENDING_DELIVERY_COMMANDS_CAPACITY, | |
| ) | |
| .unwrap(); | |
| let verifier = Arc::new(Verifier::new(delivery_queue).unwrap()); | |
| let key = 42u64; | |
| let leaves = [[0xABu8; 32], [0xCDu8; 32]]; | |
| verifier.register_blob(key, &leaves, 96 * 1024).expect("register_blob failed"); | |
| let msg = receiver.peek().expect("peek msg"); | |
| assert_eq!(msg.opcode, mapping::DELIVERY_REGISTER_BLOB_COMMAND); | |
| assert_eq!(msg.key, key); | |
| assert_eq!(msg.length, 64); | |
| assert_eq!(msg.read_alignment, 96 * 1024); | |
| let mut buf = vec![0u8; 64]; | |
| msg.payload_slice(msg.offset, msg.length).copy_to_slice(&mut buf); | |
| assert_eq!(&buf[..32], &[0xABu8; 32]); | |
| assert_eq!(&buf[32..], &[0xCDu8; 32]); | |
| msg.pop().expect("pop msg failed"); | |
| } | |
| #[fuchsia::test] | |
| fn test_verifier_buffer_32k_overlap() { | |
| let total_size = 128 * 1024; | |
| let sub_chunk_size = 16 * 1024; | |
| let flush_size = 32 * 1024; | |
| let delivery_queue = zx::Vmo::create(total_size as u64 * 2 + 65536).unwrap(); | |
| let mut receiver = vmo_fifo::Receiver::<RawDeliveryCommand>::new( | |
| delivery_queue.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(), | |
| PENDING_DELIVERY_COMMANDS_CAPACITY, | |
| ) | |
| .unwrap(); | |
| let verifier = Arc::new(Verifier::new(delivery_queue).unwrap()); | |
| let key = 45u64; | |
| let mut request = verifier.get_page_request(key, 0..total_size as u64); | |
| // With `read_alignment == 16 KiB`, `flush_threshold` rounds up to 32 KiB | |
| // (`VERIFICATION_FLUSH_THRESHOLD`). | |
| request.prepare(0..total_size as u64, sub_chunk_size).expect("prepare"); | |
| let sub_chunk = vec![0x55u8; sub_chunk_size]; | |
| let mut base_payload_offset = None; | |
| for i in 0..4 { | |
| // First 16 KiB commit is below the 32 KiB flush threshold, so nothing is flushed yet. | |
| request.mut_ptr_slice().subslice_mut(0..sub_chunk_size).copy_from_slice(&sub_chunk); | |
| request.commit(sub_chunk_size).expect("commit first 16 KiB"); | |
| assert!(receiver.is_empty()); | |
| // Second 16 KiB commit reaches the 32 KiB flush threshold and flushes a 32 KiB chunk. | |
| request.mut_ptr_slice().subslice_mut(0..sub_chunk_size).copy_from_slice(&sub_chunk); | |
| request.commit(sub_chunk_size).expect("commit second 16 KiB"); | |
| let msg = receiver.peek().expect("msg"); | |
| let base = *base_payload_offset.get_or_insert(msg.offset); | |
| assert_eq!(msg.offset, base + (i * flush_size) as u32); | |
| assert_eq!(msg.target_offset, (i * flush_size) as u64); | |
| assert_eq!(msg.length, flush_size as u32); | |
| msg.pop().expect("pop msg"); | |
| } | |
| } | |
| #[fuchsia::test] | |
| fn test_verifier_concurrent_out_of_order_buffers_and_precommit() { | |
| let page = zx::system_get_page_size() as usize; | |
| // 13 pages header/slots (53,248 bytes) + 8 pages payload (two 4-page buddy blocks). | |
| let delivery_queue = zx::Vmo::create((21 * page) as u64).unwrap(); | |
| let mut receiver = vmo_fifo::Receiver::<RawDeliveryCommand>::new( | |
| delivery_queue.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(), | |
| PENDING_DELIVERY_COMMANDS_CAPACITY, | |
| ) | |
| .unwrap(); | |
| let verifier = Arc::new(Verifier::new(delivery_queue).unwrap()); | |
| // Prepare two concurrent 3-page buffers (consuming both 4-page buddy blocks) and precommit | |
| // both. | |
| let mut req_a = verifier.get_page_request(100, 0..(3 * page) as u64); | |
| let mut req_b = verifier.get_page_request(200, 0..(3 * page) as u64); | |
| req_a.prepare(0..(3 * page) as u64, 3 * page).expect("prepare req_a"); | |
| req_b.prepare(0..(3 * page) as u64, 3 * page).expect("prepare req_b"); | |
| req_a.precommit().run(); | |
| req_b.precommit().run(); | |
| // Complete and retire `req_b` BEFORE `req_a` even writes its data. | |
| req_b.mut_ptr_slice().fill(0xBB); | |
| req_b.commit(3 * page).expect("commit req_b"); | |
| let msg_b = receiver.peek().expect("peek msg_b"); | |
| assert_eq!(msg_b.key, 200); | |
| assert_eq!(msg_b.length, (3 * page) as u32); | |
| msg_b.pop().expect("pop msg_b"); | |
| // Prepare `req_c` in the freed 3 pages, write to it, then drop `req_c` uncommitted. | |
| let mut req_c = verifier.get_page_request(300, 0..(3 * page) as u64); | |
| req_c.prepare(0..(3 * page) as u64, 3 * page).expect("prepare req_c"); | |
| req_c.mut_ptr_slice().fill(0xCC); | |
| drop(req_c); | |
| // `req_a`'s reservation must remain untouched; write and commit `req_a`. | |
| req_a.mut_ptr_slice().fill(0xAA); | |
| req_a.commit(3 * page).expect("commit req_a"); | |
| let msg_a = receiver.peek().expect("peek msg_a"); | |
| assert_eq!(msg_a.key, 100); | |
| assert_eq!(msg_a.length, (3 * page) as u32); | |
| let mut buf_a = vec![0u8; 3 * page]; | |
| msg_a.payload_slice(msg_a.offset, msg_a.length).copy_to_slice(&mut buf_a); | |
| assert_eq!(buf_a, vec![0xAA; 3 * page]); | |
| msg_a.pop().expect("pop msg_a"); | |
| } | |
| #[fuchsia::test] | |
| fn test_verifier_headroom_reserves_space_for_register_blob() { | |
| let delivery_queue = zx::Vmo::create(mapping::DELIVERY_VMO_SIZE).unwrap(); | |
| let mut receiver = vmo_fifo::Receiver::<RawDeliveryCommand>::new( | |
| delivery_queue.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(), | |
| PENDING_DELIVERY_COMMANDS_CAPACITY, | |
| ) | |
| .unwrap(); | |
| let verifier = Arc::new(Verifier::new(delivery_queue).unwrap()); | |
| assert_eq!(verifier.headroom, REGISTER_BLOB_HEADROOM_BYTES); | |
| // Reserve a full 1 MiB page request while holding the reservation open, and verify that | |
| // `register_blob` still succeeds using the reserved 512 KiB headroom. | |
| let len = 1024 * 1024usize; | |
| let read_alignment = 128 * 1024usize; | |
| let mut req = verifier.get_page_request(601, 0..len as u64); | |
| req.prepare(0..len as u64, read_alignment).expect("prepare 1 MiB request"); | |
| req.precommit().run(); | |
| let leaves = [[0xABu8; 32]; 4]; | |
| verifier.register_blob(602, &leaves, read_alignment).expect("register_blob with headroom"); | |
| let reg_msg = receiver.peek().expect("peek register_blob"); | |
| assert_eq!(reg_msg.opcode, mapping::DELIVERY_REGISTER_BLOB_COMMAND); | |
| assert_eq!(reg_msg.key, 602); | |
| reg_msg.pop().expect("pop register_blob"); | |
| req.mut_ptr_slice().fill(0x7A); | |
| req.commit(len).expect("commit 1 MiB request"); | |
| let data_msg = receiver.peek().expect("peek 1 MiB data"); | |
| assert_eq!(data_msg.opcode, DELIVERY_DATA_COMMAND); | |
| assert_eq!(data_msg.length, len as u32); | |
| data_msg.pop().expect("pop 1 MiB data"); | |
| } | |
| #[fuchsia::test] | |
| fn test_verifier_child_page_request_sparse_and_inline_tail() { | |
| let delivery_queue = zx::Vmo::create(128 * 1024).unwrap(); | |
| let mut receiver = vmo_fifo::Receiver::<RawDeliveryCommand>::new( | |
| delivery_queue.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(), | |
| PENDING_DELIVERY_COMMANDS_CAPACITY, | |
| ) | |
| .unwrap(); | |
| let verifier = Arc::new(Verifier::new(delivery_queue).unwrap()); | |
| let child_key = 700u64; | |
| // 1. Sparse child delivery (`length == 0`). | |
| verifier.deliver_child_sparse(child_key, 4096, 8192, 1001).expect("deliver_child_sparse"); | |
| let msg_sparse = receiver.peek().expect("peek sparse"); | |
| assert_eq!(msg_sparse.trace_flow_id, 1001); | |
| assert_eq!( | |
| DeliveryCommand::try_from(*msg_sparse).unwrap(), | |
| DeliveryCommand::ChildData { | |
| parent_offset: 0, | |
| offset: 0, | |
| length: 0, | |
| child_key, | |
| child_target_offset: 4096, | |
| in_chunk_offset: 0, | |
| child_length: 8192, | |
| } | |
| ); | |
| msg_sparse.pop().expect("pop sparse"); | |
| // 2. Child inline tail delivery. | |
| verifier.deliver_child_inline_tail(16384, child_key, 12288, 512, 1002); | |
| let msg_tail = receiver.peek().expect("peek inline tail"); | |
| assert_eq!(msg_tail.trace_flow_id, 1002); | |
| assert_eq!( | |
| DeliveryCommand::try_from(*msg_tail).unwrap(), | |
| DeliveryCommand::ChildInlineTail { | |
| parent_byte_offset: 16384, | |
| child_key, | |
| child_target_offset: 12288, | |
| tail_length: 512, | |
| } | |
| ); | |
| msg_tail.pop().expect("pop inline tail"); | |
| // 3. Child page request spanning 8192..45056 in parent prepared over 0..65536 (32 KiB | |
| // alignment): intermediate 32 KiB flush carries `trace_flow_id == 0`, and the final chunk | |
| // carries the page-fault flow ID (`1003`). | |
| let mut child_req = | |
| verifier.get_child_page_request(child_key, 4096, 8192..45056).with_trace_flow_id(1003); | |
| child_req.prepare(0..65536, 32768).expect("prepare child_req"); | |
| child_req.mut_ptr_slice().subslice_mut(0..32768).fill(0xEE); | |
| child_req.commit(32768).expect("commit child_req chunk 0"); | |
| child_req.mut_ptr_slice().fill(0xEE); | |
| child_req.commit(32768).expect("commit child_req chunk 1"); | |
| let msg_child_0 = receiver.peek().expect("peek child data chunk 0"); | |
| assert_eq!(msg_child_0.trace_flow_id, 0); | |
| match DeliveryCommand::try_from(*msg_child_0).unwrap() { | |
| DeliveryCommand::ChildData { | |
| parent_offset, | |
| length, | |
| child_key: k, | |
| child_target_offset, | |
| in_chunk_offset, | |
| child_length, | |
| .. | |
| } => { | |
| assert_eq!(parent_offset, 0); | |
| assert_eq!(length, 32768); | |
| assert_eq!(k, child_key); | |
| assert_eq!(child_target_offset, 4096); | |
| assert_eq!(in_chunk_offset, 8192); | |
| assert_eq!(child_length, 32768 - 8192); | |
| } | |
| other => panic!("unexpected command: {other:?}"), | |
| } | |
| msg_child_0.pop().expect("pop child data chunk 0"); | |
| let msg_child_1 = receiver.peek().expect("peek child data chunk 1"); | |
| assert_eq!(msg_child_1.trace_flow_id, 1003); | |
| match DeliveryCommand::try_from(*msg_child_1).unwrap() { | |
| DeliveryCommand::ChildData { | |
| parent_offset, | |
| length, | |
| child_key: k, | |
| child_target_offset, | |
| in_chunk_offset, | |
| child_length, | |
| .. | |
| } => { | |
| assert_eq!(parent_offset, 32768); | |
| assert_eq!(length, 32768); | |
| assert_eq!(k, child_key); | |
| assert_eq!(child_target_offset, 4096 + (32768 - 8192)); | |
| assert_eq!(in_chunk_offset, 0); | |
| assert_eq!(child_length, 45056 - 32768); | |
| } | |
| other => panic!("unexpected command: {other:?}"), | |
| } | |
| msg_child_1.pop().expect("pop child data chunk 1"); | |
| } | |
| #[fuchsia::test] | |
| fn test_verifier_new_invalid_vmo_returns_error() { | |
| assert_eq!( | |
| Verifier::new(zx::Vmo::create(0).unwrap()).unwrap_err(), | |
| zx::Status::INVALID_ARGS | |
| ); | |
| } | |
| } |