| // Copyright 2021 The Fuchsia Authors | |
| // | |
| // Use of this source code is governed by a MIT-style | |
| // license that can be found in the LICENSE file or at | |
| // https://opensource.org/licenses/MIT | |
| use core::cell::UnsafeCell; | |
| #[cfg(console_enabled)] | |
| use core::ffi::c_int; | |
| use core::ffi::{c_char, c_void}; | |
| use core::pin::Pin; | |
| use core::ptr::NonNull; | |
| use crate::ksync_kernel::IrqSavePolicy; | |
| #[cfg(console_enabled)] | |
| use kprint::{kprint, kprintln}; | |
| #[cfg(console_enabled)] | |
| use zx_status::Status; | |
| unsafe extern "C" { | |
| fn arch_clean_cache_range(start: usize, len: usize); | |
| } | |
| /// Persistent RAM allocation granularity configured for the kernel build. | |
| pub const PERSISTENT_RAM_ALLOCATION_GRANULARITY: usize = | |
| zr::parse_usize(env!("PERSISTENT_RAM_ALLOCATION_GRANULARITY")) | |
| .expect("PERSISTENT_RAM_ALLOCATION_GRANULARITY invalid"); | |
| // The allocation granularity of persistent RAM must be a power of two greater than 0. | |
| zr::static_assert!(PERSISTENT_RAM_ALLOCATION_GRANULARITY.is_power_of_two()); | |
| /// The amount of persistent RAM that persistent debug logging would like to have. | |
| pub const TARGET_PERSISTENT_DEBUGLOG_SIZE: usize = | |
| zr::parse_usize(env!("TARGET_PERSISTENT_DEBUGLOG_SIZE")) | |
| .expect("TARGET_PERSISTENT_DEBUGLOG_SIZE invalid"); | |
| // The persistent debuglog target size must be a multiple of the persistent RAM allocation | |
| // granularity, and fit within u32::MAX. | |
| zr::static_assert!( | |
| TARGET_PERSISTENT_DEBUGLOG_SIZE.is_multiple_of(PERSISTENT_RAM_ALLOCATION_GRANULARITY) | |
| ); | |
| zr::static_assert!(TARGET_PERSISTENT_DEBUGLOG_SIZE <= u32::MAX as usize); | |
| /// Flushes CPU data cache lines covering `bytes` to main memory so that | |
| /// persistent debuglog writes survive non-power-cycle warm resets. | |
| #[inline] | |
| fn clean_cache_range(bytes: &[u8]) { | |
| // SAFETY: `arch_clean_cache_range` flushes CPU data cache lines covering | |
| // `[addr, addr + len)` to main memory and does not dereference or mutate | |
| // Rust references. | |
| unsafe { | |
| arch_clean_cache_range(bytes.as_ptr().addr(), bytes.len()); | |
| } | |
| } | |
| /// On-RAM header at the start of the persistent debuglog region. | |
| #[repr(C, packed)] | |
| #[derive(Copy, Clone, Debug, PartialEq, Eq)] | |
| struct LogHeader { | |
| magic: u32, | |
| rd_ptr: u32, | |
| } | |
| zr::static_assert!( | |
| core::mem::size_of::<LogHeader>() == 8 && core::mem::align_of::<LogHeader>() == 1 | |
| ); | |
| // Persistent dlog header's alignment must be a power of 2 in size | |
| zr::static_assert!(core::mem::align_of::<LogHeader>().is_power_of_two()); | |
| impl LogHeader { | |
| /// FourCC `"Plog"` in little-endian byte order (`0x676f_6c50`). | |
| const MAGIC: u32 = u32::from_le_bytes(*b"Plog"); | |
| #[inline] | |
| fn as_bytes(&self) -> &[u8] { | |
| // SAFETY: `LogHeader` is `#[repr(C, packed)]` with two `u32` fields and no padding, | |
| // so all `size_of::<Self>()` bytes are initialized and valid to view as `&[u8]`. | |
| unsafe { | |
| core::slice::from_raw_parts( | |
| core::ptr::from_ref(self).cast::<u8>(), | |
| core::mem::size_of::<Self>(), | |
| ) | |
| } | |
| } | |
| #[inline] | |
| fn is_magic_valid(&self) -> bool { | |
| self.magic == Self::MAGIC | |
| } | |
| #[inline] | |
| fn validate_magic(&mut self) { | |
| self.magic = Self::MAGIC; | |
| clean_cache_range(self.as_bytes()); | |
| } | |
| #[inline] | |
| fn invalidate_magic(&mut self) { | |
| self.magic = 0; | |
| clean_cache_range(self.as_bytes()); | |
| } | |
| } | |
| /// Info about where the persisted log is in RAM, if we have a persisted log. | |
| /// Laid out to match the 16-byte C++ `plog` representation when wrapped in | |
| /// `Option<ActivePersistentLog>`. | |
| #[repr(C)] | |
| #[derive(Copy, Clone, Debug, PartialEq, Eq)] | |
| struct ActivePersistentLog { | |
| hdr: NonNull<LogHeader>, | |
| payload_size: u32, | |
| } | |
| zr::static_assert!( | |
| core::mem::size_of::<ActivePersistentLog>() == 16 | |
| && core::mem::align_of::<ActivePersistentLog>() == 8 | |
| ); | |
| zr::static_assert!( | |
| core::mem::size_of::<Option<ActivePersistentLog>>() == 16 | |
| && core::mem::align_of::<Option<ActivePersistentLog>>() == 8 | |
| ); | |
| // SAFETY: `ActivePersistentLog` encapsulates a persistent RAM mapping owned and | |
| // synchronized by `PersistentDebugLog`'s spinlock. | |
| unsafe impl Send for ActivePersistentLog {} | |
| impl ActivePersistentLog { | |
| /// Creates a new `ActivePersistentLog` over a persistent RAM allocation. | |
| /// | |
| /// # Safety | |
| /// `hdr` must point to a valid, readable and writable `LogHeader` followed immediately by | |
| /// `payload_size` readable and writable bytes within the same persistent RAM allocation, | |
| /// and all accesses through this `ActivePersistentLog` must be synchronized by | |
| /// `persistent_log_lock`. | |
| const unsafe fn new(hdr: NonNull<LogHeader>, payload_size: u32) -> Self { | |
| Self { hdr, payload_size } | |
| } | |
| /// Returns a shared reference to the `LogHeader`. | |
| #[inline] | |
| fn header(&self) -> &LogHeader { | |
| // SAFETY: `ActivePersistentLog::new` guarantees `self.hdr` is valid and non-aliased | |
| // mutably while `self` is borrowed. | |
| unsafe { self.hdr.as_ref() } | |
| } | |
| /// Returns an exclusive reference to the `LogHeader`. | |
| #[inline] | |
| fn header_mut(&mut self) -> &mut LogHeader { | |
| // SAFETY: `ActivePersistentLog::new` guarantees `self.hdr` is valid and exclusively | |
| // accessed while `self` is mutably borrowed. | |
| unsafe { self.hdr.as_mut() } | |
| } | |
| /// Returns a shared slice of the payload bytes immediately following `self.hdr`. | |
| #[inline] | |
| fn payload(&self) -> &[u8] { | |
| // SAFETY: `ActivePersistentLog::new` guarantees `self.hdr` is followed immediately by | |
| // `self.payload_size` valid bytes within the same allocation. | |
| unsafe { | |
| let payload_ptr = self.hdr.as_ptr().add(1).cast::<u8>().cast_const(); | |
| core::slice::from_raw_parts(payload_ptr, self.payload_size as usize) | |
| } | |
| } | |
| /// Returns an exclusive slice of the payload bytes immediately following `self.hdr`. | |
| #[inline] | |
| fn payload_mut(&mut self) -> &mut [u8] { | |
| // SAFETY: `ActivePersistentLog::new` guarantees `self.hdr` is followed immediately by | |
| // `self.payload_size` valid writable bytes within the same allocation. | |
| unsafe { | |
| let payload_ptr = self.hdr.as_ptr().add(1).cast::<u8>(); | |
| core::slice::from_raw_parts_mut(payload_ptr, self.payload_size as usize) | |
| } | |
| } | |
| /// Returns simultaneous exclusive references to the `LogHeader` and the disjoint payload | |
| /// byte slice immediately following it. | |
| #[inline] | |
| fn header_and_payload_mut(&mut self) -> (&mut LogHeader, &mut [u8]) { | |
| // SAFETY: `ActivePersistentLog::new` guarantees `self.hdr` points to a valid | |
| // `LogHeader` followed immediately by `self.payload_size` disjoint payload bytes in the | |
| // same allocation, and `&mut self` ensures exclusive access to both regions. | |
| unsafe { | |
| let hdr_ptr = self.hdr.as_ptr(); | |
| let payload_ptr = hdr_ptr.add(1).cast::<u8>(); | |
| ( | |
| &mut *hdr_ptr, | |
| core::slice::from_raw_parts_mut(payload_ptr, self.payload_size as usize), | |
| ) | |
| } | |
| } | |
| /// Invalidates the state of the persistent dlog. This gets called every time we | |
| /// gracefully reboot, so that we don't end up recovering a dlog after reboot and | |
| /// end up producing an unnecessary crashlog. | |
| #[inline] | |
| fn invalidate(&mut self) { | |
| self.header_mut().invalidate_magic(); | |
| } | |
| } | |
| /// We don't bother to lock this structure. The log, if present, is recovered | |
| /// during early boot while we are still running on a single core. After that, | |
| /// the recovered data is only ever accessed in a read only fashion, so there | |
| /// is no real need to provide any explicit synchronization. | |
| #[repr(C)] | |
| #[derive(Debug)] | |
| struct RecoveredPersistentLog { | |
| data: Option<NonNull<u8>>, | |
| capacity: u32, | |
| size: u32, | |
| } | |
| zr::static_assert!( | |
| core::mem::size_of::<RecoveredPersistentLog>() == 16 | |
| && core::mem::align_of::<RecoveredPersistentLog>() == 8 | |
| ); | |
| // SAFETY: `RecoveredPersistentLog` points to statically allocated recovery storage | |
| // populated during single-core early boot (or under test control) and read-only thereafter. | |
| unsafe impl Send for RecoveredPersistentLog {} | |
| // SAFETY: After single-core early boot initialization, `RecoveredPersistentLog` is accessed | |
| // in a read-only fashion. | |
| unsafe impl Sync for RecoveredPersistentLog {} | |
| impl RecoveredPersistentLog { | |
| const fn new(data: Option<NonNull<u8>>, capacity: u32) -> Self { | |
| Self { data, capacity, size: 0 } | |
| } | |
| /// Used only by testing to reset a log to its "pre SetLocation'ed" state. | |
| #[cfg(ktest)] | |
| #[inline] | |
| fn reset(&mut self) { | |
| self.size = 0; | |
| } | |
| /// Fetch a string view which references the recovered crashlog (if any). | |
| /// Returns `None` when `size == 0`. | |
| #[inline] | |
| fn as_bytes(&self) -> Option<&[u8]> { | |
| if self.size == 0 { | |
| return None; | |
| } | |
| let ptr = self.data?; | |
| // SAFETY: When `self.data` is `Some(ptr)`, it points to a backing store of at least | |
| // `self.capacity` bytes, and `self.size <= self.capacity` bytes have been initialized. | |
| Some(unsafe { core::slice::from_raw_parts(ptr.as_ptr().cast_const(), self.size as usize) }) | |
| } | |
| /// Recovers valid log contents from a circular persistent payload buffer starting at `rd_ptr`. | |
| /// | |
| /// # Safety | |
| /// If `self.data` is `Some(ptr)` and `self.capacity > 0`, `ptr` must point to a writable | |
| /// allocation of at least `self.capacity` bytes. | |
| unsafe fn recover_from(&mut self, payload: &[u8], rd_ptr: u32) { | |
| self.size = 0; | |
| let Some(destination_ptr) = self.data else { | |
| return; | |
| }; | |
| if self.capacity == 0 || payload.is_empty() { | |
| return; | |
| } | |
| // Save as much as we can in our static buffer, discarding the oldest data first if we | |
| // cannot fit it all. | |
| // C++ DEBUG_ASSERTs capacity <= u32::MAX here; capacity is a u32 so this holds by | |
| // construction. | |
| let payload_size = payload.len(); | |
| let capacity = self.capacity as usize; | |
| let (bytes_to_process, mut read_offset) = if payload_size <= capacity { | |
| (payload_size, (rd_ptr as usize) % payload_size) | |
| } else { | |
| (capacity, ((rd_ptr as usize) + (payload_size - capacity)) % payload_size) | |
| }; | |
| let mut write_offset: usize = 0; | |
| // SAFETY: Caller guarantees `destination_ptr` points to at least `self.capacity` | |
| // writable bytes. | |
| let destination = | |
| unsafe { core::slice::from_raw_parts_mut(destination_ptr.as_ptr(), capacity) }; | |
| for _ in 0..bytes_to_process { | |
| let byte = payload[read_offset]; | |
| read_offset += 1; | |
| if read_offset >= payload_size { | |
| read_offset = 0; | |
| } | |
| // Skip all zeros. | |
| if byte == 0 { | |
| continue; | |
| } | |
| destination[write_offset] = | |
| if byte.is_ascii_graphic() || byte == b' ' || byte == b'\n' { byte } else { b'?' }; | |
| write_offset += 1; | |
| } | |
| self.size = write_offset as u32; | |
| } | |
| } | |
| /// Core persistent debuglog state machine managing the spinlock-protected active persistent RAM | |
| /// ring buffer and the recovered previous-boot crashlog buffer. | |
| #[ksync::guarded] | |
| struct PersistentDebugLog { | |
| #[mutex] | |
| persistent_log_lock: ksync::KMutex<crate::ksync_kernel::TrackedSpinlock>, | |
| /// Info about where the persisted log is in RAM, if we have a persisted log. | |
| #[guarded_by(persistent_log_lock)] | |
| plog: Option<ActivePersistentLog>, | |
| /// We don't bother to lock this structure. The log, if present, is recovered | |
| /// during early boot while we are still running on a single core. After that, | |
| /// the recovered data is only ever accessed in a read only fashion, so there | |
| /// is no real need to provide any explicit synchronization. | |
| recovered_persistent_log: UnsafeCell<RecoveredPersistentLog>, | |
| } | |
| // SAFETY: `PersistentDebugLog` synchronizes all mutable access to `plog` and | |
| // `recovered_persistent_log` using `persistent_log_lock` (`IrqSavePolicy`), and | |
| // `recovered_persistent_log` is only read after single-core early boot initialization. | |
| unsafe impl Send for PersistentDebugLog {} | |
| // SAFETY: Shared references to `PersistentDebugLog` synchronize mutations via | |
| // `persistent_log_lock` (`IrqSavePolicy`) and only read `recovered_persistent_log` after | |
| // early boot recovery. | |
| unsafe impl Sync for PersistentDebugLog {} | |
| impl core::fmt::Debug for PersistentDebugLog { | |
| fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { | |
| f.debug_struct("PersistentDebugLog") | |
| .field("persistent_log_lock", &self.persistent_log_lock) | |
| .field("plog", &self.plog) | |
| .finish_non_exhaustive() | |
| } | |
| } | |
| impl PersistentDebugLog { | |
| /// Returns a pin-initializer for `PersistentDebugLog` backed by `recovered_data` of length | |
| /// `recovered_capacity`. | |
| fn init( | |
| recovered_data: Option<NonNull<u8>>, | |
| recovered_capacity: u32, | |
| ) -> impl pin_init::PinInit<Self, core::convert::Infallible> { | |
| pin_init::pin_init!(Self { | |
| persistent_log_lock <- ksync::KMutex::init(), | |
| plog: ksync::KCell::new(None), | |
| recovered_persistent_log: UnsafeCell::new(RecoveredPersistentLog::new( | |
| recovered_data, | |
| recovered_capacity, | |
| )), | |
| }) | |
| } | |
| /// Called very early in boot. Attempts to recover any previously persisted | |
| /// log by first performing consistency checks on the header, and then | |
| /// recovering as much as it can into the recovery buffer. | |
| /// | |
| /// # Safety | |
| /// If `virt` is non-null and `len > size_of::<LogHeader>()`, `virt` must point to a valid, | |
| /// readable and writable persistent RAM mapping of `len` bytes that remains valid while this | |
| /// `PersistentDebugLog` is active (or until `force_reset` is called in tests). | |
| unsafe fn set_location(&self, virt: *mut u8, len: usize) { | |
| // The location of the persistent dlog must be compatible with the header's | |
| // alignment, and the amount of space for the log payload must be positive, | |
| // otherwise we cannot effectively use the memory. | |
| let Some(virt_ptr) = NonNull::new(virt) else { | |
| return; | |
| }; | |
| let header_size = core::mem::size_of::<LogHeader>(); | |
| let header_align = core::mem::align_of::<LogHeader>(); | |
| if (virt_ptr.addr().get() & (header_align - 1)) != 0 || len <= header_size { | |
| return; | |
| } | |
| // If we already have a persistent dlog location, then this function has been | |
| // (improperly) called twice. It is tempting to assert here, but we are | |
| // currently very early in boot, which would make debugging the assert | |
| // extremely difficult. Instead, just ignore this request. | |
| // | |
| // TODO(johngro): come back here and try to put a warning/OOPS in to the dlog | |
| // buffer? | |
| ksync::lock!( | |
| let mut guard = self.lock_persistent_log_lock_policy::<IrqSavePolicy>() | |
| ); | |
| if guard.plog().is_some() { | |
| return; | |
| } | |
| // Check our log header to see if we have a recovered persistent log. If we | |
| // do, recover the log into our recovery buffer so that it can be picked up | |
| // later on and sent up to usermode in the crashlog. | |
| // | |
| // Note: we are at a point in boot where the heap has not been brought up yet, | |
| // which is why the recovery buffer needs to be statically allocated. If this | |
| // becomes an issue some day, we can shift to a strategy where we remember | |
| // where the persistent log is, but don't actually start to use it until we | |
| // get a chance to save it off into a dynamically allocated buffer. | |
| let header_ptr = virt_ptr.cast::<LogHeader>(); | |
| let payload_size = core::cmp::min(u32::MAX as usize, len - header_size) as u32; | |
| // SAFETY: Caller guarantees `virt` points to a valid persistent RAM buffer of `len` | |
| // bytes (`>= size_of::<LogHeader>() + payload_size`), and all accesses to `active_log` | |
| // are synchronized by `persistent_log_lock`. | |
| let mut active_log = unsafe { ActivePersistentLog::new(header_ptr, payload_size) }; | |
| let header = active_log.header(); | |
| let initial_rd_ptr = header.rd_ptr; | |
| if header.is_magic_valid() && initial_rd_ptr < payload_size { | |
| // This looks as good as it is going to. Our magic number is valid, and our | |
| // read pointer lies within the available payload size. Save as much as we | |
| // can in our static buffer, discarding the oldest data first if we cannot | |
| // fit it all. | |
| let payload = active_log.payload(); | |
| // SAFETY: We hold `persistent_log_lock` exclusively during early boot (or after | |
| // `force_reset` in single-threaded unit tests), and `payload` is disjoint from | |
| // `self.recovered_persistent_log`. | |
| unsafe { | |
| let recovered_log = &mut *self.recovered_persistent_log.get(); | |
| recovered_log.recover_from(payload, initial_rd_ptr); | |
| } | |
| } | |
| // Reset the log, then install it, and we are done. | |
| let header = active_log.header_mut(); | |
| header.validate_magic(); | |
| header.rd_ptr = 0; | |
| clean_cache_range(header.as_bytes()); | |
| let payload = active_log.payload_mut(); | |
| payload.fill(0); | |
| clean_cache_range(payload); | |
| *guard.as_mut().plog_mut() = Some(active_log); | |
| } | |
| /// Writes `str` to the persistent dlog, if enabled. Otherwise, this is a | |
| /// no-op. | |
| fn write(&self, str: &[u8]) { | |
| ksync::lock!( | |
| let mut guard = self.lock_persistent_log_lock_policy::<IrqSavePolicy>() | |
| ); | |
| // If we have no persistent log, just get out. | |
| let Some(plog) = guard.as_mut().plog_mut() else { | |
| return; | |
| }; | |
| // Copy the data into our log | |
| let payload_size = plog.payload_size as usize; | |
| let (hdr, payload) = plog.header_and_payload_mut(); | |
| let todo: usize; | |
| let offset: usize; | |
| if str.len() > payload_size { | |
| todo = payload_size; | |
| offset = str.len() - payload_size; | |
| } else { | |
| todo = str.len(); | |
| offset = 0; | |
| } | |
| let rd_ptr = hdr.rd_ptr as usize; | |
| let space = payload_size - rd_ptr; | |
| if space > todo { | |
| payload[rd_ptr..rd_ptr + todo].copy_from_slice(&str[offset..offset + todo]); | |
| clean_cache_range(&payload[rd_ptr..rd_ptr + todo]); | |
| hdr.rd_ptr += todo as u32; | |
| clean_cache_range(hdr.as_bytes()); | |
| } else { | |
| payload[rd_ptr..rd_ptr + space].copy_from_slice(&str[offset..offset + space]); | |
| payload[..todo - space].copy_from_slice(&str[offset + space..offset + todo]); | |
| clean_cache_range(&payload[rd_ptr..rd_ptr + space]); | |
| clean_cache_range(&payload[..todo - space]); | |
| hdr.rd_ptr = (todo - space) as u32; | |
| clean_cache_range(hdr.as_bytes()); | |
| } | |
| } | |
| /// Fetch a string view which references the recovered crashlog (if any). | |
| #[inline] | |
| fn recovered_log(&self) -> Option<&[u8]> { | |
| // SAFETY: We don't bother to lock this structure. The log, if present, is recovered | |
| // during early boot while we are still running on a single core. After that, | |
| // the recovered data is only ever accessed in a read only fashion, so there | |
| // is no real need to provide any explicit synchronization. | |
| let recovered_log = unsafe { &*self.recovered_persistent_log.get() }; | |
| recovered_log.as_bytes() | |
| } | |
| /// Invalidates the state of the persistent dlog. This gets called every time we | |
| /// gracefully reboot, so that we don't end up recovering a dlog after reboot and | |
| /// end up producing an unnecessary crashlog. | |
| fn invalidate(&self) { | |
| ksync::lock!( | |
| let mut guard = self.lock_persistent_log_lock_policy::<IrqSavePolicy>() | |
| ); | |
| if let Some(active_log) = guard.as_mut().plog_mut() { | |
| active_log.invalidate(); | |
| } | |
| } | |
| /// Used only by testing to reset a log to its "pre SetLocation'ed" state. | |
| #[cfg(ktest)] | |
| fn force_reset(&self) { | |
| ksync::lock!( | |
| let mut guard = self.lock_persistent_log_lock_policy::<IrqSavePolicy>() | |
| ); | |
| *guard.as_mut().plog_mut() = None; | |
| // SAFETY: `force_reset` is used only by single-threaded testing while holding | |
| // `persistent_log_lock` with `IrqSavePolicy`. | |
| let recovered_log = unsafe { &mut *self.recovered_persistent_log.get() }; | |
| recovered_log.reset(); | |
| } | |
| } | |
| /// Global persistent debuglog state and static recovery buffer parameterized by | |
| /// `STORAGE_SIZE`. When `STORAGE_SIZE == 0`, all operations are no-ops and | |
| /// `cmd` returns `-1`. | |
| struct PersistentDebuglogGlobals<const STORAGE_SIZE: usize> { | |
| instance: lazy_init::LazyInit<PersistentDebugLog, lazy_init::NoCheck>, | |
| storage: UnsafeCell<[u8; STORAGE_SIZE]>, | |
| } | |
| // SAFETY: `PersistentDebuglogGlobals` synchronizes active log mutations via | |
| // `PersistentDebugLog`'s spinlock (`IrqSavePolicy`) and only mutates `storage` | |
| // during single-core early boot initialization before read-only access. | |
| unsafe impl<const STORAGE_SIZE: usize> Send for PersistentDebuglogGlobals<STORAGE_SIZE> {} | |
| // SAFETY: Shared references to `PersistentDebuglogGlobals` synchronize active log | |
| // mutations via `PersistentDebugLog`'s spinlock (`IrqSavePolicy`) and only read | |
| // `storage` after single-core early boot recovery. | |
| unsafe impl<const STORAGE_SIZE: usize> Sync for PersistentDebuglogGlobals<STORAGE_SIZE> {} | |
| impl<const STORAGE_SIZE: usize> PersistentDebuglogGlobals<STORAGE_SIZE> { | |
| /// Constructs a new `PersistentDebuglogGlobals` with an internal recovery buffer of | |
| /// `STORAGE_SIZE` bytes. | |
| const fn new() -> Self { | |
| const { | |
| assert!(STORAGE_SIZE <= u32::MAX as usize); | |
| } | |
| Self { | |
| instance: lazy_init::LazyInit::uninit(), | |
| storage: UnsafeCell::new([0; STORAGE_SIZE]), | |
| } | |
| } | |
| /// Called once from lib/debuglog during _very_ early init. | |
| fn init_early(&'static self) { | |
| if STORAGE_SIZE == 0 { | |
| return; | |
| } | |
| let storage_ptr = NonNull::new(self.storage.get().cast::<u8>()); | |
| // SAFETY: `init_early` is called once from `lib/debuglog` during single-core early boot | |
| // before any other access to `self.instance`, and `self.storage` is valid for | |
| // `STORAGE_SIZE` bytes for the `'static` lifetime of `self`. | |
| let _ = unsafe { | |
| Pin::static_ref(&self.instance) | |
| .init_pin(PersistentDebugLog::init(storage_ptr, STORAGE_SIZE as u32)) | |
| }; | |
| } | |
| /// Sets the virtual address of where to store the persistent, assuming that we | |
| /// have one. This needs to happen early in boot, usually during ZBI header | |
| /// processing, before we start up the secondary CPUs. | |
| /// | |
| /// # Safety | |
| /// `init_early` must have been called before calling this method (guaranteed by | |
| /// `lib/debuglog`). If `STORAGE_SIZE > 0`, `vaddr` is non-null, and | |
| /// `len > size_of::<LogHeader>()`, `vaddr` must point to a valid, readable and | |
| /// writable persistent RAM mapping of `len` bytes that remains valid while this | |
| /// log is active. | |
| unsafe fn set_location(&self, vaddr: *mut u8, len: usize) { | |
| if STORAGE_SIZE == 0 { | |
| return; | |
| } | |
| // SAFETY: Caller guarantees `init_early` has been called before `set_location` and | |
| // upholds the memory validity contract for `vaddr` and `len`. | |
| unsafe { | |
| self.instance.get_unchecked().set_location(vaddr, len); | |
| } | |
| } | |
| /// Safe wrapper around [`PersistentDebuglogGlobals::set_location`] taking a static mutable | |
| /// byte slice. | |
| fn set_location_slice(&self, buffer: &'static mut [u8]) { | |
| if STORAGE_SIZE == 0 { | |
| return; | |
| } | |
| // SAFETY: `init_early` is guaranteed by `lib/debuglog` to run during very early boot | |
| // before platform ZBI handoff calls `set_location_slice`, and `buffer` is a live, | |
| // uniquely borrowed `'static` byte slice of `buffer.len()` bytes that remains valid for | |
| // the lifetime of the kernel. | |
| unsafe { | |
| self.set_location(buffer.as_mut_ptr(), buffer.len()); | |
| } | |
| } | |
| /// Writes a string to the persistent dlog, if enabled. Otherwise, this is a | |
| /// no-op. | |
| fn write(&self, bytes: &[u8]) { | |
| if STORAGE_SIZE == 0 { | |
| return; | |
| } | |
| // SAFETY: `init_early` is called during very early boot in `lib/debuglog` before any | |
| // debuglog or console writes occur. | |
| unsafe { | |
| self.instance.get_unchecked().write(bytes); | |
| } | |
| } | |
| /// Invalidates the state of the persistent dlog. This gets called every time we | |
| /// gracefully reboot, so that we don't end up recovering a dlog after reboot and | |
| /// end up producing an unnecessary crashlog. | |
| fn invalidate(&self) { | |
| if STORAGE_SIZE == 0 { | |
| return; | |
| } | |
| // SAFETY: `init_early` is called during very early boot before any reboot/shutdown | |
| // path calls `invalidate`. | |
| unsafe { | |
| self.instance.get_unchecked().invalidate(); | |
| } | |
| } | |
| /// Fetch a string view which references the recovered crashlog (if any). | |
| #[inline] | |
| fn recovered_log(&self) -> Option<&[u8]> { | |
| if STORAGE_SIZE == 0 { | |
| return None; | |
| } | |
| // SAFETY: `init_early` is called during very early boot before any crashlog recovery | |
| // queries `recovered_log`. | |
| unsafe { self.instance.get_unchecked().recovered_log() } | |
| } | |
| /// Executes the `"pdlog"` kernel console command. | |
| /// | |
| /// # Safety | |
| /// If `argument_count > 0`, `argument_vector` must be non-null and point to at least | |
| /// `argument_count` initialized [`crate::console::CmdArgs`] elements. | |
| #[cfg(console_enabled)] | |
| unsafe fn cmd( | |
| &self, | |
| argument_count: c_int, | |
| argument_vector: *const crate::console::CmdArgs, | |
| _flags: u32, | |
| ) -> c_int { | |
| if STORAGE_SIZE == 0 { | |
| return -1; | |
| } | |
| let argument_count = usize::try_from(argument_count).unwrap_or(0); | |
| // SAFETY: Caller guarantees `argument_vector` points to `argument_count` initialized | |
| // `CmdArgs` elements whenever `argument_count > 0`; a zero or negative count yields `&[]`. | |
| let arguments = unsafe { zr::slice_from_raw_parts(argument_vector, argument_count) }; | |
| let command_name = arguments.first().map(|arg| arg.as_str()).unwrap_or("pdlog"); | |
| let usage = || -> c_int { | |
| kprintln!("usage:"); | |
| kprintln!("{:s} dump : dump the recovered persistent debug log", command_name); | |
| Status::INTERNAL.into_raw() | |
| }; | |
| if arguments.len() < 2 { | |
| kprintln!("not enough arguments"); | |
| return usage(); | |
| } | |
| if arguments[1].as_str() == "dump" { | |
| if let Some(recovered) = self.recovered_log() | |
| && !recovered.is_empty() | |
| { | |
| kprintln!("Recovered {} bytes from the persistent debug log.", recovered.len()); | |
| kprintln!("---- BEGIN ----"); | |
| kprint!("{:s}", recovered); | |
| kprintln!("---- END ----"); | |
| } else { | |
| kprintln!("There was no persistent debug log recovered!"); | |
| } | |
| } else { | |
| kprintln!("unknown command"); | |
| return usage(); | |
| } | |
| zx_status::sys::ZX_OK | |
| } | |
| } | |
| static GLOBAL_PERSISTENT_DEBUGLOG: PersistentDebuglogGlobals<TARGET_PERSISTENT_DEBUGLOG_SIZE> = | |
| PersistentDebuglogGlobals::new(); | |
| /// Called once from lib/debuglog during _very_ early init. | |
| pub fn persistent_dlog_init_early() { | |
| GLOBAL_PERSISTENT_DEBUGLOG.init_early(); | |
| } | |
| /// Sets where the persistent debuglog lives, assuming that we have one. This | |
| /// needs to happen early in boot, usually during ZBI header processing, before | |
| /// we start up the secondary CPUs. The persistent debuglog keeps using | |
| /// `buffer` for the life of the kernel. | |
| pub fn persistent_dlog_set_location(buffer: &'static mut [u8]) { | |
| GLOBAL_PERSISTENT_DEBUGLOG.set_location_slice(buffer); | |
| } | |
| /// Writes `bytes` to the persistent debuglog, if enabled. Otherwise, this is a | |
| /// no-op. | |
| pub fn persistent_dlog_write(bytes: &[u8]) { | |
| GLOBAL_PERSISTENT_DEBUGLOG.write(bytes); | |
| } | |
| /// Invalidates the state of the persistent dlog. This gets called every time we | |
| /// gracefully reboot, so that we don't end up recovering a dlog after reboot and | |
| /// end up producing an unnecessary crashlog. | |
| pub fn persistent_dlog_invalidate() { | |
| GLOBAL_PERSISTENT_DEBUGLOG.invalidate(); | |
| } | |
| /// Fetch a string view which references the recovered crashlog (if any). | |
| pub fn persistent_dlog_get_recovered_log() -> Option<&'static [u8]> { | |
| GLOBAL_PERSISTENT_DEBUGLOG.recovered_log() | |
| } | |
| /// FFI entry point for C++ `persistent_dlog_init_early()`. | |
| #[unsafe(no_mangle)] | |
| pub extern "C" fn rust_persistent_dlog_init_early() { | |
| persistent_dlog_init_early(); | |
| } | |
| /// FFI entry point for C++ `persistent_dlog_set_location(void* vaddr, size_t len)`. | |
| /// | |
| /// # Safety | |
| /// `persistent_dlog_init_early` must have been called before calling this function. | |
| /// If `vaddr` is non-null and `len > size_of::<LogHeader>()`, `vaddr` must point to a valid, | |
| /// readable and writable persistent RAM mapping of `len` bytes that remains valid for the | |
| /// lifetime of the kernel. | |
| #[unsafe(no_mangle)] | |
| pub unsafe extern "C" fn rust_persistent_dlog_set_location(vaddr: *mut c_void, len: usize) { | |
| // SAFETY: Caller upholds the early-init ordering and persistent RAM mapping validity | |
| // contract for `vaddr` and `len`. | |
| unsafe { | |
| GLOBAL_PERSISTENT_DEBUGLOG.set_location(vaddr.cast::<u8>(), len); | |
| } | |
| } | |
| /// FFI entry point for C++ `persistent_dlog_write(ktl::string_view str)`. | |
| /// | |
| /// # Safety | |
| /// If `len > 0`, `ptr` must be non-null and point to at least `len` initialized, readable | |
| /// bytes that are not mutated for the duration of the call. | |
| #[unsafe(no_mangle)] | |
| pub unsafe extern "C" fn rust_persistent_dlog_write(ptr: *const c_char, len: usize) { | |
| // SAFETY: Caller guarantees `ptr` is valid for `len` bytes whenever `len > 0`; the helper | |
| // yields an empty slice when `len == 0` regardless of `ptr`. | |
| let bytes = unsafe { zr::slice_from_raw_parts(ptr.cast::<u8>(), len) }; | |
| persistent_dlog_write(bytes); | |
| } | |
| /// FFI entry point for C++ `persistent_dlog_invalidate()`. | |
| #[unsafe(no_mangle)] | |
| pub extern "C" fn rust_persistent_dlog_invalidate() { | |
| persistent_dlog_invalidate(); | |
| } | |
| /// FFI entry point for C++ `persistent_dlog_get_recovered_log()`. | |
| /// | |
| /// # Safety | |
| /// If `out_len` is non-null, it must point to a writable `usize`. | |
| #[unsafe(no_mangle)] | |
| pub unsafe extern "C" fn rust_persistent_dlog_get_recovered_log( | |
| out_len: *mut usize, | |
| ) -> *const c_char { | |
| match persistent_dlog_get_recovered_log() { | |
| Some(bytes) => { | |
| if let Some(out_len_ptr) = NonNull::new(out_len) { | |
| // SAFETY: Caller guarantees `out_len` is writable when non-null. | |
| unsafe { | |
| out_len_ptr.as_ptr().write(bytes.len()); | |
| } | |
| } | |
| bytes.as_ptr().cast::<c_char>() | |
| } | |
| None => { | |
| if let Some(out_len_ptr) = NonNull::new(out_len) { | |
| // SAFETY: Caller guarantees `out_len` is writable when non-null. | |
| unsafe { | |
| out_len_ptr.as_ptr().write(0); | |
| } | |
| } | |
| core::ptr::null() | |
| } | |
| } | |
| } | |
| /// Console command entry point for `"pdlog"`. | |
| /// | |
| /// # Safety | |
| /// If `argument_count > 0`, `argument_vector` must be non-null and point to at least | |
| /// `argument_count` initialized [`crate::console::CmdArgs`] elements. | |
| #[cfg(console_enabled)] | |
| unsafe extern "C" fn cmd_pdlog( | |
| argument_count: c_int, | |
| argument_vector: *const crate::console::CmdArgs, | |
| flags: u32, | |
| ) -> c_int { | |
| // SAFETY: Caller guarantees `argument_vector` points to `argument_count` initialized | |
| // `CmdArgs` elements whenever `argument_count > 0`. | |
| unsafe { GLOBAL_PERSISTENT_DEBUGLOG.cmd(argument_count, argument_vector, flags) } | |
| } | |
| // C++ compiled this command out with #if TARGET_PERSISTENT_DEBUGLOG_SIZE > 0. We cannot | |
| // #[cfg] on an env!() value, so register it with an empty availability mask instead, which | |
| // makes it unreachable from both normal and panic consoles. | |
| #[cfg(console_enabled)] | |
| crate::console::static_command!( | |
| CMD_PDLOG, | |
| c"pdlog".as_ptr(), | |
| c"pdlog".as_ptr(), | |
| cmd_pdlog, | |
| if TARGET_PERSISTENT_DEBUGLOG_SIZE != 0 { crate::console::CMD_AVAIL_ALWAYS } else { 0 } | |
| ); | |
| /// Persistent Debuglog Tests | |
| #[cfg(ktest)] | |
| #[unittest::suite(name = "pdlog_rust")] | |
| mod tests { | |
| use super::{LogHeader, PersistentDebugLog}; | |
| use core::ptr::NonNull; | |
| use unittest::{assert_eq, assert_false, assert_ge, assert_true, unwrap_some}; | |
| struct Buffer { | |
| storage: kalloc::Box<[u8]>, | |
| capacity: u32, | |
| } | |
| impl Buffer { | |
| fn setup(new_capacity: u32) -> Option<Self> { | |
| if new_capacity == 0 { | |
| return None; | |
| } | |
| let storage = kalloc::Box::<[u8]>::try_new_zeroed_slice(new_capacity as usize).ok()?; | |
| Some(Self { storage, capacity: new_capacity }) | |
| } | |
| } | |
| struct TestEnvironment { | |
| log: kalloc::Box<PersistentDebugLog>, | |
| log_buffer: Buffer, | |
| _recovered_log_buffer: Buffer, | |
| hdr: NonNull<LogHeader>, | |
| } | |
| impl TestEnvironment { | |
| fn setup(log_size: u32, recovered_size: u32) -> Option<Self> { | |
| let mut log_buffer = Buffer::setup(log_size)?; | |
| let mut recovered_log_buffer = Buffer::setup(recovered_size)?; | |
| let mut uninit_log = kalloc::Box::<PersistentDebugLog>::try_new_uninit().ok()?; | |
| let recovered_ptr = NonNull::new(recovered_log_buffer.storage.as_mut_ptr()); | |
| let init = PersistentDebugLog::init(recovered_ptr, recovered_log_buffer.capacity); | |
| // SAFETY: `uninit_log.as_mut_ptr()` points to a valid, writable heap allocation for | |
| // `PersistentDebugLog` that remains pinned inside `kalloc::Box`. | |
| unsafe { | |
| let _ = pin_init::PinInit::__pinned_init(init, uninit_log.as_mut_ptr()); | |
| } | |
| // SAFETY: `uninit_log` was initialized in-place by `__pinned_init` above. | |
| let log = unsafe { uninit_log.assume_init() }; | |
| let hdr = NonNull::new(log_buffer.storage.as_mut_ptr().cast::<LogHeader>())?; | |
| Some(Self { log, log_buffer, _recovered_log_buffer: recovered_log_buffer, hdr }) | |
| } | |
| fn set_location(&mut self) { | |
| // SAFETY: `self.log_buffer.storage` is a valid heap buffer of | |
| // `self.log_buffer.capacity` bytes that outlives `self.log`. | |
| unsafe { | |
| self.log.set_location( | |
| self.log_buffer.storage.as_mut_ptr(), | |
| self.log_buffer.capacity as usize, | |
| ); | |
| } | |
| } | |
| fn hdr(&self) -> &LogHeader { | |
| // SAFETY: `self.hdr` points to the start of `self.log_buffer.storage` (which has at | |
| // least `size_of::<LogHeader>()` bytes, and `LogHeader` has alignment 1). | |
| unsafe { self.hdr.as_ref() } | |
| } | |
| fn hdr_mut(&mut self) -> &mut LogHeader { | |
| // SAFETY: `self.hdr` points to the start of `self.log_buffer.storage` with exclusive | |
| // access via `&mut self`. | |
| unsafe { self.hdr.as_mut() } | |
| } | |
| } | |
| fn check_recovered_log_is_empty(log: &PersistentDebugLog) -> bool { | |
| (unittest::subtest!(|log: &PersistentDebugLog| { | |
| assert_true!(log.recovered_log().is_none()); | |
| }))(log) | |
| } | |
| fn check_recovered_log_matches(log: &PersistentDebugLog, test_vectors: &[&[u8]]) -> bool { | |
| (unittest::subtest!(|log: &PersistentDebugLog, test_vectors: &[&[u8]]| { | |
| // Check that our string view representation of the recovered log matches the | |
| // test vectors. | |
| let recovered = unwrap_some!(log.recovered_log()); | |
| let mut offset = 0usize; | |
| for test_vector in test_vectors { | |
| let log_amt = recovered.len() - offset; | |
| assert_ge!(log_amt, test_vector.len()); | |
| assert_true!(&recovered[offset..offset + test_vector.len()] == *test_vector); | |
| offset += test_vector.len(); | |
| } | |
| assert_eq!(recovered.len(), offset); | |
| }))(log, test_vectors) | |
| } | |
| /// Tests basic persistent debuglog initialization, writes, and crashlog recovery. | |
| #[test] | |
| fn pdlog_basic_test() { | |
| let mut env = unwrap_some!(TestEnvironment::setup(128, 128)); | |
| // Our allocated buffer starts filled with zeros. The log header should | |
| // consider this to be an invalid value. | |
| assert_false!(env.hdr().is_magic_valid()); | |
| assert_eq!(0u32, env.hdr().magic); | |
| // Set the location of the persistent log ram. This will attempt to recover | |
| // the (currently empty) log, and initialize the active log's header in the | |
| // process. | |
| env.set_location(); | |
| assert_true!(env.hdr().is_magic_valid()); | |
| assert_eq!(0u32, env.hdr().rd_ptr); | |
| // The recovered log should be empty. | |
| assert_true!(check_recovered_log_is_empty(&env.log)); | |
| // Perform some writes to the log | |
| const TEST_STRINGS: [&[u8]; 4] = | |
| [b"Test pattern 1\n", b"This has no newline", b"ABCDEF0123456789\n", b"Foo Bar Baz\n"]; | |
| let mut expected_rd_ptr = 0u32; | |
| for sv in TEST_STRINGS { | |
| env.log.write(sv); | |
| expected_rd_ptr += sv.len() as u32; | |
| } | |
| // The recovered log should be empty. | |
| assert_true!(check_recovered_log_is_empty(&env.log)); | |
| // But the read pointer should have advanced to match the length of the | |
| // strings we have written so far. | |
| assert_eq!(expected_rd_ptr, env.hdr().rd_ptr); | |
| // Simulate a reboot by resetting our debug log, then setting the location of | |
| // our persistent buffer once again. This will cause the implementation to | |
| // attempt to recover our log, at which point in time we can verify that | |
| // everything recovered correctly. | |
| env.log.force_reset(); | |
| env.set_location(); | |
| assert_true!(check_recovered_log_matches(&env.log, &TEST_STRINGS)); | |
| } | |
| /// Tests persistent debuglog recovery when writes wrap around the ring buffer. | |
| #[test] | |
| fn pdlog_logwrap_test() { | |
| let mut env = unwrap_some!(TestEnvironment::setup(128, 128)); | |
| env.set_location(); | |
| // Perform enough writes to the log that it wraps at least once. | |
| const TEST_STR: &[u8] = b"0123456789AB\n"; | |
| const REPEAT_COUNT: usize = 23; | |
| for _ in 0..REPEAT_COUNT { | |
| env.log.write(TEST_STR); | |
| } | |
| // "reboot" and recover the log. | |
| env.log.force_reset(); | |
| env.set_location(); | |
| // We have 120 bytes worth of payload, and our test string is 13 bytes long | |
| // and repeated 23 times. So, the total number of bytes we write into the | |
| // buffer is 299 meaning that we will wrap the buffer two complete times. In | |
| // addition, our recovered log will start 120 % 13 == 3 bytes from the end of | |
| // an instance of our test string. | |
| // Update test vectors to match change in header size! | |
| zr::static_assert!(core::mem::size_of::<LogHeader>() == 8); | |
| zr::static_assert!(TEST_STR.len() == 13); | |
| let test_strings: [&[u8]; 10] = [ | |
| &TEST_STR[TEST_STR.len() - 3..], | |
| TEST_STR, | |
| TEST_STR, | |
| TEST_STR, | |
| TEST_STR, | |
| TEST_STR, | |
| TEST_STR, | |
| TEST_STR, | |
| TEST_STR, | |
| TEST_STR, | |
| ]; | |
| assert_true!(check_recovered_log_matches(&env.log, &test_strings)); | |
| } | |
| // We expect that all embedded nulls get removed from strings when the log is | |
| // recovered. | |
| /// Tests that embedded null bytes are stripped when recovering the persistent debuglog. | |
| #[test] | |
| fn pdlog_zeros_removed_test() { | |
| let mut env = unwrap_some!(TestEnvironment::setup(128, 128)); | |
| env.set_location(); | |
| // Perform some writes to the log which have 0s embedded in them. | |
| const WITH_NULLS: [&[u8]; 2] = [b"This \0has\0nulls\n", b"\0\0even\0more\0nulls\0\0\0\n"]; | |
| for sv in WITH_NULLS { | |
| env.log.write(sv); | |
| } | |
| // "reboot" and recover the log. | |
| env.log.force_reset(); | |
| env.set_location(); | |
| // Verify that the nulls are removed during recovery. | |
| const WITHOUT_NULLS: [&[u8]; 2] = [b"This hasnulls\n", b"evenmorenulls\n"]; | |
| assert_true!(check_recovered_log_matches(&env.log, &WITHOUT_NULLS)); | |
| } | |
| /// Tests that persistent debuglog recovery rejects a corrupted header magic value. | |
| #[test] | |
| fn pdlog_rejects_bad_magic_test() { | |
| // Set up a log, put some data into it, then "reboot" | |
| let mut env = unwrap_some!(TestEnvironment::setup(128, 128)); | |
| env.set_location(); | |
| env.log.write(b"I'm in your base, corrupting your magic numbers!\n"); | |
| env.log.force_reset(); | |
| // Before we attempt to recover the log, deliberately corrupt the magic | |
| // number. | |
| env.hdr_mut().magic = env.hdr().magic.wrapping_add(1); | |
| // Now attempt to recover the log, and verify that we get nothing. | |
| env.set_location(); | |
| assert_true!(check_recovered_log_is_empty(&env.log)); | |
| } | |
| /// Tests that persistent debuglog recovery rejects an out-of-bounds header read pointer. | |
| #[test] | |
| fn pdlog_rejects_bad_rd_ptr_test() { | |
| // Set up a log, put some data into it, then "reboot" | |
| let mut env = unwrap_some!(TestEnvironment::setup(128, 128)); | |
| env.set_location(); | |
| env.log.write(b"I'm in your base, corrupting your read pointer!\n"); | |
| env.log.force_reset(); | |
| // Before we attempt to recover the log, set the read pointer of the log to | |
| // something impossible. | |
| env.hdr_mut().rd_ptr = 0x12356; | |
| // Now attempt to recover the log, and verify that we get nothing. | |
| env.set_location(); | |
| assert_true!(check_recovered_log_is_empty(&env.log)); | |
| } | |
| } |