| // Copyright 2025 The Fuchsia Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| use crate::mm::PAGE_SIZE; | |
| use crate::task::dynamic_thread_spawner::SpawnRequestBuilder; | |
| use anyhow::Context; | |
| use fidl_fuchsia_cpu_profiler as profiler; | |
| use fuchsia_component::client::connect_to_protocol; | |
| use fuchsia_runtime; | |
| use futures::StreamExt; | |
| use futures::channel::mpsc as future_mpsc; | |
| use std::borrow::Cow; | |
| use std::collections::HashMap; | |
| use std::error::Error; | |
| use std::sync::atomic::{AtomicU64, Ordering}; | |
| use std::sync::{Arc, OnceLock, mpsc as sync_mpsc}; | |
| use zerocopy::{Immutable, IntoBytes}; | |
| #[cfg(target_arch = "aarch64")] | |
| use zx::sys::zx_restricted_state_t; | |
| use fxt::TraceRecord; | |
| use fxt::profiler::ProfilerRecord; | |
| use fxt::session::SessionParser; | |
| use seq_lock::{SeqLock, SeqLockable, WriteSize}; | |
| use starnix_logging::{log_error, log_info, log_warn, track_stub}; | |
| use starnix_sync::{LockDepMutex, LockDepRwLock, PerfEventLevel, PerfFormatIdLookupTableLock}; | |
| use starnix_syscalls::{SUCCESS, SyscallArg, SyscallResult}; | |
| use starnix_uapi::arch32::{ | |
| PERF_EVENT_IOC_DISABLE, PERF_EVENT_IOC_ENABLE, PERF_EVENT_IOC_ID, | |
| PERF_EVENT_IOC_MODIFY_ATTRIBUTES, PERF_EVENT_IOC_PAUSE_OUTPUT, PERF_EVENT_IOC_PERIOD, | |
| PERF_EVENT_IOC_QUERY_BPF, PERF_EVENT_IOC_REFRESH, PERF_EVENT_IOC_RESET, PERF_EVENT_IOC_SET_BPF, | |
| PERF_EVENT_IOC_SET_FILTER, PERF_EVENT_IOC_SET_OUTPUT, PERF_RECORD_MISC_USER, | |
| perf_event_sample_format_PERF_SAMPLE_CALLCHAIN, perf_event_sample_format_PERF_SAMPLE_ID, | |
| perf_event_sample_format_PERF_SAMPLE_IDENTIFIER, perf_event_sample_format_PERF_SAMPLE_IP, | |
| perf_event_sample_format_PERF_SAMPLE_PERIOD, perf_event_sample_format_PERF_SAMPLE_READ, | |
| perf_event_sample_format_PERF_SAMPLE_REGS_USER, | |
| perf_event_sample_format_PERF_SAMPLE_STACK_USER, perf_event_sample_format_PERF_SAMPLE_TID, | |
| perf_event_sample_format_PERF_SAMPLE_TIME, perf_event_type_PERF_RECORD_LOST, | |
| perf_event_type_PERF_RECORD_SAMPLE, | |
| }; | |
| use starnix_uapi::auth::{CAP_PERFMON, CAP_SYS_ADMIN}; | |
| use starnix_uapi::errors::Errno; | |
| use starnix_uapi::open_flags::OpenFlags; | |
| use starnix_uapi::restricted_aspace::RESTRICTED_ASPACE_HIGHEST_ADDRESS; | |
| use starnix_uapi::uapi::{ | |
| perf_callchain_context_PERF_CONTEXT_KERNEL, perf_callchain_context_PERF_CONTEXT_USER, | |
| perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_32, perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_64, | |
| perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_NONE, | |
| }; | |
| use starnix_uapi::user_address::UserRef; | |
| use starnix_uapi::{ | |
| errno, error, from_status_like_fdio, perf_event_attr, perf_event_header, | |
| perf_event_mmap_page__bindgen_ty_1, perf_event_read_format_PERF_FORMAT_GROUP, | |
| perf_event_read_format_PERF_FORMAT_ID, perf_event_read_format_PERF_FORMAT_LOST, | |
| perf_event_read_format_PERF_FORMAT_TOTAL_TIME_ENABLED, | |
| perf_event_read_format_PERF_FORMAT_TOTAL_TIME_RUNNING, tid_t, uapi, | |
| }; | |
| use crate::security::{self, PerfEventOpenType, PerfEventTarget, PerfEventType}; | |
| use crate::task::Kernel; | |
| use crate::task::tracing::{LinuxIdentity, PidKoidSession}; | |
| static READ_FORMAT_ID_GENERATOR: AtomicU64 = AtomicU64::new(0); | |
| // Size of the VMO backing each event's metadata page and ring buffer; mmap | |
| // lengths up to this size are accepted. perf readers map one metadata page | |
| // plus a power-of-two data area, and commonly ask for megabytes (e.g. 256 | |
| // data pages or more), so leave generous headroom. With circular writes the | |
| // data area no longer needs to hold a whole session's records. | |
| // | |
| // We currently preallocate a fixed size for the VMO because we do not dynamically | |
| // resize it when mapped. | |
| // TODO(https://fxbug.dev/540986386): Support dynamic resizing or pass the requested size. | |
| const ESTIMATED_MMAP_BUFFER_SIZE: u64 = 16 * 1024 * 1024; | |
| // Size of a PERF_RECORD_LOST record in bytes: | |
| // perf_event_header (8) + sample_id (8) + lost_events (8) = 24. | |
| const LOST_RECORD_SIZE: u64 = 24; | |
| // Register indices in the profiler's register capture block. | |
| const AARCH64_REG_PC: usize = 32; | |
| const AARCH64_REG_SP: usize = 31; | |
| const AARCH64_REG_LR: usize = 30; | |
| // AArch32 state arrives in the same block, R0-R14 in x0-x14 (the Linux | |
| // compat layout); the AArch32 PC, LR and SP are R15, R14 and R13. | |
| const AARCH32_REG_PC: usize = 15; // R15 | |
| const AARCH32_REG_LR: usize = 14; // R14 | |
| const AARCH32_REG_SP: usize = 13; // R13 | |
| // Size of the 33 u64 general registers (r0-r29, lr, sp, pc) at the start of | |
| // both the profiler's register block and `zx_restricted_state_t`. | |
| const REGS_BYTES: usize = 33 * 8; | |
| #[cfg(target_arch = "aarch64")] | |
| const GUEST_ASPACE_BASE: u64 = starnix_uapi::restricted_aspace::RESTRICTED_ASPACE_BASE as u64; | |
| const GUEST_ASPACE_END: u64 = RESTRICTED_ASPACE_HIGHEST_ADDRESS as u64; | |
| #[cfg(target_arch = "aarch64")] | |
| const RESTRICTED_STATE_SIZE: usize = std::mem::size_of::<zx_restricted_state_t>(); | |
| #[cfg(target_arch = "aarch64")] | |
| const RESTRICTED_STATE_CPSR_OFFSET: usize = std::mem::offset_of!(zx_restricted_state_t, cpsr); | |
| #[cfg(target_arch = "aarch64")] | |
| const _: () = { | |
| assert!(std::mem::offset_of!(zx_restricted_state_t, sp) == AARCH64_REG_SP * 8); | |
| assert!(std::mem::offset_of!(zx_restricted_state_t, pc) == AARCH64_REG_PC * 8); | |
| assert!(REGS_BYTES <= RESTRICTED_STATE_CPSR_OFFSET); | |
| }; | |
| mod event; | |
| pub use event::{TraceEvent, TraceEventQueue, TraceEventQueueList}; | |
| #[repr(C)] | |
| #[derive(Copy, Clone, IntoBytes, Immutable)] | |
| struct LostRecord { | |
| header: perf_event_header, | |
| sample_id: u64, | |
| lost_events: u64, | |
| } | |
| #[repr(C)] | |
| #[derive(Copy, Clone, IntoBytes, Immutable)] | |
| struct PerfMetadataHeader { | |
| version: u32, | |
| compat_version: u32, | |
| } | |
| #[repr(C)] | |
| #[derive(Copy, Clone, IntoBytes, Immutable)] | |
| struct PerfMetadataValue { | |
| lock: u32, | |
| index: u32, | |
| offset: i64, | |
| time_enabled: u64, | |
| time_running: u64, | |
| __bindgen_anon_1: perf_event_mmap_page__bindgen_ty_1, | |
| pmc_width: u16, | |
| time_shift: u16, | |
| time_mult: u32, | |
| time_offset: u64, | |
| time_zero: u64, | |
| size: u32, | |
| __reserved_1: u32, | |
| time_cycles: u64, | |
| time_mask: u64, | |
| __reserved: [u8; 928usize], | |
| data_head: u64, | |
| data_tail: u64, | |
| data_offset: u64, | |
| data_size: u64, | |
| aux_head: u64, | |
| aux_tail: u64, | |
| aux_offset: u64, | |
| aux_size: u64, | |
| } | |
| // SAFETY: `PerfMetadataValue` can be safely written to shared memory in 8-byte chunks. | |
| // This is because it is composed of two u32s followed by only u64s. | |
| // The first u32 is the `lock` field, which is why HAS_INLINE_SEQUENCE is true. | |
| unsafe impl SeqLockable for PerfMetadataValue { | |
| const WRITE_SIZE: WriteSize = WriteSize::Eight; | |
| const HAS_INLINE_SEQUENCE: bool = true; | |
| const VMO_NAME: &'static [u8] = b"starnix:perf_event"; | |
| } | |
| struct PerfState { | |
| // This table maps a group leader's file object id to its unique u64 "format ID". | |
| // | |
| // When a sample is generated for any event in a group, we use this | |
| // "format ID" from the group leader as the value for *both* the | |
| // `PERF_SAMPLE_ID` and `PERF_SAMPLE_IDENTIFIER` fields. | |
| format_id_lookup_table: LockDepMutex<HashMap<FileObjectId, u64>, PerfFormatIdLookupTableLock>, | |
| } | |
| impl Default for PerfState { | |
| fn default() -> Self { | |
| Self { format_id_lookup_table: Default::default() } | |
| } | |
| } | |
| fn get_perf_state(kernel: &Arc<Kernel>) -> Arc<PerfState> { | |
| kernel.expando.get_or_init(PerfState::default) | |
| } | |
| uapi::check_arch_independent_layout! { | |
| perf_event_attr { | |
| type_, // "type" is a reserved keyword so add a trailing underscore. | |
| size, | |
| config, | |
| __bindgen_anon_1, | |
| sample_type, | |
| read_format, | |
| _bitfield_1, | |
| __bindgen_anon_2, | |
| bp_type, | |
| __bindgen_anon_3, | |
| __bindgen_anon_4, | |
| branch_sample_type, | |
| sample_regs_user, | |
| sample_stack_user, | |
| clockid, | |
| sample_regs_intr, | |
| aux_watermark, | |
| sample_max_stack, | |
| __reserved_2, | |
| aux_sample_size, | |
| __reserved_3, | |
| sig_data, | |
| config3, | |
| } | |
| } | |
| #[derive(Clone, Copy, Debug, PartialEq)] | |
| enum IoctlOp { | |
| Enable, | |
| Disable, | |
| } | |
| struct PerfEventFileState { | |
| attr: perf_event_attr, | |
| rf_value: u64, // "count" for the config we passed in for the event. | |
| // The most recent timestamp (ns) where we changed into an enabled state | |
| // i.e. the most recent time we got an ENABLE ioctl(). | |
| most_recent_enabled_time: u64, | |
| // Sum of all previous enablement segment durations (ns). If we are | |
| // currently in an enabled state, explicitly does NOT include the current | |
| // segment. | |
| total_time_running: u64, | |
| rf_id: u64, | |
| sample_id: u64, | |
| _rf_lost: u64, | |
| disabled: u64, | |
| sample_type: u64, | |
| // Handle to blob that stores all the perf data that a user may want. | |
| // At the moment it only stores some metadata and backtraces (bts). | |
| perf_data_vmo: zx::Vmo, | |
| // Channel used to send IoctlOps to start/stop sampling. | |
| ioctl_sender: future_mpsc::Sender<(IoctlOp, sync_mpsc::Sender<()>)>, | |
| } | |
| // Have an implementation for PerfEventFileState because VMO | |
| // doesn't have Default so we can't derive it. | |
| impl PerfEventFileState { | |
| fn new( | |
| attr: perf_event_attr, | |
| rf_value: u64, | |
| disabled: u64, | |
| sample_type: u64, | |
| perf_data_vmo: zx::Vmo, | |
| ioctl_sender: future_mpsc::Sender<(IoctlOp, sync_mpsc::Sender<()>)>, | |
| ) -> PerfEventFileState { | |
| PerfEventFileState { | |
| attr, | |
| rf_value, | |
| most_recent_enabled_time: 0, | |
| total_time_running: 0, | |
| rf_id: 0, | |
| sample_id: 0, | |
| _rf_lost: 0, | |
| disabled, | |
| sample_type, | |
| perf_data_vmo, | |
| ioctl_sender, | |
| } | |
| } | |
| } | |
| pub struct PerfEventFile { | |
| _tid: tid_t, | |
| _cpu: i32, | |
| perf_event_file: LockDepRwLock<PerfEventFileState, PerfEventLevel>, | |
| // The security state for this PerfEventFile. | |
| pub security_state: security::PerfEventState, | |
| seq_lock: Arc<OnceLock<Result<SeqLock<PerfMetadataHeader, PerfMetadataValue>, Errno>>>, | |
| } | |
| // PerfEventFile object that implements FileOps. | |
| // See https://man7.org/linux/man-pages/man2/perf_event_open.2.html for | |
| // implementation details. | |
| // This object can be saved as a FileDescriptor. | |
| impl FileOps for PerfEventFile { | |
| // Don't need to implement seek or sync for PerfEventFile. | |
| fileops_impl_nonseekable!(); | |
| fileops_impl_noop_sync!(); | |
| fn close(self: Box<Self>, file: &FileObjectState, current_task: &CurrentTask) { | |
| { | |
| let mut perf_event_file = self.perf_event_file.write(); | |
| // Ensure we disable so we clean up resources if the user closes without disabling. | |
| perf_event_file.disabled = 1; | |
| ping_receiver(perf_event_file.ioctl_sender.clone(), IoctlOp::Disable); | |
| } | |
| let perf_state = get_perf_state(¤t_task.kernel); | |
| let mut events = perf_state.format_id_lookup_table.lock(); | |
| events.remove(&file.id); | |
| } | |
| // See "Reading results" section of https://man7.org/linux/man-pages/man2/perf_event_open.2.html. | |
| fn read( | |
| &self, | |
| _file: &FileObject, | |
| current_task: &CurrentTask, | |
| _offset: usize, | |
| data: &mut dyn OutputBuffer, | |
| ) -> Result<usize, Errno> { | |
| security::check_perf_event_read_access(current_task, self)?; | |
| // Create/calculate and return the ReadFormatData object. | |
| // If we create it earlier we might want to change it and it's immutable once created. | |
| let read_format_data = { | |
| // Once we get the `value` or count from kernel, we can change this to a read() | |
| // call instead of write(). | |
| let mut perf_event_file = self.perf_event_file.write(); | |
| let mut total_time_running_including_curr = perf_event_file.total_time_running; | |
| // Only update values if enabled (either by perf_event_attr or ioctl ENABLE call). | |
| if perf_event_file.disabled == 0 { | |
| // Calculate the value or "count" of the config we're interested in. | |
| // This value should reflect the value we are counting (defined in the config). | |
| // E.g. for PERF_COUNT_SW_CPU_CLOCK it would return the value from the CPU clock. | |
| // For now we just return rf_value + 1. | |
| track_stub!( | |
| TODO("https://fxbug.dev/402938671"), | |
| "[perf_event_open] implement read_format value" | |
| ); | |
| perf_event_file.rf_value += 1; | |
| // Update time duration. | |
| let curr_time = zx::MonotonicInstant::get().into_nanos() as u64; | |
| total_time_running_including_curr += | |
| curr_time - perf_event_file.most_recent_enabled_time; | |
| } | |
| let mut output = Vec::<u8>::new(); | |
| let value = perf_event_file.rf_value.to_ne_bytes(); | |
| output.extend(value); | |
| let read_format = perf_event_file.attr.read_format; | |
| if (read_format & perf_event_read_format_PERF_FORMAT_TOTAL_TIME_ENABLED as u64) != 0 { | |
| // Total time (ns) event was enabled and running (currently same as TIME_RUNNING). | |
| output.extend(total_time_running_including_curr.to_ne_bytes()); | |
| } | |
| if (read_format & perf_event_read_format_PERF_FORMAT_TOTAL_TIME_RUNNING as u64) != 0 { | |
| // Total time (ns) event was enabled and running (currently same as TIME_ENABLED). | |
| output.extend(total_time_running_including_curr.to_ne_bytes()); | |
| } | |
| if (read_format & perf_event_read_format_PERF_FORMAT_ID as u64) != 0 { | |
| // Adds a 64-bit unique value that corresponds to the event group. | |
| output.extend(perf_event_file.rf_id.to_ne_bytes()); | |
| } | |
| output | |
| }; | |
| // The regular read() call allows the case where the bytes-we-want-to-read-in won't | |
| // fit in the output buffer. However, for perf_event_open's read(), "If you attempt to read | |
| // into a buffer that is not big enough to hold the data, the error ENOSPC results." | |
| if data.available() < read_format_data.len() { | |
| return error!(ENOSPC); | |
| } | |
| track_stub!( | |
| TODO("https://fxbug.dev/402453955"), | |
| "[perf_event_open] implement remaining error handling" | |
| ); | |
| data.write(&read_format_data) | |
| } | |
| fn ioctl( | |
| &self, | |
| _file: &FileObject, | |
| current_task: &CurrentTask, | |
| op: u32, | |
| _arg: SyscallArg, | |
| ) -> Result<SyscallResult, Errno> { | |
| track_stub!( | |
| TODO("https://fxbug.dev/405463320"), | |
| "[perf_event_open] implement PERF_IOC_FLAG_GROUP" | |
| ); | |
| security::check_perf_event_write_access(current_task, self)?; | |
| let mut perf_event_file = self.perf_event_file.write(); | |
| match op { | |
| PERF_EVENT_IOC_ENABLE => { | |
| if perf_event_file.disabled != 0 { | |
| perf_event_file.disabled = 0; // 0 = false. | |
| perf_event_file.most_recent_enabled_time = | |
| zx::MonotonicInstant::get().into_nanos() as u64; | |
| } | |
| // If we are sampling, start a profiler session. Sampling is time-based | |
| // (CounterId::Nanoseconds) regardless of the event type, and the profiler allows | |
| // only one session at a time, so other sampling events fail to start their own. | |
| // Some mmap metadata is still hardcoded; see create_seq_lock. | |
| if sampling_period(&perf_event_file.attr).is_some() { | |
| ping_receiver(perf_event_file.ioctl_sender.clone(), IoctlOp::Enable); | |
| } | |
| return Ok(SUCCESS); | |
| } | |
| PERF_EVENT_IOC_DISABLE => { | |
| if perf_event_file.disabled == 0 { | |
| perf_event_file.disabled = 1; // 1 = true. | |
| // Update total_time_running now that the segment has ended. | |
| let curr_time = zx::MonotonicInstant::get().into_nanos() as u64; | |
| perf_event_file.total_time_running += | |
| curr_time - perf_event_file.most_recent_enabled_time; | |
| } | |
| if sampling_period(&perf_event_file.attr).is_some() { | |
| ping_receiver(perf_event_file.ioctl_sender.clone(), IoctlOp::Disable); | |
| } | |
| return Ok(SUCCESS); | |
| } | |
| PERF_EVENT_IOC_RESET => { | |
| perf_event_file.rf_value = 0; | |
| return Ok(SUCCESS); | |
| } | |
| PERF_EVENT_IOC_REFRESH | |
| | PERF_EVENT_IOC_PERIOD | |
| | PERF_EVENT_IOC_SET_OUTPUT | |
| | PERF_EVENT_IOC_SET_FILTER | |
| | PERF_EVENT_IOC_ID | |
| | PERF_EVENT_IOC_SET_BPF | |
| | PERF_EVENT_IOC_PAUSE_OUTPUT | |
| | PERF_EVENT_IOC_MODIFY_ATTRIBUTES | |
| | PERF_EVENT_IOC_QUERY_BPF => { | |
| track_stub!( | |
| TODO("https://fxbug.dev/404941053"), | |
| "[perf_event_open] implement remaining ioctl() calls" | |
| ); | |
| return error!(ENOSYS); | |
| } | |
| _ => error!(ENOTTY), | |
| } | |
| } | |
| // TODO(https://fxbug.dev/460245383) match behavior when mmap() is called multiple times. | |
| // Gets called when mmap() is called. | |
| // Immediately before sampling, this should get called by the user (e.g. the test | |
| // or Perfetto). We will then write the metadata to the VMO and return the pointer to it. | |
| fn get_memory( | |
| &self, | |
| _file: &FileObject, | |
| current_task: &CurrentTask, | |
| length: Option<usize>, | |
| prot: ProtectionFlags, | |
| ) -> Result<Arc<MemoryObject>, Errno> { | |
| if prot.contains(ProtectionFlags::EXEC) { | |
| return error!(EPERM); | |
| } | |
| let buffer_size: u64 = length.unwrap_or(0) as u64; | |
| let page_size = zx::system_get_page_size() as u64; | |
| if buffer_size <= page_size || buffer_size > ESTIMATED_MMAP_BUFFER_SIZE { | |
| return error!(EINVAL); | |
| } | |
| let data_size = buffer_size - page_size; | |
| if !data_size.is_power_of_two() { | |
| return error!(EINVAL); | |
| } | |
| security::check_perf_event_read_access(current_task, self)?; | |
| self.seq_lock | |
| .get_or_init(|| { | |
| let perf_event_file = self.perf_event_file.read(); | |
| let vmo_copy = perf_event_file | |
| .perf_data_vmo | |
| .as_handle_ref() | |
| .duplicate_handle(zx::Rights::SAME_RIGHTS) | |
| .map_err(|status| from_status_like_fdio!(status))?; | |
| // SAFETY: See safety requirements on `create_seq_lock`. | |
| Ok(unsafe { create_seq_lock(&vmo_copy, buffer_size) }) | |
| }) | |
| .as_ref() | |
| .map_err(|e| e.clone())?; | |
| // Write to a MemoryObject and return it (expected return type for get_memory()). | |
| let perf_event_file = self.perf_event_file.read(); | |
| match perf_event_file | |
| .perf_data_vmo | |
| .as_handle_ref() | |
| .duplicate_handle(zx::Rights::SAME_RIGHTS) | |
| { | |
| Ok(vmo) => { | |
| let vmo: zx::Vmo = vmo.into(); | |
| let memory = MemoryObject::from(vmo); | |
| return Ok(Arc::new(memory)); | |
| } | |
| Err(_) => { | |
| track_stub!( | |
| TODO("https://fxbug.dev/416323134"), | |
| "[perf_event_open] handle get_memory() errors" | |
| ); | |
| return error!(EINVAL); | |
| } | |
| }; | |
| } | |
| // The perf data buffer is mapped through memory shared with Starnix, which isn't executable. | |
| fn can_map_executable(&self) -> bool { | |
| false | |
| } | |
| fn write( | |
| &self, | |
| _file: &FileObject, | |
| _current_task: &CurrentTask, | |
| _offset: usize, | |
| _data: &mut dyn InputBuffer, | |
| ) -> Result<usize, Errno> { | |
| track_stub!( | |
| TODO("https://fxbug.dev/394960158"), | |
| "[perf_event_open] implement perf event functions" | |
| ); | |
| error!(ENOSYS) | |
| } | |
| } | |
| // Given a PerfRecordSample struct, write it via the correct output format | |
| // (per https://man7.org/linux/man-pages/man2/perf_event_open.2.html) to the VMO. | |
| // We don't currently support all the sample_types listed in the docs. | |
| // Input: | |
| // PerfRecordSample { pid: 5, tid: 10, nr: 3, ips[nr]: [111, 222, 333] } | |
| // Human-understandable output: | |
| // 9 1 40 111 5 10 3 111 222 333 | |
| // Actual output (no spaces or \n in real output, just making it more readable): | |
| // 0x0000 0x0009 <-- starts at `offset` bytes | |
| // 0x0001 | |
| // 0x0040 | |
| // 0x0000 0x0000 0x0000 0x006F <-- starts at `offset` + 8 bytes | |
| // 0x0000 0x0000 0x0000 0x0005 | |
| // 0x0000 0x0000 0x0000 0x0010 | |
| // 0x0000 0x0000 0x0000 0x0003 | |
| // 0x0000 0x0000 0x0000 0x006F | |
| // 0x0000 0x0000 0x0000 0x00DE | |
| // 0x0000 0x0000 0x0000 0x014D | |
| // | |
| // Returns the length of bytes written. In above case, 8 + 28 = 36. | |
| // This information is used to increment the global offset. | |
| // | |
| // If writing to the VMO fails, we log a warning and return the number of | |
| // bytes successfully written so far (e.g. only the LOST record if that | |
| // succeeded, or 0). The caller uses this returned length to increment the | |
| // VMO write offset and update `data_head`, meaning failed writes are | |
| // effectively skipped and not exposed to the reader. | |
| fn write_record_to_vmo( | |
| perf_record_sample: PerfRecordSample<'_>, | |
| perf_data_vmo: &zx::Vmo, | |
| sample_type: u64, | |
| sample_id: u64, | |
| sample_period: u64, | |
| read_format: u64, | |
| head: u64, | |
| metadata: &PerfMetadataValue, | |
| lost_events: &mut u64, | |
| ) -> u64 { | |
| let ring_buffer_size = metadata.data_size; | |
| if ring_buffer_size == 0 { | |
| return 0; | |
| } | |
| // First, build record to determine its size (so that we can fill out `size` in header). | |
| let mut sample = Vec::<u8>::new(); | |
| // sample_id | |
| if (sample_type & perf_event_sample_format_PERF_SAMPLE_IDENTIFIER as u64) != 0 { | |
| sample.extend(sample_id.to_ne_bytes()); | |
| } | |
| // ip | |
| if (sample_type & perf_event_sample_format_PERF_SAMPLE_IP as u64) != 0 { | |
| let ip = perf_record_sample.ips.first().copied().unwrap_or(0); | |
| sample.extend(ip.to_ne_bytes()); | |
| } | |
| if (sample_type & perf_event_sample_format_PERF_SAMPLE_TID as u64) != 0 { | |
| // pid | |
| sample.extend(perf_record_sample.pid.unwrap_or(0).to_ne_bytes()); | |
| // tid | |
| sample.extend(perf_record_sample.tid.unwrap_or(0).to_ne_bytes()); | |
| } | |
| // time: when the sample was taken. | |
| if (sample_type & perf_event_sample_format_PERF_SAMPLE_TIME as u64) != 0 { | |
| sample.extend((perf_record_sample.time.into_nanos() as u64).to_ne_bytes()); | |
| } | |
| // id | |
| if (sample_type & perf_event_sample_format_PERF_SAMPLE_ID as u64) != 0 { | |
| sample.extend(sample_id.to_ne_bytes()); | |
| } | |
| // sample period | |
| if (sample_type & perf_event_sample_format_PERF_SAMPLE_PERIOD as u64) != 0 { | |
| sample.extend(sample_period.to_ne_bytes()); | |
| } | |
| // read_format value. | |
| if (sample_type & perf_event_sample_format_PERF_SAMPLE_READ as u64) != 0 { | |
| if (read_format & perf_event_read_format_PERF_FORMAT_GROUP as u64) != 0 { | |
| // Group reads start with the number of events followed by each | |
| // event's value; only the timebase event exists. | |
| sample.extend(1u64.to_ne_bytes()); | |
| sample.extend(0u64.to_ne_bytes()); | |
| } else { | |
| sample.extend(0u64.to_ne_bytes()); | |
| } | |
| } | |
| if (sample_type & perf_event_sample_format_PERF_SAMPLE_CALLCHAIN as u64) != 0 { | |
| // Callchains are structured into sections, each introduced by a | |
| // PERF_CONTEXT_* marker, and readers rely on these markers to split | |
| // kernel and user frames. | |
| // | |
| // Starnix normal-mode execution (>= GUEST_ASPACE_END, above | |
| // restricted_return_loop) represents kernel execution from the guest's | |
| // perspective and belongs in PERF_CONTEXT_KERNEL; restricted-mode | |
| // guest execution (< GUEST_ASPACE_END) belongs in PERF_CONTEXT_USER. | |
| // Both sections are always emitted; readers that unwind user stacks | |
| // themselves (DWARF) stop at the PERF_CONTEXT_USER marker. | |
| // Reserve nr, then fill it in once the markers and ips are written. | |
| let nr_offset = sample.len(); | |
| sample.extend(0u64.to_ne_bytes()); | |
| let callchain_start = sample.len(); | |
| sample.extend(perf_callchain_context_PERF_CONTEXT_KERNEL.to_ne_bytes()); | |
| for ip in perf_record_sample.ips.iter().filter(|&&ip| ip >= GUEST_ASPACE_END) { | |
| sample.extend(ip.to_ne_bytes()); | |
| } | |
| sample.extend(perf_callchain_context_PERF_CONTEXT_USER.to_ne_bytes()); | |
| for ip in perf_record_sample.ips.iter().filter(|&&ip| ip < GUEST_ASPACE_END) { | |
| sample.extend(ip.to_ne_bytes()); | |
| } | |
| // nr, counting the markers and ips. | |
| let nr = ((sample.len() - callchain_start) / std::mem::size_of::<u64>()) as u64; | |
| sample[nr_offset..callchain_start].copy_from_slice(&nr.to_ne_bytes()); | |
| } | |
| // User registers (PERF_SAMPLE_REGS_USER): the ABI tag, then one u64 per | |
| // bit set in attr.sample_regs_user, in perf register-index order. | |
| // Readers request the mask for their own architecture -- a 64-bit reader | |
| // requests the arm64 mask even when profiling 32-bit tasks (and relocates | |
| // the PC from the PERF_REG_ARM64_PC index itself, matching the Linux | |
| // kernel's behavior of capturing native registers). | |
| // If the mask is 0 or the ABI is PERF_SAMPLE_REGS_ABI_NONE, no register | |
| // values are output. | |
| if (sample_type & perf_event_sample_format_PERF_SAMPLE_REGS_USER as u64) != 0 { | |
| if perf_record_sample.regs_abi == perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_NONE as u64 | |
| || perf_record_sample.sample_regs_user == 0 | |
| || perf_record_sample.regs.is_empty() | |
| { | |
| sample.extend((perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_NONE as u64).to_ne_bytes()); | |
| } else { | |
| sample.extend(perf_record_sample.regs_abi.to_ne_bytes()); | |
| sample.extend_from_slice(perf_record_sample.regs); | |
| } | |
| } | |
| // User stack (PERF_SAMPLE_STACK_USER): size, the raw bytes starting at | |
| // the sampled stack pointer, then the filled size. Readers overlay these | |
| // bytes at the SP reported in REGS_USER, which is why the snapshot must | |
| // begin exactly at it. | |
| if (sample_type & perf_event_sample_format_PERF_SAMPLE_STACK_USER as u64) != 0 { | |
| let fixed_len = std::mem::size_of::<perf_event_header>() + sample.len() + 16; | |
| let header_space = (u16::MAX as usize).saturating_sub(fixed_len) & !7; | |
| let requested = (perf_record_sample.sample_stack_user as usize) & !7; | |
| let dest_len = requested.min(header_space); | |
| sample.extend((dest_len as u64).to_ne_bytes()); | |
| if dest_len > 0 { | |
| let data_len = perf_record_sample.stack.len().min(dest_len) & !7; | |
| sample.extend_from_slice(&perf_record_sample.stack[..data_len]); | |
| // Pad the rest with zeros. | |
| sample.resize(sample.len() + (dest_len - data_len), 0); | |
| // dyn_size | |
| sample.extend((data_len as u64).to_ne_bytes()); | |
| } | |
| } | |
| // The remaining sample_type fields are not implemented. | |
| // Now that we know the sample size, we can calculate the record size. | |
| // record_size = perf_event_header_size + sample_size. | |
| // perf_event_header is defined to be 8 bytes. | |
| let record_len = std::mem::size_of::<perf_event_header>() + sample.len(); | |
| // Every field above is a u64 (or a u32 pair), so record sizes are always | |
| // multiples of 8: ring positions stay 8-byte aligned. Since the | |
| // perf_event_header is also 8 bytes, it can never straddle the end of | |
| // the data area (which is page-aligned), meaning readers can always | |
| // read the header contiguously. | |
| if record_len % 8 != 0 { | |
| log_error!("Record length {} is not 8-byte aligned, dropping", record_len); | |
| *lost_events += 1; | |
| return 0; | |
| } | |
| // The header's size field is a u16. A record that exceeds it would | |
| // silently wrap the size and desynchronize every record after it, so | |
| // drop it instead. | |
| let Ok(record_size) = u16::try_from(record_len) else { | |
| log_warn!("Dropping {} byte perf sample record: exceeds u16 record size", record_len); | |
| *lost_events += 1; | |
| return 0; | |
| }; | |
| let perf_event_header = perf_event_header { | |
| // These are samples of user-space execution; readers take the | |
| // sample's cpu mode from the misc bits. | |
| type_: perf_event_type_PERF_RECORD_SAMPLE, | |
| misc: PERF_RECORD_MISC_USER as u16, | |
| size: record_size, | |
| }; | |
| // data_tail is advanced by userspace as it consumes records and is untrusted. | |
| // Calculate free space using the standard circular buffer formula, matching Linux's | |
| // CIRC_SPACE. This naturally handles wrap-around and invalid future tails. | |
| // ring_buffer_size is guaranteed to be a power of two by checks in get_memory(). | |
| let free_space = | |
| (metadata.data_tail.wrapping_sub(head).wrapping_sub(1)) & (ring_buffer_size - 1); | |
| // Drop the sample if the ring buffer is full, matching Linux's | |
| // non-overwrite mode; the drop is reported via PERF_RECORD_LOST once | |
| // space frees up. | |
| if free_space < record_len as u64 { | |
| *lost_events += 1; | |
| return 0; | |
| } | |
| let mut bytes_written: u64 = 0; | |
| // If records were dropped earlier, surface a PERF_RECORD_LOST record as | |
| // soon as there is room for it alongside the current sample. | |
| if *lost_events > 0 && free_space >= record_len as u64 + LOST_RECORD_SIZE { | |
| let lost_header = perf_event_header { | |
| type_: perf_event_type_PERF_RECORD_LOST, | |
| misc: 0, | |
| size: LOST_RECORD_SIZE as u16, | |
| }; | |
| let lost_record = LostRecord { header: lost_header, sample_id, lost_events: *lost_events }; | |
| if write_circular( | |
| perf_data_vmo, | |
| metadata.data_offset, | |
| ring_buffer_size, | |
| head, | |
| lost_record.as_bytes(), | |
| ) | |
| .is_ok() | |
| { | |
| *lost_events = 0; | |
| bytes_written += LOST_RECORD_SIZE; | |
| } | |
| } | |
| let mut record = Vec::with_capacity(record_len); | |
| record.extend_from_slice(perf_event_header.as_bytes()); | |
| record.extend_from_slice(&sample); | |
| match write_circular( | |
| perf_data_vmo, | |
| metadata.data_offset, | |
| ring_buffer_size, | |
| head + bytes_written, | |
| &record, | |
| ) { | |
| // Return the total size we wrote so the caller can advance data_head. | |
| Ok(()) => bytes_written + record_len as u64, | |
| Err(e) => { | |
| log_warn!("Failed to write PerfRecordSample to VMO due to: {}", e); | |
| bytes_written | |
| } | |
| } | |
| } | |
| // Writes `data` into the ring buffer's data area at the position | |
| // corresponding to `head` (a free-running count of bytes ever written), | |
| // splitting the write across the end of the data area when it wraps around. | |
| // Readers read records by checking the header size, reading contiguously, | |
| // and wrapping around to the beginning of the data area if the record is split. | |
| fn write_circular( | |
| vmo: &zx::Vmo, | |
| data_offset: u64, | |
| ring_buffer_size: u64, | |
| head: u64, | |
| data: &[u8], | |
| ) -> Result<(), zx::Status> { | |
| let position = head % ring_buffer_size; | |
| let vmo_offset = data_offset + position; | |
| if position + data.len() as u64 <= ring_buffer_size { | |
| vmo.write(data, vmo_offset)?; | |
| } else { | |
| let first_len = (ring_buffer_size - position) as usize; | |
| vmo.write(&data[..first_len], vmo_offset)?; | |
| vmo.write(&data[first_len..], data_offset)?; | |
| } | |
| Ok(()) | |
| } | |
| /// Represents a PERF_RECORD_SAMPLE payload to be serialized into the VMO ring buffer. | |
| /// Fields follow the perf ABI order: TID, TIME, ID, PERIOD, READ, CALLCHAIN, REGS_USER, STACK_USER. | |
| #[derive(Debug, Clone)] | |
| struct PerfRecordSample<'a> { | |
| pid: Option<u32>, | |
| tid: Option<u32>, | |
| // Timestamp of when the sample was taken, for PERF_SAMPLE_TIME. | |
| time: zx::BootInstant, | |
| // Instruction pointers (currently this is the address). First one is `ip` param. | |
| ips: Vec<u64>, | |
| regs: &'a [u8], | |
| stack: &'a [u8], | |
| // PERF_SAMPLE_REGS_ABI_32 (1) or PERF_SAMPLE_REGS_ABI_64 (2) for `regs`. | |
| regs_abi: u64, | |
| sample_regs_user: u64, | |
| sample_stack_user: u64, | |
| } | |
| fn callgraph_config(sample_type: u64, sample_stack_user: u32) -> profiler::CallgraphConfig { | |
| // Events that asked for register and stack state (PERF_SAMPLE_REGS_USER / | |
| // PERF_SAMPLE_STACK_USER) need raw stack snapshots, which only the DWARF | |
| // strategy captures. Events that just want a callchain | |
| // (PERF_SAMPLE_CALLCHAIN) are served by the in-kernel sampler, which | |
| // unwinds via frame pointers without suspending the sampled thread. | |
| let want_stack_samples = (sample_type | |
| & (perf_event_sample_format_PERF_SAMPLE_REGS_USER as u64 | |
| | perf_event_sample_format_PERF_SAMPLE_STACK_USER as u64)) | |
| != 0; | |
| let strategy = if want_stack_samples { | |
| profiler::CallgraphStrategy::Dwarf | |
| } else { | |
| profiler::CallgraphStrategy::FramePointer | |
| }; | |
| let stack_size = ((sample_type & perf_event_sample_format_PERF_SAMPLE_STACK_USER as u64) != 0) | |
| .then_some(u64::from(sample_stack_user)); | |
| profiler::CallgraphConfig { strategy: Some(strategy), stack_size, ..Default::default() } | |
| } | |
| async fn set_up_profiler( | |
| sample_period: zx::MonotonicDuration, | |
| callgraph: profiler::CallgraphConfig, | |
| ) -> Result<(profiler::SessionProxy, fidl::AsyncSocket), Errno> { | |
| let sample = profiler::Sample { callgraph: Some(callgraph), ..Default::default() }; | |
| let sampling_config = profiler::SamplingConfig { | |
| period: Some(sample_period.into_nanos() as u64), | |
| timebase: Some(profiler::Counter::PlatformIndependent(profiler::CounterId::Nanoseconds)), | |
| sample: Some(sample), | |
| ..Default::default() | |
| }; | |
| // Every Linux task in this container is a process in the Starnix kernel's own job, so | |
| // profiling that job covers all Linux tasks. Profiling system-wide would only add samples | |
| // from native Fuchsia threads, which cannot be mapped to a Linux pid/tid and are dropped. | |
| let job = fuchsia_runtime::job_default(); | |
| let koid = job.koid().map_err(|e| errno!(EINVAL, e.to_string()))?; | |
| let tasks = vec![ | |
| // Should return ~1300 samples for 1000 millis. | |
| profiler::Task::Job(koid.raw_koid()), | |
| ]; | |
| let targets = profiler::TargetConfig::Tasks(tasks); | |
| let config = profiler::Config { | |
| configs: Some(vec![sampling_config]), | |
| target: Some(targets), | |
| ..Default::default() | |
| }; | |
| let (client, server) = fidl::Socket::create_stream(); | |
| let configure = profiler::SessionConfigureRequest { | |
| output: Some(server), | |
| config: Some(config), | |
| ..Default::default() | |
| }; | |
| let proxy = connect_to_protocol::<profiler::SessionMarker>() | |
| .context("Error connecting to Profiler protocol"); | |
| let session_proxy: profiler::SessionProxy = match proxy { | |
| Ok(p) => p.clone(), | |
| Err(e) => return error!(EINVAL, e), | |
| }; | |
| // Must configure before sampling start(). | |
| let config_request = session_proxy.configure(configure).await; | |
| match config_request { | |
| Ok(_) => Ok((session_proxy, fidl::AsyncSocket::from_socket(client))), | |
| Err(e) => return error!(EINVAL, e), | |
| } | |
| } | |
| // Returns true if `cpsr` indicates AArch32 execution. | |
| #[cfg(target_arch = "aarch64")] | |
| fn cpsr_is_aarch32(cpsr: u64) -> bool { | |
| (cpsr & zx::sys::ZX_REG_CPSR_ARCH_32_MASK) == zx::sys::ZX_REG_CPSR_ARCH_32_MASK | |
| } | |
| #[cfg(not(target_arch = "aarch64"))] | |
| fn cpsr_is_aarch32(_cpsr: u64) -> bool { | |
| false | |
| } | |
| // Searches `chunks` for the thread's `zx_restricted_state_t` capture and | |
| // returns the saved guest registers (`x0..x30, sp, pc`) and `cpsr`. | |
| // | |
| // A chunk matches if it is page-aligned in the Starnix address space, has | |
| // the size of `zx_restricted_state_t`, and holds a guest PC and SP within | |
| // the restricted address space. | |
| #[cfg(target_arch = "aarch64")] | |
| fn guest_regs_from_restricted_state<'a>(chunks: &[(u64, &'a [u8])]) -> Option<(&'a [u8], u64)> { | |
| chunks.iter().find_map(|&(base, data)| { | |
| if base < GUEST_ASPACE_END || base % *PAGE_SIZE != 0 || data.len() != RESTRICTED_STATE_SIZE | |
| { | |
| return None; | |
| } | |
| let read_reg = | |
| |index: usize| u64::from_ne_bytes(data[index * 8..index * 8 + 8].try_into().unwrap()); | |
| let cpsr = u32::from_ne_bytes( | |
| data[RESTRICTED_STATE_CPSR_OFFSET..RESTRICTED_STATE_CPSR_OFFSET + 4] | |
| .try_into() | |
| .unwrap(), | |
| ); | |
| let cpsr = u64::from(cpsr); | |
| // AArch32 execution keeps its stack pointer in R13 (x13). | |
| let sp = | |
| if cpsr_is_aarch32(cpsr) { read_reg(AARCH32_REG_SP) } else { read_reg(AARCH64_REG_SP) }; | |
| let pc = read_reg(AARCH64_REG_PC); | |
| if !(GUEST_ASPACE_BASE..GUEST_ASPACE_END).contains(&pc) || sp >= GUEST_ASPACE_END { | |
| return None; | |
| } | |
| Some((&data[..REGS_BYTES], cpsr)) | |
| }) | |
| } | |
| #[cfg(not(target_arch = "aarch64"))] | |
| fn guest_regs_from_restricted_state<'a>(_chunks: &[(u64, &'a [u8])]) -> Option<(&'a [u8], u64)> { | |
| None | |
| } | |
| // Converts one FXT record from the profiler into a PERF_RECORD_SAMPLE in | |
| // the ring buffer and publishes the new data_head. Non-sample records are | |
| // ignored. | |
| fn process_fxt_record( | |
| record: TraceRecord, | |
| seq_lock_wrapper: &SeqLock<PerfMetadataHeader, PerfMetadataValue>, | |
| perf_data_vmo: &zx::Vmo, | |
| sample_type: u64, | |
| sample_id: u64, | |
| sample_period: u64, | |
| read_format: u64, | |
| sample_regs_user: u64, | |
| sample_stack_user: u64, | |
| koid_session: Option<&PidKoidSession>, | |
| vmo_write_offset: &mut u64, | |
| lost_events: &mut u64, | |
| ) { | |
| match record { | |
| TraceRecord::Profiler(ProfilerRecord::Backtrace(backtrace)) => { | |
| let ips: Vec<u64> = backtrace.data; | |
| // Resolve the sampled koids to Linux pid/tid against the live | |
| // shared map (one read lock per record; collection is off the hot | |
| // path and a live read sees every thread that recorded itself | |
| // before it was sampled). If the sample cannot be resolved (e.g. | |
| // native Fuchsia thread or without a session), drop the sample. | |
| let Some(LinuxIdentity::Thread { pid, tid }) = koid_session.and_then(|s| { | |
| s.resolve_koids( | |
| zx::Koid::from_raw(backtrace.process.0), | |
| zx::Koid::from_raw(backtrace.thread.0), | |
| ) | |
| }) else { | |
| return; | |
| }; | |
| let time = zx::BootInstant::from_nanos(backtrace.timestamp.max(0)); | |
| let perf_record_sample = PerfRecordSample { | |
| pid: Some(pid as u32), | |
| tid: Some(tid as u32), | |
| time, | |
| ips, | |
| regs: &[], | |
| stack: &[], | |
| regs_abi: perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_NONE as u64, | |
| sample_regs_user, | |
| sample_stack_user, | |
| }; | |
| let metadata = seq_lock_wrapper.get(); | |
| let bytes_written = write_record_to_vmo( | |
| perf_record_sample, | |
| perf_data_vmo, | |
| sample_type, | |
| sample_id, | |
| sample_period, | |
| read_format, | |
| *vmo_write_offset, | |
| &metadata, | |
| lost_events, | |
| ); | |
| // Publish data_head after writing; set_value's | |
| // release-ordered stores make the record contents | |
| // visible to a reader that observes the new head. | |
| if bytes_written > 0 { | |
| *vmo_write_offset += bytes_written; | |
| let mut metadata = seq_lock_wrapper.get(); | |
| metadata.data_head = *vmo_write_offset; | |
| seq_lock_wrapper.set_value(metadata); | |
| } | |
| } | |
| TraceRecord::LargeBlob(large_blob) => { | |
| // The DWARF strategy delivers each sample as a | |
| // "stack_sample" blob: [u64 regs_size][regs bytes] | |
| // followed by memory chunks of [u64 base][u64 size] | |
| // [bytes] (see the profiler's StackSampler). | |
| if large_blob.name != "stack_sample" { | |
| return; | |
| } | |
| let Some(blob_metadata) = large_blob.metadata else { | |
| return; | |
| }; | |
| let bytes = &large_blob.bytes; | |
| if bytes.len() < 8 { | |
| return; | |
| } | |
| let regs_size = u64::from_ne_bytes(bytes[0..8].try_into().unwrap()) as usize; | |
| let mut offset = 8; | |
| if regs_size == 0 || bytes.len() < offset + regs_size { | |
| return; | |
| } | |
| // 33 u64 general registers (r0-r29, lr, sp, pc), with | |
| // cpsr following them in the zircon thread state. | |
| // Register state of other widths (e.g. an x86_64 | |
| // thread state) is not supported and skipped by this | |
| // size check. | |
| if regs_size < REGS_BYTES + 8 { | |
| return; | |
| } | |
| // The size check above guarantees the 8 cpsr bytes are present. | |
| let thread_cpsr = u64::from_ne_bytes( | |
| bytes[offset + REGS_BYTES..offset + REGS_BYTES + 8].try_into().unwrap(), | |
| ); | |
| let thread_regs = &bytes[offset..offset + REGS_BYTES]; | |
| offset += regs_size; | |
| // The memory captures that follow the registers: the | |
| // thread-state stack, then, for a thread sampled while | |
| // running Starnix code, the restricted-state struct and | |
| // (if it could be read) the stack at the guest SP. | |
| let mut chunks = Vec::<(u64, &[u8])>::new(); | |
| while offset + 16 <= bytes.len() { | |
| let chunk_base = u64::from_ne_bytes(bytes[offset..offset + 8].try_into().unwrap()); | |
| offset += 8; | |
| let chunk_size = u64::from_ne_bytes(bytes[offset..offset + 8].try_into().unwrap()); | |
| offset += 8; | |
| if offset + chunk_size as usize > bytes.len() { | |
| break; | |
| } | |
| chunks.push((chunk_base, &bytes[offset..offset + chunk_size as usize])); | |
| offset += chunk_size as usize; | |
| } | |
| // If the thread was sampled in Starnix (e.g. during a syscall), | |
| // report the guest registers saved in its restricted state rather | |
| // than Starnix's own registers. If no valid restricted-state | |
| // chunk is present, fall back to the thread registers. | |
| let pc_offset = AARCH64_REG_PC * 8; | |
| let thread_pc = | |
| u64::from_ne_bytes(thread_regs[pc_offset..pc_offset + 8].try_into().unwrap()); | |
| let in_starnix = thread_pc >= GUEST_ASPACE_END; | |
| let guest_state = | |
| if in_starnix { guest_regs_from_restricted_state(&chunks) } else { None }; | |
| let reports_guest_state = guest_state.is_some(); | |
| let (regs, cpsr) = guest_state.unwrap_or((thread_regs, thread_cpsr)); | |
| let is_32bit = cpsr_is_aarch32(cpsr); | |
| // Borrow the register bytes directly; only AArch32 needs a | |
| // mutable copy to populate the sp/lr/r15 slots below. | |
| let mut blob_regs = Cow::Borrowed(regs); | |
| if is_32bit { | |
| let blob_regs = blob_regs.to_mut(); | |
| // Zircon reports the AArch32 PC in the pc slot | |
| // (index 32). 64-bit readers take it from there, | |
| // per the Linux compat layout (AArch32 R0-R14 | |
| // arrive in x0-x14); mirror it into the arm32 R15 | |
| // slot (index 15) too for 32-bit readers, which | |
| // only consume indices 0-15 -- x15 carries no | |
| // meaningful value for AArch32 state. | |
| let r15_offset = AARCH32_REG_PC * 8; | |
| blob_regs.copy_within(pc_offset..pc_offset + 8, r15_offset); | |
| // Likewise, AArch32 execution keeps its stack pointer | |
| // and link register in R13 and R14; the sp and lr | |
| // slots hold SP_EL0 and X30, which it never updates. | |
| // Report R13/R14 in the sp/lr slots, as Linux does for | |
| // compat tasks (compat_sp / compat_lr). | |
| let r13_offset = AARCH32_REG_SP * 8; | |
| let sp_offset = AARCH64_REG_SP * 8; | |
| blob_regs.copy_within(r13_offset..r13_offset + 8, sp_offset); | |
| let r14_offset = AARCH32_REG_LR * 8; | |
| let lr_offset = AARCH64_REG_LR * 8; | |
| blob_regs.copy_within(r14_offset..r14_offset + 8, lr_offset); | |
| } | |
| let read_reg = |index: usize| { | |
| u64::from_ne_bytes(blob_regs[index * 8..index * 8 + 8].try_into().unwrap()) | |
| }; | |
| let sp = if is_32bit { | |
| // The arm32 stack pointer is R13. | |
| read_reg(AARCH32_REG_SP) | |
| } else { | |
| read_reg(AARCH64_REG_SP) | |
| }; | |
| let pc = read_reg(AARCH64_REG_PC); | |
| // Select the memory chunk that contains the sampled | |
| // stack pointer and trim it to start exactly there: | |
| // readers overlay the STACK_USER bytes at the SP | |
| // reported in REGS_USER. The blob can carry several | |
| // captures (the thread-state stack, the | |
| // restricted-state struct, and a restricted-SP | |
| // stack); selecting by SP keeps the registers and the | |
| // stack bytes coherent. If no chunk covers the SP, emit | |
| // the sample with an empty stack. | |
| let stack: &[u8] = chunks | |
| .iter() | |
| .find_map(|&(chunk_base, data)| { | |
| (chunk_base <= sp && sp - chunk_base < data.len() as u64) | |
| .then(|| &data[(sp - chunk_base) as usize..]) | |
| }) | |
| .unwrap_or(&[]); | |
| let Some(LinuxIdentity::Thread { pid, tid }) = koid_session.and_then(|s| { | |
| s.resolve_koids( | |
| zx::Koid::from_raw(blob_metadata.process.0), | |
| zx::Koid::from_raw(blob_metadata.thread.0), | |
| ) | |
| }) else { | |
| return; | |
| }; | |
| let time = zx::BootInstant::from_nanos(blob_metadata.timestamp.max(0)); | |
| let regs_abi = if is_32bit { | |
| perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_32 as u64 | |
| } else { | |
| perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_64 as u64 | |
| }; | |
| let perf_record_sample = PerfRecordSample { | |
| pid: Some(pid as u32), | |
| tid: Some(tid as u32), | |
| time, | |
| // The Starnix PC (if any) goes to the kernel section | |
| // of the callchain, the user PC to the user section. | |
| ips: if reports_guest_state { vec![thread_pc, pc] } else { vec![pc] }, | |
| regs: &blob_regs, | |
| stack, | |
| regs_abi, | |
| sample_regs_user, | |
| sample_stack_user, | |
| }; | |
| let metadata = seq_lock_wrapper.get(); | |
| let bytes_written = write_record_to_vmo( | |
| perf_record_sample, | |
| perf_data_vmo, | |
| sample_type, | |
| sample_id, | |
| sample_period, | |
| read_format, | |
| *vmo_write_offset, | |
| &metadata, | |
| lost_events, | |
| ); | |
| // Publish data_head after writing; set_value's | |
| // release-ordered stores make the record contents | |
| // visible to a reader that observes the new head. | |
| if bytes_written > 0 { | |
| *vmo_write_offset += bytes_written; | |
| let mut metadata = seq_lock_wrapper.get(); | |
| metadata.data_head = *vmo_write_offset; | |
| seq_lock_wrapper.set_value(metadata); | |
| } | |
| } | |
| _ => {} | |
| } | |
| } | |
| /// Returns the sampling period requested by `attr`, or `None` if this is not a | |
| /// sampling event. `attr` carries either a frequency (Hz) or a period in the | |
| /// same union, selected by the `freq` bit; the timebase counts nanoseconds. | |
| fn sampling_period(attr: &perf_event_attr) -> Option<zx::MonotonicDuration> { | |
| // SAFETY: Reading a union field requires its bytes to be initialized and to | |
| // be a valid value for the field's type. `attr` is filled in by `read_object` | |
| // (or safe/Default construction), and this union's only fields, `sample_period` | |
| // and `sample_freq`, are both 8-byte `u64`s, so all 8 bytes are initialized | |
| // regardless of which field was written. Any 8 bytes are a valid `u64`, so | |
| // this read is sound. | |
| let value = unsafe { attr.__bindgen_anon_1.sample_period }; | |
| if value == 0 { | |
| return None; | |
| } | |
| let nanos = if attr.freq() != 0 { (1_000_000_000 / value).max(1) } else { value }; | |
| Some(zx::MonotonicDuration::from_nanos(nanos as i64)) | |
| } | |
| // Notifies other thread that we should start/stop sampling. | |
| // Once sampling is complete, that profiler session is no longer needed. | |
| // At that point, send back notification so that this is no longer blocking | |
| // (e.g. so that other profiler sessions can start). | |
| fn ping_receiver( | |
| mut ioctl_sender: future_mpsc::Sender<(IoctlOp, sync_mpsc::Sender<()>)>, | |
| command: IoctlOp, | |
| ) { | |
| log_info!("[perf_event_open] Received sampling command: {:?}", command); | |
| let (profiling_complete_sender, profiling_complete_receiver) = sync_mpsc::channel::<()>(); | |
| match ioctl_sender.try_send((command, profiling_complete_sender)) { | |
| Ok(_) => (), | |
| Err(e) => { | |
| if e.is_full() { | |
| log_warn!("[perf_event_open] Failed to send {:?}: Channel full", command); | |
| } else if e.is_disconnected() { | |
| log_warn!("[perf_event_open] Failed to send {:?}: Receiver disconnected", command); | |
| } else { | |
| log_warn!("[perf_event_open] Failed to send {:?} due to {:?}", command, e.source()); | |
| } | |
| } | |
| }; | |
| // Block on / wait until profiling is complete before returning. | |
| // This notifies that the profiler is free to be used for another session. | |
| let _ = profiling_complete_receiver.recv().unwrap(); | |
| } | |
| // Creates a seq lock for the given VMO. Initializes the seq lock with | |
| // known initial values (unknown values default to 0). | |
| // Does NOT actually save this as a memory object until mmap() is called. | |
| // | |
| // # Safety | |
| // | |
| // The caller must ensure that the kernel maintains exclusive write access to this VMO and | |
| // there are only atomic accesses to this memory (see seq_lock lib.rs for details). | |
| unsafe fn create_seq_lock( | |
| vmo_handle_ref: &zx::NullableHandle, | |
| buffer_size: u64, | |
| ) -> SeqLock<PerfMetadataHeader, PerfMetadataValue> { | |
| // Currently we hardcode everything just to get something E2E working. | |
| let metadata_header = PerfMetadataHeader { version: 1, compat_version: 2 }; | |
| let page_size = *PAGE_SIZE; | |
| let metadata_value = PerfMetadataValue { | |
| lock: 0, | |
| index: 3, | |
| offset: 19337, | |
| time_enabled: 0, | |
| time_running: 0, | |
| __bindgen_anon_1: perf_event_mmap_page__bindgen_ty_1 { capabilities: 30 }, | |
| pmc_width: 0, | |
| time_shift: 0, | |
| time_mult: 0, | |
| time_offset: 0, | |
| time_zero: 0, | |
| size: 0, | |
| __reserved_1: 0, | |
| time_cycles: 0, | |
| time_mask: 0, | |
| __reserved: [0; 928usize], | |
| // This first page (metadata) has finished writing. Start data_head at 0. | |
| data_head: 0, | |
| // Start reading from 0; it is the user's responsibility to increment on their end. | |
| data_tail: 0, | |
| // We know the data will start after 1 page size so we can set this now. | |
| data_offset: page_size, | |
| data_size: buffer_size - page_size, | |
| aux_head: 0, | |
| aux_tail: 0, | |
| aux_offset: 0, | |
| aux_size: 0, | |
| }; | |
| let vmo = zx::Vmo::from(vmo_handle_ref.duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap()); | |
| // Create a SeqLock and safely initialize the `header` and `value` for it. | |
| // SeqLock is formatted thusly: | |
| // header_struct : any size, params `version` and `compat_version` should not change | |
| // sequence_counter : u32, this is the lock and should increment | |
| // value_struct : any size, each param can change | |
| // | |
| // SAFETY: See safety requirements on `create_seq_lock`. | |
| unsafe { | |
| SeqLock::new_from_vmo(metadata_header, metadata_value, vmo) | |
| .expect("failed to create seq_lock for perf metadata") | |
| } | |
| } | |
| pub fn sys_perf_event_open( | |
| current_task: &CurrentTask, | |
| attr: UserRef<perf_event_attr>, | |
| // Note that this is pid in Linux docs. | |
| tid: tid_t, | |
| cpu: i32, | |
| group_fd: FdNumber, | |
| _flags: u64, | |
| ) -> Result<SyscallResult, Errno> { | |
| // So far, the implementation only sets the read_data_format according to the "Reading results" | |
| // section of https://man7.org/linux/man-pages/man2/perf_event_open.2.html for a single event. | |
| // Other features will be added in the future (see below track_stubs). | |
| let perf_event_attrs: perf_event_attr = current_task.read_object(attr)?; | |
| if (perf_event_attrs.sample_type & perf_event_sample_format_PERF_SAMPLE_STACK_USER as u64) != 0 | |
| { | |
| if perf_event_attrs.sample_stack_user % 8 != 0 { | |
| return error!(EINVAL); | |
| } | |
| } | |
| if tid == -1 && cpu == -1 { | |
| return error!(EINVAL); | |
| } | |
| let target = match tid { | |
| -1 => PerfEventTarget::AllTasks, | |
| 0 => PerfEventTarget::CurrentTask, | |
| _ => { | |
| track_stub!(TODO("https://fxbug.dev/409621963"), "[perf_event_open] implement tid > 0"); | |
| return error!(ENOSYS); | |
| } | |
| }; | |
| let event_type: PerfEventType = perf_event_attrs.type_.try_into()?; | |
| // Check access to open any perf_event first, before any capability check, so that it is | |
| // evaluated and audited first. | |
| security::check_perf_event_open_access(current_task, PerfEventOpenType::Open)?; | |
| // Only require CAP_PERFMON (or CAP_SYS_ADMIN) when `perf_event_paranoid` exceeds the | |
| // threshold for the requested event. | |
| // TODO(https://fxbug.dev/427246425): Starnix currently starts a job-wide profiler session | |
| // regardless of `target`, so apply the CPU-wide threshold (`paranoid > 0`) to all targets | |
| // until sampling is scoped to the target task. | |
| let paranoid = current_task.kernel().perf_event_paranoid.load(Ordering::Relaxed); | |
| let requires_capability = (paranoid > 1 && perf_event_attrs.exclude_kernel() == 0) | |
| || paranoid > 0 | |
| || (paranoid > -1 && event_type == PerfEventType::Tracepoint); | |
| // Exceptionally, a tracepoint perf event on the current task that excludes the kernel never | |
| // requires a capability. The kernel and CPU access checks below do not apply to such an event. | |
| let is_exempt = matches!( | |
| (event_type, perf_event_attrs.exclude_kernel(), &target), | |
| (PerfEventType::Tracepoint, 1, PerfEventTarget::CurrentTask) | |
| ); | |
| // CAP_SYS_ADMIN is checked without auditing, so that nothing is audit-logged if the caller has | |
| // either capability, and only CAP_PERFMON is audit-logged if it has neither. | |
| if requires_capability | |
| && !is_exempt | |
| && !security::is_task_capable_noaudit(current_task, CAP_SYS_ADMIN) | |
| { | |
| security::check_task_capable(current_task, CAP_PERFMON).map_err(|_| errno!(EACCES))?; | |
| } | |
| // Check access to monitor the kernel when `exclude_kernel` is 0. | |
| if perf_event_attrs.exclude_kernel() == 0 { | |
| security::check_perf_event_open_access(current_task, PerfEventOpenType::Kernel)?; | |
| } | |
| // Check access to monitor CPU-wide activity when | |
| // - type is PERF_TYPE_SOFTWARE or | |
| // - type is PERF_TYPE_HARDWARE or | |
| // - type is in [CACHE, TRACEPOINT, BREAKPOINT, RAW) and pid == -1 | |
| // - or when a low `perf_event_paranoid` waived the capability requirement | |
| // (TODO(https://fxbug.dev/427246425): sampling is currently job-wide regardless of | |
| // target task or event type). | |
| let check_cpu = !requires_capability | |
| || match event_type { | |
| PerfEventType::Software | PerfEventType::Hardware => true, | |
| PerfEventType::HwCache | |
| | PerfEventType::Tracepoint | |
| | PerfEventType::Breakpoint | |
| | PerfEventType::Raw => matches!(target, PerfEventTarget::AllTasks), | |
| }; | |
| if check_cpu { | |
| security::check_perf_event_open_access(current_task, PerfEventOpenType::Cpu)?; | |
| } | |
| // Check access to monitor tracepoints when the type is PERF_TYPE_TRACEPOINT. | |
| if event_type == PerfEventType::Tracepoint { | |
| security::check_perf_event_open_access(current_task, PerfEventOpenType::Tracepoint)?; | |
| } | |
| // Channel used to send info between notifier and spawned task thread. | |
| // We somewhat arbitrarily picked 8 for now in case we get a bunch of ioctls that are in | |
| // quick succession (instead of something lower). | |
| let (sender, mut receiver) = future_mpsc::channel(8); | |
| let mut perf_event_file = PerfEventFileState::new( | |
| perf_event_attrs, | |
| 0, | |
| perf_event_attrs.disabled(), | |
| perf_event_attrs.sample_type, | |
| zx::Vmo::create(ESTIMATED_MMAP_BUFFER_SIZE).unwrap(), | |
| sender, | |
| ); | |
| let read_format = perf_event_attrs.read_format; | |
| if (read_format & perf_event_read_format_PERF_FORMAT_TOTAL_TIME_ENABLED as u64) != 0 | |
| || (read_format & perf_event_read_format_PERF_FORMAT_TOTAL_TIME_RUNNING as u64) != 0 | |
| { | |
| // Only keep track of most_recent_enabled_time if we are currently in ENABLED state, | |
| // as otherwise this param shouldn't be used for calculating anything. | |
| if perf_event_file.disabled == 0 { | |
| perf_event_file.most_recent_enabled_time = | |
| zx::MonotonicInstant::get().into_nanos() as u64; | |
| } | |
| // Initialize this to 0 as we will need to return a time duration later during read(). | |
| perf_event_file.total_time_running = 0; | |
| } | |
| let event_id = READ_FORMAT_ID_GENERATOR.fetch_add(1, Ordering::Relaxed); | |
| perf_event_file.rf_id = event_id; | |
| if group_fd.raw() == -1 { | |
| perf_event_file.sample_id = event_id; | |
| } else { | |
| let group_file = current_task.files().get(group_fd)?; | |
| let group_file_object_id = group_file.id; | |
| let perf_state = get_perf_state(¤t_task.kernel); | |
| let events = perf_state.format_id_lookup_table.lock(); | |
| if let Some(rf_id) = events.get(&group_file_object_id) { | |
| perf_event_file.sample_id = *rf_id; | |
| } else { | |
| return error!(EINVAL); | |
| } | |
| } | |
| if (read_format & perf_event_read_format_PERF_FORMAT_GROUP as u64) != 0 { | |
| track_stub!( | |
| TODO("https://fxbug.dev/402238049"), | |
| "[perf_event_open] implement read_format group" | |
| ); | |
| return error!(ENOSYS); | |
| } | |
| if (read_format & perf_event_read_format_PERF_FORMAT_LOST as u64) != 0 { | |
| track_stub!( | |
| TODO("https://fxbug.dev/402260383"), | |
| "[perf_event_open] implement read_format lost" | |
| ); | |
| } | |
| // Set up notifier for handling ioctl calls to enable/disable sampling. | |
| let mut vmo_handle_copy = | |
| perf_event_file.perf_data_vmo.as_handle_ref().duplicate_handle(zx::Rights::SAME_RIGHTS); | |
| let sample_period = sampling_period(&perf_event_file.attr); | |
| // SeqLock does not get instantiated with metadata values until mmap() is called. | |
| let seq_lock = | |
| Arc::new(OnceLock::<Result<SeqLock<PerfMetadataHeader, PerfMetadataValue>, Errno>>::new()); | |
| let cloned_seq_lock = Arc::clone(&seq_lock); | |
| let mut vmo_write_offset = 0; | |
| let closure = async move |kthread_task: &CurrentTask| { | |
| let mut lost_events: u64 = 0; | |
| // Each iteration waits for an Enable and then runs one session. | |
| while let Some((command, profiling_complete_receiver)) = receiver.next().await { | |
| // We only expect Enable when no session is active. | |
| if command != IoctlOp::Enable { | |
| let _ = profiling_complete_receiver.send(()); | |
| continue; | |
| } | |
| // Enable is only sent for sampling events, but guard anyway. | |
| let Some(sample_period) = sample_period else { | |
| let _ = profiling_complete_receiver.send(()); | |
| continue; | |
| }; | |
| let callgraph = callgraph_config( | |
| perf_event_file.sample_type, | |
| perf_event_file.attr.sample_stack_user, | |
| ); | |
| let (session_proxy, client) = match set_up_profiler(sample_period, callgraph).await { | |
| Ok(session) => session, | |
| Err(e) => { | |
| log_warn!("Failed to profile: {}", e); | |
| let _ = profiling_complete_receiver.send(()); | |
| continue; | |
| } | |
| }; | |
| // Record pid/koid mappings before the profiler starts sampling | |
| // so every sampled thread can be resolved. Dropping the session | |
| // at the end of the profiling session ends the recording interest. | |
| let pid_koid_session = kthread_task.kernel().trace_event_manager.open(); | |
| let start_request = | |
| profiler::SessionStartRequest { buffer_results: Some(false), ..Default::default() }; | |
| if let Err(e) = session_proxy.start(&start_request).await { | |
| log_warn!("Failed to start profiler: {:?}", e); | |
| let _ = profiling_complete_receiver.send(()); | |
| continue; | |
| } | |
| let _ = profiling_complete_receiver.send(()); | |
| let vmo = zx::Vmo::from( | |
| vmo_handle_copy | |
| .as_mut() | |
| .expect("Failed to get VMO handle") | |
| .as_handle_ref() | |
| .duplicate_handle(zx::Rights::SAME_RIGHTS) | |
| .unwrap(), | |
| ); | |
| let mut handle_record = |record: TraceRecord| { | |
| // Records can only be written once mmap() has set up the | |
| // ring buffer. | |
| if let Some(Ok(seq_lock_wrapper)) = cloned_seq_lock.get() { | |
| process_fxt_record( | |
| record, | |
| seq_lock_wrapper, | |
| &vmo, | |
| perf_event_file.sample_type, | |
| perf_event_file.sample_id, | |
| sample_period.into_nanos() as u64, | |
| perf_event_file.attr.read_format, | |
| perf_event_file.attr.sample_regs_user, | |
| perf_event_file.attr.sample_stack_user as u64, | |
| Some(&pid_koid_session), | |
| &mut vmo_write_offset, | |
| &mut lost_events, | |
| ); | |
| } | |
| }; | |
| // Pump records from the profiler into the ring buffer until a | |
| // Disable arrives: readers poll the ring during the session, and | |
| // the profiler's socket must be drained continuously. | |
| let (stream, _parser_task) = SessionParser::new_async(client); | |
| let mut stream = stream.fuse(); | |
| let mut stream_ended = false; | |
| let disable_ack = loop { | |
| if stream_ended { | |
| // The profiler closed the socket early; only commands | |
| // remain. | |
| match receiver.next().await { | |
| Some((IoctlOp::Disable, ack)) => break Some(ack), | |
| Some((IoctlOp::Enable, ack)) => { | |
| // A session is already active. | |
| let _ = ack.send(()); | |
| } | |
| None => break None, | |
| } | |
| } else { | |
| futures::select_biased! { | |
| cmd = receiver.next() => match cmd { | |
| Some((IoctlOp::Disable, ack)) => break Some(ack), | |
| Some((IoctlOp::Enable, ack)) => { | |
| // A session is already active. | |
| let _ = ack.send(()); | |
| } | |
| None => break None, | |
| }, | |
| record = stream.next() => match record { | |
| Some(Ok(record)) => handle_record(record), | |
| Some(Err(e)) => { | |
| // The stream is desynchronized and the parser | |
| // would keep returning this error, so end the | |
| // session. The teardown below drops the | |
| // socket, which unblocks the profiler if it is | |
| // stalled writing to it. | |
| log_warn!("[perf_event_open] Error parsing FXT: {:?}", e); | |
| stream_ended = true; | |
| break None; | |
| } | |
| None => { | |
| log_warn!("[perf_event_open] Profiler stream ended mid-session"); | |
| stream_ended = true; | |
| } | |
| }, | |
| } | |
| } | |
| }; | |
| // Tear the session down while still draining: the profiler is a | |
| // separate process whose socket writes block, so if its socket is | |
| // full it is stalled mid-write and cannot service stop() or | |
| // reset() until we keep reading. Awaiting either without pumping | |
| // would deadlock the two. | |
| let stop_and_reset = async { | |
| match session_proxy.stop().await { | |
| Ok(stats) => log_info!( | |
| "[perf_event_open] profiler samples_collected: {:?}", | |
| stats.samples_collected.unwrap_or(0) | |
| ), | |
| Err(e) => log_warn!("[perf_event_open] Failed to stop profiler: {:?}", e), | |
| } | |
| // Reset flushes the remaining data and closes the socket, | |
| // which is what ends the drain below. | |
| let _ = session_proxy.reset().await; | |
| }; | |
| let drain = async move { | |
| if !stream_ended { | |
| while let Some(record) = stream.next().await { | |
| match record { | |
| Ok(record) => handle_record(record), | |
| Err(e) => { | |
| // The stream is desynchronized, so further | |
| // records cannot be parsed. | |
| log_warn!("[perf_event_open] Error parsing FXT: {:?}", e); | |
| break; | |
| } | |
| } | |
| } | |
| } | |
| // Hand the profiler a closed socket so that a write it is | |
| // blocked on fails instead of hanging forever. | |
| drop(stream); | |
| drop(_parser_task); | |
| }; | |
| futures::join!(stop_and_reset, drain); | |
| // The Disable ioctl returns only after the drain above, so the | |
| // reader sees every record once the ioctl completes. | |
| if let Some(ack) = disable_ack { | |
| let _ = ack.send(()); | |
| } | |
| } | |
| }; | |
| let req = SpawnRequestBuilder::new() | |
| .with_debug_name("perf-event-sampler") | |
| .with_async_closure(closure) | |
| .build(); | |
| current_task.kernel().kthreads.spawner().spawn_from_request(req); | |
| let file = Box::new(PerfEventFile { | |
| _tid: tid, | |
| _cpu: cpu, | |
| perf_event_file: perf_event_file.into(), | |
| security_state: security::perf_event_alloc(current_task), | |
| seq_lock: seq_lock, | |
| }); | |
| // TODO: https://fxbug.dev/404739824 - Confirm whether to handle this as a "private" node. | |
| let file_handle = Anon::new_private_file(current_task, file, OpenFlags::RDWR, "[perf_event]"); | |
| let file_object_id = file_handle.id; | |
| let file_descriptor: Result<FdNumber, Errno> = | |
| current_task.add_file(file_handle, FdFlags::empty()); | |
| match file_descriptor { | |
| Ok(fd) => { | |
| if group_fd.raw() == -1 { | |
| let perf_state = get_perf_state(¤t_task.kernel); | |
| let mut events = perf_state.format_id_lookup_table.lock(); | |
| events.insert(file_object_id, event_id); | |
| } | |
| Ok(fd.into()) | |
| } | |
| Err(_) => { | |
| track_stub!( | |
| TODO("https://fxbug.dev/402453955"), | |
| "[perf_event_open] implement remaining error handling" | |
| ); | |
| error!(EMFILE) | |
| } | |
| } | |
| } | |
| // Syscalls for arch32 usage | |
| #[cfg(target_arch = "aarch64")] | |
| mod arch32 { | |
| pub use super::sys_perf_event_open as sys_arch32_perf_event_open; | |
| } | |
| #[cfg(target_arch = "aarch64")] | |
| pub use arch32::*; | |
| use crate::mm::memory::MemoryObject; | |
| use crate::mm::{MemoryAccessorExt, ProtectionFlags}; | |
| use crate::task::CurrentTask; | |
| use crate::vfs::{ | |
| Anon, FdFlags, FdNumber, FileObject, FileObjectId, FileObjectState, FileOps, InputBuffer, | |
| OutputBuffer, | |
| }; | |
| use crate::{fileops_impl_nonseekable, fileops_impl_noop_sync}; | |
| #[cfg(test)] | |
| mod tests { | |
| use super::*; | |
| use crate::task::tracing::{TracePerformanceEventManager, ZirconIdentity}; | |
| #[::fuchsia::test] | |
| async fn test_process_fxt_record_resolves_pid_tid() { | |
| let manager = Arc::new(TracePerformanceEventManager::new(std::sync::Weak::new())); | |
| let session = manager.open(); | |
| manager.record( | |
| 42, | |
| 43, | |
| ZirconIdentity { process: zx::Koid::from_raw(1001), thread: zx::Koid::from_raw(1002) }, | |
| ); | |
| let perf_data_vmo = zx::Vmo::create(ESTIMATED_MMAP_BUFFER_SIZE).unwrap(); | |
| let vmo_handle_copy = | |
| perf_data_vmo.as_handle_ref().duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(); | |
| // SAFETY: The test maintains exclusive write access to this VMO. | |
| let seq_lock = unsafe { create_seq_lock(&vmo_handle_copy, ESTIMATED_MMAP_BUFFER_SIZE) }; | |
| let sample_type = (perf_event_sample_format_PERF_SAMPLE_IP | |
| | perf_event_sample_format_PERF_SAMPLE_TID) as u64; | |
| let mut vmo_write_offset = 0; | |
| let mut lost_events = 0; | |
| // Mapped sample: process 1001, thread 1002 -> should resolve to pid 42, tid 43. | |
| let mapped_record = | |
| TraceRecord::Profiler(ProfilerRecord::Backtrace(fxt::profiler::BacktraceRecord { | |
| timestamp: 1000, | |
| process: fxt::ProcessKoid(1001), | |
| thread: fxt::ThreadKoid(1002), | |
| num_records: 1, | |
| data: vec![0x12345678], | |
| })); | |
| process_fxt_record( | |
| mapped_record, | |
| &seq_lock, | |
| &perf_data_vmo, | |
| sample_type, | |
| 0, | |
| 0, | |
| 0, | |
| 0, | |
| 0, | |
| Some(&session), | |
| &mut vmo_write_offset, | |
| &mut lost_events, | |
| ); | |
| // Header (8 bytes) + IP (8 bytes) + PID/TID (8 bytes) = 24 bytes. | |
| let expected_record_size: u64 = 24; | |
| assert_eq!(vmo_write_offset, expected_record_size); | |
| let metadata = seq_lock.get(); | |
| assert_eq!(metadata.data_head, expected_record_size); | |
| let mut record_bytes = [0u8; 24]; | |
| perf_data_vmo.read(&mut record_bytes, metadata.data_offset).unwrap(); | |
| let record_type = u32::from_ne_bytes(record_bytes[0..4].try_into().unwrap()); | |
| assert_eq!(record_type, perf_event_type_PERF_RECORD_SAMPLE); | |
| let misc = u16::from_ne_bytes(record_bytes[4..6].try_into().unwrap()); | |
| assert_eq!(misc, PERF_RECORD_MISC_USER as u16); | |
| let size = u16::from_ne_bytes(record_bytes[6..8].try_into().unwrap()); | |
| assert_eq!(size, expected_record_size as u16); | |
| let ip = u64::from_ne_bytes(record_bytes[8..16].try_into().unwrap()); | |
| assert_eq!(ip, 0x12345678); | |
| let pid = u32::from_ne_bytes(record_bytes[16..20].try_into().unwrap()); | |
| assert_eq!(pid, 42); | |
| let tid = u32::from_ne_bytes(record_bytes[20..24].try_into().unwrap()); | |
| assert_eq!(tid, 43); | |
| // Unmapped sample: process 9999, thread 9998 -> should be dropped. | |
| let unmapped_record = | |
| TraceRecord::Profiler(ProfilerRecord::Backtrace(fxt::profiler::BacktraceRecord { | |
| timestamp: 2000, | |
| process: fxt::ProcessKoid(9999), | |
| thread: fxt::ThreadKoid(9998), | |
| num_records: 1, | |
| data: vec![0x87654321], | |
| })); | |
| process_fxt_record( | |
| unmapped_record, | |
| &seq_lock, | |
| &perf_data_vmo, | |
| sample_type, | |
| 0, | |
| 0, | |
| 0, | |
| 0, | |
| 0, | |
| Some(&session), | |
| &mut vmo_write_offset, | |
| &mut lost_events, | |
| ); | |
| // Verify that unmapped sample was dropped and no extra bytes were written. | |
| assert_eq!(vmo_write_offset, expected_record_size); | |
| let metadata = seq_lock.get(); | |
| assert_eq!(metadata.data_head, expected_record_size); | |
| let mut trailing_bytes = [0u8; 24]; | |
| perf_data_vmo | |
| .read(&mut trailing_bytes, metadata.data_offset + expected_record_size) | |
| .unwrap(); | |
| assert_eq!(trailing_bytes, [0u8; 24]); | |
| } | |
| #[::fuchsia::test] | |
| async fn test_write_record_to_vmo_regs_and_stack() { | |
| let perf_data_vmo = zx::Vmo::create(ESTIMATED_MMAP_BUFFER_SIZE).unwrap(); | |
| let vmo_handle_copy = | |
| perf_data_vmo.as_handle_ref().duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(); | |
| // SAFETY: The test maintains exclusive write access to this VMO. | |
| let seq_lock = unsafe { create_seq_lock(&vmo_handle_copy, ESTIMATED_MMAP_BUFFER_SIZE) }; | |
| let metadata = seq_lock.get(); | |
| let mut lost_events = 0; | |
| let regs_data = [0x11u8; 16]; | |
| let stack_data = [0x22u8; 16]; | |
| let sample = PerfRecordSample { | |
| pid: Some(10), | |
| tid: Some(20), | |
| time: zx::BootInstant::from_nanos(123_456_789), | |
| ips: vec![0xdeadbeef], | |
| regs: ®s_data, | |
| stack: &stack_data, | |
| regs_abi: perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_32 as u64, | |
| sample_regs_user: 3, | |
| sample_stack_user: 16, | |
| }; | |
| let sample_type = (perf_event_sample_format_PERF_SAMPLE_IP | |
| | perf_event_sample_format_PERF_SAMPLE_TID | |
| | perf_event_sample_format_PERF_SAMPLE_TIME | |
| | perf_event_sample_format_PERF_SAMPLE_REGS_USER | |
| | perf_event_sample_format_PERF_SAMPLE_STACK_USER) as u64; | |
| let written = write_record_to_vmo( | |
| sample, | |
| &perf_data_vmo, | |
| sample_type, | |
| 0, | |
| 0, | |
| 0, | |
| 0, | |
| &metadata, | |
| &mut lost_events, | |
| ); | |
| assert!(written > 0); | |
| let mut record_bytes = vec![0u8; written as usize]; | |
| perf_data_vmo.read(&mut record_bytes, metadata.data_offset).unwrap(); | |
| let record_type = u32::from_ne_bytes(record_bytes[0..4].try_into().unwrap()); | |
| assert_eq!(record_type, perf_event_type_PERF_RECORD_SAMPLE); | |
| let misc = u16::from_ne_bytes(record_bytes[4..6].try_into().unwrap()); | |
| assert_eq!(misc, PERF_RECORD_MISC_USER as u16); | |
| let size = u16::from_ne_bytes(record_bytes[6..8].try_into().unwrap()); | |
| assert_eq!(size as usize, written as usize); | |
| let ip = u64::from_ne_bytes(record_bytes[8..16].try_into().unwrap()); | |
| assert_eq!(ip, 0xdeadbeef); | |
| let pid = u32::from_ne_bytes(record_bytes[16..20].try_into().unwrap()); | |
| assert_eq!(pid, 10); | |
| let tid = u32::from_ne_bytes(record_bytes[20..24].try_into().unwrap()); | |
| assert_eq!(tid, 20); | |
| let time = u64::from_ne_bytes(record_bytes[24..32].try_into().unwrap()); | |
| assert_eq!(time, 123_456_789); | |
| // REGS_USER: abi (8 bytes) + regs (16 bytes) | |
| let abi = u64::from_ne_bytes(record_bytes[32..40].try_into().unwrap()); | |
| assert_eq!(abi, perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_32 as u64); | |
| assert_eq!(&record_bytes[40..56], ®s_data); | |
| // STACK_USER: dest_len (8 bytes) + data (16 bytes) + dyn_size (8 bytes) | |
| let stack_len = u64::from_ne_bytes(record_bytes[56..64].try_into().unwrap()); | |
| assert_eq!(stack_len, 16); | |
| assert_eq!(&record_bytes[64..80], &stack_data); | |
| let dyn_size = u64::from_ne_bytes(record_bytes[80..88].try_into().unwrap()); | |
| assert_eq!(dyn_size, 16); | |
| } | |
| #[::fuchsia::test] | |
| async fn test_write_record_to_vmo_regs_abi_64() { | |
| let perf_data_vmo = zx::Vmo::create(ESTIMATED_MMAP_BUFFER_SIZE).unwrap(); | |
| let vmo_handle_copy = | |
| perf_data_vmo.as_handle_ref().duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(); | |
| // SAFETY: The test maintains exclusive write access to this VMO. | |
| let seq_lock = unsafe { create_seq_lock(&vmo_handle_copy, ESTIMATED_MMAP_BUFFER_SIZE) }; | |
| let metadata = seq_lock.get(); | |
| let mut lost_events = 0; | |
| let regs_data = [0x44u8; 16]; | |
| let stack_data = [0x55u8; 16]; | |
| let sample = PerfRecordSample { | |
| pid: Some(10), | |
| tid: Some(20), | |
| time: zx::BootInstant::from_nanos(123_456_789), | |
| ips: vec![0xdeadbeef], | |
| regs: ®s_data, | |
| stack: &stack_data, | |
| regs_abi: perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_64 as u64, | |
| sample_regs_user: 3, | |
| sample_stack_user: 16, | |
| }; | |
| let sample_type = (perf_event_sample_format_PERF_SAMPLE_IP | |
| | perf_event_sample_format_PERF_SAMPLE_TID | |
| | perf_event_sample_format_PERF_SAMPLE_TIME | |
| | perf_event_sample_format_PERF_SAMPLE_REGS_USER | |
| | perf_event_sample_format_PERF_SAMPLE_STACK_USER) as u64; | |
| let written = write_record_to_vmo( | |
| sample, | |
| &perf_data_vmo, | |
| sample_type, | |
| 0, | |
| 0, | |
| 0, | |
| 0, | |
| &metadata, | |
| &mut lost_events, | |
| ); | |
| assert!(written > 0); | |
| let mut record_bytes = vec![0u8; written as usize]; | |
| perf_data_vmo.read(&mut record_bytes, metadata.data_offset).unwrap(); | |
| // REGS_USER: abi (8 bytes) + regs (16 bytes) | |
| let abi = u64::from_ne_bytes(record_bytes[32..40].try_into().unwrap()); | |
| assert_eq!(abi, perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_64 as u64); | |
| assert_eq!(&record_bytes[40..56], ®s_data); | |
| } | |
| #[::fuchsia::test] | |
| async fn test_write_record_to_vmo_regs_abi_none_when_empty() { | |
| let perf_data_vmo = zx::Vmo::create(ESTIMATED_MMAP_BUFFER_SIZE).unwrap(); | |
| let vmo_handle_copy = | |
| perf_data_vmo.as_handle_ref().duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(); | |
| // SAFETY: The test maintains exclusive write access to this VMO. | |
| let seq_lock = unsafe { create_seq_lock(&vmo_handle_copy, ESTIMATED_MMAP_BUFFER_SIZE) }; | |
| let metadata = seq_lock.get(); | |
| let mut lost_events = 0; | |
| let stack_data = [0x33u8; 8]; | |
| let sample = PerfRecordSample { | |
| pid: Some(10), | |
| tid: Some(20), | |
| time: zx::BootInstant::from_nanos(999), | |
| ips: vec![0x1000], | |
| regs: &[], | |
| stack: &stack_data, | |
| regs_abi: perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_NONE as u64, | |
| sample_regs_user: 3, | |
| sample_stack_user: 8, | |
| }; | |
| let sample_type = (perf_event_sample_format_PERF_SAMPLE_IP | |
| | perf_event_sample_format_PERF_SAMPLE_TID | |
| | perf_event_sample_format_PERF_SAMPLE_TIME | |
| | perf_event_sample_format_PERF_SAMPLE_REGS_USER | |
| | perf_event_sample_format_PERF_SAMPLE_STACK_USER) as u64; | |
| let written = write_record_to_vmo( | |
| sample, | |
| &perf_data_vmo, | |
| sample_type, | |
| 0, | |
| 0, | |
| 0, | |
| 0, | |
| &metadata, | |
| &mut lost_events, | |
| ); | |
| assert!(written > 0); | |
| let mut record_bytes = vec![0u8; written as usize]; | |
| perf_data_vmo.read(&mut record_bytes, metadata.data_offset).unwrap(); | |
| // REGS_USER: abi (8 bytes) should be PERF_SAMPLE_REGS_ABI_NONE (0), and no regs follow. | |
| let abi = u64::from_ne_bytes(record_bytes[32..40].try_into().unwrap()); | |
| assert_eq!(abi, perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_NONE as u64); | |
| // STACK_USER immediately follows the ABI tag at offset 40. | |
| let stack_len = u64::from_ne_bytes(record_bytes[40..48].try_into().unwrap()); | |
| assert_eq!(stack_len, 8); | |
| assert_eq!(&record_bytes[48..56], &stack_data); | |
| let dyn_size = u64::from_ne_bytes(record_bytes[56..64].try_into().unwrap()); | |
| assert_eq!(dyn_size, 8); | |
| } | |
| // Stack-sample (DWARF) blobs, laid out as the profiler's StackSampler | |
| // emits them: [u64 regs_size][thread-state regs][{u64 base, u64 size, | |
| // bytes}...]. | |
| const STARNIX_PC: u64 = GUEST_ASPACE_END + 0x2222_0558; | |
| const STARNIX_SP: u64 = GUEST_ASPACE_END + 0x4000_1000; | |
| const RESTRICTED_STATE_ADDR: u64 = GUEST_ASPACE_END + 0x5000_0000; | |
| const GUEST_PC: u64 = 0x7a00_1234; | |
| const GUEST_SP: u64 = 0x7f00_1000; | |
| const ALL_REGS_MASK: u64 = (1 << 33) - 1; | |
| const SAMPLE_STACK_USER: u64 = 256; | |
| fn u64s_to_bytes(values: &[u64]) -> Vec<u8> { | |
| values.iter().flat_map(|value| value.to_ne_bytes()).collect() | |
| } | |
| fn read_u64(bytes: &[u8], pos: &mut usize) -> u64 { | |
| let value = u64::from_ne_bytes(bytes[*pos..*pos + 8].try_into().unwrap()); | |
| *pos += 8; | |
| value | |
| } | |
| // An arm64 zx_thread_state_general_regs_t: r0-r29, lr, sp, pc, cpsr, | |
| // tpidr. | |
| fn thread_state_regs(pc: u64, sp: u64, cpsr: u64) -> Vec<u64> { | |
| let mut regs: Vec<u64> = (0..31).map(|i| 0xa000 + i).collect(); | |
| regs.extend([sp, pc, cpsr, 0x7777]); | |
| regs | |
| } | |
| fn stack_sample_blob(thread_regs: &[u64], chunks: &[(u64, &[u8])]) -> Vec<u8> { | |
| let regs_bytes = u64s_to_bytes(thread_regs); | |
| let mut blob = (regs_bytes.len() as u64).to_ne_bytes().to_vec(); | |
| blob.extend(regs_bytes); | |
| for (base, data) in chunks { | |
| blob.extend(base.to_ne_bytes()); | |
| blob.extend((data.len() as u64).to_ne_bytes()); | |
| blob.extend_from_slice(data); | |
| } | |
| blob | |
| } | |
| // A zx_restricted_state_t with x0-x30 from `x`, laid out per the | |
| // struct's field offsets. | |
| #[cfg(target_arch = "aarch64")] | |
| fn restricted_state_bytes(x: &[u64; 31], sp: u64, pc: u64, cpsr: u32) -> Vec<u8> { | |
| use zx::sys::zx_restricted_state_t; | |
| let mut state = vec![0u8; std::mem::size_of::<zx_restricted_state_t>()]; | |
| state[..31 * 8].copy_from_slice(&u64s_to_bytes(x)); | |
| let sp_offset = std::mem::offset_of!(zx_restricted_state_t, sp); | |
| state[sp_offset..sp_offset + 8].copy_from_slice(&sp.to_ne_bytes()); | |
| let pc_offset = std::mem::offset_of!(zx_restricted_state_t, pc); | |
| state[pc_offset..pc_offset + 8].copy_from_slice(&pc.to_ne_bytes()); | |
| let cpsr_offset = std::mem::offset_of!(zx_restricted_state_t, cpsr); | |
| state[cpsr_offset..cpsr_offset + 4].copy_from_slice(&cpsr.to_ne_bytes()); | |
| state | |
| } | |
| fn stack_bytes(seed: u8, len: usize) -> Vec<u8> { | |
| (0..len).map(|i| seed.wrapping_add(i as u8)).collect() | |
| } | |
| struct ParsedStackSample { | |
| ip: u64, | |
| pid: u32, | |
| tid: u32, | |
| callchain: Vec<u64>, | |
| abi: u64, | |
| regs: Vec<u64>, | |
| stack: Vec<u8>, | |
| } | |
| // Runs `blob` through process_fxt_record as a sample of process 1001 / | |
| // thread 1002 (pid 42 / tid 43) and parses the one record it writes. | |
| fn process_stack_sample(blob: Vec<u8>, sample_regs_user: u64) -> ParsedStackSample { | |
| let manager = Arc::new(TracePerformanceEventManager::new(std::sync::Weak::new())); | |
| let session = manager.open(); | |
| manager.record( | |
| 42, | |
| 43, | |
| ZirconIdentity { process: zx::Koid::from_raw(1001), thread: zx::Koid::from_raw(1002) }, | |
| ); | |
| let perf_data_vmo = zx::Vmo::create(ESTIMATED_MMAP_BUFFER_SIZE).unwrap(); | |
| let vmo_handle_copy = | |
| perf_data_vmo.as_handle_ref().duplicate_handle(zx::Rights::SAME_RIGHTS).unwrap(); | |
| // SAFETY: The test maintains exclusive write access to this VMO. | |
| let seq_lock = unsafe { create_seq_lock(&vmo_handle_copy, ESTIMATED_MMAP_BUFFER_SIZE) }; | |
| let sample_type = (perf_event_sample_format_PERF_SAMPLE_IP | |
| | perf_event_sample_format_PERF_SAMPLE_TID | |
| | perf_event_sample_format_PERF_SAMPLE_CALLCHAIN | |
| | perf_event_sample_format_PERF_SAMPLE_REGS_USER | |
| | perf_event_sample_format_PERF_SAMPLE_STACK_USER) as u64; | |
| let record = TraceRecord::LargeBlob(fxt::LargeBlobRecord { | |
| provider: None, | |
| ty: fxt::BlobType::Data, | |
| category: "cpu_profiler".into(), | |
| name: "stack_sample".into(), | |
| bytes: blob, | |
| metadata: Some(fxt::LargeBlobMetadata { | |
| timestamp: 1000, | |
| process: fxt::ProcessKoid(1001), | |
| thread: fxt::ThreadKoid(1002), | |
| args: vec![], | |
| }), | |
| }); | |
| let mut vmo_write_offset = 0; | |
| let mut lost_events = 0; | |
| process_fxt_record( | |
| record, | |
| &seq_lock, | |
| &perf_data_vmo, | |
| sample_type, | |
| 0, | |
| 0, | |
| 0, | |
| sample_regs_user, | |
| SAMPLE_STACK_USER, | |
| Some(&session), | |
| &mut vmo_write_offset, | |
| &mut lost_events, | |
| ); | |
| assert!(vmo_write_offset > 0, "no record written"); | |
| assert_eq!(lost_events, 0); | |
| let mut record = vec![0u8; vmo_write_offset as usize]; | |
| perf_data_vmo.read(&mut record, seq_lock.get().data_offset).unwrap(); | |
| assert_eq!( | |
| u32::from_ne_bytes(record[0..4].try_into().unwrap()), | |
| perf_event_type_PERF_RECORD_SAMPLE | |
| ); | |
| assert_eq!(u16::from_ne_bytes(record[6..8].try_into().unwrap()) as usize, record.len()); | |
| let mut pos = 8; | |
| let ip = read_u64(&record, &mut pos); | |
| let pid = u32::from_ne_bytes(record[pos..pos + 4].try_into().unwrap()); | |
| let tid = u32::from_ne_bytes(record[pos + 4..pos + 8].try_into().unwrap()); | |
| pos += 8; | |
| let nr = read_u64(&record, &mut pos); | |
| let callchain = (0..nr).map(|_| read_u64(&record, &mut pos)).collect(); | |
| let abi = read_u64(&record, &mut pos); | |
| let regs = if abi == perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_NONE as u64 { | |
| vec![] | |
| } else { | |
| (0..sample_regs_user.count_ones()).map(|_| read_u64(&record, &mut pos)).collect() | |
| }; | |
| let stack_size = read_u64(&record, &mut pos) as usize; | |
| assert_eq!(stack_size, SAMPLE_STACK_USER as usize); | |
| let stack_region = record[pos..pos + stack_size].to_vec(); | |
| pos += stack_size; | |
| let dyn_size = read_u64(&record, &mut pos) as usize; | |
| assert_eq!(pos, record.len()); | |
| ParsedStackSample { | |
| ip, | |
| pid, | |
| tid, | |
| callchain, | |
| abi, | |
| regs, | |
| stack: stack_region[..dyn_size].to_vec(), | |
| } | |
| } | |
| // Checks that `sample` fell back to reporting the Starnix thread registers | |
| // and stack. | |
| fn assert_reports_thread_state(sample: &ParsedStackSample, stack: &[u8]) { | |
| assert_eq!((sample.pid, sample.tid), (42, 43)); | |
| assert_eq!(sample.ip, STARNIX_PC); | |
| assert_eq!( | |
| sample.callchain, | |
| vec![ | |
| perf_callchain_context_PERF_CONTEXT_KERNEL, | |
| STARNIX_PC, | |
| perf_callchain_context_PERF_CONTEXT_USER, | |
| ] | |
| ); | |
| assert_eq!(sample.abi, perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_64 as u64); | |
| assert_eq!(sample.regs, thread_state_regs(STARNIX_PC, STARNIX_SP, 0)[..33].to_vec()); | |
| assert_eq!(sample.stack, stack); | |
| } | |
| #[cfg(target_arch = "aarch64")] | |
| #[::fuchsia::test] | |
| async fn test_stack_sample_in_starnix_reports_guest_regs_and_stack() { | |
| let starnix_stack = stack_bytes(0x10, 64); | |
| let guest_x: [u64; 31] = std::array::from_fn(|i| 0xb000 + i as u64); | |
| let restricted_state = restricted_state_bytes(&guest_x, GUEST_SP, GUEST_PC, 0); | |
| let guest_stack = stack_bytes(0x80, 48); | |
| let blob = stack_sample_blob( | |
| &thread_state_regs(STARNIX_PC, STARNIX_SP, 0), | |
| &[ | |
| (STARNIX_SP, &starnix_stack), | |
| (RESTRICTED_STATE_ADDR, &restricted_state), | |
| (GUEST_SP, &guest_stack), | |
| ], | |
| ); | |
| let sample = process_stack_sample(blob, ALL_REGS_MASK); | |
| assert_eq!((sample.pid, sample.tid), (42, 43)); | |
| // The Starnix PC is only reported as kernel execution; the user | |
| // section starts at the guest PC. | |
| assert_eq!(sample.ip, STARNIX_PC); | |
| assert_eq!( | |
| sample.callchain, | |
| vec![ | |
| perf_callchain_context_PERF_CONTEXT_KERNEL, | |
| STARNIX_PC, | |
| perf_callchain_context_PERF_CONTEXT_USER, | |
| GUEST_PC, | |
| ] | |
| ); | |
| assert_eq!(sample.abi, perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_64 as u64); | |
| let mut expected_regs = guest_x.to_vec(); | |
| expected_regs.extend([GUEST_SP, GUEST_PC]); | |
| assert_eq!(sample.regs, expected_regs); | |
| assert_eq!(sample.stack, guest_stack); | |
| } | |
| #[cfg(target_arch = "aarch64")] | |
| #[::fuchsia::test] | |
| async fn test_stack_sample_in_starnix_reports_aarch32_guest_regs() { | |
| const GUEST_PC32: u64 = 0x0040_1234; | |
| const GUEST_SP32: u64 = 0xff80_1000; | |
| let starnix_stack = stack_bytes(0x10, 64); | |
| let mut guest_x: [u64; 31] = std::array::from_fn(|i| 0xc000 + i as u64); | |
| guest_x[AARCH32_REG_SP] = GUEST_SP32; | |
| // AArch32 execution does not update the sp field; it holds a stale | |
| // value that no capture covers. | |
| let restricted_state = restricted_state_bytes( | |
| &guest_x, | |
| 0xdead_0000, | |
| GUEST_PC32, | |
| zx::sys::ZX_REG_CPSR_ARCH_32_MASK as u32, | |
| ); | |
| let guest_stack = stack_bytes(0x40, 32); | |
| let blob = stack_sample_blob( | |
| &thread_state_regs(STARNIX_PC, STARNIX_SP, 0), | |
| &[ | |
| (STARNIX_SP, &starnix_stack), | |
| (RESTRICTED_STATE_ADDR, &restricted_state), | |
| (GUEST_SP32, &guest_stack), | |
| ], | |
| ); | |
| let sample = process_stack_sample(blob, ALL_REGS_MASK); | |
| assert_eq!(sample.abi, perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_32 as u64); | |
| assert_eq!( | |
| sample.callchain, | |
| vec![ | |
| perf_callchain_context_PERF_CONTEXT_KERNEL, | |
| STARNIX_PC, | |
| perf_callchain_context_PERF_CONTEXT_USER, | |
| GUEST_PC32, | |
| ] | |
| ); | |
| assert_eq!(sample.regs.len(), 33); | |
| assert_eq!(sample.regs[AARCH32_REG_SP], GUEST_SP32); | |
| assert_eq!(sample.regs[AARCH32_REG_PC], GUEST_PC32); | |
| assert_eq!(sample.regs[AARCH64_REG_PC], GUEST_PC32); | |
| // The sp and lr slots carry R13 and R14. | |
| assert_eq!(sample.regs[AARCH64_REG_SP], GUEST_SP32); | |
| assert_eq!(sample.regs[AARCH64_REG_LR], guest_x[AARCH32_REG_LR]); | |
| assert_eq!(sample.regs[0], 0xc000); | |
| assert_eq!(sample.stack, guest_stack); | |
| } | |
| #[::fuchsia::test] | |
| async fn test_stack_sample_in_guest_reports_thread_regs() { | |
| // A thread suspended in restricted mode reports the guest registers | |
| // itself; the restricted state then holds stale values and is | |
| // ignored. | |
| let guest_stack = stack_bytes(0x20, 64); | |
| let stale_stack = stack_bytes(0x90, 64); | |
| const STALE_SP: u64 = GUEST_SP + 0x10000; | |
| #[cfg(target_arch = "aarch64")] | |
| let stale_state = restricted_state_bytes(&[0x5a; 31], STALE_SP, GUEST_PC + 0x100, 0); | |
| #[cfg(not(target_arch = "aarch64"))] | |
| let stale_state = vec![0x5au8; 280]; | |
| let thread_regs = thread_state_regs(GUEST_PC, GUEST_SP + 16, 0); | |
| let blob = stack_sample_blob( | |
| &thread_regs, | |
| &[ | |
| (GUEST_SP, &guest_stack), | |
| (RESTRICTED_STATE_ADDR, &stale_state), | |
| (STALE_SP, &stale_stack), | |
| ], | |
| ); | |
| let sample = process_stack_sample(blob, ALL_REGS_MASK); | |
| assert_eq!(sample.ip, GUEST_PC); | |
| assert_eq!( | |
| sample.callchain, | |
| vec![ | |
| perf_callchain_context_PERF_CONTEXT_KERNEL, | |
| perf_callchain_context_PERF_CONTEXT_USER, | |
| GUEST_PC, | |
| ] | |
| ); | |
| assert_eq!(sample.abi, perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_64 as u64); | |
| assert_eq!(sample.regs, thread_regs[..33].to_vec()); | |
| // The stack starts exactly at the SP, inside the first capture. | |
| assert_eq!(sample.stack, guest_stack[16..].to_vec()); | |
| } | |
| #[cfg(target_arch = "aarch64")] | |
| #[::fuchsia::test] | |
| async fn test_stack_sample_in_guest_aarch32_reports_sp_lr_from_r13_r14() { | |
| // A thread suspended in AArch32 restricted mode: Zircon reports its | |
| // stack pointer in r[13] and link register in r[14]; the sp and lr | |
| // slots hold SP_EL0 and X30, which AArch32 execution never updates. | |
| const GUEST_PC32: u64 = 0x0040_1234; | |
| const GUEST_SP32: u64 = 0xff80_1000; | |
| const GUEST_LR32: u64 = 0x0040_5678; | |
| const STALE_SP: u64 = 0xdead_0000; | |
| const STALE_LR: u64 = 0xbeef_0000; | |
| let guest_stack = stack_bytes(0x30, 64); | |
| let mut thread_regs = | |
| thread_state_regs(GUEST_PC32, STALE_SP, zx::sys::ZX_REG_CPSR_ARCH_32_MASK as u64); | |
| thread_regs[AARCH32_REG_SP] = GUEST_SP32; | |
| thread_regs[AARCH32_REG_LR] = GUEST_LR32; | |
| thread_regs[AARCH64_REG_LR] = STALE_LR; | |
| let blob = stack_sample_blob(&thread_regs, &[(GUEST_SP32, &guest_stack)]); | |
| let sample = process_stack_sample(blob, ALL_REGS_MASK); | |
| assert_eq!(sample.ip, GUEST_PC32); | |
| assert_eq!( | |
| sample.callchain, | |
| vec![ | |
| perf_callchain_context_PERF_CONTEXT_KERNEL, | |
| perf_callchain_context_PERF_CONTEXT_USER, | |
| GUEST_PC32, | |
| ] | |
| ); | |
| assert_eq!(sample.abi, perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_32 as u64); | |
| assert_eq!(sample.regs.len(), 33); | |
| assert_eq!(sample.regs[AARCH64_REG_SP], GUEST_SP32); | |
| assert_eq!(sample.regs[AARCH64_REG_LR], GUEST_LR32); | |
| assert_eq!(sample.regs[AARCH32_REG_SP], GUEST_SP32); | |
| assert_eq!(sample.regs[AARCH32_REG_LR], GUEST_LR32); | |
| assert_eq!(sample.regs[AARCH32_REG_PC], GUEST_PC32); | |
| assert_eq!(sample.regs[AARCH64_REG_PC], GUEST_PC32); | |
| assert_eq!(sample.regs[0], 0xa000); | |
| // The stack is selected by R13, not by the stale sp slot. | |
| assert_eq!(sample.stack, guest_stack); | |
| } | |
| #[::fuchsia::test] | |
| async fn test_stack_sample_in_starnix_without_restricted_state_reports_thread_regs() { | |
| let starnix_stack = stack_bytes(0x10, 64); | |
| let thread_regs = thread_state_regs(STARNIX_PC, STARNIX_SP, 0); | |
| let blob = stack_sample_blob(&thread_regs, &[(STARNIX_SP, &starnix_stack)]); | |
| let sample = process_stack_sample(blob, ALL_REGS_MASK); | |
| assert_reports_thread_state(&sample, &starnix_stack); | |
| } | |
| #[::fuchsia::test] | |
| async fn test_stack_sample_in_starnix_without_captures_is_reported() { | |
| let thread_regs = thread_state_regs(STARNIX_PC, STARNIX_SP, 0); | |
| let blob = stack_sample_blob(&thread_regs, &[]); | |
| let sample = process_stack_sample(blob, ALL_REGS_MASK); | |
| assert_reports_thread_state(&sample, &[]); | |
| } | |
| #[cfg(target_arch = "aarch64")] | |
| #[::fuchsia::test] | |
| async fn test_stack_sample_in_starnix_ignores_unusable_restricted_state() { | |
| let starnix_stack = stack_bytes(0x10, 64); | |
| let guest_stack = stack_bytes(0x80, 48); | |
| let guest_x: [u64; 31] = std::array::from_fn(|i| 0xb000 + i as u64); | |
| let valid_state = restricted_state_bytes(&guest_x, GUEST_SP, GUEST_PC, 0); | |
| let cases: [(&str, u64, Vec<u8>); 6] = [ | |
| ( | |
| "pc above the guest address space", | |
| RESTRICTED_STATE_ADDR, | |
| restricted_state_bytes(&guest_x, GUEST_SP, STARNIX_PC, 0), | |
| ), | |
| ( | |
| "pc below the guest address space", | |
| RESTRICTED_STATE_ADDR, | |
| restricted_state_bytes(&guest_x, GUEST_SP, 0x1000, 0), | |
| ), | |
| ( | |
| "sp above the guest address space", | |
| RESTRICTED_STATE_ADDR, | |
| restricted_state_bytes(&guest_x, STARNIX_SP, GUEST_PC, 0), | |
| ), | |
| ("not yet written", RESTRICTED_STATE_ADDR, restricted_state_bytes(&[0; 31], 0, 0, 0)), | |
| ("not page-aligned", RESTRICTED_STATE_ADDR + 8, valid_state.clone()), | |
| ("short read", RESTRICTED_STATE_ADDR, valid_state[..valid_state.len() - 8].to_vec()), | |
| ]; | |
| for (name, state_addr, state) in cases { | |
| let blob = stack_sample_blob( | |
| &thread_state_regs(STARNIX_PC, STARNIX_SP, 0), | |
| &[(STARNIX_SP, &starnix_stack), (state_addr, &state), (GUEST_SP, &guest_stack)], | |
| ); | |
| let sample = process_stack_sample(blob, ALL_REGS_MASK); | |
| println!("case: {name}"); | |
| assert_reports_thread_state(&sample, &starnix_stack); | |
| } | |
| } | |
| #[cfg(target_arch = "aarch64")] | |
| #[::fuchsia::test] | |
| async fn test_stack_sample_in_starnix_without_guest_stack_reports_guest_regs() { | |
| // Even if the guest stack chunk is missing (e.g. the guest SP was | |
| // unmapped when sampled), guest registers are still reported with an | |
| // empty stack. | |
| let starnix_stack = stack_bytes(0x10, 64); | |
| let guest_x: [u64; 31] = std::array::from_fn(|i| 0xb000 + i as u64); | |
| let restricted_state = restricted_state_bytes(&guest_x, GUEST_SP, GUEST_PC, 0); | |
| let thread_regs = thread_state_regs(STARNIX_PC, STARNIX_SP, 0); | |
| let blob = stack_sample_blob( | |
| &thread_regs, | |
| &[(STARNIX_SP, &starnix_stack), (RESTRICTED_STATE_ADDR, &restricted_state)], | |
| ); | |
| let sample = process_stack_sample(blob, ALL_REGS_MASK); | |
| assert_eq!((sample.pid, sample.tid), (42, 43)); | |
| assert_eq!(sample.ip, STARNIX_PC); | |
| assert_eq!( | |
| sample.callchain, | |
| vec![ | |
| perf_callchain_context_PERF_CONTEXT_KERNEL, | |
| STARNIX_PC, | |
| perf_callchain_context_PERF_CONTEXT_USER, | |
| GUEST_PC, | |
| ] | |
| ); | |
| assert_eq!(sample.abi, perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_64 as u64); | |
| let mut expected_regs = guest_x.to_vec(); | |
| expected_regs.extend([GUEST_SP, GUEST_PC]); | |
| assert_eq!(sample.regs, expected_regs); | |
| assert!(sample.stack.is_empty()); | |
| } | |
| #[cfg(target_arch = "aarch64")] | |
| #[::fuchsia::test] | |
| async fn test_stack_sample_in_starnix_with_uncovered_guest_sp_reports_guest_regs() { | |
| // If no chunk covers the guest SP, report guest registers with an | |
| // empty stack rather than picking the Starnix stack. | |
| let starnix_stack = stack_bytes(0x10, 64); | |
| let other_stack = stack_bytes(0x80, 48); | |
| let guest_x: [u64; 31] = std::array::from_fn(|i| 0xb000 + i as u64); | |
| let restricted_state = restricted_state_bytes(&guest_x, GUEST_SP + 0x10000, GUEST_PC, 0); | |
| let blob = stack_sample_blob( | |
| &thread_state_regs(STARNIX_PC, STARNIX_SP, 0), | |
| &[ | |
| (STARNIX_SP, &starnix_stack), | |
| (RESTRICTED_STATE_ADDR, &restricted_state), | |
| (GUEST_SP, &other_stack), | |
| ], | |
| ); | |
| let sample = process_stack_sample(blob, ALL_REGS_MASK); | |
| assert_eq!(sample.abi, perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_64 as u64); | |
| assert_eq!(sample.regs[AARCH64_REG_SP], GUEST_SP + 0x10000); | |
| assert_eq!(sample.regs[AARCH64_REG_PC], GUEST_PC); | |
| assert!(sample.stack.is_empty()); | |
| assert_eq!( | |
| sample.callchain, | |
| vec![ | |
| perf_callchain_context_PERF_CONTEXT_KERNEL, | |
| STARNIX_PC, | |
| perf_callchain_context_PERF_CONTEXT_USER, | |
| GUEST_PC, | |
| ] | |
| ); | |
| } | |
| #[::fuchsia::test] | |
| async fn test_stack_sample_in_guest_without_stack_capture_is_reported() { | |
| let thread_regs = thread_state_regs(GUEST_PC, GUEST_SP, 0); | |
| let blob = stack_sample_blob(&thread_regs, &[]); | |
| let sample = process_stack_sample(blob, ALL_REGS_MASK); | |
| assert_eq!( | |
| sample.callchain, | |
| vec![ | |
| perf_callchain_context_PERF_CONTEXT_KERNEL, | |
| perf_callchain_context_PERF_CONTEXT_USER, | |
| GUEST_PC, | |
| ] | |
| ); | |
| assert_eq!(sample.abi, perf_sample_regs_abi_PERF_SAMPLE_REGS_ABI_64 as u64); | |
| assert_eq!(sample.regs, thread_regs[..33].to_vec()); | |
| assert!(sample.stack.is_empty()); | |
| } | |
| #[::fuchsia::test] | |
| async fn test_callgraph_config_stack_size() { | |
| let callchain_only = | |
| callgraph_config(perf_event_sample_format_PERF_SAMPLE_CALLCHAIN as u64, 65280); | |
| assert_eq!(callchain_only.strategy, Some(profiler::CallgraphStrategy::FramePointer)); | |
| assert_eq!(callchain_only.stack_size, None); | |
| let regs_only = | |
| callgraph_config(perf_event_sample_format_PERF_SAMPLE_REGS_USER as u64, 65280); | |
| assert_eq!(regs_only.strategy, Some(profiler::CallgraphStrategy::Dwarf)); | |
| assert_eq!(regs_only.stack_size, None); | |
| let zero_stack = | |
| callgraph_config(perf_event_sample_format_PERF_SAMPLE_STACK_USER as u64, 0); | |
| assert_eq!(zero_stack.strategy, Some(profiler::CallgraphStrategy::Dwarf)); | |
| assert_eq!(zero_stack.stack_size, Some(0)); | |
| let custom_stack = | |
| callgraph_config(perf_event_sample_format_PERF_SAMPLE_STACK_USER as u64, 65280); | |
| assert_eq!(custom_stack.strategy, Some(profiler::CallgraphStrategy::Dwarf)); | |
| assert_eq!(custom_stack.stack_size, Some(65280)); | |
| } | |
| } |