| // Copyright 2021 The Fuchsia Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| use crate::bpf::EbpfState; | |
| use crate::device::remote_block_device::RemoteBlockDeviceRegistry; | |
| use crate::device::{DeviceMode, DeviceRegistry}; | |
| use crate::execution::CrashReporter; | |
| use crate::mm::{FutexTable, MappingSummary, MlockPinFlavor}; | |
| use crate::power::{SuspendResumeManagerHandle, create_watcher_for_wake_events}; | |
| use crate::ptrace::StopState; | |
| use crate::security::{self, AuditLogger}; | |
| use crate::task::container_namespace::{ | |
| ContainerNamespace, NamespaceNodePurpose, check_ns_dir_path, | |
| }; | |
| use crate::task::limits::SystemLimits; | |
| use crate::task::memory_attribution::MemoryAttributionManager; | |
| use crate::task::net::NetstackDevices; | |
| use crate::task::tracing::TracePerformanceEventManager; | |
| use crate::task::{ | |
| AbstractUnixSocketNamespace, AbstractVsockSocketNamespace, CurrentTask, DelayedReleaser, | |
| ExitStatus, IpTables, KernelCgroups, KernelStats, KernelThreads, PidTable, SchedulerManager, | |
| Syslog, Task, ThreadGroup, UtsNamespace, UtsNamespaceHandle, | |
| }; | |
| use crate::time::{HrTimerManager, HrTimerManagerHandle}; | |
| use crate::vdso::vdso_loader::Vdso; | |
| use crate::vfs::fs_args::MountParams; | |
| use crate::vfs::socket::{ | |
| GenericMessage, GenericNetlink, NetlinkAccessControl, NetlinkContextImpl, | |
| NetlinkToClientSender, SocketAddress, SocketTokensStore, | |
| }; | |
| use crate::vfs::{CacheConfig, FileOps, FsNodeHandle, FsString, Mounts, NamespaceNode}; | |
| use bstr::{BString, ByteSlice}; | |
| use devicetree::types::Devicetree; | |
| use expando::Expando; | |
| use fidl::endpoints::{ClientEnd, DiscoverableProtocolMarker, ProtocolMarker, create_endpoints}; | |
| use fidl_fuchsia_component_runner::{ComponentControllerControlHandle, ComponentStopInfo}; | |
| use fidl_fuchsia_feedback::CrashReporterProxy; | |
| use fidl_fuchsia_io as fio; | |
| use fidl_fuchsia_memory_attribution as fattribution; | |
| use fidl_fuchsia_net_power as fnet_power; | |
| use fidl_fuchsia_net_resources as fnet_resources; | |
| use fidl_fuchsia_time_external::AdjustSynchronousProxy; | |
| use fuchsia_async as fasync; | |
| use fuchsia_inspect::ArrayProperty; | |
| use fuchsia_rcu::RcuReadScope; | |
| use futures::FutureExt; | |
| use netlink::interfaces::InterfacesHandler; | |
| use netlink::{NETLINK_LOG_TAG, Netlink}; | |
| use once_cell::sync::OnceCell; | |
| use scopeguard::ScopeGuard; | |
| use smallvec::SmallVec; | |
| use starnix_lifecycle::AtomicCounter; | |
| use starnix_logging::{SyscallLogFilter, log_debug, log_error, log_info, log_warn}; | |
| use starnix_sync::{ | |
| ComponentControllerLock, KernelSwapFiles, LockDepGuard, LockDepMutex, MountsLevel, RwSeqLock, | |
| RwSeqLockGuard, SyscallLogFiltersLock, WriteProof, | |
| }; | |
| use starnix_uapi::device_id::DeviceId; | |
| use starnix_uapi::errors::{Errno, errno}; | |
| use starnix_uapi::open_flags::OpenFlags; | |
| use starnix_uapi::signals::SIGKILL; | |
| use starnix_uapi::{VMADDR_CID_HOST, from_status_like_fdio}; | |
| use std::borrow::Cow; | |
| use std::cell::RefCell; | |
| use std::collections::{HashMap, HashSet}; | |
| use std::num::NonZeroU64; | |
| use std::ops::Deref; | |
| use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicU16, Ordering}; | |
| use std::sync::{Arc, OnceLock, Weak}; | |
| use zx::{CpuFeatureFlags, CpuSveFeatureFlags}; | |
| /// Host kernel free-memory watermarks (in MiB) configured via Product Assembly | |
| /// (`platform.kernel.oom`) and routed to `starnix_kernel` as structured configuration. | |
| #[derive(Debug, Clone, Copy, PartialEq, Eq)] | |
| pub struct OomWatermarksMb { | |
| pub out_of_memory_mb: u32, | |
| pub critical_mb: u32, | |
| pub warning_mb: u32, | |
| } | |
| /// Kernel features are specified in the component manifest of the starnix container | |
| /// or explicitly provided to the kernel constructor in tests. | |
| #[derive(Debug, Default, Clone)] | |
| pub struct KernelFeatures { | |
| pub bpf_v2: bool, | |
| /// Whether the `devtmpfs` filesystem and automatic device node creation are enabled. | |
| pub devtmpfs: bool, | |
| /// Whether the kernel supports the S_ISUID and S_ISGID bits. | |
| /// | |
| /// For example, these bits are used by `sudo`. | |
| /// | |
| /// Enabling this feature is potentially a security risk because they allow privilege | |
| /// escalation. | |
| pub enable_suid: bool, | |
| /// Whether io_uring is enabled. | |
| /// | |
| /// TODO(https://fxbug.dev/297431387): Enabled by default once the feature is completed. | |
| pub io_uring: bool, | |
| /// Whether the kernel should return an error to userspace, rather than panicking, if `reboot()` | |
| /// is requested but cannot be enacted because the kernel lacks the relevant capabilities. | |
| pub error_on_failed_reboot: bool, | |
| /// The default seclabel that is applied to components that are run in this kernel. | |
| /// | |
| /// Components can override this by setting the `seclabel` field in their program block. | |
| pub default_seclabel: Option<String>, | |
| /// Whether the kernel is being used to run the SELinux Test Suite. | |
| /// | |
| /// TODO: https://fxbug.dev/388077431 - remove this once we no longer need workarounds for the | |
| /// SELinux Test Suite. | |
| pub selinux_test_suite: bool, | |
| /// The default mount options to use when mounting directories from a component's namespace. | |
| /// | |
| /// The key is the path in the component's namespace, and the value is the mount options | |
| /// string. | |
| pub default_ns_mount_options: Option<HashMap<String, String>>, | |
| /// The default uid that is applied to components that are run in this kernel. | |
| /// | |
| /// Components can override this by setting the `uid` field in their program block. | |
| pub default_uid: u32, | |
| /// mlock() never prefaults pages. | |
| pub mlock_always_onfault: bool, | |
| /// Implementation of mlock() to use for this kernel instance. | |
| pub mlock_pin_flavor: MlockPinFlavor, | |
| /// Whether excessive crash reports should be throttled. | |
| pub crash_report_throttling: bool, | |
| /// Whether or not to serve wifi support to Android. | |
| pub wifi: bool, | |
| /// The number of bytes to cache in pages for reading zx::MapInfo from VMARs. | |
| pub cached_zx_map_info_bytes: u32, | |
| /// The size of the Dirent LRU cache. | |
| pub dirent_cache_size: u32, | |
| /// Whether to expose a stub '/dev/ion' node, as a temporary workaround for compatibility. | |
| // TODO(https://fxbug.dev/485370648) remove when unnecessary | |
| pub fake_ion: bool, | |
| /// Whether `remotevol` mounts the volume in-process rather than over FIDL. | |
| pub remotevol_in_process: bool, | |
| /// Whether to log unexpected lookup misses in `sysfs` via `track_stub_log!`. | |
| pub log_sysfs_lookup_misses: bool, | |
| /// Free-memory watermarks (in MiB) for `/proc/zoneinfo`. | |
| pub oom_watermarks_mb: Option<OomWatermarksMb>, | |
| /// Paths of executables whose processes are stopped only after all other processes except init | |
| /// have exited when the kernel shuts down, e.g. one that writes crash dumps for processes that | |
| /// crash during shutdown. See `Kernel::run_shutdown()`. | |
| pub stop_last_on_shutdown: Vec<FsString>, | |
| } | |
| impl KernelFeatures { | |
| /// Returns the `MountParams` to use when mounting the specified path from a component's | |
| /// namespace. This mechanism is also used to specified options for mounts created via | |
| /// container features, by specifying a pseudo-path e.g. "#container". | |
| pub fn ns_mount_options(&self, ns_path: &str) -> Result<MountParams, Errno> { | |
| if let Some(all_options) = &self.default_ns_mount_options { | |
| if let Some(options) = all_options.get(ns_path) { | |
| return MountParams::parse(options.as_bytes().into()); | |
| } | |
| } | |
| Ok(MountParams::default()) | |
| } | |
| } | |
| /// Kernel command line or module argument structure (`name` or `name=value`). | |
| #[derive(Debug, Clone, Copy, PartialEq, Eq)] | |
| pub struct ArgNameAndValue<'a> { | |
| pub name: &'a str, | |
| pub value: Option<&'a str>, | |
| } | |
| impl<'a> From<&'a str> for ArgNameAndValue<'a> { | |
| fn from(arg: &'a str) -> Self { | |
| match arg.split_once('=') { | |
| Some((name, value)) => Self { name, value: Some(value) }, | |
| None => Self { name: arg, value: None }, | |
| } | |
| } | |
| } | |
| type DeferredDropCallback = Box<dyn FnOnce() + Send + Sync>; | |
| /// A proof token representing the global lock over the namespace mount topology. | |
| /// | |
| /// Functions that take `&MountsWriteToken` require the caller to hold the | |
| /// `Kernel::mounts_lock` to ensure safe modification of the global mount tree. | |
| pub struct MountsWriteToken { | |
| deferred_drops: RefCell<SmallVec<[DeferredDropCallback; 8]>>, | |
| } | |
| impl MountsWriteToken { | |
| fn new() -> Self { | |
| Self { deferred_drops: RefCell::new(SmallVec::new()) } | |
| } | |
| fn defer_drop_internal<T: Send + Sync + 'static>(&self, value: T) { | |
| self.deferred_drops.borrow_mut().push(Box::new(move || { | |
| drop(value); | |
| })); | |
| } | |
| /// We cannot block while holding a [MountsWriteGuard] as it is a spinlock. If a Drop requires | |
| /// a blocking operation, use [Self::defer_drop] to defer the blocking operation until | |
| /// after the [MountsWriteGuard] is released. | |
| pub fn defer_drop<T: Send + Sync + 'static>( | |
| &self, | |
| value: T, | |
| ) -> ScopeGuard<T, impl FnOnce(T) + '_> { | |
| scopeguard::guard(value, |value| { | |
| self.defer_drop_internal(value); | |
| }) | |
| } | |
| /// Helper method to [Self::defer_drop] specifically for [Arc<T>]s. | |
| pub fn retain<T: Send + Sync + 'static>(&self, value: &Arc<T>) -> Arc<T> { | |
| self.defer_drop_internal(value.clone()); | |
| value.clone() | |
| } | |
| fn take_deferred_drops(&mut self) -> SmallVec<[DeferredDropCallback; 8]> { | |
| std::mem::take(self.deferred_drops.get_mut()) | |
| } | |
| } | |
| impl WriteProof<MountsWriteToken> for MountsWriteToken {} | |
| /// A guard special cased for mount operations to defer destroying mounts until after all | |
| /// modifications of the global mount tree are completed. | |
| pub struct MountsWriteGuard<'a> { | |
| guard: Option<RwSeqLockGuard<'a, LockDepGuard<'a, MountsWriteToken>>>, | |
| } | |
| impl WriteProof<MountsWriteToken> for MountsWriteGuard<'_> {} | |
| impl<'a> MountsWriteGuard<'a> { | |
| fn new(guard: RwSeqLockGuard<'a, LockDepGuard<'a, MountsWriteToken>>) -> Self { | |
| Self { guard: Some(guard) } | |
| } | |
| } | |
| impl<'a> Deref for MountsWriteGuard<'a> { | |
| type Target = MountsWriteToken; | |
| fn deref(&self) -> &Self::Target { | |
| self.guard.as_ref().unwrap() | |
| } | |
| } | |
| impl<'a> Drop for MountsWriteGuard<'a> { | |
| fn drop(&mut self) { | |
| if let Some(mut guard) = self.guard.take() { | |
| let drops = guard.take_deferred_drops(); | |
| drop(guard); | |
| for callback in drops { | |
| callback(); | |
| } | |
| } | |
| } | |
| } | |
| /// Maximum time after shutdown starts before deferred-shutdown processes receive SIGKILL. | |
| /// | |
| /// Deferred-shutdown processes are processes that are still handling a core-dump signal | |
| /// when their normal shutdown SIGKILL timeout expires (e.g., while writing a crash dump), | |
| /// along with any child processes they spawn. See `ThreadGroup::shut_down()`. | |
| /// | |
| /// `component_manager` forcibly kills the container if it does not stop within 5s by | |
| /// default, which skips filesystem unmounts and `on_stop`. Setting this timeout to 3s | |
| /// gives crash dumps time to finish (typically 1-2.5s in testing) while leaving 2s for | |
| /// the remaining shutdown steps (`stop_last_on_shutdown` processes, `init`, unmounts, | |
| /// and `on_stop`, which took up to 0.6s in testing). | |
| // TODO(https://fxbug.dev/574471394): Remove once non-forced shutdown lets the container | |
| // run its own shutdown sequence. | |
| const DEFERRED_SHUTDOWN_TIMEOUT: zx::MonotonicDuration = zx::MonotonicDuration::from_seconds(3); | |
| /// The shared, mutable state for the entire Starnix kernel. | |
| /// | |
| /// The `Kernel` object holds all kernel threads, userspace tasks, and file system resources for a | |
| /// single instance of the Starnix kernel. In production, there is one instance of this object for | |
| /// the entire Starnix kernel. However, multiple instances of this object can be created in one | |
| /// process during unit testing. | |
| /// | |
| /// The structure of this object will likely need to evolve as we implement more namespacing and | |
| /// isolation mechanisms, such as `namespaces(7)` and `pid_namespaces(7)`. | |
| pub struct Kernel { | |
| /// Weak reference to self. Allows to not have to pass &Arc<Kernel> in apis. | |
| pub weak_self: Weak<Kernel>, | |
| /// The kernel threads running on behalf of this kernel. | |
| pub kthreads: KernelThreads, | |
| /// The features enabled for this kernel. | |
| pub features: KernelFeatures, | |
| /// The processes and threads running in this kernel, organized by pid_t. | |
| pub pids: PidTable, | |
| /// A weak reference to the init task (PID 1). | |
| pub init_task: OnceLock<Weak<Task>>, | |
| /// Shared manager of pid/tid to koid mappings for tracing and profiling clients. | |
| /// A direct field (rather than an expando entry) so the task-creation fast path is a | |
| /// single relaxed atomic load with no locks when nothing is recording. | |
| pub trace_event_manager: Arc<TracePerformanceEventManager>, | |
| /// Subsystem-specific properties that hang off the Kernel object. | |
| /// | |
| /// Instead of adding yet another property to the Kernel object, consider storing the property | |
| /// in an expando if that property is only used by one part of the system, such as a module. | |
| pub expando: Expando, | |
| /// The default namespace for abstract AF_UNIX sockets in this kernel. | |
| /// | |
| /// Rather than use this default namespace, abstract socket addresses | |
| /// should be looked up in the AbstractSocketNamespace on each Task | |
| /// object because some Task objects might have a non-default namespace. | |
| pub default_abstract_socket_namespace: Arc<AbstractUnixSocketNamespace>, | |
| /// The default namespace for abstract AF_VSOCK sockets in this kernel. | |
| pub default_abstract_vsock_namespace: Arc<AbstractVsockSocketNamespace>, | |
| /// The kernel command line. Shows up in /proc/cmdline. | |
| pub cmdline: BString, | |
| pub device_tree: Option<Devicetree>, | |
| // Global state held by the Linux Security Modules subsystem. | |
| pub security_state: security::KernelState, | |
| /// The registry of device drivers. | |
| pub device_registry: DeviceRegistry, | |
| /// Mapping of top-level namespace entries to an associated proxy. | |
| /// For example, "/svc" to the respective proxy. Only the namespace entries | |
| /// which were known at component startup will be available by the kernel. | |
| pub container_namespace: ContainerNamespace, | |
| /// The global lock for the mount tree. | |
| /// | |
| /// This lock protects against concurrent modifications to the mount topology. It uses | |
| /// an `RwSeqLock` to allow readers (like path walking traversing mount points) to get a | |
| /// consistent, lock-free snapshot of the RCU-protected mount table using `read_seq`. | |
| /// Writers must acquire the lock before mutating filesystems, moving mounts, or | |
| /// propagating peer groups. The returned `MountsWriteToken` is used as a proof token | |
| /// throughout the `namespace` module to statically enforce exclusive write access. | |
| pub mounts_lock: RwSeqLock<LockDepMutex<MountsWriteToken, MountsLevel>>, | |
| /// The registry of block devices backed by a remote fuchsia.io file. | |
| pub remote_block_device_registry: Arc<RemoteBlockDeviceRegistry>, | |
| /// The iptables used for filtering network packets. | |
| iptables: OnceLock<IpTables>, | |
| /// The futexes across all processes in the kernel. | |
| pub shared_futexes: FutexTable, | |
| /// The default UTS namespace for all tasks. | |
| /// | |
| /// Because each task can have its own UTS namespace, you probably want to use | |
| /// the UTS namespace handle of the task, which may/may not point to this one. | |
| pub root_uts_ns: UtsNamespaceHandle, | |
| /// A struct containing a VMO with a vDSO implementation, if implemented for a given architecture, and possibly an offset for a sigreturn function. | |
| pub vdso: Vdso, | |
| /// A struct containing a VMO with a arch32-vDSO implementation, if implemented for a given architecture. | |
| // TODO(https://fxbug.dev/380431743) This could be made less clunky -- maybe a Vec<Vdso> above or | |
| // something else | |
| pub vdso_arch32: Option<Vdso>, | |
| /// The table of devices installed on the netstack and their associated | |
| /// state local to this `Kernel`. | |
| pub netstack_devices: Arc<NetstackDevices>, | |
| /// Files that are currently available for swapping. | |
| /// Note: Starnix never actually swaps memory to these files. We just need to track them | |
| /// to pass conformance tests. | |
| pub swap_files: LockDepMutex<Vec<FsNodeHandle>, KernelSwapFiles>, | |
| /// The implementation of generic Netlink protocol families. | |
| generic_netlink: OnceLock<GenericNetlink<NetlinkToClientSender<GenericMessage>>>, | |
| /// The implementation of networking-related Netlink protocol families. | |
| network_netlink: OnceLock<Netlink<NetlinkContextImpl>>, | |
| /// Inspect instrumentation for this kernel instance. | |
| pub inspect_node: fuchsia_inspect::Node, | |
| /// The kinds of seccomp action that gets logged, stored as a bit vector. | |
| /// Each potential SeccompAction gets a bit in the vector, as specified by | |
| /// SeccompAction::logged_bit_offset. If the bit is set, that means the | |
| /// action should be logged when it is taken, subject to the caveats | |
| /// described in seccomp(2). The value of the bit vector is exposed to users | |
| /// in a text form in the file /proc/sys/kernel/seccomp/actions_logged. | |
| pub actions_logged: AtomicU16, | |
| /// The manager for suspend/resume. | |
| pub suspend_resume_manager: SuspendResumeManagerHandle, | |
| /// Unique IDs for new mounts and mount namespaces. | |
| pub next_mount_id: AtomicCounter<u64>, | |
| pub next_peer_group_id: AtomicCounter<u64>, | |
| pub next_namespace_id: AtomicCounter<u64>, | |
| /// Unique IDs for file objects. | |
| pub next_file_object_id: AtomicCounter<u64>, | |
| /// Controls which processes a process is allowed to ptrace. See Documentation/security/Yama.txt | |
| pub ptrace_scope: AtomicU8, | |
| // The Fuchsia build version returned by `fuchsia.buildinfo.Provider`. | |
| pub build_version: OnceCell<String>, | |
| pub stats: Arc<KernelStats>, | |
| /// Resource limits that are exposed, for example, via sysctl. | |
| pub system_limits: SystemLimits, | |
| // The service to handle delayed releases. This is required for elements that requires to | |
| // execute some code when released and requires a known context (both in term of lock context, | |
| // as well as `CurrentTask`). | |
| pub delayed_releaser: DelayedReleaser, | |
| /// Manages task priorities. | |
| pub scheduler: SchedulerManager, | |
| /// The syslog manager. | |
| pub syslog: Syslog, | |
| /// All mounts. | |
| pub mounts: Mounts, | |
| /// The manager for creating and managing high-resolution timers. | |
| pub hrtimer_manager: HrTimerManagerHandle, | |
| /// The manager for monitoring and reporting resources used by the kernel. | |
| pub memory_attribution_manager: MemoryAttributionManager, | |
| /// Handler for crashing Linux processes. | |
| pub crash_reporter: CrashReporter, | |
| /// When this kernel started shutting down, if it has. See `run_shutdown()`. | |
| shutdown_start: OnceLock<zx::MonotonicInstant>, | |
| /// True to disable syslog access to unprivileged callers. This also controls whether read | |
| /// access to /dev/kmsg requires privileged capabilities. | |
| pub restrict_dmesg: AtomicBool, | |
| /// Determines whether unprivileged BPF is permitted, or can be re-enabled. | |
| /// 0 - Unprivileged BPF is permitted. | |
| /// 1 - Unprivileged BPF is not permitted, and cannot be enabled. | |
| /// 2 - Unprivileged BPF is not permitted, but can be enabled by a privileged task. | |
| pub disable_unprivileged_bpf: AtomicU8, | |
| /// Controls use of perf_event_open(2) by tasks without CAP_PERFMON (or CAP_SYS_ADMIN): | |
| /// -1 - No capability required. | |
| /// 0 - Capability required for tracepoint events. | |
| /// 1 - Capability additionally required for CPU-wide events. | |
| /// 2 - Capability additionally required for kernel profiling (default). | |
| pub perf_event_paranoid: AtomicI32, | |
| /// Control handle to the running container's ComponentController. | |
| pub container_control_handle: | |
| LockDepMutex<Option<ComponentControllerControlHandle>, ComponentControllerLock>, | |
| /// eBPF state: loaded programs, eBPF maps, etc. | |
| pub ebpf_state: EbpfState, | |
| /// Cgroups of the kernel. | |
| pub cgroups: KernelCgroups, | |
| /// Used to communicate requests to adjust system time from within a Starnix | |
| /// container. Used from syscalls. | |
| pub time_adjustment_proxy: Option<AdjustSynchronousProxy>, | |
| /// A token for the wake group we have registered with the netstack to | |
| /// receive wakeup notifications on incoming data while suspended. | |
| /// | |
| /// A value of `None` means we were unable to communicate with the netstack | |
| /// to create the wake group. In that case, we assume something has gone | |
| /// wrong with the netstack (failed to start, crash, etc) and won't retry. | |
| pub netstack_wake_group: OnceLock<Option<fnet_resources::WakeGroupToken>>, | |
| /// Used to store tokens for sockets, particularly per-uid sharing domain sockets. | |
| pub socket_tokens_store: SocketTokensStore, | |
| /// Hardware capabilities to push onto stack when loading an ELF binary. | |
| pub hwcaps: HwCaps, | |
| /// Filters for syscall logging. Processes with names matching these filters will have syscalls | |
| /// logged at INFO level. | |
| pub syscall_log_filters: LockDepMutex<Vec<SyscallLogFilter>, SyscallLogFiltersLock>, | |
| } | |
| /// Hardware capabilities. | |
| #[derive(Debug, Clone, Copy, Default)] | |
| pub struct HwCap { | |
| /// The value for `AT_HWCAP`. | |
| pub hwcap: u32, | |
| /// The value for `AT_HWCAP2`. | |
| pub hwcap2: u32, | |
| } | |
| /// Hardware capabilities for both 32-bit and 64-bit ELF binaries. | |
| #[derive(Debug, Clone, Copy, Default)] | |
| pub struct HwCaps { | |
| /// For 32-bit binaries. | |
| #[cfg(target_arch = "aarch64")] | |
| pub arch32: HwCap, | |
| /// For 64-bit binaries. | |
| pub arch64: HwCap, | |
| } | |
| /// An implementation of [`InterfacesHandler`]. | |
| /// | |
| /// This holds a `Weak<Kernel>` because it is held within a [`Netlink`] which | |
| /// is itself held within an `Arc<Kernel>`. Holding an `Arc<T>` within an | |
| /// `Arc<T>` prevents the `Arc`'s ref count from ever reaching 0, causing a | |
| /// leak. | |
| struct InterfacesHandlerImpl(Weak<Kernel>); | |
| impl InterfacesHandlerImpl { | |
| fn kernel(&self) -> Option<Arc<Kernel>> { | |
| self.0.upgrade() | |
| } | |
| } | |
| impl InterfacesHandler for InterfacesHandlerImpl { | |
| fn handle_new_link(&mut self, name: &str, interface_id: NonZeroU64) { | |
| if let Some(kernel) = self.kernel() { | |
| kernel.netstack_devices.add_device(&kernel, name.into(), interface_id); | |
| } | |
| } | |
| fn handle_deleted_link(&mut self, name: &str) { | |
| if let Some(kernel) = self.kernel() { | |
| kernel.netstack_devices.remove_device(kernel.kthreads.system_task(), name.into()); | |
| } | |
| } | |
| fn handle_idle_event(&mut self) { | |
| let Some(kernel) = self.kernel() else { | |
| log_error!("kernel went away while netlink is initializing"); | |
| return; | |
| }; | |
| let (initialized, wq) = &kernel.netstack_devices.initialized_and_wq; | |
| if initialized.swap(true, Ordering::SeqCst) { | |
| log_error!("netlink initial devices should only be reported once"); | |
| return; | |
| } | |
| wq.notify_all() | |
| } | |
| } | |
| impl Kernel { | |
| pub fn new( | |
| cmdline: BString, | |
| features: KernelFeatures, | |
| system_limits: SystemLimits, | |
| container_namespace: ContainerNamespace, | |
| scheduler: SchedulerManager, | |
| crash_reporter_proxy: Option<CrashReporterProxy>, | |
| inspect_node: fuchsia_inspect::Node, | |
| security_state: security::KernelState, | |
| time_adjustment_proxy: Option<AdjustSynchronousProxy>, | |
| device_tree: Option<Devicetree>, | |
| ) -> Result<Arc<Kernel>, zx::Status> { | |
| let unix_address_maker = | |
| Box::new(|x: FsString| -> SocketAddress { SocketAddress::Unix(x) }); | |
| let vsock_address_maker = Box::new(|x: u32| -> SocketAddress { | |
| SocketAddress::Vsock { port: x, cid: VMADDR_CID_HOST } | |
| }); | |
| let crash_reporter = CrashReporter::new( | |
| &inspect_node, | |
| crash_reporter_proxy, | |
| zx::Duration::from_minutes(8), | |
| features.crash_report_throttling, | |
| ); | |
| let hrtimer_manager = HrTimerManager::new(&inspect_node); | |
| let cpu_feature_flags = | |
| zx::system_get_feature_flags::<CpuFeatureFlags>().unwrap_or_else(|e| { | |
| log_debug!("CPU feature flags are only supported on ARM64: {}, reporting 0", e); | |
| CpuFeatureFlags::empty() | |
| }); | |
| let cpu_sve_feature_flags = zx::system_get_feature_flags::<CpuSveFeatureFlags>() | |
| .unwrap_or_else(|e| { | |
| log_debug!("CPU SVE feature flags failed: {}, reporting 0", e); | |
| CpuSveFeatureFlags::empty() | |
| }); | |
| let hwcaps = HwCaps::from_cpu_feature_flags(cpu_feature_flags, cpu_sve_feature_flags); | |
| let this = Arc::new_cyclic(|kernel| Kernel { | |
| weak_self: kernel.clone(), | |
| kthreads: KernelThreads::new(kernel.clone()), | |
| features, | |
| pids: Default::default(), | |
| init_task: OnceLock::new(), | |
| trace_event_manager: Arc::new(TracePerformanceEventManager::new(kernel.clone())), | |
| expando: Default::default(), | |
| default_abstract_socket_namespace: AbstractUnixSocketNamespace::new(unix_address_maker), | |
| default_abstract_vsock_namespace: AbstractVsockSocketNamespace::new( | |
| vsock_address_maker, | |
| ), | |
| cmdline, | |
| device_tree, | |
| security_state, | |
| device_registry: Default::default(), | |
| container_namespace, | |
| mounts_lock: RwSeqLock::new(MountsWriteToken::new().into()), | |
| remote_block_device_registry: Default::default(), | |
| iptables: OnceLock::new(), | |
| shared_futexes: FutexTable::default(), | |
| root_uts_ns: Arc::new(UtsNamespace::default().into()), | |
| vdso: Vdso::new(), | |
| vdso_arch32: Vdso::new_arch32(), | |
| netstack_devices: Arc::default(), | |
| swap_files: Default::default(), | |
| generic_netlink: OnceLock::new(), | |
| network_netlink: OnceLock::new(), | |
| inspect_node, | |
| actions_logged: AtomicU16::new(0), | |
| suspend_resume_manager: Default::default(), | |
| next_mount_id: AtomicCounter::<u64>::new(1), | |
| next_peer_group_id: AtomicCounter::<u64>::new(1), | |
| next_namespace_id: AtomicCounter::<u64>::new(1), | |
| next_file_object_id: Default::default(), | |
| system_limits, | |
| ptrace_scope: AtomicU8::new(0), // Disable YAMA checks by default. | |
| restrict_dmesg: AtomicBool::new(false), | |
| disable_unprivileged_bpf: AtomicU8::new(0), // Enable unprivileged BPF by default. | |
| perf_event_paranoid: AtomicI32::new(2), | |
| build_version: OnceCell::new(), | |
| stats: Arc::new(KernelStats::default()), | |
| delayed_releaser: Default::default(), | |
| scheduler, | |
| syslog: Default::default(), | |
| mounts: Mounts::new(), | |
| hrtimer_manager, | |
| memory_attribution_manager: MemoryAttributionManager::new(kernel.clone()), | |
| crash_reporter, | |
| shutdown_start: OnceLock::new(), | |
| container_control_handle: Default::default(), | |
| ebpf_state: Default::default(), | |
| cgroups: Default::default(), | |
| time_adjustment_proxy, | |
| netstack_wake_group: OnceLock::new(), | |
| socket_tokens_store: Default::default(), | |
| hwcaps, | |
| syscall_log_filters: Default::default(), | |
| }); | |
| // Initialize the device registry before registering any devices. | |
| // | |
| // We will create sysfs recursively within this function. | |
| this.device_registry.objects.init(&this); | |
| // Make a copy of this Arc for the inspect lazy node to use but don't create an Arc cycle | |
| // because the inspect node that owns this reference is owned by the kernel. | |
| let kernel = Arc::downgrade(&this); | |
| this.inspect_node.record_lazy_child("thread_groups", move || { | |
| if let Some(kernel) = kernel.upgrade() { | |
| let inspector = kernel.get_thread_groups_inspect(); | |
| async move { Ok(inspector) }.boxed() | |
| } else { | |
| async move { Err(anyhow::format_err!("kernel was dropped")) }.boxed() | |
| } | |
| }); | |
| let kernel = Arc::downgrade(&this); | |
| this.inspect_node.record_lazy_child("cgroupv2", move || { | |
| if let Some(kernel) = kernel.upgrade() { | |
| async move { Ok(kernel.cgroups.cgroup2.get_cgroup_inspect()) }.boxed() | |
| } else { | |
| async move { Err(anyhow::format_err!("kernel was dropped")) }.boxed() | |
| } | |
| }); | |
| Ok(this) | |
| } | |
| /// Returns the init task for this kernel. | |
| pub fn get_init_task(&self) -> Result<Arc<Task>, Errno> { | |
| self.init_task.get().and_then(|t| t.upgrade()).ok_or_else(|| errno!(EINVAL)) | |
| } | |
| /// Shuts down userspace and the kernel in an orderly fashion, eventually terminating the root | |
| /// kernel process. | |
| pub fn shut_down(self: &Arc<Self>) { | |
| // Run shutdown code on a kthread in the main process so that it can be the last process | |
| // alive. | |
| self.kthreads.spawn_future( | |
| { | |
| let kernel = self.clone(); | |
| move || async move { | |
| kernel.run_shutdown().await; | |
| } | |
| }, | |
| "run_shutdown", | |
| ); | |
| } | |
| /// Starts shutting down the Starnix kernel and any running container. Only one thread can drive | |
| /// shutdown at a time. This function will return immediately if shut down is already under way. | |
| /// | |
| /// Shutdown happens in several phases: | |
| /// | |
| /// 1. Stop processes from being created, except by deferred shutdown processes until the | |
| /// deferred shutdown deadline | |
| /// 2. Shut down individual ThreadGroups until only the init and system tasks remain, stopping | |
| /// the `stop_last_on_shutdown` ones last | |
| /// 3. Repeat the above for the init task | |
| /// 4. Clean up kernel-internal structures that can hold processes alive | |
| /// 5. Ensure this process is the only one running in the kernel job. | |
| /// 6. Unmounts the kernel's mounts' FileSystems. | |
| /// 7. Tell CF the container component has stopped | |
| /// 8. Exit this process | |
| /// | |
| /// If a ThreadGroup does not shut down on its own (including after SIGKILL), that phase of | |
| /// shutdown will hang. To gracefully shut down any further we need the other kernel processes | |
| /// to do controlled exits that properly release access to shared state. If our orderly shutdown | |
| /// does hang, eventually CF will kill the container component which will lead to the job of | |
| /// this process being killed and shutdown will still complete. | |
| async fn run_shutdown(&self) { | |
| const INIT_PID: i32 = 1; | |
| const SYSTEM_TASK_PID: i32 = 2; | |
| // Step 1: From here on, only deferred shutdown processes may create processes, and only | |
| // until the deferred shutdown deadline (see `create_zircon_process()`). This keeps Step 2 | |
| // finite: each round waits for its thread groups to exit, and after the deadline no new | |
| // ones can appear. | |
| if self.shutdown_start.set(zx::MonotonicInstant::get()).is_err() { | |
| log_info!("Additional thread tried to initiate shutdown while already in-progress."); | |
| return; | |
| } | |
| let deferred_shutdown_deadline = | |
| self.deferred_shutdown_deadline().expect("shutdown has started"); | |
| log_info!("Shutting down Starnix kernel."); | |
| // Step 2: Shut down thread groups in a loop until init and the system task are all that | |
| // remain. Processes running a `stop_last_on_shutdown` executable are kept until everything | |
| // else has exited, e.g. so that processes that crash during shutdown can still have their | |
| // crash dumps written. | |
| // TODO(https://fxbug.dev/574471394): Remove once a shutdown that isn't forced lets the | |
| // container run its own shutdown sequence. | |
| let stop_last = &self.features.stop_last_on_shutdown; | |
| let is_stop_last = |tg: &ThreadGroup| { | |
| let exe = tg.read().first_task().and_then(|task| task.mm().ok()); | |
| let exe = exe.and_then(|mm| mm.executable_node()); | |
| exe.is_some_and(|exe| stop_last.contains(&exe.path_escaping_chroot())) | |
| }; | |
| let mut keep_stop_last = !stop_last.is_empty(); | |
| loop { | |
| // Collect under the pid table lock so that no process is caught while it's being | |
| // created. `is_stop_last()` runs after the lock is released. | |
| let tgs = { | |
| let pids = self.pids.lock(); | |
| pids.get_thread_groups(&RcuReadScope::new()) | |
| .filter(|tg| tg.leader.id != SYSTEM_TASK_PID && tg.leader.id != INIT_PID) | |
| .collect::<Vec<_>>() | |
| }; | |
| let tgs = tgs | |
| .into_iter() | |
| .filter(|tg| !(keep_stop_last && is_stop_last(tg))) | |
| .collect::<Vec<_>>(); | |
| if tgs.is_empty() { | |
| if keep_stop_last { | |
| keep_stop_last = false; | |
| continue; | |
| } | |
| log_info!("pid table is empty except init and system task"); | |
| break; | |
| } | |
| log_info!(tgs:?; "shutting down thread groups"); | |
| let mut tasks = vec![]; | |
| for tg in tgs { | |
| let task = fasync::Task::local(ThreadGroup::shut_down( | |
| Arc::downgrade(&tg), | |
| deferred_shutdown_deadline, | |
| )); | |
| tasks.push(task); | |
| } | |
| futures::future::join_all(tasks).await; | |
| } | |
| // Step 3: Terminate the init process. | |
| let maybe_init = self.get_init_task().ok().map(|t| Arc::downgrade(&t.thread_group)); | |
| if let Some(init) = maybe_init { | |
| log_info!("shutting down init"); | |
| ThreadGroup::shut_down(init, deferred_shutdown_deadline).await; | |
| // If init forked while handling a core-dump signal, its children can outlive it. | |
| self.kill_processes_created_during_shutdown(); | |
| } else { | |
| log_info!("init already terminated"); | |
| } | |
| // Step 4: Clean up any structures that can keep non-Linux processes live in our job. | |
| log_info!("cleaning up pinned memory"); | |
| self.expando.remove::<crate::mm::InfoCacheShadowProcess>(); | |
| self.expando.remove::<crate::mm::MlockShadowProcess>(); | |
| // Step 5: Make sure this is the only process running in the job. We already should have | |
| // cleared up all processes other than the system task at this point, but wait on any that | |
| // might be around for good measure. | |
| // | |
| // Use unwrap liberally since we're shutting down anyway and errors will still tear down the | |
| // kernel. | |
| let kernel_job = fuchsia_runtime::job_default(); | |
| assert_eq!(kernel_job.children().unwrap(), &[], "starnix does not create any child jobs"); | |
| let own_koid = fuchsia_runtime::process_self().koid().unwrap(); | |
| log_info!("waiting for this to be the only process in the job"); | |
| let mut terminated_pids = HashSet::from([own_koid]); | |
| loop { | |
| let mut remaining_processes = kernel_job | |
| .processes() | |
| .unwrap() | |
| .into_iter() | |
| // Don't wait for ourselves or already-terminated processes to exit. | |
| // In Zircon, a dying process asserts `ZX_PROCESS_TERMINATED` before removing | |
| // itself from its parent job's process list (`ProcessDispatcher::FinishDeadTransition`), | |
| // so `kernel_job.processes()` can briefly still return a process whose | |
| // `PROCESS_TERMINATED` signal has already been awaited. | |
| .filter(|pid| terminated_pids.insert(*pid)) | |
| .peekable(); | |
| if remaining_processes.peek().is_none() { | |
| log_info!("No stray Zircon processes."); | |
| break; | |
| } | |
| let mut terminated_signals = vec![]; | |
| for pid in remaining_processes { | |
| let handle = match kernel_job | |
| .get_child(&pid, zx::Rights::BASIC | zx::Rights::PROPERTY | zx::Rights::DESTROY) | |
| { | |
| Ok(h) => h, | |
| Err(e) => { | |
| log_info!(pid:?, e:?; "failed to get child process from job"); | |
| continue; | |
| } | |
| }; | |
| log_info!( | |
| pid:?, | |
| name:? = handle.get_name(); | |
| "waiting on process terminated signal" | |
| ); | |
| terminated_signals | |
| .push(fuchsia_async::OnSignals::new(handle, zx::Signals::PROCESS_TERMINATED)); | |
| } | |
| log_info!("waiting on process terminated signals"); | |
| futures::future::join_all(terminated_signals).await; | |
| } | |
| // Step 6: Forcibly unmounts the mounts' FileSystems. | |
| log_info!("clearing mounts"); | |
| self.mounts.clear(); | |
| // Step 7: Tell CF the container stopped. | |
| log_info!("all non-root processes killed, notifying CF container is stopped"); | |
| if let Some(control_handle) = self.container_control_handle.lock().take() { | |
| log_info!("Notifying CF that the container has stopped."); | |
| control_handle | |
| .send_on_stop(ComponentStopInfo { | |
| termination_status: Some(zx::sys::ZX_OK), | |
| exit_code: Some(0), | |
| ..ComponentStopInfo::default() | |
| }) | |
| .unwrap(); | |
| control_handle.shutdown_with_epitaph(Ok(())); | |
| } else { | |
| log_warn!("Shutdown invoked without a container controller control handle."); | |
| } | |
| // Step 8: exiting this process. | |
| log_info!("All tasks killed, exiting Starnix kernel root process."); | |
| // Normally a Rust program exits its process by calling `std::process::exit()` which goes | |
| // through libc to exit the program. This runs drop impls on any thread-local variables | |
| // which can cause issues during Starnix shutdown when we haven't yet integrated every | |
| // subsystem with the shutdown flow. While those issues are indicative of underlying | |
| // problems, we can't solve them without finishing the implementation of graceful shutdown. | |
| // Instead, ask Zircon to exit our process directly, bypassing any libc atexit handlers. | |
| // TODO(https://fxbug.dev/295073633) return from main instead of avoiding atexit handlers | |
| zx::Process::exit(0); | |
| } | |
| pub fn is_shutting_down(&self) -> bool { | |
| self.shutdown_start.get().is_some() | |
| } | |
| /// Until when deferred shutdown processes may keep running and create processes, or `None` if | |
| /// the kernel is not shutting down. See `ThreadGroup::shut_down()`. | |
| pub fn deferred_shutdown_deadline(&self) -> Option<zx::MonotonicInstant> { | |
| self.shutdown_start.get().map(|start| *start + DEFERRED_SHUTDOWN_TIMEOUT) | |
| } | |
| /// SIGKILLs every process created during shutdown, all of which deferred shutdown processes | |
| /// created (e.g. crash dumpers). See `ThreadGroup::shut_down()`. | |
| /// | |
| /// A dumper that outlives the process it dumps fails, gets reported as a crash and can keep | |
| /// that process stopped under ptrace while it does. Processes still handling a core-dump signal | |
| /// aren't killed here: after their dumper exits, their handler re-raises the signal so Starnix | |
| /// files a crash report, the only record of a crash whose dump missed the deadline. Their own | |
| /// `ThreadGroup::shut_down()` gives them a moment to do so before sending SIGKILL. | |
| pub fn kill_processes_created_during_shutdown(&self) { | |
| // Collect under the pid table lock so that a process being created (which holds it from | |
| // the deadline check until its first task is added) is either fully added or not created. | |
| let created = { | |
| let pids = self.pids.lock(); | |
| pids.get_thread_groups(&RcuReadScope::new()) | |
| .filter(|tg| tg.created_during_shutdown.load(Ordering::Relaxed)) | |
| .collect::<Vec<_>>() | |
| }; | |
| for tg in created { | |
| tg.kill(ExitStatus::Kill(SIGKILL), None); | |
| } | |
| } | |
| pub fn allow_unprivileged_bpf(&self) -> bool { | |
| self.disable_unprivileged_bpf.load(Ordering::Relaxed) == 0 | |
| } | |
| /// Opens a device file (driver) identified by `dev`. | |
| pub fn open_device( | |
| &self, | |
| current_task: &CurrentTask, | |
| node: &NamespaceNode, | |
| flags: OpenFlags, | |
| dev: DeviceId, | |
| mode: DeviceMode, | |
| ) -> Result<Box<dyn FileOps>, Errno> { | |
| self.device_registry.open_device(current_task, node, flags, dev, mode) | |
| } | |
| /// Return a reference to the Audit Framework | |
| /// | |
| /// This function follows the lazy initialization pattern. | |
| pub fn audit_logger(&self) -> Arc<AuditLogger> { | |
| self.expando.get_or_init(|| AuditLogger::new(self)) | |
| } | |
| /// Return a reference to the GenericNetlink implementation. | |
| /// | |
| /// This function follows the lazy initialization pattern, where the first | |
| /// call will instantiate the Generic Netlink server in a separate kthread. | |
| pub fn generic_netlink(&self) -> &GenericNetlink<NetlinkToClientSender<GenericMessage>> { | |
| self.generic_netlink.get_or_init(|| { | |
| let (generic_netlink, worker_params) = GenericNetlink::new(); | |
| let enable_nl80211 = self.features.wifi; | |
| self.kthreads.spawn_future( | |
| move || async move { | |
| crate::vfs::socket::run_generic_netlink_worker(worker_params, enable_nl80211) | |
| .await; | |
| log_error!("Generic Netlink future unexpectedly exited"); | |
| }, | |
| "generic_netlink_worker", | |
| ); | |
| generic_netlink | |
| }) | |
| } | |
| /// Return a reference to the [`netlink::Netlink`] implementation. | |
| /// | |
| /// This function follows the lazy initialization pattern, where the first | |
| /// call will instantiate the Netlink implementation. | |
| pub fn network_netlink(self: &Arc<Self>) -> &Netlink<NetlinkContextImpl> { | |
| self.network_netlink.get_or_init(|| { | |
| let (network_netlink, worker_params) = | |
| Netlink::new(InterfacesHandlerImpl(self.weak_self.clone())); | |
| let kernel = self.clone(); | |
| self.kthreads.spawn_future( | |
| move || async move { | |
| netlink::run_netlink_worker( | |
| worker_params, | |
| NetlinkAccessControl::new(kernel.kthreads.system_task()), | |
| ) | |
| .await; | |
| log_error!(tag = NETLINK_LOG_TAG; "Netlink async worker unexpectedly exited"); | |
| }, | |
| "network_netlink_worker", | |
| ); | |
| network_netlink | |
| }) | |
| } | |
| /// Return a reference to the token representing our wake group registered | |
| /// with the netstack. | |
| /// | |
| /// This function lazily initializes the wake group with the netstack and | |
| /// the wake group as a wake source with the Starnix runner. | |
| pub(crate) fn netstack_wake_group(&self) -> Option<&fnet_resources::WakeGroupToken> { | |
| self.netstack_wake_group | |
| .get_or_init(|| { | |
| // The signal the netstack raises when it wants the container to | |
| // wake up. | |
| const GROUP_WAKEUP_SIGNAL: zx::Signals = | |
| zx::Signals::from_bits(fnet_power::GROUP_WAKEUP_SIGNAL).unwrap(); | |
| let provider = fuchsia_component::client::connect_to_protocol_sync::< | |
| fnet_power::WakeGroupProviderMarker, | |
| >() | |
| .expect("connect to WakeGroupProvider"); | |
| let (wake_watcher_waiter, wake_watcher_signaller) = zx::EventPair::create(); | |
| create_watcher_for_wake_events(wake_watcher_signaller); | |
| let token = match provider.create_wake_group( | |
| &fnet_power::WakeGroupOptions { | |
| debug_name: Some(String::from("starnix")), | |
| ..Default::default() | |
| }, | |
| wake_watcher_waiter, | |
| zx::Instant::INFINITE, | |
| ) { | |
| Ok(fnet_power::CreateWakeGroupResponse { token, .. }) => token, | |
| Err(e) => { | |
| log_error!("failed to create wake group with the netstack: {e}"); | |
| return None; | |
| } | |
| }; | |
| let token = token.expect("netstack provides a wake group token"); | |
| let wake_source = token | |
| .token | |
| .duplicate_handle(zx::Rights::SAME_RIGHTS) | |
| .expect("duplicate handle to token"); | |
| self.suspend_resume_manager | |
| .add_external_wake_source( | |
| wake_source.into(), | |
| GROUP_WAKEUP_SIGNAL, | |
| "netstack-wake-group".into(), | |
| ) | |
| .expect("add wake group as wake source"); | |
| Some(token) | |
| }) | |
| .as_ref() | |
| } | |
| pub fn iptables(&self) -> &IpTables { | |
| self.iptables.get_or_init(|| IpTables::new()) | |
| } | |
| /// Returns a Proxy to the service used by the container at `filename`. | |
| #[allow(unused)] | |
| pub fn connect_to_named_protocol_at_container_svc<P: ProtocolMarker>( | |
| &self, | |
| filename: &str, | |
| ) -> Result<ClientEnd<P>, Errno> { | |
| match self.container_namespace.get_namespace_channel("/svc") { | |
| Ok(channel) => { | |
| let (client_end, server_end) = create_endpoints::<P>(); | |
| fdio::service_connect_at(channel.as_ref(), filename, server_end.into_channel()) | |
| .map_err(|status| from_status_like_fdio!(status))?; | |
| Ok(client_end) | |
| } | |
| Err(err) => { | |
| log_error!("Unable to get /svc namespace channel! {}", err); | |
| Err(errno!(ENOENT)) | |
| } | |
| } | |
| } | |
| /// Returns a Proxy to the service `P` used by the container. | |
| pub fn connect_to_protocol_at_container_svc<P: DiscoverableProtocolMarker>( | |
| &self, | |
| ) -> Result<ClientEnd<P>, Errno> { | |
| self.connect_to_named_protocol_at_container_svc::<P>(P::PROTOCOL_NAME) | |
| } | |
| pub fn add_syscall_log_filter(&self, name: &str) { | |
| let filter = SyscallLogFilter::new(name.to_string()); | |
| { | |
| let mut filters = self.syscall_log_filters.lock(); | |
| if filters.contains(&filter) { | |
| return; | |
| } | |
| filters.push(filter); | |
| } | |
| for headers in self.pids.get_thread_groups(&RcuReadScope::new()) { | |
| headers.sync_syscall_log_level(); | |
| } | |
| } | |
| pub fn clear_syscall_log_filters(&self) { | |
| { | |
| let mut filters = self.syscall_log_filters.lock(); | |
| if filters.is_empty() { | |
| return; | |
| } | |
| filters.clear(); | |
| } | |
| for headers in self.pids.get_thread_groups(&RcuReadScope::new()) { | |
| headers.sync_syscall_log_level(); | |
| } | |
| } | |
| fn get_thread_groups_inspect(&self) -> fuchsia_inspect::Inspector { | |
| let inspector = fuchsia_inspect::Inspector::default(); | |
| let thread_groups = inspector.root(); | |
| let mut mm_summary = MappingSummary::default(); | |
| let mut mms_summarized = HashSet::new(); | |
| for thread_group in self.pids.get_thread_groups(&RcuReadScope::new()) { | |
| // Avoid holding the state lock while summarizing. | |
| let (ppid, tasks) = { | |
| let tg = thread_group.read(); | |
| (tg.get_ppid() as i64, tg.tasks()) | |
| }; | |
| let tg_node = thread_groups.create_child(format!("{}", thread_group.leader)); | |
| if let Ok(koid) = thread_group.process.koid() { | |
| tg_node.record_int("koid", koid.raw_koid() as i64); | |
| } | |
| tg_node.record_int("pid", thread_group.leader.id as i64); | |
| tg_node.record_int("ppid", ppid); | |
| tg_node.record_bool("stopped", thread_group.load_stopped() == StopState::GroupStopped); | |
| let tasks_node = tg_node.create_child("tasks"); | |
| for task in tasks { | |
| if let Ok(mm) = task.mm() { | |
| if mms_summarized.insert(Arc::as_ptr(&mm) as usize) { | |
| mm.summarize(&mut mm_summary); | |
| } | |
| } | |
| let set_properties = |node: &fuchsia_inspect::Node| { | |
| node.record_string("command", task.command().to_string()); | |
| let scheduler_state = task.scheduler_state.lock().clone(); | |
| if !scheduler_state.is_default() { | |
| node.record_child("sched", |node| { | |
| node.record_string( | |
| "role_name", | |
| self.scheduler | |
| .role_name(&task, &scheduler_state) | |
| .map(|n| Cow::Borrowed(n)) | |
| .unwrap_or_else(|e| Cow::Owned(e.to_string())), | |
| ); | |
| node.record_string("state", format!("{scheduler_state:?}")); | |
| }); | |
| } | |
| }; | |
| if task.tid == thread_group.leader { | |
| let mut argv = task.read_argv(256).unwrap_or_default(); | |
| // Any runtime that overwrites argv is likely to leave a lot of trailing | |
| // nulls, no need to print those in inspect. | |
| argv.retain(|arg| !arg.is_empty()); | |
| let inspect_argv = tg_node.create_string_array("argv", argv.len()); | |
| for (i, arg) in argv.iter().enumerate() { | |
| inspect_argv.set(i, arg.to_string()); | |
| } | |
| tg_node.record(inspect_argv); | |
| set_properties(&tg_node); | |
| } else { | |
| tasks_node.record_child(task.tid.to_string(), |task_node| { | |
| set_properties(task_node); | |
| }); | |
| }; | |
| } | |
| tg_node.record(tasks_node); | |
| thread_groups.record(tg_node); | |
| } | |
| thread_groups.record_child("memory_managers", |node| mm_summary.record(node)); | |
| inspector | |
| } | |
| pub fn new_memory_attribution_observer( | |
| &self, | |
| control_handle: fattribution::ProviderControlHandle, | |
| ) -> attribution_server::Observer { | |
| self.memory_attribution_manager.new_observer(control_handle) | |
| } | |
| /// Opens and returns a directory proxy from the container's namespace, at | |
| /// the requested path, using the provided flags. This method will open the | |
| /// closest existing path from the namespace hierarchy, and then attempt | |
| /// initialize an open on the remaining subdirectory path, using the given open_flags. | |
| /// | |
| /// For example, given the parameter provided is `/path/to/foo/bar` and there | |
| /// are namespace entries already for `/path/to/foo` and `/path/to`. The entry | |
| /// for /path/to/foo will be opened, and then the /bar will attempt to be opened | |
| /// underneath that directory with the given open_flags. The returned value | |
| /// will be the proxy to the parent (/path/to/foo) and the string to the child | |
| /// path (/bar). The caller of this method can expect /bar to be initialized. | |
| pub fn open_ns_dir( | |
| &self, | |
| path: &str, | |
| open_flags: fio::Flags, | |
| purpose: NamespaceNodePurpose, | |
| ) -> Result<(fio::DirectorySynchronousProxy, String), Errno> { | |
| check_ns_dir_path(path, purpose)?; | |
| match self.container_namespace.find_closest_channel(path) { | |
| Ok((root_channel, remaining_subdir)) => { | |
| let root_proxy = fio::DirectorySynchronousProxy::new(root_channel); | |
| if !remaining_subdir.is_empty() { | |
| syncio::directory_open_directory_sync( | |
| &root_proxy, | |
| &remaining_subdir, | |
| open_flags, | |
| zx::MonotonicInstant::INFINITE, | |
| ) | |
| .map_err(|status| { | |
| log_error!("Failed to initialize the subdirs: {}", status); | |
| from_status_like_fdio!(status) | |
| })?; | |
| } | |
| Ok((root_proxy, remaining_subdir)) | |
| } | |
| Err(err) => { | |
| log_error!("Unable to find a channel for {}. Received error: {}", path, err); | |
| Err(errno!(ENOENT)) | |
| } | |
| } | |
| } | |
| /// Returns an iterator of the command line arguments. | |
| pub fn cmdline_args_iter(&self) -> impl Iterator<Item = ArgNameAndValue<'_>> { | |
| parse_cmdline(self.cmdline.to_str().unwrap_or_default()).map(ArgNameAndValue::from) | |
| } | |
| /// Returns the container-configured CacheConfig. | |
| pub fn fs_cache_config(&self) -> CacheConfig { | |
| CacheConfig { capacity: self.features.dirent_cache_size as usize } | |
| } | |
| pub fn mounts_lock(&self) -> MountsWriteGuard<'_> { | |
| MountsWriteGuard::new(self.mounts_lock.lock()) | |
| } | |
| } | |
| pub fn parse_cmdline(cmdline: &str) -> impl Iterator<Item = &str> { | |
| let mut args = Vec::new(); | |
| let mut arg_start: Option<usize> = None; | |
| let mut in_quotes = false; | |
| let mut previous_char = ' '; | |
| for (i, c) in cmdline.char_indices() { | |
| if let Some(start) = arg_start { | |
| match c { | |
| ' ' if !in_quotes => { | |
| args.push(&cmdline[start..i]); | |
| arg_start = None; | |
| } | |
| '"' if previous_char != '\\' => { | |
| in_quotes = !in_quotes; | |
| } | |
| _ => {} | |
| } | |
| } else if c != ' ' { | |
| arg_start = Some(i); | |
| if c == '"' { | |
| in_quotes = true; | |
| } | |
| } | |
| previous_char = c; | |
| } | |
| if let Some(start) = arg_start { | |
| args.push(&cmdline[start..]); | |
| } | |
| args.into_iter() | |
| } | |
| impl std::fmt::Debug for Kernel { | |
| fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { | |
| f.debug_struct("Kernel").finish() | |
| } | |
| } | |
| // TODO(https://fxbug.dev/380427153): move arch dependent code to `kernel/core/arch/*`. | |
| #[cfg(target_arch = "aarch64")] | |
| fn arm32_hwcap(cpu_feature_flags: CpuFeatureFlags) -> HwCap { | |
| use starnix_uapi::arch32; | |
| const COMPAT_ARM32_ELF_HWCAP: u32 = arch32::HWCAP_HALF | |
| | arch32::HWCAP_THUMB | |
| | arch32::HWCAP_FAST_MULT | |
| | arch32::HWCAP_EDSP | |
| | arch32::HWCAP_TLS | |
| | arch32::HWCAP_IDIV // == IDIVA | IDIVT. | |
| | arch32::HWCAP_LPAE | |
| | arch32::HWCAP_EVTSTRM; | |
| let mut hwcap = COMPAT_ARM32_ELF_HWCAP; | |
| let mut hwcap2 = 0; | |
| for feature in cpu_feature_flags.iter() { | |
| match feature { | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_ASIMD => hwcap |= arch32::HWCAP_NEON, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_AES => hwcap2 |= arch32::HWCAP2_AES, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_PMULL => hwcap2 |= arch32::HWCAP2_PMULL, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_SHA1 => hwcap2 |= arch32::HWCAP2_SHA1, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_SHA256 => hwcap2 |= arch32::HWCAP2_SHA2, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_CRC32 => hwcap2 |= arch32::HWCAP2_CRC32, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_I8MM => hwcap |= arch32::HWCAP_I8MM, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_FHM => hwcap |= arch32::HWCAP_ASIMDFHM, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_DP => hwcap |= arch32::HWCAP_ASIMDDP, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_FP => { | |
| hwcap |= arch32::HWCAP_VFP | arch32::HWCAP_VFPv3 | arch32::HWCAP_VFPv4 | |
| } | |
| _ => {} | |
| } | |
| } | |
| // Both FEAT_FHM and FEAT_SVE architecturally require FEAT_FP16 (which is defined for both | |
| // AArch64 and AArch32 execution states). | |
| // TODO(https://fxbug.dev/42081850): Expose a dedicated `ZX_ARM64_FEATURE_ISA_FP16` flag from | |
| // Zircon so standalone Armv8.2 FEAT_FP16 cores without FHM or SVE also report FP16 hwcaps. | |
| if cpu_feature_flags | |
| .intersects(CpuFeatureFlags::ARM64_FEATURE_ISA_FHM | CpuFeatureFlags::ARM64_FEATURE_ISA_SVE) | |
| { | |
| hwcap |= arch32::HWCAP_FPHP | arch32::HWCAP_ASIMDHP; | |
| } | |
| HwCap { hwcap, hwcap2 } | |
| } | |
| #[cfg(target_arch = "aarch64")] | |
| fn arm64_hwcap( | |
| cpu_feature_flags: CpuFeatureFlags, | |
| cpu_sve_feature_flags: CpuSveFeatureFlags, | |
| ) -> HwCap { | |
| // See https://docs.kernel.org/arch/arm64/elf_hwcaps.html for details. | |
| use starnix_uapi; | |
| let mut hwcap = 0; | |
| let mut hwcap2 = 0; | |
| for feature in cpu_feature_flags.iter() { | |
| match feature { | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_FP => hwcap |= starnix_uapi::HWCAP_FP, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_ASIMD => hwcap |= starnix_uapi::HWCAP_ASIMD, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_AES => hwcap |= starnix_uapi::HWCAP_AES, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_PMULL => hwcap |= starnix_uapi::HWCAP_PMULL, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_SHA1 => hwcap |= starnix_uapi::HWCAP_SHA1, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_SHA256 => hwcap |= starnix_uapi::HWCAP_SHA2, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_CRC32 => hwcap |= starnix_uapi::HWCAP_CRC32, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_I8MM => hwcap2 |= starnix_uapi::HWCAP2_I8MM, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_FHM => hwcap |= starnix_uapi::HWCAP_ASIMDFHM, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_DP => hwcap |= starnix_uapi::HWCAP_ASIMDDP, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_SM3 => hwcap |= starnix_uapi::HWCAP_SM3, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_SM4 => hwcap |= starnix_uapi::HWCAP_SM4, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_SHA3 => hwcap |= starnix_uapi::HWCAP_SHA3, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_SHA512 => hwcap |= starnix_uapi::HWCAP_SHA512, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_ATOMICS => hwcap |= starnix_uapi::HWCAP_ATOMICS, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_RDM => hwcap |= starnix_uapi::HWCAP_ASIMDRDM, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_TS => hwcap |= starnix_uapi::HWCAP_FLAGM, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_DPB => hwcap |= starnix_uapi::HWCAP_DCPOP, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_RNDR => hwcap2 |= starnix_uapi::HWCAP2_RNG, | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_SVE => hwcap |= starnix_uapi::HWCAP_SVE, | |
| _ => {} | |
| } | |
| } | |
| // Both FEAT_FHM and FEAT_SVE architecturally require FEAT_FP16 (HWCAP_FPHP and HWCAP_ASIMDHP). | |
| // Per ACLE Function Multi-Versioning (FMV) dependency rules (used by LLVM/compiler-rt), "sve" | |
| // and "sve2" targets transitively require FEAT_FP | FEAT_FP16 | FEAT_SVE, where compiler-rt | |
| // derives FEAT_FP16 from HWCAP_FPHP. | |
| // TODO(https://fxbug.dev/42081850): Expose a dedicated `ZX_ARM64_FEATURE_ISA_FP16` flag from | |
| // Zircon so standalone Armv8.2 FEAT_FP16 cores without FHM or SVE also report FP16 hwcaps. | |
| if cpu_feature_flags | |
| .intersects(CpuFeatureFlags::ARM64_FEATURE_ISA_FHM | CpuFeatureFlags::ARM64_FEATURE_ISA_SVE) | |
| { | |
| hwcap |= starnix_uapi::HWCAP_FPHP | starnix_uapi::HWCAP_ASIMDHP; | |
| } | |
| if cpu_feature_flags.contains(CpuFeatureFlags::ARM64_FEATURE_ISA_SVE) { | |
| for feature in cpu_sve_feature_flags.iter() { | |
| match feature { | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEI8MM => { | |
| hwcap2 |= starnix_uapi::HWCAP2_SVEI8MM | |
| } | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEF32MM => { | |
| hwcap2 |= starnix_uapi::HWCAP2_SVEF32MM | |
| } | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEF64MM => { | |
| hwcap2 |= starnix_uapi::HWCAP2_SVEF64MM | |
| } | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEBF16 => { | |
| hwcap2 |= starnix_uapi::HWCAP2_SVEBF16 | |
| } | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVE2 => hwcap2 |= starnix_uapi::HWCAP2_SVE2, | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEAES => { | |
| hwcap2 |= starnix_uapi::HWCAP2_SVEAES | |
| } | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEPMULL => { | |
| hwcap2 |= starnix_uapi::HWCAP2_SVEPMULL | |
| } | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEBITPERM => { | |
| hwcap2 |= starnix_uapi::HWCAP2_SVEBITPERM | |
| } | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVESHA3 => { | |
| hwcap2 |= starnix_uapi::HWCAP2_SVESHA3 | |
| } | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVESM4 => { | |
| hwcap2 |= starnix_uapi::HWCAP2_SVESM4 | |
| } | |
| _ => {} | |
| } | |
| } | |
| } | |
| HwCap { hwcap, hwcap2 } | |
| } | |
| impl HwCaps { | |
| #[cfg(target_arch = "aarch64")] | |
| pub fn from_cpu_feature_flags( | |
| cpu_feature_flags: CpuFeatureFlags, | |
| cpu_sve_feature_flags: CpuSveFeatureFlags, | |
| ) -> Self { | |
| Self { | |
| arch32: arm32_hwcap(cpu_feature_flags), | |
| arch64: arm64_hwcap(cpu_feature_flags, cpu_sve_feature_flags), | |
| } | |
| } | |
| #[cfg(not(target_arch = "aarch64"))] | |
| pub fn from_cpu_feature_flags( | |
| _cpu_feature_flags: CpuFeatureFlags, | |
| _cpu_sve_feature_flags: CpuSveFeatureFlags, | |
| ) -> Self { | |
| Self { arch64: HwCap::default() } | |
| } | |
| } | |
| #[cfg(test)] | |
| mod test { | |
| use super::*; | |
| #[test] | |
| fn test_parse_cmdline() { | |
| let cmdline = | |
| r#"first second=third "fourth fifth" sixth="seventh eighth" "ninth\" tenth" eleventh"#; | |
| let expected = vec![ | |
| "first", | |
| "second=third", | |
| "\"fourth fifth\"", | |
| "sixth=\"seventh eighth\"", | |
| "\"ninth\\\" tenth\"", | |
| "eleventh", | |
| ]; | |
| assert_eq!(parse_cmdline(cmdline).collect::<Vec<_>>(), expected); | |
| } | |
| #[cfg(target_arch = "aarch64")] | |
| #[test] | |
| fn test_arm64_hwcap_sve() { | |
| let empty = | |
| HwCaps::from_cpu_feature_flags(CpuFeatureFlags::empty(), CpuSveFeatureFlags::empty()); | |
| assert_eq!(empty.arch64.hwcap, 0); | |
| assert_eq!(empty.arch64.hwcap2, 0); | |
| let expected_sve_hwcap = | |
| starnix_uapi::HWCAP_SVE | starnix_uapi::HWCAP_FPHP | starnix_uapi::HWCAP_ASIMDHP; | |
| let sve_only = HwCaps::from_cpu_feature_flags( | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_SVE, | |
| CpuSveFeatureFlags::empty(), | |
| ); | |
| assert_eq!(sve_only.arch64.hwcap, expected_sve_hwcap); | |
| assert_eq!(sve_only.arch64.hwcap2, 0); | |
| assert_eq!( | |
| sve_only.arch32.hwcap, | |
| empty.arch32.hwcap | |
| | starnix_uapi::arch32::HWCAP_FPHP | |
| | starnix_uapi::arch32::HWCAP_ASIMDHP | |
| ); | |
| assert_eq!(sve_only.arch32.hwcap2, 0); | |
| let fhm_only = HwCaps::from_cpu_feature_flags( | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_FHM, | |
| CpuSveFeatureFlags::empty(), | |
| ); | |
| assert_eq!( | |
| fhm_only.arch64.hwcap, | |
| starnix_uapi::HWCAP_ASIMDFHM | starnix_uapi::HWCAP_FPHP | starnix_uapi::HWCAP_ASIMDHP | |
| ); | |
| assert_eq!(fhm_only.arch64.hwcap2, 0); | |
| assert_eq!( | |
| fhm_only.arch32.hwcap, | |
| empty.arch32.hwcap | |
| | starnix_uapi::arch32::HWCAP_ASIMDFHM | |
| | starnix_uapi::arch32::HWCAP_FPHP | |
| | starnix_uapi::arch32::HWCAP_ASIMDHP | |
| ); | |
| assert_eq!(fhm_only.arch32.hwcap2, 0); | |
| let cases = [ | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEI8MM, starnix_uapi::HWCAP2_SVEI8MM), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEF32MM, starnix_uapi::HWCAP2_SVEF32MM), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEF64MM, starnix_uapi::HWCAP2_SVEF64MM), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEBF16, starnix_uapi::HWCAP2_SVEBF16), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVE2, starnix_uapi::HWCAP2_SVE2), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEAES, starnix_uapi::HWCAP2_SVEAES), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEPMULL, starnix_uapi::HWCAP2_SVEPMULL), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEBITPERM, starnix_uapi::HWCAP2_SVEBITPERM), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVESHA3, starnix_uapi::HWCAP2_SVESHA3), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVESM4, starnix_uapi::HWCAP2_SVESM4), | |
| ]; | |
| let mut all_sve_flags = CpuSveFeatureFlags::empty(); | |
| let mut expected_hwcap2 = 0; | |
| for (flag, hwcap2_bit) in cases { | |
| let without_sve = HwCaps::from_cpu_feature_flags(CpuFeatureFlags::empty(), flag); | |
| assert_eq!(without_sve.arch64.hwcap, 0); | |
| assert_eq!(without_sve.arch64.hwcap2, 0); | |
| let caps = HwCaps::from_cpu_feature_flags(CpuFeatureFlags::ARM64_FEATURE_ISA_SVE, flag); | |
| assert_eq!(caps.arch64.hwcap, expected_sve_hwcap); | |
| assert_eq!(caps.arch64.hwcap2, hwcap2_bit); | |
| all_sve_flags |= flag; | |
| expected_hwcap2 |= hwcap2_bit; | |
| } | |
| let all_caps = | |
| HwCaps::from_cpu_feature_flags(CpuFeatureFlags::ARM64_FEATURE_ISA_SVE, all_sve_flags); | |
| assert_eq!(all_caps.arch64.hwcap, expected_sve_hwcap); | |
| assert_eq!(all_caps.arch64.hwcap2, expected_hwcap2); | |
| } | |
| #[cfg(target_arch = "aarch64")] | |
| #[test] | |
| fn test_arm64_hwcap_sve_gating_and_linux_abi() { | |
| // The HWCAP bit positions are part of the Linux ABI | |
| // (`arch/arm64/include/uapi/asm/hwcap.h` and `arch/arm/include/uapi/asm/hwcap.h`). Pin | |
| // them, so that a wrong uapi binding or a transposed mapping in `arm64_hwcap` / | |
| // `arm32_hwcap` is caught immediately. | |
| assert_eq!(starnix_uapi::HWCAP_FP, 1 << 0); | |
| assert_eq!(starnix_uapi::HWCAP_ASIMD, 1 << 1); | |
| assert_eq!(starnix_uapi::HWCAP_FPHP, 1 << 9); | |
| assert_eq!(starnix_uapi::HWCAP_ASIMDHP, 1 << 10); | |
| assert_eq!(starnix_uapi::HWCAP_SVE, 1 << 22); | |
| assert_eq!(starnix_uapi::HWCAP_ASIMDFHM, 1 << 23); | |
| assert_eq!(starnix_uapi::arch32::HWCAP_FPHP, 1 << 22); | |
| assert_eq!(starnix_uapi::arch32::HWCAP_ASIMDHP, 1 << 23); | |
| assert_eq!(starnix_uapi::arch32::HWCAP_ASIMDFHM, 1 << 25); | |
| let expected_sve_hwcap = | |
| starnix_uapi::HWCAP_SVE | starnix_uapi::HWCAP_FPHP | starnix_uapi::HWCAP_ASIMDHP; | |
| let sve_mappings = [ | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVE2, starnix_uapi::HWCAP2_SVE2, 1 << 1), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEAES, starnix_uapi::HWCAP2_SVEAES, 1 << 2), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEPMULL, starnix_uapi::HWCAP2_SVEPMULL, 1 << 3), | |
| ( | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEBITPERM, | |
| starnix_uapi::HWCAP2_SVEBITPERM, | |
| 1 << 4, | |
| ), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVESHA3, starnix_uapi::HWCAP2_SVESHA3, 1 << 5), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVESM4, starnix_uapi::HWCAP2_SVESM4, 1 << 6), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEI8MM, starnix_uapi::HWCAP2_SVEI8MM, 1 << 9), | |
| ( | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEF32MM, | |
| starnix_uapi::HWCAP2_SVEF32MM, | |
| 1 << 10, | |
| ), | |
| ( | |
| CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEF64MM, | |
| starnix_uapi::HWCAP2_SVEF64MM, | |
| 1 << 11, | |
| ), | |
| (CpuSveFeatureFlags::ARM64_FEATURE_ISA_SVEBF16, starnix_uapi::HWCAP2_SVEBF16, 1 << 12), | |
| ]; | |
| let mut all_sve_flags = CpuSveFeatureFlags::empty(); | |
| let mut all_sve_hwcap2 = 0u32; | |
| for (flag, hwcap2_bit, linux_bit) in sve_mappings { | |
| assert_eq!(hwcap2_bit, linux_bit, "{flag:?}"); | |
| assert_eq!(all_sve_hwcap2 & hwcap2_bit, 0, "{flag:?} HWCAP2 bit is not unique"); | |
| all_sve_flags |= flag; | |
| all_sve_hwcap2 |= hwcap2_bit; | |
| } | |
| // Every named SVE subfeature reported by Zircon has a mapping. | |
| let all_named_sve_flags = <CpuSveFeatureFlags as bitflags::Flags>::FLAGS | |
| .iter() | |
| .filter(|f| f.is_named()) | |
| .fold(CpuSveFeatureFlags::empty(), |acc, f| acc | *f.value()); | |
| assert_eq!(all_sve_flags, all_named_sve_flags); | |
| // Linux never reports HWCAP2_SVE* without HWCAP_SVE. Without ZX_ARM64_FEATURE_ISA_SVE, the | |
| // SVE subfeatures (all of them at once, or even unknown bits in ZX_FEATURE_KIND_CPU_SVE) | |
| // must be ignored, while unrelated hwcaps are unaffected. | |
| let non_sve_cpu_flags = CpuFeatureFlags::ARM64_FEATURE_ISA_FP | |
| | CpuFeatureFlags::ARM64_FEATURE_ISA_ASIMD | |
| | CpuFeatureFlags::ARM64_FEATURE_ISA_I8MM | |
| | CpuFeatureFlags::ARM64_FEATURE_ISA_RNDR; | |
| let non_sve_hwcap = starnix_uapi::HWCAP_FP | starnix_uapi::HWCAP_ASIMD; | |
| let non_sve_hwcap2 = starnix_uapi::HWCAP2_I8MM | starnix_uapi::HWCAP2_RNG; | |
| let baseline = | |
| HwCaps::from_cpu_feature_flags(non_sve_cpu_flags, CpuSveFeatureFlags::empty()); | |
| assert_eq!(baseline.arch64.hwcap, non_sve_hwcap); | |
| assert_eq!(baseline.arch64.hwcap2, non_sve_hwcap2); | |
| for sve_flags in [all_sve_flags, CpuSveFeatureFlags::from_bits_retain(!0)] { | |
| let gated = HwCaps::from_cpu_feature_flags(CpuFeatureFlags::empty(), sve_flags); | |
| assert_eq!(gated.arch64.hwcap, 0, "{sve_flags:?}"); | |
| assert_eq!(gated.arch64.hwcap2, 0, "{sve_flags:?}"); | |
| let gated = HwCaps::from_cpu_feature_flags(non_sve_cpu_flags, sve_flags); | |
| assert_eq!(gated.arch64.hwcap, non_sve_hwcap, "{sve_flags:?}"); | |
| assert_eq!(gated.arch64.hwcap & starnix_uapi::HWCAP_SVE, 0, "{sve_flags:?}"); | |
| assert_eq!(gated.arch64.hwcap2, non_sve_hwcap2, "{sve_flags:?}"); | |
| assert_eq!(gated.arch64.hwcap2 & all_sve_hwcap2, 0, "{sve_flags:?}"); | |
| // The 32-bit (compat) hwcaps never include SVE. | |
| assert_eq!(gated.arch32.hwcap, baseline.arch32.hwcap); | |
| assert_eq!(gated.arch32.hwcap2, baseline.arch32.hwcap2); | |
| // With ZX_ARM64_FEATURE_ISA_SVE, exactly the known subfeature bits are added on top of | |
| // the unrelated hwcaps, and unknown bits in ZX_FEATURE_KIND_CPU_SVE are still ignored. | |
| let enabled = HwCaps::from_cpu_feature_flags( | |
| non_sve_cpu_flags | CpuFeatureFlags::ARM64_FEATURE_ISA_SVE, | |
| sve_flags, | |
| ); | |
| assert_eq!(enabled.arch64.hwcap, non_sve_hwcap | expected_sve_hwcap); | |
| assert_eq!(enabled.arch64.hwcap2, non_sve_hwcap2 | all_sve_hwcap2); | |
| assert_eq!( | |
| enabled.arch32.hwcap, | |
| baseline.arch32.hwcap | |
| | starnix_uapi::arch32::HWCAP_FPHP | |
| | starnix_uapi::arch32::HWCAP_ASIMDHP | |
| ); | |
| assert_eq!(enabled.arch32.hwcap2, baseline.arch32.hwcap2); | |
| } | |
| // Leaving out a single subfeature clears exactly its HWCAP2 bit, which catches duplicated | |
| // or transposed mappings that `test_arm64_hwcap_sve`'s single-flag checks would not. | |
| for (flag, hwcap2_bit, _) in sve_mappings { | |
| let all_but_one = HwCaps::from_cpu_feature_flags( | |
| CpuFeatureFlags::ARM64_FEATURE_ISA_SVE, | |
| all_sve_flags.difference(flag), | |
| ); | |
| assert_eq!(all_but_one.arch64.hwcap, expected_sve_hwcap, "{flag:?}"); | |
| assert_eq!(all_but_one.arch64.hwcap2, all_sve_hwcap2 & !hwcap2_bit, "{flag:?}"); | |
| } | |
| } | |
| } |