| // Copyright 2023 The Fuchsia Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| use crate::execution::create_kernel_thread; | |
| use crate::mm::PrefetchThreadPool; | |
| use crate::task::dynamic_thread_spawner::DynamicThreadSpawner; | |
| use crate::task::{CurrentTask, DelayedReleaser, Kernel, Task, ThreadGroup}; | |
| use fragile::Fragile; | |
| use fuchsia_async as fasync; | |
| use fuchsia_sync::Completion; | |
| use pin_project::pin_project; | |
| use scopeguard::ScopeGuard; | |
| use starnix_task_command::TaskCommand; | |
| use starnix_uapi::errors::Errno; | |
| use starnix_uapi::{errno, error}; | |
| use std::future::Future; | |
| use std::pin::Pin; | |
| use std::sync::{Arc, OnceLock, Weak}; | |
| use std::task::{Context, Poll}; | |
| /// The threads that the kernel runs internally. | |
| /// | |
| /// These threads run in the main starnix process and outlive any specific userspace process. | |
| pub struct KernelThreads { | |
| /// The main starnix process. This process is used to create new processes when using the | |
| /// restricted executor. | |
| pub starnix_process: zx::Process, | |
| /// A handle to the async executor running in `starnix_process`. | |
| /// | |
| /// You can spawn tasks on this executor using `spawn_future`. However, those task must not | |
| /// block. If you need to block, you can spawn a worker thread using `spawner`. | |
| ehandle: fasync::EHandle, | |
| /// The background thread pool for memory prefetch operations (e.g. from madvise). Order above | |
| /// the `spawner` so that the threads are joined before the `spawner` is torn down. | |
| prefetch_pool: OnceLock<PrefetchThreadPool>, | |
| /// The thread pool to spawn blocking calls to. | |
| spawner: OnceLock<DynamicThreadSpawner>, | |
| /// Information about the main system task that is bound to the kernel main thread. | |
| system_task: OnceLock<SystemTask>, | |
| /// A weak reference to the kernel owning this struct. | |
| kernel: Weak<Kernel>, | |
| /// The RCU advancer thread running callbacks in the background. | |
| rcu_advancer: OnceLock<RcuAdvancer>, | |
| } | |
| /// The minimum time between running rcu callbacks. This allows us to limit wake ups and batch | |
| /// callbacks together. | |
| const RCU_RATE_LIMIT: std::time::Duration = std::time::Duration::from_millis(10); | |
| struct RcuAdvancer { | |
| stop: Arc<Completion>, | |
| thread: Option<std::thread::JoinHandle<()>>, | |
| } | |
| impl Drop for RcuAdvancer { | |
| fn drop(&mut self) { | |
| self.stop.signal(); | |
| fuchsia_rcu::rcu_advancer_wake(); | |
| if let Some(thread) = self.thread.take() { | |
| let _ = thread.join(); | |
| } | |
| } | |
| } | |
| impl KernelThreads { | |
| /// Create a KernelThreads object for the given Kernel. | |
| /// | |
| /// Must be called in the initial Starnix process on a thread with an async executor. This | |
| /// function captures the async executor for this thread for use with spawned futures. | |
| /// | |
| /// Used during kernel boot. | |
| pub fn new(kernel: Weak<Kernel>) -> Self { | |
| KernelThreads { | |
| starnix_process: fuchsia_runtime::process_self() | |
| .duplicate_handle(zx::Rights::SAME_RIGHTS) | |
| .expect("Failed to duplicate process self"), | |
| ehandle: fasync::EHandle::local(), | |
| spawner: Default::default(), | |
| system_task: Default::default(), | |
| kernel, | |
| rcu_advancer: Default::default(), | |
| prefetch_pool: Default::default(), | |
| } | |
| } | |
| /// Initialize this object with the system task that will be used for spawned threads. | |
| /// | |
| /// This function must be called before this object is used to spawn threads. | |
| pub fn init(&self, system_task: CurrentTask) -> Result<(), Errno> { | |
| self.system_task.set(SystemTask::new(system_task)).map_err(|_| errno!(EEXIST))?; | |
| self.spawner | |
| .set(DynamicThreadSpawner::new(2, self.system_task().weak_task(), "kthreadd/init")) | |
| .map_err(|_| errno!(EEXIST))?; | |
| let stop = Arc::new(Completion::new()); | |
| let stop_clone = Arc::clone(&stop); | |
| let thread = std::thread::Builder::new() | |
| .name("starnix-rcu".to_string()) | |
| .spawn(move || { | |
| let _rcu_registration = fuchsia_rcu::register_thread(); | |
| while !stop_clone.is_signaled() { | |
| fuchsia_rcu::rcu_advancer_wait_for_work(); | |
| while !stop_clone.is_signaled() { | |
| let start = std::time::Instant::now(); | |
| if !fuchsia_rcu::rcu_run_callbacks() { | |
| break; | |
| } | |
| let elapsed = start.elapsed(); | |
| if elapsed < RCU_RATE_LIMIT { | |
| if stop_clone.wait_for(RCU_RATE_LIMIT - elapsed) { | |
| break; | |
| } | |
| } | |
| } | |
| } | |
| // Run remaining ready callbacks before exiting. | |
| fuchsia_rcu::rcu_run_callbacks(); | |
| }) | |
| .expect("failed to spawn rcu advancer thread"); | |
| self.rcu_advancer | |
| .set(RcuAdvancer { stop, thread: Some(thread) }) | |
| .map_err(|_| errno!(EEXIST))?; | |
| self.prefetch_pool | |
| .set(PrefetchThreadPool::new(self.spawner())) | |
| .map_err(|_| errno!(EEXIST))?; | |
| Ok(()) | |
| } | |
| /// Spawn an async task in the main async executor to await the given future. | |
| /// | |
| /// Use this function to run async tasks in the background. These tasks cannot block or else | |
| /// they will starve the main async executor. | |
| /// | |
| /// Prefer this function to `spawn` for non-blocking work. | |
| pub fn spawn_future( | |
| &self, | |
| future: impl AsyncFnOnce() -> () + Send + 'static, | |
| name: &'static str, | |
| ) { | |
| self.ehandle.spawn_detached(WrappedMainFuture::new( | |
| self.kernel.clone(), | |
| async move { fasync::Task::local(future()).await }, | |
| name, | |
| )); | |
| } | |
| /// The dynamic thread spawner used to spawn threads. | |
| /// | |
| /// To spawn a thread in this thread pool, use `spawn()`. | |
| pub fn spawner(&self) -> &DynamicThreadSpawner { | |
| self.spawner.get().as_ref().unwrap() | |
| } | |
| /// Access the `CurrentTask` for the kernel main thread. | |
| /// | |
| /// This function can only be called from the kernel main thread itself. | |
| pub fn system_task(&self) -> &CurrentTask { | |
| self.system_task.get().expect("KernelThreads::init must be called").system_task.get() | |
| } | |
| /// Access the `ThreadGroup` for the system tasks. | |
| /// | |
| /// This function can be safely called from anywhere as soon as `KernelThreads::init` has been | |
| /// called. | |
| pub fn system_thread_group(&self) -> Arc<ThreadGroup> { | |
| self.system_task | |
| .get() | |
| .expect("KernelThreads::init must be called") | |
| .system_thread_group | |
| .upgrade() | |
| .expect("System task must be still alive") | |
| } | |
| /// Access the background thread pool for memory prefetch operations. | |
| pub fn prefetch_pool(&self) -> &PrefetchThreadPool { | |
| self.prefetch_pool.get().expect("KernelThreads::init must be called") | |
| } | |
| } | |
| impl Drop for KernelThreads { | |
| // TODO: Replace with .release. This is not actually safe, since locks | |
| // may be held elsewhere on this thread. | |
| fn drop(&mut self) { | |
| let _ = self.prefetch_pool.take(); | |
| if let Some(system_task) = self.system_task.take() { | |
| system_task.system_task.into_inner().release(()); | |
| } | |
| } | |
| } | |
| /// Create a new system task, register it on the thread and run the given closure with it. | |
| pub fn with_new_current_task<F, R>( | |
| system_task: &Weak<Task>, | |
| task_name: String, | |
| f: F, | |
| ) -> Result<R, Errno> | |
| where | |
| F: FnOnce(&CurrentTask) -> R, | |
| { | |
| let current_task = { | |
| let Some(system_task) = system_task.upgrade() else { | |
| return error!(ESRCH); | |
| }; | |
| create_kernel_thread(&system_task, TaskCommand::new(task_name.as_bytes())).unwrap() | |
| }; | |
| let result = f(¤t_task); | |
| current_task.release(()); | |
| // Ensure that no releasables are registered after this point as we unwind the stack. | |
| DelayedReleaser::finalize(); | |
| Ok(result) | |
| } | |
| struct SystemTask { | |
| /// The system task is bound to the kernel main thread. `Fragile` ensures a runtime crash if it | |
| /// is accessed from any other thread. | |
| system_task: Fragile<CurrentTask>, | |
| /// The system `ThreadGroup` is accessible from everywhere. | |
| system_thread_group: Weak<ThreadGroup>, | |
| } | |
| impl SystemTask { | |
| fn new(system_task: CurrentTask) -> Self { | |
| let system_thread_group = Arc::downgrade(&system_task.thread_group()); | |
| Self { system_task: system_task.into(), system_thread_group } | |
| } | |
| } | |
| // The order is important here. Rust will drop fields in declaration order and we want | |
| // the future to be dropped before the ScopeGuard runs. | |
| #[pin_project] | |
| pub(crate) struct WrappedFuture<F, C: Clone> { | |
| #[pin] | |
| fut: F, | |
| cleaner: fn(C), | |
| context: ScopeGuard<C, fn(C)>, | |
| name: &'static str, | |
| } | |
| impl<F, C: Clone> WrappedFuture<F, C> { | |
| pub(crate) fn new_with_cleaner(context: C, cleaner: fn(C), fut: F, name: &'static str) -> Self { | |
| // We need the ScopeGuard in case the future queues releasers when dropped. | |
| Self { fut, cleaner, context: ScopeGuard::with_strategy(context, cleaner), name } | |
| } | |
| } | |
| impl<F: Future, C: Clone> Future for WrappedFuture<F, C> { | |
| type Output = F::Output; | |
| fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { | |
| let this = self.project(); | |
| fuchsia_trace::duration!(starnix_logging::CATEGORY_STARNIX, &*this.name); | |
| let result = this.fut.poll(cx); | |
| (this.cleaner)(this.context.clone()); | |
| result | |
| } | |
| } | |
| type WrappedMainFuture<F> = WrappedFuture<F, Weak<Kernel>>; | |
| impl<F> WrappedMainFuture<F> { | |
| fn new(kernel: Weak<Kernel>, fut: F, name: &'static str) -> Self { | |
| Self::new_with_cleaner(kernel, trigger_delayed_releaser, fut, name) | |
| } | |
| } | |
| fn trigger_delayed_releaser(kernel: Weak<Kernel>) { | |
| if let Some(kernel) = kernel.upgrade() { | |
| if let Some(system_task) = kernel.kthreads.system_task.get() { | |
| system_task.system_task.get().trigger_delayed_releaser(); | |
| } | |
| } | |
| } |