| // Copyright 2016 The Fuchsia Authors | |
| // | |
| // Use of this source code is governed by a MIT-style | |
| // license that can be found in the LICENSE file or at | |
| // https://opensource.org/licenses/MIT | |
| #ifndef ZIRCON_KERNEL_OBJECT_INCLUDE_OBJECT_THREAD_DISPATCHER_H_ | |
| #define ZIRCON_KERNEL_OBJECT_INCLUDE_OBJECT_THREAD_DISPATCHER_H_ | |
| #include <arch.h> | |
| #include <lib/object-constants.h> | |
| #include <platform.h> | |
| #include <sys/types.h> | |
| #include <zircon/compiler.h> | |
| #include <zircon/syscalls/debug.h> | |
| #include <zircon/syscalls/exception.h> | |
| #include <zircon/syscalls/object.h> | |
| #include <zircon/types.h> | |
| #include <arch/exception.h> | |
| #include <fbl/intrusive_double_list.h> | |
| #include <fbl/ref_counted.h> | |
| #include <fbl/ref_ptr.h> | |
| #include <kernel/event.h> | |
| #include <kernel/ffi.h> | |
| #include <kernel/owned_wait_queue.h> | |
| #include <kernel/task_runtime_stats.h> | |
| #include <kernel/thread.h> | |
| #include <ktl/atomic.h> | |
| #include <ktl/string_view.h> | |
| #include <object/dispatcher.h> | |
| #include <object/exceptionate.h> | |
| #include <object/futex_context.h> | |
| #include <object/handle.h> | |
| #include <vm/vm_address_region.h> | |
| class ExceptionDispatcher; | |
| class ProcessDispatcher; | |
| class ThreadDispatcher; | |
| // These are declared ahead of the class because ThreadDispatcher befriends | |
| // cpp_thread_dispatcher_alloc, and a friend declaration requires the declaration it names to | |
| // already be visible. | |
| extern "C" { | |
| void* rust_thread_dispatcher_get_core_thread_lock(const ThreadDispatcher* thread); | |
| // Allocates and constructs a ThreadDispatcher. The Rust ThreadDispatcher::create() calls this | |
| // and then attaches a lower-level kernel Thread to the result. | |
| zx_status_t cpp_thread_dispatcher_alloc( | |
| ProcessDispatcher* process, ffi::Uninitialized<KernelHandle<ThreadDispatcher>>* out_handle); | |
| void cpp_thread_dispatcher_set_core_thread(ThreadDispatcher* thread, Thread* core_thread); | |
| Thread* cpp_thread_dispatcher_get_core_thread(const ThreadDispatcher* thread); | |
| void cpp_thread_dispatcher_set_user_entry(ThreadDispatcher* thread, const UserEntryState* entry); | |
| int cpp_thread_start_routine_entry(void* thread); | |
| // Rust FFI prototypes: | |
| void rust_thread_dispatcher_state_init(void* state, const ThreadDispatcher* disp, | |
| ProcessDispatcher* process); | |
| void rust_thread_dispatcher_state_destroy(void* state); | |
| Lock<CriticalMutex>* rust_thread_dispatcher_state_get_lock(const void* state); | |
| zx_koid_t rust_thread_dispatcher_get_related_koid(const ThreadDispatcher* thread); | |
| ProcessDispatcher* rust_thread_dispatcher_get_process(const ThreadDispatcher* thread); | |
| void rust_thread_dispatcher_assert_canary(const ThreadDispatcher* thread); | |
| bool rust_thread_dispatcher_is_dying_or_dead(const ThreadDispatcher* thread); | |
| zx_status_t rust_thread_dispatcher_set_name(const ThreadDispatcher* thread, const char* name, | |
| size_t len); | |
| zx_status_t rust_thread_dispatcher_get_name(const ThreadDispatcher* thread, | |
| char (*out_name)[ZX_MAX_NAME_LEN]); | |
| zx_status_t rust_thread_dispatcher_make_runnable(const ThreadDispatcher* thread, | |
| const UserEntryState* entry, bool suspended); | |
| void rust_thread_dispatcher_kill(const ThreadDispatcher* thread); | |
| zx_status_t rust_thread_dispatcher_suspend(const ThreadDispatcher* thread); | |
| void rust_thread_dispatcher_resume(const ThreadDispatcher* thread); | |
| void rust_thread_dispatcher_suspending(const ThreadDispatcher* thread); | |
| void rust_thread_dispatcher_resuming(const ThreadDispatcher* thread); | |
| bool rust_thread_dispatcher_handle_single_shot_exception( | |
| ThreadDispatcher* thread, Exceptionate* exceptionate, zx_excp_type_t exception_type, | |
| const arch_exception_context_t* arch_context); | |
| void cpp_thread_handle_single_shot_exiting(ProcessDispatcher* process, ThreadDispatcher* thread); | |
| void cpp_thread_dispatcher_reset_core_thread(ThreadDispatcher* thread); | |
| void rust_thread_dispatcher_get_compensated_task_runtime_stats(const ThreadDispatcher* thread, | |
| TaskRuntimeStats* out_stats); | |
| void rust_thread_dispatcher_exiting_current(const ThreadDispatcher* thread); | |
| uint32_t rust_thread_dispatcher_set_blocked_reason(uint32_t reason); | |
| void* rust_thread_dispatcher_get_dll_node(ThreadDispatcher* thread); | |
| } | |
| struct ThreadDispatcherListTraits { | |
| using NodeState = fbl::DoublyLinkedListNodeState<ThreadDispatcher*, fbl::NodeOptions::None>; | |
| static_assert(sizeof(NodeState) == 2 * sizeof(uintptr_t)); | |
| static_assert(alignof(NodeState) == alignof(uintptr_t)); | |
| using PtrType = ThreadDispatcher*; | |
| using TagType = fbl::DefaultObjectTag; | |
| using PtrTraits = fbl::internal::ContainerPtrTraits<PtrType>; | |
| static NodeState& node_state(ThreadDispatcher& obj) { | |
| return *reinterpret_cast<NodeState*>(rust_thread_dispatcher_get_dll_node(&obj)); | |
| } | |
| }; | |
| class ThreadDispatcher final : public Dispatcher { | |
| public: | |
| // When in a blocking syscall, or blocked in an exception, the blocking reason. | |
| // There is one of these for each syscall marked "blocking". | |
| // See //zircon/vdso. | |
| enum class Blocked : uint32_t { | |
| // Not blocked. | |
| NONE, | |
| // The thread is blocked in an exception. | |
| EXCEPTION, | |
| // The thread is sleeping (zx_nanosleep). | |
| SLEEPING, | |
| // zx_futex_wait | |
| FUTEX, | |
| // zx_port_wait | |
| PORT, | |
| // zx_channel_call | |
| CHANNEL, | |
| // zx_object_wait_one | |
| WAIT_ONE, | |
| // zx_object_wait_many | |
| WAIT_MANY, | |
| // zx_interrupt_wait | |
| INTERRUPT, | |
| // pager | |
| PAGER, | |
| }; | |
| using EntryState = UserEntryState; | |
| ~ThreadDispatcher(); | |
| static ThreadDispatcher* GetCurrent() { return rust_thread_current_user_thread(); } | |
| // Terminates the current thread. Does not return. | |
| static void ExitCurrent() __NO_RETURN { Thread::Current::Exit(0); } | |
| // Asserts that the Rust ThreadDispatcherState's canary is valid. | |
| void AssertCanary() const { rust_thread_dispatcher_assert_canary(this); } | |
| // Dispatcher implementation. | |
| zx_obj_type_t get_type() const final { return ZX_OBJ_TYPE_THREAD; } | |
| zx_koid_t get_related_koid() const final { | |
| AssertCanary(); | |
| return rust_thread_dispatcher_get_related_koid(this); | |
| } | |
| bool is_waitable() const final { return ZX_DEFAULT_THREAD_RIGHTS & ZX_RIGHT_WAIT; } | |
| zx_status_t user_signal_self(uint32_t clear_mask, uint32_t set_mask) final { | |
| return UserSignalSelfSolo(this, clear_mask, set_mask, 0); | |
| } | |
| zx_status_t user_signal_peer(uint32_t clear_mask, uint32_t set_mask) final { | |
| return ZX_ERR_NOT_SUPPORTED; | |
| } | |
| // Sets whether or not this is the initial thread in its process. | |
| // Should only be called by ProcessDispatcher upon adding the initialized thread. | |
| void set_is_initial_thread(bool is_initial_thread) { is_initial_thread_ = is_initial_thread; } | |
| // Transitions a thread from the INITIALIZED state to either the RUNNING or SUSPENDED state. | |
| // Is the caller's responsibility to ensure this thread is registered with the parent process, | |
| // as such this is only expected to be called from the ProcessDispatcher. | |
| zx_status_t MakeRunnable(const EntryState& entry, bool suspended) { | |
| return rust_thread_dispatcher_make_runnable(this, &entry, suspended); | |
| } | |
| void Kill() { rust_thread_dispatcher_kill(this); } | |
| // Suspends the thread. | |
| // Returns ZX_OK on success, or ZX_ERR_BAD_STATE iff the thread is dying or dead. | |
| zx_status_t Suspend() { return rust_thread_dispatcher_suspend(this); } | |
| void Resume() { rust_thread_dispatcher_resume(this); } | |
| // accessors | |
| ProcessDispatcher* process() const { return rust_thread_dispatcher_get_process(this); } | |
| // Returns the lower level kernel thread backing this dispatcher, or nullptr once | |
| // ExitingCurrent() has cleared it. | |
| Thread* core_thread() const TA_REQ(get_lock()) { return core_thread_; } | |
| // Stores the register values userspace will start executing with. Only meaningful before the | |
| // thread has started, so this is not guarded. | |
| void set_user_entry(const EntryState& entry) { user_entry_ = entry; } | |
| // Returns true if the thread is dying or dead. Threads never return to a previous state | |
| // from dying/dead so once this is true it will never flip back to false. | |
| bool IsDyingOrDead() const TA_EXCL(get_lock()) { | |
| return rust_thread_dispatcher_is_dying_or_dead(this); | |
| } | |
| [[nodiscard]] zx_status_t set_name(const char* name, size_t len) final __NONNULL((2)) | |
| TA_EXCL(get_lock()) { | |
| return rust_thread_dispatcher_set_name(this, name, len); | |
| } | |
| [[nodiscard]] zx_status_t get_name(char (&out_name)[ZX_MAX_NAME_LEN]) const final | |
| TA_EXCL(get_lock()) { | |
| return rust_thread_dispatcher_get_name(this, &out_name); | |
| } | |
| // Sends a single-shot exception (which is sent to at most one handler, e.g. | |
| // ZX_EXCP_THREAD_STARTING) over |exceptionate| and blocks for a response. | |
| // | |
| // Internally sets up the required exception state and creates the exception | |
| // object from |exception_type| and |context|. | |
| // | |
| // Returns true if the exception was sent. | |
| bool HandleSingleShotException(Exceptionate* exceptionate, zx_excp_type_t exception_type, | |
| const arch_exception_context_t& context) TA_EXCL(get_lock()) { | |
| return rust_thread_dispatcher_handle_single_shot_exception(this, exceptionate, exception_type, | |
| &context); | |
| } | |
| // Fetch a consistent snapshot of the runtime stats, compensated for unaccumulated runtime in the | |
| // ready or running state. | |
| TaskRuntimeStats GetCompensatedTaskRuntimeStats() const { | |
| TaskRuntimeStats stats; | |
| rust_thread_dispatcher_get_compensated_task_runtime_stats(this, &stats); | |
| return stats; | |
| } | |
| // Blocking syscalls, once they commit to a path that will likely block the | |
| // thread, use this helper class to properly set/restore |blocked_reason_|. | |
| class AutoBlocked final { | |
| public: | |
| explicit AutoBlocked(Blocked reason) | |
| : prev_reason_(ThreadDispatcher::SetBlockedReason(reason)) { | |
| DEBUG_ASSERT(reason != Blocked::NONE); | |
| } | |
| ~AutoBlocked() { ThreadDispatcher::SetBlockedReason(prev_reason_); } | |
| private: | |
| const Blocked prev_reason_; | |
| }; | |
| static Blocked SetBlockedReason(Blocked reason) { | |
| return static_cast<Blocked>( | |
| rust_thread_dispatcher_set_blocked_reason(static_cast<uint32_t>(reason))); | |
| } | |
| // This is called from Thread as it is exiting, just before it stops for good. | |
| // It is an error to call this on anything other than the current thread. | |
| void ExitingCurrent() { rust_thread_dispatcher_exiting_current(this); } | |
| // callback from kernel when thread is suspending | |
| void Suspending() { rust_thread_dispatcher_suspending(this); } | |
| // callback from kernel when thread is resuming | |
| void Resuming() { rust_thread_dispatcher_resuming(this); } | |
| class CoreThreadObservation { | |
| public: | |
| CoreThreadObservation() = default; | |
| ~CoreThreadObservation() { Release(); } | |
| void Release() { | |
| if (core_thread_ != nullptr) { | |
| DEBUG_ASSERT(lock_ != nullptr); | |
| lock_->AssertHeld(); | |
| [this]() TA_NO_THREAD_SAFETY_ANALYSIS { lock_->Release(); }(); | |
| core_thread_ = nullptr; | |
| lock_ = nullptr; | |
| } else { | |
| DEBUG_ASSERT(lock_ == nullptr); | |
| } | |
| } | |
| Thread* core_thread() const { return core_thread_; } | |
| CoreThreadObservation(const CoreThreadObservation&) = delete; | |
| CoreThreadObservation& operator=(const CoreThreadObservation&) = delete; | |
| CoreThreadObservation(CoreThreadObservation&& other) { *this = ktl::move(other); } | |
| CoreThreadObservation& operator=(CoreThreadObservation&& other) { | |
| // We should only ever be moving into a CoreThreadObservation instance | |
| // which has been default constructed (eg, empty) | |
| DEBUG_ASSERT((core_thread_ == nullptr) && (lock_ == nullptr)); | |
| core_thread_ = other.core_thread_; | |
| lock_ = other.lock_; | |
| other.core_thread_ = nullptr; | |
| other.lock_ = nullptr; | |
| return *this; | |
| } | |
| private: | |
| friend class ThreadDispatcher; | |
| CoreThreadObservation(Thread* core_thread, SpinLock* lock) | |
| : core_thread_(core_thread), lock_(lock) { | |
| if (lock_ != nullptr) { | |
| [this]() TA_NO_THREAD_SAFETY_ANALYSIS { lock_->Acquire(); }(); | |
| } | |
| } | |
| Thread* core_thread_{nullptr}; | |
| SpinLock* lock_{nullptr}; | |
| }; | |
| CoreThreadObservation ObserveCoreThread() TA_REQ(get_lock()) { | |
| SpinLock* lock = | |
| &static_cast<Lock<SpinLock>*>(rust_thread_dispatcher_get_core_thread_lock(this))->lock(); | |
| return CoreThreadObservation{core_thread_, lock}; | |
| } | |
| // WARNING: The members below, through `get_lock`, are public *only* so that the Rust port of | |
| // FutexContext (object/futex_context.rs) can reach them through FFI shims. In C++ they were | |
| // private and accessed exclusively by FutexContext via `friend class FutexContext`. | |
| // | |
| // It is *not* generally correct to use any of them outside of FutexContext. In particular, | |
| // holding the locks named in their thread-safety annotations (if any) is strictly insufficient | |
| // for safe usage: callers must also follow FutexContext's synchronization protocol (holding the | |
| // owning FutexContext's state lock, holding pending-operation references on the involved | |
| // threads, and obeying its chain-lock ordering). Do not add new callers. | |
| // | |
| // TODO(https://fxbug.dev/565800859): Find a better way to hide these from everything other than | |
| // FutexContext. | |
| // Accessors for `blocking_futex_id_`, the ID of the futex this thread is currently waiting on. | |
| // These are guarded by FutexContext's own locking protocol, which cannot be expressed with | |
| // static thread-safety annotations (see the comment on `blocking_futex_id_`). | |
| // | |
| // Only intended for use by FutexContext; see the WARNING above. | |
| FutexId blocking_futex_id() const { return blocking_futex_id_; } | |
| void set_blocking_futex_id(FutexId id) { blocking_futex_id_ = id; } | |
| // Exposes this thread's dispatcher lock so that FutexContext can acquire it (as a "null-able" | |
| // guard on a potential futex owner) through FFI shims. The lock itself is supplied by the Rust | |
| // ThreadDispatcherState's KMutex. | |
| // | |
| // Only intended for use by FutexContext; see the WARNING above. | |
| Lock<CriticalMutex>* get_lock() const final; | |
| private: | |
| // The constructor, StartRoutine, and set_core_thread stay private. They are only reachable | |
| // through these shims, and friendship does not follow the caller across the FFI boundary, so | |
| // the shims themselves are the friends rather than the Rust code that calls them. | |
| friend zx_status_t cpp_thread_dispatcher_alloc( | |
| ProcessDispatcher* process, ffi::Uninitialized<KernelHandle<ThreadDispatcher>>* out_handle); | |
| friend void cpp_thread_dispatcher_set_core_thread(ThreadDispatcher* thread, Thread* core_thread); | |
| friend int cpp_thread_start_routine_entry(void* thread); | |
| explicit ThreadDispatcher(fbl::RefPtr<ProcessDispatcher> process); | |
| ThreadDispatcher(const ThreadDispatcher&) = delete; | |
| ThreadDispatcher& operator=(const ThreadDispatcher&) = delete; | |
| friend void cpp_thread_dispatcher_reset_core_thread(ThreadDispatcher* thread) | |
| TA_REQ(thread->get_lock()); | |
| // kernel level entry point | |
| static int StartRoutine(void* arg); | |
| // Sets the lower level kernel thread backing this dispatcher. | |
| // | |
| // Thread safety analysis is disabled here because this runs before the dispatcher has been | |
| // published anywhere, so there is nothing to race with and no need to take the lock. | |
| void set_core_thread(Thread* core_thread) TA_NO_THREAD_SAFETY_ANALYSIS { | |
| core_thread_ = core_thread; | |
| } | |
| // The thread as understood by the lower kernel. This is set to nullptr when | |
| // the thread's lifecycle transitions to DEAD. | |
| Thread* core_thread_ TA_GUARDED(get_lock()) = nullptr; | |
| // User thread starting register values. | |
| EntryState user_entry_; | |
| // If true and ancestor job has a debugger attached, thread will block on | |
| // start and will send a process start exception. | |
| bool is_initial_thread_ = false; | |
| // The ID of the futex we are currently waiting on, or 0 if we are not | |
| // waiting on any futex at the moment. | |
| // | |
| // TODO(johngro): figure out some way to apply clang static thread analysis | |
| // to this. Right now, there is no good (cost free) way for the compiler to | |
| // figure out that this thread belongs to a specific process/futex-context, | |
| // and therefor the thread's futex-context lock can be used to guard this | |
| // futex ID. | |
| FutexId blocking_futex_id_{FutexId::Null()}; | |
| // Stores the Rust ThreadDispatcherState instance. It currently holds the canary, parent | |
| // process reference, state machine, runtime stats, blocked reason, exceptionate, exception | |
| // dispatcher, exceptionate type, suspend count, channel waiter, core thread lock, process | |
| // thread list node, and state lock; the remaining fields above migrate into it as the methods | |
| // that use them move to Rust. | |
| OpaqueStorage<kThreadDispatcherStateSize, kThreadDispatcherStateAlign> opaque_storage_; | |
| }; | |
| extern "C" { | |
| uintptr_t cpp_thread_dispatcher_blocking_futex_id(const ThreadDispatcher* thread); | |
| void cpp_thread_dispatcher_set_blocking_futex_id(ThreadDispatcher* thread, uintptr_t id); | |
| bool cpp_thread_dispatcher_lock_acquire(ThreadDispatcher* thread); | |
| void cpp_thread_dispatcher_lock_release(ThreadDispatcher* thread, bool should_clear); | |
| void cpp_thread_dispatcher_observe_core_thread( | |
| ThreadDispatcher* thread, ffi::Uninitialized<ThreadDispatcher::CoreThreadObservation>* out_obs); | |
| void cpp_thread_dispatcher_release_core_thread_observation( | |
| ThreadDispatcher::CoreThreadObservation* obs); | |
| zx_status_t cpp_sys_thread_raise_exception(uint32_t options, zx_excp_type_t type, | |
| const zx_exception_context_t* user_context); | |
| } | |
| #endif // ZIRCON_KERNEL_OBJECT_INCLUDE_OBJECT_THREAD_DISPATCHER_H_ |