blob: 652e8eaee33398cd0f15f064ddc4bbe24d5ac68a [file]
// Copyright 2016 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT
#ifndef ZIRCON_KERNEL_OBJECT_INCLUDE_OBJECT_THREAD_DISPATCHER_H_
#define ZIRCON_KERNEL_OBJECT_INCLUDE_OBJECT_THREAD_DISPATCHER_H_
#include <arch.h>
#include <lib/object-constants.h>
#include <platform.h>
#include <sys/types.h>
#include <zircon/compiler.h>
#include <zircon/syscalls/debug.h>
#include <zircon/syscalls/exception.h>
#include <zircon/syscalls/object.h>
#include <zircon/types.h>
#include <arch/exception.h>
#include <fbl/intrusive_double_list.h>
#include <fbl/ref_counted.h>
#include <fbl/ref_ptr.h>
#include <kernel/event.h>
#include <kernel/ffi.h>
#include <kernel/owned_wait_queue.h>
#include <kernel/task_runtime_stats.h>
#include <kernel/thread.h>
#include <ktl/atomic.h>
#include <ktl/string_view.h>
#include <object/dispatcher.h>
#include <object/exceptionate.h>
#include <object/futex_context.h>
#include <object/handle.h>
#include <vm/vm_address_region.h>
class ExceptionDispatcher;
class ProcessDispatcher;
class ThreadDispatcher;
// These are declared ahead of the class because ThreadDispatcher befriends
// cpp_thread_dispatcher_alloc, and a friend declaration requires the declaration it names to
// already be visible.
extern "C" {
void* rust_thread_dispatcher_get_core_thread_lock(const ThreadDispatcher* thread);
// Allocates and constructs a ThreadDispatcher. The Rust ThreadDispatcher::create() calls this
// and then attaches a lower-level kernel Thread to the result.
zx_status_t cpp_thread_dispatcher_alloc(
ProcessDispatcher* process, ffi::Uninitialized<KernelHandle<ThreadDispatcher>>* out_handle);
void cpp_thread_dispatcher_set_core_thread(ThreadDispatcher* thread, Thread* core_thread);
Thread* cpp_thread_dispatcher_get_core_thread(const ThreadDispatcher* thread);
void cpp_thread_dispatcher_set_user_entry(ThreadDispatcher* thread, const UserEntryState* entry);
int cpp_thread_start_routine_entry(void* thread);
// Rust FFI prototypes:
void rust_thread_dispatcher_state_init(void* state, const ThreadDispatcher* disp,
ProcessDispatcher* process);
void rust_thread_dispatcher_state_destroy(void* state);
Lock<CriticalMutex>* rust_thread_dispatcher_state_get_lock(const void* state);
zx_koid_t rust_thread_dispatcher_get_related_koid(const ThreadDispatcher* thread);
ProcessDispatcher* rust_thread_dispatcher_get_process(const ThreadDispatcher* thread);
void rust_thread_dispatcher_assert_canary(const ThreadDispatcher* thread);
bool rust_thread_dispatcher_is_dying_or_dead(const ThreadDispatcher* thread);
zx_status_t rust_thread_dispatcher_set_name(const ThreadDispatcher* thread, const char* name,
size_t len);
zx_status_t rust_thread_dispatcher_get_name(const ThreadDispatcher* thread,
char (*out_name)[ZX_MAX_NAME_LEN]);
zx_status_t rust_thread_dispatcher_make_runnable(const ThreadDispatcher* thread,
const UserEntryState* entry, bool suspended);
void rust_thread_dispatcher_kill(const ThreadDispatcher* thread);
zx_status_t rust_thread_dispatcher_suspend(const ThreadDispatcher* thread);
void rust_thread_dispatcher_resume(const ThreadDispatcher* thread);
void rust_thread_dispatcher_suspending(const ThreadDispatcher* thread);
void rust_thread_dispatcher_resuming(const ThreadDispatcher* thread);
bool rust_thread_dispatcher_handle_single_shot_exception(
ThreadDispatcher* thread, Exceptionate* exceptionate, zx_excp_type_t exception_type,
const arch_exception_context_t* arch_context);
void cpp_thread_handle_single_shot_exiting(ProcessDispatcher* process, ThreadDispatcher* thread);
void cpp_thread_dispatcher_reset_core_thread(ThreadDispatcher* thread);
void rust_thread_dispatcher_get_compensated_task_runtime_stats(const ThreadDispatcher* thread,
TaskRuntimeStats* out_stats);
void rust_thread_dispatcher_exiting_current(const ThreadDispatcher* thread);
uint32_t rust_thread_dispatcher_set_blocked_reason(uint32_t reason);
void* rust_thread_dispatcher_get_dll_node(ThreadDispatcher* thread);
}
struct ThreadDispatcherListTraits {
using NodeState = fbl::DoublyLinkedListNodeState<ThreadDispatcher*, fbl::NodeOptions::None>;
static_assert(sizeof(NodeState) == 2 * sizeof(uintptr_t));
static_assert(alignof(NodeState) == alignof(uintptr_t));
using PtrType = ThreadDispatcher*;
using TagType = fbl::DefaultObjectTag;
using PtrTraits = fbl::internal::ContainerPtrTraits<PtrType>;
static NodeState& node_state(ThreadDispatcher& obj) {
return *reinterpret_cast<NodeState*>(rust_thread_dispatcher_get_dll_node(&obj));
}
};
class ThreadDispatcher final : public Dispatcher {
public:
// When in a blocking syscall, or blocked in an exception, the blocking reason.
// There is one of these for each syscall marked "blocking".
// See //zircon/vdso.
enum class Blocked : uint32_t {
// Not blocked.
NONE,
// The thread is blocked in an exception.
EXCEPTION,
// The thread is sleeping (zx_nanosleep).
SLEEPING,
// zx_futex_wait
FUTEX,
// zx_port_wait
PORT,
// zx_channel_call
CHANNEL,
// zx_object_wait_one
WAIT_ONE,
// zx_object_wait_many
WAIT_MANY,
// zx_interrupt_wait
INTERRUPT,
// pager
PAGER,
};
using EntryState = UserEntryState;
~ThreadDispatcher();
static ThreadDispatcher* GetCurrent() { return rust_thread_current_user_thread(); }
// Terminates the current thread. Does not return.
static void ExitCurrent() __NO_RETURN { Thread::Current::Exit(0); }
// Asserts that the Rust ThreadDispatcherState's canary is valid.
void AssertCanary() const { rust_thread_dispatcher_assert_canary(this); }
// Dispatcher implementation.
zx_obj_type_t get_type() const final { return ZX_OBJ_TYPE_THREAD; }
zx_koid_t get_related_koid() const final {
AssertCanary();
return rust_thread_dispatcher_get_related_koid(this);
}
bool is_waitable() const final { return ZX_DEFAULT_THREAD_RIGHTS & ZX_RIGHT_WAIT; }
zx_status_t user_signal_self(uint32_t clear_mask, uint32_t set_mask) final {
return UserSignalSelfSolo(this, clear_mask, set_mask, 0);
}
zx_status_t user_signal_peer(uint32_t clear_mask, uint32_t set_mask) final {
return ZX_ERR_NOT_SUPPORTED;
}
// Sets whether or not this is the initial thread in its process.
// Should only be called by ProcessDispatcher upon adding the initialized thread.
void set_is_initial_thread(bool is_initial_thread) { is_initial_thread_ = is_initial_thread; }
// Transitions a thread from the INITIALIZED state to either the RUNNING or SUSPENDED state.
// Is the caller's responsibility to ensure this thread is registered with the parent process,
// as such this is only expected to be called from the ProcessDispatcher.
zx_status_t MakeRunnable(const EntryState& entry, bool suspended) {
return rust_thread_dispatcher_make_runnable(this, &entry, suspended);
}
void Kill() { rust_thread_dispatcher_kill(this); }
// Suspends the thread.
// Returns ZX_OK on success, or ZX_ERR_BAD_STATE iff the thread is dying or dead.
zx_status_t Suspend() { return rust_thread_dispatcher_suspend(this); }
void Resume() { rust_thread_dispatcher_resume(this); }
// accessors
ProcessDispatcher* process() const { return rust_thread_dispatcher_get_process(this); }
// Returns the lower level kernel thread backing this dispatcher, or nullptr once
// ExitingCurrent() has cleared it.
Thread* core_thread() const TA_REQ(get_lock()) { return core_thread_; }
// Stores the register values userspace will start executing with. Only meaningful before the
// thread has started, so this is not guarded.
void set_user_entry(const EntryState& entry) { user_entry_ = entry; }
// Returns true if the thread is dying or dead. Threads never return to a previous state
// from dying/dead so once this is true it will never flip back to false.
bool IsDyingOrDead() const TA_EXCL(get_lock()) {
return rust_thread_dispatcher_is_dying_or_dead(this);
}
[[nodiscard]] zx_status_t set_name(const char* name, size_t len) final __NONNULL((2))
TA_EXCL(get_lock()) {
return rust_thread_dispatcher_set_name(this, name, len);
}
[[nodiscard]] zx_status_t get_name(char (&out_name)[ZX_MAX_NAME_LEN]) const final
TA_EXCL(get_lock()) {
return rust_thread_dispatcher_get_name(this, &out_name);
}
// Sends a single-shot exception (which is sent to at most one handler, e.g.
// ZX_EXCP_THREAD_STARTING) over |exceptionate| and blocks for a response.
//
// Internally sets up the required exception state and creates the exception
// object from |exception_type| and |context|.
//
// Returns true if the exception was sent.
bool HandleSingleShotException(Exceptionate* exceptionate, zx_excp_type_t exception_type,
const arch_exception_context_t& context) TA_EXCL(get_lock()) {
return rust_thread_dispatcher_handle_single_shot_exception(this, exceptionate, exception_type,
&context);
}
// Fetch a consistent snapshot of the runtime stats, compensated for unaccumulated runtime in the
// ready or running state.
TaskRuntimeStats GetCompensatedTaskRuntimeStats() const {
TaskRuntimeStats stats;
rust_thread_dispatcher_get_compensated_task_runtime_stats(this, &stats);
return stats;
}
// Blocking syscalls, once they commit to a path that will likely block the
// thread, use this helper class to properly set/restore |blocked_reason_|.
class AutoBlocked final {
public:
explicit AutoBlocked(Blocked reason)
: prev_reason_(ThreadDispatcher::SetBlockedReason(reason)) {
DEBUG_ASSERT(reason != Blocked::NONE);
}
~AutoBlocked() { ThreadDispatcher::SetBlockedReason(prev_reason_); }
private:
const Blocked prev_reason_;
};
static Blocked SetBlockedReason(Blocked reason) {
return static_cast<Blocked>(
rust_thread_dispatcher_set_blocked_reason(static_cast<uint32_t>(reason)));
}
// This is called from Thread as it is exiting, just before it stops for good.
// It is an error to call this on anything other than the current thread.
void ExitingCurrent() { rust_thread_dispatcher_exiting_current(this); }
// callback from kernel when thread is suspending
void Suspending() { rust_thread_dispatcher_suspending(this); }
// callback from kernel when thread is resuming
void Resuming() { rust_thread_dispatcher_resuming(this); }
class CoreThreadObservation {
public:
CoreThreadObservation() = default;
~CoreThreadObservation() { Release(); }
void Release() {
if (core_thread_ != nullptr) {
DEBUG_ASSERT(lock_ != nullptr);
lock_->AssertHeld();
[this]() TA_NO_THREAD_SAFETY_ANALYSIS { lock_->Release(); }();
core_thread_ = nullptr;
lock_ = nullptr;
} else {
DEBUG_ASSERT(lock_ == nullptr);
}
}
Thread* core_thread() const { return core_thread_; }
CoreThreadObservation(const CoreThreadObservation&) = delete;
CoreThreadObservation& operator=(const CoreThreadObservation&) = delete;
CoreThreadObservation(CoreThreadObservation&& other) { *this = ktl::move(other); }
CoreThreadObservation& operator=(CoreThreadObservation&& other) {
// We should only ever be moving into a CoreThreadObservation instance
// which has been default constructed (eg, empty)
DEBUG_ASSERT((core_thread_ == nullptr) && (lock_ == nullptr));
core_thread_ = other.core_thread_;
lock_ = other.lock_;
other.core_thread_ = nullptr;
other.lock_ = nullptr;
return *this;
}
private:
friend class ThreadDispatcher;
CoreThreadObservation(Thread* core_thread, SpinLock* lock)
: core_thread_(core_thread), lock_(lock) {
if (lock_ != nullptr) {
[this]() TA_NO_THREAD_SAFETY_ANALYSIS { lock_->Acquire(); }();
}
}
Thread* core_thread_{nullptr};
SpinLock* lock_{nullptr};
};
CoreThreadObservation ObserveCoreThread() TA_REQ(get_lock()) {
SpinLock* lock =
&static_cast<Lock<SpinLock>*>(rust_thread_dispatcher_get_core_thread_lock(this))->lock();
return CoreThreadObservation{core_thread_, lock};
}
// WARNING: The members below, through `get_lock`, are public *only* so that the Rust port of
// FutexContext (object/futex_context.rs) can reach them through FFI shims. In C++ they were
// private and accessed exclusively by FutexContext via `friend class FutexContext`.
//
// It is *not* generally correct to use any of them outside of FutexContext. In particular,
// holding the locks named in their thread-safety annotations (if any) is strictly insufficient
// for safe usage: callers must also follow FutexContext's synchronization protocol (holding the
// owning FutexContext's state lock, holding pending-operation references on the involved
// threads, and obeying its chain-lock ordering). Do not add new callers.
//
// TODO(https://fxbug.dev/565800859): Find a better way to hide these from everything other than
// FutexContext.
// Accessors for `blocking_futex_id_`, the ID of the futex this thread is currently waiting on.
// These are guarded by FutexContext's own locking protocol, which cannot be expressed with
// static thread-safety annotations (see the comment on `blocking_futex_id_`).
//
// Only intended for use by FutexContext; see the WARNING above.
FutexId blocking_futex_id() const { return blocking_futex_id_; }
void set_blocking_futex_id(FutexId id) { blocking_futex_id_ = id; }
// Exposes this thread's dispatcher lock so that FutexContext can acquire it (as a "null-able"
// guard on a potential futex owner) through FFI shims. The lock itself is supplied by the Rust
// ThreadDispatcherState's KMutex.
//
// Only intended for use by FutexContext; see the WARNING above.
Lock<CriticalMutex>* get_lock() const final;
private:
// The constructor, StartRoutine, and set_core_thread stay private. They are only reachable
// through these shims, and friendship does not follow the caller across the FFI boundary, so
// the shims themselves are the friends rather than the Rust code that calls them.
friend zx_status_t cpp_thread_dispatcher_alloc(
ProcessDispatcher* process, ffi::Uninitialized<KernelHandle<ThreadDispatcher>>* out_handle);
friend void cpp_thread_dispatcher_set_core_thread(ThreadDispatcher* thread, Thread* core_thread);
friend int cpp_thread_start_routine_entry(void* thread);
explicit ThreadDispatcher(fbl::RefPtr<ProcessDispatcher> process);
ThreadDispatcher(const ThreadDispatcher&) = delete;
ThreadDispatcher& operator=(const ThreadDispatcher&) = delete;
friend void cpp_thread_dispatcher_reset_core_thread(ThreadDispatcher* thread)
TA_REQ(thread->get_lock());
// kernel level entry point
static int StartRoutine(void* arg);
// Sets the lower level kernel thread backing this dispatcher.
//
// Thread safety analysis is disabled here because this runs before the dispatcher has been
// published anywhere, so there is nothing to race with and no need to take the lock.
void set_core_thread(Thread* core_thread) TA_NO_THREAD_SAFETY_ANALYSIS {
core_thread_ = core_thread;
}
// The thread as understood by the lower kernel. This is set to nullptr when
// the thread's lifecycle transitions to DEAD.
Thread* core_thread_ TA_GUARDED(get_lock()) = nullptr;
// User thread starting register values.
EntryState user_entry_;
// If true and ancestor job has a debugger attached, thread will block on
// start and will send a process start exception.
bool is_initial_thread_ = false;
// The ID of the futex we are currently waiting on, or 0 if we are not
// waiting on any futex at the moment.
//
// TODO(johngro): figure out some way to apply clang static thread analysis
// to this. Right now, there is no good (cost free) way for the compiler to
// figure out that this thread belongs to a specific process/futex-context,
// and therefor the thread's futex-context lock can be used to guard this
// futex ID.
FutexId blocking_futex_id_{FutexId::Null()};
// Stores the Rust ThreadDispatcherState instance. It currently holds the canary, parent
// process reference, state machine, runtime stats, blocked reason, exceptionate, exception
// dispatcher, exceptionate type, suspend count, channel waiter, core thread lock, process
// thread list node, and state lock; the remaining fields above migrate into it as the methods
// that use them move to Rust.
OpaqueStorage<kThreadDispatcherStateSize, kThreadDispatcherStateAlign> opaque_storage_;
};
extern "C" {
uintptr_t cpp_thread_dispatcher_blocking_futex_id(const ThreadDispatcher* thread);
void cpp_thread_dispatcher_set_blocking_futex_id(ThreadDispatcher* thread, uintptr_t id);
bool cpp_thread_dispatcher_lock_acquire(ThreadDispatcher* thread);
void cpp_thread_dispatcher_lock_release(ThreadDispatcher* thread, bool should_clear);
void cpp_thread_dispatcher_observe_core_thread(
ThreadDispatcher* thread, ffi::Uninitialized<ThreadDispatcher::CoreThreadObservation>* out_obs);
void cpp_thread_dispatcher_release_core_thread_observation(
ThreadDispatcher::CoreThreadObservation* obs);
zx_status_t cpp_sys_thread_raise_exception(uint32_t options, zx_excp_type_t type,
const zx_exception_context_t* user_context);
}
#endif // ZIRCON_KERNEL_OBJECT_INCLUDE_OBJECT_THREAD_DISPATCHER_H_