| // Copyright 2016 The Fuchsia Authors |
| // Copyright (c) 2008-2015 Travis Geiselbrecht |
| // |
| // Use of this source code is governed by a MIT-style |
| // license that can be found in the LICENSE file or at |
| // https://opensource.org/licenses/MIT |
| |
| #pragma once |
| |
| #include <arch/defines.h> |
| #include <arch/ops.h> |
| #include <arch/thread.h> |
| #include <debug.h> |
| #include <kernel/cpu.h> |
| #include <kernel/spinlock.h> |
| #include <kernel/timer.h> |
| #include <kernel/wait.h> |
| #include <list.h> |
| #include <sys/types.h> |
| #include <vm/kstack.h> |
| #include <zircon/compiler.h> |
| #include <zircon/types.h> |
| |
| // fwd decl for the user_thread member below. |
| class ThreadDispatcher; |
| |
| __BEGIN_CDECLS |
| |
| enum thread_state { |
| THREAD_INITIAL = 0, |
| THREAD_READY, |
| THREAD_RUNNING, |
| THREAD_BLOCKED, |
| THREAD_SLEEPING, |
| THREAD_SUSPENDED, |
| THREAD_DEATH, |
| }; |
| |
| enum thread_user_state_change { |
| THREAD_USER_STATE_EXIT, |
| THREAD_USER_STATE_SUSPEND, |
| THREAD_USER_STATE_RESUME, |
| }; |
| |
| typedef int (*thread_start_routine)(void* arg); |
| typedef void (*thread_trampoline_routine)(void) __NO_RETURN; |
| typedef void (*thread_user_callback_t)(enum thread_user_state_change new_state, |
| struct thread* thread_context); |
| typedef void (*thread_tls_callback_t)(void* tls_value); |
| |
| // clang-format off |
| #define THREAD_FLAG_DETACHED (1 << 0) |
| #define THREAD_FLAG_FREE_STRUCT (1 << 1) |
| #define THREAD_FLAG_REAL_TIME (1 << 2) |
| #define THREAD_FLAG_IDLE (1 << 3) |
| |
| #define THREAD_SIGNAL_KILL (1 << 0) |
| #define THREAD_SIGNAL_SUSPEND (1 << 1) |
| #define THREAD_SIGNAL_POLICY_EXCEPTION (1 << 2) |
| // clang-format on |
| |
| #define THREAD_MAGIC (0x74687264) // 'thrd' |
| |
| // This includes the trailing NUL. |
| // N.B. This must match ZX_MAX_NAME_LEN. |
| #define THREAD_NAME_LENGTH 32 |
| |
| #define THREAD_LINEBUFFER_LENGTH 128 |
| |
| // Number of kernel tls slots. |
| #define THREAD_MAX_TLS_ENTRY 2 |
| |
| struct vmm_aspace; |
| |
| // This is a parallel structure to lockdep::ThreadLockState to work around the |
| // fact that this header is included by C code and cannot reference C++ types |
| // directly. This structure MUST NOT be touched by code outside of the lockdep |
| // implementation and MUST be kept in sync with the C++ counterpart. |
| typedef struct lockdep_state { |
| uintptr_t acquired_locks; |
| uint16_t reporting_disabled_count; |
| uint8_t last_result; |
| } lockdep_state_t; |
| |
| typedef struct thread { |
| int magic; |
| struct list_node thread_list_node; |
| |
| // active bits |
| struct list_node queue_node; |
| enum thread_state state; |
| zx_time_t last_started_running; |
| zx_duration_t remaining_time_slice; |
| unsigned int flags; |
| unsigned int signals; |
| |
| // Total time in THREAD_RUNNING state. If the thread is currently in |
| // THREAD_RUNNING state, this excludes the time it has accrued since it |
| // left the scheduler. |
| zx_duration_t runtime_ns; |
| |
| // priority: in the range of [MIN_PRIORITY, MAX_PRIORITY], from low to high. |
| // base_priority is set at creation time, and can be tuned with thread_set_priority(). |
| // priority_boost is a signed value that is moved around within a range by the scheduler. |
| // inherited_priority is temporarily set to >0 when inheriting a priority from another |
| // thread blocked on a locking primitive this thread holds. -1 means no inherit. |
| // effective_priority is MAX(base_priority + priority boost, inherited_priority) and is |
| // the working priority for run queue decisions. |
| int effec_priority; |
| int base_priority; |
| int priority_boost; |
| int inherited_priority; |
| |
| // current cpu the thread is either running on or in the ready queue, undefined otherwise |
| cpu_num_t curr_cpu; |
| cpu_num_t last_cpu; // last cpu the thread ran on, INVALID_CPU if it's never run |
| cpu_mask_t cpu_affinity; // mask of cpus that this thread can run on |
| |
| // if blocked, a pointer to the wait queue |
| struct wait_queue* blocking_wait_queue; |
| |
| // list of other wait queue heads if we're a head |
| struct list_node wait_queue_heads_node; |
| |
| // return code if woken up abnormally from suspend, sleep, or block |
| zx_status_t blocked_status; |
| |
| // are we allowed to be interrupted on the current thing we're blocked/sleeping on |
| bool interruptable; |
| |
| // number of mutexes we currently hold |
| int mutexes_held; |
| |
| #if WITH_LOCK_DEP |
| // state for runtime lock validation when in thread context |
| lockdep_state_t lock_state; |
| #endif |
| |
| // pointer to the kernel address space this thread is associated with |
| struct vmm_aspace* aspace; |
| |
| // pointer to user thread if one exists for this thread |
| ThreadDispatcher* user_thread; |
| uint64_t user_tid; |
| uint64_t user_pid; |
| |
| // callback for user thread state changes; do not invoke directly, use invoke_user_callback |
| // helper function instead |
| thread_user_callback_t user_callback; |
| |
| // non-NULL if stopped in an exception |
| const struct arch_exception_context* exception_context; |
| |
| // architecture stuff |
| struct arch_thread arch; |
| |
| kstack_t stack; |
| |
| // entry point |
| thread_start_routine entry; |
| void* arg; |
| |
| // return code |
| int retcode; |
| struct wait_queue retcode_wait_queue; |
| |
| // disable_counts contains two fields: |
| // |
| // * Bottom 16 bits: the preempt_disable counter. See |
| // thread_preempt_disable(). |
| // * Top 16 bits: the resched_disable counter. See |
| // thread_resched_disable(). |
| // |
| // This is a single field so that both counters can be compared against |
| // zero with a single memory access and comparison. |
| // |
| // disable_counts is modified by interrupt handlers, but it is always |
| // restored to its original value before the interrupt handler returns, |
| // so modifications are not visible to the interrupted thread. Despite |
| // that, "volatile" is still technically needed. Otherwise the |
| // compiler is technically allowed to compile |
| // "++thread->disable_counts" into code that stores a junk value into |
| // preempt_disable temporarily. |
| volatile uint32_t disable_counts; |
| |
| // preempt_pending tracks whether a thread reschedule is pending. |
| // |
| // This is volatile because it can be changed asynchronously by an |
| // interrupt handler: If preempt_disable is set, an interrupt handler |
| // may change this from false to true. Otherwise, if resched_disable |
| // is set, an interrupt handler may change this from true to false. |
| // |
| // preempt_pending should only be true: |
| // * if preempt_disable or resched_disable are non-zero, or |
| // * after preempt_disable or resched_disable have been decremented, |
| // while preempt_pending is being checked. |
| volatile bool preempt_pending; |
| |
| // thread local storage, initialized to zero |
| void* tls[THREAD_MAX_TLS_ENTRY]; |
| |
| // callback for cleanup of tls slots |
| thread_tls_callback_t tls_callback[THREAD_MAX_TLS_ENTRY]; |
| |
| char name[THREAD_NAME_LENGTH]; |
| #if WITH_DEBUG_LINEBUFFER |
| // buffering for debug/klog output |
| size_t linebuffer_pos; |
| char linebuffer[THREAD_LINEBUFFER_LENGTH]; |
| #endif |
| } thread_t; |
| |
| // thread priority |
| #define NUM_PRIORITIES (32) |
| #define LOWEST_PRIORITY (0) |
| #define HIGHEST_PRIORITY (NUM_PRIORITIES - 1) |
| #define DPC_PRIORITY (NUM_PRIORITIES - 2) |
| #define IDLE_PRIORITY LOWEST_PRIORITY |
| #define LOW_PRIORITY (NUM_PRIORITIES / 4) |
| #define DEFAULT_PRIORITY (NUM_PRIORITIES / 2) |
| #define HIGH_PRIORITY ((NUM_PRIORITIES / 4) * 3) |
| |
| // stack size |
| #ifdef CUSTOM_DEFAULT_STACK_SIZE |
| #define DEFAULT_STACK_SIZE CUSTOM_DEFAULT_STACK_SIZE |
| #else |
| #define DEFAULT_STACK_SIZE ARCH_DEFAULT_STACK_SIZE |
| #endif |
| |
| // functions |
| void thread_init_early(void); |
| void thread_become_idle(void) __NO_RETURN; |
| void thread_secondary_cpu_init_early(thread_t* t); |
| void thread_secondary_cpu_entry(void) __NO_RETURN; |
| void thread_construct_first(thread_t* t, const char* name); |
| thread_t* thread_create_idle_thread(uint cpu_num); |
| void thread_set_name(const char* name); |
| void thread_set_priority(thread_t* t, int priority); |
| void thread_set_user_callback(thread_t* t, thread_user_callback_t cb); |
| thread_t* thread_create(const char* name, thread_start_routine entry, void* arg, int priority); |
| thread_t* thread_create_etc(thread_t* t, const char* name, thread_start_routine entry, void* arg, |
| int priority, thread_trampoline_routine alt_trampoline); |
| void thread_resume(thread_t*); |
| zx_status_t thread_suspend(thread_t*); |
| void thread_signal_policy_exception(void); |
| void thread_exit(int retcode) __NO_RETURN; |
| void thread_forget(thread_t*); |
| |
| // set the mask of valid cpus to run the thread on. migrates the thread to satisfy |
| // the new constraint |
| void thread_set_cpu_affinity(thread_t* t, cpu_mask_t mask); |
| |
| // migrates the current thread to the CPU identified by target_cpu |
| void thread_migrate_to_cpu(cpu_num_t target_cpuid); |
| |
| zx_status_t thread_detach(thread_t* t); |
| zx_status_t thread_join(thread_t* t, int* retcode, zx_time_t deadline); |
| zx_status_t thread_detach_and_resume(thread_t* t); |
| zx_status_t thread_set_real_time(thread_t* t); |
| |
| // scheduler routines to be used by regular kernel code |
| void thread_yield(void); // give up the cpu and time slice voluntarily |
| void thread_preempt(void); // get preempted at irq time |
| void thread_reschedule(void); // re-evaluate the run queue on the current cpu |
| |
| void thread_owner_name(thread_t* t, char out_name[THREAD_NAME_LENGTH]); |
| |
| // print the backtrace on the current thread |
| void thread_print_current_backtrace(void); |
| |
| // append the backtrace of the current thread to the passed in char pointer up |
| // to `len' characters. |
| // return the number of chars appended. |
| size_t thread_append_current_backtrace(char* out, size_t len); |
| |
| // print the backtrace on the current thread at the given frame |
| void thread_print_current_backtrace_at_frame(void* caller_frame); |
| |
| // print the backtrace of the passed in thread, if possible |
| zx_status_t thread_print_backtrace(thread_t* t); |
| |
| // Return true if stopped in an exception. |
| static inline bool thread_stopped_in_exception(const thread_t* thread) { |
| return !!thread->exception_context; |
| } |
| |
| // wait until after the specified deadline. if interruptable, may return early with |
| // ZX_ERR_INTERNAL_INTR_KILLED if thread is signaled for kill. |
| zx_status_t thread_sleep_etc(zx_time_t deadline, |
| slack_mode slack_type, |
| zx_duration_t slack, |
| bool interruptable, |
| zx_time_t now); |
| |
| // non-interruptable version of thread_sleep_etc |
| zx_status_t thread_sleep(zx_time_t deadline); |
| |
| // non-interruptable relative delay version of thread_sleep |
| zx_status_t thread_sleep_relative(zx_duration_t delay); |
| |
| // interruptable version of thread_sleep |
| zx_status_t thread_sleep_interruptable(zx_time_t deadline); |
| |
| // return the number of nanoseconds a thread has been running for |
| zx_duration_t thread_runtime(const thread_t* t); |
| |
| // deliver a kill signal to a thread |
| void thread_kill(thread_t* t); |
| |
| // return true if thread has been signaled |
| static inline bool thread_is_signaled(thread_t* t) { |
| return t->signals != 0; |
| } |
| |
| // process pending signals, may never return because of kill signal |
| void thread_process_pending_signals(void); |
| |
| void dump_thread(thread_t* t, bool full); |
| void arch_dump_thread(thread_t* t); |
| void dump_all_threads(bool full); |
| void dump_all_threads_locked(bool full); |
| void dump_thread_user_tid(uint64_t tid, bool full); |
| void dump_thread_user_tid_locked(uint64_t tid, bool full); |
| |
| // find a thread based on the thread id |
| // NOTE: used only for debugging, its a slow linear search through the |
| // global thread list |
| thread_t* thread_id_to_thread_slow(uint64_t tid); |
| |
| static inline bool thread_is_realtime(thread_t* t) { |
| return (t->flags & THREAD_FLAG_REAL_TIME) && t->base_priority > DEFAULT_PRIORITY; |
| } |
| |
| static inline bool thread_is_idle(thread_t* t) { |
| return !!(t->flags & THREAD_FLAG_IDLE); |
| } |
| |
| static inline bool thread_is_real_time_or_idle(thread_t* t) { |
| return !!(t->flags & (THREAD_FLAG_REAL_TIME | THREAD_FLAG_IDLE)); |
| } |
| |
| // the current thread |
| #include <arch/current_thread.h> |
| thread_t* get_current_thread(void); |
| void set_current_thread(thread_t*); |
| |
| // scheduler lock |
| extern spin_lock_t thread_lock; |
| |
| static inline bool thread_lock_held(void) { |
| return spin_lock_held(&thread_lock); |
| } |
| |
| // Thread local storage. See tls_slots.h in the object layer above for |
| // the current slot usage. |
| |
| static inline void* tls_get(uint entry) { |
| return get_current_thread()->tls[entry]; |
| } |
| |
| static inline void* tls_set(uint entry, void* val) { |
| thread_t* curr_thread = get_current_thread(); |
| void* oldval = curr_thread->tls[entry]; |
| curr_thread->tls[entry] = val; |
| return oldval; |
| } |
| |
| // set the callback that is issued when the thread exits |
| static inline void tls_set_callback(uint entry, thread_tls_callback_t cb) { |
| get_current_thread()->tls_callback[entry] = cb; |
| } |
| |
| void thread_check_preempt_pending(void); |
| |
| static inline uint32_t thread_preempt_disable_count(void) { |
| return get_current_thread()->disable_counts & 0xffff; |
| } |
| |
| static inline uint32_t thread_resched_disable_count(void) { |
| return get_current_thread()->disable_counts >> 16; |
| } |
| |
| // thread_preempt_disable() increments the preempt_disable counter for the |
| // current thread. While preempt_disable is non-zero, preemption of the |
| // thread is disabled, including preemption from interrupt handlers. |
| // During this time, any call to thread_reschedule() or sched_reschedule() |
| // will only record that a reschedule is pending, and won't do a context |
| // switch. |
| // |
| // Note that this does not disallow blocking operations |
| // (e.g. mutex_acquire()). Disabling preemption does not prevent switching |
| // away from the current thread if it blocks. |
| // |
| // A call to thread_preempt_disable() must be matched by a later call to |
| // thread_preempt_reenable() to decrement the preempt_disable counter. |
| static inline void thread_preempt_disable(void) { |
| DEBUG_ASSERT(thread_preempt_disable_count() < 0xffff); |
| |
| thread_t* current_thread = get_current_thread(); |
| atomic_signal_fence(); |
| ++current_thread->disable_counts; |
| atomic_signal_fence(); |
| } |
| |
| // thread_preempt_reenable() decrements the preempt_disable counter. See |
| // thread_preempt_disable(). |
| static inline void thread_preempt_reenable(void) { |
| DEBUG_ASSERT(!arch_blocking_disallowed()); |
| DEBUG_ASSERT(thread_preempt_disable_count() > 0); |
| |
| thread_t* current_thread = get_current_thread(); |
| atomic_signal_fence(); |
| uint32_t new_count = --current_thread->disable_counts; |
| atomic_signal_fence(); |
| |
| if (new_count == 0) { |
| thread_check_preempt_pending(); |
| } |
| } |
| |
| // This is the same as thread_preempt_reenable(), except that it does not |
| // check for any pending reschedules. This is useful in interrupt handlers |
| // when we know that no reschedules should have become pending since |
| // calling thread_preempt_disable(). |
| static inline void thread_preempt_reenable_no_resched(void) { |
| DEBUG_ASSERT(thread_preempt_disable_count() > 0); |
| |
| thread_t* current_thread = get_current_thread(); |
| atomic_signal_fence(); |
| --current_thread->disable_counts; |
| atomic_signal_fence(); |
| } |
| |
| // thread_resched_disable() increments the resched_disable counter for the |
| // current thread. When resched_disable is non-zero, preemption of the |
| // thread from outside interrupt handlers is disabled. However, interrupt |
| // handlers may still preempt the thread. |
| // |
| // This is a weaker version of thread_preempt_disable(). |
| // |
| // As with preempt_disable, blocking operations are still allowed while |
| // resched_disable is non-zero. |
| // |
| // A call to thread_resched_disable() must be matched by a later call to |
| // thread_resched_reenable() to decrement the preempt_disable counter. |
| static inline void thread_resched_disable(void) { |
| DEBUG_ASSERT(thread_resched_disable_count() < 0xffff); |
| |
| thread_t* current_thread = get_current_thread(); |
| atomic_signal_fence(); |
| current_thread->disable_counts += 1 << 16; |
| atomic_signal_fence(); |
| } |
| |
| // thread_resched_reenable() decrements the preempt_disable counter. See |
| // thread_resched_disable(). |
| static inline void thread_resched_reenable(void) { |
| DEBUG_ASSERT(!arch_blocking_disallowed()); |
| DEBUG_ASSERT(thread_resched_disable_count() > 0); |
| |
| thread_t* current_thread = get_current_thread(); |
| atomic_signal_fence(); |
| uint32_t new_count = current_thread->disable_counts - (1 << 16); |
| current_thread->disable_counts = new_count; |
| atomic_signal_fence(); |
| |
| if (new_count == 0) { |
| thread_check_preempt_pending(); |
| } |
| } |
| |
| // thread_preempt_set_pending() marks a preemption as pending for the |
| // current CPU. |
| // |
| // This is similar to thread_reschedule(), except that it may only be |
| // used inside an interrupt handler while interrupts and preemption |
| // are disabled, between thread_preempt_disable() and |
| // thread_preempt_reenable(). It is similar to sched_reschedule(), |
| // except that it does not need to be called with thread_lock held. |
| static inline void thread_preempt_set_pending(void) { |
| DEBUG_ASSERT(arch_ints_disabled()); |
| DEBUG_ASSERT(arch_blocking_disallowed()); |
| thread_t* current_thread = get_current_thread(); |
| DEBUG_ASSERT(thread_preempt_disable_count() > 0); |
| |
| current_thread->preempt_pending = true; |
| } |
| |
| __END_CDECLS |
| |
| #ifdef __cplusplus |
| |
| #include <fbl/macros.h> |
| |
| // AutoReschedDisable is an RAII helper for disabling rescheduling |
| // using thread_resched_disable()/thread_resched_reenable(). |
| // |
| // A typical use case is when we wake another thread while holding a |
| // mutex. If the other thread is likely to claim the same mutex when |
| // runs (either immediately or later), then it is useful to defer |
| // waking the thread until after we have released the mutex. We can |
| // do that by disabling rescheduling while holding the lock. This is |
| // beneficial when there are no free CPUs for running the woken thread |
| // on. |
| // |
| // Example usage: |
| // |
| // AutoReschedDisable resched_disable; |
| // AutoLock al(&lock_); |
| // // Do some initial computation... |
| // resched_disable.Disable(); |
| // // Possibly wake another thread... |
| // |
| // The AutoReschedDisable must be placed before the AutoLock to ensure that |
| // rescheduling is re-enabled only after releasing the mutex. |
| class AutoReschedDisable { |
| public: |
| AutoReschedDisable() {} |
| ~AutoReschedDisable() { |
| if (started_) { |
| thread_resched_reenable(); |
| } |
| } |
| |
| void Disable() { |
| if (!started_) { |
| thread_resched_disable(); |
| started_ = true; |
| } |
| } |
| |
| DISALLOW_COPY_ASSIGN_AND_MOVE(AutoReschedDisable); |
| |
| private: |
| bool started_ = false; |
| }; |
| |
| #endif // __cplusplus |