blob: 397fd6e7a025aa18f925767cc9d1201b70621e6f [file] [log] [blame]
// Copyright 2016 The Fuchsia Authors
// Copyright (c) 2008 Travis Geiselbrecht
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
#include <align.h>
#include <debug.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <trace.h>
#include <arch/arm64.h>
#include <arch/arm64/mp.h>
#include <arch/arm64/uarch.h>
#include <kernel/thread.h>
#define LOCAL_TRACE 0
// assert that the context switch frame is a multiple of 16 to maintain
// stack alignment requirements per ABI
static_assert(sizeof(arm64_context_switch_frame) % 16 == 0, "");
void arch_thread_initialize(Thread* t, vaddr_t entry_point) {
// zero out the entire arch state
t->arch() = {};
// create a default stack frame on the stack
vaddr_t stack_top = t->stack().top();
// make sure the top of the stack is 16 byte aligned for EABI compliance
DEBUG_ASSERT(IS_ALIGNED(stack_top, 16));
struct arm64_context_switch_frame* frame = (struct arm64_context_switch_frame*)(stack_top);
// fill in the entry point
frame->lr = entry_point;
// This is really a global (boot-time) constant value.
// But it's stored in each thread struct to satisfy the
t->arch().stack_guard = Thread::Current::Get()->arch().stack_guard;
// set the stack pointer
t->arch().sp = (vaddr_t)frame;
#if __has_feature(safe_stack)
DEBUG_ASSERT(IS_ALIGNED(t->stack_.unsafe_top(), 16));
t->arch().unsafe_sp = t->stack_.unsafe_top();
#if __has_feature(shadow_call_stack)
// The shadow call stack grows up.
t->arch().shadow_call_sp = reinterpret_cast<uintptr_t*>(t->stack().shadow_call_base());
__NO_SAFESTACK void arch_thread_construct_first(Thread* t) {
// Propagate the values from the fake arch_thread that the thread
// pointer points to now (set up in start.S) into the real thread
// structure being set up now.
Thread* fake = Thread::Current::Get();
const auto& fake_arch = fake->arch();
// Cache refs to the arch parts of the thread to avoid function calls to
// accessor routines because of the NO_STAFESTACK attribute here.
auto& arch = t->arch();
arch.stack_guard = fake_arch.stack_guard;
arch.unsafe_sp = fake_arch.unsafe_sp;
// make sure the thread saves a copy of the current cpu pointer
arch.current_percpu_ptr = arm64_read_percpu_ptr();
// Force the thread pointer immediately to the real struct. This way
// our callers don't have to avoid safe-stack code or risk losing track
// of the unsafe_sp value. The caller's unsafe_sp value is visible at
// TPIDR_EL1 + ZX_TLS_UNSAFE_SP_OFFSET as expected, though TPIDR_EL1
// happens to have changed. (We're assuming that the compiler doesn't
// decide to cache the TPIDR_EL1 value across this function call, which
// would be pointless since it's just one instruction to fetch it afresh.)
static void arm64_tpidr_save_state(Thread* thread) {
thread->arch().tpidr_el0 = __arm_rsr64("tpidr_el0");
thread->arch().tpidrro_el0 = __arm_rsr64("tpidrro_el0");
static void arm64_tpidr_restore_state(Thread* thread) {
__arm_wsr64("tpidr_el0", thread->arch().tpidr_el0);
__arm_wsr64("tpidrro_el0", thread->arch().tpidrro_el0);
static void arm64_debug_restore_state(Thread* thread) {
// If the thread has debug state, then install it, replacing the current contents.
if (unlikely(thread->arch().track_debug_state)) {
static void arm64_context_switch_spec_mitigations(Thread* oldthread, Thread* newthread) {
// Spectre V2: Flush Indirect Branch Predictor State, if:
// 0) Speculative Execution infoleak mitigations are enabled AND
// 1) The current CPU requires Spectre V2 mitigations AND
// 2a) We are switching between threads in different address spaces AND
// 2b) the old address space is not nullptr (not a kernel thread).
// If the old thread is a kernel thread, it can be trusted not to attack userspace
if (arm64_read_percpu_ptr()->should_invalidate_bp_on_context_switch &&
((oldthread->aspace() != newthread->aspace()) && oldthread->aspace())) {
void arch_context_switch(Thread* oldthread, Thread* newthread) {
LTRACEF("old %p (%s), new %p (%s)\n", oldthread, oldthread->name(), newthread, newthread->name());
// DSB here to make sure any pending TLB or cache operations that we may be
// preempting are complete before the thread switch. This avoids a problem in
// case the thread is moved to a new cpu.
// Set the current cpu pointer in the new thread's structure so it can be
// restored on exception entry.
newthread->arch().current_percpu_ptr = arm64_read_percpu_ptr();
if (likely(!oldthread->IsUserStateSavedLocked())) {
arm64_fpu_context_switch(oldthread, newthread);
// Not saving debug state because the arch_thread_t's debug state is authoritative.
} else {
// Nothing left to save for |oldthread|, so just restore |newthread|. Technically, we could
// skip restoring here since we know a higher layer will restore before leaving the kernel. We
// restore anyway so we don't leave |oldthread|'s state lingering in the hardware registers.
// The thinking is that:
// 1. The performance cost is tolerable - This code path is only executed by threads that have
// taken a (zircon) exception or are being debugged so there should be no performance impact to
// "normal" threads.
// 2. We want to avoid confusion - When, for example, the kernel panics and prints user register
// state to a log, a future maintainer might be confused to find that some other thread's user
// register state is present on a CPU that was executing an unrelated thread.
// 3. We want an extra layer of security - If we make a mistake and don't properly restore the
// state before returning we might expose one thread's register state to another thread. By
// restoring early, that's less likely to happen (think belt and suspenders).
arm64_context_switch_spec_mitigations(oldthread, newthread);
// Call into the inner assembly context switch routine to save integer registers on the old stack
// and swap to the new stack.
#if __has_feature(shadow_call_stack)
arm64_context_switch(&oldthread->arch().sp, newthread->arch().sp,
&oldthread->arch().shadow_call_sp, newthread->arch().shadow_call_sp);
arm64_context_switch(&oldthread->arch().sp, newthread->arch().sp,
void arch_dump_thread(Thread* t) {
if (t->state() != THREAD_RUNNING) {
dprintf(INFO, "\tarch: ");
dprintf(INFO, "sp 0x%lx\n", t->arch().sp);
void* arch_thread_get_blocked_fp(Thread* t) {
return nullptr;
struct arm64_context_switch_frame* frame = arm64_get_context_switch_frame(t);
return (void*)frame->r29;
arm64_context_switch_frame* arm64_get_context_switch_frame(Thread* thread) {
return reinterpret_cast<struct arm64_context_switch_frame*>(thread->arch().sp);
void arch_save_user_state(Thread* thread) {
// Not saving debug state because the arch_thread_t's debug state is authoritative.
void arch_restore_user_state(Thread* thread) {
void arch_set_suspended_general_regs(struct Thread* thread, GeneralRegsSource source,
void* iframe) {
DEBUG_ASSERT(thread->arch().suspended_general_regs == nullptr);
DEBUG_ASSERT(iframe != nullptr);
DEBUG_ASSERT_MSG(source == GeneralRegsSource::Iframe, "invalid source %u\n",
thread->arch().suspended_general_regs = static_cast<iframe_t*>(iframe);
void arch_reset_suspended_general_regs(struct Thread* thread) {
thread->arch().suspended_general_regs = nullptr;