blob: d74bf50f3cc9f9f147a6eb235c0591bc7b6a5a70 [file] [log] [blame]
// Copyright 2021 The Fuchsia Authors
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
#include <lib/zircon-internal/macros.h>
#include <kernel/thread.h>
#include <ktl/limits.h>
#include <object/thread_dispatcher.h>
#include <vm/page.h>
#include <vm/stack_owned_loaned_pages_interval.h>
#include <ktl/enforce.h>
void StackOwnedLoanedPagesInterval::PrepareForWaiter() {
// For now we don't need a CAS loop in here because thread_lock is held by callers of
// PrepareForWaiter() and PrepareForWaiter() is the only mutator of is_ready_for_waiter_. Even if
// we did have a CAS loop, the caller would still need to guarantee somehow that the interval
// won't get deleted out from under this call. Currently that's guaranteed by the current
// thread_lock hold interval being the same interval that set kObjectOrStackOwnerHasWaiter.
// Because all setters of is_ready_for_waiter_ hold thread_lock, we could use memory_order_relaxed
// here, but for now we're using acquire for all loads of is_ready_for_waiter_.
if (is_ready_for_waiter_.load(ktl::memory_order_acquire)) {
// Thanks to thread_lock, we know that the current thread is the only thread setting
// is_ready_for_waiter_, so we can just set it using a store(). We also need to prepare the
// owned_wait_queue_ to have a waiter that can transmit its priority via priority inheritance to
// the stack-owning thread.
DEBUG_ASSERT(Thread::Current::Get() != owning_thread_);
// The memory_order_release isn't really needed here thanks to release of thread_lock by this
// thread shortly and acquire of thread_lock by any thread removing the
// StackOwnedLoanedPagesInterval from the page (before deleting the interval), but for now we're
// using release for all stores to is_ready_for_waiter_., ktl::memory_order_release);
// static
StackOwnedLoanedPagesInterval& StackOwnedLoanedPagesInterval::current() {
Thread* current_thread = Thread::Current::Get();
// The caller should only call current() when the caller knows there must be a current interval,
// and just needs to know which interval is the outer-most on this thread's stack.
// Stack ownership of a loaned page requires having a StackOwnedLoanedPagesInterval on the
// caller's stack.
"StackOwnedLoanedPagesInterval missing");
return *current_thread->stack_owned_loaned_pages_interval_;
// static
StackOwnedLoanedPagesInterval* StackOwnedLoanedPagesInterval::maybe_current() {
Thread* current_thread = Thread::Current::Get();
return current_thread->stack_owned_loaned_pages_interval_;
// static
void StackOwnedLoanedPagesInterval::WaitUntilContiguousPageNotStackOwned(vm_page_t* page) {
// Due to not holding the PmmNode lock, we can't check loaned directly, and it may have been unset
// recently in any case, but in that case we'll notice via !is_stack_owned() instead.
// Need to take thread_lock at this point, because avoiding deletion of the OwnedWaitQueue
// requires holding the thread_lock while applying kObjectOrStackOwnerHasWaiter to the page, to
// prevent the StackOwnedLoanedPagesInterval thread from removing the stack_owner from the page
// and deleting the OwnedWaitQueue. We also need the thread_lock to block on the
// OwnedWaitQueue.
// Before we acquire the thread_lock we do a check whether a stack_owner is still set. This is
// just to avoid acquiring the thread lock on the off chance that the stack ownership interval
// is already over. This isn't particularly likely to be the case, and we'd be fine without
// this check. But since we're about to take the thread_lock let's avoid an unnecessary acquire
// if we can.
if (!page->object.is_stack_owned()) {
// StackOwnedLoanedPagesInterval is already removed from the page, so no need to
// acquire the thread_lock. Go around and observe the new page state.
// Acquire thread_lock since that's required to ensure ~StackOwnedLoanedPagesInterval doesn't
// miss that this thread is blocked waiting, along with kObjectOrStackOwnerHasWaiter.
Guard<MonitoredSpinLock, IrqSave> thread_lock_guard{ThreadLock::Get(), SOURCE_TAG};
// Holding the thread_lock doesn't guarantee that the stack_owner won't be cleared, but holding
// thread_lock and successfully ensuring that kObjectOrStackOwnerHasWaiter is set does guarantee
// the stack_owner won't be cleared.
auto maybe_try_set_has_waiter_result = page->object.try_set_has_waiter();
if (!maybe_try_set_has_waiter_result) {
// stack_owner was cleared; no need to wait.
// ~thread_lock_guard
auto& try_set_has_waiter_result = maybe_try_set_has_waiter_result.value();
auto& stack_owner = *try_set_has_waiter_result.stack_owner;
// By doing PrepareForWaiter() only when necessary, we avoid pressure on the thread_lock in the
// case where there's no page reclaiming thread needing to wait / transmit priority.
if (try_set_has_waiter_result.first_setter) {
// PrepareForWaiter() was called previously, either by this thread or a different thread.
// At this point we know that the stack_owner won't be changed on the page while we hold
// thread_lock, which means the OwnedWaitQueue can't be deleted yet either, since deletion is
// after uninstalling from the page. So now we just need to block on the OwnedWaitQueue, which
// requires holding the thread_lock during the call anyway. We don't really care if this
// OwnedWaitQueue is relevant to moving from cow to cow, or cow to FREE, or during ALLOC state.
// In all those possible cases, we want to block on the OwnedWaitQueue. The fact that the
// OwnedWaitQueue is there is reason enough to block on it, since we want to wait for the page
// to be outside any stack ownership interval.
DEBUG_ASSERT(stack_owner.owned_wait_queue_->owner() != Thread::Current::Get());
// This is a brief wait that's guaranteed not to get stuck (short of bugs elsewhere), with
// priority inheritance propagated to the owning thread. So no need for a deadline or
// interruptible.
zx_status_t block_status =
stack_owner.owned_wait_queue_->Block(Deadline::infinite(), Interruptible::No);
// For this wait queue, no other status is possible since no other status is ever passed to
// OwnedWaitQueue::WakeAll() for this wait queue and Block() doesn't have any other sources of
// failures assuming no bugs here.
DEBUG_ASSERT(block_status == ZX_OK);
void StackOwnedLoanedPagesInterval::WakeWaitersAndClearOwner(Thread* current_thread) {
DEBUG_ASSERT(current_thread == Thread::Current::Get());
auto hook = [](Thread* woken, void* ctx) -> OwnedWaitQueue::Hook::Action {
return OwnedWaitQueue::Hook::Action::SelectAndKeepGoing;
Guard<MonitoredSpinLock, IrqSave> thread_lock_guard{ThreadLock::Get(), SOURCE_TAG};
DEBUG_ASSERT(owned_wait_queue_->owner() == current_thread);
owned_wait_queue_->WakeThreads(ktl::numeric_limits<uint32_t>::max(), {hook, nullptr});