| // Copyright 2021 The Fuchsia Authors | |
| // | |
| // Use of this source code is governed by a MIT-style | |
| // license that can be found in the LICENSE file or at | |
| // https://opensource.org/licenses/MIT | |
| #include <lib/zircon-internal/macros.h> | |
| #include <kernel/thread.h> | |
| #include <ktl/limits.h> | |
| #include <object/thread_dispatcher.h> | |
| #include <vm/page.h> | |
| #include <vm/stack_owned_loaned_pages_interval.h> | |
| #include <ktl/enforce.h> | |
| void StackOwnedLoanedPagesInterval::PrepareForWaiter() { | |
| canary_.Assert(); | |
| // For now we don't need a CAS loop in here because thread_lock is held by callers of | |
| // PrepareForWaiter() and PrepareForWaiter() is the only mutator of is_ready_for_waiter_. Even if | |
| // we did have a CAS loop, the caller would still need to guarantee somehow that the interval | |
| // won't get deleted out from under this call. Currently that's guaranteed by the current | |
| // thread_lock hold interval being the same interval that set kObjectOrStackOwnerHasWaiter. | |
| // | |
| // Because all setters of is_ready_for_waiter_ hold thread_lock, we could use memory_order_relaxed | |
| // here, but for now we're using acquire for all loads of is_ready_for_waiter_. | |
| if (is_ready_for_waiter_.load(ktl::memory_order_acquire)) { | |
| return; | |
| } | |
| // Thanks to thread_lock, we know that the current thread is the only thread setting | |
| // is_ready_for_waiter_, so we can just set it using a store(). We also need to prepare the | |
| // owned_wait_queue_ to have a waiter that can transmit its priority via priority inheritance to | |
| // the stack-owning thread. | |
| DEBUG_ASSERT(owning_thread_); | |
| DEBUG_ASSERT(Thread::Current::Get() != owning_thread_); | |
| owned_wait_queue_.emplace(); | |
| owned_wait_queue_->AssignOwner(owning_thread_); | |
| // The memory_order_release isn't really needed here thanks to release of thread_lock by this | |
| // thread shortly and acquire of thread_lock by any thread removing the | |
| // StackOwnedLoanedPagesInterval from the page (before deleting the interval), but for now we're | |
| // using release for all stores to is_ready_for_waiter_. | |
| is_ready_for_waiter_.store(true, ktl::memory_order_release); | |
| } | |
| // static | |
| StackOwnedLoanedPagesInterval& StackOwnedLoanedPagesInterval::current() { | |
| Thread* current_thread = Thread::Current::Get(); | |
| // The caller should only call current() when the caller knows there must be a current interval, | |
| // and just needs to know which interval is the outer-most on this thread's stack. | |
| // | |
| // Stack ownership of a loaned page requires having a StackOwnedLoanedPagesInterval on the | |
| // caller's stack. | |
| DEBUG_ASSERT_MSG(current_thread->stack_owned_loaned_pages_interval(), | |
| "StackOwnedLoanedPagesInterval missing"); | |
| return *current_thread->stack_owned_loaned_pages_interval_; | |
| } | |
| // static | |
| StackOwnedLoanedPagesInterval* StackOwnedLoanedPagesInterval::maybe_current() { | |
| Thread* current_thread = Thread::Current::Get(); | |
| return current_thread->stack_owned_loaned_pages_interval_; | |
| } | |
| // static | |
| void StackOwnedLoanedPagesInterval::WaitUntilContiguousPageNotStackOwned(vm_page_t* page) { | |
| // Due to not holding the PmmNode lock, we can't check loaned directly, and it may have been unset | |
| // recently in any case, but in that case we'll notice via !is_stack_owned() instead. | |
| // | |
| // Need to take thread_lock at this point, because avoiding deletion of the OwnedWaitQueue | |
| // requires holding the thread_lock while applying kObjectOrStackOwnerHasWaiter to the page, to | |
| // prevent the StackOwnedLoanedPagesInterval thread from removing the stack_owner from the page | |
| // and deleting the OwnedWaitQueue. We also need the thread_lock to block on the | |
| // OwnedWaitQueue. | |
| // | |
| // Before we acquire the thread_lock we do a check whether a stack_owner is still set. This is | |
| // just to avoid acquiring the thread lock on the off chance that the stack ownership interval | |
| // is already over. This isn't particularly likely to be the case, and we'd be fine without | |
| // this check. But since we're about to take the thread_lock let's avoid an unnecessary acquire | |
| // if we can. | |
| if (!page->object.is_stack_owned()) { | |
| // StackOwnedLoanedPagesInterval is already removed from the page, so no need to | |
| // acquire the thread_lock. Go around and observe the new page state. | |
| return; | |
| } | |
| // Acquire thread_lock since that's required to ensure ~StackOwnedLoanedPagesInterval doesn't | |
| // miss that this thread is blocked waiting, along with kObjectOrStackOwnerHasWaiter. | |
| AnnotatedAutoPreemptDisabler aapd; | |
| Guard<MonitoredSpinLock, IrqSave> thread_lock_guard{ThreadLock::Get(), SOURCE_TAG}; | |
| // Holding the thread_lock doesn't guarantee that the stack_owner won't be cleared, but holding | |
| // thread_lock and successfully ensuring that kObjectOrStackOwnerHasWaiter is set does guarantee | |
| // the stack_owner won't be cleared. | |
| auto maybe_try_set_has_waiter_result = page->object.try_set_has_waiter(); | |
| if (!maybe_try_set_has_waiter_result) { | |
| // stack_owner was cleared; no need to wait. | |
| // | |
| // ~thread_lock_guard | |
| return; | |
| } | |
| auto& try_set_has_waiter_result = maybe_try_set_has_waiter_result.value(); | |
| auto& stack_owner = *try_set_has_waiter_result.stack_owner; | |
| // By doing PrepareForWaiter() only when necessary, we avoid pressure on the thread_lock in the | |
| // case where there's no page reclaiming thread needing to wait / transmit priority. | |
| if (try_set_has_waiter_result.first_setter) { | |
| stack_owner.PrepareForWaiter(); | |
| } | |
| // PrepareForWaiter() was called previously, either by this thread or a different thread. | |
| DEBUG_ASSERT(stack_owner.is_ready_for_waiter_.load(ktl::memory_order_acquire)); | |
| // At this point we know that the stack_owner won't be changed on the page while we hold | |
| // thread_lock, which means the OwnedWaitQueue can't be deleted yet either, since deletion is | |
| // after uninstalling from the page. So now we just need to block on the OwnedWaitQueue, which | |
| // requires holding the thread_lock during the call anyway. We don't really care if this | |
| // OwnedWaitQueue is relevant to moving from cow to cow, or cow to FREE, or during ALLOC state. | |
| // In all those possible cases, we want to block on the OwnedWaitQueue. The fact that the | |
| // OwnedWaitQueue is there is reason enough to block on it, since we want to wait for the page | |
| // to be outside any stack ownership interval. | |
| DEBUG_ASSERT(stack_owner.owned_wait_queue_->owner()); | |
| DEBUG_ASSERT(stack_owner.owned_wait_queue_->owner() != Thread::Current::Get()); | |
| // This is a brief wait that's guaranteed not to get stuck (short of bugs elsewhere), with | |
| // priority inheritance propagated to the owning thread. So no need for a deadline or | |
| // interruptible. | |
| zx_status_t block_status = | |
| stack_owner.owned_wait_queue_->Block(Deadline::infinite(), Interruptible::No); | |
| // For this wait queue, no other status is possible since no other status is ever passed to | |
| // OwnedWaitQueue::WakeAll() for this wait queue and Block() doesn't have any other sources of | |
| // failures assuming no bugs here. | |
| DEBUG_ASSERT(block_status == ZX_OK); | |
| } | |
| void StackOwnedLoanedPagesInterval::WakeWaitersAndClearOwner(Thread* current_thread) { | |
| DEBUG_ASSERT(current_thread == Thread::Current::Get()); | |
| auto hook = [](Thread* woken, void* ctx) -> OwnedWaitQueue::Hook::Action { | |
| return OwnedWaitQueue::Hook::Action::SelectAndKeepGoing; | |
| }; | |
| AnnotatedAutoPreemptDisabler aapd; | |
| Guard<MonitoredSpinLock, IrqSave> thread_lock_guard{ThreadLock::Get(), SOURCE_TAG}; | |
| DEBUG_ASSERT(owned_wait_queue_->owner() == current_thread); | |
| owned_wait_queue_->WakeThreads(ktl::numeric_limits<uint32_t>::max(), {hook, nullptr}); | |
| owned_wait_queue_->AssignOwner(nullptr); | |
| } |