blob: 9b838919903637190b0b02c7b95cc37a8f3aa979 [file] [log] [blame]
// Copyright 2023 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <lib/kconcurrent/chainlock_transaction.h>
#include <kernel/thread.h>
namespace kconcurrent {
void ChainLockTransaction::PreemptDisable() {
Thread::Current::preemption_state().PreemptDisable();
}
void ChainLockTransaction::PreemptReenable() {
Thread::Current::preemption_state().PreemptReenable();
}
void ChainLockTransaction::EagerReschedDisable() {
Thread::Current::preemption_state().EagerReschedDisable();
}
void ChainLockTransaction::EagerReschedReenable() {
Thread::Current::preemption_state().EagerReschedReenable();
}
RescheduleContext SchedulerUtils::PrepareForReschedule() TA_REQ(chainlock_transaction_token) {
DEBUG_ASSERT(arch_ints_disabled());
Thread* const current_thread = Thread::Current::Get();
current_thread->get_lock().AssertHeld();
ChainLockTransaction* const orig_clt = ChainLockTransaction::Active();
orig_clt->debug_.AssertNumLocksHeld(1);
const ChainLock::Token orig_token{orig_clt->active_token_};
const ChainLock::Token sched_token{ChainLock::CreateSchedToken()};
ChainLock::AssertTokenIsNotSchedToken(orig_token);
ChainLock::ReplaceToken(orig_clt->active_token_, sched_token);
current_thread->get_lock().ReplaceLockToken(sched_token);
return RescheduleContext{orig_clt, orig_token};
}
void SchedulerUtils::RestoreRescheduleContext(const RescheduleContext& ctx) {
// Called after a reschedule operation where the current thread kept running
// (no context switch). Things which should be true right now.
//
// 1) There should be an active transaction, and the token being used by the
// active transaction should be a special "scheduler token"
// 2) There should be exactly one lock held.
// 3) The one lock held should be current thread's lock.
// 4) The active transaction should not have changed.
// 5) The token we are restoring has to be a "non-scheduler" token.
//
ChainLockTransaction* const active_clt = ChainLockTransaction::Active();
Thread* const current_thread = Thread::Current::Get();
ChainLock::AssertTokenIsSchedToken(active_clt->active_token_); // #1
active_clt->debug_.AssertNumLocksHeld(1); // #2
current_thread->get_lock().AssertHeld(); // #3
DEBUG_ASSERT(active_clt == ctx.orig_transaction); // #4
ChainLock::AssertTokenIsNotSchedToken(ctx.orig_token); // #5
// Go ahead and restore transaction, as well as the token in both the
// transaction and the current thread's lock.
ChainLock::ReplaceToken(active_clt->active_token_, ctx.orig_token);
current_thread->get_lock().ReplaceLockToken(ctx.orig_token);
}
void SchedulerUtils::PostContextSwitchLockHandoff(const RescheduleContext& ctx,
Thread* previous_thread) {
// Called after a reschedule operation where a new thread was selected to run.
// Things which should be true right now.
//
// 1) There should be an active transaction, and the token being used by the
// active transaction should be a special "scheduler token"
// 2) There should be exactly two locks held.
// 3) One of the locks held should be current thread's lock.
// 4) The other lock held should be the previous_thread's lock (should be
// guaranteed by the TA_REL static annotation on this method.
// 5) The active transaction should have changed to the new current thread's
// active transaction.
// 6) The token we are restoring has to be a "non-scheduler" token.
//
[[maybe_unused]] ChainLockTransaction* const active_clt = ChainLockTransaction::Active();
Thread* const current_thread = Thread::Current::Get();
ChainLock::AssertTokenIsSchedToken(active_clt->active_token_); // #1
active_clt->debug_.AssertNumLocksHeld(2); // #2
current_thread->get_lock().AssertHeld(); // #3
DEBUG_ASSERT((active_clt != ctx.orig_transaction) && // #5
(ctx.orig_transaction != nullptr)); // #5
ChainLock::AssertTokenIsNotSchedToken(ctx.orig_token); // #6
// Restore the original lock token.
ChainLock::ReplaceToken(ctx.orig_transaction->active_token_, ctx.orig_token);
current_thread->get_lock().ReplaceLockToken(ctx.orig_token);
previous_thread->get_lock().ReplaceLockToken(ctx.orig_token);
// Swap out the active transaction and drop the previous thread's lock.
arch_get_curr_percpu()->active_cl_transaction = ctx.orig_transaction;
previous_thread->get_lock().Release();
}
void SchedulerUtils::TrampolineLockHandoff() {
ChainLockTransaction::AssertActive();
// Assert that the current thread's lock was "acquired" here. It was
// actually acquired during the context switch from
// Scheduler::RescheduleCommon, but we now need to release it before letting
// the new thread run. The currently registered CLT should be using the
// scheduler token that was used to acquire the new thread's lock during
// reschedule common.
Thread* const current_thread = Thread::Current::Get();
current_thread->get_lock().AssertAcquired();
// Construct the fake chainlock transaction we will be restoring. It should:
//
// 1) Not attempt to register itself with the per-cpu data structure. There
// should already be a registered CLT, and attempting to instantiate a new
// one using the typical constructor would trigger an ASSERT. This should
// look like a transaction which had been sitting on a stack, but swapped
// out using PrepareForReschedule at some point in the past.
// 2) Report that there are currently two locks held (because there are).
// We have the previously running thread's lock, and the current (new)
// thread's lock held (obtained by Scheduler::RescheduleCommon).
// 3) Report that the transaction has been finalized, as it would have been
// had we come in via RescheduleCommon.
//
ChainLockTransactionNoIrqSave clt{internal::TrampolineTransactionTag{}};
// Now perform our lock handoff. This will release our previous thread's
// lock, and restore the transaction we just instantiated as the active
// transaction, and replace the current thread's lock's token with the new
// non-scheduler token.
Scheduler::LockHandoffInternal(RescheduleContext{&clt, clt.active_token_}, current_thread);
// Finally, we just need to release the current thread's lock and we are
// finished.
current_thread->get_lock().Release();
}
} // namespace kconcurrent