blob: 0d03aacc7f0f1086552b7452518e1aa41d2f4492 [file]
// Copyright 2018 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT
#include <inttypes.h>
#include <lib/fit/defer.h>
#include <lib/kconcurrent/chainlock_transaction.h>
#include <lib/unittest/unittest.h>
#include <platform.h>
#include <zircon/errors.h>
#include <zircon/types.h>
#include <arch/interrupt.h>
#include <dev/power.h>
#include <fbl/alloc_checker.h>
#include <kernel/cpu.h>
#include <kernel/dpc.h>
#include <kernel/event.h>
#include <kernel/mp.h>
#include <kernel/mutex.h>
#include <kernel/scheduler.h>
#include <kernel/thread.h>
#include <ktl/atomic.h>
#include <ktl/iterator.h>
#include <ktl/unique_ptr.h>
#include "tests.h"
#include <ktl/enforce.h>
static int resume_cpu_test_thread(void* arg) {
*reinterpret_cast<cpu_num_t*>(arg) = arch_curr_cpu_num();
return 0;
}
// "Unplug" online secondary (non-BOOT) cores
static zx_status_t unplug_all_cores() {
cpu_mask_t cpumask = mp_get_online_mask() & ~cpu_num_to_mask(BOOT_CPU_ID);
return mp_unplug_cpu_mask(cpumask, ZX_TIME_INFINITE);
}
static zx_status_t hotplug_core(cpu_num_t i) {
cpu_mask_t cpumask = cpu_num_to_mask(i);
return mp_hotplug_cpu_mask(cpumask);
}
static unsigned get_num_cpus_online() {
unsigned count = 0;
cpu_mask_t online = mp_get_online_mask();
while (online) {
online >>= 1;
++count;
}
return count;
}
static zx_status_t wait_for_cpu_offline(cpu_num_t i) {
zx_instant_mono_t print_time = zx_time_add_duration(current_mono_time(), ZX_SEC(5));
while (true) {
zx::result<power_cpu_state> res = platform_get_cpu_state(i);
if (res.is_error()) {
if (res.error_value() == ZX_ERR_NOT_SUPPORTED) {
// x86 does not implement platform_get_cpu_state, so return OK if the call returns
// ZX_ERR_NOT_SUPPORTED.
return ZX_OK;
}
return res.error_value();
} else if (res.value() == power_cpu_state::OFF || res.value() == power_cpu_state::STOPPED) {
return ZX_OK;
}
if (current_mono_time() > print_time) {
print_time = zx_time_add_duration(current_mono_time(), ZX_SEC(5));
printf("Still waiting for CPU %u to go offline, waiting 5 more seconds\n", i);
}
Thread::Current::SleepRelative(ZX_USEC(200));
}
}
static void wait_for_cpu_active(cpu_num_t i) {
zx_instant_mono_t print_time = zx_time_add_duration(current_mono_time(), ZX_SEC(5));
while (true) {
if (Scheduler::PeekIsActive(i)) {
return;
}
if (current_mono_time() > print_time) {
print_time = zx_time_add_duration(current_mono_time(), ZX_SEC(5));
printf("Still waiting for CPU %u to become active, waiting 5 more seconds\n", i);
}
Thread::Current::SleepRelative(ZX_USEC(200));
}
}
// Wait until |i|'s DpcRunner is running, by queuing a Dpc on each of |i|'s Dpc queues and waiting
// for them to execute.
//
// A hotplugged core becomes active (see wait_for_cpu_active) before it has created its Dpc threads,
// and a core must not be unplugged before then (mp_unplug_cpu_mask shuts down its Dpc threads).
// The general queue's thread is created (and may start executing Dpcs) before the low latency
// queue's thread, so both queues must be checked.
static zx_status_t wait_for_cpu_dpc_runner(cpu_num_t i) {
Thread* t = Thread::Create(
"wait-for-dpc-runner",
[](void*) -> int {
constexpr DpcRunner::QueueType kQueueTypes[] = {DpcRunner::QueueType::General,
DpcRunner::QueueType::LowLatency};
for (const DpcRunner::QueueType type : kQueueTypes) {
Event event;
Dpc dpc([](Dpc* d) { d->arg<Event>()->Signal(); }, &event);
const zx_status_t status = DpcRunner::Enqueue(dpc, type);
if (status != ZX_OK) {
return status;
}
// Wait non-interruptibly, since |dpc| and |event| live on this thread's stack.
event.Wait();
}
return ZX_OK;
},
nullptr, DEFAULT_PRIORITY);
if (t == nullptr) {
return ZX_ERR_NO_MEMORY;
}
t->SetCpuAffinity(cpu_num_to_mask(i));
t->Resume();
int retcode;
const zx_status_t status = t->Join(&retcode, ZX_TIME_INFINITE);
return status != ZX_OK ? status : retcode;
}
// Unplug all cores (except for Boot core), then hotplug
// the cores one by one and make sure that we can schedule
// tasks on that core.
[[maybe_unused]] static bool mp_hotplug_test() {
BEGIN_TEST;
// TODO(https://fxbug.dev/42086046): Re-enable test on RISC-V.
#if defined(__riscv)
printf("skipping test mp_hotplug, hotplug only suported on x64 and arm64\n");
END_TEST;
#endif
uint num_cores = get_num_cpus_online();
if (num_cores < 2) {
printf("skipping test mp_hotplug, not enough online cpus\n");
END_TEST;
}
Thread::Current::MigrateToCpu(BOOT_CPU_ID);
// "Unplug" online secondary (non-BOOT) cores
ASSERT_OK(unplug_all_cores(), "unplugging all cores failed");
for (cpu_num_t i = 0; i < num_cores; i++) {
if (i == BOOT_CPU_ID) {
continue;
}
// Wait until this core is fully offline.
ASSERT_OK(wait_for_cpu_offline(i), "waiting for core to go offline failed");
// Hotplug this core.
ASSERT_OK(hotplug_core(i), "hotplugging core failed");
// Wait until the core is active.
wait_for_cpu_active(i);
// Unplug shuts down DPC threads, which start after the core becomes active.
ASSERT_OK(wait_for_cpu_dpc_runner(i), "waiting for core's dpc runner failed");
// Create a thread, affine it to the core just hotplugged
// and make sure the thread does get scheduled there.
cpu_num_t running_core{INVALID_CPU};
Thread* nt = Thread::Create("resume-test-thread", resume_cpu_test_thread, &running_core,
DEFAULT_PRIORITY);
ASSERT_NE(nullptr, nt, "Thread create failed");
nt->SetCpuAffinity(cpu_num_to_mask(i));
nt->SetMigrateFn([](auto...) {});
nt->Resume();
ASSERT_OK(nt->Join(nullptr, ZX_TIME_INFINITE), "thread join failed");
ASSERT_EQ(i, running_core, "Thread not running on hotplugged core");
}
END_TEST;
}
// Repeatedly unplug and hotplug a core while another thread repeatedly calls mp_sync_exec targeting
// it, and make sure that every mp_sync_exec call returns.
//
// A task may be queued for the core after it has disabled interrupts to go offline, in which case
// the task is drained when the core is marked offline rather than executed. Otherwise, the task is
// executed by the core before it goes offline. Either way, mp_sync_exec must not wait
// indefinitely for the core, and must not return while its task is still queued (which
// mp_sync_exec checks with a DEBUG_ASSERT).
[[maybe_unused]] static bool mp_hotplug_sync_exec_test() {
BEGIN_TEST;
// TODO(https://fxbug.dev/42086046): Re-enable test on RISC-V.
#if defined(__riscv)
printf("skipping test mp_hotplug_sync_exec, hotplug only supported on x64 and arm64\n");
END_TEST;
#endif
const cpu_mask_t online_secondaries = mp_get_online_mask() & ~cpu_num_to_mask(BOOT_CPU_ID);
if (online_secondaries == 0) {
printf("skipping test mp_hotplug_sync_exec, not enough online cpus\n");
END_TEST;
}
const cpu_num_t target_cpu = highest_cpu_set(online_secondaries);
auto restore_affinity = fit::defer(
[previous_affinity = Thread::Current::SetCpuAffinity(cpu_num_to_mask(BOOT_CPU_ID))] {
Thread::Current::SetCpuAffinity(previous_affinity);
});
// If an assertion below fails while |target_cpu| is unplugged, hotplug it again so that
// subsequent tests run with the same set of cores.
auto restore_target_cpu = fit::defer([target_cpu] {
if (!mp_is_cpu_online(target_cpu)) {
const zx_status_t status = hotplug_core(target_cpu);
ASSERT_MSG(status == ZX_OK, "hotplugging core %u failed: %d", target_cpu, status);
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
const zx_status_t dpc_status = wait_for_cpu_dpc_runner(target_cpu);
ASSERT_MSG(dpc_status == ZX_OK, "waiting for core %u's dpc runner failed: %d", target_cpu,
dpc_status);
}
});
struct State {
const cpu_num_t target_cpu;
ktl::atomic<bool> stop{false};
ktl::atomic<uint64_t> calls{0};
} state{.target_cpu = target_cpu};
// The sender may run on any online core other than the one being unplugged and hotplugged.
Thread* sender = Thread::Create(
"mp-hotplug-sync-exec-sender",
[](void* arg) -> int {
auto* s = static_cast<State*>(arg);
while (!s->stop.load(ktl::memory_order_relaxed)) {
// Alternate between targeting just the core being unplugged and hotplugged, and all
// other cores.
const uint64_t calls = s->calls.load(ktl::memory_order_relaxed);
if (calls % 2 == 0) {
mp_sync_exec(
mp_ipi_target::MASK, cpu_num_to_mask(s->target_cpu), [](void*) {}, nullptr);
} else {
// Targeting all but the local core requires interrupts to be disabled.
InterruptDisableGuard irqd;
mp_sync_exec(mp_ipi_target::ALL_BUT_LOCAL, 0, [](void*) {}, nullptr);
}
s->calls.store(calls + 1, ktl::memory_order_relaxed);
// While the target core is offline, every other call returns immediately, so yield to
// avoid monopolizing this core.
Thread::Current::Yield();
}
return 0;
},
&state, DEFAULT_PRIORITY);
ASSERT_NONNULL(sender, "Thread create failed");
sender->SetCpuAffinity(mp_get_online_mask() & ~cpu_num_to_mask(target_cpu));
sender->Resume();
// Stop the sender, and wait for it to finish, before |state| goes out of scope, even if an
// assertion below fails. If any mp_sync_exec call fails to return, the sender never observes
// |stop|, and this test hangs.
auto stop_sender = fit::defer([&state, sender] {
state.stop.store(true, ktl::memory_order_relaxed);
sender->Join(nullptr, ZX_TIME_INFINITE);
});
constexpr int kIterations = 20;
for (int i = 0; i < kIterations; ++i) {
ASSERT_OK(mp_unplug_cpu_mask(cpu_num_to_mask(target_cpu), ZX_TIME_INFINITE),
"unplugging core failed");
ASSERT_OK(wait_for_cpu_offline(target_cpu), "waiting for core to go offline failed");
ASSERT_OK(hotplug_core(target_cpu), "hotplugging core failed");
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
ASSERT_OK(wait_for_cpu_dpc_runner(target_cpu), "waiting for core's dpc runner failed");
}
stop_sender.call();
EXPECT_GT(state.calls.load(ktl::memory_order_relaxed), 0u);
END_TEST;
}
static void wait_for_thread_state(const Thread& thread, thread_state expected_state) {
zx_instant_mono_t print_time = zx_time_add_duration(current_mono_time(), ZX_SEC(5));
while (true) {
{
SingleChainLockGuard guard{IrqSaveOption, thread.get_lock(),
CLT_TAG("mp_hotplug_wait_for_thread_state")};
if (thread.state() == expected_state) {
return;
}
}
if (current_mono_time() > print_time) {
print_time = zx_time_add_duration(current_mono_time(), ZX_SEC(5));
printf("Still waiting for thread to enter state %d, waiting 5 more seconds\n",
static_cast<int>(expected_state));
}
Thread::Current::SleepRelative(ZX_USEC(100));
}
}
// Regression test for https://fxbug.dev/578020457:
// Verify that threads hard-pinned to a single secondary CPU can be woken (both from an
// OwnedWaitQueue via Mutex release and from a WaitQueue via Event::Signal) while their pinned CPU
// is offline or remain THREAD_READY across unplug (running MigrateStage::Save before power-off),
// stay queued on that inactive CPU's scheduler without panicking or dereferencing INVALID_CPU, and
// execute on that CPU once it is hotplugged back online.
[[maybe_unused]] static bool mp_hotplug_unblock_pinned_while_offline_test() {
BEGIN_TEST;
// TODO(https://fxbug.dev/42086046): Re-enable test on RISC-V.
#if defined(__riscv)
printf(
"skipping test mp_hotplug_unblock_pinned_while_offline, hotplug only supported on x64 and "
"arm64\n");
END_TEST;
#endif
const cpu_mask_t online_secondaries = mp_get_online_mask() & ~cpu_num_to_mask(BOOT_CPU_ID);
if (online_secondaries == 0) {
printf("skipping test mp_hotplug_unblock_pinned_while_offline, not enough online cpus\n");
END_TEST;
}
const cpu_num_t target_cpu = highest_cpu_set(online_secondaries);
auto restore_affinity = fit::defer(
[previous_affinity = Thread::Current::SetCpuAffinity(cpu_num_to_mask(BOOT_CPU_ID))] {
Thread::Current::SetCpuAffinity(previous_affinity);
});
// Relaxed atomic loads/stores on |State| are synchronized by Event::Wait(),
// wait_for_thread_state() / wait_for_cpu_offline() lock acquisitions, and Thread::Join().
// Heap-allocate |State| to keep this function's kernel stack frame small.
struct State {
DECLARE_MUTEX(State) lock;
Event holder_locked_event;
Event release_lock_event;
Event holder_released_event;
Event wake_event;
Event ready_pinned_started_event;
ktl::atomic<bool> observe_post_hotplug_cpu{false};
ktl::atomic<bool> stop_ready_pinned{false};
ktl::atomic<cpu_num_t> mutex_waiter_before_cpu{INVALID_CPU};
ktl::atomic<cpu_num_t> mutex_waiter_after_cpu{INVALID_CPU};
ktl::atomic<cpu_num_t> event_waiter_before_cpu{INVALID_CPU};
ktl::atomic<cpu_num_t> event_waiter_after_cpu{INVALID_CPU};
ktl::atomic<cpu_num_t> ready_pinned_before_cpu{INVALID_CPU};
ktl::atomic<cpu_num_t> ready_pinned_after_cpu{INVALID_CPU};
ktl::atomic<uint32_t> migrate_save_calls{0};
ktl::atomic<uint32_t> migrate_restore_calls{0};
ktl::atomic<uint32_t> ready_migrate_save_calls{0};
ktl::atomic<uint32_t> ready_migrate_restore_calls{0};
};
fbl::AllocChecker ac;
auto state = ktl::make_unique<State>(&ac);
ASSERT_TRUE(ac.check());
Thread* mutex_holder = nullptr;
Thread* mutex_waiter = nullptr;
Thread* event_waiter = nullptr;
Thread* ready_pinned = nullptr;
auto cleanup_threads =
fit::defer([state = state.get(), &mutex_holder, &mutex_waiter, &event_waiter, &ready_pinned] {
state->stop_ready_pinned.store(true, ktl::memory_order_relaxed);
state->release_lock_event.Signal();
state->wake_event.Signal();
if (mutex_holder != nullptr) {
mutex_holder->Join(nullptr, ZX_TIME_INFINITE);
}
if (mutex_waiter != nullptr) {
mutex_waiter->Join(nullptr, ZX_TIME_INFINITE);
}
if (event_waiter != nullptr) {
event_waiter->Join(nullptr, ZX_TIME_INFINITE);
}
if (ready_pinned != nullptr) {
ready_pinned->Join(nullptr, ZX_TIME_INFINITE);
}
});
// Declared after |cleanup_threads| so |target_cpu| is brought back online first during unwinding,
// allowing any threads pinned to |target_cpu| to run and exit when |cleanup_threads| joins them.
auto restore_target_cpu = fit::defer([target_cpu] {
if (!mp_is_cpu_online(target_cpu)) {
const zx_status_t status = hotplug_core(target_cpu);
ASSERT_MSG(status == ZX_OK, "hotplugging core %u failed: %d", target_cpu, status);
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
const zx_status_t dpc_status = wait_for_cpu_dpc_runner(target_cpu);
ASSERT_MSG(dpc_status == ZX_OK, "waiting for core %u's dpc runner failed: %d", target_cpu,
dpc_status);
}
});
mutex_holder = Thread::Create(
"pinned-offline-mutex-holder",
[](void* arg) -> int {
auto* s = static_cast<State*>(arg);
{
Guard<Mutex> guard{&s->lock};
s->holder_locked_event.Signal();
s->release_lock_event.Wait();
}
s->holder_released_event.Signal();
return 0;
},
state.get(), DEFAULT_PRIORITY);
ASSERT_NONNULL(mutex_holder, "mutex_holder create failed");
mutex_holder->SetCpuAffinity(cpu_num_to_mask(BOOT_CPU_ID));
mutex_holder->Resume();
ASSERT_OK(state->holder_locked_event.Wait(), "waiting for holder_locked_event failed");
mutex_waiter = Thread::Create(
"pinned-offline-mutex-waiter",
[](void* arg) -> int {
auto* s = static_cast<State*>(arg);
s->mutex_waiter_before_cpu.store(arch_curr_cpu_num(), ktl::memory_order_relaxed);
Guard<Mutex> guard{&s->lock};
s->mutex_waiter_after_cpu.store(arch_curr_cpu_num(), ktl::memory_order_relaxed);
return 0;
},
state.get(), DEFAULT_PRIORITY);
ASSERT_NONNULL(mutex_waiter, "mutex_waiter create failed");
mutex_waiter->SetCpuAffinity(cpu_num_to_mask(target_cpu));
mutex_waiter->Resume();
event_waiter = Thread::Create(
"pinned-offline-event-waiter",
[](void* arg) -> int {
auto* s = static_cast<State*>(arg);
s->event_waiter_before_cpu.store(arch_curr_cpu_num(), ktl::memory_order_relaxed);
s->wake_event.Wait();
s->event_waiter_after_cpu.store(arch_curr_cpu_num(), ktl::memory_order_relaxed);
return 0;
},
state.get(), DEFAULT_PRIORITY);
ASSERT_NONNULL(event_waiter, "event_waiter create failed");
event_waiter->SetCpuAffinity(cpu_num_to_mask(target_cpu));
event_waiter->SetMigrateFn([state = state.get()](Thread*, Thread::MigrateStage stage) {
if (stage == Thread::MigrateStage::Save) {
state->migrate_save_calls.fetch_add(1, ktl::memory_order_relaxed);
} else if (stage == Thread::MigrateStage::Restore) {
state->migrate_restore_calls.fetch_add(1, ktl::memory_order_relaxed);
}
});
event_waiter->Resume();
ready_pinned = Thread::Create(
"pinned-offline-ready-worker",
[](void* arg) -> int {
auto* s = static_cast<State*>(arg);
s->ready_pinned_before_cpu.store(arch_curr_cpu_num(), ktl::memory_order_relaxed);
s->ready_pinned_started_event.Signal();
while (!s->stop_ready_pinned.load(ktl::memory_order_relaxed)) {
if (s->observe_post_hotplug_cpu.load(ktl::memory_order_relaxed)) {
s->ready_pinned_after_cpu.store(arch_curr_cpu_num(), ktl::memory_order_relaxed);
break;
}
Thread::Current::Yield();
}
return 0;
},
state.get(), LOW_PRIORITY);
ASSERT_NONNULL(ready_pinned, "ready_pinned create failed");
ready_pinned->SetCpuAffinity(cpu_num_to_mask(target_cpu));
ready_pinned->SetMigrateFn([state = state.get()](Thread*, Thread::MigrateStage stage) {
if (stage == Thread::MigrateStage::Save) {
state->ready_migrate_save_calls.fetch_add(1, ktl::memory_order_relaxed);
} else if (stage == Thread::MigrateStage::Restore) {
state->ready_migrate_restore_calls.fetch_add(1, ktl::memory_order_relaxed);
}
});
ready_pinned->Resume();
// Wait until both waiters have run on |target_cpu| and entered THREAD_BLOCKED, and
// |ready_pinned| has started executing on |target_cpu|.
wait_for_thread_state(*mutex_waiter, THREAD_BLOCKED);
wait_for_thread_state(*event_waiter, THREAD_BLOCKED);
ASSERT_OK(state->ready_pinned_started_event.Wait(),
"waiting for ready_pinned_started_event failed");
EXPECT_EQ(target_cpu, state->mutex_waiter_before_cpu.load(ktl::memory_order_relaxed));
EXPECT_EQ(target_cpu, state->event_waiter_before_cpu.load(ktl::memory_order_relaxed));
EXPECT_EQ(target_cpu, state->ready_pinned_before_cpu.load(ktl::memory_order_relaxed));
// Take |target_cpu| offline while the waiters are blocked and |ready_pinned| is runnable on
// |target_cpu| (preempted into THREAD_READY during unplug).
ASSERT_OK(mp_unplug_cpu_mask(cpu_num_to_mask(target_cpu), ZX_TIME_INFINITE),
"unplugging core failed");
ASSERT_OK(wait_for_cpu_offline(target_cpu), "waiting for core to go offline failed");
ASSERT_FALSE(Scheduler::PeekIsActive(target_cpu));
wait_for_thread_state(*ready_pinned, THREAD_READY);
EXPECT_EQ(1u, state->ready_migrate_save_calls.load(ktl::memory_order_relaxed));
EXPECT_EQ(0u, state->ready_migrate_restore_calls.load(ktl::memory_order_relaxed));
state->observe_post_hotplug_cpu.store(true, ktl::memory_order_relaxed);
// Unblock both pinned waiters while |target_cpu| is offline:
// 1. Releasing |state->lock| wakes |mutex_waiter| via OwnedWaitQueue::WakeThreadsLocked.
// 2. Signaling |state->wake_event| wakes |event_waiter| via WaitQueue::WakeOne.
state->release_lock_event.Signal();
ASSERT_OK(state->holder_released_event.Wait(), "waiting for holder_released_event failed");
state->wake_event.Signal();
// All three pinned threads should now be queued in THREAD_READY on inactive |target_cpu|, and
// must not have run on any other CPU while |target_cpu| is offline.
wait_for_thread_state(*mutex_waiter, THREAD_READY);
wait_for_thread_state(*event_waiter, THREAD_READY);
EXPECT_EQ(INVALID_CPU, state->mutex_waiter_after_cpu.load(ktl::memory_order_relaxed));
EXPECT_EQ(INVALID_CPU, state->event_waiter_after_cpu.load(ktl::memory_order_relaxed));
EXPECT_EQ(INVALID_CPU, state->ready_pinned_after_cpu.load(ktl::memory_order_relaxed));
// Hotplug |target_cpu| back online and verify that all queued pinned threads execute on it.
ASSERT_OK(hotplug_core(target_cpu), "hotplugging core failed");
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
ASSERT_OK(wait_for_cpu_dpc_runner(target_cpu), "waiting for core's dpc runner failed");
ASSERT_OK(mutex_holder->Join(nullptr, ZX_TIME_INFINITE), "joining mutex_holder failed");
mutex_holder = nullptr;
ASSERT_OK(mutex_waiter->Join(nullptr, ZX_TIME_INFINITE), "joining mutex_waiter failed");
mutex_waiter = nullptr;
ASSERT_OK(event_waiter->Join(nullptr, ZX_TIME_INFINITE), "joining event_waiter failed");
event_waiter = nullptr;
ASSERT_OK(ready_pinned->Join(nullptr, ZX_TIME_INFINITE), "joining ready_pinned failed");
ready_pinned = nullptr;
EXPECT_EQ(target_cpu, state->mutex_waiter_after_cpu.load(ktl::memory_order_relaxed));
EXPECT_EQ(target_cpu, state->event_waiter_after_cpu.load(ktl::memory_order_relaxed));
EXPECT_EQ(target_cpu, state->ready_pinned_after_cpu.load(ktl::memory_order_relaxed));
EXPECT_EQ(1u, state->migrate_save_calls.load(ktl::memory_order_relaxed));
EXPECT_EQ(1u, state->migrate_restore_calls.load(ktl::memory_order_relaxed));
EXPECT_EQ(1u, state->ready_migrate_save_calls.load(ktl::memory_order_relaxed));
EXPECT_EQ(1u, state->ready_migrate_restore_calls.load(ktl::memory_order_relaxed));
END_TEST;
}
// Regression test for https://fxbug.dev/578020457:
// Verify that unpinned threads with migration functions neither livelock nor spin-deadlock when
// woken (or unpinned via SetCpuAffinity) while their last_cpu is offline or deactivating:
// 1. When last_cpu is already offline (!is_active && !cpu_deactivating) and a migration function is
// installed after CallMigrateFnForCpu has already completed, waking a blocked thread or widening
// an offline-enqueued thread's affinity places it directly onto an active CPU without hanging or
// running MigrateStage::Save on the wrong CPU.
// 2. When last_cpu is deactivating in Scheduler::MigrateUnpinnedThreads, concurrent wakeups from
// other CPUs queue onto last_cpu's save_state_list_ without deadlocking against
// Thread::CallMigrateFnForCpu and preserve strict Save-on-last_cpu / Restore ordering.
[[maybe_unused]] static bool mp_hotplug_unblock_unpinned_migrate_while_offline_test() {
BEGIN_TEST;
// TODO(https://fxbug.dev/42086046): Re-enable test on RISC-V.
#if defined(__riscv)
printf(
"skipping test mp_hotplug_unblock_unpinned_migrate_while_offline, hotplug only supported on "
"x64 and arm64\n");
END_TEST;
#endif
const cpu_mask_t online_secondaries = mp_get_online_mask() & ~cpu_num_to_mask(BOOT_CPU_ID);
if (online_secondaries == 0) {
printf(
"skipping test mp_hotplug_unblock_unpinned_migrate_while_offline, not enough online "
"cpus\n");
END_TEST;
}
const cpu_num_t target_cpu = highest_cpu_set(online_secondaries);
auto restore_affinity = fit::defer(
[previous_affinity = Thread::Current::SetCpuAffinity(cpu_num_to_mask(BOOT_CPU_ID))] {
Thread::Current::SetCpuAffinity(previous_affinity);
});
// Relaxed atomic loads/stores on |State| are synchronized by Event::Wait(),
// wait_for_thread_state() / wait_for_cpu_offline() lock acquisitions, and Thread::Join();
// per-worker |save_pending| and |exited| fields are accessed exclusively under each worker
// thread's ChainLock inside CallMigrateFnLocked. Heap-allocate |State| to keep this function's
// kernel stack frame small.
struct State;
struct WorkerSlot {
Thread* thread{nullptr};
State* state{nullptr};
Event started_event;
AutounsignalEvent wake_event;
ktl::atomic<uint64_t> wakeups{0};
ktl::atomic<uint64_t> save_calls{0};
ktl::atomic<uint64_t> restore_calls{0};
ktl::atomic<bool> migrate_order_ok{true};
bool save_pending{false};
bool exited{false};
};
static constexpr size_t kNumWorkers = 4;
struct State {
Event offline_waiter_started_event;
Event offline_waiter_wake_event;
Event pinned_enqueued_started_event;
ktl::atomic<bool> unpinned_observe_cpu{false};
ktl::atomic<bool> stop_pinned_enqueued{false};
ktl::atomic<bool> stop_workers{false};
ktl::atomic<cpu_num_t> offline_waiter_before_cpu{INVALID_CPU};
ktl::atomic<cpu_num_t> offline_waiter_after_cpu{INVALID_CPU};
ktl::atomic<cpu_num_t> pinned_enqueued_before_cpu{INVALID_CPU};
ktl::atomic<cpu_num_t> pinned_enqueued_after_cpu{INVALID_CPU};
ktl::atomic<uint32_t> offline_waiter_save_calls{0};
ktl::atomic<uint32_t> offline_waiter_restore_calls{0};
ktl::atomic<uint32_t> offline_waiter_exiting_calls{0};
ktl::atomic<bool> offline_waiter_migrate_ok{true};
ktl::atomic<uint32_t> pinned_enqueued_save_calls{0};
ktl::atomic<uint32_t> pinned_enqueued_restore_calls{0};
ktl::atomic<uint32_t> pinned_enqueued_exiting_calls{0};
ktl::atomic<bool> pinned_enqueued_migrate_ok{true};
WorkerSlot workers[kNumWorkers];
};
fbl::AllocChecker ac;
auto state = ktl::make_unique<State>(&ac);
ASSERT_TRUE(ac.check());
Thread* offline_waiter = nullptr;
Thread* pinned_enqueued = nullptr;
Thread* waker = nullptr;
auto cleanup_threads =
fit::defer([state = state.get(), &offline_waiter, &pinned_enqueued, &waker] {
state->stop_pinned_enqueued.store(true, ktl::memory_order_relaxed);
state->stop_workers.store(true, ktl::memory_order_relaxed);
state->offline_waiter_wake_event.Signal();
for (WorkerSlot& slot : state->workers) {
slot.wake_event.Signal();
}
if (waker != nullptr) {
waker->Join(nullptr, ZX_TIME_INFINITE);
waker = nullptr;
}
for (WorkerSlot& slot : state->workers) {
if (slot.thread != nullptr) {
slot.wake_event.Signal();
slot.thread->Join(nullptr, ZX_TIME_INFINITE);
slot.thread = nullptr;
}
}
if (offline_waiter != nullptr) {
offline_waiter->Join(nullptr, ZX_TIME_INFINITE);
offline_waiter = nullptr;
}
if (pinned_enqueued != nullptr) {
pinned_enqueued->Join(nullptr, ZX_TIME_INFINITE);
pinned_enqueued = nullptr;
}
});
// Declared after |cleanup_threads| so |target_cpu| is brought back online first during unwinding.
auto restore_target_cpu = fit::defer([target_cpu] {
if (!mp_is_cpu_online(target_cpu)) {
const zx_status_t status = hotplug_core(target_cpu);
ASSERT_MSG(status == ZX_OK, "hotplugging core %u failed: %d", target_cpu, status);
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
const zx_status_t dpc_status = wait_for_cpu_dpc_runner(target_cpu);
ASSERT_MSG(dpc_status == ZX_OK, "waiting for core %u's dpc runner failed: %d", target_cpu,
dpc_status);
}
});
// Phase 1: Wake an unpinned thread (and unpin an offline-enqueued thread) whose |last_cpu| is
// already offline and had a migration function installed after |CallMigrateFnForCpu| ran.
offline_waiter = Thread::Create(
"unpinned-offline-migrate-waiter",
[](void* arg) -> int {
auto* s = static_cast<State*>(arg);
s->offline_waiter_before_cpu.store(arch_curr_cpu_num(), ktl::memory_order_relaxed);
s->offline_waiter_started_event.Signal();
s->offline_waiter_wake_event.Wait();
s->offline_waiter_after_cpu.store(arch_curr_cpu_num(), ktl::memory_order_relaxed);
return 0;
},
state.get(), DEFAULT_PRIORITY);
ASSERT_NONNULL(offline_waiter, "offline_waiter create failed");
offline_waiter->SetSoftCpuAffinity(cpu_num_to_mask(target_cpu));
offline_waiter->Resume();
pinned_enqueued = Thread::Create(
"unpin-while-offline-migrate-worker",
[](void* arg) -> int {
auto* s = static_cast<State*>(arg);
s->pinned_enqueued_before_cpu.store(arch_curr_cpu_num(), ktl::memory_order_relaxed);
s->pinned_enqueued_started_event.Signal();
while (!s->stop_pinned_enqueued.load(ktl::memory_order_relaxed)) {
if (s->unpinned_observe_cpu.load(ktl::memory_order_relaxed)) {
s->pinned_enqueued_after_cpu.store(arch_curr_cpu_num(), ktl::memory_order_relaxed);
break;
}
Thread::Current::Yield();
}
return 0;
},
state.get(), LOW_PRIORITY);
ASSERT_NONNULL(pinned_enqueued, "pinned_enqueued create failed");
pinned_enqueued->SetCpuAffinity(cpu_num_to_mask(target_cpu));
pinned_enqueued->Resume();
ASSERT_OK(state->offline_waiter_started_event.Wait(),
"waiting for offline_waiter_started_event failed");
wait_for_thread_state(*offline_waiter, THREAD_BLOCKED);
ASSERT_OK(state->pinned_enqueued_started_event.Wait(),
"waiting for pinned_enqueued_started_event failed");
EXPECT_EQ(target_cpu, state->offline_waiter_before_cpu.load(ktl::memory_order_relaxed));
EXPECT_EQ(target_cpu, state->pinned_enqueued_before_cpu.load(ktl::memory_order_relaxed));
ASSERT_OK(mp_unplug_cpu_mask(cpu_num_to_mask(target_cpu), ZX_TIME_INFINITE),
"unplugging core failed");
ASSERT_OK(wait_for_cpu_offline(target_cpu), "waiting for core to go offline failed");
ASSERT_FALSE(Scheduler::PeekIsActive(target_cpu));
wait_for_thread_state(*pinned_enqueued, THREAD_READY);
// Install migration functions while |target_cpu| is offline so |last_cpu == target_cpu| with
// |!migrate_pending|.
offline_waiter->SetMigrateFn([state = state.get()](Thread* t, Thread::MigrateStage stage) {
ChainLockTransaction::AssertActive();
t->get_lock().AssertHeld();
switch (stage) {
case Thread::MigrateStage::Save:
if (arch_curr_cpu_num() != t->LastCpuLocked()) {
state->offline_waiter_migrate_ok.store(false, ktl::memory_order_relaxed);
}
state->offline_waiter_save_calls.fetch_add(1, ktl::memory_order_relaxed);
break;
case Thread::MigrateStage::Restore:
state->offline_waiter_restore_calls.fetch_add(1, ktl::memory_order_relaxed);
state->offline_waiter_migrate_ok.store(false, ktl::memory_order_relaxed);
break;
case Thread::MigrateStage::Exiting:
state->offline_waiter_exiting_calls.fetch_add(1, ktl::memory_order_relaxed);
break;
}
});
state->offline_waiter_wake_event.Signal();
ASSERT_OK(offline_waiter->Join(nullptr, ZX_TIME_INFINITE), "joining offline_waiter failed");
offline_waiter = nullptr;
const cpu_num_t waiter_after_cpu =
state->offline_waiter_after_cpu.load(ktl::memory_order_relaxed);
EXPECT_NE(INVALID_CPU, waiter_after_cpu);
EXPECT_NE(target_cpu, waiter_after_cpu);
EXPECT_EQ(0u, state->offline_waiter_save_calls.load(ktl::memory_order_relaxed));
EXPECT_EQ(0u, state->offline_waiter_restore_calls.load(ktl::memory_order_relaxed));
EXPECT_EQ(1u, state->offline_waiter_exiting_calls.load(ktl::memory_order_relaxed));
EXPECT_TRUE(state->offline_waiter_migrate_ok.load(ktl::memory_order_relaxed));
pinned_enqueued->SetMigrateFn([state = state.get()](Thread* t, Thread::MigrateStage stage) {
ChainLockTransaction::AssertActive();
t->get_lock().AssertHeld();
switch (stage) {
case Thread::MigrateStage::Save:
if (arch_curr_cpu_num() != t->LastCpuLocked()) {
state->pinned_enqueued_migrate_ok.store(false, ktl::memory_order_relaxed);
}
state->pinned_enqueued_save_calls.fetch_add(1, ktl::memory_order_relaxed);
break;
case Thread::MigrateStage::Restore:
state->pinned_enqueued_restore_calls.fetch_add(1, ktl::memory_order_relaxed);
state->pinned_enqueued_migrate_ok.store(false, ktl::memory_order_relaxed);
break;
case Thread::MigrateStage::Exiting:
state->pinned_enqueued_exiting_calls.fetch_add(1, ktl::memory_order_relaxed);
break;
}
});
state->unpinned_observe_cpu.store(true, ktl::memory_order_relaxed);
pinned_enqueued->SetCpuAffinity(CPU_MASK_ALL);
ASSERT_OK(pinned_enqueued->Join(nullptr, ZX_TIME_INFINITE), "joining pinned_enqueued failed");
pinned_enqueued = nullptr;
const cpu_num_t enqueued_after_cpu =
state->pinned_enqueued_after_cpu.load(ktl::memory_order_relaxed);
EXPECT_NE(INVALID_CPU, enqueued_after_cpu);
EXPECT_NE(target_cpu, enqueued_after_cpu);
EXPECT_EQ(0u, state->pinned_enqueued_save_calls.load(ktl::memory_order_relaxed));
EXPECT_EQ(0u, state->pinned_enqueued_restore_calls.load(ktl::memory_order_relaxed));
EXPECT_EQ(1u, state->pinned_enqueued_exiting_calls.load(ktl::memory_order_relaxed));
EXPECT_TRUE(state->pinned_enqueued_migrate_ok.load(ktl::memory_order_relaxed));
ASSERT_OK(hotplug_core(target_cpu), "hotplugging core failed");
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
ASSERT_OK(wait_for_cpu_dpc_runner(target_cpu), "waiting for core's dpc runner failed");
// Phase 2: Repeatedly unplug and hotplug |target_cpu| while unpinned workers with migration
// functions block on |target_cpu| and are woken concurrently from |BOOT_CPU_ID|.
for (size_t i = 0; i < kNumWorkers; ++i) {
WorkerSlot& slot = state->workers[i];
slot.state = state.get();
slot.thread = Thread::Create(
"unpinned-deactivating-migrate-worker",
[](void* arg) -> int {
auto* w = static_cast<WorkerSlot*>(arg);
State* s = w->state;
w->started_event.Signal();
uint64_t iter = 0;
while (!s->stop_workers.load(ktl::memory_order_relaxed)) {
if ((++iter & 1u) == 0u) {
w->wake_event.Wait(Deadline::after_mono(ZX_USEC(150)));
} else {
Thread::Current::SleepRelative(ZX_USEC(50));
}
w->wakeups.fetch_add(1, ktl::memory_order_relaxed);
}
return 0;
},
&slot, DEFAULT_PRIORITY);
ASSERT_NONNULL(slot.thread, "worker create failed");
slot.thread->SetCpuAffinity(cpu_num_to_mask(target_cpu));
slot.thread->SetMigrateFn([&slot](Thread* t, Thread::MigrateStage stage) {
ChainLockTransaction::AssertActive();
t->get_lock().AssertHeld();
switch (stage) {
case Thread::MigrateStage::Save:
if (slot.exited || slot.save_pending || arch_curr_cpu_num() != t->LastCpuLocked()) {
slot.migrate_order_ok.store(false, ktl::memory_order_relaxed);
}
slot.save_pending = true;
slot.save_calls.fetch_add(1, ktl::memory_order_relaxed);
break;
case Thread::MigrateStage::Restore:
if (slot.exited || !slot.save_pending) {
slot.migrate_order_ok.store(false, ktl::memory_order_relaxed);
}
slot.save_pending = false;
slot.restore_calls.fetch_add(1, ktl::memory_order_relaxed);
break;
case Thread::MigrateStage::Exiting:
if (slot.exited || slot.save_pending) {
slot.migrate_order_ok.store(false, ktl::memory_order_relaxed);
}
slot.exited = true;
break;
}
});
slot.thread->Resume();
}
for (WorkerSlot& slot : state->workers) {
ASSERT_OK(slot.started_event.Wait(), "waiting for worker to start on target_cpu failed");
slot.thread->SetCpuAffinity(CPU_MASK_ALL);
slot.thread->SetSoftCpuAffinity(cpu_num_to_mask(target_cpu));
}
waker = Thread::Create(
"unpinned-deactivating-migrate-waker",
[](void* arg) -> int {
auto* s = static_cast<State*>(arg);
while (!s->stop_workers.load(ktl::memory_order_relaxed)) {
for (WorkerSlot& slot : s->workers) {
slot.wake_event.Signal();
}
Thread::Current::SleepRelative(ZX_USEC(25));
}
return 0;
},
state.get(), DEFAULT_PRIORITY);
ASSERT_NONNULL(waker, "waker create failed");
waker->SetCpuAffinity(cpu_num_to_mask(BOOT_CPU_ID));
waker->Resume();
constexpr int kUnplugCycles = 20;
for (int i = 0; i < kUnplugCycles; ++i) {
Thread::Current::SleepRelative(ZX_USEC(250));
ASSERT_OK(mp_unplug_cpu_mask(cpu_num_to_mask(target_cpu), ZX_TIME_INFINITE),
"unplugging core failed");
ASSERT_OK(wait_for_cpu_offline(target_cpu), "waiting for core to go offline failed");
ASSERT_OK(hotplug_core(target_cpu), "hotplugging core failed");
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
ASSERT_OK(wait_for_cpu_dpc_runner(target_cpu), "waiting for core's dpc runner failed");
}
cleanup_threads.call();
uint64_t total_wakeups = 0;
uint64_t total_save_calls = 0;
uint64_t total_restore_calls = 0;
for (const WorkerSlot& slot : state->workers) {
total_wakeups += slot.wakeups.load(ktl::memory_order_relaxed);
total_save_calls += slot.save_calls.load(ktl::memory_order_relaxed);
total_restore_calls += slot.restore_calls.load(ktl::memory_order_relaxed);
EXPECT_TRUE(slot.migrate_order_ok.load(ktl::memory_order_relaxed));
}
EXPECT_GE(total_wakeups, kNumWorkers);
EXPECT_GE(total_save_calls, kNumWorkers);
EXPECT_EQ(total_save_calls, total_restore_calls);
END_TEST;
}
// Regression test for https://fxbug.dev/578020457 and https://fxbug.dev/448121736:
// Repeatedly unplug and hotplug a secondary CPU while unpinned fair threads of widely differing
// weights (driving fair_affine_transform.slope < 1.0) actively run on and migrate off that CPU.
// Verifies that Scheduler::MigrateUnpinnedThreads converts fair start/finish times from variable to
// monotonic time only once rather than corrupting the fair timeline across unplug cycles.
[[maybe_unused]] static bool mp_hotplug_migrate_fair_timeline_test() {
BEGIN_TEST;
// TODO(https://fxbug.dev/42086046): Re-enable test on RISC-V.
#if defined(__riscv)
printf(
"skipping test mp_hotplug_migrate_fair_timeline, hotplug only supported on x64 and arm64\n");
END_TEST;
#endif
const cpu_mask_t online_secondaries = mp_get_online_mask() & ~cpu_num_to_mask(BOOT_CPU_ID);
if (online_secondaries == 0) {
printf("skipping test mp_hotplug_migrate_fair_timeline, not enough online cpus\n");
END_TEST;
}
const cpu_num_t target_cpu = highest_cpu_set(online_secondaries);
auto restore_affinity = fit::defer(
[previous_affinity = Thread::Current::SetCpuAffinity(cpu_num_to_mask(BOOT_CPU_ID))] {
Thread::Current::SetCpuAffinity(previous_affinity);
});
// |stop| is a monotonic cancellation flag and |iterations| is read after Thread::Join().
struct State {
ktl::atomic<bool> stop{false};
ktl::atomic<uint64_t> iterations{0};
} state;
// Combining LOWEST_PRIORITY and HIGHEST_PRIORITY fair threads with soft affinity preferring
// |target_cpu| guarantees min_weight < weight_total * kMinFairUtilization on |target_cpu|,
// forcing fair_affine_transform.slope well below 1.0 while keeping hard_affinity unpinned so
// MigrateUnpinnedThreads migrates them on each unplug cycle.
constexpr int kPriorities[] = {
LOWEST_PRIORITY, LOWEST_PRIORITY, DEFAULT_PRIORITY, HIGHEST_PRIORITY, HIGHEST_PRIORITY,
};
Thread* workers[ktl::size(kPriorities)] = {};
auto stop_workers = fit::defer([&] {
state.stop.store(true, ktl::memory_order_relaxed);
for (Thread*& worker : workers) {
if (worker != nullptr) {
worker->Join(nullptr, ZX_TIME_INFINITE);
worker = nullptr;
}
}
});
auto restore_target_cpu = fit::defer([target_cpu] {
if (!mp_is_cpu_online(target_cpu)) {
const zx_status_t status = hotplug_core(target_cpu);
ASSERT_MSG(status == ZX_OK, "hotplugging core %u failed: %d", target_cpu, status);
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
const zx_status_t dpc_status = wait_for_cpu_dpc_runner(target_cpu);
ASSERT_MSG(dpc_status == ZX_OK, "waiting for core %u's dpc runner failed: %d", target_cpu,
dpc_status);
}
});
for (size_t i = 0; i < ktl::size(kPriorities); ++i) {
workers[i] = Thread::Create(
"mp-hotplug-fair-timeline-worker",
[](void* arg) -> int {
auto* s = static_cast<State*>(arg);
while (!s->stop.load(ktl::memory_order_relaxed)) {
s->iterations.fetch_add(1, ktl::memory_order_relaxed);
Thread::Current::Yield();
}
return 0;
},
&state, kPriorities[i]);
ASSERT_NONNULL(workers[i], "worker create failed");
workers[i]->SetSoftCpuAffinity(cpu_num_to_mask(target_cpu));
workers[i]->Resume();
}
constexpr int kUnplugCycles = 25;
for (int i = 0; i < kUnplugCycles; ++i) {
Thread::Current::SleepRelative(ZX_MSEC(1));
ASSERT_OK(mp_unplug_cpu_mask(cpu_num_to_mask(target_cpu), ZX_TIME_INFINITE),
"unplugging core failed");
ASSERT_OK(wait_for_cpu_offline(target_cpu), "waiting for core to go offline failed");
ASSERT_OK(hotplug_core(target_cpu), "hotplugging core failed");
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
ASSERT_OK(wait_for_cpu_dpc_runner(target_cpu), "waiting for core's dpc runner failed");
}
stop_workers.call();
EXPECT_GT(state.iterations.load(ktl::memory_order_relaxed), 0u);
END_TEST;
}
// Regression test for https://fxbug.dev/578020457:
// Repeatedly unplug and hotplug |target_cpu| while worker threads on other active CPUs transition
// their own soft affinity to |target_cpu| and yield. If |target_cpu| deactivates after
// Scheduler::NeedsMigrationLocal initiates migration away from the current CPU, target selection
// falls back to hard affinity and may re-select the current CPU; verifies that RescheduleCommon
// does not assert, invoke a spurious MigrateStage::Save without leaving the CPU, or lose the local
// reschedule.
[[maybe_unused]] static bool mp_hotplug_soft_affinity_migration_fallback_test() {
BEGIN_TEST;
// TODO(https://fxbug.dev/42086046): Re-enable test on RISC-V.
#if defined(__riscv)
printf(
"skipping test mp_hotplug_soft_affinity_migration_fallback, hotplug only supported on x64 "
"and arm64\n");
END_TEST;
#endif
const cpu_mask_t online_secondaries = mp_get_online_mask() & ~cpu_num_to_mask(BOOT_CPU_ID);
if (online_secondaries == 0) {
printf("skipping test mp_hotplug_soft_affinity_migration_fallback, not enough online cpus\n");
END_TEST;
}
const cpu_num_t target_cpu = highest_cpu_set(online_secondaries);
auto restore_affinity = fit::defer(
[previous_affinity = Thread::Current::SetCpuAffinity(cpu_num_to_mask(BOOT_CPU_ID))] {
Thread::Current::SetCpuAffinity(previous_affinity);
});
// |stop| is a monotonic cancellation flag, |probe_target| is an advisory toggle carrying no
// dependent state, and |iterations|, |migrate_save_calls|, and |migrate_order_ok| are read after
// Thread::Join(). Heap-allocate |State| to keep this function's kernel stack frame small.
struct State;
struct WorkerSlot {
Thread* thread{nullptr};
State* state{nullptr};
cpu_num_t home_cpu{INVALID_CPU};
ktl::atomic<uint64_t> migrate_save_calls{0};
ktl::atomic<bool> migrate_order_ok{true};
bool save_pending{false};
bool exited{false};
};
struct State {
ktl::atomic<bool> stop{false};
ktl::atomic<bool> probe_target{false};
ktl::atomic<uint64_t> iterations{0};
WorkerSlot workers[SMP_MAX_CPUS]{};
cpu_num_t target_cpu{INVALID_CPU};
};
fbl::AllocChecker ac;
auto state = ktl::make_unique<State>(&ac);
ASSERT_TRUE(ac.check(), "failed to allocate test state");
state->target_cpu = target_cpu;
auto stop_workers = fit::defer([state = state.get()] {
state->stop.store(true, ktl::memory_order_relaxed);
for (WorkerSlot& slot : state->workers) {
if (slot.thread != nullptr) {
slot.thread->Join(nullptr, ZX_TIME_INFINITE);
slot.thread = nullptr;
}
}
});
auto restore_target_cpu = fit::defer([target_cpu] {
if (!mp_is_cpu_online(target_cpu)) {
const zx_status_t status = hotplug_core(target_cpu);
ASSERT_MSG(status == ZX_OK, "hotplugging core %u failed: %d", target_cpu, status);
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
const zx_status_t dpc_status = wait_for_cpu_dpc_runner(target_cpu);
ASSERT_MSG(dpc_status == ZX_OK, "waiting for core %u's dpc runner failed: %d", target_cpu,
dpc_status);
}
});
cpu_mask_t remaining_cpus = mp_get_online_mask() & ~cpu_num_to_mask(target_cpu);
size_t worker_index = 0;
while (remaining_cpus != 0) {
const cpu_num_t home_cpu = lowest_cpu_set(remaining_cpus);
remaining_cpus &= ~cpu_num_to_mask(home_cpu);
WorkerSlot& slot = state->workers[worker_index];
slot.state = state.get();
slot.home_cpu = home_cpu;
slot.thread = Thread::Create(
"mp-hotplug-soft-affinity-worker",
[](void* arg) -> int {
auto* w = static_cast<WorkerSlot*>(arg);
State* s = w->state;
const cpu_mask_t home_mask = cpu_num_to_mask(w->home_cpu);
const cpu_mask_t target_mask = cpu_num_to_mask(s->target_cpu);
Thread* self = Thread::Current::Get();
uint64_t step = 0;
while (!s->stop.load(ktl::memory_order_relaxed)) {
if (s->probe_target.load(ktl::memory_order_relaxed) && ((++step & 1u) == 0u)) {
self->SetSoftCpuAffinity(target_mask);
} else {
self->SetSoftCpuAffinity(home_mask);
}
s->iterations.fetch_add(1, ktl::memory_order_relaxed);
Thread::Current::Yield();
}
return 0;
},
&slot, DEFAULT_PRIORITY);
ASSERT_NONNULL(slot.thread, "worker create failed");
slot.thread->SetCpuAffinity(cpu_num_to_mask(home_cpu) | cpu_num_to_mask(target_cpu));
slot.thread->SetSoftCpuAffinity(cpu_num_to_mask(home_cpu));
if ((worker_index % 2) == 0) {
slot.thread->SetMigrateFn([&slot](Thread*, Thread::MigrateStage stage) {
switch (stage) {
case Thread::MigrateStage::Save:
if (slot.exited || slot.save_pending) {
slot.migrate_order_ok.store(false, ktl::memory_order_relaxed);
}
slot.save_pending = true;
slot.migrate_save_calls.fetch_add(1, ktl::memory_order_relaxed);
break;
case Thread::MigrateStage::Restore:
if (slot.exited || !slot.save_pending) {
slot.migrate_order_ok.store(false, ktl::memory_order_relaxed);
}
slot.save_pending = false;
break;
case Thread::MigrateStage::Exiting:
if (slot.exited) {
slot.migrate_order_ok.store(false, ktl::memory_order_relaxed);
}
slot.exited = true;
break;
}
});
}
slot.thread->Resume();
++worker_index;
}
constexpr int kUnplugCycles = 30;
for (int i = 0; i < kUnplugCycles; ++i) {
state->probe_target.store(false, ktl::memory_order_relaxed);
Thread::Current::SleepRelative(ZX_USEC(250));
state->probe_target.store(true, ktl::memory_order_relaxed);
ASSERT_OK(mp_unplug_cpu_mask(cpu_num_to_mask(target_cpu), ZX_TIME_INFINITE),
"unplugging core failed");
state->probe_target.store(false, ktl::memory_order_relaxed);
ASSERT_OK(wait_for_cpu_offline(target_cpu), "waiting for core to go offline failed");
ASSERT_OK(hotplug_core(target_cpu), "hotplugging core failed");
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
ASSERT_OK(wait_for_cpu_dpc_runner(target_cpu), "waiting for core's dpc runner failed");
}
stop_workers.call();
EXPECT_GT(state->iterations.load(ktl::memory_order_relaxed), 0u);
uint64_t total_migrate_save_calls = 0;
for (const WorkerSlot& slot : state->workers) {
total_migrate_save_calls += slot.migrate_save_calls.load(ktl::memory_order_relaxed);
EXPECT_TRUE(slot.migrate_order_ok.load(ktl::memory_order_relaxed));
}
EXPECT_GT(total_migrate_save_calls, 0u);
END_TEST;
}
// Regression test for https://fxbug.dev/578020457:
// Concurrently mutate soft and hard CPU affinity on runnable worker threads queued on |target_cpu|
// while repeatedly unplugging and hotplugging |target_cpu|. Verifies that
// Scheduler::MigrateUnpinnedThreads and Scheduler::SetCpuAffinity do not race on run-queue removal,
// unblock_list_node_, or thread migration state when queue_lock_ is released.
[[maybe_unused]] static bool mp_hotplug_set_affinity_during_unplug_test() {
BEGIN_TEST;
// TODO(https://fxbug.dev/42086046): Re-enable test on RISC-V.
#if defined(__riscv)
printf(
"skipping test mp_hotplug_set_affinity_during_unplug, hotplug only supported on x64 and "
"arm64\n");
END_TEST;
#endif
const cpu_mask_t online_secondaries = mp_get_online_mask() & ~cpu_num_to_mask(BOOT_CPU_ID);
if (online_secondaries == 0) {
printf("skipping test mp_hotplug_set_affinity_during_unplug, not enough online cpus\n");
END_TEST;
}
const cpu_num_t target_cpu = highest_cpu_set(online_secondaries);
auto restore_affinity = fit::defer(
[previous_affinity = Thread::Current::SetCpuAffinity(cpu_num_to_mask(BOOT_CPU_ID))] {
Thread::Current::SetCpuAffinity(previous_affinity);
});
// |stop| is a monotonic cancellation flag; counters are checked after Thread::Join().
struct State {
ktl::atomic<bool> stop{false};
ktl::atomic<uint64_t> worker_iterations{0};
ktl::atomic<uint64_t> affinity_updates{0};
Thread* workers[4]{};
cpu_num_t target_cpu{INVALID_CPU};
} state;
state.target_cpu = target_cpu;
Thread* mutator = nullptr;
auto cleanup_threads = fit::defer([&] {
state.stop.store(true, ktl::memory_order_relaxed);
if (mutator != nullptr) {
mutator->Join(nullptr, ZX_TIME_INFINITE);
mutator = nullptr;
}
for (Thread*& worker : state.workers) {
if (worker != nullptr) {
worker->Join(nullptr, ZX_TIME_INFINITE);
worker = nullptr;
}
}
});
auto restore_target_cpu = fit::defer([target_cpu] {
if (!mp_is_cpu_online(target_cpu)) {
const zx_status_t status = hotplug_core(target_cpu);
ASSERT_MSG(status == ZX_OK, "hotplugging core %u failed: %d", target_cpu, status);
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
const zx_status_t dpc_status = wait_for_cpu_dpc_runner(target_cpu);
ASSERT_MSG(dpc_status == ZX_OK, "waiting for core %u's dpc runner failed: %d", target_cpu,
dpc_status);
}
});
for (Thread*& worker : state.workers) {
worker = Thread::Create(
"mp-hotplug-affinity-worker",
[](void* arg) -> int {
auto* s = static_cast<State*>(arg);
while (!s->stop.load(ktl::memory_order_relaxed)) {
s->worker_iterations.fetch_add(1, ktl::memory_order_relaxed);
Thread::Current::Yield();
}
return 0;
},
&state, DEFAULT_PRIORITY);
ASSERT_NONNULL(worker, "worker create failed");
worker->SetSoftCpuAffinity(cpu_num_to_mask(target_cpu));
worker->Resume();
}
mutator = Thread::Create(
"mp-hotplug-affinity-mutator",
[](void* arg) -> int {
auto* s = static_cast<State*>(arg);
const cpu_mask_t target_mask = cpu_num_to_mask(s->target_cpu);
const cpu_mask_t boot_or_target = cpu_num_to_mask(BOOT_CPU_ID) | target_mask;
uint64_t step = 0;
while (!s->stop.load(ktl::memory_order_relaxed)) {
for (Thread* worker : s->workers) {
worker->SetSoftCpuAffinity((step & 1u) ? target_mask : CPU_MASK_ALL);
worker->SetCpuAffinity((step & 2u) ? boot_or_target : CPU_MASK_ALL);
s->affinity_updates.fetch_add(1, ktl::memory_order_relaxed);
}
++step;
Thread::Current::Yield();
}
return 0;
},
&state, DEFAULT_PRIORITY);
ASSERT_NONNULL(mutator, "mutator create failed");
mutator->SetCpuAffinity(cpu_num_to_mask(BOOT_CPU_ID));
mutator->Resume();
constexpr int kUnplugCycles = 20;
for (int i = 0; i < kUnplugCycles; ++i) {
Thread::Current::SleepRelative(ZX_MSEC(1));
ASSERT_OK(mp_unplug_cpu_mask(cpu_num_to_mask(target_cpu), ZX_TIME_INFINITE),
"unplugging core failed");
ASSERT_OK(wait_for_cpu_offline(target_cpu), "waiting for core to go offline failed");
ASSERT_OK(hotplug_core(target_cpu), "hotplugging core failed");
wait_for_cpu_active(target_cpu);
// Unplug shuts down DPC threads, which start after the core becomes active.
ASSERT_OK(wait_for_cpu_dpc_runner(target_cpu), "waiting for core's dpc runner failed");
}
cleanup_threads.call();
EXPECT_GT(state.worker_iterations.load(ktl::memory_order_relaxed), 0u);
EXPECT_GT(state.affinity_updates.load(ktl::memory_order_relaxed), 0u);
END_TEST;
}
UNITTEST_START_TESTCASE(mp_hotplug_tests)
UNITTEST("test unplug and hotplug cores one by one", mp_hotplug_test)
UNITTEST("test mp_sync_exec while unplugging and hotplugging a core", mp_hotplug_sync_exec_test)
UNITTEST("test unblocking pinned threads while their core is offline",
mp_hotplug_unblock_pinned_while_offline_test)
UNITTEST("test unblocking unpinned migrating threads while their last core is offline",
mp_hotplug_unblock_unpinned_migrate_while_offline_test)
UNITTEST("test fair timeline integrity across repeated core unplug migrations",
mp_hotplug_migrate_fair_timeline_test)
UNITTEST("test soft affinity migration fallback during core unplug",
mp_hotplug_soft_affinity_migration_fallback_test)
UNITTEST("test concurrent SetCpuAffinity during core unplug migrations",
mp_hotplug_set_affinity_during_unplug_test)
UNITTEST_END_TESTCASE(mp_hotplug_tests, "hotplug", "Tests for unplugging and hotplugging cores")