blob: 5ad360e21bc4c4703a2976ab5787323006ea4187 [file]
// Copyright 2026 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#![deny(
dead_code,
unused_parens,
unused_results,
unsafe_op_in_unsafe_fn,
missing_unsafe_on_extern,
unsafe_attr_outside_unsafe,
non_local_definitions,
mismatched_lifetime_syntaxes,
clippy::all,
clippy::missing_safety_doc,
clippy::undocumented_unsafe_blocks,
clippy::not_unsafe_ptr_arg_deref,
clippy::new_without_default
)]
#[cfg(ktest)]
#[unittest::suite]
/// Tests for Rust ksync bindings
mod ksync_tests {
use crate::kernel::brwlock::{BrwLock, BrwLockEnablePi};
use crate::kernel::mutex::Mutex;
use crate::kernel::spin_tracing::{
IdStringRef, RWLOCK_NAME_TRACING_ENABLED, RwLockNameStorage,
};
use crate::kernel::spinlock::{MonitoredSpinLock, SpinLock};
use crate::ksync_kernel::{
CriticalMutexOrderedPolicy, CriticalMutexPolicy, IrqSavePolicy, MutexSpinMaxDurationPolicy,
NoIrqSavePolicy, TrackedBrwLockNoPi, TrackedBrwLockPi, TrackedCriticalMutex,
TrackedMonitoredSpinlock, TrackedMutex, TrackedSpinlock, TryAcquireResult,
declare_singleton_critical_mutex, declare_singleton_monitored_spinlock,
declare_singleton_mutex, declare_singleton_spinlock,
};
use kprint::kprintln;
use pin_init::{pin_data, pin_init, stack_pin_init};
use unittest::{assert_true, expect_eq, expect_true};
#[ksync::guarded]
#[fbl::ref_counted]
#[derive(fbl::Recyclable)]
#[pin_data]
#[repr(C)]
struct GuardedMutexObj {
#[mutex]
mu: ksync::KMutex<TrackedMutex>,
#[guarded_by(mu)]
value: u32,
}
#[ksync::guarded]
struct GuardedSpinlockObj {
#[mutex]
mu: ksync::KMutex<TrackedSpinlock>,
#[guarded_by(mu)]
value: u32,
}
#[ksync::guarded]
struct GuardedSpinlockObj2 {
#[mutex]
mu: ksync::KMutex<TrackedSpinlock>,
#[guarded_by(mu)]
value: u32,
}
#[ksync::guarded]
struct GuardedMonitoredSpinlockObj {
#[mutex]
mu: ksync::KMutex<TrackedMonitoredSpinlock>,
#[guarded_by(mu)]
value: u32,
}
#[ksync::guarded]
struct GuardedCriticalMutexObj {
#[mutex]
mu: ksync::KMutex<TrackedCriticalMutex>,
#[guarded_by(mu)]
value: u32,
}
#[ksync::guarded]
struct GuardedNestableCriticalMutexObj {
#[mutex(flags = ksync::LOCK_FLAGS_NESTABLE | lockdep::LOCK_FLAGS_RE_ACQUIRE_FATAL)]
mu: ksync::KMutex<TrackedCriticalMutex>,
#[guarded_by(mu)]
value: u32,
}
#[ksync::guarded]
struct GuardedNonFatalNestableCriticalMutexObj {
#[mutex(flags = ksync::LOCK_FLAGS_NESTABLE)]
mu: ksync::KMutex<TrackedCriticalMutex>,
#[guarded_by(mu)]
value: u32,
}
#[ksync::guarded]
struct GuardedUntrackedMutexObj {
#[mutex]
mu: ksync::KMutex<Mutex>,
#[guarded_by(mu)]
value: u32,
}
#[ksync::guarded]
struct GuardedUntrackedSpinlockObj {
#[mutex]
mu: ksync::KMutex<SpinLock>,
#[guarded_by(mu)]
value: u32,
}
#[ksync::guarded]
struct GuardedUntrackedMonitoredSpinlockObj {
#[mutex]
mu: ksync::KMutex<MonitoredSpinLock>,
#[guarded_by(mu)]
value: u32,
}
#[ksync::guarded]
struct GuardedBrwLockObj {
#[brwlock]
lock: ksync::BrwLockPi<TrackedBrwLockPi>,
#[guarded_by(lock)]
value: u32,
}
#[ksync::guarded]
struct GuardedBrwLockNoPiObj {
#[brwlock]
lock: ksync::BrwLockPi<TrackedBrwLockNoPi>,
#[guarded_by(lock)]
value: u32,
}
#[ksync::guarded]
struct GuardedBrwLockMultiAcquireObj {
#[brwlock(flags = lockdep::LOCK_FLAGS_MULTI_ACQUIRE)]
lock: ksync::BrwLockPi<TrackedBrwLockPi>,
#[guarded_by(lock)]
value: u32,
}
#[ksync::guarded]
struct GuardedOnceObj {
#[mutex]
mu: ksync::KMutex<TrackedMutex>,
#[guarded_by(mu)]
once_val: ksync::KOnceCell<u32>,
}
#[ksync::guarded]
struct GuardedPhantomObj {
#[mutex(GuardedMutexObjMuClass)]
mu: ksync::KMutex<ksync::PhantomMutex>,
#[guarded_by(mu)]
target: fbl::RefPtr<GuardedMutexObj>,
}
#[ksync::guarded]
#[fbl::ref_counted]
#[derive(fbl::Recyclable)]
#[pin_data]
#[repr(C)]
struct GuardedGenericObj<T> {
#[mutex]
mu: ksync::KMutex<TrackedMutex>,
#[guarded_by(mu)]
value: T,
}
#[ksync::guarded]
struct GuardedGenericPhantomObj<T> {
#[mutex(GuardedGenericObjMuClass<T>)]
mu: ksync::KMutex<ksync::PhantomMutex>,
#[guarded_by(mu)]
target: fbl::RefPtr<GuardedGenericObj<T>>,
}
unsafe extern "C" {
fn cpp_verify_mutex_id(
lock: *const core::ffi::c_void,
expected_id: *const core::ffi::c_void,
) -> bool;
fn cpp_verify_critical_mutex_id(
lock: *const core::ffi::c_void,
expected_id: *const core::ffi::c_void,
) -> bool;
fn cpp_verify_spinlock_id(
lock: *const core::ffi::c_void,
expected_id: *const core::ffi::c_void,
) -> bool;
fn cpp_verify_monitored_spinlock_id(
lock: *const core::ffi::c_void,
expected_id: *const core::ffi::c_void,
) -> bool;
fn cpp_get_test_singleton_mutex_size() -> usize;
fn cpp_get_test_singleton_mutex_raw_lock_ptr() -> *const core::ffi::c_void;
fn cpp_get_test_singleton_mutex_raw_lock_size() -> usize;
fn cpp_get_test_singleton_critical_mutex_size() -> usize;
fn cpp_get_test_singleton_critical_mutex_raw_lock_ptr() -> *const core::ffi::c_void;
fn cpp_get_test_singleton_critical_mutex_raw_lock_size() -> usize;
fn cpp_ksync_test_lock_validation_enabled() -> bool;
fn cpp_ksync_test_lockdep_last_result() -> u8;
}
// LINT.IfChange
const LOCKDEP_RESULT_SUCCESS: u8 = 0;
const LOCKDEP_RESULT_INVALID_NESTING: u8 = 3;
// LINT.ThenChange(//zircon/kernel/tests/ksync_tests.cc)
/// Simulates losing the order across `call_unlocked` by reacquiring at order 0.
#[derive(Debug, Default, Clone, Copy)]
struct LostOrderPolicy;
// SAFETY: Every operation delegates to `CriticalMutexOrderedPolicy`, which provides mutual
// exclusion; only the lockdep order passed on reacquire differs.
unsafe impl ksync::LockPolicy<TrackedCriticalMutex> for LostOrderPolicy {
type AcquireArgs = usize;
type GuardState =
<CriticalMutexOrderedPolicy as ksync::LockPolicy<TrackedCriticalMutex>>::GuardState;
const EXTERNAL_ORDER: bool = true;
unsafe fn acquire(lock: &TrackedCriticalMutex, state: &mut Self::GuardState, order: usize) {
// SAFETY: The caller upholds `LockPolicy::acquire`'s contract, which is identical for
// `CriticalMutexOrderedPolicy`.
unsafe {
<CriticalMutexOrderedPolicy as ksync::LockPolicy<_>>::acquire(lock, state, order);
}
}
unsafe fn reacquire(lock: &TrackedCriticalMutex, state: &mut Self::GuardState) {
// Reset `state` after the preceding `release` so that it satisfies `acquire`'s
// precondition of a freshly `Default`-initialized `GuardState` in the same pinned
// slot. Deliberately reacquiring at order 0 instead of `state.order` is what this
// policy tests; it only affects lockdep validation, not mutual exclusion.
*state = Self::GuardState::default();
// SAFETY: `state` is freshly `Default`-initialized and remains pinned in place until
// the matching `release` (or `move_state`) call per `LockPolicy::reacquire`'s contract.
unsafe {
<CriticalMutexOrderedPolicy as ksync::LockPolicy<_>>::acquire(lock, state, 0);
}
}
unsafe fn untrack(lock: &TrackedCriticalMutex, state: &mut Self::GuardState) {
// SAFETY: Caller upholds `LockPolicy::untrack`'s contract.
unsafe {
<CriticalMutexOrderedPolicy as ksync::LockPolicy<_>>::untrack(lock, state);
}
}
unsafe fn retrack(lock: &TrackedCriticalMutex, state: &mut Self::GuardState) {
// SAFETY: Caller upholds `LockPolicy::retrack`'s contract.
unsafe {
<CriticalMutexOrderedPolicy as ksync::LockPolicy<_>>::retrack(lock, state);
}
}
unsafe fn move_state(src: &mut Self::GuardState, dst: &mut Self::GuardState) {
// SAFETY: Caller upholds `LockPolicy::move_state`'s contract.
unsafe {
<CriticalMutexOrderedPolicy as ksync::LockPolicy<TrackedCriticalMutex>>::move_state(
src, dst,
);
}
}
unsafe fn release(lock: &TrackedCriticalMutex, state: &mut Self::GuardState) {
// SAFETY: The caller upholds `LockPolicy::release`'s contract; `state` came from this
// policy's `acquire`/`reacquire`, which produce `CriticalMutexOrderedPolicy` state.
unsafe {
<CriticalMutexOrderedPolicy as ksync::LockPolicy<_>>::release(lock, state);
}
}
}
// SAFETY: `LostOrderPolicy` delegates lock acquisition and release to
// `CriticalMutexOrderedPolicy`, which guarantees exclusive mutual exclusion.
unsafe impl ksync::ExclusiveLockPolicy<TrackedCriticalMutex> for LostOrderPolicy {}
/// test Rust KMutex ID
#[test]
fn mutex_id() {
let obj = fbl::pin_make_ref_counted!(GuardedMutexObj {
mu <- ksync::KMutex::init(),
value: 0.into(),
})
.unwrap();
// SAFETY: `obj.mu` is an initialized `KMutex`.
unsafe {
assert_true!(cpp_verify_mutex_id(
(&raw const obj.mu).cast::<core::ffi::c_void>(),
<GuardedMutexObjMuClass as ksync::LockClass>::ID,
));
}
}
/// test Rust KCriticalMutex ID
#[test]
fn critical_mutex_id() {
stack_pin_init!(let obj = pin_init!(GuardedCriticalMutexObj {
mu <- ksync::KMutex::init(),
value: 0.into(),
}));
// SAFETY: `obj.mu` is an initialized `KMutex<_, TrackedCriticalMutex>`.
unsafe {
assert_true!(cpp_verify_critical_mutex_id(
(&raw const obj.mu).cast::<core::ffi::c_void>(),
<GuardedCriticalMutexObjMuClass as ksync::LockClass>::ID,
));
}
}
/// test ordered TrackedCriticalMutex acquisition and call_unlocked reacquire
#[test]
fn critical_mutex_ordered() {
stack_pin_init!(let a = pin_init!(GuardedNestableCriticalMutexObj {
mu <- ksync::KMutex::init(),
value: 1.into(),
}));
stack_pin_init!(let b = pin_init!(GuardedNestableCriticalMutexObj {
mu <- ksync::KMutex::init(),
value: 2.into(),
}));
// Acquire `a` (order 0) then `b` (order 1), and test `call_unlocked` on `b` while holding
// `a`.
{
ksync::lock!(
let mut ga = a.lock_mu_policy_with::<CriticalMutexOrderedPolicy>(0)
);
ksync::lock!(
let mut gb = b.lock_mu_policy_with::<CriticalMutexOrderedPolicy>(1)
);
let sum = gb.as_mut().call_unlocked(|| *ga.value() + 10);
*gb.as_mut().value_mut() = sum;
*ga.as_mut().value_mut() = 3;
}
// Acquire in reverse instance order (`b` at order 0, then `a` at order 1).
{
ksync::lock!(
let gb = b.mu.lock_policy_with::<CriticalMutexOrderedPolicy>(0)
);
ksync::lock!(
let ga = a.mu.lock_policy_with::<CriticalMutexOrderedPolicy>(1)
);
expect_true!(*b.guard_mu_from_guard(&gb).value() == 11);
expect_true!(*a.guard_mu_from_guard(&ga).value() == 3);
}
// Exercise `lock_<mu>_aliased_policy_with` with `CriticalMutexOrderedPolicy`.
{
type Ordered = CriticalMutexOrderedPolicy;
ksync::lock!(let ga = a.lock_mu_aliased_policy_with::<_, _, Ordered>(&a.mu, 0));
ksync::lock!(let gb = b.lock_mu_policy_with::<Ordered>(1));
let (t1, _t2) = ga.tokens();
// SAFETY: `t1` was acquired from `a.mu`.
let guarded_a = unsafe { a.guard_mu(t1) };
expect_true!(*guarded_a.value() == 3);
expect_true!(*gb.value() == 11);
}
}
/// test that ordered reacquire in call_unlocked and retrack in call_untracked pass lockdep
#[test]
fn critical_mutex_ordered_call_unlocked() {
// SAFETY: No preconditions.
if unsafe { cpp_ksync_test_lock_validation_enabled() } {
let last = || {
// SAFETY: Runs on a valid kernel thread; reads only the current thread's
// `ThreadLockState`.
unsafe { cpp_ksync_test_lockdep_last_result() }
};
stack_pin_init!(let a = pin_init!(GuardedNonFatalNestableCriticalMutexObj {
mu <- ksync::KMutex::init(),
value: 0.into(),
}));
stack_pin_init!(let b = pin_init!(GuardedNonFatalNestableCriticalMutexObj {
mu <- ksync::KMutex::init(),
value: 7.into(),
}));
{
ksync::lock!(
let ga = a.lock_mu_policy_with::<CriticalMutexOrderedPolicy>(0)
);
ksync::lock!(
let mut gb = b.lock_mu_policy_with::<CriticalMutexOrderedPolicy>(1)
);
expect_true!(last() == LOCKDEP_RESULT_SUCCESS);
gb.as_mut().call_unlocked(|| ());
expect_true!(last() == LOCKDEP_RESULT_SUCCESS);
// The lockdep entry (and its order) survives `call_untracked`, so retracking `b`
// at order 1 while holding `a` at order 0 validates successfully.
let value = gb.as_mut().call_untracked(|token| {
// SAFETY: `token` was acquired from `b.mu`.
*unsafe { b.guard_mu(token) }.value()
});
expect_true!(value == 7);
expect_true!(last() == LOCKDEP_RESULT_SUCCESS);
expect_true!(*ga.value() == 0);
}
}
}
/// test that lockdep detects invalid nesting and a lost order on ordered reacquire
#[test]
fn critical_mutex_ordered_lost_order_detected() {
// SAFETY: No preconditions.
if unsafe { cpp_ksync_test_lock_validation_enabled() } {
struct LockdepExceptBlockGuard;
impl Drop for LockdepExceptBlockGuard {
fn drop(&mut self) {
kprintln!("[===KSYNC LOCKDEP EXCEPT BLOCK END===]");
}
}
kprintln!("[===KSYNC LOCKDEP EXCEPT BLOCK START===]");
let _except_block = LockdepExceptBlockGuard;
let last = || {
// SAFETY: Runs on a valid kernel thread; reads only the current thread's
// `ThreadLockState`.
unsafe { cpp_ksync_test_lockdep_last_result() }
};
stack_pin_init!(let a = pin_init!(GuardedNonFatalNestableCriticalMutexObj {
mu <- ksync::KMutex::init(),
value: 0.into(),
}));
stack_pin_init!(let b = pin_init!(GuardedNonFatalNestableCriticalMutexObj {
mu <- ksync::KMutex::init(),
value: 0.into(),
}));
// Equal orders: the second acquisition is not strictly above the first.
{
ksync::lock!(
let _ga = a.mu.lock_policy_with::<CriticalMutexOrderedPolicy>(1)
);
ksync::lock!(
let _gb = b.mu.lock_policy_with::<CriticalMutexOrderedPolicy>(1)
);
expect_true!(last() == LOCKDEP_RESULT_INVALID_NESTING);
}
// Descending orders (`a` at 1, then `b` at 0), as in the C++ lockdep tests.
{
ksync::lock!(
let _ga = a.mu.lock_policy_with::<CriticalMutexOrderedPolicy>(1)
);
expect_true!(last() == LOCKDEP_RESULT_SUCCESS);
ksync::lock!(
let _gb = b.mu.lock_policy_with::<CriticalMutexOrderedPolicy>(0)
);
expect_true!(last() == LOCKDEP_RESULT_INVALID_NESTING);
}
// Reacquiring at order 0 instead of the saved order 1 inside `call_unlocked`.
{
ksync::lock!(
let _ga = a.mu.lock_policy_with::<CriticalMutexOrderedPolicy>(0)
);
ksync::lock!(let mut gb = b.mu.lock_policy_with::<LostOrderPolicy>(1));
expect_true!(last() == LOCKDEP_RESULT_SUCCESS);
gb.as_mut().call_unlocked(|| ());
expect_true!(last() == LOCKDEP_RESULT_INVALID_NESTING);
}
}
}
/// test Rust KSpinlock ID
#[test]
fn spinlock_id() {
stack_pin_init!(let obj = pin_init!(GuardedSpinlockObj {
mu <- ksync::KMutex::init(),
value: 0.into(),
}));
// SAFETY: `obj.mu` is an initialized `KMutex<_, TrackedSpinlock>`.
unsafe {
assert_true!(cpp_verify_spinlock_id(
(&raw const obj.mu).cast::<core::ffi::c_void>(),
<GuardedSpinlockObjMuClass as ksync::LockClass>::ID,
));
}
}
/// test Rust KMonitoredSpinlock ID and acquire/release with source_tag!
#[test]
fn monitored_spinlock_id_and_ops() {
stack_pin_init!(let obj = pin_init!(GuardedMonitoredSpinlockObj {
mu <- ksync::KMutex::init(),
value: 10.into(),
}));
// SAFETY: `obj.mu` is an initialized `KMutex<_, TrackedMonitoredSpinlock>`.
unsafe {
assert_true!(cpp_verify_monitored_spinlock_id(
(&raw const obj.mu).cast::<core::ffi::c_void>(),
<GuardedMonitoredSpinlockObjMuClass as ksync::LockClass>::ID,
));
}
// Acquire with default IrqSavePolicy and source_tag!
{
ksync::lock!(let mut guard = obj.lock_mu(ksync::source_tag!()));
expect_eq!(*guard.value(), 10);
*guard.as_mut().value_mut() = 20;
expect_eq!(*guard.value(), 20);
}
// Verify persistence and acquire with explicit policy
{
ksync::lock!(
let guard = obj.lock_mu_policy::<IrqSavePolicy>(ksync::source_tag!())
);
expect_eq!(*guard.value(), 20);
}
}
/// Returns the lock class name that lock tracing records for a lock of the class `C`.
fn expected_class_name_ref<C: ksync::LockClass>() -> IdStringRef {
let mut name_storage = RwLockNameStorage::<RWLOCK_NAME_TRACING_ENABLED>::new();
// SAFETY: `C::ID` is either null or a valid static `LockClassId` (`*const LockClassState`),
// whose `#[repr(C)]` first field at offset 0 is a null or `'static InternedString` pointer
// in `__fxt_interned_string_table`.
unsafe { name_storage.set_lock_class_id(C::ID) };
name_storage.class_name_ref()
}
/// Returns whether `lock` was initialized with the lockdep lock class ID of `C` (when lock
/// class metadata is compiled in).
fn verify_brwlock_id<C: ksync::LockClass, P: BrwLockEnablePi>(
lock: &ksync::LockdepTracked<BrwLock<P>>,
) -> bool {
lock.class_id().is_none_or(|class_id| class_id == C::ID)
}
/// Returns whether `lock` was assigned a lock ID and the lock class name of `C` for lock
/// tracing (when enabled).
fn verify_brwlock_name<C: ksync::LockClass, P: BrwLockEnablePi>(lock: &BrwLock<P>) -> bool {
lock.class_name_ref() == expected_class_name_ref::<C>()
&& (!RWLOCK_NAME_TRACING_ENABLED || lock.encoded_lock_id().id() != 0)
}
/// test Rust BrwLockPi with TrackedBrwLockPi and TrackedBrwLockNoPi IDs
#[test]
fn brwlock_id() {
stack_pin_init!(let obj = pin_init!(GuardedBrwLockObj {
lock <- ksync::BrwLockPi::init(),
value: 0.into(),
}));
expect_true!(verify_brwlock_id::<GuardedBrwLockObjLockClass, _>(obj.lock.raw_lock()));
expect_true!(verify_brwlock_name::<GuardedBrwLockObjLockClass, _>(&**obj.lock.raw_lock()));
stack_pin_init!(let no_pi_obj = pin_init!(GuardedBrwLockNoPiObj {
lock <- ksync::BrwLockPi::init(),
value: 0.into(),
}));
expect_true!(verify_brwlock_id::<GuardedBrwLockNoPiObjLockClass, _>(
no_pi_obj.lock.raw_lock()
));
expect_true!(verify_brwlock_name::<GuardedBrwLockNoPiObjLockClass, _>(
&**no_pi_obj.lock.raw_lock()
));
}
/// test Rust KSpinlock
#[test]
fn spinlock() {
stack_pin_init!(let obj = pin_init!(GuardedSpinlockObj {
mu <- ksync::KMutex::init(),
value: 100.into(),
}));
stack_pin_init!(let obj2 = pin_init!(GuardedSpinlockObj2 {
mu <- ksync::KMutex::init(),
value: 100.into(),
}));
{
ksync::lock!(let mut guard = obj.lock_mu());
expect_eq!(*guard.value(), 100);
*guard.as_mut().value_mut() = 101;
}
{
ksync::lock!(let guard = obj.lock_mu());
expect_eq!(*guard.value(), 101);
}
{
ksync::lock!(let guard = obj.mu.lock());
ksync::lock!(let guard2 = obj2.mu.lock_policy::<NoIrqSavePolicy>());
// Expect a no irq save policy guard to be strictly smaller than one that had to
// remember the irq state.
expect_true!(core::mem::size_of_val(&*guard2) < core::mem::size_of_val(&*guard));
}
}
/// test Rust KMutex
#[test]
fn mutex() {
let obj = fbl::pin_make_ref_counted!(GuardedMutexObj {
mu <- ksync::KMutex::init(),
value: 42.into(),
})
.unwrap();
expect_true!(core::ptr::eq(
<TrackedMutex as ksync::RawLock>::as_mut_ptr(obj.mu.raw_mutex()),
<Mutex as ksync::RawLock>::as_mut_ptr(&**obj.mu.raw_mutex()),
));
{
ksync::lock!(let mut guard = obj.lock_mu());
expect_eq!(*guard.value(), 42);
*guard.as_mut().value_mut() = 43;
}
{
ksync::lock!(let guard = obj.lock_mu());
expect_eq!(*guard.value(), 43);
}
}
/// test untracked Mutex directly with both standard and critical policies
#[test]
fn untracked_mutex() {
stack_pin_init!(let obj = pin_init!(GuardedUntrackedMutexObj {
mu <- ksync::KMutex::init(),
value: 10.into(),
}));
expect_true!(!obj.mu.raw_mutex().is_contested());
{
ksync::lock!(let mut guard = obj.lock_mu());
expect_eq!(*guard.value(), 10);
*guard.as_mut().value_mut() = 20;
let res = guard.as_mut().call_unlocked(|| 99);
expect_eq!(res, 99);
}
{
ksync::lock!(let mut guard = obj.lock_mu_policy::<CriticalMutexPolicy>());
expect_eq!(*guard.value(), 20);
*guard.as_mut().value_mut() = 30;
}
{
ksync::lock!(let guard = obj.lock_mu());
expect_eq!(*guard.value(), 30);
}
}
/// test MutexSpinMaxDurationPolicy acquire, call_unlocked, and call_untracked
#[test]
fn mutex_spin_max_duration() {
// 1us monotonic duration, in nanoseconds.
const SPIN_MAX_DURATION: i64 = 1_000;
// SAFETY: No preconditions.
let lock_validation_enabled = unsafe { cpp_ksync_test_lock_validation_enabled() };
let last = || {
// SAFETY: Runs on a valid kernel thread; reads only the current thread's
// `ThreadLockState`.
unsafe { cpp_ksync_test_lockdep_last_result() }
};
let obj = fbl::pin_make_ref_counted!(GuardedMutexObj {
mu <- ksync::KMutex::init(),
value: 1.into(),
})
.unwrap();
{
ksync::lock!(
let mut guard =
obj.lock_mu_policy_with::<MutexSpinMaxDurationPolicy>(
SPIN_MAX_DURATION
)
);
if lock_validation_enabled {
expect_true!(last() == LOCKDEP_RESULT_SUCCESS);
}
expect_true!(obj.mu.raw_mutex().is_held());
expect_true!(*guard.value() == 1);
let held = guard.as_mut().call_unlocked(|| obj.mu.raw_mutex().is_held());
expect_true!(!held);
expect_true!(obj.mu.raw_mutex().is_held());
if lock_validation_enabled {
expect_true!(last() == LOCKDEP_RESULT_SUCCESS);
}
let (held_in_untracked, untracked_val) = guard.as_mut().call_untracked(|token| {
let held = obj.mu.raw_mutex().is_held();
// SAFETY: `token` was acquired from `obj.mu`.
*unsafe { obj.guard_mu_mut(token) }.value_mut() = 2;
// SAFETY: `token` was acquired from `obj.mu`.
(held, *unsafe { obj.guard_mu(token) }.value())
});
expect_true!(held_in_untracked);
expect_true!(untracked_val == 2);
if lock_validation_enabled {
expect_true!(last() == LOCKDEP_RESULT_SUCCESS);
}
}
expect_true!(!obj.mu.raw_mutex().is_held());
// Exercise direct `KMutex::lock_policy_with` with boundary duration `0`.
{
ksync::lock!(
let mut guard =
obj.mu.lock_policy_with::<MutexSpinMaxDurationPolicy>(0)
);
if lock_validation_enabled {
expect_true!(last() == LOCKDEP_RESULT_SUCCESS);
}
expect_true!(obj.mu.raw_mutex().is_held());
expect_true!(*obj.guard_mu_from_guard(&guard).value() == 2);
let held = guard.as_mut().call_unlocked(|| obj.mu.raw_mutex().is_held());
expect_true!(!held);
expect_true!(obj.mu.raw_mutex().is_held());
if lock_validation_enabled {
expect_true!(last() == LOCKDEP_RESULT_SUCCESS);
}
*obj.guard_mu_mut_from_guard(guard.as_mut()).value_mut() = 3;
}
expect_true!(!obj.mu.raw_mutex().is_held());
{
ksync::lock!(let guard = obj.lock_mu());
expect_true!(*guard.value() == 3);
}
// Exercise `GuardedFlaggedMutexObj` (`#[mutex(flags = ...)]`) as well as a
// `LockdepTracked` mutex with non-default `const LOCK_FLAGS` (verifying `const LOCK_FLAGS`
// in the 3-arg `impl_lockdep_tracked_lock_policy!` arm).
stack_pin_init!(let flagged = pin_init!(GuardedFlaggedMutexObj {
seek_lock <- ksync::KMutex::init(),
seek: 100.into(),
}));
{
ksync::lock!(
let mut guard =
flagged.lock_seek_lock_policy_with::<MutexSpinMaxDurationPolicy>(
SPIN_MAX_DURATION
)
);
if lock_validation_enabled {
expect_true!(last() == LOCKDEP_RESULT_SUCCESS);
}
expect_true!(flagged.seek_lock.raw_mutex().is_held());
expect_true!(*guard.seek() == 100);
*guard.as_mut().seek_mut() = 200;
}
expect_true!(!flagged.seek_lock.raw_mutex().is_held());
type FlaggedTrackedMutex =
ksync::LockdepTracked<Mutex, Mutex, { lockdep::LOCK_FLAGS_ACTIVE_LIST_DISABLED }>;
stack_pin_init!(
let const_flagged_mu =
ksync::KMutex::<GuardedFlaggedMutexObjSeekLockClass, FlaggedTrackedMutex>::init()
);
{
ksync::lock!(
let _guard = const_flagged_mu
.lock_policy_with::<MutexSpinMaxDurationPolicy>(SPIN_MAX_DURATION)
);
if lock_validation_enabled {
expect_true!(last() == LOCKDEP_RESULT_SUCCESS);
}
expect_true!(const_flagged_mu.raw_mutex().is_held());
}
expect_true!(!const_flagged_mu.raw_mutex().is_held());
stack_pin_init!(let untracked = pin_init!(GuardedUntrackedMutexObj {
mu <- ksync::KMutex::init(),
value: 10.into(),
}));
{
ksync::lock!(
let mut guard =
untracked.lock_mu_policy_with::<MutexSpinMaxDurationPolicy>(
SPIN_MAX_DURATION
)
);
expect_true!(untracked.mu.raw_mutex().is_held());
expect_true!(*guard.value() == 10);
let held = guard.as_mut().call_unlocked(|| untracked.mu.raw_mutex().is_held());
expect_true!(!held);
expect_true!(untracked.mu.raw_mutex().is_held());
let (held_in_untracked, untracked_val) = guard.as_mut().call_untracked(|token| {
let held = untracked.mu.raw_mutex().is_held();
// SAFETY: `token` was acquired from `untracked.mu`.
*unsafe { untracked.guard_mu_mut(token) }.value_mut() = 20;
// SAFETY: `token` was acquired from `untracked.mu`.
(held, *unsafe { untracked.guard_mu(token) }.value())
});
expect_true!(held_in_untracked);
expect_true!(untracked_val == 20);
}
expect_true!(!untracked.mu.raw_mutex().is_held());
{
ksync::lock!(let guard = untracked.lock_mu());
expect_true!(*guard.value() == 20);
}
}
/// test untracked SpinLock and MonitoredSpinLock directly
#[test]
fn untracked_spinlock() {
stack_pin_init!(let obj = pin_init!(GuardedUntrackedSpinlockObj {
mu <- ksync::KMutex::init(),
value: 10.into(),
}));
stack_pin_init!(let mobj = pin_init!(GuardedUntrackedMonitoredSpinlockObj {
mu <- ksync::KMutex::init(),
value: 50.into(),
}));
{
ksync::lock!(let mut guard = obj.lock_mu());
expect_eq!(*guard.value(), 10);
*guard.as_mut().value_mut() = 20;
ksync::lock!(
let mut mguard = mobj.lock_mu_policy::<NoIrqSavePolicy>(ksync::source_tag!())
);
expect_eq!(*mguard.value(), 50);
*mguard.as_mut().value_mut() = 60;
}
{
ksync::lock!(let mut mguard = mobj.lock_mu(ksync::source_tag!()));
expect_eq!(*mguard.value(), 60);
let res = mguard.as_mut().call_unlocked(|| 77);
expect_eq!(res, 77);
*mguard.as_mut().value_mut() = 70;
}
{
ksync::lock!(let guard = obj.lock_mu());
expect_eq!(*guard.value(), 20);
ksync::lock!(let mguard = mobj.lock_mu(ksync::source_tag!()));
expect_eq!(*mguard.value(), 70);
}
}
/// test raw SpinLock methods and KMutex<_, SpinLock> assert/release helpers
#[test]
fn spinlock_raw_methods() {
stack_pin_init!(let outer = SpinLock::new());
stack_pin_init!(let inner = SpinLock::new());
stack_pin_init!(let obj = pin_init!(GuardedUntrackedSpinlockObj {
mu <- ksync::KMutex::init(),
value: 10.into(),
}));
// SAFETY: `outer` is not held by this CPU, and it is released below with the returned
// state on this CPU without blocking.
let outer_state = unsafe { outer.acquire_irq_save() };
expect_true!(outer.is_held());
outer.assert_held();
// Holding `outer` keeps interrupts disabled, so the non-IRQ-saving operations on `inner`
// and `obj.mu` below are valid.
expect_true!(!inner.is_held());
// SAFETY: Interrupts are disabled, `inner` is not held, and it is released below.
unsafe { inner.acquire() };
expect_true!(inner.is_held());
inner.assert_held();
// SAFETY: Interrupts are disabled. `inner` is already held, so this must fail and not
// acquire the lock.
let result = unsafe { inner.try_acquire() };
expect_eq!(result, TryAcquireResult::Failed);
expect_true!(!result.is_acquired());
// SAFETY: Interrupts are disabled and `inner` is held by this CPU.
unsafe { inner.release() };
expect_true!(!inner.is_held());
// SAFETY: Interrupts are disabled, and if acquired `inner` is released below.
let result = unsafe { inner.try_acquire() };
expect_eq!(result, TryAcquireResult::Acquired);
expect_true!(inner.is_held());
if result.is_acquired() {
// SAFETY: Interrupts are disabled and `inner` was acquired by this CPU above.
unsafe { inner.release() };
}
expect_true!(!inner.is_held());
// SAFETY: Interrupts are disabled, `obj.mu` is not held, and it is released below via
// `KMutex::release`.
unsafe { obj.mu.raw_mutex().acquire() };
{
// SAFETY: `obj.mu` is held by this CPU with interrupts disabled, and `token` is
// consumed by `release` below with no outstanding borrows.
let token = unsafe { obj.mu.assert_held() };
// SAFETY: `token` witnesses `obj.mu`, which guards `obj.value`.
let value = *unsafe { obj.guard_mu(&token) }.value();
expect_eq!(value, 10);
// SAFETY: `obj.mu` is held by this CPU via `acquire` with interrupts disabled, `token`
// witnesses that acquisition, and no guarded borrows remain live.
unsafe { obj.mu.release(token) };
}
expect_true!(!obj.mu.raw_mutex().is_held());
// SAFETY: `outer` is held by this CPU and `outer_state` came from its acquisition.
unsafe { outer.release_irq_restore(outer_state) };
// SAFETY: `obj.mu` is not held by this CPU, and it is released below via
// `KMutex::release_irq_restore` with the returned state.
let state = unsafe { obj.mu.raw_mutex().acquire_irq_save() };
{
// SAFETY: `obj.mu` is held by this CPU with interrupts disabled, and `token` is
// consumed by `release_irq_restore` below with no outstanding borrows.
let mut token = unsafe { obj.mu.assert_held() };
// SAFETY: `token` witnesses `obj.mu`, which guards `obj.value`.
*unsafe { obj.guard_mu_mut(&mut token) }.value_mut() = 20;
// SAFETY: `token` witnesses `obj.mu`, which guards `obj.value`.
let value = *unsafe { obj.guard_mu(&token) }.value();
expect_eq!(value, 20);
// SAFETY: `obj.mu` is held by this CPU via `acquire_irq_save`, which returned `state`,
// `token` witnesses that acquisition, and no guarded borrows remain live.
unsafe { obj.mu.release_irq_restore(state, token) };
}
{
ksync::lock!(let guard = obj.lock_mu());
expect_eq!(*guard.value(), 20);
}
}
/// test KSpinlock::assert_held on a lockdep-tracked spinlock
#[test]
fn spinlock_assert_held() {
stack_pin_init!(let obj = pin_init!(GuardedSpinlockObj {
mu <- ksync::KMutex::init(),
value: 30.into(),
}));
ksync::lock!(let guard = obj.lock_mu());
// SAFETY: `guard` holds `obj.mu` with interrupts disabled, `token` does not outlive
// `guard`, and both are only used for shared access.
let token = unsafe { obj.mu.assert_held() };
// SAFETY: `token` witnesses `obj.mu`, which guards `obj.value`.
let value = *unsafe { obj.guard_mu(&token) }.value();
expect_eq!(value, 30);
expect_eq!(*guard.value(), 30);
}
/// test PhantomMutex and #[mutex(LockClass)]
#[test]
fn phantom_lock() {
// This test demonstrates how an object (`phantom_obj`) can use a zero-sized `PhantomMutex`
// to participate in `#[guarded]` type checking while physical mutual exclusion is
// provided by an external physical lock (`real_obj.mu`).
// Create an object with a real, physical `KMutex`, wrapped in an `fbl::RefPtr`.
let real_obj = fbl::pin_make_ref_counted!(GuardedMutexObj {
mu <- ksync::KMutex::init(),
value: 1.into(),
})
.unwrap();
// Create an object protected by a zero-sized `PhantomMutex` (no physical lock storage).
// Its synchronization is provided externally by `real_obj.mu`.
let target = real_obj.clone();
stack_pin_init!(let phantom_obj = pin_init!(GuardedPhantomObj {
mu: ksync::KMutex::new(ksync::PhantomMutex),
target: target.into(),
}));
{
// Acquire a guard for `phantom_obj` by passing the physical `real_obj.mu` to `lock_mu`.
// SAFETY: `real_obj.mu` is the physical lock synchronizing `phantom_obj` and `target`.
ksync::lock!(let mut guard = unsafe { phantom_obj.lock_mu(&real_obj.mu) });
// Read the reference to `real_obj` from `phantom_obj.target`.
let target = guard.target().clone();
// Using the proof token from `guard`, read `target.value`.
// SAFETY: `guard` holds `real_obj.mu`, which protects `target`.
unsafe {
expect_eq!(*target.guard_mu(guard.token()).value(), 1);
// Modify `target.value` using the mutable token from `guard`.
*target.guard_mu_mut(guard.as_mut().token_mut()).value_mut() = 200;
}
}
{
// Lock `real_obj` directly using `GuardedMutexObj`'s own guard to verify
// that we read back the same value (200) that was written via `phantom_obj`.
ksync::lock!(let guard = real_obj.lock_mu());
expect_eq!(*guard.value(), 200);
}
}
/// test PhantomMutex with generic LockClass
#[test]
fn generic_phantom_mutex() {
let real_obj = fbl::pin_make_ref_counted!(GuardedGenericObj {
mu <- ksync::KMutex::init(),
value: 42.into(),
})
.unwrap();
let real_obj_clone = real_obj.clone();
stack_pin_init!(let phantom_obj = pin_init!(GuardedGenericPhantomObj {
mu: ksync::KMutex::new(ksync::PhantomMutex),
target: ksync::KCell::new(real_obj_clone),
}));
{
// SAFETY: `real_obj.mu` is the physical lock synchronizing `phantom_obj` and `target`.
ksync::lock!(let mut guard = unsafe { phantom_obj.lock_mu(&real_obj.mu) });
let target = guard.target().clone();
// SAFETY: `guard` holds `real_obj.mu`, which protects `target`.
unsafe {
expect_eq!(*target.guard_mu(guard.token()).value(), 42);
*target.guard_mu_mut(guard.as_mut().token_mut()).value_mut() = 99;
}
}
{
ksync::lock!(let guard = real_obj.lock_mu());
expect_eq!(*guard.value(), 99);
}
}
/// test Rust BrwLockPi with TrackedBrwLockPi and TrackedBrwLockNoPi
#[test]
fn brwlock() {
stack_pin_init!(let obj = pin_init!(GuardedBrwLockObj {
lock <- ksync::BrwLockPi::init(),
value: 10.into(),
}));
{
ksync::lock!(let mut guard = obj.read_lock());
expect_eq!(*guard.value(), 10);
let res = guard.as_mut().call_unlocked(|| 55);
expect_eq!(res, 55);
expect_eq!(*guard.value(), 10);
}
{
ksync::lock!(let mut guard = obj.write_lock());
expect_eq!(*guard.value(), 10);
let res = guard.as_mut().call_unlocked(|| 66);
expect_eq!(res, 66);
*guard.as_mut().value_mut() = 20;
}
{
ksync::lock!(let guard = obj.read_lock());
expect_eq!(*guard.value(), 20);
}
stack_pin_init!(let no_pi_obj = pin_init!(GuardedBrwLockNoPiObj {
lock <- ksync::BrwLockPi::init(),
value: 30.into(),
}));
{
ksync::lock!(let mut guard = no_pi_obj.read_lock());
expect_eq!(*guard.value(), 30);
let res = guard.as_mut().call_unlocked(|| 77);
expect_eq!(res, 77);
expect_eq!(*guard.value(), 30);
}
{
ksync::lock!(let mut guard = no_pi_obj.write_lock());
expect_eq!(*guard.value(), 30);
let res = guard.as_mut().call_unlocked(|| 88);
expect_eq!(res, 88);
*guard.as_mut().value_mut() = 40;
}
{
ksync::lock!(let guard = no_pi_obj.read_lock());
expect_eq!(*guard.value(), 40);
}
}
/// test call_unlocked on KMutex and macro-generated guards
#[test]
fn call_unlocked() {
let obj = fbl::pin_make_ref_counted!(GuardedMutexObj {
mu <- ksync::KMutex::init(),
value: 10.into(),
})
.unwrap();
ksync::lock!(let mut guard = obj.lock_mu());
expect_eq!(*guard.value(), 10);
let res = guard.as_mut().call_unlocked(|| 42);
expect_eq!(res, 42);
*guard.as_mut().value_mut() = 25;
expect_eq!(*guard.value(), 25);
}
/// test AliasedLock in kernel
#[test]
fn aliased_lock() {
let real_obj = fbl::pin_make_ref_counted!(GuardedMutexObj {
mu <- ksync::KMutex::init(),
value: 10.into(),
})
.unwrap();
let target = real_obj.clone();
stack_pin_init!(let phantom_obj = pin_init!(GuardedPhantomObj {
mu: ksync::KMutex::new(ksync::PhantomMutex),
target: target.into(),
}));
{
ksync::lock!(let mut guard = ksync::aliased_lock(&real_obj.mu, &real_obj.mu));
let (t1, t2) = guard.tokens();
// SAFETY: `real_obj` and `phantom_obj` are both protected by `real_obj.mu`, and
// `_t2_mut` is unused.
unsafe {
expect_eq!(*real_obj.guard_mu(t1).value(), 10);
expect_true!(core::ptr::eq(&**phantom_obj.guard_mu(t2).target(), &*real_obj));
let (t1_mut, _t2_mut) = guard.as_mut().tokens_mut();
*real_obj.guard_mu_mut(t1_mut).value_mut() = 55;
}
let res = guard.as_mut().call_unlocked(|| 99);
expect_eq!(res, 99);
let (t1, _t2) = guard.tokens();
// SAFETY: `t1` is backed by `real_obj.mu`.
unsafe {
expect_eq!(*real_obj.guard_mu(t1).value(), 55);
}
}
ksync::lock!(let guard = real_obj.lock_mu());
expect_eq!(*guard.value(), 55);
}
#[ksync::guarded]
struct GuardedFlaggedMutexObj {
#[mutex(flags = lockdep::LOCK_FLAGS_ACTIVE_LIST_DISABLED)]
seek_lock: ksync::KMutex<TrackedMutex>,
#[guarded_by(seek_lock)]
seek: u64,
}
/// test Mutex with custom flags (e.g. LOCK_FLAGS_ACTIVE_LIST_DISABLED)
#[test]
fn flagged_mutex() {
stack_pin_init!(let obj = pin_init!(GuardedFlaggedMutexObj {
seek_lock <- ksync::KMutex::init(),
seek: 0.into(),
}));
ksync::lock!(let mut guard = obj.lock_seek_lock());
expect_eq!(*guard.seek(), 0);
*guard.as_mut().seek_mut() = 4096;
expect_eq!(*guard.seek(), 4096);
}
declare_singleton_mutex!(TestKernelSingletonMutex);
declare_singleton_critical_mutex!(TestKernelSingletonCriticalMutex);
declare_singleton_spinlock!(TestKernelSingletonSpinlock);
/// test Rust declare_singleton_mutex and declare_singleton_critical_mutex
#[test]
fn singleton_mutex() {
let lock1 = TestKernelSingletonMutex::Get();
let lock2 = TestKernelSingletonMutex::get();
expect_true!(core::ptr::eq(lock1, lock2));
// SAFETY: All singleton locks and C++ test helper pointers/sizes are valid static objects.
// The singleton locks are not acquired or released while their raw storage is compared.
unsafe {
expect_true!(cpp_verify_mutex_id(
(&raw const *lock1).cast::<core::ffi::c_void>(),
<TestKernelSingletonMutex as ksync::LockClass>::ID,
));
expect_true!(cpp_verify_critical_mutex_id(
(&raw const *TestKernelSingletonCriticalMutex::Get()).cast::<core::ffi::c_void>(),
<TestKernelSingletonCriticalMutex as ksync::LockClass>::ID,
));
expect_true!(cpp_verify_spinlock_id(
(&raw const *TestKernelSingletonSpinlock::Get()).cast::<core::ffi::c_void>(),
<TestKernelSingletonSpinlock as ksync::LockClass>::ID,
));
// Compare total sizes of Rust-declared singletons vs C++-declared singletons.
let rust_mutex_size = core::mem::size_of_val(lock1);
let cpp_mutex_size = cpp_get_test_singleton_mutex_size();
expect_eq!(rust_mutex_size, cpp_mutex_size);
let rust_critical_size =
core::mem::size_of_val(TestKernelSingletonCriticalMutex::Get());
let cpp_critical_size = cpp_get_test_singleton_critical_mutex_size();
expect_eq!(rust_critical_size, cpp_critical_size);
// Compare raw mutex storage byte-by-byte between Rust and C++.
let cpp_raw_ptr = cpp_get_test_singleton_mutex_raw_lock_ptr();
let cpp_raw_size = cpp_get_test_singleton_mutex_raw_lock_size();
let cpp_raw_bytes = core::slice::from_raw_parts(cpp_raw_ptr as *const u8, cpp_raw_size);
let rust_raw_bytes = lock1.raw_mutex().raw_storage_slice();
expect_true!(rust_raw_bytes == cpp_raw_bytes);
// Compare CriticalMutex raw storage byte-by-byte between Rust and C++.
let cpp_critical_raw_ptr = cpp_get_test_singleton_critical_mutex_raw_lock_ptr();
let cpp_critical_raw_size = cpp_get_test_singleton_critical_mutex_raw_lock_size();
let cpp_critical_raw_bytes = core::slice::from_raw_parts(
cpp_critical_raw_ptr as *const u8,
cpp_critical_raw_size,
);
let rust_critical_raw_bytes =
TestKernelSingletonCriticalMutex::Get().raw_mutex().raw_storage_slice();
expect_true!(rust_critical_raw_bytes == cpp_critical_raw_bytes);
}
{
ksync::lock!(let _guard = TestKernelSingletonMutex::Get().lock());
}
{
ksync::lock!(TestKernelSingletonMutex::lock());
}
{
ksync::lock!(let _guard = TestKernelSingletonCriticalMutex::Get().lock());
}
// Verify that a singleton spinlock registers LOCK_FLAGS_IRQ_SAFE so it can be acquired
// while holding another IRQ-safe spinlock without triggering a lockdep violation.
stack_pin_init!(let outer = pin_init!(GuardedSpinlockObj {
mu <- ksync::KMutex::init(),
value: 0.into(),
}));
{
ksync::lock!(let _outer_guard = outer.lock_mu());
ksync::lock!(let _inner_guard = TestKernelSingletonSpinlock::lock());
}
}
declare_singleton_monitored_spinlock!(TestKernelCpuAlignedSingletonSpinlock, cpu_align);
declare_singleton_monitored_spinlock!(
TestKernelCpuAlignExclusiveSingletonSpinlock,
cpu_align_exclusive,
);
/// test that `cpu_align` and `cpu_align_exclusive` place a singleton lock on its own cache line
#[test]
fn singleton_aligned() {
let lock = TestKernelCpuAlignedSingletonSpinlock::get();
expect_true!(core::ptr::from_ref(lock).addr().is_multiple_of(64));
{
ksync::lock!(TestKernelCpuAlignedSingletonSpinlock::lock(ksync::source_tag!()));
}
let lock = TestKernelCpuAlignExclusiveSingletonSpinlock::get();
expect_true!(core::ptr::from_ref(lock).addr().is_multiple_of(64));
{
ksync::lock!(TestKernelCpuAlignExclusiveSingletonSpinlock::lock(ksync::source_tag!()));
}
// `kernel.ld` places `.data.cpu_align_exclusive` at the start of `.data`, before the lock
// class registrations.
unsafe extern "C" {
static __data_start: u8;
static __start_rust_lock_classes: u8;
}
let addr = core::ptr::from_ref(lock).addr();
expect_true!(addr >= (&raw const __data_start).addr());
expect_true!(addr < (&raw const __start_rust_lock_classes).addr());
}
/// test KOnceCell lazy initialization and access
#[test]
fn konce_cell_test() {
static TEST_CELL: ksync::KOnceCell<u64, TestKernelSingletonMutex> = ksync::KOnceCell::new();
ksync::lock!(let mut guard = TestKernelSingletonMutex::lock());
// SAFETY: `guard` holds the singleton lock `TestKernelSingletonMutex` guarding `TEST_CELL`.
unsafe {
let token = guard.token();
expect_true!(!TEST_CELL.is_initialized(token));
expect_true!(TEST_CELL.get(token).is_none());
let token_mut = guard.as_mut().token_mut();
let val = TEST_CELL.get_or_init(|| 12345u64, token_mut);
expect_eq!(*val, 12345);
let token = guard.token();
expect_true!(TEST_CELL.is_initialized(token));
expect_true!(TEST_CELL.get(token) == Some(&12345));
let unchk = TEST_CELL.get_unchecked();
expect_true!(unchk == Some(&12345));
}
}
/// test KOnceCell integration with #[ksync::guarded]
#[test]
fn guarded_konce_cell_test() {
stack_pin_init!(let obj = pin_init!(GuardedOnceObj {
mu <- ksync::KMutex::init(),
once_val: ksync::KOnceCell::new(),
}));
// Before init: read accessor returns None
{
ksync::lock!(let guard = obj.lock_mu());
expect_true!(guard.once_val().is_none());
}
// Initialize through mutable guard
{
ksync::lock!(let mut guard = obj.lock_mu());
let mut once_cell = guard.as_mut().once_val_cell();
let val = once_cell.get_or_init(|| 42);
expect_eq!(*val, 42);
// Read after init
expect_true!(guard.once_val() == Some(&42));
expect_true!(guard.as_mut().once_val_mut() == Some(&mut 42));
// Attempting to set fails because it's already initialized
expect_true!(guard.as_mut().once_val_cell().set(99) == Err(99));
// Test proxy cell access
expect_true!(guard.as_mut().once_val_cell().get() == Some(&42));
// Disjoint fields access
let fields = guard.fields();
expect_true!(fields.once_val == Some(&42));
let fields_mut = guard.as_mut().fields_mut();
expect_true!(fields_mut.once_val == Some(&mut 42));
}
}
#[ksync::guarded]
struct GuardedMultiSpinlockObj {
#[mutex(flags = lockdep::LOCK_FLAGS_MULTI_ACQUIRE)]
mu: ksync::KMutex<TrackedSpinlock>,
#[guarded_by(mu)]
value: u32,
}
/// test explicit release() and KMutexGuardMultiple in kernel
#[test]
fn guard_multiple_and_release_test() {
stack_pin_init!(let s1 = pin_init!(GuardedMultiSpinlockObj {
mu <- ksync::KMutex::init(),
value: 10.into(),
}));
stack_pin_init!(let s2 = pin_init!(GuardedMultiSpinlockObj {
mu <- ksync::KMutex::init(),
value: 20.into(),
}));
// Test single guard early release
{
ksync::lock!(let mut g = s1.lock_mu());
*g.as_mut().value_mut() = 11;
g.as_mut().release();
ksync::lock!(let g2 = s1.lock_mu());
expect_eq!(*g2.value(), 11);
}
// Test multi-lock in [s1, s2] and [s2, s1] order with early release
{
ksync::lock!(
let mut guards = GuardedMultiSpinlockObj::lock_mu_multiple([&*s1, &*s2])
);
let [t1, t2] = guards.as_mut().tokens_mut();
// SAFETY: Both `s1.mu` and `s2.mu` are held by `guards`, and `s1` and `s2` are
// distinct instances.
unsafe {
*s1.guard_mu_mut(t1).value_mut() = 100;
*s2.guard_mu_mut(t2).value_mut() = 200;
}
}
{
ksync::lock!(
let mut guards = GuardedMultiSpinlockObj::lock_mu_multiple([&*s2, &*s1])
);
let [t1, t2] = guards.tokens();
// SAFETY: Both `s1.mu` and `s2.mu` are held by `guards`.
unsafe {
expect_eq!(*s1.guard_mu(t1).value(), 100);
expect_eq!(*s2.guard_mu(t2).value(), 200);
}
guards.as_mut().release();
ksync::lock!(let g1 = s1.lock_mu());
ksync::lock!(let g2 = s2.lock_mu());
expect_eq!(*g1.value(), 100);
expect_eq!(*g2.value(), 200);
}
}
#[ksync::guarded]
struct GuardedMultiLockObj {
#[mutex]
mu1: ksync::KMutex<TrackedMutex>,
#[mutex]
mu2: ksync::KMutex<TrackedMutex>,
#[guarded_by(mu1)]
val1: u32,
#[guarded_by(mu2)]
val2: u32,
#[guarded_by(mu1, mu2)]
shared: u32,
}
/// test #[guarded_by(mu1, mu2)] in kernel
#[test]
fn guarded_multi_lock_test() {
stack_pin_init!(let obj = pin_init!(GuardedMultiLockObj {
mu1 <- ksync::KMutex::init(),
mu2 <- ksync::KMutex::init(),
val1: 1.into(),
val2: 2.into(),
shared: 10.into(),
}));
{
ksync::lock!(let g1 = obj.lock_mu1());
expect_eq!(*g1.val1(), 1);
expect_eq!(*g1.shared(), 10);
}
{
ksync::lock!(let g2 = obj.lock_mu2());
expect_eq!(*g2.val2(), 2);
expect_eq!(*g2.shared(), 10);
}
{
ksync::lock!(let mut g1 = obj.lock_mu1());
ksync::lock!(let mut g2 = obj.lock_mu2());
*g1.as_mut().shared_mut(&mut g2) = 20;
expect_eq!(*g1.shared(), 20);
expect_eq!(*g2.shared(), 20);
*g2.as_mut().shared_mut(&mut g1) = 30;
expect_eq!(*g1.shared(), 30);
}
}
/// test brwlock with custom flags attribute (e.g. LOCK_FLAGS_MULTI_ACQUIRE)
#[test]
fn guarded_brwlock_multi_acquire_test() {
stack_pin_init!(let obj = pin_init!(GuardedBrwLockMultiAcquireObj {
lock <- ksync::BrwLockPi::init(),
value: 42.into(),
}));
ksync::lock!(let guard = obj.read_lock());
expect_eq!(*guard.value(), 42);
}
#[ksync::guarded]
struct GuardedGenericLockObj<L: ksync::RawLock> {
#[mutex]
mu: ksync::KMutex<L>,
#[guarded_by(mu)]
value: u32,
}
/// test #[ksync::guarded] with generic lock types
#[test]
fn generic_lock_guarded_test() {
stack_pin_init!(let mutex_obj = pin_init!(GuardedGenericLockObj::<TrackedMutex> {
mu <- ksync::KMutex::init(),
value: 10.into(),
}));
stack_pin_init!(let crit_obj = pin_init!(GuardedGenericLockObj::<TrackedCriticalMutex> {
mu <- ksync::KMutex::init(),
value: 20.into(),
}));
stack_pin_init!(let spin_obj = pin_init!(GuardedGenericLockObj::<TrackedSpinlock> {
mu <- ksync::KMutex::init(),
value: 30.into(),
}));
stack_pin_init!(
let mon_obj = pin_init!(GuardedGenericLockObj::<TrackedMonitoredSpinlock> {
mu <- ksync::KMutex::init(),
value: 40.into(),
})
);
// SAFETY: All four objects have initialized `KMutex` instances of the corresponding raw
// lock types.
unsafe {
assert_true!(cpp_verify_mutex_id(
core::ptr::from_ref(&mutex_obj.mu).cast::<core::ffi::c_void>(),
<GuardedGenericLockObjMuClass<TrackedMutex> as ksync::LockClass>::ID,
));
assert_true!(cpp_verify_critical_mutex_id(
core::ptr::from_ref(&crit_obj.mu).cast::<core::ffi::c_void>(),
<GuardedGenericLockObjMuClass<TrackedCriticalMutex> as ksync::LockClass>::ID,
));
assert_true!(cpp_verify_spinlock_id(
core::ptr::from_ref(&spin_obj.mu).cast::<core::ffi::c_void>(),
<GuardedGenericLockObjMuClass<TrackedSpinlock> as ksync::LockClass>::ID,
));
assert_true!(cpp_verify_monitored_spinlock_id(
core::ptr::from_ref(&mon_obj.mu).cast::<core::ffi::c_void>(),
<GuardedGenericLockObjMuClass<TrackedMonitoredSpinlock> as ksync::LockClass>::ID,
));
}
// Verify that IRQ-safe and non-IRQ-safe instantiations select distinct LockClass IDs
// when lockdep metadata/validation is enabled.
if !<GuardedGenericLockObjMuClass<TrackedMutex> as ksync::LockClass>::ID.is_null() {
expect_true!(
<GuardedGenericLockObjMuClass<TrackedMutex> as ksync::LockClass>::ID
!= <GuardedGenericLockObjMuClass<TrackedSpinlock> as ksync::LockClass>::ID
);
} else {
expect_true!(
<GuardedGenericLockObjMuClass<TrackedSpinlock> as ksync::LockClass>::ID.is_null()
);
}
{
ksync::lock!(let mut g = mutex_obj.lock_mu());
expect_eq!(*g.value(), 10);
*g.as_mut().value_mut() = 11;
expect_eq!(*g.value(), 11);
}
{
ksync::lock!(let mut g = crit_obj.lock_mu());
expect_eq!(*g.value(), 20);
*g.as_mut().value_mut() = 21;
expect_eq!(*g.value(), 21);
}
// Verify that GuardedGenericLockObj<TrackedSpinlock> registers LOCK_FLAGS_IRQ_SAFE so it
// can be acquired while holding another IRQ-safe spinlock without triggering a lockdep
// violation.
stack_pin_init!(let outer = pin_init!(GuardedSpinlockObj {
mu <- ksync::KMutex::init(),
value: 0.into(),
}));
{
ksync::lock!(let _outer_guard = outer.lock_mu());
ksync::lock!(let mut g = spin_obj.lock_mu());
expect_eq!(*g.value(), 30);
*g.as_mut().value_mut() = 31;
expect_eq!(*g.value(), 31);
}
{
ksync::lock!(let mut g = mon_obj.lock_mu_with(ksync::source_tag!()));
expect_eq!(*g.value(), 40);
*g.as_mut().value_mut() = 41;
expect_eq!(*g.value(), 41);
}
}
/// test pinned Adoptable and KMutexGuard adoption and move
#[test]
fn adoptable_guard_test() {
type MutexAdoptable<'a> = ksync::Adoptable<'a, GuardedMutexObjMuClass, TrackedMutex>;
type MutexGuard<'a> = ksync::KMutexGuard<'a, GuardedMutexObjMuClass, TrackedMutex>;
expect_eq!(
core::mem::size_of::<MutexAdoptable<'_>>(),
core::mem::size_of::<MutexGuard<'_>>()
);
expect_eq!(
core::mem::align_of::<MutexAdoptable<'_>>(),
core::mem::align_of::<MutexGuard<'_>>()
);
let Ok(obj) = fbl::pin_make_ref_counted!(GuardedMutexObj {
mu <- ksync::KMutex::init(),
value: 5.into(),
}) else {
assert_true!(false);
return false;
};
fn adopt_and_increment(
obj: &GuardedMutexObj,
adoptable: core::pin::Pin<&mut MutexAdoptable<'_>>,
) -> (bool, bool, u32) {
ksync::lock!(let mut guard = obj.mu.adopt(adoptable));
let is_locked = guard.is_locked_by(&obj.mu);
let ptr_eq = guard.mutex().is_some_and(|m| core::ptr::eq(m, &obj.mu));
let mut tg = obj.guard_mu_mut_from_guard(guard.as_mut());
let prev = *tg.value();
*tg.reborrow().value_mut() = 6;
guard.as_mut().call_untracked(|token| {
// SAFETY: `token` witnesses exclusive access to `obj.mu` on `obj`.
*unsafe { obj.guard_mu_mut(token) }.value_mut() = 7;
});
(is_locked, ptr_eq, prev)
}
{
ksync::lock!(let mut a1 = obj.mu.lock_adoptable());
expect_true!(a1.is_locked_by(&obj.mu));
ksync::lock!(let mut a2 = ksync::Adoptable::empty());
expect_true!(a2.is_none());
a2.as_mut().move_from(a1.as_mut());
expect_true!(a1.is_none());
expect_true!(a2.is_locked_by(&obj.mu));
let (is_locked, ptr_eq, prev) = adopt_and_increment(&obj, a2.as_mut());
expect_true!(is_locked);
expect_true!(ptr_eq);
expect_eq!(prev, 5);
expect_true!(a2.is_none());
expect_true!(!obj.mu.raw_mutex().is_held());
}
// Exercise `lock_adoptable_with`, `KMutexGuard::take`, `Adoptable::take_from`,
// `KMutexGuard::adopt_from`, `KMutexGuard::move_from`, and `MutexSpinMaxDurationPolicy`
// (`call_unlocked`).
{
ksync::lock!(let guard_with = obj.mu.lock_adoptable_with(()));
expect_true!(guard_with.mutex().is_some_and(|m| core::ptr::eq(m, &obj.mu)));
}
{
ksync::lock!(let mut g1 = obj.mu.lock());
ksync::lock!(let mut taken = g1.as_mut().take());
expect_true!(g1.is_none());
expect_true!(taken.is_locked_by(&obj.mu));
ksync::lock!(let mut g2 = ksync::KMutexGuard::adopt(taken.as_mut()));
expect_true!(taken.is_none());
expect_true!(g2.is_locked_by(&obj.mu));
g1.as_mut().move_from(g2.as_mut());
expect_true!(g2.is_none());
expect_true!(g1.is_locked_by(&obj.mu));
taken.as_mut().take_from(g1.as_mut());
expect_true!(g1.is_none());
expect_true!(taken.is_locked_by(&obj.mu));
g2.as_mut().adopt_from(taken.as_mut());
expect_true!(taken.is_none());
expect_true!(g2.is_locked_by(&obj.mu));
}
{
ksync::lock!(
let mut spin_guard =
obj.mu.lock_adoptable_policy_with::<MutexSpinMaxDurationPolicy>(10_000)
);
expect_true!(spin_guard.is_locked_by(&obj.mu));
let held = spin_guard.as_mut().call_unlocked(|| obj.mu.raw_mutex().is_held());
expect_true!(!held);
expect_true!(obj.mu.raw_mutex().is_held());
// SAFETY: `spin_guard.token()` witnesses `obj.mu` on `obj`.
expect_eq!(*unsafe { obj.guard_mu(spin_guard.token()) }.value(), 7);
}
// Exercise `NoIrqSavePolicy` on `TrackedMonitoredSpinlock` (IRQ-safe per-CPU
// `ThreadLockState` entry relocation and `SourceTag` `GuardState` preservation across
// `move_from` and `call_unlocked`).
stack_pin_init!(let outer_irq = pin_init!(GuardedSpinlockObj {
mu <- ksync::KMutex::init(),
value: 0.into(),
}));
stack_pin_init!(let mon_spin = pin_init!(GuardedMonitoredSpinlockObj {
mu <- ksync::KMutex::init(),
value: 50.into(),
}));
{
ksync::lock!(let _irq_guard = outer_irq.lock_mu());
ksync::lock!(
let mut sa1 = mon_spin
.mu
.lock_adoptable_policy_with::<NoIrqSavePolicy>(ksync::source_tag!())
);
ksync::lock!(let mut sa2 = ksync::Adoptable::empty());
sa2.as_mut().move_from(sa1.as_mut());
expect_true!(sa1.is_none());
expect_true!(sa2.is_locked_by(&mon_spin.mu));
sa2.as_mut().call_unlocked(|| {});
ksync::lock!(let mut sg = mon_spin.mu.adopt(sa2.as_mut()));
expect_true!(sa2.is_none());
let mut tg = mon_spin.guard_mu_mut_from_guard(sg.as_mut());
expect_eq!(*tg.value(), 50);
*tg.reborrow().value_mut() = 55;
}
// Exercise `TrackedCriticalMutex` and `lock_adoptable_policy::<CriticalMutexPolicy>()`.
stack_pin_init!(let crit = pin_init!(GuardedCriticalMutexObj {
mu <- ksync::KMutex::init(),
value: 11.into(),
}));
{
ksync::lock!(let mut crit_guard = crit.mu.lock_adoptable());
let held = crit_guard.as_mut().call_unlocked(|| false);
expect_true!(!held);
crit_guard.as_mut().call_untracked(|token| {
// SAFETY: `token` witnesses exclusive access to `crit.mu` on `crit`.
*unsafe { crit.guard_mu_mut(token) }.value_mut() = 12;
});
}
{
ksync::lock!(let cg = crit.lock_mu());
expect_eq!(*cg.value(), 12);
}
stack_pin_init!(let untracked = pin_init!(GuardedUntrackedMutexObj {
mu <- ksync::KMutex::init(),
value: 1.into(),
}));
{
ksync::lock!(
let mut uguard = untracked.mu.lock_adoptable_policy::<CriticalMutexPolicy>()
);
expect_true!(uguard.is_locked_by(&untracked.mu));
// SAFETY: `uguard.token_mut()` witnesses exclusive access to `untracked.mu`.
*unsafe { untracked.guard_mu_mut(uguard.as_mut().token_mut()) }.value_mut() = 2;
}
stack_pin_init!(let a = pin_init!(GuardedNestableCriticalMutexObj {
mu <- ksync::KMutex::init(),
value: 10.into(),
}));
stack_pin_init!(let b = pin_init!(GuardedNestableCriticalMutexObj {
mu <- ksync::KMutex::init(),
value: 20.into(),
}));
{
ksync::lock!(
let mut ga = a.mu.lock_adoptable_policy_with::<CriticalMutexOrderedPolicy>(0)
);
ksync::lock!(
let mut gb = b.mu.lock_adoptable_policy_with::<CriticalMutexOrderedPolicy>(1)
);
ksync::lock!(let mut moved_gb = ksync::Adoptable::from_adoptable(gb.as_mut()));
expect_true!(gb.is_none());
moved_gb.as_mut().call_unlocked(|| {});
// SAFETY: The tokens witness `a.mu` on `a` and `b.mu` on `b` respectively.
let sum =
unsafe { *a.guard_mu(ga.token()).value() + *b.guard_mu(moved_gb.token()).value() };
// SAFETY: As above, with exclusive access.
*unsafe { b.guard_mu_mut(moved_gb.as_mut().token_mut()) }.value_mut() = sum;
}
{
ksync::lock!(let gb = b.lock_mu());
expect_eq!(*gb.value(), 30);
}
// SAFETY: FFI call to query the thread-local lockdep test result.
if unsafe { cpp_ksync_test_lock_validation_enabled() } {
// SAFETY: FFI call to query the thread-local lockdep test result.
expect_eq!(unsafe { cpp_ksync_test_lockdep_last_result() }, LOCKDEP_RESULT_SUCCESS);
struct LockdepExceptBlockGuard;
impl Drop for LockdepExceptBlockGuard {
fn drop(&mut self) {
kprintln!("[===KSYNC LOCKDEP EXCEPT BLOCK END===]");
}
}
kprintln!("[===KSYNC LOCKDEP EXCEPT BLOCK START===]");
let _except_block = LockdepExceptBlockGuard;
stack_pin_init!(let na = pin_init!(GuardedNonFatalNestableCriticalMutexObj {
mu <- ksync::KMutex::init(),
value: 0.into(),
}));
stack_pin_init!(let nb = pin_init!(GuardedNonFatalNestableCriticalMutexObj {
mu <- ksync::KMutex::init(),
value: 0.into(),
}));
// Verify descending orders (`na` at 1, then `nb` at 0) are caught for `Adoptable`
// even after moving the first guard into a new pinned slot before acquiring the
// second.
{
ksync::lock!(
let mut a_na =
na.mu.lock_adoptable_policy_with::<CriticalMutexOrderedPolicy>(1)
);
ksync::lock!(let _moved_na = ksync::Adoptable::from_adoptable(a_na.as_mut()));
ksync::lock!(
let _nb = nb.mu.lock_adoptable_policy_with::<CriticalMutexOrderedPolicy>(0)
);
// SAFETY: FFI call to query the thread-local lockdep test result.
let res = unsafe { cpp_ksync_test_lockdep_last_result() };
expect_eq!(res, LOCKDEP_RESULT_INVALID_NESTING);
}
}
}
}