zircon/kernel/arch/x86/spinlock.cc

// Copyright 2019 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT

#include <lib/arch/intrin.h>

#include <arch/arch_ops.h>
#include <arch/spinlock.h>
#include <kernel/spin_tracing.h>

namespace {

void assert_lock_held(arch_spin_lock_t *lock) TA_ASSERT(lock) {}

inline void arch_spin_lock_core(arch_spin_lock_t *lock, cpu_num_t val) TA_ACQ(lock) {
  cpu_num_t expected = 0;
  while (unlikely(!lock->value.compare_exchange_weak(expected, val, ktl::memory_order_acquire,
                                                     ktl::memory_order_relaxed))) {
    expected = 0;
    do {
      arch::Yield();
    } while (unlikely(lock->value.load(ktl::memory_order_relaxed) != 0));
  }

  assert_lock_held(lock);
}

}  // namespace

void arch_spin_lock_non_instrumented(arch_spin_lock_t *lock) TA_ACQ(lock) {
  struct x86_percpu *percpu = x86_get_percpu();
  cpu_num_t val = percpu->cpu_num + 1;
  arch_spin_lock_core(lock, val);
  percpu->num_spinlocks++;
}

void arch_spin_lock_trace_instrumented(arch_spin_lock_t *lock,
                                       spin_tracing::EncodedLockId encoded_lock_id) TA_ACQ(lock) {
  struct x86_percpu *percpu = x86_get_percpu();
  cpu_num_t val = percpu->cpu_num + 1;

  // If this lock acquisition is trace instrumented, try to obtain the lock once
  // before we decide that we need to spin and produce spin trace events.
  cpu_num_t expected = 0;
  lock->value.compare_exchange_weak(expected, val, ktl::memory_order_acquire,
                                    ktl::memory_order_relaxed);
  if (expected == 0) {
    percpu->num_spinlocks++;
    assert_lock_held(lock);
    return;
  }

  spin_tracing::Tracer<true> spin_tracer;
  arch_spin_lock_core(lock, val);
  spin_tracer.Finish(spin_tracing::FinishType::kLockAcquired, encoded_lock_id);

  percpu->num_spinlocks++;
}

bool arch_spin_trylock(arch_spin_lock_t *lock) TA_NO_THREAD_SAFETY_ANALYSIS {
  struct x86_percpu *percpu = x86_get_percpu();
  cpu_num_t val = percpu->cpu_num + 1;

  cpu_num_t expected = 0;

  lock->value.compare_exchange_strong(expected, val, ktl::memory_order_acquire,
                                      ktl::memory_order_relaxed);
  if (expected == 0) {
    percpu->num_spinlocks++;
  }

  return expected != 0;
}

void arch_spin_unlock(arch_spin_lock_t *lock) TA_NO_THREAD_SAFETY_ANALYSIS {
  x86_get_percpu()->num_spinlocks--;
  lock->value.store(0, ktl::memory_order_release);
}