blob: d8ddee3d2c622b76b6dc70d8bbf24a8de9fc3398 [file]
// Copyright 2017 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT
//! Hypervisor virtual interrupt bitmap and tracker (`<hypervisor/interrupt_tracker.h>`).
use crate::ksync_kernel::TrackedSpinlock;
use bitmap::{Bitmap, FixedStorage, RawBitmapGeneric};
use core::convert::Infallible;
use core::mem::{align_of, size_of};
use ksync::{KCell, KMutex};
use pin_init::{Init, PinInit, init, pin_init};
use zr::static_assert;
use zx_status::Status;
use super::ktrace::{VcpuBlockOp, VcpuMeta, ktrace_vcpu};
use super::state_invalidator::StateInvalidator;
use crate::kernel::deadline::Deadline;
use crate::kernel::event::AutounsignalEvent;
use crate::platform_rs::timer::InstantMono;
/// Default interrupt vector count for [`InterruptBitmap`] and [`InterruptTracker`].
const DEFAULT_INTERRUPT_VECTORS: usize = 256;
/// Returns the number of `usize` words needed to store `n` interrupt bits.
pub const fn bitmap_words(n: usize) -> usize {
n.div_ceil(usize::BITS as usize)
}
/// Bitmap of `N` interrupt vectors (`hypervisor::InterruptBitmap<N>`), backed by `WORDS` words of
/// inline storage (`WORDS == bitmap_words(N)`).
#[derive(Debug)]
#[repr(C)]
pub struct InterruptBitmap<
const N: usize = DEFAULT_INTERRUPT_VECTORS,
const WORDS: usize = { bitmap_words(DEFAULT_INTERRUPT_VECTORS) },
> {
bitmap: RawBitmapGeneric<FixedStorage<WORDS>>,
}
static_assert!(size_of::<InterruptBitmap<8, 1>>() == 16);
static_assert!(align_of::<InterruptBitmap<8, 1>>() == 8);
static_assert!(size_of::<InterruptBitmap>() == 40);
static_assert!(align_of::<InterruptBitmap>() == 8);
impl<const N: usize, const WORDS: usize> Default for InterruptBitmap<N, WORDS> {
fn default() -> Self {
Self::new()
}
}
impl<const N: usize, const WORDS: usize> InterruptBitmap<N, WORDS> {
/// Creates an in-place initializer for a cleared `InterruptBitmap<N, WORDS>`.
pub fn init() -> impl Init<Self, Infallible> {
const {
assert!(N <= u32::MAX as usize, "N must fit in u32");
assert!(WORDS == bitmap_words(N), "WORDS must equal N.div_ceil(usize::BITS)");
}
init!(Self { bitmap <- RawBitmapGeneric::<FixedStorage<WORDS>>::init_fixed(N) })
}
/// Creates a new cleared `InterruptBitmap<N, WORDS>`.
pub fn new() -> Self {
const {
assert!(N <= u32::MAX as usize, "N must fit in u32");
assert!(WORDS == bitmap_words(N), "WORDS must equal N.div_ceil(usize::BITS)");
}
let mut bitmap = RawBitmapGeneric::default();
let result = bitmap.reset(N);
// `bitmap` uses static storage, so `reset` cannot fail.
debug_assert_eq!(result, Ok(()));
Self { bitmap }
}
/// Returns whether `vector` (where `vector < N`) is set in the bitmap.
#[inline]
pub fn get(&self, vector: u32) -> bool {
self.bitmap.get_one(vector as usize)
}
/// Sets `vector` in the bitmap.
#[inline]
pub fn set(&mut self, vector: u32) {
// TODO(https://fxbug.dev/572853263): Slice `RawBitmapGeneric` storage directly and add
// `#[inline]` single-word `set_one` / `clear_one` specializations (also used in `try_pop`
// and `pop`) to inline single-bit updates without iterator spills.
let _ = self.bitmap.set_one(vector as usize);
}
/// Clears all vectors in `[min, max)`.
#[inline]
pub fn clear(&mut self, min: u32, max: u32) {
let _ = self.bitmap.clear(min as usize, max as usize);
}
/// Scans for the highest-priority pending interrupt vector.
#[inline]
pub fn scan(&self) -> Option<u32> {
#[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))]
let bitoff = self.bitmap.scan(0, N, false);
#[cfg(target_arch = "x86_64")]
let bitoff = self.bitmap.reverse_scan(0, N, false);
#[cfg(not(any(target_arch = "aarch64", target_arch = "riscv64", target_arch = "x86_64")))]
compile_error!("Unsupported architecture for InterruptBitmap::scan");
bitoff.map(|off| off as u32)
}
}
/// `N` is the maximum number of interrupts to be tracked.
// Note: `#[ksync::guarded]` expands to `#[pin_init::pin_data]`, so `#[pin]` on `event` makes
// `InterruptTracker` `!Unpin` and structurally pins `event`.
#[ksync::guarded]
#[repr(C)]
pub struct InterruptTracker<
const N: usize = DEFAULT_INTERRUPT_VECTORS,
const WORDS: usize = { bitmap_words(DEFAULT_INTERRUPT_VECTORS) },
> {
#[pin]
event: AutounsignalEvent,
#[mutex]
lock: KMutex<TrackedSpinlock>,
#[guarded_by(lock)]
bitmap: InterruptBitmap<N, WORDS>,
}
static_assert!(
size_of::<InterruptTracker>()
== size_of::<AutounsignalEvent>()
+ size_of::<TrackedSpinlock>().next_multiple_of(8)
+ size_of::<InterruptBitmap>()
);
static_assert!(align_of::<InterruptTracker>() == 8);
impl<const N: usize, const WORDS: usize> InterruptTracker<N, WORDS> {
/// Creates an in-place pinned initializer for `InterruptTracker<N, WORDS>`.
pub fn init() -> impl PinInit<Self, Infallible> {
pin_init!(Self {
event <- AutounsignalEvent::init_unsignaled(),
lock <- KMutex::init(),
bitmap <- KCell::init(InterruptBitmap::<N, WORDS>::init()),
})
}
/// Returns whether there are pending interrupts.
#[inline]
pub fn pending(&self) -> bool {
ksync::lock!(let guard = self.lock_lock());
guard.bitmap().scan().is_some()
}
/// Clears all vectors in the range `[min, max)`.
#[inline]
pub fn clear(&self, min: u32, max: u32) {
ksync::lock!(let mut guard = self.lock_lock());
guard.as_mut().bitmap_mut().clear(min, max);
}
/// Pops the specified vector, if it is pending.
#[inline]
pub fn try_pop(&self, vector: u32) -> bool {
// TODO(https://fxbug.dev/572853263): Once x86 `vcpu.cc` is ported to Rust, combine the
// `X86_INT_NMI` check and priority `pop` under one spinlock acquisition in `Vcpu::Enter`.
if (vector as usize) >= N {
core::hint::cold_path();
return false;
}
ksync::lock!(let mut guard = self.lock_lock());
if guard.bitmap().get(vector) {
guard.as_mut().bitmap_mut().clear(vector, vector + 1);
return true;
}
false
}
/// Pops the highest priority interrupt.
#[inline]
pub fn pop(&self) -> Option<u32> {
ksync::lock!(let mut guard = self.lock_lock());
if let Some(vector) = guard.bitmap().scan() {
guard.as_mut().bitmap_mut().clear(vector, vector + 1);
return Some(vector);
}
None
}
/// Tracks the given interrupt.
#[inline]
pub fn track(&self, vector: u32) -> Result<(), Status> {
if (vector as usize) >= N {
core::hint::cold_path();
return Err(Status::OUT_OF_RANGE);
}
ksync::lock!(let mut guard = self.lock_lock());
guard.as_mut().bitmap_mut().set(vector);
Ok(())
}
/// Tracks the compile-time validated interrupt `VECTOR`.
#[inline]
pub fn track_const<const VECTOR: u32>(&self) {
const {
assert!((VECTOR as usize) < N, "Vector out of bounds");
}
let _ = self.track(VECTOR);
}
/// Tracks the given interrupt, and signals any waiters.
#[inline]
pub fn interrupt(&self, vector: u32) -> Result<(), Status> {
let result = self.track(vector);
if result.is_ok() {
self.event.signal();
}
result
}
/// Tracks the compile-time validated interrupt `VECTOR`, and signals any waiters.
#[inline]
pub fn interrupt_const<const VECTOR: u32>(&self) {
const {
assert!((VECTOR as usize) < N, "Vector out of bounds");
}
let _ = self.interrupt(VECTOR);
}
/// Cancels a wait for an interrupt.
///
/// We signal [`Status::INTERRUPTED_RETRY`] (`ZX_ERR_INTERNAL_INTR_RETRY`), so that if the
/// status is propagated to the syscall-layer, we will retry the syscall.
#[inline]
pub fn cancel(&self) {
self.event.signal_result(Err(Status::INTERRUPTED_RETRY));
}
/// Waits for an interrupt, optionally invalidating `invalidator` before blocking.
#[inline]
pub fn wait(
&self,
deadline: InstantMono,
invalidator: Option<&mut StateInvalidator>,
) -> Result<(), Status> {
if let Some(inv) = invalidator {
inv.invalidate();
}
ktrace_vcpu(VcpuBlockOp::Block, VcpuMeta::Interrupt);
let _defer = zr::defer(|| ktrace_vcpu(VcpuBlockOp::Unblock, VcpuMeta::Interrupt));
loop {
match self.event.wait(&Deadline::no_slack(deadline.into())) {
Ok(()) => {}
// If the event timed out, return `Ok(())` to resume the VCPU.
Err(Status::TIMED_OUT) => return Ok(()),
// Otherwise, return the status.
Err(status) => {
core::hint::cold_path();
return Err(status);
}
}
if self.pending() {
break;
}
}
Ok(())
}
}
/// Unit tests for `InterruptBitmap` and `InterruptTracker`.
#[cfg(ktest)]
#[unittest::suite(name = "interrupt_tracker_rust")]
mod tests {
// Note: `#[unittest::suite]` automatically injects `use super::*;`.
use pin_init::stack_pin_init;
use unittest::{expect_eq, expect_err, expect_false, expect_ok, expect_true, unwrap_some};
/// Verifies `InterruptBitmap` set, get, clear, and scan operations.
#[test]
fn interrupt_bitmap() {
let mut bitmap = InterruptBitmap::<8, 1>::new();
expect_false!(bitmap.get(0));
expect_false!(bitmap.get(1));
expect_true!(bitmap.scan().is_none());
// Index 0.
bitmap.set(0);
expect_true!(bitmap.get(0));
expect_false!(bitmap.get(1));
expect_eq!(unwrap_some!(bitmap.scan()), 0);
bitmap.clear(0, 1);
expect_false!(bitmap.get(0));
expect_false!(bitmap.get(1));
expect_true!(bitmap.scan().is_none());
// Index 1.
bitmap.set(1);
expect_false!(bitmap.get(0));
expect_true!(bitmap.get(1));
expect_eq!(unwrap_some!(bitmap.scan()), 1);
bitmap.clear(1, 2);
expect_false!(bitmap.get(0));
expect_false!(bitmap.get(1));
expect_true!(bitmap.scan().is_none());
// Clear
bitmap.set(0);
bitmap.set(1);
bitmap.set(2);
bitmap.set(3);
bitmap.clear(1, 3);
expect_true!(bitmap.get(0));
expect_false!(bitmap.get(1));
expect_false!(bitmap.get(2));
expect_true!(bitmap.get(3));
}
/// Verifies `InterruptTracker` track, pending, and pop operations.
#[test]
fn interrupt_tracker_track_and_pop() {
stack_pin_init!(let tracker = InterruptTracker::<8, 1>::init());
expect_false!(tracker.pending());
expect_true!(tracker.pop().is_none());
expect_err!(tracker.track(8), Status::OUT_OF_RANGE);
expect_ok!(tracker.track(2));
expect_ok!(tracker.track(5));
expect_true!(tracker.pending());
expect_true!(tracker.try_pop(2));
expect_false!(tracker.try_pop(2));
expect_false!(tracker.try_pop(8));
// Re-track 2 to verify priority pop ordering with multiple items.
expect_ok!(tracker.track(2));
#[cfg(target_arch = "x86_64")]
{
expect_eq!(unwrap_some!(tracker.pop()), 5);
expect_eq!(unwrap_some!(tracker.pop()), 2);
}
#[cfg(not(target_arch = "x86_64"))]
{
expect_eq!(unwrap_some!(tracker.pop()), 2);
expect_eq!(unwrap_some!(tracker.pop()), 5);
}
expect_false!(tracker.pending());
}
}