zircon/kernel/object/msi_interrupt_dispatcher.cc - fuchsia - Git at Google

 // Copyright 2019 The Fuchsia Authors
 //
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file or at
 // https://opensource.org/licenses/MIT

 #include <lib/arch/intrin.h>
 #include <lib/counters.h>
 #include <lib/fit/defer.h>
 #include <platform.h>
 #include <reg.h>
 #include <trace.h>
 #include <zircon/errors.h>
 #include <zircon/limits.h>
 #include <zircon/rights.h>
 #include <zircon/types.h>

 #include <array>
 #include <cstring>
 #include <limits>

 #include <dev/interrupt.h>
 #include <fbl/alloc_checker.h>
 #include <kernel/auto_lock.h>
 #include <object/dispatcher.h>
 #include <object/interrupt_dispatcher.h>
 #include <object/msi_interrupt_dispatcher.h>
 #include <vm/vm_address_region.h>
 #include <vm/vm_object.h>

 #define LOCAL_TRACE 0

 KCOUNTER(dispatcher_msi_create_count, "msi_dispatcher.create")
 KCOUNTER(dispatcher_msi_interrupt_count, "msi_dispatcher.interrupts")
 KCOUNTER(dispatcher_msi_mask_count, "msi_dispatcher.mask")
 KCOUNTER(dispatcher_msi_unmask_count, "msi_dispatcher.unmask")
 KCOUNTER(dispatcher_msi_destroy_count, "msi_dispatcher.destroy")

 // Creates an a derived MsiInterruptDispatcher determined by the flags passed in
 zx_status_t MsiInterruptDispatcher::Create(fbl::RefPtr<MsiAllocation> alloc, uint32_t msi_id,
                                            const fbl::RefPtr<VmObject>& vmo, zx_paddr_t vmo_offset,
                                            uint32_t options, zx_rights_t* out_rights,
                                            KernelHandle<InterruptDispatcher>* out_interrupt,
                                            RegisterIntFn register_int_fn) {
   LTRACEF(
       "out_rights = %p, out_interrupt = %p\nvmo: %s, %s, %s\nsize = %lu, "
       "vmo_offset = %lu, options = %#x, cache policy = %u\n",
       out_rights, out_interrupt, vmo->is_paged() ? "paged" : "physical",
       vmo->is_contiguous() ? "contiguous" : "not contiguous",
       vmo->is_resizable() ? "resizable" : "not resizable", vmo->size(), vmo_offset, options,
       vmo->GetMappingCachePolicy());

   bool is_msix = (options & ZX_MSI_MODE_MSI_X) == ZX_MSI_MODE_MSI_X;
   options &= ~ZX_MSI_MODE_MSI_X;

   if (!out_rights || !out_interrupt ||
       (vmo->is_paged() && (vmo->is_resizable() || !vmo->is_contiguous())) ||
       vmo_offset >= vmo->size() || options ||
       vmo->GetMappingCachePolicy() != ZX_CACHE_POLICY_UNCACHED_DEVICE) {
     return ZX_ERR_INVALID_ARGS;
   }

   uint32_t base_irq_id = 0;
   {
     Guard<SpinLock, IrqSave> guard{&alloc->lock()};
     if (msi_id >= alloc->block().num_irq) {
       LTRACEF("msi_id %u is out of range for the block (num_irqs: %u)\n", msi_id,
               alloc->block().num_irq);
       return ZX_ERR_INVALID_ARGS;
     }
     base_irq_id = alloc->block().base_irq_id;
   }

   zx_status_t st = alloc->ReserveId(msi_id);
   if (st != ZX_OK) {
     LTRACEF("failed to reserve msi_id %u: %d\n", msi_id, st);
     return st;
   }
   auto cleanup = fit::defer([alloc, msi_id]() { alloc->ReleaseId(msi_id); });

   // To handle MSI masking we need to create a kernel mapping for the VMO handed
   // to us, this will provide access to the register controlling the given MSI.
   // The VMO must be a contiguous VMO with the cache policy already configured.
   // Size checks will come into play when we know what type of capability we're
   // working with.
   auto vmar = VmAspace::kernel_aspace()->RootVmar();
   uint32_t vector = base_irq_id + msi_id;
   ktl::array<char, ZX_MAX_NAME_LEN> name{};
   snprintf(name.data(), name.max_size(), "msi id %u (vector %u)", msi_id, vector);
   fbl::RefPtr<VmMapping> mapping;
   st = vmar->CreateVmMapping(0, vmo->size(), 0, 0, vmo, 0,
                              ARCH_MMU_FLAG_PERM_READ | ARCH_MMU_FLAG_PERM_WRITE, name.data(),
                              &mapping);
   if (st != ZX_OK) {
     LTRACEF("Failed to create MSI mapping: %d\n", st);
     return st;
   }

   st = mapping->MapRange(0, vmo->size(), true);
   if (st != ZX_OK) {
     LTRACEF("Falled to MapRange for the mapping: %d\n", st);
     return st;
   }

   LTRACEF("Mapping mapped at %#lx, size %zx, vmo size %lx, vmo_offset = %#lx\n", mapping->base(),
           mapping->size(), vmo->size(), vmo_offset);
   fbl::AllocChecker ac;
   fbl::RefPtr<MsiInterruptDispatcher> disp;

   // MSI lives inside a device's config space within an MSI Capability. MSI-X has a similar
   // capability, but has another table mapped elsewhere which contains individually maskable bits
   // per vector. The capability itself is managed by the PCI bus driver, and the mask bits are
   // handled by this dispatcher. So in the event of MSI-X there is no capability id to check, since
   // we don't touch the capability at all at this level.
   size_t add_result = 0;
   if (is_msix) {
     // Most validation for MSI-X is done in the PCI driver since it can confirm that the Table
     // Structure is appropriately large for the number of interrupts, and the allocation by now has
     // already been made.
     if (add_overflow(vmo_offset, (msi_id + 1) * sizeof(MsixTableEntry), &add_result) ||
         add_result > vmo->size()) {
       return ZX_ERR_INVALID_ARGS;
     }

     disp = fbl::AdoptRef<MsiInterruptDispatcher>(new (&ac) MsixDispatcherImpl(
         ktl::move(alloc), base_irq_id, msi_id, ktl::move(mapping), vmo_offset, register_int_fn));
   } else {
     auto* cap = reinterpret_cast<MsiCapability*>(mapping->base() + vmo_offset);
     if (cap->id != kMsiCapabilityId) {
       return ZX_ERR_INVALID_ARGS;
     }

     // MSI capabilities fit within a given device's configuration space which is either 256
     // or 4096 bytes. But in most cases the VMO containing config space is going to be at
     // least the size of a full PCI bus's worth of devices, and physical VMOs cannot be sliced
     // into children. We can validate that the capability fits within the offset given, but
     // otherwise cannot rely on the VMO's size for validation.
     if (add_overflow(vmo_offset, sizeof(MsiCapability), &add_result) || add_result > vmo->size()) {
       return ZX_ERR_INVALID_ARGS;
     }

     uint16_t ctrl_val = cap->control;
     bool has_pvm = !!(ctrl_val & kMsiPvmSupported);
     bool has_64bit = !!(ctrl_val & kMsi64bitSupported);
     disp = fbl::AdoptRef<MsiInterruptDispatcher>(new (&ac) MsiInterruptDispatcherImpl(
         ktl::move(alloc), base_irq_id, msi_id, ktl::move(mapping), vmo_offset, has_pvm, has_64bit,
         register_int_fn));
   }

   if (!ac.check()) {
     LTRACEF("Failed to allocate MsiInterruptDispatcher\n");
     return ZX_ERR_NO_MEMORY;
   }
   // If we allocated MsiInterruptDispatcher successfully then its dtor will release
   // the id if necessary.
   cleanup.cancel();

   // MSI / MSI-X interrupts share a masking approach and should be masked while
   // being serviced and unmasked while waiting for an interrupt message to arrive.
   disp->set_flags(INTERRUPT_UNMASK_PREWAIT | INTERRUPT_MASK_POSTWAIT);

   disp->UnmaskInterrupt();
   st = disp->RegisterInterruptHandler();
   if (st != ZX_OK) {
     LTRACEF("Failed to register interrupt handler for msi id %u (vector %u): %d\n", msi_id, vector,
             st);
     return st;
   }

   *out_rights = default_rights();
   out_interrupt->reset(ktl::move(disp));
   LTRACEF("MsiInterruptDispatcher successfully created.\n");
   return ZX_OK;
 }

 MsiInterruptDispatcher::MsiInterruptDispatcher(fbl::RefPtr<MsiAllocation>&& alloc,
                                                fbl::RefPtr<VmMapping>&& mapping,
                                                uint32_t base_irq_id, uint32_t msi_id,
                                                RegisterIntFn register_int_fn)
     : alloc_(ktl::move(alloc)),
       mapping_(ktl::move(mapping)),
       register_int_fn_(register_int_fn),
       base_irq_id_(base_irq_id),
       msi_id_(msi_id) {
   kcounter_add(dispatcher_msi_create_count, 1);
 }

 MsiInterruptDispatcher::~MsiInterruptDispatcher() {
   zx_status_t st = alloc_->ReleaseId(msi_id_);
   if (st != ZX_OK) {
     LTRACEF("MsiInterruptDispatcher: Failed to release MSI id %u (vector %u): %d\n", msi_id_,
             base_irq_id_ + msi_id_, st);
   }
   LTRACEF("MsiInterruptDispatcher: cleaning up MSI id %u\n", msi_id_);
   kcounter_add(dispatcher_msi_destroy_count, 1);
 }

 // This IrqHandler acts as a trampoline to call into the base
 // InterruptDispatcher's InterruptHandler() routine. Masking and signaling will
 // be handled there based on flags set in the constructor.
 interrupt_eoi MsiInterruptDispatcher::IrqHandler(void* ctx) {
   auto* self = reinterpret_cast<MsiInterruptDispatcher*>(ctx);
   self->InterruptHandler();
   kcounter_add(dispatcher_msi_interrupt_count, 1);
   return IRQ_EOI_DEACTIVATE;
 }

 zx_status_t MsiInterruptDispatcher::RegisterInterruptHandler() {
   register_int_fn_(&alloc_->block(), msi_id_, IrqHandler, this);
   return ZX_OK;
 }

 void MsiInterruptDispatcher::UnregisterInterruptHandler() {
   register_int_fn_(&alloc_->block(), msi_id_, nullptr, this);
 }

 void MsiInterruptDispatcherImpl::MaskInterrupt() {
   kcounter_add(dispatcher_msi_mask_count, 1);

   Guard<SpinLock, IrqSave> guard{&allocation()->lock()};
   if (has_platform_pvm_) {
     msi_mask_unmask(&allocation()->block(), msi_id(), true);
   }

   if (has_cap_pvm_) {
     const uint32_t mask = (1 << msi_id());
     if (has_64bit_) {
       capability_->mask_bits_64 = capability_->mask_bits_64 | mask;
     } else {
       capability_->mask_bits_32 = capability_->mask_bits_32 | mask;
     }
     arch::DeviceMemoryBarrier();
   }
 }

 void MsiInterruptDispatcherImpl::UnmaskInterrupt() {
   kcounter_add(dispatcher_msi_unmask_count, 1);

   Guard<SpinLock, IrqSave> guard{&allocation()->lock()};
   if (has_platform_pvm_) {
     msi_mask_unmask(&allocation()->block(), msi_id(), false);
   }

   if (has_cap_pvm_) {
     const uint32_t mask = ~(1 << msi_id());
     if (has_64bit_) {
       capability_->mask_bits_64 = capability_->mask_bits_64 & mask;
     } else {
       capability_->mask_bits_32 = capability_->mask_bits_32 & mask;
     }
     arch::DeviceMemoryBarrier();
   }
 }

 MsixDispatcherImpl::MsixDispatcherImpl(fbl::RefPtr<MsiAllocation>&& alloc, uint32_t base_irq_id,
                                        uint32_t msi_id, fbl::RefPtr<VmMapping>&& mapping,
                                        zx_off_t table_offset, RegisterIntFn register_int_fn)
     : MsiInterruptDispatcher(ktl::move(alloc), ktl::move(mapping), base_irq_id, msi_id,
                              register_int_fn),
       table_entries_(reinterpret_cast<MsixTableEntry*>(this->mapping()->base() + table_offset)) {
   // Disable the vector, set up the address and data registers, then re-enable
   // it for our given msi_id. Per PCI Local Bus Spec v3 section 6.8.2
   // implementation notes, all accesses to these registers must be DWORD or
   // QWORD only. We write upper and lower halves of the address unconditionally
   // because if the address is 32 bits then we want to write zeroes to the upper
   // half regardless. The msg_data field is incremented by msi_id because unlike
   // MSI, MSI-X does not adjust the data payload. This allows us to point
   // multiple table entries at the same vector, but requires us to specify the
   // vector in the data field.
   MaskInterrupt();
   writel(allocation()->block().tgt_addr & UINT32_MAX, &table_entries_[msi_id].msg_addr);
   writel(static_cast<uint32_t>(allocation()->block().tgt_addr >> 32),
          &table_entries_[msi_id].msg_upper_addr);
   writel(allocation()->block().tgt_data + msi_id, &table_entries_[msi_id].msg_data);
   arch::DeviceMemoryBarrier();
 }

 void MsixDispatcherImpl::MaskInterrupt() {
   kcounter_add(dispatcher_msi_mask_count, 1);
   RMWREG32(&table_entries_[msi_id()].vector_control, kMsixVectorControlMaskBit, 1, 1);
   arch::DeviceMemoryBarrier();
 }

 void MsixDispatcherImpl::UnmaskInterrupt() {
   kcounter_add(dispatcher_msi_unmask_count, 1);
   RMWREG32(&table_entries_[msi_id()].vector_control, kMsixVectorControlMaskBit, 1, 0);
   arch::DeviceMemoryBarrier();
 }

 MsixDispatcherImpl::~MsixDispatcherImpl() {
   MaskInterrupt();
   writel(0, &table_entries_[msi_id()].msg_addr);
   writel(0, &table_entries_[msi_id()].msg_upper_addr);
   writel(0, &table_entries_[msi_id()].msg_data);
   arch::DeviceMemoryBarrier();
 }
	// Copyright 2019 The Fuchsia Authors
	//
	// Use of this source code is governed by a MIT-style
	// license that can be found in the LICENSE file or at
	// https://opensource.org/licenses/MIT

	#include <lib/arch/intrin.h>
	#include <lib/counters.h>
	#include <lib/fit/defer.h>
	#include <platform.h>
	#include <reg.h>
	#include <trace.h>
	#include <zircon/errors.h>
	#include <zircon/limits.h>
	#include <zircon/rights.h>
	#include <zircon/types.h>

	#include <array>
	#include <cstring>
	#include <limits>

	#include <dev/interrupt.h>
	#include <fbl/alloc_checker.h>
	#include <kernel/auto_lock.h>
	#include <object/dispatcher.h>
	#include <object/interrupt_dispatcher.h>
	#include <object/msi_interrupt_dispatcher.h>
	#include <vm/vm_address_region.h>
	#include <vm/vm_object.h>

	#define LOCAL_TRACE 0

	KCOUNTER(dispatcher_msi_create_count, "msi_dispatcher.create")
	KCOUNTER(dispatcher_msi_interrupt_count, "msi_dispatcher.interrupts")
	KCOUNTER(dispatcher_msi_mask_count, "msi_dispatcher.mask")
	KCOUNTER(dispatcher_msi_unmask_count, "msi_dispatcher.unmask")
	KCOUNTER(dispatcher_msi_destroy_count, "msi_dispatcher.destroy")

	// Creates an a derived MsiInterruptDispatcher determined by the flags passed in
	zx_status_t MsiInterruptDispatcher::Create(fbl::RefPtr<MsiAllocation> alloc, uint32_t msi_id,
	const fbl::RefPtr<VmObject>& vmo, zx_paddr_t vmo_offset,
	uint32_t options, zx_rights_t* out_rights,
	KernelHandle<InterruptDispatcher>* out_interrupt,
	RegisterIntFn register_int_fn) {
	LTRACEF(
	"out_rights = %p, out_interrupt = %p\nvmo: %s, %s, %s\nsize = %lu, "
	"vmo_offset = %lu, options = %#x, cache policy = %u\n",
	out_rights, out_interrupt, vmo->is_paged() ? "paged" : "physical",
	vmo->is_contiguous() ? "contiguous" : "not contiguous",
	vmo->is_resizable() ? "resizable" : "not resizable", vmo->size(), vmo_offset, options,
	vmo->GetMappingCachePolicy());

	bool is_msix = (options & ZX_MSI_MODE_MSI_X) == ZX_MSI_MODE_MSI_X;
	options &= ~ZX_MSI_MODE_MSI_X;

	if (!out_rights \|\| !out_interrupt \|\|
	(vmo->is_paged() && (vmo->is_resizable() \|\| !vmo->is_contiguous())) \|\|
	vmo_offset >= vmo->size() \|\| options \|\|
	vmo->GetMappingCachePolicy() != ZX_CACHE_POLICY_UNCACHED_DEVICE) {
	return ZX_ERR_INVALID_ARGS;
	}

	uint32_t base_irq_id = 0;
	{
	Guard<SpinLock, IrqSave> guard{&alloc->lock()};
	if (msi_id >= alloc->block().num_irq) {
	LTRACEF("msi_id %u is out of range for the block (num_irqs: %u)\n", msi_id,
	alloc->block().num_irq);
	return ZX_ERR_INVALID_ARGS;
	}
	base_irq_id = alloc->block().base_irq_id;
	}

	zx_status_t st = alloc->ReserveId(msi_id);
	if (st != ZX_OK) {
	LTRACEF("failed to reserve msi_id %u: %d\n", msi_id, st);
	return st;
	}
	auto cleanup = fit::defer([alloc, msi_id]() { alloc->ReleaseId(msi_id); });

	// To handle MSI masking we need to create a kernel mapping for the VMO handed
	// to us, this will provide access to the register controlling the given MSI.
	// The VMO must be a contiguous VMO with the cache policy already configured.
	// Size checks will come into play when we know what type of capability we're
	// working with.
	auto vmar = VmAspace::kernel_aspace()->RootVmar();
	uint32_t vector = base_irq_id + msi_id;
	ktl::array<char, ZX_MAX_NAME_LEN> name{};
	snprintf(name.data(), name.max_size(), "msi id %u (vector %u)", msi_id, vector);
	fbl::RefPtr<VmMapping> mapping;
	st = vmar->CreateVmMapping(0, vmo->size(), 0, 0, vmo, 0,
	ARCH_MMU_FLAG_PERM_READ \| ARCH_MMU_FLAG_PERM_WRITE, name.data(),
	&mapping);
	if (st != ZX_OK) {
	LTRACEF("Failed to create MSI mapping: %d\n", st);
	return st;
	}

	st = mapping->MapRange(0, vmo->size(), true);
	if (st != ZX_OK) {
	LTRACEF("Falled to MapRange for the mapping: %d\n", st);
	return st;
	}

	LTRACEF("Mapping mapped at %#lx, size %zx, vmo size %lx, vmo_offset = %#lx\n", mapping->base(),
	mapping->size(), vmo->size(), vmo_offset);
	fbl::AllocChecker ac;
	fbl::RefPtr<MsiInterruptDispatcher> disp;

	// MSI lives inside a device's config space within an MSI Capability. MSI-X has a similar
	// capability, but has another table mapped elsewhere which contains individually maskable bits
	// per vector. The capability itself is managed by the PCI bus driver, and the mask bits are
	// handled by this dispatcher. So in the event of MSI-X there is no capability id to check, since
	// we don't touch the capability at all at this level.
	size_t add_result = 0;
	if (is_msix) {
	// Most validation for MSI-X is done in the PCI driver since it can confirm that the Table
	// Structure is appropriately large for the number of interrupts, and the allocation by now has
	// already been made.
	if (add_overflow(vmo_offset, (msi_id + 1) * sizeof(MsixTableEntry), &add_result) \|\|
	add_result > vmo->size()) {
	return ZX_ERR_INVALID_ARGS;
	}

	disp = fbl::AdoptRef<MsiInterruptDispatcher>(new (&ac) MsixDispatcherImpl(
	ktl::move(alloc), base_irq_id, msi_id, ktl::move(mapping), vmo_offset, register_int_fn));
	} else {
	auto* cap = reinterpret_cast<MsiCapability*>(mapping->base() + vmo_offset);
	if (cap->id != kMsiCapabilityId) {
	return ZX_ERR_INVALID_ARGS;
	}

	// MSI capabilities fit within a given device's configuration space which is either 256
	// or 4096 bytes. But in most cases the VMO containing config space is going to be at
	// least the size of a full PCI bus's worth of devices, and physical VMOs cannot be sliced
	// into children. We can validate that the capability fits within the offset given, but
	// otherwise cannot rely on the VMO's size for validation.
	if (add_overflow(vmo_offset, sizeof(MsiCapability), &add_result) \|\| add_result > vmo->size()) {
	return ZX_ERR_INVALID_ARGS;
	}

	uint16_t ctrl_val = cap->control;
	bool has_pvm = !!(ctrl_val & kMsiPvmSupported);
	bool has_64bit = !!(ctrl_val & kMsi64bitSupported);
	disp = fbl::AdoptRef<MsiInterruptDispatcher>(new (&ac) MsiInterruptDispatcherImpl(
	ktl::move(alloc), base_irq_id, msi_id, ktl::move(mapping), vmo_offset, has_pvm, has_64bit,
	register_int_fn));
	}

	if (!ac.check()) {
	LTRACEF("Failed to allocate MsiInterruptDispatcher\n");
	return ZX_ERR_NO_MEMORY;
	}
	// If we allocated MsiInterruptDispatcher successfully then its dtor will release
	// the id if necessary.
	cleanup.cancel();

	// MSI / MSI-X interrupts share a masking approach and should be masked while
	// being serviced and unmasked while waiting for an interrupt message to arrive.
	disp->set_flags(INTERRUPT_UNMASK_PREWAIT \| INTERRUPT_MASK_POSTWAIT);

	disp->UnmaskInterrupt();
	st = disp->RegisterInterruptHandler();
	if (st != ZX_OK) {
	LTRACEF("Failed to register interrupt handler for msi id %u (vector %u): %d\n", msi_id, vector,
	st);
	return st;
	}

	*out_rights = default_rights();
	out_interrupt->reset(ktl::move(disp));
	LTRACEF("MsiInterruptDispatcher successfully created.\n");
	return ZX_OK;
	}

	MsiInterruptDispatcher::MsiInterruptDispatcher(fbl::RefPtr<MsiAllocation>&& alloc,
	fbl::RefPtr<VmMapping>&& mapping,
	uint32_t base_irq_id, uint32_t msi_id,
	RegisterIntFn register_int_fn)
	: alloc_(ktl::move(alloc)),
	mapping_(ktl::move(mapping)),
	register_int_fn_(register_int_fn),
	base_irq_id_(base_irq_id),
	msi_id_(msi_id) {
	kcounter_add(dispatcher_msi_create_count, 1);
	}

	MsiInterruptDispatcher::~MsiInterruptDispatcher() {
	zx_status_t st = alloc_->ReleaseId(msi_id_);
	if (st != ZX_OK) {
	LTRACEF("MsiInterruptDispatcher: Failed to release MSI id %u (vector %u): %d\n", msi_id_,
	base_irq_id_ + msi_id_, st);
	}
	LTRACEF("MsiInterruptDispatcher: cleaning up MSI id %u\n", msi_id_);
	kcounter_add(dispatcher_msi_destroy_count, 1);
	}

	// This IrqHandler acts as a trampoline to call into the base
	// InterruptDispatcher's InterruptHandler() routine. Masking and signaling will
	// be handled there based on flags set in the constructor.
	interrupt_eoi MsiInterruptDispatcher::IrqHandler(void* ctx) {
	auto* self = reinterpret_cast<MsiInterruptDispatcher*>(ctx);
	self->InterruptHandler();
	kcounter_add(dispatcher_msi_interrupt_count, 1);
	return IRQ_EOI_DEACTIVATE;
	}

	zx_status_t MsiInterruptDispatcher::RegisterInterruptHandler() {
	register_int_fn_(&alloc_->block(), msi_id_, IrqHandler, this);
	return ZX_OK;
	}

	void MsiInterruptDispatcher::UnregisterInterruptHandler() {
	register_int_fn_(&alloc_->block(), msi_id_, nullptr, this);
	}

	void MsiInterruptDispatcherImpl::MaskInterrupt() {
	kcounter_add(dispatcher_msi_mask_count, 1);

	Guard<SpinLock, IrqSave> guard{&allocation()->lock()};
	if (has_platform_pvm_) {
	msi_mask_unmask(&allocation()->block(), msi_id(), true);
	}

	if (has_cap_pvm_) {
	const uint32_t mask = (1 << msi_id());
	if (has_64bit_) {
	capability_->mask_bits_64 = capability_->mask_bits_64 \| mask;
	} else {
	capability_->mask_bits_32 = capability_->mask_bits_32 \| mask;
	}
	arch::DeviceMemoryBarrier();
	}
	}

	void MsiInterruptDispatcherImpl::UnmaskInterrupt() {
	kcounter_add(dispatcher_msi_unmask_count, 1);

	Guard<SpinLock, IrqSave> guard{&allocation()->lock()};
	if (has_platform_pvm_) {
	msi_mask_unmask(&allocation()->block(), msi_id(), false);
	}

	if (has_cap_pvm_) {
	const uint32_t mask = ~(1 << msi_id());
	if (has_64bit_) {
	capability_->mask_bits_64 = capability_->mask_bits_64 & mask;
	} else {
	capability_->mask_bits_32 = capability_->mask_bits_32 & mask;
	}
	arch::DeviceMemoryBarrier();
	}
	}

	MsixDispatcherImpl::MsixDispatcherImpl(fbl::RefPtr<MsiAllocation>&& alloc, uint32_t base_irq_id,
	uint32_t msi_id, fbl::RefPtr<VmMapping>&& mapping,
	zx_off_t table_offset, RegisterIntFn register_int_fn)
	: MsiInterruptDispatcher(ktl::move(alloc), ktl::move(mapping), base_irq_id, msi_id,
	register_int_fn),
	table_entries_(reinterpret_cast<MsixTableEntry*>(this->mapping()->base() + table_offset)) {
	// Disable the vector, set up the address and data registers, then re-enable
	// it for our given msi_id. Per PCI Local Bus Spec v3 section 6.8.2
	// implementation notes, all accesses to these registers must be DWORD or
	// QWORD only. We write upper and lower halves of the address unconditionally
	// because if the address is 32 bits then we want to write zeroes to the upper
	// half regardless. The msg_data field is incremented by msi_id because unlike
	// MSI, MSI-X does not adjust the data payload. This allows us to point
	// multiple table entries at the same vector, but requires us to specify the
	// vector in the data field.
	MaskInterrupt();
	writel(allocation()->block().tgt_addr & UINT32_MAX, &table_entries_[msi_id].msg_addr);
	writel(static_cast<uint32_t>(allocation()->block().tgt_addr >> 32),
	&table_entries_[msi_id].msg_upper_addr);
	writel(allocation()->block().tgt_data + msi_id, &table_entries_[msi_id].msg_data);
	arch::DeviceMemoryBarrier();
	}

	void MsixDispatcherImpl::MaskInterrupt() {
	kcounter_add(dispatcher_msi_mask_count, 1);
	RMWREG32(&table_entries_[msi_id()].vector_control, kMsixVectorControlMaskBit, 1, 1);
	arch::DeviceMemoryBarrier();
	}

	void MsixDispatcherImpl::UnmaskInterrupt() {
	kcounter_add(dispatcher_msi_unmask_count, 1);
	RMWREG32(&table_entries_[msi_id()].vector_control, kMsixVectorControlMaskBit, 1, 0);
	arch::DeviceMemoryBarrier();
	}

	MsixDispatcherImpl::~MsixDispatcherImpl() {
	MaskInterrupt();
	writel(0, &table_entries_[msi_id()].msg_addr);
	writel(0, &table_entries_[msi_id()].msg_upper_addr);
	writel(0, &table_entries_[msi_id()].msg_data);
	arch::DeviceMemoryBarrier();
	}