| // Copyright 2016 The Fuchsia Authors |
| // Copyright (c) 2016, Google, Inc. All rights reserved |
| // |
| // Use of this source code is governed by a MIT-style |
| // license that can be found in the LICENSE file or at |
| // https://opensource.org/licenses/MIT |
| |
| #include <inttypes.h> |
| #include <lib/pci/pio.h> |
| #include <trace.h> |
| |
| #include <dev/pcie_bridge.h> |
| #include <dev/pcie_bus_driver.h> |
| #include <dev/pcie_device.h> |
| #include <dev/pcie_root.h> |
| #include <fbl/algorithm.h> |
| #include <fbl/alloc_checker.h> |
| #include <kernel/mutex.h> |
| #include <ktl/iterator.h> |
| #include <ktl/limits.h> |
| #include <ktl/move.h> |
| #include <vm/vm_aspace.h> |
| |
| /* TODO(johngro) : figure this out someday. |
| * |
| * In theory, BARs which map PIO regions for devices are supposed to be able to |
| * use bits [2, 31] to describe the programmable section of the PIO window. On |
| * real x86/64 systems, however, using the write-1s-readback technique to |
| * determine programmable bits of the BAR's address (and therefor the size of the |
| * I/O window) shows that the upper 16 bits are not programmable. This makes |
| * sense for x86 (where I/O space is only 16-bits), but fools the system into |
| * thinking that the I/O window is enormous. |
| * |
| * For now, just define a mask which can be used during PIO window space |
| * calculations which limits the size to 16 bits for x86/64 systems. non-x86 |
| * systems are still free to use all of the bits for their PIO addresses |
| * (although, it is still a bit unclear what it would mean to generate an IO |
| * space cycle on an architecture which has no such thing as IO space). |
| */ |
| constexpr size_t PcieBusDriver::REGION_BOOKKEEPING_SLAB_SIZE; |
| constexpr size_t PcieBusDriver::REGION_BOOKKEEPING_MAX_MEM; |
| |
| fbl::RefPtr<PcieBusDriver> PcieBusDriver::driver_; |
| |
| PcieBusDriver::PcieBusDriver(PciePlatformInterface& platform) : platform_(platform) {} |
| PcieBusDriver::~PcieBusDriver() { |
| // TODO(johngro): For now, if the bus driver is shutting down and unloading, |
| // ASSERT that there are no currently claimed devices out there. In the |
| // long run, we need to gracefully handle disconnecting from all user mode |
| // drivers (probably using a simulated hot-unplug) if we unload the bus |
| // driver. |
| ForeachDevice( |
| [](const fbl::RefPtr<PcieDevice>& dev, void* ctx, uint level) -> bool { |
| DEBUG_ASSERT(dev); |
| return true; |
| }, |
| nullptr); |
| |
| /* Shut off all of our IRQs and free all of our bookkeeping */ |
| ShutdownIrqs(); |
| |
| // Free the device tree |
| ForeachRoot( |
| [](const fbl::RefPtr<PcieRoot>& root, void* ctx) -> bool { |
| root->UnplugDownstream(); |
| return true; |
| }, |
| nullptr); |
| roots_.clear(); |
| |
| // Release the region bookkeeping memory. |
| region_bookkeeping_.reset(); |
| } |
| |
| zx_status_t PcieBusDriver::AddRoot(fbl::RefPtr<PcieRoot>&& root) { |
| if (root == nullptr) |
| return ZX_ERR_INVALID_ARGS; |
| |
| // Make sure that we are not already started. |
| if (!IsNotStarted()) { |
| TRACEF("Cannot add more PCIe roots once the bus driver has been started!\n"); |
| return ZX_ERR_BAD_STATE; |
| } |
| |
| // Attempt to add it to the collection of roots. |
| { |
| Guard<Mutex> guard{&bus_topology_lock_}; |
| if (!roots_.insert_or_find(ktl::move(root))) { |
| TRACEF("Failed to add PCIe root for bus %u, root already exists!\n", root->managed_bus_id()); |
| return ZX_ERR_ALREADY_EXISTS; |
| } |
| } |
| |
| return ZX_OK; |
| } |
| |
| zx_status_t PcieBusDriver::SetAddressTranslationProvider( |
| ktl::unique_ptr<PcieAddressProvider> provider) { |
| if (!IsNotStarted()) { |
| TRACEF("Cannot set an address provider if the driver is already running\n"); |
| return ZX_ERR_BAD_STATE; |
| } |
| |
| if (provider == nullptr) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| addr_provider_ = ktl::move(provider); |
| |
| return ZX_OK; |
| } |
| |
| zx_status_t PcieBusDriver::RescanDevices() { |
| if (!IsOperational()) { |
| TRACEF("Cannot rescan devices until the bus driver is operational!\n"); |
| return ZX_ERR_BAD_STATE; |
| } |
| |
| Guard<Mutex> guard{&bus_rescan_lock_}; |
| |
| // Scan each root looking for for devices and other bridges. |
| ForeachRoot( |
| [](const fbl::RefPtr<PcieRoot>& root, void* ctx) -> bool { |
| root->ScanDownstream(); |
| return true; |
| }, |
| nullptr); |
| |
| // Attempt to allocate any unallocated BARs |
| ForeachRoot( |
| [](const fbl::RefPtr<PcieRoot>& root, void* ctx) -> bool { |
| root->AllocateDownstreamBars(); |
| return true; |
| }, |
| nullptr); |
| |
| return ZX_OK; |
| } |
| |
| bool PcieBusDriver::IsNotStarted(bool allow_quirks_phase) const { |
| Guard<Mutex> guard{&start_lock_}; |
| |
| if ((state_ != State::NOT_STARTED) && |
| (!allow_quirks_phase || (state_ != State::STARTING_RUNNING_QUIRKS))) |
| return false; |
| |
| return true; |
| } |
| |
| bool PcieBusDriver::AdvanceState(State expected, State next) { |
| Guard<Mutex> guard{&start_lock_}; |
| |
| if (state_ != expected) { |
| TRACEF( |
| "Failed to advance PCIe bus driver state to %u. " |
| "Expected state (%u) does not match current state (%u)\n", |
| static_cast<uint>(next), static_cast<uint>(expected), static_cast<uint>(state_)); |
| return false; |
| } |
| |
| state_ = next; |
| return true; |
| } |
| |
| zx_status_t PcieBusDriver::StartBusDriver() { |
| if (!AdvanceState(State::NOT_STARTED, State::STARTING_SCANNING)) |
| return ZX_ERR_BAD_STATE; |
| |
| { |
| Guard<Mutex> guard{&bus_rescan_lock_}; |
| |
| // Scan each root looking for for devices and other bridges. |
| ForeachRoot( |
| [](const fbl::RefPtr<PcieRoot>& root, void* ctx) -> bool { |
| root->ScanDownstream(); |
| return true; |
| }, |
| nullptr); |
| |
| if (!AdvanceState(State::STARTING_SCANNING, State::STARTING_RUNNING_QUIRKS)) |
| return ZX_ERR_BAD_STATE; |
| |
| // Run registered quirk handlers for any newly discovered devices. |
| ForeachDevice( |
| [](const fbl::RefPtr<PcieDevice>& dev, void* ctx, uint level) -> bool { |
| PcieBusDriver::RunQuirks(dev); |
| return true; |
| }, |
| nullptr); |
| |
| // Indicate to the registered quirks handlers that we are finished with the |
| // quirks phase. |
| PcieBusDriver::RunQuirks(nullptr); |
| |
| if (!AdvanceState(State::STARTING_RUNNING_QUIRKS, State::STARTING_RESOURCE_ALLOCATION)) |
| return ZX_ERR_BAD_STATE; |
| |
| // Attempt to allocate any unallocated BARs |
| ForeachRoot( |
| [](const fbl::RefPtr<PcieRoot>& root, void* ctx) -> bool { |
| root->AllocateDownstreamBars(); |
| return true; |
| }, |
| nullptr); |
| } |
| |
| if (!AdvanceState(State::STARTING_RESOURCE_ALLOCATION, State::OPERATIONAL)) |
| return ZX_ERR_BAD_STATE; |
| |
| return ZX_OK; |
| } |
| |
| fbl::RefPtr<PcieDevice> PcieBusDriver::GetNthDevice(uint32_t index) { |
| struct GetNthDeviceState { |
| uint32_t index; |
| fbl::RefPtr<PcieDevice> ret; |
| } state; |
| |
| state.index = index; |
| |
| ForeachDevice( |
| [](const fbl::RefPtr<PcieDevice>& dev, void* ctx, uint level) -> bool { |
| DEBUG_ASSERT(dev && ctx); |
| |
| auto state = reinterpret_cast<GetNthDeviceState*>(ctx); |
| if (!state->index) { |
| state->ret = dev; |
| return false; |
| } |
| |
| state->index--; |
| return true; |
| }, |
| &state); |
| |
| return ktl::move(state.ret); |
| } |
| |
| void PcieBusDriver::LinkDeviceToUpstream(PcieDevice& dev, PcieUpstreamNode& upstream) { |
| Guard<Mutex> guard{&bus_topology_lock_}; |
| |
| // Have the device hold a reference to its upstream bridge. |
| DEBUG_ASSERT(dev.upstream_ == nullptr); |
| dev.upstream_ = fbl::RefPtr(&upstream); |
| |
| // Have the bridge hold a reference to the device |
| uint ndx = (dev.dev_id() * PCIE_MAX_FUNCTIONS_PER_DEVICE) + dev.func_id(); |
| DEBUG_ASSERT(ndx < ktl::size(upstream.downstream_)); |
| DEBUG_ASSERT(upstream.downstream_[ndx] == nullptr); |
| upstream.downstream_[ndx] = fbl::RefPtr(&dev); |
| } |
| |
| void PcieBusDriver::UnlinkDeviceFromUpstream(PcieDevice& dev) { |
| Guard<Mutex> guard{&bus_topology_lock_}; |
| |
| if (dev.upstream_ != nullptr) { |
| uint ndx = (dev.dev_id() * PCIE_MAX_FUNCTIONS_PER_DEVICE) + dev.func_id(); |
| DEBUG_ASSERT(ndx < ktl::size(dev.upstream_->downstream_)); |
| DEBUG_ASSERT(&dev == dev.upstream_->downstream_[ndx].get()); |
| |
| // Let go of the upstream's reference to the device |
| dev.upstream_->downstream_[ndx] = nullptr; |
| |
| // Let go of the device's reference to its upstream |
| dev.upstream_ = nullptr; |
| } |
| } |
| |
| fbl::RefPtr<PcieUpstreamNode> PcieBusDriver::GetUpstream(PcieDevice& dev) { |
| Guard<Mutex> guard{&bus_topology_lock_}; |
| auto ret = dev.upstream_; |
| return ret; |
| } |
| |
| fbl::RefPtr<PcieDevice> PcieBusDriver::GetDownstream(PcieUpstreamNode& upstream, uint ndx) { |
| DEBUG_ASSERT(ndx <= ktl::size(upstream.downstream_)); |
| Guard<Mutex> guard{&bus_topology_lock_}; |
| auto ret = upstream.downstream_[ndx]; |
| return ret; |
| } |
| |
| fbl::RefPtr<PcieDevice> PcieBusDriver::GetRefedDevice(uint bus_id, uint dev_id, uint func_id) { |
| struct GetRefedDeviceState { |
| uint bus_id; |
| uint dev_id; |
| uint func_id; |
| fbl::RefPtr<PcieDevice> ret; |
| } state; |
| |
| state.bus_id = bus_id, state.dev_id = dev_id, state.func_id = func_id, |
| |
| ForeachDevice( |
| [](const fbl::RefPtr<PcieDevice>& dev, void* ctx, uint level) -> bool { |
| DEBUG_ASSERT(dev && ctx); |
| auto state = reinterpret_cast<GetRefedDeviceState*>(ctx); |
| |
| if ((state->bus_id == dev->bus_id()) && (state->dev_id == dev->dev_id()) && |
| (state->func_id == dev->func_id())) { |
| state->ret = dev; |
| return false; |
| } |
| |
| return true; |
| }, |
| &state); |
| |
| return ktl::move(state.ret); |
| } |
| |
| void PcieBusDriver::ForeachRoot(ForeachRootCallback cbk, void* ctx) { |
| DEBUG_ASSERT(cbk); |
| |
| // Iterate over the roots, calling the registered callback for each one. |
| // Hold a reference to each root while we do this, but do not hold the |
| // topology lock. Note that this requires some slightly special handling |
| // when it comes to advancing the iterator as the root we are holding the |
| // reference to could (in theory) be removed from the collection during the |
| // callback.. |
| Guard<Mutex> guard{&bus_topology_lock_}; |
| |
| auto iter = roots_.begin(); |
| bool keep_going = true; |
| while (iter.IsValid()) { |
| // Grab our ref. |
| auto root_ref = iter.CopyPointer(); |
| |
| // Perform our callback. |
| guard.CallUnlocked([&keep_going, &cbk, &root_ref, &ctx] { keep_going = cbk(root_ref, ctx); }); |
| if (!keep_going) { |
| break; |
| } |
| |
| // If the root is still in the collection, simply advance the iterator. |
| // Otherwise, find the root (if any) with the next higher managed bus |
| // id. |
| if (root_ref->InContainer()) { |
| ++iter; |
| } else { |
| iter = roots_.upper_bound(root_ref->GetKey()); |
| } |
| } |
| } |
| |
| void PcieBusDriver::ForeachDevice(ForeachDeviceCallback cbk, void* ctx) { |
| DEBUG_ASSERT(cbk); |
| |
| struct ForeachDeviceCtx { |
| PcieBusDriver* driver; |
| ForeachDeviceCallback dev_cbk; |
| void* dev_ctx; |
| }; |
| |
| ForeachDeviceCtx foreach_device_ctx = { |
| .driver = this, |
| .dev_cbk = cbk, |
| .dev_ctx = ctx, |
| }; |
| |
| ForeachRoot( |
| [](const fbl::RefPtr<PcieRoot>& root, void* ctx_) -> bool { |
| auto ctx = static_cast<ForeachDeviceCtx*>(ctx_); |
| return ctx->driver->ForeachDownstreamDevice(root, 0, ctx->dev_cbk, ctx->dev_ctx); |
| }, |
| &foreach_device_ctx); |
| } |
| |
| zx_status_t PcieBusDriver::AllocBookkeeping() { |
| // Create the RegionPool we will use to supply the memory for the |
| // bookkeeping for all of our region tracking and allocation needs. Then |
| // assign it to each of our allocators. |
| region_bookkeeping_ = RegionAllocator::RegionPool::Create(REGION_BOOKKEEPING_MAX_MEM); |
| if (region_bookkeeping_ == nullptr) { |
| TRACEF("Failed to create pool allocator for Region bookkeeping!\n"); |
| return ZX_ERR_NO_MEMORY; |
| } |
| |
| mmio_lo_regions_.SetRegionPool(region_bookkeeping_); |
| mmio_hi_regions_.SetRegionPool(region_bookkeeping_); |
| pio_regions_.SetRegionPool(region_bookkeeping_); |
| |
| return ZX_OK; |
| } |
| |
| bool PcieBusDriver::ForeachDownstreamDevice(const fbl::RefPtr<PcieUpstreamNode>& upstream, |
| uint level, ForeachDeviceCallback cbk, void* ctx) { |
| DEBUG_ASSERT(upstream && cbk); |
| bool keep_going = true; |
| |
| for (uint i = 0; keep_going && (i < ktl::size(upstream->downstream_)); ++i) { |
| auto dev = upstream->GetDownstream(i); |
| |
| if (!dev) |
| continue; |
| |
| keep_going = cbk(dev, ctx, level); |
| |
| // It should be impossible to have a bridge topology such that we could |
| // recurse more than 256 times. |
| if (keep_going && (level < 256)) { |
| if (dev->is_bridge()) { |
| // TODO(johngro): eliminate the need to hold this extra ref. If |
| // we had the ability to up and downcast when moving RefPtrs, we |
| // could just ktl::move dev into a PcieBridge pointer and then |
| // down into a PcieUpstreamNode pointer. |
| fbl::RefPtr<PcieUpstreamNode> downstream_bridge( |
| static_cast<PcieUpstreamNode*>(static_cast<PcieBridge*>(dev.get()))); |
| keep_going = ForeachDownstreamDevice(downstream_bridge, level + 1, cbk, ctx); |
| } |
| } |
| } |
| |
| return keep_going; |
| } |
| |
| zx_status_t PcieBusDriver::AddSubtractBusRegion(uint64_t base, uint64_t size, PciAddrSpace aspace, |
| bool add_op) { |
| if (!IsNotStarted(true)) { |
| TRACEF("Cannot add/subtract bus regions once the bus driver has been started!\n"); |
| return ZX_ERR_BAD_STATE; |
| } |
| |
| if (!size) |
| return ZX_ERR_INVALID_ARGS; |
| |
| uint64_t end = base + size - 1; |
| |
| auto AddSub = add_op ? |
| [](RegionAllocator& allocator, const ralloc_region_t& region) { |
| return allocator.AddRegion(region, RegionAllocator::AllowOverlap::Yes); |
| } |
| : |
| [](RegionAllocator& allocator, const ralloc_region_t& region) { |
| return allocator.SubtractRegion(region, RegionAllocator::AllowIncomplete::Yes); |
| }; |
| |
| if (aspace == PciAddrSpace::MMIO) { |
| // Figure out if this goes in the low region, the high region, or needs |
| // to be split into two regions. |
| constexpr uint64_t U32_MAX = ktl::numeric_limits<uint32_t>::max(); |
| |
| if (end <= U32_MAX) { |
| return AddSub(mmio_lo_regions_, {.base = base, .size = size}); |
| } else if (base > U32_MAX) { |
| return AddSub(mmio_hi_regions_, {.base = base, .size = size}); |
| } else { |
| uint64_t lo_base = base; |
| uint64_t hi_base = U32_MAX + 1; |
| uint64_t lo_size = hi_base - lo_base; |
| uint64_t hi_size = size - lo_size; |
| zx_status_t res; |
| |
| res = AddSub(mmio_lo_regions_, {.base = lo_base, .size = lo_size}); |
| if (res != ZX_OK) { |
| return res; |
| } |
| |
| return AddSub(mmio_hi_regions_, {.base = hi_base, .size = hi_size}); |
| } |
| } else { |
| DEBUG_ASSERT(aspace == PciAddrSpace::PIO); |
| |
| if ((base | end) & ~PCIE_PIO_ADDR_SPACE_MASK) |
| return ZX_ERR_INVALID_ARGS; |
| |
| return AddSub(pio_regions_, {.base = base, .size = size}); |
| } |
| } |
| |
| zx_status_t PcieBusDriver::InitializeDriver(PciePlatformInterface& platform) { |
| Guard<Mutex> guard{PcieBusDriverLock::Get()}; |
| |
| if (driver_ != nullptr) { |
| TRACEF("Failed to initialize PCIe bus driver; driver already initialized\n"); |
| return ZX_ERR_BAD_STATE; |
| } |
| |
| fbl::AllocChecker ac; |
| driver_ = fbl::AdoptRef(new (&ac) PcieBusDriver(platform)); |
| if (!ac.check()) { |
| TRACEF("Failed to allocate PCIe bus driver\n"); |
| return ZX_ERR_NO_MEMORY; |
| } |
| |
| zx_status_t ret = driver_->AllocBookkeeping(); |
| if (ret != ZX_OK) |
| driver_.reset(); |
| |
| return ret; |
| } |
| |
| void PcieBusDriver::ShutdownDriver() { |
| fbl::RefPtr<PcieBusDriver> driver; |
| |
| { |
| Guard<Mutex> guard{PcieBusDriverLock::Get()}; |
| driver = ktl::move(driver_); |
| } |
| |
| driver.reset(); |
| } |
| |
| /******************************************************************************* |
| * |
| * ECAM support |
| * |
| ******************************************************************************/ |
| /* TODO(cja): The bus driver owns all configs as well as devices so the |
| * lifecycle of both are already dependent. Should this still return a refptr? |
| */ |
| const PciConfig* PcieBusDriver::GetConfig(uint bus_id, uint dev_id, uint func_id, |
| paddr_t* out_cfg_phys) { |
| DEBUG_ASSERT(bus_id < PCIE_MAX_BUSSES); |
| DEBUG_ASSERT(dev_id < PCIE_MAX_DEVICES_PER_BUS); |
| DEBUG_ASSERT(func_id < PCIE_MAX_FUNCTIONS_PER_DEVICE); |
| |
| if (!addr_provider_) { |
| TRACEF("Cannot get state if no address translation provider is set\n"); |
| return nullptr; |
| } |
| |
| if (out_cfg_phys) { |
| *out_cfg_phys = 0; |
| } |
| |
| uintptr_t addr; |
| zx_status_t result = |
| addr_provider_->Translate(static_cast<uint8_t>(bus_id), static_cast<uint8_t>(dev_id), |
| static_cast<uint8_t>(func_id), &addr, out_cfg_phys); |
| if (result != ZX_OK) { |
| return nullptr; |
| } |
| |
| // Check if we already have this config space cached somewhere. |
| auto cfg_iter = configs_.find_if([addr](const PciConfig& cfg) { return (cfg.base() == addr); }); |
| |
| if (cfg_iter.IsValid()) { |
| return &(*cfg_iter); |
| } |
| |
| // Nothing found, create a new PciConfig for this address |
| auto cfg = addr_provider_->CreateConfig(addr); |
| configs_.push_front(cfg); |
| |
| return cfg.get(); |
| } |
| |
| // External references to the quirks handler table. |
| extern const PcieBusDriver::QuirkHandler pcie_quirk_handlers[]; |
| void PcieBusDriver::RunQuirks(const fbl::RefPtr<PcieDevice>& dev) { |
| if (dev && dev->quirks_done()) |
| return; |
| |
| for (size_t i = 0; pcie_quirk_handlers[i] != nullptr; i++) { |
| pcie_quirk_handlers[i](dev); |
| } |
| |
| if (dev != nullptr) |
| dev->SetQuirksDone(); |
| } |
| |
| // Workaround to disable all devices on the bus for mexec. This should not be |
| // used for any other reason due to it intentionally leaving drivers in a bad |
| // state (some may crash). |
| // TODO(cja): The paradise serial workaround in particular may need a smarter |
| // way of being handled in the future because it is not uncommon to have serial |
| // bus devices initialized by the bios that we need to retain in zedboot/crash |
| // situations. |
| void PcieBusDriver::DisableBus() { |
| Guard<Mutex> guard{PcieBusDriverLock::Get()}; |
| ForeachDevice( |
| [](const fbl::RefPtr<PcieDevice>& dev, void* ctx, uint level) -> bool { |
| if (!dev->is_bridge() && !(dev->vendor_id() == 0x8086 && dev->device_id() == 0x9d66)) { |
| TRACEF("Disabling device %#02x:%#02x.%01x - VID %#04x DID %#04x\n", dev->dev_id(), |
| dev->bus_id(), dev->func_id(), dev->vendor_id(), dev->device_id()); |
| dev->EnableBusMaster(false); |
| dev->Disable(); |
| } else { |
| TRACEF("Skipping LP Serial disable!"); |
| } |
| return true; |
| }, |
| nullptr); |
| } |