// Copyright 2016 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "usb-xhci.h"

#include <lib/device-protocol/pci.h>
#include <lib/fit/promise.h>
#include <lib/zx/bti.h>
#include <lib/zx/interrupt.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <threads.h>
#include <unistd.h>
#include <zircon/syscalls.h>
#include <zircon/types.h>

#include <algorithm>
#include <memory>
#include <string>

#include <ddk/binding.h>
#include <ddk/debug.h>
#include <ddk/metadata.h>
#include <ddk/platform-defs.h>
#include <fbl/alloc_checker.h>
#include <fbl/auto_call.h>
#include <hw/arch_ops.h>
#include <hw/reg.h>

namespace usb_xhci {

// Computes the interval value for a specified endpoint.
static int ComputeInterval(const usb_endpoint_descriptor_t* ep, usb_speed_t speed) {
  uint8_t ep_type = ep->bmAttributes & USB_ENDPOINT_TYPE_MASK;
  uint8_t interval = ep->bInterval;

  if (ep_type == USB_ENDPOINT_CONTROL || ep_type == USB_ENDPOINT_BULK) {
    if (speed == USB_SPEED_HIGH) {
      return Log2(interval);
    } else {
      return 0;
    }
  }

  // now we deal with interrupt and isochronous endpoints
  // first make sure bInterval is in legal range
  // See table 6-12 in xHCI specification section 6.2.3.6
  if (ep_type == USB_ENDPOINT_INTERRUPT && (speed == USB_SPEED_LOW || speed == USB_SPEED_FULL)) {
    interval = static_cast<uint8_t>(std::clamp(static_cast<int>(interval), 1, 255));
  } else {
    interval = static_cast<uint8_t>(std::clamp(static_cast<int>(interval), 1, 16));
  }

  switch (speed) {
    case USB_SPEED_LOW:
      return Log2(interval) + 3;  // + 3 to convert 125us to 1ms
    case USB_SPEED_FULL:
      if (ep_type == USB_ENDPOINT_ISOCHRONOUS) {
        return (interval - 1) + 3;
      } else {
        return Log2(interval) + 3;
      }
    case USB_SPEED_SUPER:
    case USB_SPEED_HIGH:
      return interval - 1;
    default:
      return 0;
  }
}

TRBPromise UsbXhci::DisableSlotCommand(uint32_t slot_id) {
  uint8_t port;
  bool connected_to_hub = false;
  {
    fbl::AutoLock _(&device_state_[slot_id - 1].transaction_lock());
    device_state_[slot_id - 1].Disconnect();
    port = device_state_[slot_id - 1].GetPort();
    connected_to_hub = static_cast<bool>(device_state_[slot_id - 1].GetHub());
  }
  DisableSlot cmd;
  cmd.set_slot(slot_id);
  auto context = command_ring_.AllocateContext();
  if (!context) {
    return fit::make_result_promise(fit::result<TRB*, zx_status_t>(fit::error(ZX_ERR_BAD_STATE)))
        .box();
  }
  if (!connected_to_hub) {
    port_state_[port - 1].slot_id = 0;
  }

  return SubmitCommand(cmd, std::move(context))
      .then([slot_id,
             this](fit::result<TRB*, zx_status_t>& result) -> fit::result<TRB*, zx_status_t> {
        if (result.is_error()) {
          return result;
        }
        TRB* trb = result.value();
        auto completion_event = reinterpret_cast<CommandCompletionEvent*>(trb);
        if (completion_event->CompletionCode() != CommandCompletionEvent::Success) {
          return fit::error(ZX_ERR_BAD_STATE);
        }
        dcbaa_[completion_event->SlotID()] = 0;
        {
          fbl::AutoLock _(&device_state_[slot_id - 1].transaction_lock());
          device_state_[slot_id - 1].reset();
        }
        return fit::ok(trb);
      })
      .box();
}

TRBPromise UsbXhci::EnableSlotCommand() {
  TRB trb;
  Control::Get().FromValue(0).set_Type(Control::EnableSlot).ToTrb(&trb);
  auto context = command_ring_.AllocateContext();
  return SubmitCommand(trb, std::move(context));
}

fit::promise<OwnedRequest, void> UsbXhci::UsbHciRequestQueue(OwnedRequest usb_request) {
  fit::bridge<OwnedRequest, void> bridge;
  usb_request_complete_t completion;
  completion.callback = [](void* ctx, usb_request_t* req) {
    auto completer = static_cast<fit::completer<OwnedRequest, void>*>(ctx);
    completer->complete_ok(OwnedRequest(req, sizeof(usb_request_t)));
    delete completer;
  };
  completion.ctx = new fit::completer<OwnedRequest, void>(std::move(bridge.completer));
  UsbHciRequestQueue(usb_request.take(), &completion);
  return bridge.consumer.promise().box();
}

zx_status_t DeviceState::InitializeSlotBuffer(const UsbXhci& hci, uint8_t slot_id, uint8_t port_id,
                                              const std::optional<HubInfo>& hub_info,
                                              std::unique_ptr<dma_buffer::PagedBuffer>* out) {
  // Section 4.3.3
  // 6.2.5 (Input Context initialization)
  std::unique_ptr<dma_buffer::PagedBuffer> buffer;
  zx_status_t status = hci.buffer_factory().CreatePaged(hci.bti(), ZX_PAGE_SIZE, false, &buffer);
  if (status != ZX_OK) {
    return status;
  }
  if (hci.get_is_32_bit_controller() && (buffer->phys()[0] >= UINT32_MAX)) {
    return ZX_ERR_NO_MEMORY;
  }
  // 6.2.5.1 -- Initialize input control context
  // NOTE: Input Control Context consumes 64 bytes if CSZ is 1
  // Enable bit starts at offset 2
  auto control = static_cast<uint32_t*>(buffer->virt());
  control[1] = 0x3;  // Enable both slot and endpoint context.
  size_t slot_size = (hci.CSZ()) ? 64 : 32;
  // Initialize input slot context data structure (6.2.2) with 1 context entry
  // Set root hub port number to port number and context entries to 1
  auto slot_context =
      reinterpret_cast<SlotContext*>(reinterpret_cast<unsigned char*>(control) + slot_size);
  if (hub_info) {
    slot_context->set_CONTEXT_ENTRIES(1)
        .set_ROUTE_STRING(hub_info->route_string)
        .set_PORTNO(hub_info->rh_port)
        .set_SPEED(hub_info->speed);
  } else {
    slot_context->set_CONTEXT_ENTRIES(1).set_PORTNO(port_id).set_SPEED(hci.GetPortSpeed(port_id));
  }
  *out = std::move(buffer);
  return ZX_OK;
}

zx_status_t DeviceState::InitializeEndpointContext(const UsbXhci& hci, uint8_t slot_id,
                                                   uint8_t port_id,
                                                   const std::optional<HubInfo>& hub_info,
                                                   dma_buffer::PagedBuffer* slot_context_buffer) {
  CRCR trb_phys = tr_.phys(hci.CapLength());
  auto* control = static_cast<uint32_t*>(slot_context_buffer->virt());
  size_t slot_size = (hci.CSZ()) ? 64 : 32;
  auto slot_context =
      reinterpret_cast<SlotContext*>(reinterpret_cast<unsigned char*>(control) + slot_size);
  // Initialize endpoint context 0
  // Set CERR to 3, TR dequeue pointer, max packet size, EP type = control, DCS = 1
  auto endpoint_context = reinterpret_cast<EndpointContext*>(
      reinterpret_cast<unsigned char*>(control) + (slot_size * 2));
  uint16_t mps = 8;
  uint8_t speed;
  if (hub_info) {
    speed = static_cast<uint8_t>(hub_info->speed);
    // TODO (ZX-4579): USB 3.1 support. Section 6.2.2
    if (((speed == USB_SPEED_LOW) || (speed == USB_SPEED_FULL)) &&
        (hub_info->hub_speed == USB_SPEED_HIGH)) {
      slot_context->set_PARENT_HUB_SLOT_ID(hci.DeviceIdToSlotId(hub_info->hub_id))
          .set_PARENT_PORT_NUMBER(speed);
    }
  } else {
    speed = hci.GetPortSpeed(port_id);
  }
  switch (speed) {
    case USB_SPEED_SUPER:
      mps = 512;
      break;
    case USB_SPEED_FULL:
    case USB_SPEED_HIGH:
      mps = 64;
      break;
    case USB_SPEED_LOW:
    default:
      mps = 8;
      break;
  }
  endpoint_context->Init(EndpointContext::Control, trb_phys, mps);
  return ZX_OK;
}

zx_status_t DeviceState::InitializeOutputContextBuffer(
    const UsbXhci& hci, uint8_t slot_id, uint8_t port_id, const std::optional<HubInfo>& hub_info,
    uint64_t* dcbaa, std::unique_ptr<dma_buffer::PagedBuffer>* out) {
  // Allocate an output device context data structure (6.2.1)
  // Update the DCBAA entry for this slot.
  std::unique_ptr<dma_buffer::PagedBuffer> output_context_buffer;
  zx_status_t status =
      hci.buffer_factory().CreatePaged(hci.bti(), ZX_PAGE_SIZE, false, &output_context_buffer);
  if (status != ZX_OK) {
    return status;
  }
  if (hci.get_is_32_bit_controller() && (output_context_buffer->phys()[0] >= UINT32_MAX)) {
    return ZX_ERR_NO_MEMORY;
  }
  dcbaa[slot_id] = output_context_buffer->phys()[0];
  if (!hub_info) {
    slot_id = static_cast<uint8_t>(slot_id);
  }
  hub_ = hub_info;
  hw_mb();
  *out = std::move(output_context_buffer);
  return ZX_OK;
}

TRBPromise DeviceState::AddressDeviceCommand(UsbXhci* hci, uint8_t slot, uint8_t port,
                                             std::optional<HubInfo> hub_info, uint64_t* dcbaa,
                                             EventRing* event_ring, CommandRing* command_ring,
                                             ddk::MmioBuffer* mmio, bool bsr) {
  if (!hub_info.has_value()) {
    hci->get_port_state()[port - 1].slot_id = slot;
  }
  std::unique_ptr<dma_buffer::PagedBuffer> slot_context_buffer;
  std::unique_ptr<dma_buffer::PagedBuffer> output_context_buffer;
  zx_status_t status = InitializeSlotBuffer(*hci, slot, port, hub_info, &slot_context_buffer);
  if (status != ZX_OK) {
    return hci->ResultToTRBPromise(fit::error(status));
  }

  // Allocate the transfer ring (see section 4.9)
  // TODO (bbosak): Assign an Interrupter from the pool
  fbl::AutoLock _(&transaction_lock_);
  status = tr_.Init(hci->get_page_size(), hci->bti(), event_ring, hci->get_is_32_bit_controller(),
                    mmio, *hci);
  if (status != ZX_OK) {
    return fit::make_result_promise(fit::result<TRB*, zx_status_t>(fit::error(status))).box();
  }

  status = InitializeEndpointContext(*hci, slot, port, hub_info, slot_context_buffer.get());
  if (status != ZX_OK) {
    return fit::make_result_promise(fit::result<TRB*, zx_status_t>(fit::error(status))).box();
  }

  status = InitializeOutputContextBuffer(*hci, slot, port, hub_info, dcbaa, &output_context_buffer);
  if (status != ZX_OK) {
    return fit::make_result_promise(fit::result<TRB*, zx_status_t>(fit::error(status))).box();
  }

  // Issue an address device command for the device slot
  // See sections 3.3.4 and 6.4.3.4
  usb_xhci::AddressDeviceStruct command;
  command.ptr = slot_context_buffer->phys()[0];
  command.set_SlotID(slot);
  command.set_BSR(bsr);
  auto command_context = command_ring->AllocateContext();
  if (!command_context) {
    return fit::make_result_promise(fit::result<TRB*, zx_status_t>(fit::error(ZX_ERR_NO_MEMORY)))
        .box();
  }
  command_context->port_number = port;
  hw_mb();
  input_context_ = std::move(slot_context_buffer);
  device_context_ = std::move(output_context_buffer);
  return hci->SubmitCommand(command, std::move(command_context));
}

TRBPromise UsbXhci::AddressDeviceCommand(uint8_t slot_id) {
  usb_xhci::AddressDeviceStruct cmd;
  cmd.set_BSR(0);
  cmd.set_SlotID(slot_id);
  return SubmitCommand(cmd, command_ring_.AllocateContext());
}

usb_speed_t UsbXhci::GetDeviceSpeed(uint8_t slot) {
  {
    fbl::AutoLock _(&device_state_[slot - 1].transaction_lock());
    if (device_state_[slot - 1].GetHub()) {
      return device_state_[slot - 1].GetHub()->speed;
    }
  }
  return PORTSC::Get(cap_length_, device_state_[slot - 1].GetPort())
      .ReadFrom(&mmio_.value())
      .PortSpeed();
}

zx_status_t Interrupter::Start(uint32_t interrupter, const RuntimeRegisterOffset& offset,
                               ddk::MmioView mmio_view, UsbXhci* hci) {
  hci_ = hci;
  interrupter_ = interrupter;
  ERDP erdp = ERDP::Get(offset, interrupter).ReadFrom(&mmio_view);
  if (!event_ring_.erdp_phys()) {
    return ZX_ERR_BAD_STATE;
  }
  erdp.set_reg_value(event_ring_.erdp_phys());
  erdp.WriteTo(&mmio_view);
  ERSTBA ba = ERSTBA::Get(offset, interrupter).ReadFrom(&mmio_view);
  // This enables the interrupter
  ba.set_Pointer(event_ring_.erst()).WriteTo(&mmio_view);
  IMAN::Get(offset, interrupter).FromValue(0).set_IE(1).WriteTo(&mmio_view);
  thread_ = std::thread([this]() { IrqThread(); });
  return ZX_OK;
}

uint8_t UsbXhci::GetPortSpeed(uint8_t port_id) const {
  return static_cast<uint8_t>(
      PORTSC::Get(cap_length_, port_id).ReadFrom(&mmio_.value()).PortSpeed());
}

TRBPromise UsbXhci::AddressDeviceCommand(uint8_t slot_id, uint8_t port_id,
                                         std::optional<HubInfo> hub_info, bool bsr) {
  return device_state_[slot_id - 1].AddressDeviceCommand(this, slot_id, port_id, hub_info, dcbaa_,
                                                         &interrupters_[0].ring(), &command_ring_,
                                                         &mmio_.value(), bsr);
}

void UsbXhci::SetDeviceInformation(uint8_t slot, uint8_t port, const std::optional<HubInfo>& hub) {
  fbl::AutoLock _(&device_state_[slot - 1].transaction_lock());
  device_state_[slot - 1].SetDeviceInformation(slot, port, hub);
  if (hub) {
    uint8_t hub_id = hub->hub_id;
    fbl::AutoLock _(&device_state_[hub_id].transaction_lock());
    device_state_[hub_id].GetHub()->port_to_device[port - 1] = static_cast<uint8_t>(slot - 1);
  }
}

TRBPromise UsbXhci::SetMaxPacketSizeCommand(uint8_t slot_id, uint8_t bMaxPacketSize0) {
  auto& state = device_state_[slot_id - 1];
  usb_xhci::AddressDeviceStruct cmd;
  {
    fbl::AutoLock _(&state.transaction_lock());
    auto control = reinterpret_cast<uint32_t*>(state.GetInputContext()->virt());
    size_t slot_size = (hcc_.CSZ()) ? 64 : 32;
    auto endpoint_context = reinterpret_cast<EndpointContext*>(
        reinterpret_cast<unsigned char*>(control) + (slot_size * 2));
    endpoint_context->set_MAX_PACKET_SIZE(bMaxPacketSize0);
    Control::Get().FromValue(0).set_Type(Control::EvaluateContextCommand).ToTrb(&cmd);
    cmd.set_SlotID(slot_id);

    cmd.ptr = state.GetInputContext()->phys()[0];
  }
  auto context = command_ring_.AllocateContext();
  return SubmitCommand(cmd, std::move(context));
}

zx_status_t UsbXhci::DeviceOnline(uint32_t slot, uint16_t port, usb_speed_t speed) {
  bool is_usb_3 = false;
  {
    fbl::AutoLock transaction_lock(&device_state_[slot - 1].transaction_lock());
    if (device_state_[slot - 1].GetHub()) {
      transaction_lock.release();
      return PostCallback([=](const ddk::UsbBusInterfaceProtocolClient& bus) {
        uint32_t hub_id;
        {
          fbl::AutoLock _(&get_device_state()[slot - 1].transaction_lock());
          if (!get_device_state()[slot - 1].GetHub()) {
            // Race condition -- device was unplugged before we got a chance to notify the bus
            // driver.
            return ZX_OK;
          }
          hub_id = get_device_state()[slot - 1].GetHub()->hub_id;
        }
        bus.AddDevice(slot - 1, hub_id, speed);
        return ZX_OK;
      });
    }
    is_usb_3 = get_port_state()[port].is_USB3;
  }
  return PostCallback([=](const ddk::UsbBusInterfaceProtocolClient& bus) {
    bus.AddDevice(slot - 1,
                  static_cast<uint32_t>(is_usb_3 ? UsbHciGetMaxDeviceCount() - 1
                                                 : UsbHciGetMaxDeviceCount() - 2),
                  speed);
    return ZX_OK;
  });
}

TRBPromise UsbXhci::DeviceOffline(uint32_t slot, TRB* continuation) {
  fbl::AutoLock _(&device_state_[slot - 1].transaction_lock());
  device_state_[slot - 1].Disconnect();
  fit::bridge<TRB*, zx_status_t> bridge;
  zx_status_t status = PostCallback([this, slot, cb = std::move(bridge.completer), continuation](
                                        const ddk::UsbBusInterfaceProtocolClient& bus) mutable {
    for (size_t i = 0; i < kMaxEndpoints; i++) {
      fbl::AutoLock _(&device_state_[slot - 1].transaction_lock());
      auto trbs = device_state_[slot - 1].GetTransferRing(i).TakePendingTRBs();
      for (auto& trb : trbs) {
        trb.request->Complete(ZX_ERR_IO_NOT_PRESENT, 0);
      }
    }
    fbl::DoublyLinkedList<std::unique_ptr<TRBContext>> trbs;
    {
      fbl::AutoLock _(&device_state_[slot - 1].transaction_lock());
      trbs = device_state_[slot - 1].GetTransferRing().TakePendingTRBs();
    }
    for (auto& trb : trbs) {
      trb.request->Complete(ZX_ERR_IO_NOT_PRESENT, 0);
    }
    zx_status_t status = bus.RemoveDevice(slot - 1);
    if (status != ZX_OK) {
      cb.complete_error(status);
      return status;
    }
    cb.complete_ok(continuation);
    return status;
  });
  if (status != ZX_OK) {
    return ResultToTRBPromise(fit::error(status));
  }
  return bridge.consumer.promise().box();
}

void UsbXhci::ResetPort(uint16_t port) {
  PORTSC sc = PORTSC::Get(cap_length_, port).ReadFrom(&mmio_.value());
  PORTSC::Get(cap_length_, port)
      .FromValue(0)
      .set_CCS(sc.CCS())
      .set_PortSpeed(sc.PortSpeed())
      .set_PIC(sc.PIC())
      .set_PLS(sc.PLS())
      .set_PP(sc.PP())
      .set_PR(1)
      .WriteTo(&mmio_.value());
}

TRBPromise UsbXhci::UsbHciHubDeviceAddedAsync(uint32_t device_id, uint32_t port,
                                              usb_speed_t speed) {
  auto state = &device_state_[device_id];
  // Acquire a slot
  HubInfo hub;
  {
    fbl::AutoLock _(&state->transaction_lock());
    hub.hub_id = static_cast<uint8_t>(device_id);
    hub.speed = speed;
    hub.multi_tt = state->GetHub()->multi_tt;
    hub.route_string =
        (state->GetHub()->route_string) | ((port) << (state->GetHub()->hub_depth * 4));
    hub.parent_port_number = static_cast<uint8_t>(port);
    hub.hub_depth = static_cast<uint8_t>(state->GetHub()->hub_depth);
    hub.hub_speed = static_cast<uint8_t>(state->GetHub()->speed);
    hub.rh_port = state->GetHub()->rh_port;
  }
  return EnumerateDevice(this, static_cast<uint8_t>(port), std::make_optional(hub));
}

TRBPromise UsbXhci::ConfigureHubAsync(uint32_t device_id, usb_speed_t speed,
                                      const usb_hub_descriptor_t* desc, bool multi_tt) {
  auto state = &device_state_[device_id];
  HubInfo hub;
  struct AddressDeviceStruct cmd;
  std::unique_ptr<TRBContext> context;
  {
    fbl::AutoLock _(&state->transaction_lock());
    hub.hub_id = static_cast<uint8_t>(device_id);
    hub.speed = speed;
    hub.hub_speed = static_cast<uint8_t>(speed);
    hub.multi_tt = multi_tt;
    hub.rh_port = state->GetPort();
    if (state->GetHub()) {
      hub.parent_port_number = state->GetHub()->parent_port_number;
      hub.route_string = state->GetHub()->route_string;
      hub.hub_depth = static_cast<uint8_t>(state->GetHub()->hub_depth + 1);
      hub.rh_port = state->GetHub()->rh_port;
    }
    state->GetHub() = hub;
    uint8_t slot = state->GetSlot();
    size_t slot_size = (hcc_.CSZ()) ? 64 : 32;
    // Initialize input slot context data structure (6.2.2) with 1 context entry
    // Set root hub port number to port number and context entries to 1
    auto control = static_cast<uint32_t*>(state->GetInputContext()->virt());
    // Evaluate slot context
    control[0] = 0;
    control[1] = 1;
    auto slot_context =
        reinterpret_cast<SlotContext*>(reinterpret_cast<unsigned char*>(control) + slot_size);
    slot_context->set_SPEED(speed)
        .set_MULTI_TT(multi_tt)
        .set_HUB(1)
        .set_PORT_COUNT(desc->bNbrPorts)
        .set_TTT((speed == USB_SPEED_HIGH) ? ((desc->wHubCharacteristics >> 5) & 3) : 0);
    Control::Get().FromValue(0).set_Type(Control::EvaluateContextCommand).ToTrb(&cmd);
    cmd.set_SlotID(slot).set_BSR(0);
    cmd.ptr = state->GetInputContext()->phys()[0];
    hw_mb();
    context = command_ring_.AllocateContext();
  }
  return SubmitCommand(cmd, std::move(context))
      .then([=](fit::result<TRB*, zx_status_t>& result) {
        if (result.is_error()) {
          return ResultToTRBPromise(result);
        }
        auto completion = reinterpret_cast<CommandCompletionEvent*>(result.value());
        if (completion->CompletionCode() != CommandCompletionEvent::Success) {
          return ResultToTRBPromise(fit::error(ZX_ERR_IO));
        }
        if (speed == USB_SPEED_SUPER) {
          std::optional<usb::Request<void>> request_wrapper;
          zx_status_t status =
              usb::Request<void>::Alloc(&request_wrapper, 0, 0, sizeof(usb_request_t));
          if (status != ZX_OK) {
            return ResultToTRBPromise(fit::error(status));
          }
          usb_request_t* request = request_wrapper->request();
          request->direct = true;
          request->header.device_id = device_id;
          request->header.ep_address = 0;
          request->setup.bmRequestType = USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_DEVICE;
          {
            fbl::AutoLock _(&state->transaction_lock());
            request->setup.wValue = state->GetHub()->hub_depth;
          }
          request->setup.wIndex = 0;
          request->setup.bRequest = USB_HUB_SET_DEPTH;
          request->setup.wLength = 0;
          return USBRequestToTRBPromise(UsbHciRequestQueue(std::move(*request_wrapper)).box());
        }
        return ResultToTRBPromise(result);
      })
      .box();
}

fit::promise<OwnedRequest, void> UsbXhci::TRBToUSBRequestPromise(TRBPromise promise,
                                                                 OwnedRequest req) {
  return promise
      .then([request = std::move(req)](fit::result<TRB*, zx_status_t>& result) mutable {
        if (result.is_error()) {
          request.request()->response.status = result.error();
        }
        auto completion_code =
            static_cast<CommandCompletionEvent*>(result.value())->CompletionCode();
        if (completion_code != CommandCompletionEvent::Success) {
          request.request()->response.status = ZX_ERR_IO;
        }
        return fit::ok(std::move(request));
      })
      .box();
}

TRBPromise UsbXhci::USBRequestToTRBPromise(fit::promise<OwnedRequest, void> promise) {
  return promise
      .then([=](fit::result<OwnedRequest, void>& result) -> fit::result<TRB*, zx_status_t> {
        if (result.value().request()->response.status != ZX_OK) {
          return fit::error(result.value().request()->response.status);
        }
        return fit::ok<TRB*>(nullptr);
      })
      .box();
}

void UsbXhci::DdkSuspend(ddk::SuspendTxn txn) {
  sync_completion_wait(&init_complete_, ZX_TIME_INFINITE);
  if (!mmio_.has_value()) {
    txn.Reply(ZX_ERR_BAD_STATE, 0);
    return;
  }
  // TODO(fxb/42612) do different things based on the requested_state and suspend reason.
  // for now we shutdown the driver in preparation for mexec
  USBCMD::Get(cap_length_).ReadFrom(&mmio_.value()).set_ENABLE(0).WriteTo(&mmio_.value());
  while (!USBSTS::Get(cap_length_).ReadFrom(&mmio_.value()).HCHalted()) {
  }
  txn.Reply(ZX_OK, 0);
}

void UsbXhci::DdkUnbindNew(ddk::UnbindTxn txn) {
  // Prevent anything external to us from queueing any more work during shutdown.
  sync_completion_wait(&init_complete_, ZX_TIME_INFINITE);

  running_ = false;
  PostCallback([this, transaction = std::move(txn)](
                   const ddk::UsbBusInterfaceProtocolClient& client) mutable {
    ddk_interaction_loop_.Quit();
    USBCMD::Get(cap_length_).ReadFrom(&mmio_.value()).set_ENABLE(0).WriteTo(&mmio_.value());
    while (!USBSTS::Get(cap_length_).ReadFrom(&mmio_.value()).HCHalted()) {
    }
    // Disable all interrupters

    uint32_t active_interrupters;
    {
      // This is safe because no new interrupters will be added after the running
      // bit is set to false.
      fbl::AutoLock _(&scheduler_lock_);
      active_interrupters = active_interrupters_;
    }
    for (size_t i = 0; i < active_interrupters; i++) {
      interrupters_[i].Stop();
    }
    // Should now be safe to terminate everything on the command ring
    bool pending;
    do {
      pending = false;
      auto trbs = command_ring_.TakePendingTRBs();
      for (auto& trb : trbs) {
        pending = true;
        CommandCompletionEvent evt;
        evt.ptr = 0;
        evt.set_Type(Control::CommandCompletionEvent);
        evt.set_CompletionCode(CommandCompletionEvent::CommandRingStopped);
        if (trb.completer.has_value()) {
          trb.completer.value().complete_ok(trb.trb);
        }
      }
      // Ensure that we've actually invoked the completions above
      // before moving to the next step.
      // TODO (fxb/44375): Migrate to joins
      RunUntilIdle();
      for (size_t i = 0; i < max_slots_; i++) {
        fbl::DoublyLinkedList<std::unique_ptr<TRBContext>> trbs;
        {
          fbl::AutoLock _(&device_state_[i].transaction_lock());
          trbs = device_state_[i].GetTransferRing().TakePendingTRBs();
        }
        for (auto& trb : trbs) {
          pending = true;
          trb.request->Complete(ZX_ERR_IO_NOT_PRESENT, 0);
        }
        for (size_t c = 0; c < 32; c++) {
          fbl::DoublyLinkedList<std::unique_ptr<TRBContext>> trbs;
          {
            fbl::AutoLock _(&device_state_[i].transaction_lock());
            trbs = device_state_[i].GetTransferRing(c).TakePendingTRBs();
          }
          for (auto& trb : trbs) {
            pending = true;
            trb.request->Complete(ZX_ERR_IO_NOT_PRESENT, 0);
          }
        }
      }
      // Flush any outstanding async I/O
      // TODO (fxb/44375): Migrate to joins
      RunUntilIdle();
    } while (pending);
    interrupters_.reset();
    transaction.Reply();
    return ZX_OK;
  });
}

void UsbXhci::DdkRelease() {
  int output;
  if (ddk_interaction_thread_.has_value()) {
    thrd_join(*ddk_interaction_thread_, &output);
  }
  if (init_thread_.has_value()) {
    thrd_join(*init_thread_, &output);
  }
  delete this;
}

// USB HCI protocol implementation.
void UsbXhci::UsbHciRequestQueue(usb_request_t* usb_request_,
                                 const usb_request_complete_t* complete_cb_) {
  Request request(usb_request_, *complete_cb_, sizeof(usb_request_t));

  if (!running_) {
    request.Complete(ZX_ERR_IO_NOT_PRESENT, 0);
    return;
  }
  if (request.request()->header.device_id >= params_.MaxSlots()) {
    request.Complete(ZX_ERR_INVALID_ARGS, 0);
    return;
  }
  {
    fbl::AutoLock _(&device_state_[request.request()->header.device_id].transaction_lock());
    if (!device_state_[request.request()->header.device_id].GetSlot()) {
      request.Complete(ZX_ERR_IO_NOT_PRESENT, 0);
      return;
    }
  }
  if (unlikely(request.request()->header.ep_address == 0)) {
    UsbHciControlRequestQueue(std::move(request));
  } else {
    UsbHciNormalRequestQueue(std::move(request));
  }
}

void UsbXhci::WaitForIsochronousReady(UsbRequestState* state) {
  // Cannot schedule more than 895 microseconds into the future per section 4.11.2.5
  // in the xHCI specification (revision 1.2)
  constexpr int kMaxSchedulingInterval = 895;
  if (state->context->request->request()->header.frame) {
    uint64_t frame = UsbHciGetCurrentFrame();
    while (static_cast<int32_t>(state->context->request->request()->header.frame - frame) >=
           kMaxSchedulingInterval) {
      uint32_t time =
          static_cast<uint32_t>((state->context->request->request()->header.frame - frame) -
                                kMaxSchedulingInterval) *
          1000;
      zx::nanosleep(zx::deadline_after(zx::msec(time)));
      frame = UsbHciGetCurrentFrame();
    }

    if (state->context->request->request()->header.frame < frame) {
      state->complete = true;
      state->status = ZX_ERR_IO;
      state->bytes_transferred = 0;
    }
  }
}

void UsbXhci::StartNormalTransaction(UsbRequestState* state) {
  size_t packet_count = 0;

  // Normal transfer
  zx_status_t status = state->context->request->PhysMap(bti_);
  if (status != ZX_OK) {
    state->complete = true;
    state->status = status;
    state->bytes_transferred = 0;
    return;
  }
  TRB* current_trb = nullptr;
  bool first_cycle = false;
  size_t pending_len = state->context->request->request()->header.length;
  uint32_t total_len = 0;
  for (auto [paddr, len] : state->context->request->phys_iter(0)) {
    if (len > pending_len) {
      len = pending_len;
    }
    if (!paddr) {
      break;
    }
    if (!len) {
      continue;
    }
    total_len += static_cast<uint32_t>(len);
    packet_count++;
    TRB* prev = current_trb;
    pending_len -= len;
    zx_status_t status = state->transfer_ring->AllocateTRB(&current_trb, nullptr);
    if (!pending_len) {
      state->last_trb = current_trb;
    }
    if (status != ZX_OK) {
      state->complete = true;
      state->status = status;
      state->bytes_transferred = 0;
      return;
    }

    static_assert(sizeof(TRB*) == sizeof(uint64_t));
    if (likely(prev)) {
      prev->ptr = reinterpret_cast<uint64_t>(current_trb);
    } else {
      state->first_trb = current_trb;
      first_cycle = current_trb->status;
    }
  }

  if (pending_len) {
    // Something doesn't add up here....
    state->complete = true;
    state->status = ZX_ERR_BAD_STATE;
    state->bytes_transferred = 0;
    return;
  }
  state->total_len = total_len;
  state->packet_count = packet_count;
  state->first_cycle = first_cycle;
}

void UsbXhci::ContinueNormalTransaction(UsbRequestState* state) {
  // TODO(fxb/42611): Assign an interrupter dynamically from the pool
  // Data stage
  if (state->first_trb) {
    TRB* current = state->first_trb;
    for (auto [paddr, len] : state->context->request->phys_iter(0)) {
      if (!len) {
        break;
      }
      state->packet_count--;
      TRB* next = reinterpret_cast<TRB*>(current->ptr);
      uint32_t pcs = current->status;
      current->status = 0;
      enum Control::Type type;
      if (((state->is_isochronous_transfer) && state->first_trb == current)) {
        // Force direct mode as workaround for USB audio latency issue.
        type = Control::Isoch;
        Isoch* data = reinterpret_cast<Isoch*>(current);
        // Burst size is number of packets, not bytes
        uint32_t burst_size = state->burst_size;
        uint32_t packet_size = state->max_packet_size;
        uint32_t packet_count = state->total_len / packet_size;
        if (!packet_count) {
          packet_count = 1;
        }
        // Number of bursts - 1
        uint32_t burst_count = packet_count / burst_size;
        if (burst_count) {
          burst_count--;
        }
        // Zero-based last-burst-packet count (where 0 == 1 packet)
        uint32_t last_burst_packet_count = packet_count % burst_size;
        if (last_burst_packet_count) {
          last_burst_packet_count--;
        }
        data->set_CHAIN(next != nullptr)
            .set_SIA(state->context->request->request()->header.frame == 0)
            .set_TLBPC(last_burst_packet_count)
            .set_FrameID(
                static_cast<uint32_t>(state->context->request->request()->header.frame % 2048))
            .set_TBC(burst_count)
            .set_INTERRUPTER(0)
            .set_LENGTH(static_cast<uint16_t>(len))
            .set_SIZE(static_cast<uint32_t>(packet_count))
            .set_NO_SNOOP(!has_coherent_cache_)
            .set_IOC(next == nullptr)
            .set_ISP(next != nullptr);
      } else {
        type = Control::Normal;
        Normal* data = reinterpret_cast<Normal*>(current);
        data->set_CHAIN(next != nullptr)
            .set_INTERRUPTER(0)
            .set_LENGTH(static_cast<uint16_t>(len))
            .set_SIZE(static_cast<uint32_t>(state->packet_count))
            .set_NO_SNOOP(!has_coherent_cache_)
            .set_IOC(next == nullptr)
            .set_ISP(next != nullptr);
      }

      current->ptr = paddr;
      Control::FromTRB(current)
          .set_Cycle(unlikely(current == state->first_trb) ? !pcs : pcs)
          .set_Type(type)
          .ToTrb(current);
      current = next;
    }
  }
}

void UsbXhci::CommitNormalTransaction(UsbRequestState* state) {
  hw_mb();
  // Start the transaction!
  if (!has_coherent_cache_) {
    usb_request_cache_flush_invalidate(state->context->request->request(), 0,
                                       state->context->request->request()->header.length);
  }
  state->transfer_ring->AssignContext(state->last_trb, std::move(state->context), state->first_trb);
  Control::FromTRB(state->first_trb).set_Cycle(state->first_cycle).ToTrb(state->first_trb);

  state->transfer_ring->CommitTransaction(state->transaction);
  DOORBELL::Get(doorbell_offset_, state->slot)
      .FromValue(0)
      .set_Target(2 + state->index)
      .WriteTo(&mmio_.value());
}

bool UsbXhci::UsbRequestState::Complete() {
  if (complete) {
    context->request->Complete(status, bytes_transferred);
    return true;
  }
  return false;
}

void UsbXhci::UsbHciNormalRequestQueue(Request request) {
  UsbRequestState pending_transfer;
  uint8_t index = static_cast<uint8_t>(XhciEndpointIndex(request.request()->header.ep_address) - 1);
  auto& state = device_state_[request.request()->header.device_id];
  fbl::AutoLock transaction_lock(&state.transaction_lock());
  auto* control = reinterpret_cast<uint32_t*>(state.GetInputContext()->virt());
  size_t slot_size = (hcc_.CSZ()) ? 64 : 32;
  auto endpoint_context = reinterpret_cast<EndpointContext*>(
      reinterpret_cast<unsigned char*>(control) + (slot_size * (2 + (index + 1))));
  if (!state.GetTransferRing((index)).active()) {
    return;
  }
  pending_transfer.is_isochronous_transfer = state.GetTransferRing(index).IsIsochronous();
  pending_transfer.transfer_ring = &state.GetTransferRing(index);
  pending_transfer.burst_size = endpoint_context->MaxBurstSize() + 1;
  pending_transfer.max_packet_size = endpoint_context->MAX_PACKET_SIZE();
  pending_transfer.slot_size = (hcc_.CSZ()) ? 64 : 32;
  pending_transfer.complete = false;
  pending_transfer.index = index;
  pending_transfer.context = state.GetTransferRing(index).AllocateContext();
  pending_transfer.context->request = std::move(request);
  pending_transfer.slot =
      device_state_[pending_transfer.context->request->request()->header.device_id].GetSlot();
  if (!pending_transfer.context) {
    transaction_lock.release();
    pending_transfer.context->request->Complete(ZX_ERR_NO_MEMORY, 0);
    return;
  }

  if (pending_transfer.is_isochronous_transfer) {
    // Release the lock while we're sleeping to avoid blocking
    // other operations.
    state.transaction_lock().Release();
    WaitForIsochronousReady(&pending_transfer);
    if (pending_transfer.Complete()) {
      state.transaction_lock().Acquire();
      return;
    }
    state.transaction_lock().Acquire();
  }

  // Start the transaction
  pending_transfer.transaction = state.GetTransferRing(index).SaveState();
  auto rollback_transaction = [&]() __TA_NO_THREAD_SAFETY_ANALYSIS {
    state.GetTransferRing(index).Restore(pending_transfer.transaction);
  };
  StartNormalTransaction(&pending_transfer);
  if (pending_transfer.complete) {
    rollback_transaction();
    transaction_lock.release();
    pending_transfer.Complete();
    return;
  }
  // Continue the transaction
  ContinueNormalTransaction(&pending_transfer);
  if (pending_transfer.complete) {
    rollback_transaction();
    transaction_lock.release();
    pending_transfer.Complete();
    return;
  }
  // Commit the transaction -- starting the actual transfer
  CommitNormalTransaction(&pending_transfer);
}

void UsbXhci::UsbHciControlRequestQueue(Request req) {
  auto device_state = &device_state_[req.request()->header.device_id];
  fbl::AutoLock transaction_lock(&device_state->transaction_lock());
  auto context = device_state->GetTransferRing().AllocateContext();
  if (!context) {
    transaction_lock.release();
    req.Complete(ZX_ERR_NO_MEMORY, 0);
    return;
  }
  TransferRing::State transaction;
  TRB* setup;
  zx_status_t status = device_state->GetTransferRing().AllocateTRB(&setup, &transaction);
  auto rollback_transaction = [=]() __TA_NO_THREAD_SAFETY_ANALYSIS {
    device_state->GetTransferRing().Restore(transaction);
  };
  if (status != ZX_OK) {
    rollback_transaction();
    transaction_lock.release();
    req.Complete(status, 0);
    return;
  }

  context->request = std::move(req);
  UsbRequestState pending_transfer;
  pending_transfer.context = std::move(context);
  pending_transfer.setup = setup;
  pending_transfer.transaction = transaction;
  pending_transfer.transfer_ring = &device_state->GetTransferRing();
  pending_transfer.slot = device_state->GetSlot();
  ControlRequestAllocationPhase(&pending_transfer);
  fbl::AutoCall call([&]() __TA_NO_THREAD_SAFETY_ANALYSIS {
    rollback_transaction();
    transaction_lock.release();
    pending_transfer.Complete();
  });
  if (pending_transfer.complete) {
    return;
  }
  ControlRequestStatusPhase(&pending_transfer);
  if (pending_transfer.complete) {
    return;
  }
  ControlRequestDataPhase(&pending_transfer);
  if (pending_transfer.complete) {
    return;
  }
  ControlRequestSetupPhase(&pending_transfer);
  if (pending_transfer.complete) {
    return;
  }
  ControlRequestCommit(&pending_transfer);
  call.cancel();
}

void UsbXhci::ControlRequestAllocationPhase(UsbRequestState* state) {
  state->setup_cycle = state->setup->status;
  state->setup->status = 0;
  if (state->context->request->request()->header.length) {
    zx_status_t status = state->context->request->PhysMap(bti_);
    if (status != ZX_OK) {
      state->status = status;
      state->complete = true;
      state->bytes_transferred = 0;
      return;
    }
    TRB* current_trb = nullptr;
    for (auto [paddr, len] : state->context->request->phys_iter(0)) {
      if (!len) {
        break;
      }
      state->packet_count++;
      TRB* prev = current_trb;
      zx_status_t status = state->transfer_ring->AllocateTRB(&current_trb, nullptr);
      if (status != ZX_OK) {
        state->status = status;
        state->complete = true;
        state->bytes_transferred = 0;
        return;
      }
      static_assert(sizeof(TRB*) == sizeof(uint64_t));
      if (likely(prev)) {
        prev->ptr = reinterpret_cast<uint64_t>(current_trb);
      } else {
        state->first_trb = current_trb;
      }
    }
  }
}

void UsbXhci::ControlRequestStatusPhase(UsbRequestState* state) {
  // TODO(fxb/42611): Assign an interrupter dynamically from the pool
  state->interrupter = 0;
  bool status_in = true;
  // See table 4-7 in section 4.11.2.2
  if (state->first_trb && (state->context->request->request()->setup.bmRequestType & USB_DIR_IN)) {
    status_in = false;
  }
  zx_status_t status = state->transfer_ring->AllocateTRB(&state->status_trb_ptr, nullptr);
  if (status != ZX_OK) {
    state->status = status;
    state->complete = true;
    state->bytes_transferred = 0;
    return;
  }
  Control::FromTRB(state->status_trb_ptr)
      .set_Cycle(state->status_trb_ptr->status)
      .set_Type(Control::Status)
      .ToTrb(state->status_trb_ptr);
  state->status_trb_ptr->status = 0;
  auto* status_trb = static_cast<Status*>(state->status_trb_ptr);
  status_trb->set_DIRECTION(status_in).set_INTERRUPTER(state->interrupter).set_IOC(1);
}

void UsbXhci::ControlRequestDataPhase(UsbRequestState* state) {
  // Data stage
  if (state->first_trb) {
    TRB* current = state->first_trb;
    for (auto [paddr, len] : state->context->request->phys_iter(0)) {
      if (!len) {
        break;
      }
      state->packet_count--;
      TRB* next = reinterpret_cast<TRB*>(current->ptr);
      uint32_t pcs = current->status;
      current->status = 0;
      enum Control::Type type;
      if (current == state->first_trb) {
        type = Control::Data;
        ControlData* data = reinterpret_cast<ControlData*>(current);
        // Control transfers always get interrupter 0 (we consider those to be low-priority)
        // TODO (ZX-4288): Change bus snooping options based on input from higher-level drivers.
        data->set_CHAIN(next != nullptr)
            .set_DIRECTION((state->context->request->request()->setup.bmRequestType & USB_DIR_IN) !=
                           0)
            .set_INTERRUPTER(0)
            .set_LENGTH(static_cast<uint16_t>(len))
            .set_SIZE(static_cast<uint32_t>(state->packet_count))
            .set_ISP(true)
            .set_NO_SNOOP(!has_coherent_cache_);
      } else {
        type = Control::Normal;
        Normal* data = reinterpret_cast<Normal*>(current);
        data->set_CHAIN(next != nullptr)
            .set_INTERRUPTER(0)
            .set_LENGTH(static_cast<uint16_t>(len))
            .set_SIZE(static_cast<uint32_t>(state->packet_count))
            .set_ISP(true)
            .set_NO_SNOOP(!has_coherent_cache_);
      }
      current->ptr = paddr;
      Control::FromTRB(current).set_Cycle(pcs).set_Type(type).ToTrb(current);
      current = next;
    }
  }
}

void UsbXhci::ControlRequestSetupPhase(UsbRequestState* state) {
  // Setup phase (4.11.2.2)
  memcpy(&state->setup->ptr, &state->context->request->request()->setup,
         sizeof(state->context->request->request()->setup));
  Setup* setup_trb = reinterpret_cast<Setup*>(state->setup);
  setup_trb->set_INTERRUPTER(state->interrupter)
      .set_length(8)
      .set_IDT(1)
      .set_TRT(((state->context->request->request()->setup.bmRequestType & USB_DIR_IN) != 0)
                   ? Setup::IN
                   : Setup::OUT);
  hw_mb();
}

void UsbXhci::ControlRequestCommit(UsbRequestState* state) {
  // Start the transaction!
  if (!has_coherent_cache_) {
    usb_request_cache_flush_invalidate(state->context->request->request(), 0,
                                       state->context->request->request()->header.length);
  }
  state->transfer_ring->AssignContext(state->status_trb_ptr, std::move(state->context),
                                      state->first_trb);
  Control::FromTRB(state->setup)
      .set_Type(Control::Setup)
      .set_Cycle(state->setup_cycle)
      .ToTrb(state->setup);
  state->transfer_ring->CommitTransaction(state->transaction);
  DOORBELL::Get(doorbell_offset_, state->slot).FromValue(0).set_Target(1).WriteTo(&mmio_.value());
}

void UsbXhci::UsbHciSetBusInterface(const usb_bus_interface_protocol_t* bus_intf) {
  // We must be unbinding if the bus is currently valid.
  if (bus_.is_valid()) {
    // Assert that we've started unbinding and are no longer accepting
    // any requests to prevent a use-after-free situation.
    ZX_ASSERT(!running_);
    return;
  }
  ZX_ASSERT(bus_intf != nullptr);
  bus_ = bus_intf;
  sync_completion_signal(&bus_completion);
}
size_t UsbXhci::UsbHciGetMaxDeviceCount() {
  // Last two slots represent the virtual hubs (USB 2.0 and 3.0 respectively)
  return params_.MaxSlots() + 2;
}

zx_status_t UsbXhci::UsbHciEnableEndpoint(uint32_t device_id,
                                          const usb_endpoint_descriptor_t* ep_desc,
                                          const usb_ss_ep_comp_descriptor_t* ss_com_desc,
                                          bool enable) {
  if (!running_) {
    return ZX_ERR_IO_NOT_PRESENT;
  }
  if (device_id >= params_.MaxSlots()) {
    // TODO: Root hub endpoint support
    return ZX_ERR_OUT_OF_RANGE;
  }
  if (!enable) {
    return TRBWait(UsbHciDisableEndpoint(device_id, ep_desc, ss_com_desc));
  }
  return TRBWait(UsbHciEnableEndpoint(device_id, ep_desc, ss_com_desc));
}

TRBPromise UsbXhci::UsbHciEnableEndpoint(uint32_t device_id,
                                         const usb_endpoint_descriptor_t* ep_desc,
                                         const usb_ss_ep_comp_descriptor_t* ss_com_desc) {
  auto context = command_ring_.AllocateContext();
  auto state = &device_state_[device_id];
  SlotContext* slot_context;
  TRB trb;
  uint32_t context_entries;
  uint8_t index;
  {
    fbl::AutoLock _(&state->transaction_lock());
    auto control = reinterpret_cast<uint32_t*>(state->GetInputContext()->virt());
    size_t slot_size = (hcc_.CSZ()) ? 64 : 32;
    // Initialize input slot context data structure (6.2.2) with 1 context entry
    // Set root hub port number to port number and context entries to 1
    slot_context =
        reinterpret_cast<SlotContext*>(reinterpret_cast<unsigned char*>(control) + slot_size);
    context_entries = slot_context->CONTEXT_ENTRIES();
    index = XhciEndpointIndex(ep_desc->bEndpointAddress);
    if (index >= context_entries) {
      slot_context->set_CONTEXT_ENTRIES(index + 1);
    }
    // Allocate the transfer ring (see section 4.9)
    // TODO (bbosak): Assign an Interrupter from the pool
    control[0] = 0;
    control[1] = 1 | (1 << (index + 1));
    // TODO(bbosak): Dynamically assign an event ring
    uint32_t event_ring = 0;
    zx_status_t status = state->GetTransferRing(index - 1).Init(
        page_size_, bti_, &this->interrupters_[event_ring].ring(), is_32bit_, &mmio_.value(),
        *this);
    if (status != ZX_OK) {
      return ResultToTRBPromise(fit::error(status));
    }
    CRCR trb_phys = state->GetTransferRing(index - 1).phys(cap_length_);
    // Initialize endpoint context 0
    // Set CERR to 3, TR dequeue pointer, max packet size, EP type = control, DCS = 1
    auto endpoint_context = reinterpret_cast<EndpointContext*>(
        reinterpret_cast<unsigned char*>(control) + (slot_size * (2 + index)));

    // See section 4.3.6
    uint32_t ep_type = ep_desc->bmAttributes & USB_ENDPOINT_TYPE_MASK;
    if (ep_type == USB_ENDPOINT_ISOCHRONOUS) {
      state->GetTransferRing(index - 1).SetIsochronous();
    }
    uint32_t ep_index = ep_type;
    if ((ep_desc->bEndpointAddress & USB_ENDPOINT_DIR_MASK) == USB_ENDPOINT_IN) {
      ep_index += 4;
    }
    endpoint_context->Init(static_cast<EndpointContext::EndpointType>(ep_index), trb_phys,
                           ep_desc->wMaxPacketSize & 0x07FF);
    int interval = ComputeInterval(ep_desc, slot_context->SPEED());
    if (interval == -1) {
      interval = 1;
    }
    endpoint_context->set_Interval(interval);
    // Section 6.2.3.4
    uint32_t max_burst = 0;
    if (ss_com_desc) {
      max_burst = ss_com_desc->bMaxBurst;
    } else {
      // TODO: Handle special case for interrupt/isochronous endpoints
      if ((slot_context->SPEED() == USB_SPEED_HIGH) && (ep_type == USB_ENDPOINT_ISOCHRONOUS)) {
        max_burst = (le16toh((ep_desc)->wMaxPacketSize) >> 11) & 3;
      }
    }
    endpoint_context->set_MaxBurstSize(max_burst);
    if (ep_type == USB_ENDPOINT_ISOCHRONOUS) {
      endpoint_context->set_MAX_ESIT_PAYLOAD_LOW((ep_desc->wMaxPacketSize & 0x07FF) * max_burst);
    }
    trb.ptr = state->GetInputContext()->phys()[0];
    Control::Get()
        .FromValue((device_id + 1) << 24)
        .set_Type(Control::ConfigureEndpointCommand)
        .ToTrb(&trb);
  }
  // TODO (ZX-4363): Implement async support
  hw_mb();
  return SubmitCommand(trb, std::move(context))
      .then([=](fit::result<TRB*, zx_status_t>& result) {
        fbl::AutoCall free_buffers([=]() {
          fbl::AutoLock _(&state->transaction_lock());
          state->GetTransferRing(index - 1).Deinit();
          slot_context->set_CONTEXT_ENTRIES(context_entries);
        });
        if (result.is_error()) {
          return result;
        }
        auto completion = static_cast<CommandCompletionEvent*>(result.value());
        bool success = completion->CompletionCode() == CommandCompletionEvent::Success;
        if (success) {
          free_buffers.cancel();
        } else {
          return fit::result<TRB*, zx_status_t>(fit::error(ZX_ERR_IO));
        }
        return result;
      })
      .box();
}

TRBPromise UsbXhci::UsbHciDisableEndpoint(uint32_t device_id,
                                          const usb_endpoint_descriptor_t* ep_desc,
                                          const usb_ss_ep_comp_descriptor_t* ss_com_desc) {
  auto context = command_ring_.AllocateContext();
  auto state = &device_state_[device_id];
  size_t slot_size = (hcc_.CSZ()) ? 64 : 32;
  uint8_t index = XhciEndpointIndex(ep_desc->bEndpointAddress);
  TRB trb;
  uint32_t* control;
  {
    fbl::AutoLock _(&state->transaction_lock());
    control = reinterpret_cast<uint32_t*>(state->GetInputContext()->virt());
    // Initialize input slot context data structure (6.2.2) with 1 context entry
    // Set root hub port number to port number and context entries to 1
    control[0] = (1 << (index + 1));
    control[1] = 1;
    trb.ptr = state->GetInputContext()->phys()[0];
    Control::Get()
        .FromValue((device_id + 1) << 24)
        .set_Type(Control::ConfigureEndpointCommand)
        .ToTrb(&trb);
  }
  // TODO (ZX-4363): Implement async support
  hw_mb();
  return SubmitCommand(trb, std::move(context))
      .then([=](fit::result<TRB*, zx_status_t>& result) -> fit::result<TRB*, zx_status_t> {
        if (result.is_error()) {
          return fit::error(ZX_ERR_BAD_STATE);
        }
        auto completion = reinterpret_cast<CommandCompletionEvent*>(result.value());
        bool success = completion->CompletionCode() == CommandCompletionEvent::Success;
        if (!success) {
          return fit::error(ZX_ERR_BAD_STATE);
        }
        auto endpoint_context = reinterpret_cast<EndpointContext*>(
            reinterpret_cast<unsigned char*>(control) + (slot_size * (2 + index)));
        endpoint_context->Deinit();
        fbl::AutoLock _(&state->transaction_lock());
        zx_status_t status = state->GetTransferRing(index - 1).Deinit();
        // If we can't deinit the ring something is seriously wrong.
        if (status != ZX_OK) {
          return fit::error(ZX_ERR_BAD_STATE);
        }
        return result;
      })
      .box();
}

uint64_t UsbXhci::UsbHciGetCurrentFrame() {
  if (!running_) {
    return 0;
  }
  uint32_t mfindex = MFINDEX::Get(runtime_offset_).ReadFrom(&mmio_.value()).INDEX();
  if (mfindex < last_mfindex_) {
    // Wrapped
    wrap_count_++;
  }

  last_mfindex_ = mfindex;
  uint64_t wrap_count = wrap_count_;
  // shift three to convert from 125us microframes to 1ms frames
  return ((wrap_count * (1 << 14)) + mfindex) >> 3;
}

zx_status_t UsbXhci::UsbHciConfigureHub(uint32_t device_id, usb_speed_t speed,
                                        const usb_hub_descriptor_t* desc, bool multi_tt) {
  if (!running_) {
    return ZX_ERR_IO_NOT_PRESENT;
  }
  sync_completion_t completion;
  zx_status_t hub_status = ZX_OK;
  ScheduleTask(ConfigureHubAsync(device_id, speed, desc, multi_tt)
                   .then([&](fit::result<TRB*, zx_status_t>& result) {
                     if (result.is_ok()) {
                       hub_status = ZX_OK;
                     } else {
                       hub_status = result.error();
                     }
                     sync_completion_signal(&completion);
                     return result;
                   })
                   .box());
  sync_completion_wait(&completion, ZX_TIME_INFINITE);
  return hub_status;
}
zx_status_t UsbXhci::UsbHciHubDeviceAdded(uint32_t device_id, uint32_t port, usb_speed_t speed) {
  if (!running_) {
    return ZX_ERR_IO_NOT_PRESENT;
  }
  sync_completion_t completion;
  zx_status_t out_status;
  ScheduleTask(UsbHciHubDeviceAddedAsync(device_id, port, speed)
                   .then([&](fit::result<TRB*, zx_status_t>& result) {
                     if (result.is_ok()) {
                       out_status = ZX_OK;
                     } else {
                       out_status = result.error();
                     }
                     sync_completion_signal(&completion);
                     return result;
                   })
                   .box());
  sync_completion_wait(&completion, ZX_TIME_INFINITE);
  return ZX_OK;
}

zx_status_t UsbXhci::UsbHciHubDeviceRemoved(uint32_t hub_id, uint32_t port) {
  if (!running_) {
    return ZX_ERR_IO_NOT_PRESENT;
  }
  auto hub_state = &device_state_[hub_id];
  uint32_t slot;
  {
    fbl::AutoLock _(&hub_state->transaction_lock());
    // In the case where the hub itself is unplugged,
    // we will likely have torn down the hub state
    // prior to teardown of devices connected to said hub.
    // If this is the case, just return OK.
    // Teardown of child devices will complete asynchronously
    if (!hub_state->GetHub()) {
      return ZX_OK;
    }
    uint32_t device_id = hub_state->GetHub()->port_to_device[port - 1];
    auto device_state = &device_state_[device_id];
    slot = device_state->GetSlot();
  }
  bool success = false;
  sync_completion_t event;
  for (size_t i = 0; i < 32; i++) {
    fbl::DoublyLinkedList<std::unique_ptr<TRBContext>> trbs;
    {
      fbl::AutoLock _(&device_state_[slot - 1].transaction_lock());
      trbs = device_state_[slot - 1].GetTransferRing(i).TakePendingTRBs();
    }
    for (auto& trb : trbs) {
      trb.request->Complete(ZX_ERR_IO_NOT_PRESENT, 0);
    }
  }
  RunUntilIdle();
  fbl::DoublyLinkedList<std::unique_ptr<TRBContext>> trbs;
  {
    fbl::AutoLock _(&device_state_[slot - 1].transaction_lock());
    trbs = device_state_[slot - 1].GetTransferRing().TakePendingTRBs();
  }
  for (auto& trb : trbs) {
    trb.request->Complete(ZX_ERR_IO_NOT_PRESENT, 0);
  }
  RunUntilIdle();
  // Bus should always be valid since we're getting a callback from a hub
  // that is a child of the bus.
  ZX_ASSERT(bus_.is_valid());
  zx_status_t status = bus_.RemoveDevice(slot - 1);
  if (status != ZX_OK) {
    return status;
  }
  ScheduleTask(DisableSlotCommand(slot)
                   .then([&](fit::result<TRB*, zx_status_t>& result) {
                     if (result.is_error()) {
                       success = false;
                       return result;
                     }
                     auto completion = static_cast<CommandCompletionEvent*>(result.value());
                     success = completion->CompletionCode() == CommandCompletionEvent::Success;
                     sync_completion_signal(&event);
                     return result;
                   })
                   .box());
  sync_completion_wait(&event, ZX_TIME_INFINITE);
  return success ? ZX_OK : ZX_ERR_IO;
}

zx_status_t UsbXhci::UsbHciHubDeviceReset(uint32_t device_id, uint32_t port) {
  return ZX_ERR_NOT_SUPPORTED;
}

zx_status_t UsbXhci::UsbHciResetEndpoint(uint32_t device_id, uint8_t ep_address) {
  if (device_id >= params_.MaxSlots()) {
    return ZX_ERR_NOT_SUPPORTED;
  }
  auto state = &device_state_[device_id];
  ResetEndpoint reset_command;
  {
    fbl::AutoLock _(&state->transaction_lock());
    reset_command.set_ENDPOINT(XhciEndpointIndex(ep_address));
    reset_command.set_SLOT(state->GetSlot());
  }
  auto context = command_ring_.AllocateContext();
  if (!context) {
    return ZX_ERR_NO_MEMORY;
  }
  zx_status_t status;
  sync_completion_t completion;
  ScheduleTask(
      SubmitCommand(reset_command, std::move(context))
          .then([&](fit::result<TRB*, zx_status_t>& result) -> fit::result<TRB*, zx_status_t> {
            if (result.is_error()) {
              status = result.error();
              sync_completion_signal(&completion);
              return result;
            }
            CommandCompletionEvent* evt = static_cast<CommandCompletionEvent*>(result.value());
            if (evt->CompletionCode() != CommandCompletionEvent::Success) {
              status = ZX_ERR_IO;
              sync_completion_signal(&completion);
              return fit::error(ZX_ERR_IO);
            }
            sync_completion_signal(&completion);
            return result;
          })
          .box());
  sync_completion_wait(&completion, ZX_TIME_INFINITE);
  if (status != ZX_OK) {
    return status;
  }
  return status;
}

// TODO (ZX-4864): Either decide what these reset methods should do,
// or get rid of them.
zx_status_t UsbXhci::UsbHciResetDevice(uint32_t hub_address, uint32_t device_id) {
  return ZX_ERR_NOT_SUPPORTED;
}

size_t UsbXhci::UsbHciGetMaxTransferSize(uint32_t device_id, uint8_t ep_address) {
  if (device_id >= params_.MaxSlots()) {
    // TODO: Root hub endpoint support
    return 0;
  }
  auto state = &device_state_[device_id];

  fbl::AutoLock _(&state->transaction_lock());
  auto control = reinterpret_cast<uint32_t*>(state->GetInputContext()->virt());
  size_t slot_size = (hcc_.CSZ()) ? 64 : 32;
  uint8_t index = XhciEndpointIndex(ep_address);
  auto endpoint_context = reinterpret_cast<EndpointContext*>(
      reinterpret_cast<unsigned char*>(control) + (slot_size * (2 + index)));
  return endpoint_context->MAX_PACKET_SIZE();
}

zx_status_t UsbXhci::UsbHciCancelAll(uint32_t device_id, uint8_t ep_address) {
  return TRBWait(UsbHciCancelAllAsync(device_id, ep_address));
}

TRBPromise UsbXhci::UsbHciCancelAllAsync(uint32_t device_id, uint8_t ep_address) {
  auto state = &device_state_[device_id];
  StopEndpoint stop;
  {
    fbl::AutoLock state_lock(&state->transaction_lock());
    uint8_t index = static_cast<uint8_t>(XhciEndpointIndex(ep_address) + 1);
    stop.set_ENDPOINT(index);
    stop.set_SLOT(state->GetSlot());
  }
  auto context = command_ring_.AllocateContext();
  return SubmitCommand(stop, std::move(context))
      .then([=](fit::result<TRB*, zx_status_t>& result) {
        if (result.is_error()) {
          return ResultToTRBPromise(result);
        }
        auto completion_event = static_cast<CommandCompletionEvent*>(result.value());
        auto completion_code = completion_event->CompletionCode();
        zx_status_t status =
            (completion_code == CommandCompletionEvent::Success) ? ZX_OK : ZX_ERR_IO;
        if (status != ZX_OK) {
          return fit::make_result_promise(fit::result<TRB*, zx_status_t>(fit::error(status))).box();
        }
        // We can now move everything off of the transfer ring starting at the dequeue pointer
        uint8_t index;
        fbl::DoublyLinkedList<std::unique_ptr<TRBContext>> trbs;
        zx_paddr_t new_ptr_phys = 0;
        {
          TRB* new_ptr = nullptr;
          fbl::AutoLock _(&state->transaction_lock());
          index = static_cast<uint8_t>(XhciEndpointIndex(ep_address) - 1);
          trbs = state->GetTransferRing(index).TakePendingTRBs();
          for (auto& trb : trbs) {
            new_ptr = trb.trb;
            auto control = Control::FromTRB(trb.trb);
            control.set_Cycle(!control.Cycle());
          }
          if (new_ptr) {
            new_ptr_phys = state->GetTransferRing(index).VirtToPhys(new_ptr + 1);
          }
        }
        for (auto& trb : trbs) {
          trb.request->Complete(ZX_ERR_CANCELED, 0);
        }
        // It's possible that the dequeue pointer was in the middle of a multi-TRB TD when we
        // stopped. If this is the case, we need to adjust the dequeue pointer to point to the index
        // of the first TRB that we know about.
        if (new_ptr_phys) {
          SetTRDequeuePointer cmd;
          cmd.set_ENDPOINT(index + 2);
          cmd.set_SLOT(state->GetSlot());
          cmd.ptr = new_ptr_phys;
          auto context = command_ring_.AllocateContext();
          return SubmitCommand(cmd, std::move(context))
              .then([=](fit::result<TRB*, zx_status_t>& result) -> fit::result<TRB*, zx_status_t> {
                if (result.is_error()) {
                  return result;
                }
                auto completion_event = static_cast<CommandCompletionEvent*>(result.value());
                auto completion_code = completion_event->CompletionCode();
                bool command_success = completion_code == CommandCompletionEvent::Success;
                zx_status_t status = command_success ? ZX_OK : ZX_ERR_IO;
                if (status == ZX_OK) {
                  return fit::ok(result.value());
                } else {
                  return fit::error(status);
                }
              })
              .box();
        } else {
          return fit::make_result_promise(fit::result<TRB*, zx_status_t>(fit::ok(result.value())))
              .box();
        }
      })
      .box();
}

size_t UsbXhci::UsbHciGetRequestSize() { return Request::RequestSize(sizeof(usb_request_t)); }

void UsbXhci::Shutdown(zx_status_t status) {
  USBCMD::Get(cap_length_).ReadFrom(&mmio_.value()).set_ENABLE(0).WriteTo(&mmio_.value());
  while (!USBSTS::Get(cap_length_).ReadFrom(&mmio_.value()).HCHalted()) {
  }
  if (status != ZX_OK) {
    // If we're shutting down due to an error (not just regular unbind)
    // ensure that
    DdkAsyncRemove();
  }
}

void UsbXhci::InitQuirks() {
  zx_pcie_device_info_t info;
  pci_.GetDeviceInfo(&info);
  if ((info.vendor_id == 0x1033) && (info.device_id == 0x194)) {
    qemu_quirk_ = true;
  }
  if ((info.vendor_id) == 0x8086 && (info.device_id == 0x8C31)) {
    // TODO (bbosak): Implement stub EHCI driver so we can properly
    // do the handoff in case the BIOS is managing a device on EHCI.
    // Quirk for some older Intel chipsets
    // Switch ports from EHCI to XHCI.
    uint32_t ports_available;
    pci_.ConfigRead32(0xdc, &ports_available);
    if (ports_available) {
      pci_.ConfigWrite32(0xd8, ports_available);
    }
    // Route power and data lines for USB 2.0 ports
    pci_.ConfigRead32(0xd4, &ports_available);
    if (ports_available) {
      pci_.ConfigWrite32(0xD0, ports_available);
    }
    // Handoff takes 5 seconds if we're contending with the EHCI controller.
    // (have to wait for enumeration to time out)
    sleep(5);
  }
}

zx_status_t UsbXhci::InitPci() {
  // Perform vendor-specific workarounds.
  InitQuirks();
  // PCIe interface supports cache snooping
  has_coherent_cache_ = true;
  // Initialize MMIO
  std::optional<ddk::MmioBuffer> buffer;
  zx_status_t status = pci_.MapMmio(0, ZX_CACHE_POLICY_UNCACHED_DEVICE, &buffer);
  if (status != ZX_OK) {
    return status;
  }
  mmio_ = std::move(*buffer);
  // TODO (bbosak): Convert to MSI-X when we have kernel support
  // for it (see kpci.c TODO).
  uint32_t irq_count;
  status = pci_.QueryIrqMode(ZX_PCIE_IRQ_MODE_MSI, &irq_count);
  if (status != ZX_OK) {
    return status;
  }
  if (pci_.SetIrqMode(ZX_PCIE_IRQ_MODE_MSI, irq_count) != ZX_OK) {
    return status;
  }
  irq_count_ = irq_count;
  fbl::AllocChecker ac;
  interrupters_.reset(new (&ac) Interrupter[irq_count]);
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }
  for (uint32_t i = 0; i < irq_count; i++) {
    status = pci_.MapInterrupt(i, &interrupters_[i].GetIrq());
    if (status != ZX_OK) {
      return status;
    }
  }
  status = pci_.EnableBusMaster(true);
  if (status != ZX_OK) {
    return status;
  }
  return ZX_OK;
}

zx_status_t UsbXhci::InitMmio() {
  if (!pdev_.is_valid()) {
    return 0;
  }
  std::optional<ddk::MmioBuffer> mmio;
  if (pdev_.MapMmio(0, &mmio) != ZX_OK) {
    return 0;
  }
  mmio_ = std::move(*mmio);
  uint32_t irq_count = 1;
  irq_count_ = irq_count;
  fbl::AllocChecker ac;
  interrupters_.reset(new (&ac) Interrupter[irq_count]);
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }
  for (uint32_t i = 0; i < irq_count; i++) {
    if (pdev_.GetInterrupt(i, &interrupters_[i].GetIrq()) != ZX_OK) {
      return 0;
    }
  }
  return 0;
}

void UsbXhci::BiosHandoff() {
  HCCPARAMS1 hcc = HCCPARAMS1::Get().ReadFrom(&mmio_.value());
  if (hcc.ReadFrom(&mmio_.value()).xECP()) {
    XECP current = XECP::Get(hcc).ReadFrom(&mmio_.value());
    while (true) {
      if (current.ID() == XECP::UsbLegacySupport) {
        current.set_reg_value(current.reg_value() | 1 << 24).WriteTo(&mmio_.value());
        while ((current = current.ReadFrom(&mmio_.value())).reg_value() & 1 << 16) {
        };
      }
      if (!current.NEXT()) {
        break;
      }
      current = current.Next().ReadFrom(&mmio_.value());
    }
  }
}

void UsbXhci::ResetController() {
  USBCMD::Get(cap_length_).ReadFrom(&mmio_.value()).set_ENABLE(0).WriteTo(&mmio_.value());
  while (!USBSTS::Get(cap_length_).ReadFrom(&mmio_.value()).HCHalted()) {
    zx::nanosleep(zx::deadline_after(zx::msec(1)));
  }
  while (USBSTS::Get(cap_length_).ReadFrom(&mmio_.value()).CNR()) {
    zx::nanosleep(zx::deadline_after(zx::msec(1)));
  }
  USBCMD::Get(cap_length_).ReadFrom(&mmio_.value()).set_RESET(1).WriteTo(&mmio_.value());
  while (USBCMD::Get(cap_length_).ReadFrom(&mmio_.value()).RESET()) {
    zx::nanosleep(zx::deadline_after(zx::msec(1)));
  }
  while (USBSTS::Get(cap_length_).ReadFrom(&mmio_.value()).CNR()) {
    zx::nanosleep(zx::deadline_after(zx::msec(1)));
  }
}

int UsbXhci::InitThread(std::unique_ptr<dma_buffer::BufferFactory> factory) {
  fbl::AutoCall call([=]() { DdkAsyncRemove(); });
  fbl::AutoCall init_completer([=]() { sync_completion_signal(&init_complete_); });
  // Initialize either the PCI or MMIO structures first
  if (pci_.is_valid()) {
    zx_status_t status = InitPci();
    if (status != 0) {
      zxlogf(ERROR, "PCI initialization failed with code %i", (int)status);
      return status;
    }
  } else {
    int status = InitMmio();
    if (status != 0) {
      zxlogf(ERROR, "MMIO initialization failed with code %i", (int)status);
      return status;
    }
  }
  // Perform the BIOS handoff if necessary
  BiosHandoff();
  // At startup the device is in an unknown state
  // Reset the xHCI to place everything in its well-defined
  // initial state.
  uint8_t cap_length = CapLength::Get().ReadFrom(&mmio_.value()).Length();
  cap_length_ = cap_length;
  buffer_factory_ = std::move(factory);
  // Perform xHCI reset process
  ResetController();
  // Start DDK interaction thread
  thrd_t thrd;
  int thread_status = thrd_create_with_name(
      &thrd,
      [](void* ctx) {
        auto hci = static_cast<UsbXhci*>(ctx);
        hci->ddk_interaction_loop_.Run();
        return 0;
      },
      this, "ddk_interaction_thread");
  if (thread_status != thrd_success) {
    return thread_status;
  }
  ddk_interaction_thread_ = thrd;
  // Finish HCI initialization
  zx_status_t status = HciFinalize();
  if (status != ZX_OK) {
    zxlogf(ERROR, "xHCI initialization failed with code %i", (int)status);
    return status;
  }
  call.cancel();
  return 0;
}

zx_status_t UsbXhci::HciFinalize() {
  hcc_ = HCCPARAMS1::Get().ReadFrom(&mmio_.value());
  HCSPARAMS1 hcsparams1 = HCSPARAMS1::Get().ReadFrom(&mmio_.value());
  is_32bit_ = !hcc_.AC64();
  params_ = hcsparams1;
  CONFIG::Get(cap_length_)
      .ReadFrom(&mmio_.value())
      .set_MaxSlotsEn(hcsparams1.MaxSlots())
      .WriteTo(&mmio_.value());
  {
    zx::bti bti;
    if (pci_.is_valid()) {
      if (pci_.GetBti(0, &bti) != ZX_OK) {
        return 0;
      }
    } else {
      if (pdev_.GetBti(0, &bti) != ZX_OK) {
        return 0;
      }
    }
    bti_ = std::move(bti);
  }
  uint32_t page_size = USB_PAGESIZE::Get(cap_length_).ReadFrom(&mmio_.value()).PageSize() << 12;
  page_size_ = page_size;
  // TODO (bbosak): Correct this to use variable alignment when we get kernel
  // support for this.
  if (page_size != ZX_PAGE_SIZE) {
    return 0;
  }
  uint32_t align_log2 = 0;
  if (buffer_factory_->CreatePaged(bti_, ZX_PAGE_SIZE, false, &dcbaa_buffer_) != ZX_OK) {
    return 0;
  }
  if (is_32bit_ && (dcbaa_buffer_->phys()[0] >= UINT32_MAX)) {
    return 0;
  }
  dcbaa_ = static_cast<uint64_t*>(dcbaa_buffer_->virt());
  fbl::AllocChecker ac;
  HCSPARAMS2 hcsparams2 = HCSPARAMS2::Get().ReadFrom(&mmio_.value());
  RuntimeRegisterOffset offset = RuntimeRegisterOffset::Get().ReadFrom(&mmio_.value());
  runtime_offset_ = offset;
  uint32_t buffers = hcsparams2.MAX_SCRATCHPAD_BUFFERS_LOW() |
                     ((hcsparams2.MAX_SCRATCHPAD_BUFFERS_HIGH() << 5) + 1);
  scratchpad_buffers_ = std::make_unique<std::unique_ptr<dma_buffer::ContiguousBuffer>[]>(buffers);
  if (fbl::round_up(buffers * sizeof(uint64_t), ZX_PAGE_SIZE) > ZX_PAGE_SIZE) {
    // We can't create multi-page contiguously physical uncached buffers.
    // This is presently not supported in the kernel.
    return ZX_ERR_NOT_SUPPORTED;
  }
  if (buffer_factory_->CreatePaged(bti_, ZX_PAGE_SIZE, false, &scratchpad_buffer_array_) != ZX_OK) {
    return 0;
  }
  if (is_32bit_ && (scratchpad_buffer_array_->phys()[0] >= UINT32_MAX)) {
    return 0;
  }
  uint64_t* scratchpad_buffer_array = static_cast<uint64_t*>(scratchpad_buffer_array_->virt());
  for (size_t i = 0; i < buffers - 1; i++) {
    if (buffer_factory_->CreateContiguous(bti_, page_size, align_log2, &scratchpad_buffers_[i]) !=
        ZX_OK) {
      return 0;
    }
    if (is_32bit_ && (scratchpad_buffers_[i]->phys() >= UINT32_MAX)) {
      return 0;
    }
    scratchpad_buffer_array[i] = scratchpad_buffers_[i]->phys();
  }
  static_cast<uint64_t*>(dcbaa_buffer_->virt())[0] = scratchpad_buffer_array_->phys()[0];
  max_slots_ = hcsparams1.MaxSlots();
  device_state_.reset(new (&ac) DeviceState[max_slots_]);
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }
  port_state_.reset(new (&ac) PortState[hcsparams1.MaxPorts()]);
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }
  hw_mb();
  DCBAAP::Get(cap_length_).FromValue(0).set_PTR(dcbaa_buffer_->phys()[0]).WriteTo(&mmio_.value());
  // Initialize command ring
  doorbell_offset_ = DoorbellOffset::Get().ReadFrom(&mmio_.value());
  // Interrupt moderation interval == 30 microseconds (optimal value derived from scheduler trace)
  // TODO: Change this based on P state (performance states) for power management
  IMODI::Get(offset, 0).ReadFrom(&mmio_.value()).set_MODI(240).WriteTo(&mmio_.value());
  if (interrupters_[0].ring().Init(
          page_size, bti_, &mmio_.value(), is_32bit_, 1 << hcsparams2.ERST_MAX(),
          ERSTSZ::Get(offset, 0).ReadFrom(&mmio_.value()),
          ERDP::Get(offset, 0).ReadFrom(&mmio_.value()), IMAN::Get(offset, 0).FromValue(0),
          cap_length_, HCSPARAMS1::Get().ReadFrom(&mmio_.value()), &command_ring_, doorbell_offset_,
          this, hcc_, dcbaa_) != ZX_OK) {
    return 0;
  }
  if (command_ring_.Init(page_size, &bti_, &interrupters_[0].ring(), is_32bit_, &mmio_.value(),
                         this) != ZX_OK) {
    return 0;
  }
  CRCR cr = command_ring_.phys(cap_length_);
  cr.WriteTo(&mmio_.value());
  // Initialize initial interrupter. We will later demand-allocate interrupters
  // as we get additional load.
  {
    fbl::AutoLock sched_lock(&scheduler_lock_);
    active_interrupters_ = 1;
  }
  if (interrupters_[0].Start(0, offset, mmio_.value().View(0), this) != ZX_OK) {
    return 0;
  }
  DdkMakeVisible();
  sync_completion_wait(&bus_completion, ZX_TIME_INFINITE);
  USBCMD::Get(cap_length_)
      .ReadFrom(&mmio_.value())
      .set_ENABLE(1)
      .set_INTE(1)
      .set_HSEE(1)
      .set_EWE(1)
      .WriteTo(&mmio_.value());
  while (USBSTS::Get(cap_length_).ReadFrom(&mmio_.value()).HCHalted()) {
    zx::nanosleep(zx::deadline_after(zx::msec(1)));
  }
  sync_completion_signal(&bringup_);
  return 0;
}

zx_status_t UsbXhci::Init() {
  if (!(pci_.is_valid() || pdev_.is_valid())) {
    return ZX_ERR_IO_INVALID;
  }
  zx_status_t status =
      device_get_profile(zxdev_, /*HIGH_PRIORITY*/ 31, "src/devices/usb/drivers/xhci/usb-xhci",
                         profile_.reset_and_get_address());
  if (status != ZX_OK) {
    zxlogf(WARN, "Failed to obtain scheduler profile for high priority completer (res %d)", status);
  }
  thrd_t init_thread;
  status = DdkAdd("xhci", DEVICE_ADD_INVISIBLE);
  if (status != ZX_OK) {
    return status;
  }
  if (thrd_create_with_name(
          &init_thread,
          [](void* ctx) {
            UsbXhci* hci = static_cast<UsbXhci*>(ctx);
            return hci->InitThread(dma_buffer::CreateBufferFactory());
          },
          this, "xhci-init-thread") != thrd_success) {
    DdkAsyncRemove();
    // We've successfully called DdkAdd, so return ZX_OK
    // since ownership was passed to the DDK
    // and we need to wait for DdkAsyncRemove to complete
    // before the object can be freed.
    return ZX_OK;
  }
  init_thread_ = init_thread;

  return ZX_OK;
}

TRBPromise UsbXhci::SubmitCommand(const TRB& command, std::unique_ptr<TRBContext> trb_context) {
  fit::bridge<TRB*, zx_status_t> bridge;
  trb_context->completer = std::move(bridge.completer);
  zx_status_t status = command_ring_.AddTRB(command, std::move(trb_context));
  if (status != ZX_OK) {
    return fit::make_result_promise(fit::result<TRB*, zx_status_t>(fit::error(status))).box();
  }
  // Ring the doorbell
  DOORBELL::Get(doorbell_offset_, 0).FromValue(0).WriteTo(&mmio_.value());
  return bridge.consumer.promise().box();
}

zx_status_t UsbXhci::Create(void* ctx, zx_device_t* parent) {
  fbl::AllocChecker ac;
  auto dev = std::unique_ptr<UsbXhci>(new (&ac) UsbXhci(parent));
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }
  if (dev->composite_.is_valid()) {
    enum { PDEV = 0, PHY = 1, COUNT = 2 };
    zx_device_t* fragments[COUNT];
    size_t actual;

    // Retrieve platform device protocol from our first fragment.
    dev->composite_.GetFragments(fragments, COUNT, &actual);
    // We need at least a PDEV, but the PHY is optional
    // for devices not implementing OTG.
    if (actual < 1) {
      return ZX_ERR_NOT_SUPPORTED;
    }
    if (actual > 1) {
      ddk::UsbPhyProtocolClient phy_proto(fragments[PHY]);
      dev->phy_ = std::move(phy_proto);
    }
    dev->pdev_ = fragments[PDEV];
    if (!dev->pdev_.is_valid()) {
      zxlogf(ERROR, "UsbXhci::Init: could not get platform device protocol");
      return ZX_ERR_NOT_SUPPORTED;
    }
  }
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }

  auto status = dev->Init();
  if (status != ZX_OK) {
    return status;
  }

  // devmgr is now in charge of the device.
  __UNUSED auto* dummy = dev.release();
  return ZX_OK;
}

static constexpr zx_driver_ops_t driver_ops = []() {
  zx_driver_ops_t ops = {};
  ops.version = DRIVER_OPS_VERSION;
  ops.bind = UsbXhci::Create;
  return ops;
}();

}  // namespace usb_xhci

// clang-format off
ZIRCON_DRIVER_BEGIN(usb_xhci, usb_xhci::driver_ops, "zircon", "0.1", 17)
    BI_GOTO_IF(EQ, BIND_PROTOCOL, ZX_PROTOCOL_PDEV, 0),
    BI_GOTO_IF(EQ, BIND_PROTOCOL, ZX_PROTOCOL_COMPOSITE, 1),

    // PCI binding support
    BI_ABORT_IF(NE, BIND_PROTOCOL, ZX_PROTOCOL_PCI),
    BI_ABORT_IF(NE, BIND_PCI_CLASS, 0x0C),
    BI_ABORT_IF(NE, BIND_PCI_SUBCLASS, 0x03),
    BI_MATCH_IF(EQ, BIND_PCI_INTERFACE, 0x30),
    BI_ABORT(),

    // platform bus support
    BI_LABEL(0),
    BI_ABORT_IF(NE, BIND_PLATFORM_DEV_VID, PDEV_VID_GENERIC),
    BI_ABORT_IF(NE, BIND_PLATFORM_DEV_PID, PDEV_PID_GENERIC),
    BI_MATCH_IF(EQ, BIND_PLATFORM_DEV_DID, PDEV_DID_USB_XHCI),
    BI_ABORT(),

    // composite binding support
    BI_LABEL(1),
    BI_ABORT_IF(NE, BIND_PLATFORM_DEV_VID, PDEV_VID_GENERIC),
    BI_ABORT_IF(NE, BIND_PLATFORM_DEV_PID, PDEV_PID_GENERIC),
    BI_MATCH_IF(EQ, BIND_PLATFORM_DEV_DID, PDEV_DID_USB_XHCI_COMPOSITE),

    BI_ABORT(),
ZIRCON_DRIVER_END(usb_xhci)