src/devices/usb/drivers/xhci/xhci.cc

// Copyright 2016 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "xhci.h"

#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <threads.h>
#include <unistd.h>
#include <zircon/errors.h>
#include <zircon/process.h>
#include <zircon/syscalls.h>
#include <zircon/types.h>

#include <bits/limits.h>
#include <ddk/debug.h>
#include <ddk/io-buffer.h>
#include <fbl/alloc_checker.h>
#include <fbl/auto_lock.h>
#include <hw/arch_ops.h>
#include <hw/reg.h>
#include <usb/usb-request.h>

#include "xhci-device-manager.h"
#include "xhci-root-hub.h"
#include "xhci-transfer.h"
#include "xhci-util.h"

namespace usb_xhci {

#define ROUNDUP_TO(x, multiple) ((x + multiple - 1) & ~(multiple - 1))
#define PAGE_ROUNDUP(x) ROUNDUP_TO(x, PAGE_SIZE)

// The Interrupter Moderation Interval prevents the controller from sending interrupts too often.
// According to XHCI Rev 1.1 4.17.2, the default is 4000 (= 1 ms). We set it to 1000 (= 250 us) to
// get better latency on completions for bulk transfers; setting it too low seems to destabilize the
// system.
#define XHCI_IMODI_VAL 1000

uint8_t xhci_endpoint_index(uint8_t ep_address) {
  if (ep_address == 0)
    return 0;
  uint8_t index = static_cast<uint8_t>(2 * (ep_address & ~USB_ENDPOINT_DIR_MASK));
  if ((ep_address & USB_ENDPOINT_DIR_MASK) == USB_ENDPOINT_OUT)
    index--;
  return index;
}

// returns index into xhci->root_hubs[], or -1 if not a root hub
int xhci_get_root_hub_index(xhci_t* xhci, uint32_t device_id) {
  // regular devices have IDs 1 through xhci->max_slots
  // root hub IDs start at xhci->max_slots + 1
  if (device_id > xhci->max_slots) {
    return static_cast<int>(device_id - xhci->max_slots - 1);
  } else {
    return -1;
  }
}

static void xhci_read_extended_caps(xhci_t* xhci) {
  uint32_t* cap_ptr = nullptr;
  // TODO(fxbug.dev/56253): Add MMIO_PTR to cast.
  while ((cap_ptr = xhci_get_next_ext_cap((void*)xhci->mmio->get(), cap_ptr, nullptr))) {
    uint32_t cap_id =
        XHCI_GET_BITS32(cap_ptr, EXT_CAP_CAPABILITY_ID_START, EXT_CAP_CAPABILITY_ID_BITS);

    if (cap_id == EXT_CAP_SUPPORTED_PROTOCOL) {
      uint32_t rev_major =
          XHCI_GET_BITS32(cap_ptr, EXT_CAP_SP_REV_MAJOR_START, EXT_CAP_SP_REV_MAJOR_BITS);
      uint32_t rev_minor =
          XHCI_GET_BITS32(cap_ptr, EXT_CAP_SP_REV_MINOR_START, EXT_CAP_SP_REV_MINOR_BITS);
      zxlogf(DEBUG, "EXT_CAP_SUPPORTED_PROTOCOL %d.%d", rev_major, rev_minor);

      uint32_t psic = XHCI_GET_BITS32(&cap_ptr[2], EXT_CAP_SP_PSIC_START, EXT_CAP_SP_PSIC_BITS);
      // psic = count of PSI registers
      uint32_t compat_port_offset = XHCI_GET_BITS32(
          &cap_ptr[2], EXT_CAP_SP_COMPAT_PORT_OFFSET_START, EXT_CAP_SP_COMPAT_PORT_OFFSET_BITS);
      uint32_t compat_port_count = XHCI_GET_BITS32(&cap_ptr[2], EXT_CAP_SP_COMPAT_PORT_COUNT_START,
                                                   EXT_CAP_SP_COMPAT_PORT_COUNT_BITS);

      zxlogf(DEBUG, "compat_port_offset: %d compat_port_count: %d psic: %d", compat_port_offset,
             compat_port_count, psic);

      uint8_t rh_index;
      if (rev_major == 3) {
        rh_index = XHCI_RH_USB_3;
      } else if (rev_major == 2) {
        rh_index = XHCI_RH_USB_2;
      } else {
        zxlogf(ERROR, "unsupported rev_major in XHCI extended capabilities");
        break;
      }
      for (off_t i = 0; i < compat_port_count; i++) {
        off_t index = compat_port_offset + i - 1;
        if (index >= xhci->rh_num_ports) {
          zxlogf(ERROR, "port index out of range in xhci_read_extended_caps");
          break;
        }
        xhci->rh_map[index] = rh_index;
      }

      uint32_t* psi = &cap_ptr[4];
      for (uint32_t i = 0; i < psic; i++, psi++) {
        uint32_t psiv = XHCI_GET_BITS32(psi, EXT_CAP_SP_PSIV_START, EXT_CAP_SP_PSIV_BITS);
        uint32_t psie = XHCI_GET_BITS32(psi, EXT_CAP_SP_PSIE_START, EXT_CAP_SP_PSIE_BITS);
        uint32_t plt = XHCI_GET_BITS32(psi, EXT_CAP_SP_PLT_START, EXT_CAP_SP_PLT_BITS);
        uint32_t psim = XHCI_GET_BITS32(psi, EXT_CAP_SP_PSIM_START, EXT_CAP_SP_PSIM_BITS);
        zxlogf(DEBUG, "PSI[%d] psiv: %d psie: %d plt: %d psim: %d", i, psiv, psie, plt, psim);
      }
    } else if (cap_id == EXT_CAP_USB_LEGACY_SUPPORT) {
      xhci->usb_legacy_support_cap = (xhci_usb_legacy_support_cap_t*)cap_ptr;
    }
  }
}

static zx_status_t xhci_claim_ownership(xhci_t* xhci) {
  xhci_usb_legacy_support_cap_t* cap = xhci->usb_legacy_support_cap;
  if (cap == nullptr) {
    return ZX_OK;
  }

  // The XHCI spec defines this handoff protocol.  We need to wait at most one
  // second for the BIOS to respond.
  //
  // Note that bios_owned_sem and os_owned_sem are adjacent 1-byte fields, so
  // must be written to as single bytes to prevent the OS from modifying the
  // BIOS semaphore.  Additionally, all bits besides bit 0 in the OS semaphore
  // are RsvdP, so we need to preserve them on modification.
  cap->os_owned_sem |= 1;
  zx_time_t now = zx_clock_get_monotonic();
  zx_time_t deadline = now + ZX_SEC(1);
  while ((cap->bios_owned_sem & 1) && now < deadline) {
    zx_nanosleep(zx_deadline_after(ZX_MSEC(10)));
    now = zx_clock_get_monotonic();
  }

  if (cap->bios_owned_sem & 1) {
    cap->os_owned_sem &= static_cast<uint8_t>(~1u);
    return ZX_ERR_TIMED_OUT;
  }
  return ZX_OK;
}

zx_status_t xhci_init(xhci_t* xhci, xhci_mode_t mode, uint32_t num_interrupts) {
  zx_status_t result = ZX_OK;

  list_initialize(&xhci->command_queue);
  sync_completion_reset(&xhci->command_queue_completion);

  usb_request_pool_init(&xhci->free_reqs,
                        sizeof(usb_request_t) + offsetof(xhci_usb_request_internal_t, node));

  // TODO(fxbug.dev/56253): Add MMIO_PTR to cast.
  xhci->cap_regs = (xhci_cap_regs_t*)xhci->mmio->get();
  xhci->op_regs = (xhci_op_regs_t*)((uint8_t*)xhci->cap_regs + xhci->cap_regs->length);
  xhci->doorbells = (uint32_t*)((uint8_t*)xhci->cap_regs + xhci->cap_regs->dboff);
  xhci->runtime_regs = (xhci_runtime_regs_t*)((uint8_t*)xhci->cap_regs + xhci->cap_regs->rtsoff);
  volatile uint32_t* hcsparams1 = &xhci->cap_regs->hcsparams1;
  volatile uint32_t* hcsparams2 = &xhci->cap_regs->hcsparams2;
  volatile uint32_t* hccparams1 = &xhci->cap_regs->hccparams1;
  volatile uint32_t* hccparams2 = &xhci->cap_regs->hccparams2;

  xhci->mode = mode;
  xhci->num_interrupts = num_interrupts;

  xhci->max_slots =
      XHCI_GET_BITS32(hcsparams1, HCSPARAMS1_MAX_SLOTS_START, HCSPARAMS1_MAX_SLOTS_BITS);
  xhci->rh_num_ports =
      XHCI_GET_BITS32(hcsparams1, HCSPARAMS1_MAX_PORTS_START, HCSPARAMS1_MAX_PORTS_BITS);
  xhci->context_size = (XHCI_READ32(hccparams1) & HCCPARAMS1_CSZ ? 64 : 32);
  xhci->large_esit = !!(XHCI_READ32(hccparams2) & HCCPARAMS2_LEC);

  uint32_t scratch_pad_bufs =
      XHCI_GET_BITS32(hcsparams2, HCSPARAMS2_MAX_SBBUF_HI_START, HCSPARAMS2_MAX_SBBUF_HI_BITS);
  scratch_pad_bufs <<= HCSPARAMS2_MAX_SBBUF_LO_BITS;
  scratch_pad_bufs |=
      XHCI_GET_BITS32(hcsparams2, HCSPARAMS2_MAX_SBBUF_LO_START, HCSPARAMS2_MAX_SBBUF_LO_BITS);
  uint32_t max_erst_count =
      1 << XHCI_GET_BITS32(hcsparams2, HCSPARAMS2_ERST_MAX_START, HCSPARAMS2_ERST_MAX_BITS);
  if (max_erst_count * sizeof(erst_entry_t) > PAGE_SIZE) {
    max_erst_count = PAGE_SIZE / sizeof(erst_entry_t);
    // TODO(bbosak): Implement a mechanism to allocate many physically contiguous pages
    // reliably.
  }
  xhci->page_size = XHCI_READ32(&xhci->op_regs->pagesize) << 12;

  fbl::AllocChecker ac;

  // allocate array to hold our slots
  // add 1 to allow 1-based indexing of slots
  xhci->slots.reset(new (&ac) xhci_slot_t[xhci->max_slots + 1], xhci->max_slots + 1);
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }
  xhci->rh_map.reset(new (&ac) uint8_t[xhci->rh_num_ports], xhci->rh_num_ports);
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }
  xhci->rh_port_map.reset(new (&ac) uint8_t[xhci->rh_num_ports], xhci->rh_num_ports);
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }

  xhci_read_extended_caps(xhci);

  // We need to claim before we write to any other registers on the
  // controller, but after we've read the extended capabilities.
  result = xhci_claim_ownership(xhci);
  if (result != ZX_OK) {
    zxlogf(ERROR, "xhci_claim_ownership failed");
    goto fail;
  }

  // Allocate DMA memory for various things
  result = xhci->dcbaa_erst_buffer.Init(xhci->bti_handle.get(), PAGE_SIZE,
                                        IO_BUFFER_RW | IO_BUFFER_CONTIG | XHCI_IO_BUFFER_UNCACHED);
  if (result != ZX_OK) {
    zxlogf(ERROR, "io_buffer_init failed for xhci->dcbaa_erst_buffer");
    goto fail;
  }
  result = xhci->input_context_buffer.Init(
      xhci->bti_handle.get(), PAGE_SIZE, IO_BUFFER_RW | IO_BUFFER_CONTIG | XHCI_IO_BUFFER_UNCACHED);
  if (result != ZX_OK) {
    zxlogf(ERROR, "io_buffer_init failed for xhci->input_context_buffer");
    goto fail;
  }

  bool scratch_pad_is_contig;
  scratch_pad_is_contig = false;
  if (scratch_pad_bufs > 0) {
    // map scratchpad buffers read-only
    uint32_t flags = IO_BUFFER_RO;
    if (xhci->page_size > PAGE_SIZE) {
      flags |= IO_BUFFER_CONTIG;
      scratch_pad_is_contig = true;
    }
    size_t scratch_pad_pages_size = scratch_pad_bufs * xhci->page_size;
    result =
        xhci->scratch_pad_pages_buffer.Init(xhci->bti_handle.get(), scratch_pad_pages_size, flags);
    if (result != ZX_OK) {
      zxlogf(ERROR, "io_buffer_init failed for xhci->scratch_pad_pages_buffer");
      goto fail;
    }
    if (!scratch_pad_is_contig) {
      result = xhci->scratch_pad_pages_buffer.PhysMap();
      if (result != ZX_OK) {
        zxlogf(ERROR, "io_buffer_physmap failed for xhci->scratch_pad_pages_buffer");
        goto fail;
      }
    }
    size_t scratch_pad_index_size = PAGE_ROUNDUP(scratch_pad_bufs * sizeof(uint64_t));
    result = xhci->scratch_pad_index_buffer.Init(
        xhci->bti_handle.get(), scratch_pad_index_size,
        IO_BUFFER_RW | IO_BUFFER_CONTIG | XHCI_IO_BUFFER_UNCACHED);
    if (result != ZX_OK) {
      zxlogf(ERROR, "io_buffer_init failed for xhci->scratch_pad_index_buffer");
      goto fail;
    }
  }

  // set up DCBAA, ERST array and input context
  xhci->dcbaa = static_cast<uint64_t*>(xhci->dcbaa_erst_buffer.virt());
  xhci->dcbaa_phys = xhci->dcbaa_erst_buffer.phys();
  xhci->input_context = static_cast<uint8_t*>(xhci->input_context_buffer.virt());
  xhci->input_context_phys = xhci->input_context_buffer.phys();

  // DCBAA can only be 256 * sizeof(uint64_t) = 2048 bytes, so we have room for ERST array after
  // DCBAA MSI only supports up to 32 interrupts, so the required ERST arrays will fit within the
  // page. Potentially more pages will need to be allocated for MSI-X. Max number of contiguous
  // physical pages per interrupt: 8
  for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
    result = xhci->erst_buffers[i].Init(xhci->bti_handle.get(),
                                        PAGE_ROUNDUP(max_erst_count * sizeof(erst_entry_t)),
                                        IO_BUFFER_RW | IO_BUFFER_CONTIG | XHCI_IO_BUFFER_UNCACHED);
    if (result != ZX_OK) {
      zxlogf(ERROR, "io_buffer_init failed for xhci->erst_buffer");
      goto fail;
    }
    xhci->erst_sizes[i] = max_erst_count;
    xhci->erst_arrays[i] = reinterpret_cast<erst_entry_t*>(xhci->erst_buffers[i].virt());
    xhci->erst_arrays_phys[i] = xhci->erst_buffers[i].phys();
  }

  if (scratch_pad_bufs > 0) {
    uint64_t* scratch_pad_index = static_cast<uint64_t*>(xhci->scratch_pad_index_buffer.virt());
    off_t offset = 0;
    for (uint32_t i = 0; i < scratch_pad_bufs; i++) {
      zx_paddr_t scratch_pad_phys;
      if (scratch_pad_is_contig) {
        scratch_pad_phys = xhci->scratch_pad_pages_buffer.phys() + offset;
      } else {
        size_t index = offset / PAGE_SIZE;
        size_t suboffset = offset & (PAGE_SIZE - 1);
        scratch_pad_phys = xhci->scratch_pad_pages_buffer.phys_list()[index] + suboffset;
      }

      scratch_pad_index[i] = scratch_pad_phys;
      offset += xhci->page_size;
    }

    zx_paddr_t scratch_pad_index_phys = xhci->scratch_pad_index_buffer.phys();
    xhci->dcbaa[0] = scratch_pad_index_phys;
  } else {
    xhci->dcbaa[0] = 0;
  }

  result = xhci_transfer_ring_init(&xhci->command_ring, xhci->bti_handle.get(), COMMAND_RING_SIZE);
  if (result != ZX_OK) {
    zxlogf(ERROR, "xhci_command_ring_init failed");
    goto fail;
  }

  for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
    result = xhci_event_ring_init(&xhci->event_rings[i], xhci->bti_handle.get(),
                                  xhci->erst_arrays[i], EVENT_RING_SIZE);
    if (result != ZX_OK) {
      zxlogf(ERROR, "xhci_event_ring_init failed");
      goto fail;
    }
  }

  // initialize slots and endpoints
  for (uint32_t i = 1; i <= xhci->max_slots; i++) {
    xhci_slot_t* slot = &xhci->slots[i];
    xhci_endpoint_t* eps = slot->eps;
    for (int j = 0; j < XHCI_NUM_EPS; j++) {
      xhci_endpoint_t* ep = &eps[j];
      list_initialize(&ep->queued_reqs);
      list_initialize(&ep->pending_reqs);
      ep->current_req = nullptr;
    }
  }
  // initialize virtual root hub devices
  for (int i = 0; i < XHCI_RH_COUNT; i++) {
    result = xhci_root_hub_init(xhci, i);
    if (result != ZX_OK)
      goto fail;
  }

  return ZX_OK;

fail:
  xhci_free(xhci);
  return result;
}

uint32_t xhci_get_max_interrupters(xhci_t* xhci) {
  // TODO(fxbug.dev/56253): Add MMIO_PTR to cast.
  auto* cap_regs = static_cast<xhci_cap_regs_t*>((void*)xhci->mmio->get());
  volatile uint32_t* hcsparams1 = &cap_regs->hcsparams1;
  return XHCI_GET_BITS32(hcsparams1, HCSPARAMS1_MAX_INTRS_START, HCSPARAMS1_MAX_INTRS_BITS);
}

int xhci_get_slot_ctx_state(xhci_slot_t* slot) {
  return XHCI_GET_BITS32(&slot->sc->sc3, SLOT_CTX_SLOT_STATE_START, SLOT_CTX_CONTEXT_ENTRIES_BITS);
}

int xhci_get_ep_ctx_state(xhci_slot_t* slot, xhci_endpoint_t* ep) {
  if (!ep->epc) {
    return EP_CTX_STATE_DISABLED;
  }
  return XHCI_GET_BITS32(&ep->epc->epc0, EP_CTX_EP_STATE_START, EP_CTX_EP_STATE_BITS);
}

static void xhci_update_erdp(xhci_t* xhci, int interrupter) {
  xhci_event_ring_t* er = &xhci->event_rings[interrupter];
  xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter];

  uint64_t erdp = xhci_event_ring_current_phys(er);
  erdp |= ERDP_EHB;  // clear event handler busy
  XHCI_WRITE64(&intr_regs->erdp, erdp);
}

static void xhci_interrupter_init(xhci_t* xhci, int interrupter) {
  xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter];

  xhci_update_erdp(xhci, interrupter);

  XHCI_SET32(&intr_regs->iman, IMAN_IE, IMAN_IE);
  XHCI_SET32(&intr_regs->imod, IMODI_MASK, XHCI_IMODI_VAL);
  XHCI_SET32(&intr_regs->erstsz, ERSTSZ_MASK, ERST_ARRAY_SIZE);
  XHCI_WRITE64(&intr_regs->erstba, xhci->erst_arrays_phys[interrupter]);
}

void xhci_wait_bits(volatile uint32_t* ptr, uint32_t bits, uint32_t expected) {
  uint32_t value = XHCI_READ32(ptr);
  while ((value & bits) != expected) {
    usleep(1000);
    value = XHCI_READ32(ptr);
  }
}

void xhci_wait_bits64(volatile uint64_t* ptr, uint64_t bits, uint64_t expected) {
  uint64_t value = XHCI_READ64(ptr);
  while ((value & bits) != expected) {
    usleep(1000);
    value = XHCI_READ64(ptr);
  }
}

void xhci_set_dbcaa(xhci_t* xhci, uint32_t slot_id, zx_paddr_t paddr) {
  XHCI_WRITE64(&xhci->dcbaa[slot_id], paddr);
}

zx_status_t xhci_start(xhci_t* xhci) {
  volatile uint32_t* usbcmd = &xhci->op_regs->usbcmd;
  volatile uint32_t* usbsts = &xhci->op_regs->usbsts;

  xhci_wait_bits(usbsts, USBSTS_CNR, 0);

  // stop controller
  XHCI_SET32(usbcmd, USBCMD_RS, 0);
  // wait until USBSTS_HCH signals we stopped
  xhci_wait_bits(usbsts, USBSTS_HCH, USBSTS_HCH);

  XHCI_SET32(usbcmd, USBCMD_HCRST, USBCMD_HCRST);
  xhci_wait_bits(usbcmd, USBCMD_HCRST, 0);
  xhci_wait_bits(usbsts, USBSTS_CNR, 0);

  if (xhci->mode == XHCI_PCI) {
    // enable bus master
    zx_status_t status = pci_enable_bus_master(&xhci->pci, true);
    if (status < 0) {
      zxlogf(ERROR, "usb_xhci_bind enable_bus_master failed %d", status);
      return status;
    }
  }

  // setup operational registers
  xhci_op_regs_t* op_regs = xhci->op_regs;
  // initialize command ring
  uint64_t crcr = xhci->command_ring.buffers.front()->phys_list()[0];
  if (xhci->command_ring.pcs) {
    crcr |= CRCR_RCS;
  }
  XHCI_WRITE64(&op_regs->crcr, crcr);

  XHCI_WRITE64(&op_regs->dcbaap, xhci->dcbaa_phys);
  XHCI_SET_BITS32(&op_regs->config, CONFIG_MAX_SLOTS_ENABLED_START, CONFIG_MAX_SLOTS_ENABLED_BITS,
                  xhci->max_slots);

  // initialize interrupters
  for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
    xhci_interrupter_init(xhci, i);
  }

  // start the controller with interrupts and mfindex wrap events enabled
  uint32_t start_flags = USBCMD_RS | USBCMD_INTE | USBCMD_EWE;
  XHCI_SET32(usbcmd, start_flags, start_flags);
  xhci_wait_bits(usbsts, USBSTS_HCH, 0);

  xhci_start_device_thread(xhci);

  return ZX_OK;
}

static void xhci_slot_stop(xhci_slot_t* slot, xhci_t* xhci) {
  for (int i = 0; i < XHCI_NUM_EPS; i++) {
    xhci_endpoint_t* ep = &slot->eps[i];

    list_node_t pending_reqs = LIST_INITIAL_VALUE(pending_reqs);
    list_node_t queued_reqs = LIST_INITIAL_VALUE(queued_reqs);
    {
      fbl::AutoLock al(&ep->lock);

      if (ep->state != EP_STATE_DEAD) {
        list_move(&ep->pending_reqs, &pending_reqs);
        list_move(&ep->queued_reqs, &queued_reqs);
        ep->state = EP_STATE_DEAD;
      }
    }
    usb_request_t* req = nullptr;
    xhci_usb_request_internal_t* req_int = nullptr;
    while ((req_int = list_remove_head_type(&pending_reqs, xhci_usb_request_internal_t, node)) !=
           nullptr) {
      req = XHCI_INTERNAL_TO_USB_REQ(req_int);
      usb_request_complete(req, ZX_ERR_IO_NOT_PRESENT, 0, &req_int->complete_cb);
    }
    while ((req_int = list_remove_head_type(&queued_reqs, xhci_usb_request_internal_t, node)) !=
           nullptr) {
      req = XHCI_INTERNAL_TO_USB_REQ(req_int);
      usb_request_complete(req, ZX_ERR_IO_NOT_PRESENT, 0, &req_int->complete_cb);
    }
  }
}

void xhci_stop(xhci_t* xhci) {
  volatile uint32_t* usbcmd = &xhci->op_regs->usbcmd;
  volatile uint32_t* usbsts = &xhci->op_regs->usbsts;

  // stop device thread and root hubs before turning off controller
  xhci_stop_device_thread(xhci);
  xhci_stop_root_hubs(xhci);

  // stop controller
  XHCI_SET32(usbcmd, USBCMD_RS, 0);
  // wait until USBSTS_HCH signals we stopped
  xhci_wait_bits(usbsts, USBSTS_HCH, USBSTS_HCH);

  for (uint32_t i = 1; i <= xhci->max_slots; i++) {
    xhci_slot_stop(&xhci->slots[i], xhci);
  }
}

void xhci_free(xhci_t* xhci) { delete xhci; }

zx_status_t xhci_post_command(xhci_t* xhci, uint32_t command, uint64_t ptr, uint32_t control_bits,
                              xhci_command_context_t* context) {
  fbl::AutoLock al(&xhci->command_ring_lock);

  xhci_transfer_ring_t* cr = &xhci->command_ring;
  size_t free_count = xhci_transfer_ring_free_trbs(cr);

  zxlogf(DEBUG, "xhci_post_command: free_count: %zu command: %u ptr: %lx control_bits %x",
         free_count, command, ptr, control_bits);

  if (free_count == 0) {
    zxlogf(ERROR, "xhci_post_command: command ring full!");
    return ZX_ERR_NO_RESOURCES;
  }

  xhci_trb_t* trb = cr->current_trb;
  auto index = trb - cr->start;
  xhci->command_contexts[index] = context;

  XHCI_WRITE64(&trb->ptr, ptr);
  XHCI_WRITE32(&trb->status, 0);
  trb_set_control(trb, command, control_bits);

  xhci_increment_ring(cr);
  context->next_trb = cr->current_trb;

  hw_mb();
  XHCI_WRITE32(&xhci->doorbells[0], 0);

  return ZX_OK;
}

static void xhci_handle_command_complete_event(xhci_t* xhci, xhci_trb_t* event_trb) {
  xhci_trb_t* command_trb = xhci_read_trb_ptr(&xhci->command_ring, event_trb);
  uint32_t cc = XHCI_GET_BITS32(&event_trb->status, EVT_TRB_CC_START, EVT_TRB_CC_BITS);
  zxlogf(DEBUG, "xhci_handle_command_complete_event slot_id: %d command: %d cc: %d",
         (event_trb->control >> TRB_SLOT_ID_START), trb_get_type(command_trb), cc);

  size_t index = command_trb - xhci->command_ring.start;

  if (cc == TRB_CC_COMMAND_RING_STOPPED) {
    // TRB_CC_COMMAND_RING_STOPPED is generated after aborting a command.
    // Ignore this, since it is unrelated to the next command in the command ring.
    return;
  }

  xhci_command_context_t* context;
  {
    fbl::AutoLock al(&xhci->command_ring_lock);
    context = xhci->command_contexts[index];
    xhci_set_dequeue_ptr(&xhci->command_ring, context->next_trb);
    xhci->command_contexts[index] = nullptr;
  }

  context->callback(context->data, cc, command_trb, event_trb);
}

static void xhci_handle_mfindex_wrap(xhci_t* xhci) {
  fbl::AutoLock al(&xhci->mfindex_mutex);
  xhci->mfindex_wrap_count++;
  xhci->last_mfindex_wrap = zx_clock_get_monotonic();
}

uint64_t xhci_get_current_frame(xhci_t* xhci) {
  fbl::AutoLock al(&xhci->mfindex_mutex);

  uint32_t mfindex = XHCI_READ32(&xhci->runtime_regs->mfindex) & ((1 << XHCI_MFINDEX_BITS) - 1);
  uint64_t wrap_count = xhci->mfindex_wrap_count;
  // try to detect race condition where mfindex has wrapped but we haven't processed wrap event yet
  if (mfindex < 500) {
    if (zx_clock_get_monotonic() - xhci->last_mfindex_wrap > ZX_MSEC(1000)) {
      zxlogf(DEBUG, "woah, mfindex wrapped before we got the event!");
      wrap_count++;
    }
  }

  // shift three to convert from 125us microframes to 1ms frames
  return ((wrap_count * (1 << XHCI_MFINDEX_BITS)) + mfindex) >> 3;
}

static void xhci_handle_events(xhci_t* xhci, int interrupter) {
  xhci_event_ring_t* er = &xhci->event_rings[interrupter];
  // process all TRBs with cycle bit matching our CCS
  while ((XHCI_READ32(&er->current->control) & TRB_C) == er->ccs) {
    uint32_t type = trb_get_type(er->current);
    switch (type) {
      case TRB_EVENT_COMMAND_COMP:
        xhci_handle_command_complete_event(xhci, er->current);
        break;
      case TRB_EVENT_PORT_STATUS_CHANGE:
        xhci_handle_root_hub_change(xhci);
        break;
      case TRB_EVENT_TRANSFER:
        xhci_handle_transfer_event(xhci, er->current);
        break;
      case TRB_EVENT_MFINDEX_WRAP:
        xhci_handle_mfindex_wrap(xhci);
        break;
      default:
        zxlogf(ERROR, "xhci_handle_events: unhandled event type %d", type);
        break;
    }
    fbl::AutoLock l(&er->xfer_lock);
    if (er->current + 1 == er->end) {
      er->current = er->start;
      er->ccs ^= TRB_C;
    } else {
      er->current = xhci_next_evt(er, er->current);
    }
    xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter];
    // Update the dequeue pointer to allow other events to come in
    uint64_t erdp = xhci_event_ring_current_phys(er);
    XHCI_WRITE64(&intr_regs->erdp, erdp);
  }

  // update event ring dequeue pointer and clear event handler busy flag
  xhci_update_erdp(xhci, interrupter);
}

void xhci_handle_interrupt(xhci_t* xhci, uint32_t interrupter) {
  // clear the interrupt pending flag
  xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter];
  XHCI_WRITE32(&intr_regs->iman, IMAN_IE | IMAN_IP);
  xhci_handle_events(xhci, interrupter);
}

bool xhci_add_to_list_tail(xhci_t* xhci, list_node_t* list, usb_request_t* req) {
  uint64_t node_offset = sizeof(usb_request_t) + offsetof(xhci_usb_request_internal_t, node);
  list_add_tail(list, (list_node_t*)((uintptr_t)req + node_offset));
  return true;
}

bool xhci_add_to_list_head(xhci_t* xhci, list_node_t* list, usb_request_t* req) {
  uint64_t node_offset = sizeof(usb_request_t) + offsetof(xhci_usb_request_internal_t, node);
  list_add_head(list, (list_node_t*)((uintptr_t)req + node_offset));
  return true;
}

bool xhci_remove_from_list_head(xhci_t* xhci, list_node_t* list, usb_request_t** req) {
  uint64_t node_offset = sizeof(usb_request_t) + offsetof(xhci_usb_request_internal_t, node);
  list_node_t* node = list_remove_head(list);
  if (!node) {
    *req = NULL;
    return false;
  }
  *req = (usb_request_t*)((uintptr_t)node - node_offset);
  return true;
}

bool xhci_remove_from_list_tail(xhci_t* xhci, list_node_t* list, usb_request_t** req) {
  uint64_t node_offset = sizeof(usb_request_t) + offsetof(xhci_usb_request_internal_t, node);
  list_node_t* node = list_remove_tail(list);
  if (!node) {
    *req = NULL;
    return false;
  }
  *req = (usb_request_t*)((uintptr_t)node - node_offset);
  return true;
}

void xhci_delete_req_node(xhci_t* xhci, usb_request_t* req) {
  uint64_t node_offset = sizeof(usb_request_t) + offsetof(xhci_usb_request_internal_t, node);
  list_node_t* node = (list_node_t*)((uintptr_t)req + node_offset);
  list_delete(node);
}

}  // namespace usb_xhci