// Copyright 2016 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <ddk/debug.h>
#include <hw/reg.h>
#include <zircon/types.h>
#include <zircon/syscalls.h>
#include <zircon/process.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <threads.h>
#include <unistd.h>

#include "xhci.h"
#include "xhci-device-manager.h"
#include "xhci-root-hub.h"
#include "xhci-transfer.h"
#include "xhci-util.h"

#define ROUNDUP_TO(x, multiple) ((x + multiple - 1) & ~(multiple - 1))
#define PAGE_ROUNDUP(x) ROUNDUP_TO(x, PAGE_SIZE)

#ifndef MIN
#define MIN(a, b) ((a) < (b) ? (a) : (b))
#endif

// The Interrupter Moderation Interval prevents the controller from sending interrupts too often.
// According to XHCI Rev 1.1 4.17.2, the default is 4000 (= 1 ms). We set it to 1000 (= 250 us) to
// get better latency on completions for bulk transfers; setting it too low seems to destabilize the
// system.
#define XHCI_IMODI_VAL      1000

uint8_t xhci_endpoint_index(uint8_t ep_address) {
    if (ep_address == 0) return 0;
    uint32_t index = 2 * (ep_address & ~USB_ENDPOINT_DIR_MASK);
    if ((ep_address & USB_ENDPOINT_DIR_MASK) == USB_ENDPOINT_OUT)
        index--;
    return index;
}

// returns index into xhci->root_hubs[], or -1 if not a root hub
int xhci_get_root_hub_index(xhci_t* xhci, uint32_t device_id) {
    // regular devices have IDs 1 through xhci->max_slots
    // root hub IDs start at xhci->max_slots + 1
    int index = device_id - (xhci->max_slots + 1);
    if (index < 0 || index >= XHCI_RH_COUNT) return -1;
    return index;
}

static void xhci_read_extended_caps(xhci_t* xhci, volatile uint32_t* hccparams1) {
    uint32_t offset = XHCI_GET_BITS32(hccparams1, HCCPARAMS1_EXT_CAP_PTR_START,
                                      HCCPARAMS1_EXT_CAP_PTR_BITS);
    if (!offset) return;
    // offset is 32-bit words from MMIO base
    uint32_t* cap_ptr = (uint32_t *)(xhci->mmio + (offset << 2));

    while (cap_ptr) {
        uint32_t cap_id = XHCI_GET_BITS32(cap_ptr, EXT_CAP_CAPABILITY_ID_START,
                                          EXT_CAP_CAPABILITY_ID_BITS);

        if (cap_id == EXT_CAP_SUPPORTED_PROTOCOL) {
            uint32_t rev_major = XHCI_GET_BITS32(cap_ptr, EXT_CAP_SP_REV_MAJOR_START,
                                                 EXT_CAP_SP_REV_MAJOR_BITS);
            uint32_t rev_minor = XHCI_GET_BITS32(cap_ptr, EXT_CAP_SP_REV_MINOR_START,
                                                 EXT_CAP_SP_REV_MINOR_BITS);
            zxlogf(TRACE, "EXT_CAP_SUPPORTED_PROTOCOL %d.%d\n", rev_major, rev_minor);

            uint32_t psic = XHCI_GET_BITS32(&cap_ptr[2], EXT_CAP_SP_PSIC_START,
                                            EXT_CAP_SP_PSIC_BITS);
            // psic = count of PSI registers
            uint32_t compat_port_offset = XHCI_GET_BITS32(&cap_ptr[2],
                                                          EXT_CAP_SP_COMPAT_PORT_OFFSET_START,
                                                          EXT_CAP_SP_COMPAT_PORT_OFFSET_BITS);
            uint32_t compat_port_count = XHCI_GET_BITS32(&cap_ptr[2],
                                                         EXT_CAP_SP_COMPAT_PORT_COUNT_START,
                                                         EXT_CAP_SP_COMPAT_PORT_COUNT_BITS);

            zxlogf(TRACE, "compat_port_offset: %d compat_port_count: %d psic: %d\n",
                    compat_port_offset, compat_port_count, psic);

            int rh_index;
            if (rev_major == 3) {
                rh_index = XHCI_RH_USB_3;
            } else if (rev_major == 2) {
                rh_index = XHCI_RH_USB_2;
            } else {
                zxlogf(ERROR, "unsupported rev_major in XHCI extended capabilities\n");
                rh_index = -1;
            }
            for (off_t i = 0; i < compat_port_count; i++) {
                off_t index = compat_port_offset + i - 1;
                if (index >= xhci->rh_num_ports) {
                    zxlogf(ERROR, "port index out of range in xhci_read_extended_caps\n");
                    break;
                }
                xhci->rh_map[index] = rh_index;
            }

            uint32_t* psi = &cap_ptr[4];
            for (uint32_t i = 0; i < psic; i++, psi++) {
                uint32_t psiv = XHCI_GET_BITS32(psi, EXT_CAP_SP_PSIV_START, EXT_CAP_SP_PSIV_BITS);
                uint32_t psie = XHCI_GET_BITS32(psi, EXT_CAP_SP_PSIE_START, EXT_CAP_SP_PSIE_BITS);
                uint32_t plt = XHCI_GET_BITS32(psi, EXT_CAP_SP_PLT_START, EXT_CAP_SP_PLT_BITS);
                uint32_t psim = XHCI_GET_BITS32(psi, EXT_CAP_SP_PSIM_START, EXT_CAP_SP_PSIM_BITS);
                zxlogf(TRACE, "PSI[%d] psiv: %d psie: %d plt: %d psim: %d\n", i, psiv, psie, plt, psim);
            }
        } else if (cap_id == EXT_CAP_USB_LEGACY_SUPPORT) {
            xhci->usb_legacy_support_cap = (xhci_usb_legacy_support_cap_t*)cap_ptr;
        }

        // offset is 32-bit words from cap_ptr
        offset = XHCI_GET_BITS32(cap_ptr, EXT_CAP_NEXT_PTR_START, EXT_CAP_NEXT_PTR_BITS);
        cap_ptr = (offset ? cap_ptr + offset : NULL);
    }
}

static zx_status_t xhci_claim_ownership(xhci_t* xhci) {
    xhci_usb_legacy_support_cap_t* cap = xhci->usb_legacy_support_cap;
    if (cap == NULL) {
        return ZX_OK;
    }

    // The XHCI spec defines this handoff protocol.  We need to wait at most one
    // second for the BIOS to respond.
    //
    // Note that bios_owned_sem and os_owned_sem are adjacent 1-byte fields, so
    // must be written to as single bytes to prevent the OS from modifying the
    // BIOS semaphore.  Additionally, all bits besides bit 0 in the OS semaphore
    // are RsvdP, so we need to preserve them on modification.
    cap->os_owned_sem |= 1;
    zx_time_t now = zx_time_get(ZX_CLOCK_MONOTONIC);
    zx_time_t deadline = now + ZX_SEC(1);
    while ((cap->bios_owned_sem & 1) && now < deadline) {
        zx_nanosleep(zx_deadline_after(ZX_MSEC(10)));
        now = zx_time_get(ZX_CLOCK_MONOTONIC);
    }

    if (cap->bios_owned_sem & 1) {
        cap->os_owned_sem &= ~1;
        return ZX_ERR_TIMED_OUT;
    }
    return ZX_OK;
}

static void xhci_vmo_release(zx_handle_t handle, zx_vaddr_t virt) {
    uint64_t size;
    zx_vmo_get_size(handle, &size);
    zx_vmar_unmap(zx_vmar_root_self(), virt, size);
    zx_handle_close(handle);
}

zx_status_t xhci_init(xhci_t* xhci, xhci_mode_t mode, uint32_t num_interrupts) {
    zx_status_t result = ZX_OK;

    list_initialize(&xhci->command_queue);
    mtx_init(&xhci->command_ring_lock, mtx_plain);
    mtx_init(&xhci->command_queue_mutex, mtx_plain);
    mtx_init(&xhci->mfindex_mutex, mtx_plain);
    mtx_init(&xhci->input_context_lock, mtx_plain);
    completion_reset(&xhci->command_queue_completion);

    usb_request_pool_init(&xhci->free_reqs);

    xhci->cap_regs = (xhci_cap_regs_t*)xhci->mmio;
    xhci->op_regs = (xhci_op_regs_t*)((uint8_t*)xhci->cap_regs + xhci->cap_regs->length);
    xhci->doorbells = (uint32_t*)((uint8_t*)xhci->cap_regs + xhci->cap_regs->dboff);
    xhci->runtime_regs = (xhci_runtime_regs_t*)((uint8_t*)xhci->cap_regs + xhci->cap_regs->rtsoff);
    volatile uint32_t* hcsparams1 = &xhci->cap_regs->hcsparams1;
    volatile uint32_t* hcsparams2 = &xhci->cap_regs->hcsparams2;
    volatile uint32_t* hccparams1 = &xhci->cap_regs->hccparams1;
    volatile uint32_t* hccparams2 = &xhci->cap_regs->hccparams2;

    xhci->mode = mode;
    uint32_t max_interrupters = XHCI_GET_BITS32(hcsparams1, HCSPARAMS1_MAX_INTRS_START,
                                                HCSPARAMS1_MAX_INTRS_BITS);
    max_interrupters = MIN(INTERRUPTER_COUNT, max_interrupters);
    xhci->num_interrupts = MIN(max_interrupters, num_interrupts);

    xhci->max_slots = XHCI_GET_BITS32(hcsparams1, HCSPARAMS1_MAX_SLOTS_START,
                                      HCSPARAMS1_MAX_SLOTS_BITS);
    xhci->rh_num_ports = XHCI_GET_BITS32(hcsparams1, HCSPARAMS1_MAX_PORTS_START,
                                         HCSPARAMS1_MAX_PORTS_BITS);
    xhci->context_size = (XHCI_READ32(hccparams1) & HCCPARAMS1_CSZ ? 64 : 32);
    xhci->large_esit = !!(XHCI_READ32(hccparams2) & HCCPARAMS2_LEC);

    uint32_t scratch_pad_bufs = XHCI_GET_BITS32(hcsparams2, HCSPARAMS2_MAX_SBBUF_HI_START,
                                                HCSPARAMS2_MAX_SBBUF_HI_BITS);
    scratch_pad_bufs <<= HCSPARAMS2_MAX_SBBUF_LO_BITS;
    scratch_pad_bufs |= XHCI_GET_BITS32(hcsparams2, HCSPARAMS2_MAX_SBBUF_LO_START,
                                        HCSPARAMS2_MAX_SBBUF_LO_BITS);
    xhci->page_size = XHCI_READ32(&xhci->op_regs->pagesize) << 12;

    // allocate array to hold our slots
    // add 1 to allow 1-based indexing of slots
    xhci->slots = (xhci_slot_t*)calloc(xhci->max_slots + 1, sizeof(xhci_slot_t));
    if (!xhci->slots) {
        result = ZX_ERR_NO_MEMORY;
        goto fail;
    }

    xhci->rh_map = (uint8_t *)calloc(xhci->rh_num_ports, sizeof(uint8_t));
    if (!xhci->rh_map) {
        result = ZX_ERR_NO_MEMORY;
        goto fail;
    }
    xhci->rh_port_map = (uint8_t *)calloc(xhci->rh_num_ports, sizeof(uint8_t));
    if (!xhci->rh_port_map) {
        result = ZX_ERR_NO_MEMORY;
        goto fail;
    }
    xhci_read_extended_caps(xhci, hccparams1);

    // We need to claim before we write to any other registers on the
    // controller, but after we've read the extended capabilities.
    result = xhci_claim_ownership(xhci);
    if (result != ZX_OK) {
        zxlogf(ERROR, "xhci_claim_ownership failed\n");
        goto fail;
    }

    // Allocate DMA memory for various things
    result = io_buffer_init(&xhci->dcbaa_erst_buffer, PAGE_SIZE, IO_BUFFER_RW);
    if (result != ZX_OK) {
        zxlogf(ERROR, "io_buffer_init failed for xhci->dcbaa_erst_buffer\n");
        goto fail;
    }
    result = io_buffer_init(&xhci->input_context_buffer, PAGE_SIZE, IO_BUFFER_RW);
    if (result != ZX_OK) {
        zxlogf(ERROR, "io_buffer_init failed for xhci->input_context_buffer\n");
        goto fail;
    }

    if (scratch_pad_bufs > 0) {
        // map scratchpad buffers read-only
        uint32_t flags = xhci->page_size > PAGE_SIZE ?
                            (IO_BUFFER_RO | IO_BUFFER_CONTIG) : IO_BUFFER_RO;
        size_t scratch_pad_pages_size = scratch_pad_bufs * xhci->page_size;
        result = io_buffer_init(&xhci->scratch_pad_pages_buffer, scratch_pad_pages_size, flags);
        if (result != ZX_OK) {
            zxlogf(ERROR, "xhci_vmo_init failed for xhci->scratch_pad_pages_buffer\n");
            goto fail;
        }
        size_t scratch_pad_index_size = PAGE_ROUNDUP(scratch_pad_bufs * sizeof(uint64_t));
        result = io_buffer_init(&xhci->scratch_pad_index_buffer, scratch_pad_index_size,
                                IO_BUFFER_RW | IO_BUFFER_CONTIG);
        if (result != ZX_OK) {
            zxlogf(ERROR, "io_buffer_init failed for xhci->scratch_pad_index_buffer\n");
            goto fail;
        }
    }

    // set up DCBAA, ERST array and input context
    xhci->dcbaa = (uint64_t *)io_buffer_virt(&xhci->dcbaa_erst_buffer);
    xhci->dcbaa_phys = io_buffer_phys(&xhci->dcbaa_erst_buffer);
    xhci->input_context = (uint8_t *)io_buffer_virt(&xhci->input_context_buffer);
    xhci->input_context_phys = io_buffer_phys(&xhci->input_context_buffer);

    // DCBAA can only be 256 * sizeof(uint64_t) = 2048 bytes, so we have room for ERST array after DCBAA
    zx_off_t erst_offset = 256 * sizeof(uint64_t);

    size_t array_bytes = ERST_ARRAY_SIZE * sizeof(erst_entry_t);
    // MSI only supports up to 32 interupts, so the required ERST arrays will fit
    // within the page. Potentially more pages will need to be allocated for MSI-X.
    for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
        // Ran out of space in page.
        if (erst_offset + array_bytes > PAGE_SIZE) {
            zxlogf(ERROR, "only have space for %u ERST arrays, want %u\n", i,
                    xhci->num_interrupts);
            goto fail;
        }
        xhci->erst_arrays[i] = (void *)xhci->dcbaa + erst_offset;
        xhci->erst_arrays_phys[i] = xhci->dcbaa_phys + erst_offset;
        // ERST arrays must be 64 byte aligned - see Table 54 in XHCI spec.
        // dcbaa_phys is already page (and hence 64 byte) aligned, so only
        // need to round the offset.
        erst_offset = ROUNDUP_TO(erst_offset + array_bytes, 64);
    }

    if (scratch_pad_bufs > 0) {
        uint64_t* scratch_pad_index = (uint64_t *)io_buffer_virt(&xhci->scratch_pad_index_buffer);
        off_t offset = 0;
        for (uint32_t i = 0; i < scratch_pad_bufs; i++) {
            zx_paddr_t scratch_pad_phys;
            result = io_buffer_physmap_range(&xhci->scratch_pad_pages_buffer, offset, PAGE_SIZE,
                                             sizeof(scratch_pad_phys), &scratch_pad_phys);
            if (result != ZX_OK) {
                zxlogf(ERROR, "io_buffer_physmap failed for xhci->scratch_pad_pages_buffer\n");
                goto fail;
            }
            scratch_pad_index[i] = scratch_pad_phys;
            offset += xhci->page_size;
        }

        zx_paddr_t scratch_pad_index_phys = io_buffer_phys(&xhci->scratch_pad_index_buffer);
        xhci->dcbaa[0] = scratch_pad_index_phys;
    } else {
        xhci->dcbaa[0] = 0;
    }

    result = xhci_transfer_ring_init(&xhci->command_ring, COMMAND_RING_SIZE);
    if (result != ZX_OK) {
        zxlogf(ERROR, "xhci_command_ring_init failed\n");
        goto fail;
    }

    for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
        result = xhci_event_ring_init(xhci, i, EVENT_RING_SIZE);
        if (result != ZX_OK) {
            zxlogf(ERROR, "xhci_event_ring_init failed\n");
            goto fail;
        }
    }

    // initialize slots and endpoints
    for (uint32_t i = 1; i <= xhci->max_slots; i++) {
        xhci_slot_t* slot = &xhci->slots[i];
        xhci_endpoint_t* eps = slot->eps;
        for (int j = 0; j < XHCI_NUM_EPS; j++) {
            xhci_endpoint_t* ep = &eps[j];
            mtx_init(&ep->lock, mtx_plain);
            list_initialize(&ep->queued_reqs);
            list_initialize(&ep->pending_reqs);
            ep->current_req = NULL;
        }
    }

    // initialize virtual root hub devices
    for (int i = 0; i < XHCI_RH_COUNT; i++) {
        result = xhci_root_hub_init(xhci, i);
        if (result != ZX_OK) goto fail;
    }

    return ZX_OK;

fail:
    xhci_free(xhci);
    return result;
}

int xhci_get_ep_ctx_state(xhci_slot_t* slot, xhci_endpoint_t* ep) {
    if (!ep->epc) {
        return EP_CTX_STATE_DISABLED;
    }
#if XHCI_USE_CACHE_OPS
    uint32_t offset = (void *)&ep->epc->epc0 - io_buffer_virt(&slot->buffer);
    io_buffer_cache_op(&slot->buffer, ZX_VMO_OP_CACHE_INVALIDATE, offset, sizeof(uint32_t));
#endif
    return XHCI_GET_BITS32(&ep->epc->epc0, EP_CTX_EP_STATE_START, EP_CTX_EP_STATE_BITS);
}

static void xhci_update_erdp(xhci_t* xhci, int interrupter) {
    xhci_event_ring_t* er = &xhci->event_rings[interrupter];
    xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter];

    uint64_t erdp = xhci_event_ring_current_phys(er);
    erdp |= ERDP_EHB; // clear event handler busy
    XHCI_WRITE64(&intr_regs->erdp, erdp);
}

static void xhci_interrupter_init(xhci_t* xhci, int interrupter) {
    xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter];

    xhci_update_erdp(xhci, interrupter);

    XHCI_SET32(&intr_regs->iman, IMAN_IE, IMAN_IE);
    XHCI_SET32(&intr_regs->imod, IMODI_MASK, XHCI_IMODI_VAL);
    XHCI_SET32(&intr_regs->erstsz, ERSTSZ_MASK, ERST_ARRAY_SIZE);
    XHCI_WRITE64(&intr_regs->erstba, xhci->erst_arrays_phys[interrupter]);
}

void xhci_wait_bits(volatile uint32_t* ptr, uint32_t bits, uint32_t expected) {
    uint32_t value = XHCI_READ32(ptr);
    while ((value & bits) != expected) {
        usleep(1000);
        value = XHCI_READ32(ptr);
    }
}

void xhci_wait_bits64(volatile uint64_t* ptr, uint64_t bits, uint64_t expected) {
    uint64_t value = XHCI_READ64(ptr);
    while ((value & bits) != expected) {
        usleep(1000);
        value = XHCI_READ64(ptr);
    }
}

void xhci_set_dbcaa(xhci_t* xhci, uint32_t slot_id, zx_paddr_t paddr) {
    XHCI_WRITE64(&xhci->dcbaa[slot_id], paddr);
#if XHCI_USE_CACHE_OPS
    uint32_t offset = (void *)&xhci->dcbaa[slot_id] - io_buffer_virt(&xhci->dcbaa_erst_buffer);
    io_buffer_cache_op(&xhci->dcbaa_erst_buffer, ZX_VMO_OP_CACHE_CLEAN, offset, sizeof(paddr));
#endif
}


static int heartbeat_thread(void *arg) {
    xhci_t* xhci = arg;

    while (1) {
        sleep(2);
        xhci_sync_command_t command;
        xhci_sync_command_init(&command);
        xhci_post_command(xhci, TRB_CMD_NOOP, 0, 0, &command.context);
        int cc = xhci_sync_command_wait(&command);
        printf("TRB_CMD_NOOP got %d\n", cc);
    }

    return 0;
}

zx_status_t xhci_start(xhci_t* xhci) {
    volatile uint32_t* usbcmd = &xhci->op_regs->usbcmd;
    volatile uint32_t* usbsts = &xhci->op_regs->usbsts;

    xhci_wait_bits(usbsts, USBSTS_CNR, 0);

    // stop controller
    XHCI_SET32(usbcmd, USBCMD_RS, 0);
    // wait until USBSTS_HCH signals we stopped
    xhci_wait_bits(usbsts, USBSTS_HCH, USBSTS_HCH);

    XHCI_SET32(usbcmd, USBCMD_HCRST, USBCMD_HCRST);
    xhci_wait_bits(usbcmd, USBCMD_HCRST, 0);
    xhci_wait_bits(usbsts, USBSTS_CNR, 0);

    if (xhci->mode == XHCI_PCI_MSI || xhci->mode == XHCI_PCI_LEGACY) {
        // enable bus master
        zx_status_t status = pci_enable_bus_master(&xhci->pci, true);
        if (status < 0) {
            zxlogf(ERROR, "usb_xhci_bind enable_bus_master failed %d\n", status);
            return status;
        }
    }

    // setup operational registers
    xhci_op_regs_t* op_regs = xhci->op_regs;
    // initialize command ring
    uint64_t crcr = xhci_transfer_ring_start_phys(&xhci->command_ring);
    if (xhci->command_ring.pcs) {
        crcr |= CRCR_RCS;
    }
    XHCI_WRITE64(&op_regs->crcr, crcr);

    XHCI_WRITE64(&op_regs->dcbaap, xhci->dcbaa_phys);
    XHCI_SET_BITS32(&op_regs->config, CONFIG_MAX_SLOTS_ENABLED_START,
                    CONFIG_MAX_SLOTS_ENABLED_BITS, xhci->max_slots);

    // initialize interrupters
    for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
        xhci_interrupter_init(xhci, i);
    }

    // start the controller with interrupts and mfindex wrap events enabled
    uint32_t start_flags = USBCMD_RS | USBCMD_INTE | USBCMD_EWE;
    XHCI_SET32(usbcmd, start_flags, start_flags);
    xhci_wait_bits(usbsts, USBSTS_HCH, 0);

    xhci_start_device_thread(xhci);

    thrd_t thread;
    thrd_create_with_name(&thread, heartbeat_thread, xhci, "heartbeat_thread");

    return ZX_OK;
}

static void xhci_slot_stop(xhci_slot_t* slot) {
    for (int i = 0; i < XHCI_NUM_EPS; i++) {
        xhci_endpoint_t* ep = &slot->eps[i];

        mtx_lock(&ep->lock);
        if (ep->state != EP_STATE_DEAD) {
            usb_request_t* req;
            while ((req = list_remove_tail_type(&ep->pending_reqs, usb_request_t, node)) != NULL) {
                usb_request_complete(req, ZX_ERR_IO_NOT_PRESENT, 0);
            }
            while ((req = list_remove_tail_type(&ep->queued_reqs, usb_request_t, node)) != NULL) {
                usb_request_complete(req, ZX_ERR_IO_NOT_PRESENT, 0);
            }
            ep->state = EP_STATE_DEAD;
        }
        mtx_unlock(&ep->lock);
    }
}

void xhci_stop(xhci_t* xhci) {
    volatile uint32_t* usbcmd = &xhci->op_regs->usbcmd;
    volatile uint32_t* usbsts = &xhci->op_regs->usbsts;

    xhci_stop_root_hubs(xhci);

    // stop controller
    XHCI_SET32(usbcmd, USBCMD_RS, 0);
    // wait until USBSTS_HCH signals we stopped
    xhci_wait_bits(usbsts, USBSTS_HCH, USBSTS_HCH);

    for (uint32_t i = 1; i <= xhci->max_slots; i++) {
        xhci_slot_stop(&xhci->slots[i]);
    }

    xhci_stop_device_thread(xhci);
}

void xhci_free(xhci_t* xhci) {
    for (uint32_t i = 1; i <= xhci->max_slots; i++) {
        xhci_slot_t* slot = &xhci->slots[i];
        io_buffer_release(&slot->buffer);

        for (int j = 0; j < XHCI_NUM_EPS; j++) {
            xhci_endpoint_t* ep = &slot->eps[j];
            xhci_transfer_ring_free(&ep->transfer_ring);
        }
    }
    free(xhci->slots);

     for (int i = 0; i < XHCI_RH_COUNT; i++) {
        xhci_root_hub_free(&xhci->root_hubs[i]);
    }
    free(xhci->rh_map);
    free(xhci->rh_port_map);

    for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
        xhci_event_ring_free(xhci, i);
    }

    xhci_transfer_ring_free(&xhci->command_ring);
    io_buffer_release(&xhci->dcbaa_erst_buffer);
    io_buffer_release(&xhci->input_context_buffer);
    io_buffer_release(&xhci->scratch_pad_pages_buffer);
    io_buffer_release(&xhci->scratch_pad_index_buffer);

    free(xhci);
}

void xhci_post_command(xhci_t* xhci, uint32_t command, uint64_t ptr, uint32_t control_bits,
                       xhci_command_context_t* context) {
    // FIXME - check that command ring is not full?

    mtx_lock(&xhci->command_ring_lock);

    xhci_transfer_ring_t* cr = &xhci->command_ring;
    xhci_trb_t* trb = cr->current;
    int index = trb - cr->start;
    xhci->command_contexts[index] = context;

    XHCI_WRITE64(&trb->ptr, ptr);
    XHCI_WRITE32(&trb->status, 0);
    trb_set_control(trb, command, control_bits);

    xhci_increment_ring(cr);

    XHCI_WRITE32(&xhci->doorbells[0], 0);

    mtx_unlock(&xhci->command_ring_lock);
}

static void xhci_handle_command_complete_event(xhci_t* xhci, xhci_trb_t* event_trb) {
    xhci_trb_t* command_trb = xhci_read_trb_ptr(&xhci->command_ring, event_trb);
    uint32_t cc = XHCI_GET_BITS32(&event_trb->status, EVT_TRB_CC_START, EVT_TRB_CC_BITS);
    zxlogf(TRACE, "xhci_handle_command_complete_event slot_id: %d command: %d cc: %d\n",
            (event_trb->control >> TRB_SLOT_ID_START), trb_get_type(command_trb), cc);

    int index = command_trb - xhci->command_ring.start;

    if (cc == TRB_CC_COMMAND_RING_STOPPED) {
        // TRB_CC_COMMAND_RING_STOPPED is generated after aborting a command.
        // Ignore this, since it is unrelated to the next command in the command ring.
        return;
    }

    mtx_lock(&xhci->command_ring_lock);
    xhci_command_context_t* context = xhci->command_contexts[index];
    xhci->command_contexts[index] = NULL;
    mtx_unlock(&xhci->command_ring_lock);

    context->callback(context->data, cc, command_trb, event_trb);
}

static void xhci_handle_mfindex_wrap(xhci_t* xhci) {
    mtx_lock(&xhci->mfindex_mutex);
    xhci->mfindex_wrap_count++;
    xhci->last_mfindex_wrap = zx_time_get(ZX_CLOCK_MONOTONIC);
    mtx_unlock(&xhci->mfindex_mutex);
}

uint64_t xhci_get_current_frame(xhci_t* xhci) {
    mtx_lock(&xhci->mfindex_mutex);

    uint32_t mfindex = XHCI_READ32(&xhci->runtime_regs->mfindex) & ((1 << XHCI_MFINDEX_BITS) - 1);
    uint64_t wrap_count = xhci->mfindex_wrap_count;
    // try to detect race condition where mfindex has wrapped but we haven't processed wrap event yet
    if (mfindex < 500) {
        if (zx_time_get(ZX_CLOCK_MONOTONIC) - xhci->last_mfindex_wrap > ZX_MSEC(1000)) {
            zxlogf(TRACE, "woah, mfindex wrapped before we got the event!\n");
            wrap_count++;
        }
    }
    mtx_unlock(&xhci->mfindex_mutex);

    // shift three to convert from 125us microframes to 1ms frames
    return ((wrap_count * (1 << XHCI_MFINDEX_BITS)) + mfindex) >> 3;
}

static void xhci_handle_events(xhci_t* xhci, int interrupter) {
    xhci_event_ring_t* er = &xhci->event_rings[interrupter];

#if XHCI_USE_CACHE_OPS
    io_buffer_cache_op(&er->buffer, ZX_VMO_OP_CACHE_INVALIDATE, 0, er->buffer.size);
#endif

    // process all TRBs with cycle bit matching our CCS
    while ((XHCI_READ32(&er->current->control) & TRB_C) == er->ccs) {
        uint32_t type = trb_get_type(er->current);
        switch (type) {
        case TRB_EVENT_COMMAND_COMP:
            xhci_handle_command_complete_event(xhci, er->current);
            break;
        case TRB_EVENT_PORT_STATUS_CHANGE:
            xhci_handle_root_hub_change(xhci);
            break;
        case TRB_EVENT_TRANSFER:
            xhci_handle_transfer_event(xhci, er->current);
            break;
        case TRB_EVENT_MFINDEX_WRAP:
printf("TRB_EVENT_MFINDEX_WRAP\n");
            xhci_handle_mfindex_wrap(xhci);
            break;
        default:
            zxlogf(ERROR, "xhci_handle_events: unhandled event type %d\n", type);
            break;
        }

        er->current++;
        if (er->current == er->end) {
            er->current = er->start;
            er->ccs ^= TRB_C;
        }
    }

    // update event ring dequeue pointer and clear event handler busy flag
    xhci_update_erdp(xhci, interrupter);
}

void xhci_handle_interrupt(xhci_t* xhci, uint32_t interrupter) {
    // clear the interrupt pending flag
    xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter];
    XHCI_WRITE32(&intr_regs->iman, IMAN_IE | IMAN_IP);

    xhci_handle_events(xhci, interrupter);
}