| // Copyright 2016 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "xhci.h" |
| |
| #include <ddk/debug.h> |
| #include <fbl/alloc_checker.h> |
| #include <fbl/auto_lock.h> |
| #include <hw/arch_ops.h> |
| #include <hw/reg.h> |
| #include <limits.h> |
| #include <stdio.h> |
| #include <stdlib.h> |
| #include <string.h> |
| #include <threads.h> |
| #include <unistd.h> |
| #include <usb/usb-request.h> |
| #include <zircon/process.h> |
| #include <zircon/syscalls.h> |
| #include <zircon/types.h> |
| |
| #include <bits/limits.h> |
| #include <ddk/io-buffer.h> |
| #include "xhci-device-manager.h" |
| #include "xhci-root-hub.h" |
| #include "xhci-transfer.h" |
| #include "xhci-util.h" |
| #include <zircon/errors.h> |
| |
| namespace usb_xhci { |
| |
| #define ROUNDUP_TO(x, multiple) ((x + multiple - 1) & ~(multiple - 1)) |
| #define PAGE_ROUNDUP(x) ROUNDUP_TO(x, PAGE_SIZE) |
| |
| // The Interrupter Moderation Interval prevents the controller from sending interrupts too often. |
| // According to XHCI Rev 1.1 4.17.2, the default is 4000 (= 1 ms). We set it to 1000 (= 250 us) to |
| // get better latency on completions for bulk transfers; setting it too low seems to destabilize the |
| // system. |
| #define XHCI_IMODI_VAL 1000 |
| |
| uint8_t xhci_endpoint_index(uint8_t ep_address) { |
| if (ep_address == 0) return 0; |
| uint8_t index = static_cast<uint8_t>(2 * (ep_address & ~USB_ENDPOINT_DIR_MASK)); |
| if ((ep_address & USB_ENDPOINT_DIR_MASK) == USB_ENDPOINT_OUT) |
| index--; |
| return index; |
| } |
| |
| // returns index into xhci->root_hubs[], or -1 if not a root hub |
| int xhci_get_root_hub_index(xhci_t* xhci, uint32_t device_id) { |
| // regular devices have IDs 1 through xhci->max_slots |
| // root hub IDs start at xhci->max_slots + 1 |
| if (device_id > xhci->max_slots) { |
| return static_cast<int>(device_id - xhci->max_slots - 1); |
| } else { |
| return -1; |
| } |
| } |
| |
| static void xhci_read_extended_caps(xhci_t* xhci) { |
| uint32_t* cap_ptr = nullptr; |
| while ((cap_ptr = xhci_get_next_ext_cap(xhci->mmio->get(), cap_ptr, nullptr))) { |
| uint32_t cap_id = XHCI_GET_BITS32(cap_ptr, EXT_CAP_CAPABILITY_ID_START, |
| EXT_CAP_CAPABILITY_ID_BITS); |
| |
| if (cap_id == EXT_CAP_SUPPORTED_PROTOCOL) { |
| uint32_t rev_major = XHCI_GET_BITS32(cap_ptr, EXT_CAP_SP_REV_MAJOR_START, |
| EXT_CAP_SP_REV_MAJOR_BITS); |
| uint32_t rev_minor = XHCI_GET_BITS32(cap_ptr, EXT_CAP_SP_REV_MINOR_START, |
| EXT_CAP_SP_REV_MINOR_BITS); |
| zxlogf(TRACE, "EXT_CAP_SUPPORTED_PROTOCOL %d.%d\n", rev_major, rev_minor); |
| |
| uint32_t psic = XHCI_GET_BITS32(&cap_ptr[2], EXT_CAP_SP_PSIC_START, |
| EXT_CAP_SP_PSIC_BITS); |
| // psic = count of PSI registers |
| uint32_t compat_port_offset = XHCI_GET_BITS32(&cap_ptr[2], |
| EXT_CAP_SP_COMPAT_PORT_OFFSET_START, |
| EXT_CAP_SP_COMPAT_PORT_OFFSET_BITS); |
| uint32_t compat_port_count = XHCI_GET_BITS32(&cap_ptr[2], |
| EXT_CAP_SP_COMPAT_PORT_COUNT_START, |
| EXT_CAP_SP_COMPAT_PORT_COUNT_BITS); |
| |
| zxlogf(TRACE, "compat_port_offset: %d compat_port_count: %d psic: %d\n", |
| compat_port_offset, compat_port_count, psic); |
| |
| uint8_t rh_index; |
| if (rev_major == 3) { |
| rh_index = XHCI_RH_USB_3; |
| } else if (rev_major == 2) { |
| rh_index = XHCI_RH_USB_2; |
| } else { |
| zxlogf(ERROR, "unsupported rev_major in XHCI extended capabilities\n"); |
| break; |
| } |
| for (off_t i = 0; i < compat_port_count; i++) { |
| off_t index = compat_port_offset + i - 1; |
| if (index >= xhci->rh_num_ports) { |
| zxlogf(ERROR, "port index out of range in xhci_read_extended_caps\n"); |
| break; |
| } |
| xhci->rh_map[index] = rh_index; |
| } |
| |
| uint32_t* psi = &cap_ptr[4]; |
| for (uint32_t i = 0; i < psic; i++, psi++) { |
| uint32_t psiv = XHCI_GET_BITS32(psi, EXT_CAP_SP_PSIV_START, EXT_CAP_SP_PSIV_BITS); |
| uint32_t psie = XHCI_GET_BITS32(psi, EXT_CAP_SP_PSIE_START, EXT_CAP_SP_PSIE_BITS); |
| uint32_t plt = XHCI_GET_BITS32(psi, EXT_CAP_SP_PLT_START, EXT_CAP_SP_PLT_BITS); |
| uint32_t psim = XHCI_GET_BITS32(psi, EXT_CAP_SP_PSIM_START, EXT_CAP_SP_PSIM_BITS); |
| zxlogf(TRACE, "PSI[%d] psiv: %d psie: %d plt: %d psim: %d\n", i, psiv, psie, plt, psim); |
| } |
| } else if (cap_id == EXT_CAP_USB_LEGACY_SUPPORT) { |
| xhci->usb_legacy_support_cap = (xhci_usb_legacy_support_cap_t*)cap_ptr; |
| } |
| } |
| } |
| |
| static zx_status_t xhci_claim_ownership(xhci_t* xhci) { |
| xhci_usb_legacy_support_cap_t* cap = xhci->usb_legacy_support_cap; |
| if (cap == nullptr) { |
| return ZX_OK; |
| } |
| |
| // The XHCI spec defines this handoff protocol. We need to wait at most one |
| // second for the BIOS to respond. |
| // |
| // Note that bios_owned_sem and os_owned_sem are adjacent 1-byte fields, so |
| // must be written to as single bytes to prevent the OS from modifying the |
| // BIOS semaphore. Additionally, all bits besides bit 0 in the OS semaphore |
| // are RsvdP, so we need to preserve them on modification. |
| cap->os_owned_sem |= 1; |
| zx_time_t now = zx_clock_get_monotonic(); |
| zx_time_t deadline = now + ZX_SEC(1); |
| while ((cap->bios_owned_sem & 1) && now < deadline) { |
| zx_nanosleep(zx_deadline_after(ZX_MSEC(10))); |
| now = zx_clock_get_monotonic(); |
| } |
| |
| if (cap->bios_owned_sem & 1) { |
| cap->os_owned_sem &= static_cast<uint8_t>(~1u); |
| return ZX_ERR_TIMED_OUT; |
| } |
| return ZX_OK; |
| } |
| |
| static void xhci_vmo_release(zx_handle_t handle, zx_vaddr_t virt) { |
| uint64_t size; |
| zx_vmo_get_size(handle, &size); |
| zx_vmar_unmap(zx_vmar_root_self(), virt, size); |
| zx_handle_close(handle); |
| } |
| |
| zx_status_t xhci_init(xhci_t* xhci, xhci_mode_t mode, uint32_t num_interrupts) { |
| zx_status_t result = ZX_OK; |
| |
| list_initialize(&xhci->command_queue); |
| sync_completion_reset(&xhci->command_queue_completion); |
| |
| usb_request_pool_init(&xhci->free_reqs, sizeof(usb_request_t) + |
| offsetof(xhci_usb_request_internal_t, node)); |
| |
| xhci->cap_regs = (xhci_cap_regs_t*)xhci->mmio->get(); |
| xhci->op_regs = (xhci_op_regs_t*)((uint8_t*)xhci->cap_regs + xhci->cap_regs->length); |
| xhci->doorbells = (uint32_t*)((uint8_t*)xhci->cap_regs + xhci->cap_regs->dboff); |
| xhci->runtime_regs = (xhci_runtime_regs_t*)((uint8_t*)xhci->cap_regs + xhci->cap_regs->rtsoff); |
| volatile uint32_t* hcsparams1 = &xhci->cap_regs->hcsparams1; |
| volatile uint32_t* hcsparams2 = &xhci->cap_regs->hcsparams2; |
| volatile uint32_t* hccparams1 = &xhci->cap_regs->hccparams1; |
| volatile uint32_t* hccparams2 = &xhci->cap_regs->hccparams2; |
| |
| xhci->mode = mode; |
| xhci->num_interrupts = num_interrupts; |
| |
| xhci->max_slots = XHCI_GET_BITS32(hcsparams1, HCSPARAMS1_MAX_SLOTS_START, |
| HCSPARAMS1_MAX_SLOTS_BITS); |
| xhci->rh_num_ports = XHCI_GET_BITS32(hcsparams1, HCSPARAMS1_MAX_PORTS_START, |
| HCSPARAMS1_MAX_PORTS_BITS); |
| xhci->context_size = (XHCI_READ32(hccparams1) & HCCPARAMS1_CSZ ? 64 : 32); |
| xhci->large_esit = !!(XHCI_READ32(hccparams2) & HCCPARAMS2_LEC); |
| |
| uint32_t scratch_pad_bufs = XHCI_GET_BITS32(hcsparams2, HCSPARAMS2_MAX_SBBUF_HI_START, |
| HCSPARAMS2_MAX_SBBUF_HI_BITS); |
| scratch_pad_bufs <<= HCSPARAMS2_MAX_SBBUF_LO_BITS; |
| scratch_pad_bufs |= |
| XHCI_GET_BITS32(hcsparams2, HCSPARAMS2_MAX_SBBUF_LO_START, HCSPARAMS2_MAX_SBBUF_LO_BITS); |
| uint32_t max_erst_count = |
| 1 << XHCI_GET_BITS32(hcsparams2, HCSPARAMS2_ERST_MAX_START, HCSPARAMS2_ERST_MAX_BITS); |
| if (max_erst_count * sizeof(erst_entry_t) > PAGE_SIZE) { |
| max_erst_count = PAGE_SIZE / sizeof(erst_entry_t); |
| // TODO(bbosak): Implement a mechanism to allocate many physically contiguous pages |
| // reliably. |
| } |
| xhci->page_size = XHCI_READ32(&xhci->op_regs->pagesize) << 12; |
| |
| fbl::AllocChecker ac; |
| |
| // allocate array to hold our slots |
| // add 1 to allow 1-based indexing of slots |
| xhci->slots.reset(new (&ac) xhci_slot_t[xhci->max_slots + 1], xhci->max_slots + 1); |
| if (!ac.check()) { |
| return ZX_ERR_NO_MEMORY; |
| } |
| xhci->rh_map.reset(new (&ac) uint8_t[xhci->rh_num_ports], xhci->rh_num_ports); |
| if (!ac.check()) { |
| return ZX_ERR_NO_MEMORY; |
| } |
| xhci->rh_port_map.reset(new (&ac) uint8_t[xhci->rh_num_ports], xhci->rh_num_ports); |
| if (!ac.check()) { |
| return ZX_ERR_NO_MEMORY; |
| } |
| |
| xhci_read_extended_caps(xhci); |
| |
| // We need to claim before we write to any other registers on the |
| // controller, but after we've read the extended capabilities. |
| result = xhci_claim_ownership(xhci); |
| if (result != ZX_OK) { |
| zxlogf(ERROR, "xhci_claim_ownership failed\n"); |
| goto fail; |
| } |
| |
| // Allocate DMA memory for various things |
| result = xhci->dcbaa_erst_buffer.Init(xhci->bti_handle.get(), PAGE_SIZE, |
| IO_BUFFER_RW | IO_BUFFER_CONTIG | |
| XHCI_IO_BUFFER_UNCACHED); |
| if (result != ZX_OK) { |
| zxlogf(ERROR, "io_buffer_init failed for xhci->dcbaa_erst_buffer\n"); |
| goto fail; |
| } |
| result = xhci->input_context_buffer.Init(xhci->bti_handle.get(), PAGE_SIZE, |
| IO_BUFFER_RW | IO_BUFFER_CONTIG | |
| XHCI_IO_BUFFER_UNCACHED); |
| if (result != ZX_OK) { |
| zxlogf(ERROR, "io_buffer_init failed for xhci->input_context_buffer\n"); |
| goto fail; |
| } |
| |
| bool scratch_pad_is_contig; |
| scratch_pad_is_contig = false; |
| if (scratch_pad_bufs > 0) { |
| // map scratchpad buffers read-only |
| uint32_t flags = IO_BUFFER_RO; |
| if (xhci->page_size > PAGE_SIZE) { |
| flags |= IO_BUFFER_CONTIG; |
| scratch_pad_is_contig = true; |
| } |
| size_t scratch_pad_pages_size = scratch_pad_bufs * xhci->page_size; |
| result = xhci->scratch_pad_pages_buffer.Init(xhci->bti_handle.get(), |
| scratch_pad_pages_size, flags); |
| if (result != ZX_OK) { |
| zxlogf(ERROR, "io_buffer_init failed for xhci->scratch_pad_pages_buffer\n"); |
| goto fail; |
| } |
| if (!scratch_pad_is_contig) { |
| result = xhci->scratch_pad_pages_buffer.PhysMap(); |
| if (result != ZX_OK) { |
| zxlogf(ERROR, "io_buffer_physmap failed for xhci->scratch_pad_pages_buffer\n"); |
| goto fail; |
| } |
| } |
| size_t scratch_pad_index_size = PAGE_ROUNDUP(scratch_pad_bufs * sizeof(uint64_t)); |
| result = xhci->scratch_pad_index_buffer.Init(xhci->bti_handle.get(), |
| scratch_pad_index_size, |
| IO_BUFFER_RW | IO_BUFFER_CONTIG | XHCI_IO_BUFFER_UNCACHED); |
| if (result != ZX_OK) { |
| zxlogf(ERROR, "io_buffer_init failed for xhci->scratch_pad_index_buffer\n"); |
| goto fail; |
| } |
| } |
| |
| // set up DCBAA, ERST array and input context |
| xhci->dcbaa = static_cast<uint64_t*>(xhci->dcbaa_erst_buffer.virt()); |
| xhci->dcbaa_phys = xhci->dcbaa_erst_buffer.phys(); |
| xhci->input_context = static_cast<uint8_t*>(xhci->input_context_buffer.virt()); |
| xhci->input_context_phys = xhci->input_context_buffer.phys(); |
| |
| // DCBAA can only be 256 * sizeof(uint64_t) = 2048 bytes, so we have room for ERST array after |
| // DCBAA MSI only supports up to 32 interrupts, so the required ERST arrays will fit within the |
| // page. Potentially more pages will need to be allocated for MSI-X. Max number of contiguous |
| // physical pages per interrupt: 8 |
| for (uint32_t i = 0; i < xhci->num_interrupts; i++) { |
| result = xhci->erst_buffers[i].Init( |
| xhci->bti_handle.get(), PAGE_ROUNDUP(max_erst_count * sizeof(erst_entry_t)), |
| IO_BUFFER_RW | IO_BUFFER_CONTIG | XHCI_IO_BUFFER_UNCACHED); |
| if (result != ZX_OK) { |
| zxlogf(ERROR, "io_buffer_init failed for xhci->erst_buffer\n"); |
| goto fail; |
| } |
| xhci->erst_sizes[i] = max_erst_count; |
| xhci->erst_arrays[i] = reinterpret_cast<erst_entry_t*>(xhci->erst_buffers[i].virt()); |
| xhci->erst_arrays_phys[i] = xhci->erst_buffers[i].phys(); |
| } |
| |
| if (scratch_pad_bufs > 0) { |
| uint64_t* scratch_pad_index = static_cast<uint64_t*>(xhci->scratch_pad_index_buffer.virt()); |
| off_t offset = 0; |
| for (uint32_t i = 0; i < scratch_pad_bufs; i++) { |
| zx_paddr_t scratch_pad_phys; |
| if (scratch_pad_is_contig) { |
| scratch_pad_phys = xhci->scratch_pad_pages_buffer.phys() + offset; |
| } else { |
| size_t index = offset / PAGE_SIZE; |
| size_t suboffset = offset & (PAGE_SIZE - 1); |
| scratch_pad_phys = xhci->scratch_pad_pages_buffer.phys_list()[index] + suboffset; |
| } |
| |
| scratch_pad_index[i] = scratch_pad_phys; |
| offset += xhci->page_size; |
| } |
| |
| zx_paddr_t scratch_pad_index_phys = xhci->scratch_pad_index_buffer.phys(); |
| xhci->dcbaa[0] = scratch_pad_index_phys; |
| } else { |
| xhci->dcbaa[0] = 0; |
| } |
| |
| result = xhci_transfer_ring_init(&xhci->command_ring, xhci->bti_handle.get(), |
| COMMAND_RING_SIZE); |
| if (result != ZX_OK) { |
| zxlogf(ERROR, "xhci_command_ring_init failed\n"); |
| goto fail; |
| } |
| |
| for (uint32_t i = 0; i < xhci->num_interrupts; i++) { |
| result = xhci_event_ring_init(&xhci->event_rings[i], xhci->bti_handle.get(), |
| xhci->erst_arrays[i], EVENT_RING_SIZE); |
| if (result != ZX_OK) { |
| zxlogf(ERROR, "xhci_event_ring_init failed\n"); |
| goto fail; |
| } |
| } |
| |
| // initialize slots and endpoints |
| for (uint32_t i = 1; i <= xhci->max_slots; i++) { |
| xhci_slot_t* slot = &xhci->slots[i]; |
| xhci_endpoint_t* eps = slot->eps; |
| for (int j = 0; j < XHCI_NUM_EPS; j++) { |
| xhci_endpoint_t* ep = &eps[j]; |
| list_initialize(&ep->queued_reqs); |
| list_initialize(&ep->pending_reqs); |
| ep->current_req = nullptr; |
| } |
| } |
| // initialize virtual root hub devices |
| for (int i = 0; i < XHCI_RH_COUNT; i++) { |
| result = xhci_root_hub_init(xhci, i); |
| if (result != ZX_OK) goto fail; |
| } |
| |
| return ZX_OK; |
| |
| fail: |
| xhci_free(xhci); |
| return result; |
| } |
| |
| uint32_t xhci_get_max_interrupters(xhci_t* xhci) { |
| auto* cap_regs = static_cast<xhci_cap_regs_t*>(xhci->mmio->get()); |
| volatile uint32_t* hcsparams1 = &cap_regs->hcsparams1; |
| return XHCI_GET_BITS32(hcsparams1, HCSPARAMS1_MAX_INTRS_START, |
| HCSPARAMS1_MAX_INTRS_BITS); |
| } |
| |
| int xhci_get_slot_ctx_state(xhci_slot_t* slot) { |
| return XHCI_GET_BITS32(&slot->sc->sc3, SLOT_CTX_SLOT_STATE_START, |
| SLOT_CTX_CONTEXT_ENTRIES_BITS); |
| } |
| |
| int xhci_get_ep_ctx_state(xhci_slot_t* slot, xhci_endpoint_t* ep) { |
| if (!ep->epc) { |
| return EP_CTX_STATE_DISABLED; |
| } |
| return XHCI_GET_BITS32(&ep->epc->epc0, EP_CTX_EP_STATE_START, EP_CTX_EP_STATE_BITS); |
| } |
| |
| static void xhci_update_erdp(xhci_t* xhci, int interrupter) { |
| xhci_event_ring_t* er = &xhci->event_rings[interrupter]; |
| xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter]; |
| |
| uint64_t erdp = xhci_event_ring_current_phys(er); |
| erdp |= ERDP_EHB; // clear event handler busy |
| XHCI_WRITE64(&intr_regs->erdp, erdp); |
| } |
| |
| static void xhci_interrupter_init(xhci_t* xhci, int interrupter) { |
| xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter]; |
| |
| xhci_update_erdp(xhci, interrupter); |
| |
| XHCI_SET32(&intr_regs->iman, IMAN_IE, IMAN_IE); |
| XHCI_SET32(&intr_regs->imod, IMODI_MASK, XHCI_IMODI_VAL); |
| XHCI_SET32(&intr_regs->erstsz, ERSTSZ_MASK, ERST_ARRAY_SIZE); |
| XHCI_WRITE64(&intr_regs->erstba, xhci->erst_arrays_phys[interrupter]); |
| } |
| |
| void xhci_wait_bits(volatile uint32_t* ptr, uint32_t bits, uint32_t expected) { |
| uint32_t value = XHCI_READ32(ptr); |
| while ((value & bits) != expected) { |
| usleep(1000); |
| value = XHCI_READ32(ptr); |
| } |
| } |
| |
| void xhci_wait_bits64(volatile uint64_t* ptr, uint64_t bits, uint64_t expected) { |
| uint64_t value = XHCI_READ64(ptr); |
| while ((value & bits) != expected) { |
| usleep(1000); |
| value = XHCI_READ64(ptr); |
| } |
| } |
| |
| void xhci_set_dbcaa(xhci_t* xhci, uint32_t slot_id, zx_paddr_t paddr) { |
| XHCI_WRITE64(&xhci->dcbaa[slot_id], paddr); |
| } |
| |
| zx_status_t xhci_start(xhci_t* xhci) { |
| volatile uint32_t* usbcmd = &xhci->op_regs->usbcmd; |
| volatile uint32_t* usbsts = &xhci->op_regs->usbsts; |
| |
| xhci_wait_bits(usbsts, USBSTS_CNR, 0); |
| |
| // stop controller |
| XHCI_SET32(usbcmd, USBCMD_RS, 0); |
| // wait until USBSTS_HCH signals we stopped |
| xhci_wait_bits(usbsts, USBSTS_HCH, USBSTS_HCH); |
| |
| XHCI_SET32(usbcmd, USBCMD_HCRST, USBCMD_HCRST); |
| xhci_wait_bits(usbcmd, USBCMD_HCRST, 0); |
| xhci_wait_bits(usbsts, USBSTS_CNR, 0); |
| |
| if (xhci->mode == XHCI_PCI_MSI || xhci->mode == XHCI_PCI_LEGACY) { |
| // enable bus master |
| zx_status_t status = pci_enable_bus_master(&xhci->pci, true); |
| if (status < 0) { |
| zxlogf(ERROR, "usb_xhci_bind enable_bus_master failed %d\n", status); |
| return status; |
| } |
| } |
| |
| // setup operational registers |
| xhci_op_regs_t* op_regs = xhci->op_regs; |
| // initialize command ring |
| uint64_t crcr = xhci->command_ring.buffers.front()->phys_list()[0]; |
| if (xhci->command_ring.pcs) { |
| crcr |= CRCR_RCS; |
| } |
| XHCI_WRITE64(&op_regs->crcr, crcr); |
| |
| XHCI_WRITE64(&op_regs->dcbaap, xhci->dcbaa_phys); |
| XHCI_SET_BITS32(&op_regs->config, CONFIG_MAX_SLOTS_ENABLED_START, |
| CONFIG_MAX_SLOTS_ENABLED_BITS, xhci->max_slots); |
| |
| // initialize interrupters |
| for (uint32_t i = 0; i < xhci->num_interrupts; i++) { |
| xhci_interrupter_init(xhci, i); |
| } |
| |
| // start the controller with interrupts and mfindex wrap events enabled |
| uint32_t start_flags = USBCMD_RS | USBCMD_INTE | USBCMD_EWE; |
| XHCI_SET32(usbcmd, start_flags, start_flags); |
| xhci_wait_bits(usbsts, USBSTS_HCH, 0); |
| |
| xhci_start_device_thread(xhci); |
| |
| return ZX_OK; |
| } |
| |
| static void xhci_slot_stop(xhci_slot_t* slot, xhci_t* xhci) { |
| for (int i = 0; i < XHCI_NUM_EPS; i++) { |
| xhci_endpoint_t* ep = &slot->eps[i]; |
| |
| list_node_t pending_reqs = LIST_INITIAL_VALUE(pending_reqs); |
| list_node_t queued_reqs = LIST_INITIAL_VALUE(queued_reqs); |
| { |
| fbl::AutoLock al(&ep->lock); |
| |
| if (ep->state != EP_STATE_DEAD) { |
| list_move(&ep->pending_reqs, &pending_reqs); |
| list_move(&ep->queued_reqs, &queued_reqs); |
| ep->state = EP_STATE_DEAD; |
| } |
| } |
| usb_request_t* req = nullptr; |
| xhci_usb_request_internal_t* req_int = nullptr; |
| while ((req_int = list_remove_head_type(&pending_reqs, |
| xhci_usb_request_internal_t, |
| node)) != nullptr) { |
| req = XHCI_INTERNAL_TO_USB_REQ(req_int); |
| usb_request_complete(req, ZX_ERR_IO_NOT_PRESENT, 0, &req_int->complete_cb); |
| } |
| while ((req_int = list_remove_head_type(&queued_reqs, |
| xhci_usb_request_internal_t, |
| node)) != nullptr) { |
| req = XHCI_INTERNAL_TO_USB_REQ(req_int); |
| usb_request_complete(req, ZX_ERR_IO_NOT_PRESENT, 0, &req_int->complete_cb); |
| } |
| } |
| } |
| |
| void xhci_stop(xhci_t* xhci) { |
| volatile uint32_t* usbcmd = &xhci->op_regs->usbcmd; |
| volatile uint32_t* usbsts = &xhci->op_regs->usbsts; |
| |
| // stop device thread and root hubs before turning off controller |
| xhci_stop_device_thread(xhci); |
| xhci_stop_root_hubs(xhci); |
| |
| // stop controller |
| XHCI_SET32(usbcmd, USBCMD_RS, 0); |
| // wait until USBSTS_HCH signals we stopped |
| xhci_wait_bits(usbsts, USBSTS_HCH, USBSTS_HCH); |
| |
| for (uint32_t i = 1; i <= xhci->max_slots; i++) { |
| xhci_slot_stop(&xhci->slots[i], xhci); |
| } |
| } |
| |
| void xhci_free(xhci_t* xhci) { |
| delete xhci; |
| } |
| |
| zx_status_t xhci_post_command(xhci_t* xhci, uint32_t command, uint64_t ptr, uint32_t control_bits, |
| xhci_command_context_t* context) { |
| fbl::AutoLock al(&xhci->command_ring_lock); |
| |
| xhci_transfer_ring_t* cr = &xhci->command_ring; |
| size_t free_count = xhci_transfer_ring_free_trbs(cr); |
| |
| zxlogf(TRACE, "xhci_post_command: free_count: %zu command: %u ptr: %lx control_bits %x\n", |
| free_count, command, ptr, control_bits); |
| |
| if (free_count == 0) { |
| zxlogf(ERROR, "xhci_post_command: command ring full!\n"); |
| return ZX_ERR_NO_RESOURCES; |
| } |
| |
| xhci_trb_t* trb = cr->current_trb; |
| auto index = trb - cr->start; |
| xhci->command_contexts[index] = context; |
| |
| XHCI_WRITE64(&trb->ptr, ptr); |
| XHCI_WRITE32(&trb->status, 0); |
| trb_set_control(trb, command, control_bits); |
| |
| xhci_increment_ring(cr); |
| context->next_trb = cr->current_trb; |
| |
| hw_mb(); |
| XHCI_WRITE32(&xhci->doorbells[0], 0); |
| |
| return ZX_OK; |
| } |
| |
| static void xhci_handle_command_complete_event(xhci_t* xhci, xhci_trb_t* event_trb) { |
| xhci_trb_t* command_trb = xhci_read_trb_ptr(&xhci->command_ring, event_trb); |
| uint32_t cc = XHCI_GET_BITS32(&event_trb->status, EVT_TRB_CC_START, EVT_TRB_CC_BITS); |
| zxlogf(TRACE, "xhci_handle_command_complete_event slot_id: %d command: %d cc: %d\n", |
| (event_trb->control >> TRB_SLOT_ID_START), trb_get_type(command_trb), cc); |
| |
| size_t index = command_trb - xhci->command_ring.start; |
| |
| if (cc == TRB_CC_COMMAND_RING_STOPPED) { |
| // TRB_CC_COMMAND_RING_STOPPED is generated after aborting a command. |
| // Ignore this, since it is unrelated to the next command in the command ring. |
| return; |
| } |
| |
| xhci_command_context_t* context; |
| { |
| fbl::AutoLock al(&xhci->command_ring_lock); |
| context = xhci->command_contexts[index]; |
| xhci_set_dequeue_ptr(&xhci->command_ring, context->next_trb); |
| xhci->command_contexts[index] = nullptr; |
| } |
| |
| context->callback(context->data, cc, command_trb, event_trb); |
| } |
| |
| static void xhci_handle_mfindex_wrap(xhci_t* xhci) { |
| fbl::AutoLock al(&xhci->mfindex_mutex); |
| xhci->mfindex_wrap_count++; |
| xhci->last_mfindex_wrap = zx_clock_get_monotonic(); |
| } |
| |
| uint64_t xhci_get_current_frame(xhci_t* xhci) { |
| fbl::AutoLock al(&xhci->mfindex_mutex); |
| |
| uint32_t mfindex = XHCI_READ32(&xhci->runtime_regs->mfindex) & ((1 << XHCI_MFINDEX_BITS) - 1); |
| uint64_t wrap_count = xhci->mfindex_wrap_count; |
| // try to detect race condition where mfindex has wrapped but we haven't processed wrap event yet |
| if (mfindex < 500) { |
| if (zx_clock_get_monotonic() - xhci->last_mfindex_wrap > ZX_MSEC(1000)) { |
| zxlogf(TRACE, "woah, mfindex wrapped before we got the event!\n"); |
| wrap_count++; |
| } |
| } |
| |
| // shift three to convert from 125us microframes to 1ms frames |
| return ((wrap_count * (1 << XHCI_MFINDEX_BITS)) + mfindex) >> 3; |
| } |
| |
| static void xhci_handle_events(xhci_t* xhci, int interrupter) { |
| xhci_event_ring_t* er = &xhci->event_rings[interrupter]; |
| // process all TRBs with cycle bit matching our CCS |
| while ((XHCI_READ32(&er->current->control) & TRB_C) == er->ccs) { |
| uint32_t type = trb_get_type(er->current); |
| switch (type) { |
| case TRB_EVENT_COMMAND_COMP: |
| xhci_handle_command_complete_event(xhci, er->current); |
| break; |
| case TRB_EVENT_PORT_STATUS_CHANGE: |
| xhci_handle_root_hub_change(xhci); |
| break; |
| case TRB_EVENT_TRANSFER: |
| xhci_handle_transfer_event(xhci, er->current); |
| break; |
| case TRB_EVENT_MFINDEX_WRAP: |
| xhci_handle_mfindex_wrap(xhci); |
| break; |
| default: |
| zxlogf(ERROR, "xhci_handle_events: unhandled event type %d\n", type); |
| break; |
| } |
| fbl::AutoLock l(&er->xfer_lock); |
| if (er->current + 1 == er->end) { |
| er->current = er->start; |
| er->ccs ^= TRB_C; |
| } else { |
| er->current = xhci_next_evt(er, er->current); |
| } |
| xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter]; |
| // Update the dequeue pointer to allow other events to come in |
| uint64_t erdp = xhci_event_ring_current_phys(er); |
| XHCI_WRITE64(&intr_regs->erdp, erdp); |
| } |
| |
| // update event ring dequeue pointer and clear event handler busy flag |
| xhci_update_erdp(xhci, interrupter); |
| } |
| |
| void xhci_handle_interrupt(xhci_t* xhci, uint32_t interrupter) { |
| // clear the interrupt pending flag |
| xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter]; |
| XHCI_WRITE32(&intr_regs->iman, IMAN_IE | IMAN_IP); |
| xhci_handle_events(xhci, interrupter); |
| } |
| |
| bool xhci_add_to_list_tail(xhci_t* xhci, list_node_t* list, usb_request_t* req) { |
| uint64_t node_offset = sizeof(usb_request_t) + offsetof(xhci_usb_request_internal_t, node); |
| list_add_tail(list, (list_node_t*)((uintptr_t)req + node_offset)); |
| return true; |
| } |
| |
| bool xhci_add_to_list_head(xhci_t* xhci, list_node_t* list, usb_request_t* req) { |
| uint64_t node_offset = sizeof(usb_request_t) + offsetof(xhci_usb_request_internal_t, node); |
| list_add_head(list, (list_node_t*)((uintptr_t)req + node_offset)); |
| return true; |
| } |
| |
| bool xhci_remove_from_list_head(xhci_t* xhci, list_node_t* list, usb_request_t** req) { |
| uint64_t node_offset = sizeof(usb_request_t) + offsetof(xhci_usb_request_internal_t, node); |
| list_node_t* node = list_remove_head(list); |
| if (!node) { |
| *req = NULL; |
| return false; |
| } |
| *req = (usb_request_t*)((uintptr_t)node - node_offset); |
| return true; |
| } |
| |
| bool xhci_remove_from_list_tail(xhci_t* xhci, list_node_t* list, usb_request_t** req) { |
| uint64_t node_offset = sizeof(usb_request_t) + offsetof(xhci_usb_request_internal_t, node); |
| list_node_t* node = list_remove_tail(list); |
| if (!node) { |
| *req = NULL; |
| return false; |
| } |
| *req = (usb_request_t*)((uintptr_t)node - node_offset); |
| return true; |
| } |
| |
| void xhci_delete_req_node(xhci_t* xhci, usb_request_t* req) { |
| uint64_t node_offset = sizeof(usb_request_t) + offsetof(xhci_usb_request_internal_t, node); |
| list_node_t* node = (list_node_t*)((uintptr_t)req + node_offset); |
| list_delete(node); |
| } |
| |
| } // namespace usb_xhci |