system/dev/usb/xhci/usb-xhci.cpp - zircon/ - Git at Google

 // Copyright 2016 The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 #include <ddk/binding.h>
 #include <ddk/debug.h>
 #include <ddk/driver.h>
 #include <ddk/phys-iter.h>
 #include <ddk/platform-defs.h>
 #include <ddk/protocol/usb/hci.h>

 #include <hw/arch_ops.h>
 #include <hw/reg.h>
 #include <zircon/syscalls.h>
 #include <zircon/types.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <threads.h>

 #include "xhci-device-manager.h"
 #include "xhci-root-hub.h"
 #include "xhci-util.h"
 #include "xhci.h"

 #ifndef MIN
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 #endif

 #define MAX_SLOTS 255

 #define DEFAULT_PRIORITY 16
 #define HIGH_PRIORITY    24

 #define PDEV_MMIO_INDEX  0
 #define PDEV_IRQ_INDEX   0

 zx_status_t xhci_add_device(xhci_t* xhci, int slot_id, int hub_address, int speed) {
     zxlogf(TRACE, "xhci_add_new_device\n");

     if (!xhci->bus.ops) {
         zxlogf(ERROR, "no bus device in xhci_add_device\n");
         return ZX_ERR_INTERNAL;
     }

     return usb_bus_interface_add_device(&xhci->bus, slot_id, hub_address, speed);
 }

 void xhci_remove_device(xhci_t* xhci, int slot_id) {
     zxlogf(TRACE, "xhci_remove_device %d\n", slot_id);

     if (!xhci->bus.ops) {
         zxlogf(ERROR, "no bus device in xhci_remove_device\n");
         return;
     }

     usb_bus_interface_remove_device(&xhci->bus, slot_id);
 }

 static void xhci_hci_request_queue(void* ctx, usb_request_t* usb_request,
                                    const usb_request_complete_t* complete_cb) {
     auto* xhci = static_cast<xhci_t*>(ctx);
     xhci_request_queue(xhci, usb_request, complete_cb);
 }

 static void xhci_set_bus_interface(void* ctx, const usb_bus_interface_t* bus) {
     auto* xhci = static_cast<xhci_t*>(ctx);

     if (bus) {
         memcpy(&xhci->bus, bus, sizeof(xhci->bus));
         // wait until bus driver has started before doing this
         xhci_queue_start_root_hubs(xhci);
     } else {
         memset(&xhci->bus, 0, sizeof(xhci->bus));
     }
 }

 static size_t xhci_get_max_device_count(void* ctx) {
     auto* xhci = static_cast<xhci_t*>(ctx);
     // add one to allow device IDs to be 1-based
     return xhci->max_slots + XHCI_RH_COUNT + 1;
 }

 static zx_status_t xhci_enable_ep(void* ctx, uint32_t device_id,
                                   usb_endpoint_descriptor_t* ep_desc,
                                   usb_ss_ep_comp_descriptor_t* ss_comp_desc, bool enable) {
     auto* xhci = static_cast<xhci_t*>(ctx);
     return xhci_enable_endpoint(xhci, device_id, ep_desc, ss_comp_desc, enable);
 }

 static uint64_t xhci_get_frame(void* ctx) {
     auto* xhci = static_cast<xhci_t*>(ctx);
     return xhci_get_current_frame(xhci);
 }

 static zx_status_t xhci_config_hub(void* ctx, uint32_t device_id, usb_speed_t speed,
                                    const usb_hub_descriptor_t* descriptor) {
     auto* xhci = static_cast<xhci_t*>(ctx);
     return xhci_configure_hub(xhci, device_id, speed, descriptor);
 }

 static zx_status_t xhci_hub_device_added(void* ctx, uint32_t hub_address, uint32_t port,
                                   usb_speed_t speed) {
     auto* xhci = static_cast<xhci_t*>(ctx);
     return xhci_enumerate_device(xhci, hub_address, port, speed);
 }

 static zx_status_t xhci_hub_device_removed(void* ctx, uint32_t hub_address, uint32_t port) {
     auto* xhci = static_cast<xhci_t*>(ctx);
     xhci_device_disconnected(xhci, hub_address, port);
     return ZX_OK;
 }

 static zx_status_t xhci_reset_ep(void* ctx, uint32_t device_id, uint8_t ep_address) {
     auto* xhci = static_cast<xhci_t*>(ctx);
     return xhci_reset_endpoint(xhci, device_id, ep_address);
 }

 static size_t xhci_get_max_transfer_size(void* ctx, uint32_t device_id, uint8_t ep_address) {
     if (ep_address == 0) {
         // control requests have uint16 length field so we need to support UINT16_MAX
         // we require one setup, status and data event TRB in addition to data transfer TRBs
         // and subtract one more to account for the link TRB
         static_assert(PAGE_SIZE * (TRANSFER_RING_SIZE - 4) >= UINT16_MAX, "TRANSFER_RING_SIZE too small");
         return UINT16_MAX;
     }
     // non-control transfers consist of normal transfer TRBs plus one data event TRB
     // Subtract 2 to reserve a TRB for data event and to account for the link TRB
     return PAGE_SIZE * (TRANSFER_RING_SIZE - 2);
 }

 static zx_status_t xhci_cancel_all(void* ctx, uint32_t device_id, uint8_t ep_address) {
     auto* xhci = static_cast<xhci_t*>(ctx);
     return xhci_cancel_transfers(xhci, device_id, xhci_endpoint_index(ep_address));
 }

 //Returns the public request size plus its private context size.
 static size_t xhci_get_request_size(void* ctx) {
     return sizeof(xhci_usb_request_internal_t) + sizeof(usb_request_t);
 }

 usb_hci_protocol_ops_t xhci_hci_protocol = {
     .request_queue = xhci_hci_request_queue,
     .set_bus_interface = xhci_set_bus_interface,
     .get_max_device_count = xhci_get_max_device_count,
     .enable_endpoint = xhci_enable_ep,
     .get_current_frame = xhci_get_frame,
     .configure_hub = xhci_config_hub,
     .hub_device_added = xhci_hub_device_added,
     .hub_device_removed = xhci_hub_device_removed,
     .reset_endpoint = xhci_reset_ep,
     .get_max_transfer_size = xhci_get_max_transfer_size,
     .cancel_all = xhci_cancel_all,
     .get_request_size = xhci_get_request_size,
 };

 void xhci_request_queue(xhci_t* xhci, usb_request_t* req,
                         const usb_request_complete_t* complete_cb) {
     zx_status_t status;

     xhci_usb_request_internal_t* req_int = USB_REQ_TO_XHCI_INTERNAL(req);
     req_int->complete_cb = *complete_cb;
     if (req->header.length > xhci_get_max_transfer_size(xhci->zxdev, req->header.device_id,
                                                         req->header.ep_address)) {
         status = ZX_ERR_INVALID_ARGS;
     } else {
         status = xhci_queue_transfer(xhci, req);
     }

     if (status != ZX_OK && status != ZX_ERR_BUFFER_TOO_SMALL) {
         usb_request_complete_new(req, status, 0, complete_cb);
     }
 }

 static void xhci_shutdown(xhci_t* xhci) {
     // stop the controller and our device thread
     xhci_stop(xhci);
     xhci->suspended.store(true);
     // stop our interrupt threads
     for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
         zx_interrupt_destroy(xhci->irq_handles[i]);
         thrd_join(xhci->completer_threads[i], nullptr);
         zx_handle_close(xhci->irq_handles[i]);
     }
 }

 static zx_status_t xhci_suspend(void* ctx, uint32_t flags) {
     zxlogf(TRACE, "xhci_suspend %u\n", flags);
     auto* xhci = static_cast<xhci_t*>(ctx);

     // TODO(voydanoff) do different things based on the flags.
     // for now we shutdown the driver in preparation for mexec
     xhci_shutdown(xhci);

     return ZX_OK;
 }

 static void xhci_unbind(void* ctx) {
     zxlogf(INFO, "xhci_unbind\n");
     auto* xhci = static_cast<xhci_t*>(ctx);

     xhci_shutdown(xhci);
     device_remove(xhci->zxdev);
 }

 static void xhci_release(void* ctx) {
     zxlogf(INFO, "xhci_release\n");
     auto* xhci = static_cast<xhci_t*>(ctx);
     mmio_buffer_release(&xhci->mmio);
     zx_handle_close(xhci->cfg_handle);
     xhci_free(xhci);
 }
 static zx_protocol_device_t xhci_device_ops = []() {
     zx_protocol_device_t device;
     device.version = DEVICE_OPS_VERSION;
     device.suspend = xhci_suspend,
     device.unbind = xhci_unbind,
     device.release = xhci_release;
     return device;
 }();

 typedef struct completer {
     xhci_t *xhci;
     uint32_t interrupter;
     uint32_t priority;
 } completer_t;

 static int completer_thread(void *arg) {
     completer_t* completer = (completer_t*)arg;
     zx_handle_t irq_handle = completer->xhci->irq_handles[completer->interrupter];

     // TODO(johngro): See ZX-940.  Get rid of this.  For now we need thread
     // priorities so that realtime transactions use the completer which ends
     // up getting realtime latency guarantees.
     zx_thread_set_priority(completer->priority);

     while (1) {
         zx_status_t wait_res;
         wait_res = zx_interrupt_wait(irq_handle, nullptr);
         if (wait_res != ZX_OK) {
             if (wait_res != ZX_ERR_CANCELED) {
                 zxlogf(ERROR, "unexpected zx_interrupt_wait failure (%d)\n", wait_res);
             }
             break;
         }
         if (completer->xhci->suspended.load()) {
             // TODO(ravoorir): Remove this hack once the interrupt signalling bug
             // is resolved.
             zxlogf(ERROR, "race in zx_interrupt_cancel triggered. Kick off workaround for now\n");
             break;
         }
         xhci_handle_interrupt(completer->xhci, completer->interrupter);
     }
     zxlogf(TRACE, "xhci completer %u thread done\n", completer->interrupter);
     free(completer);
     return 0;
 }

 static int xhci_start_thread(void* arg) {
     xhci_t* xhci = (xhci_t*)arg;
     zxlogf(TRACE, "xhci_start_thread start\n");

     zx_status_t status;
     completer_t* completers[xhci->num_interrupts];
     uint32_t num_completers_initialized = 0;
     for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
         auto* completer = static_cast<completer_t*>(calloc(1, sizeof(completer_t)));
         if (completer == nullptr) {
             status = ZX_ERR_NO_MEMORY;
             goto error_return;
         }
         completers[i] = completer;
         completer->interrupter = i;
         completer->xhci = xhci;
         // We need a high priority thread for isochronous transfers.
         // If there is only one interrupt available, that thread will need
         // to be high priority.
         completer->priority = (i == ISOCH_INTERRUPTER || xhci->num_interrupts == 1) ?
                               HIGH_PRIORITY : DEFAULT_PRIORITY;
         num_completers_initialized++;
     }

     // xhci_start will block, so do this part here instead of in usb_xhci_bind
     status = xhci_start(xhci);
     if (status != ZX_OK) {
         goto error_return;
     }

     device_make_visible(xhci->zxdev);
     for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
         thrd_create_with_name(&xhci->completer_threads[i], completer_thread, completers[i],
                               "completer_thread");
     }

     zxlogf(TRACE, "xhci_start_thread done\n");
     return 0;

 error_return:
     device_remove(xhci->zxdev);
     free(xhci);
     for (uint32_t i = 0; i < num_completers_initialized; i++) {
         free(completers[i]);
     }
     return status;
 }

 static zx_status_t xhci_finish_bind(xhci_t* xhci, zx_device_t* parent) {
     device_add_args_t args = {};
     args.version = DEVICE_ADD_ARGS_VERSION;
     args.name = "xhci";
     args.ctx = xhci;
     args.ops = &xhci_device_ops;
     args.proto_id = ZX_PROTOCOL_USB_HCI;
     args.proto_ops = &xhci_hci_protocol;
     args.flags = DEVICE_ADD_INVISIBLE;

     zx_status_t status = device_add(parent, &args, &xhci->zxdev);
     if (status != ZX_OK) {
         return status;
     }

     thrd_t thread;
     thrd_create_with_name(&thread, xhci_start_thread, xhci, "xhci_start_thread");
     thrd_detach(thread);

     return ZX_OK;
 }

 static zx_status_t usb_xhci_bind_pci(zx_device_t* parent, pci_protocol_t* pci) {
     zx_handle_t cfg_handle = ZX_HANDLE_INVALID;
     xhci_t* xhci = nullptr;
     uint32_t num_irq_handles_initialized = 0;
     zx_status_t status;

     xhci = static_cast<xhci_t*>(calloc(1, sizeof(xhci_t)));
     if (!xhci) {
         status = ZX_ERR_NO_MEMORY;
         goto error_return;
     }

     status = pci_get_bti(pci, 0, &xhci->bti_handle);
     if (status != ZX_OK) {
         goto error_return;
     }

     /*
      * eXtensible Host Controller Interface revision 1.1, section 5, xhci
      * should only use BARs 0 and 1. 0 for 32 bit addressing, and 0+1 for 64 bit addressing.
      */
     zx_pci_bar_t bar;
     status = pci_get_bar(pci, 0u, &bar);
     if (status != ZX_OK) {
         zxlogf(ERROR, "usb_xhci_bind could not find bar\n");
         goto error_return;
     }
     ZX_ASSERT(bar.type == ZX_PCI_BAR_TYPE_MMIO);
     status = mmio_buffer_init(&xhci->mmio, 0, bar.size, bar.handle, ZX_CACHE_POLICY_UNCACHED);
     if (status != ZX_OK) {
         zxlogf(ERROR, "usb_xhci_bind could not map bar\n");
         goto error_return;
     }

     uint32_t irq_cnt;
     irq_cnt = 0;
     status = pci_query_irq_mode(pci, ZX_PCIE_IRQ_MODE_MSI, &irq_cnt);
     if (status != ZX_OK) {
         zxlogf(ERROR, "pci_query_irq_mode failed %d\n", status);
         goto error_return;
     }

     // Cap IRQ count at the number of interrupters we want to use and
     // the number of interrupters supported by XHCI.
     irq_cnt = MIN(irq_cnt, INTERRUPTER_COUNT);
     irq_cnt = MIN(irq_cnt, xhci_get_max_interrupters(xhci));

     // select our IRQ mode
     xhci_mode_t mode;
     mode = XHCI_PCI_MSI;
     status = pci_set_irq_mode(pci, ZX_PCIE_IRQ_MODE_MSI, irq_cnt);
     if (status < 0) {
         zxlogf(ERROR, "MSI interrupts not available, irq_cnt: %d, err: %d\n",
                irq_cnt, status);
         zx_status_t status_legacy = pci_set_irq_mode(pci, ZX_PCIE_IRQ_MODE_LEGACY, 1);

         if (status_legacy < 0) {
             zxlogf(ERROR, "usb_xhci_bind Failed to set IRQ mode to either MSI "
                    "(err = %d) or Legacy (err = %d)\n",
                    status, status_legacy);
             goto error_return;
         }

         mode = XHCI_PCI_LEGACY;
         irq_cnt = 1;
     }

     for (uint32_t i = 0; i < irq_cnt; i++) {
         // register for interrupts
         status = pci_map_interrupt(pci, i, &xhci->irq_handles[i]);
         if (status != ZX_OK) {
             zxlogf(ERROR, "usb_xhci_bind map_interrupt failed %d\n", status);
             goto error_return;
         }
         num_irq_handles_initialized++;
     }
     xhci->cfg_handle = cfg_handle;

     // used for enabling bus mastering
     memcpy(&xhci->pci, pci, sizeof(pci_protocol_t));

     status = xhci_init(xhci, mode, irq_cnt);
     if (status != ZX_OK) {
         goto error_return;
     }
     status = xhci_finish_bind(xhci, parent);
     if (status != ZX_OK) {
         goto error_return;
     }

     return ZX_OK;

 error_return:
     zx_handle_close(xhci->bti_handle);
     for (uint32_t i = 0; i < num_irq_handles_initialized; i++) {
         zx_handle_close(xhci->irq_handles[i]);
     }
     mmio_buffer_release(&xhci->mmio);
     zx_handle_close(xhci->cfg_handle);
     free(xhci);
     return status;
 }


 static zx_status_t usb_xhci_bind_pdev(zx_device_t* parent, pdev_protocol_t* pdev) {
     zx_handle_t irq_handle = ZX_HANDLE_INVALID;
     xhci_t* xhci = nullptr;
     zx_status_t status;

     xhci = static_cast<xhci_t*>(calloc(1, sizeof(xhci_t)));
     if (!xhci) {
         status = ZX_ERR_NO_MEMORY;
         goto error_return;
     }

     status = pdev_get_bti(pdev, 0, &xhci->bti_handle);
     if (status != ZX_OK) {
         goto error_return;
     }

     status = pdev_map_mmio_buffer2(pdev, PDEV_MMIO_INDEX, ZX_CACHE_POLICY_UNCACHED_DEVICE,
                                   &xhci->mmio);
     if (status != ZX_OK) {
         zxlogf(ERROR, "usb_xhci_bind_pdev: pdev_map_mmio failed\n");
         goto error_return;
     }

     status = pdev_map_interrupt(pdev, PDEV_IRQ_INDEX, &irq_handle);
     if (status != ZX_OK) {
         zxlogf(ERROR, "usb_xhci_bind_pdev: pdev_map_interrupt failed\n");
         goto error_return;
     }

     xhci->irq_handles[0] = irq_handle;

     status = xhci_init(xhci, XHCI_PDEV, 1);
     if (status != ZX_OK) {
         goto error_return;
     }
     status = xhci_finish_bind(xhci, parent);
     if (status != ZX_OK) {
         goto error_return;
     }

     return ZX_OK;

 error_return:
     zx_handle_close(xhci->bti_handle);
     mmio_buffer_release(&xhci->mmio);
     free(xhci);
     zx_handle_close(irq_handle);
     return status;
 }

 zx_status_t usb_xhci_bind(void* ctx, zx_device_t* parent) {
     pci_protocol_t pci;
     pdev_protocol_t pdev;
     zx_status_t status;

     if ((status = device_get_protocol(parent, ZX_PROTOCOL_PCI, &pci)) == ZX_OK) {
         return usb_xhci_bind_pci(parent, &pci);
     }
     if ((status = device_get_protocol(parent, ZX_PROTOCOL_PDEV, &pdev)) == ZX_OK) {
         return usb_xhci_bind_pdev(parent, &pdev);
     }

     return status;
 }

 static zx_driver_ops_t xhci_driver_ops = [](){
     zx_driver_ops_t ops;
     ops.version = DRIVER_OPS_VERSION;
     ops.bind = usb_xhci_bind;
     return ops;
 }();

 // clang-format off
 ZIRCON_DRIVER_BEGIN(usb_xhci, xhci_driver_ops, "zircon", "0.1", 9)
     // PCI binding support
     BI_GOTO_IF(NE, BIND_PROTOCOL, ZX_PROTOCOL_PCI, 0),
     BI_ABORT_IF(NE, BIND_PCI_CLASS, 0x0C),
     BI_ABORT_IF(NE, BIND_PCI_SUBCLASS, 0x03),
     BI_MATCH_IF(EQ, BIND_PCI_INTERFACE, 0x30),

     // platform bus binding support
     BI_LABEL(0),
     BI_ABORT_IF(NE, BIND_PLATFORM_DEV_VID, PDEV_VID_GENERIC),
     BI_ABORT_IF(NE, BIND_PLATFORM_DEV_PID, PDEV_PID_GENERIC),
     BI_MATCH_IF(EQ, BIND_PLATFORM_DEV_DID, PDEV_DID_USB_XHCI),

     BI_ABORT(),
 ZIRCON_DRIVER_END(usb_xhci)
	// Copyright 2016 The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	#include <ddk/binding.h>
	#include <ddk/debug.h>
	#include <ddk/driver.h>
	#include <ddk/phys-iter.h>
	#include <ddk/platform-defs.h>
	#include <ddk/protocol/usb/hci.h>

	#include <hw/arch_ops.h>
	#include <hw/reg.h>
	#include <zircon/syscalls.h>
	#include <zircon/types.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <threads.h>

	#include "xhci-device-manager.h"
	#include "xhci-root-hub.h"
	#include "xhci-util.h"
	#include "xhci.h"

	#ifndef MIN
	#define MIN(a, b) ((a) < (b) ? (a) : (b))
	#endif

	#define MAX_SLOTS 255

	#define DEFAULT_PRIORITY 16
	#define HIGH_PRIORITY 24

	#define PDEV_MMIO_INDEX 0
	#define PDEV_IRQ_INDEX 0

	zx_status_t xhci_add_device(xhci_t* xhci, int slot_id, int hub_address, int speed) {
	zxlogf(TRACE, "xhci_add_new_device\n");

	if (!xhci->bus.ops) {
	zxlogf(ERROR, "no bus device in xhci_add_device\n");
	return ZX_ERR_INTERNAL;
	}

	return usb_bus_interface_add_device(&xhci->bus, slot_id, hub_address, speed);
	}

	void xhci_remove_device(xhci_t* xhci, int slot_id) {
	zxlogf(TRACE, "xhci_remove_device %d\n", slot_id);

	if (!xhci->bus.ops) {
	zxlogf(ERROR, "no bus device in xhci_remove_device\n");
	return;
	}

	usb_bus_interface_remove_device(&xhci->bus, slot_id);
	}

	static void xhci_hci_request_queue(void* ctx, usb_request_t* usb_request,
	const usb_request_complete_t* complete_cb) {
	auto* xhci = static_cast<xhci_t*>(ctx);
	xhci_request_queue(xhci, usb_request, complete_cb);
	}

	static void xhci_set_bus_interface(void* ctx, const usb_bus_interface_t* bus) {
	auto* xhci = static_cast<xhci_t*>(ctx);

	if (bus) {
	memcpy(&xhci->bus, bus, sizeof(xhci->bus));
	// wait until bus driver has started before doing this
	xhci_queue_start_root_hubs(xhci);
	} else {
	memset(&xhci->bus, 0, sizeof(xhci->bus));
	}
	}

	static size_t xhci_get_max_device_count(void* ctx) {
	auto* xhci = static_cast<xhci_t*>(ctx);
	// add one to allow device IDs to be 1-based
	return xhci->max_slots + XHCI_RH_COUNT + 1;
	}

	static zx_status_t xhci_enable_ep(void* ctx, uint32_t device_id,
	usb_endpoint_descriptor_t* ep_desc,
	usb_ss_ep_comp_descriptor_t* ss_comp_desc, bool enable) {
	auto* xhci = static_cast<xhci_t*>(ctx);
	return xhci_enable_endpoint(xhci, device_id, ep_desc, ss_comp_desc, enable);
	}

	static uint64_t xhci_get_frame(void* ctx) {
	auto* xhci = static_cast<xhci_t*>(ctx);
	return xhci_get_current_frame(xhci);
	}

	static zx_status_t xhci_config_hub(void* ctx, uint32_t device_id, usb_speed_t speed,
	const usb_hub_descriptor_t* descriptor) {
	auto* xhci = static_cast<xhci_t*>(ctx);
	return xhci_configure_hub(xhci, device_id, speed, descriptor);
	}

	static zx_status_t xhci_hub_device_added(void* ctx, uint32_t hub_address, uint32_t port,
	usb_speed_t speed) {
	auto* xhci = static_cast<xhci_t*>(ctx);
	return xhci_enumerate_device(xhci, hub_address, port, speed);
	}

	static zx_status_t xhci_hub_device_removed(void* ctx, uint32_t hub_address, uint32_t port) {
	auto* xhci = static_cast<xhci_t*>(ctx);
	xhci_device_disconnected(xhci, hub_address, port);
	return ZX_OK;
	}

	static zx_status_t xhci_reset_ep(void* ctx, uint32_t device_id, uint8_t ep_address) {
	auto* xhci = static_cast<xhci_t*>(ctx);
	return xhci_reset_endpoint(xhci, device_id, ep_address);
	}

	static size_t xhci_get_max_transfer_size(void* ctx, uint32_t device_id, uint8_t ep_address) {
	if (ep_address == 0) {
	// control requests have uint16 length field so we need to support UINT16_MAX
	// we require one setup, status and data event TRB in addition to data transfer TRBs
	// and subtract one more to account for the link TRB
	static_assert(PAGE_SIZE * (TRANSFER_RING_SIZE - 4) >= UINT16_MAX, "TRANSFER_RING_SIZE too small");
	return UINT16_MAX;
	}
	// non-control transfers consist of normal transfer TRBs plus one data event TRB
	// Subtract 2 to reserve a TRB for data event and to account for the link TRB
	return PAGE_SIZE * (TRANSFER_RING_SIZE - 2);
	}

	static zx_status_t xhci_cancel_all(void* ctx, uint32_t device_id, uint8_t ep_address) {
	auto* xhci = static_cast<xhci_t*>(ctx);
	return xhci_cancel_transfers(xhci, device_id, xhci_endpoint_index(ep_address));
	}

	//Returns the public request size plus its private context size.
	static size_t xhci_get_request_size(void* ctx) {
	return sizeof(xhci_usb_request_internal_t) + sizeof(usb_request_t);
	}

	usb_hci_protocol_ops_t xhci_hci_protocol = {
	.request_queue = xhci_hci_request_queue,
	.set_bus_interface = xhci_set_bus_interface,
	.get_max_device_count = xhci_get_max_device_count,
	.enable_endpoint = xhci_enable_ep,
	.get_current_frame = xhci_get_frame,
	.configure_hub = xhci_config_hub,
	.hub_device_added = xhci_hub_device_added,
	.hub_device_removed = xhci_hub_device_removed,
	.reset_endpoint = xhci_reset_ep,
	.get_max_transfer_size = xhci_get_max_transfer_size,
	.cancel_all = xhci_cancel_all,
	.get_request_size = xhci_get_request_size,
	};

	void xhci_request_queue(xhci_t* xhci, usb_request_t* req,
	const usb_request_complete_t* complete_cb) {
	zx_status_t status;

	xhci_usb_request_internal_t* req_int = USB_REQ_TO_XHCI_INTERNAL(req);
	req_int->complete_cb = *complete_cb;
	if (req->header.length > xhci_get_max_transfer_size(xhci->zxdev, req->header.device_id,
	req->header.ep_address)) {
	status = ZX_ERR_INVALID_ARGS;
	} else {
	status = xhci_queue_transfer(xhci, req);
	}

	if (status != ZX_OK && status != ZX_ERR_BUFFER_TOO_SMALL) {
	usb_request_complete_new(req, status, 0, complete_cb);
	}
	}

	static void xhci_shutdown(xhci_t* xhci) {
	// stop the controller and our device thread
	xhci_stop(xhci);
	xhci->suspended.store(true);
	// stop our interrupt threads
	for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
	zx_interrupt_destroy(xhci->irq_handles[i]);
	thrd_join(xhci->completer_threads[i], nullptr);
	zx_handle_close(xhci->irq_handles[i]);
	}
	}

	static zx_status_t xhci_suspend(void* ctx, uint32_t flags) {
	zxlogf(TRACE, "xhci_suspend %u\n", flags);
	auto* xhci = static_cast<xhci_t*>(ctx);

	// TODO(voydanoff) do different things based on the flags.
	// for now we shutdown the driver in preparation for mexec
	xhci_shutdown(xhci);

	return ZX_OK;
	}

	static void xhci_unbind(void* ctx) {
	zxlogf(INFO, "xhci_unbind\n");
	auto* xhci = static_cast<xhci_t*>(ctx);

	xhci_shutdown(xhci);
	device_remove(xhci->zxdev);
	}

	static void xhci_release(void* ctx) {
	zxlogf(INFO, "xhci_release\n");
	auto* xhci = static_cast<xhci_t*>(ctx);
	mmio_buffer_release(&xhci->mmio);
	zx_handle_close(xhci->cfg_handle);
	xhci_free(xhci);
	}
	static zx_protocol_device_t xhci_device_ops = []() {
	zx_protocol_device_t device;
	device.version = DEVICE_OPS_VERSION;
	device.suspend = xhci_suspend,
	device.unbind = xhci_unbind,
	device.release = xhci_release;
	return device;
	}();

	typedef struct completer {
	xhci_t *xhci;
	uint32_t interrupter;
	uint32_t priority;
	} completer_t;

	static int completer_thread(void *arg) {
	completer_t* completer = (completer_t*)arg;
	zx_handle_t irq_handle = completer->xhci->irq_handles[completer->interrupter];

	// TODO(johngro): See ZX-940. Get rid of this. For now we need thread
	// priorities so that realtime transactions use the completer which ends
	// up getting realtime latency guarantees.
	zx_thread_set_priority(completer->priority);

	while (1) {
	zx_status_t wait_res;
	wait_res = zx_interrupt_wait(irq_handle, nullptr);
	if (wait_res != ZX_OK) {
	if (wait_res != ZX_ERR_CANCELED) {
	zxlogf(ERROR, "unexpected zx_interrupt_wait failure (%d)\n", wait_res);
	}
	break;
	}
	if (completer->xhci->suspended.load()) {
	// TODO(ravoorir): Remove this hack once the interrupt signalling bug
	// is resolved.
	zxlogf(ERROR, "race in zx_interrupt_cancel triggered. Kick off workaround for now\n");
	break;
	}
	xhci_handle_interrupt(completer->xhci, completer->interrupter);
	}
	zxlogf(TRACE, "xhci completer %u thread done\n", completer->interrupter);
	free(completer);
	return 0;
	}

	static int xhci_start_thread(void* arg) {
	xhci_t* xhci = (xhci_t*)arg;
	zxlogf(TRACE, "xhci_start_thread start\n");

	zx_status_t status;
	completer_t* completers[xhci->num_interrupts];
	uint32_t num_completers_initialized = 0;
	for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
	auto* completer = static_cast<completer_t*>(calloc(1, sizeof(completer_t)));
	if (completer == nullptr) {
	status = ZX_ERR_NO_MEMORY;
	goto error_return;
	}
	completers[i] = completer;
	completer->interrupter = i;
	completer->xhci = xhci;
	// We need a high priority thread for isochronous transfers.
	// If there is only one interrupt available, that thread will need
	// to be high priority.
	completer->priority = (i == ISOCH_INTERRUPTER \|\| xhci->num_interrupts == 1) ?
	HIGH_PRIORITY : DEFAULT_PRIORITY;
	num_completers_initialized++;
	}

	// xhci_start will block, so do this part here instead of in usb_xhci_bind
	status = xhci_start(xhci);
	if (status != ZX_OK) {
	goto error_return;
	}

	device_make_visible(xhci->zxdev);
	for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
	thrd_create_with_name(&xhci->completer_threads[i], completer_thread, completers[i],
	"completer_thread");
	}

	zxlogf(TRACE, "xhci_start_thread done\n");
	return 0;

	error_return:
	device_remove(xhci->zxdev);
	free(xhci);
	for (uint32_t i = 0; i < num_completers_initialized; i++) {
	free(completers[i]);
	}
	return status;
	}

	static zx_status_t xhci_finish_bind(xhci_t* xhci, zx_device_t* parent) {
	device_add_args_t args = {};
	args.version = DEVICE_ADD_ARGS_VERSION;
	args.name = "xhci";
	args.ctx = xhci;
	args.ops = &xhci_device_ops;
	args.proto_id = ZX_PROTOCOL_USB_HCI;
	args.proto_ops = &xhci_hci_protocol;
	args.flags = DEVICE_ADD_INVISIBLE;

	zx_status_t status = device_add(parent, &args, &xhci->zxdev);
	if (status != ZX_OK) {
	return status;
	}

	thrd_t thread;
	thrd_create_with_name(&thread, xhci_start_thread, xhci, "xhci_start_thread");
	thrd_detach(thread);

	return ZX_OK;
	}

	static zx_status_t usb_xhci_bind_pci(zx_device_t* parent, pci_protocol_t* pci) {
	zx_handle_t cfg_handle = ZX_HANDLE_INVALID;
	xhci_t* xhci = nullptr;
	uint32_t num_irq_handles_initialized = 0;
	zx_status_t status;

	xhci = static_cast<xhci_t*>(calloc(1, sizeof(xhci_t)));
	if (!xhci) {
	status = ZX_ERR_NO_MEMORY;
	goto error_return;
	}

	status = pci_get_bti(pci, 0, &xhci->bti_handle);
	if (status != ZX_OK) {
	goto error_return;
	}

	/*
	* eXtensible Host Controller Interface revision 1.1, section 5, xhci
	* should only use BARs 0 and 1. 0 for 32 bit addressing, and 0+1 for 64 bit addressing.
	*/
	zx_pci_bar_t bar;
	status = pci_get_bar(pci, 0u, &bar);
	if (status != ZX_OK) {
	zxlogf(ERROR, "usb_xhci_bind could not find bar\n");
	goto error_return;
	}
	ZX_ASSERT(bar.type == ZX_PCI_BAR_TYPE_MMIO);
	status = mmio_buffer_init(&xhci->mmio, 0, bar.size, bar.handle, ZX_CACHE_POLICY_UNCACHED);
	if (status != ZX_OK) {
	zxlogf(ERROR, "usb_xhci_bind could not map bar\n");
	goto error_return;
	}

	uint32_t irq_cnt;
	irq_cnt = 0;
	status = pci_query_irq_mode(pci, ZX_PCIE_IRQ_MODE_MSI, &irq_cnt);
	if (status != ZX_OK) {
	zxlogf(ERROR, "pci_query_irq_mode failed %d\n", status);
	goto error_return;
	}

	// Cap IRQ count at the number of interrupters we want to use and
	// the number of interrupters supported by XHCI.
	irq_cnt = MIN(irq_cnt, INTERRUPTER_COUNT);
	irq_cnt = MIN(irq_cnt, xhci_get_max_interrupters(xhci));

	// select our IRQ mode
	xhci_mode_t mode;
	mode = XHCI_PCI_MSI;
	status = pci_set_irq_mode(pci, ZX_PCIE_IRQ_MODE_MSI, irq_cnt);
	if (status < 0) {
	zxlogf(ERROR, "MSI interrupts not available, irq_cnt: %d, err: %d\n",
	irq_cnt, status);
	zx_status_t status_legacy = pci_set_irq_mode(pci, ZX_PCIE_IRQ_MODE_LEGACY, 1);

	if (status_legacy < 0) {
	zxlogf(ERROR, "usb_xhci_bind Failed to set IRQ mode to either MSI "
	"(err = %d) or Legacy (err = %d)\n",
	status, status_legacy);
	goto error_return;
	}

	mode = XHCI_PCI_LEGACY;
	irq_cnt = 1;
	}

	for (uint32_t i = 0; i < irq_cnt; i++) {
	// register for interrupts
	status = pci_map_interrupt(pci, i, &xhci->irq_handles[i]);
	if (status != ZX_OK) {
	zxlogf(ERROR, "usb_xhci_bind map_interrupt failed %d\n", status);
	goto error_return;
	}
	num_irq_handles_initialized++;
	}
	xhci->cfg_handle = cfg_handle;

	// used for enabling bus mastering
	memcpy(&xhci->pci, pci, sizeof(pci_protocol_t));

	status = xhci_init(xhci, mode, irq_cnt);
	if (status != ZX_OK) {
	goto error_return;
	}
	status = xhci_finish_bind(xhci, parent);
	if (status != ZX_OK) {
	goto error_return;
	}

	return ZX_OK;

	error_return:
	zx_handle_close(xhci->bti_handle);
	for (uint32_t i = 0; i < num_irq_handles_initialized; i++) {
	zx_handle_close(xhci->irq_handles[i]);
	}
	mmio_buffer_release(&xhci->mmio);
	zx_handle_close(xhci->cfg_handle);
	free(xhci);
	return status;
	}


	static zx_status_t usb_xhci_bind_pdev(zx_device_t* parent, pdev_protocol_t* pdev) {
	zx_handle_t irq_handle = ZX_HANDLE_INVALID;
	xhci_t* xhci = nullptr;
	zx_status_t status;

	xhci = static_cast<xhci_t*>(calloc(1, sizeof(xhci_t)));
	if (!xhci) {
	status = ZX_ERR_NO_MEMORY;
	goto error_return;
	}

	status = pdev_get_bti(pdev, 0, &xhci->bti_handle);
	if (status != ZX_OK) {
	goto error_return;
	}

	status = pdev_map_mmio_buffer2(pdev, PDEV_MMIO_INDEX, ZX_CACHE_POLICY_UNCACHED_DEVICE,
	&xhci->mmio);
	if (status != ZX_OK) {
	zxlogf(ERROR, "usb_xhci_bind_pdev: pdev_map_mmio failed\n");
	goto error_return;
	}

	status = pdev_map_interrupt(pdev, PDEV_IRQ_INDEX, &irq_handle);
	if (status != ZX_OK) {
	zxlogf(ERROR, "usb_xhci_bind_pdev: pdev_map_interrupt failed\n");
	goto error_return;
	}

	xhci->irq_handles[0] = irq_handle;

	status = xhci_init(xhci, XHCI_PDEV, 1);
	if (status != ZX_OK) {
	goto error_return;
	}
	status = xhci_finish_bind(xhci, parent);
	if (status != ZX_OK) {
	goto error_return;
	}

	return ZX_OK;

	error_return:
	zx_handle_close(xhci->bti_handle);
	mmio_buffer_release(&xhci->mmio);
	free(xhci);
	zx_handle_close(irq_handle);
	return status;
	}

	zx_status_t usb_xhci_bind(void* ctx, zx_device_t* parent) {
	pci_protocol_t pci;
	pdev_protocol_t pdev;
	zx_status_t status;

	if ((status = device_get_protocol(parent, ZX_PROTOCOL_PCI, &pci)) == ZX_OK) {
	return usb_xhci_bind_pci(parent, &pci);
	}
	if ((status = device_get_protocol(parent, ZX_PROTOCOL_PDEV, &pdev)) == ZX_OK) {
	return usb_xhci_bind_pdev(parent, &pdev);
	}

	return status;
	}

	static zx_driver_ops_t xhci_driver_ops = [](){
	zx_driver_ops_t ops;
	ops.version = DRIVER_OPS_VERSION;
	ops.bind = usb_xhci_bind;
	return ops;
	}();

	// clang-format off
	ZIRCON_DRIVER_BEGIN(usb_xhci, xhci_driver_ops, "zircon", "0.1", 9)
	// PCI binding support
	BI_GOTO_IF(NE, BIND_PROTOCOL, ZX_PROTOCOL_PCI, 0),
	BI_ABORT_IF(NE, BIND_PCI_CLASS, 0x0C),
	BI_ABORT_IF(NE, BIND_PCI_SUBCLASS, 0x03),
	BI_MATCH_IF(EQ, BIND_PCI_INTERFACE, 0x30),

	// platform bus binding support
	BI_LABEL(0),
	BI_ABORT_IF(NE, BIND_PLATFORM_DEV_VID, PDEV_VID_GENERIC),
	BI_ABORT_IF(NE, BIND_PLATFORM_DEV_PID, PDEV_PID_GENERIC),
	BI_MATCH_IF(EQ, BIND_PLATFORM_DEV_DID, PDEV_DID_USB_XHCI),

	BI_ABORT(),
	ZIRCON_DRIVER_END(usb_xhci)