[zircon] Fix XHCI, PHY tuning, and CDC Ethernet issues
This patch fixes various XHCI issues that caused problems
with CDC-Ethernet on certain boards with high-latency interrupts.
The following changes have been made in this patch:
* Add a direct mode for USB URBs to bypass re-assembly
and buffering throughout the pipeline and take the
most "direct" path from an interrupt handler to driver code.
* Enable direct mode for CDC Ethernet TX and RX paths.
* Increase the buffer size used for CDC Ethernet.
* Implement multi-page TRB support in XHCI.
* Increase the size of the transfer and event ring buffers.
* Revert workaround in pack-images.py
CONN-50
ZX-3957
Test: Validate no software-related packet loss in XHCI and
CDC Ethernet drivers on affected boards. Ran additional tests described
in CONN-50.
Change-Id: Ice7429ee63109db67b254488d7127a8bd95394ab
diff --git a/zircon/system/banjo/ddk.protocol.usb.request/usb-request.banjo b/zircon/system/banjo/ddk.protocol.usb.request/usb-request.banjo
index 5e1172d..b0d918c 100644
--- a/zircon/system/banjo/ddk.protocol.usb.request/usb-request.banjo
+++ b/zircon/system/banjo/ddk.protocol.usb.request/usb-request.banjo
@@ -77,6 +77,15 @@
/// |silent_completions_count| to know how many consecutive requests prior to this one
/// (in relation to queue order for the endpoint) have completed successfully.
bool cb_on_error_only;
+ /// Direct mode -- if set to true, this packet is handled with high priority directly
+ /// in interrupt context. It will NOT be safe to block in any callbacks, and all layers
+ /// should take the most direct path to route the packet to the requesting driver from
+ /// hardware.
+ bool direct;
+ /// If true, resets an endpoint and does not transfer any data.
+ bool reset;
+ /// The address of the endpoint to reset.
+ uint8 reset_address;
};
[Layout="ddk-callback"]
diff --git a/zircon/system/core/netsvc/BUILD.gn b/zircon/system/core/netsvc/BUILD.gn
index c47a522..28dfe83 100644
--- a/zircon/system/core/netsvc/BUILD.gn
+++ b/zircon/system/core/netsvc/BUILD.gn
@@ -11,17 +11,14 @@
"netsvc.cpp",
]
if (enable_netsvc_debugging_features) {
- sources += [
- "debug-command-enabled.cpp",
- ]
+ sources += [ "debug-command-enabled.cpp" ]
} else {
- sources += [
- "debug-command-disabled.cpp",
- ]
+ sources += [ "debug-command-disabled.cpp" ]
}
deps = [
":netsvc_common",
"$zx/system/fidl/fuchsia-hardware-ethernet:c",
+ "$zx/system/ulib/fbl",
"$zx/system/ulib/fdio",
"$zx/system/ulib/inet6",
"$zx/system/ulib/sync",
@@ -39,8 +36,8 @@
sources = [
"board-name.cpp",
"file-api.cpp",
- "netcp.cpp",
"netboot.cpp",
+ "netcp.cpp",
"paver.cpp",
"payload-streamer.cpp",
"tftp.cpp",
@@ -51,8 +48,8 @@
"$zx/system/fidl/fuchsia-device-manager:c",
"$zx/system/fidl/fuchsia-hardware-block:c",
"$zx/system/fidl/fuchsia-sysinfo:c",
- "$zx/system/ulib/chromeos-disk-setup",
"$zx/system/ulib/async-loop:async-loop-cpp",
+ "$zx/system/ulib/chromeos-disk-setup",
"$zx/system/ulib/fdio",
"$zx/system/ulib/gpt",
"$zx/system/ulib/libzbi",
diff --git a/zircon/system/dev/board/astro/astro-usb.c b/zircon/system/dev/board/astro/astro-usb.c
index 7864656..0721150 100644
--- a/zircon/system/dev/board/astro/astro-usb.c
+++ b/zircon/system/dev/board/astro/astro-usb.c
@@ -63,22 +63,6 @@
return status;
}
- volatile void* base = buf.vaddr;
-
- if (default_val) {
- writel(0, base + 0x38);
- writel(PLL_SETTING_5, base + 0x34);
- } else {
- writel(PLL_SETTING_3, base + 0x50);
- writel(PLL_SETTING_4, base + 0x10);
- if (host) {
- writel(PLL_SETTING_6, base + 0x38);
- } else {
- writel(PLL_SETTING_7, base + 0x38);
- }
- writel(PLL_SETTING_5, base + 0x34);
- }
-
mmio_buffer_release(&buf);
return ZX_OK;
}
diff --git a/zircon/system/dev/ethernet/ethernet/ethernet.cpp b/zircon/system/dev/ethernet/ethernet/ethernet.cpp
index 144ef3a..c75411a 100644
--- a/zircon/system/dev/ethernet/ethernet/ethernet.cpp
+++ b/zircon/system/dev/ethernet/ethernet/ethernet.cpp
@@ -143,10 +143,13 @@
if (status != ZX_OK) {
if (status == ZX_ERR_SHOULD_WAIT) {
fail_receive_read_ += 1;
- if (fail_receive_read_ == 1 ||
- (fail_receive_read_ % kFailureReportRate) == 0) {
- zxlogf(WARN, "eth [%s]: warning: no rx buffers available, frame dropped "
- "(%u time%s)\n",
+ if (fail_receive_read_ == 1 || (fail_receive_read_ % kFailureReportRate) == 0) {
+ // TODO(bbosak): Printing this warning
+ // can result in more dropped packets.
+ // Find a better way to log this.
+ zxlogf(WARN,
+ "eth [%s]: warning: no rx buffers available, frame dropped "
+ "(%u time%s)\n",
name_, fail_receive_read_, fail_receive_read_ > 1 ? "s" : "");
}
} else {
diff --git a/zircon/system/dev/ethernet/usb-cdc-ecm/usb-cdc-ecm.c b/zircon/system/dev/ethernet/usb-cdc-ecm/usb-cdc-ecm.c
index 013115b..589c9cd 100644
--- a/zircon/system/dev/ethernet/usb-cdc-ecm/usb-cdc-ecm.c
+++ b/zircon/system/dev/ethernet/usb-cdc-ecm/usb-cdc-ecm.c
@@ -24,7 +24,7 @@
// The maximum amount of memory we are willing to allocate to transaction buffers
#define MAX_TX_BUF_SZ 32768
-#define MAX_RX_BUF_SZ 32768
+#define MAX_RX_BUF_SZ 1500 * 2048
#define ETHMAC_MAX_TRANSMIT_DELAY 100
#define ETHMAC_MAX_RECV_DELAY 100
@@ -249,34 +249,61 @@
return ZX_OK;
}
-static void usb_write_complete(void* cookie, usb_request_t* request) {
+// Write completion callback. Normally -- this will simply acquire the TX lock, release it,
+// and re-queue the USB request.
+// The error case is a bit more complicated. We set the reset bit on the request, and queue
+// a packet that triggers a reset (asynchronously). We then immediately return to the interrupt
+// thread with the lock held to allow for interrupt processing to take place. Once the reset
+// completes, this function is called again with the lock still held, and request processing
+// continues normally. It is necessary to keep the lock held after returning in the error case
+// because we do not want other packets to get queued out-of-order while the asynchronous operation
+// is in progress.
+static void usb_write_complete(void* cookie,
+ usb_request_t* request) __TA_NO_THREAD_SAFETY_ANALYSIS {
ecm_ctx_t* ctx = cookie;
if (request->response.status == ZX_ERR_IO_NOT_PRESENT) {
usb_request_release(request);
return;
}
-
- mtx_lock(&ctx->tx_mutex);
-
- // Return transmission buffer to pool
- zx_status_t status = usb_req_list_add_tail(&ctx->tx_txn_bufs, request, ctx->parent_req_size);
- ZX_DEBUG_ASSERT(status == ZX_OK);
-
- if (request->response.status == ZX_ERR_IO_REFUSED) {
- zxlogf(TRACE, "%s: resetting transmit endpoint\n", module_name);
- usb_reset_endpoint(&ctx->usb, ctx->tx_endpoint.addr);
- }
-
- if (request->response.status == ZX_ERR_IO_INVALID) {
- zxlogf(TRACE, "%s: slowing down the requests by %d usec."
- "Resetting the transmit endpoint\n",
- module_name, ETHMAC_TRANSMIT_DELAY);
- if (ctx->tx_endpoint_delay < ETHMAC_MAX_TRANSMIT_DELAY) {
- ctx->tx_endpoint_delay += ETHMAC_TRANSMIT_DELAY;
+ // If reset, we still hold the TX mutex.
+ if (!request->reset) {
+ mtx_lock(&ctx->tx_mutex);
+ // Return transmission buffer to pool
+ zx_status_t status =
+ usb_req_list_add_tail(&ctx->tx_txn_bufs, request, ctx->parent_req_size);
+ ZX_DEBUG_ASSERT(status == ZX_OK);
+ if (request->response.status == ZX_ERR_IO_REFUSED) {
+ zxlogf(TRACE, "%s: resetting transmit endpoint\n", module_name);
+ request->reset = true;
+ request->reset_address = ctx->tx_endpoint.addr;
+ usb_request_complete_t complete = {
+ .callback = usb_write_complete,
+ .ctx = ctx,
+ };
+ usb_request_queue(&ctx->usb, request, &complete);
+ return;
}
- usb_reset_endpoint(&ctx->usb, ctx->tx_endpoint.addr);
+
+ if (request->response.status == ZX_ERR_IO_INVALID) {
+ zxlogf(TRACE,
+ "%s: slowing down the requests by %d usec."
+ "Resetting the transmit endpoint\n",
+ module_name, ETHMAC_TRANSMIT_DELAY);
+ if (ctx->tx_endpoint_delay < ETHMAC_MAX_TRANSMIT_DELAY) {
+ ctx->tx_endpoint_delay += ETHMAC_TRANSMIT_DELAY;
+ }
+ request->reset = true;
+ request->reset_address = ctx->tx_endpoint.addr;
+ usb_request_complete_t complete = {
+ .callback = usb_write_complete,
+ .ctx = ctx,
+ };
+ usb_request_queue(&ctx->usb, request, &complete);
+ return;
+ }
}
+ request->reset = false;
bool additional_tx_queued = false;
txn_info_t* txn;
@@ -321,12 +348,11 @@
mtx_unlock(&ctx->ethmac_mutex);
}
-static void usb_read_complete(void* cookie, usb_request_t* request) {
+static void usb_read_complete(void* cookie, usb_request_t* request) __TA_NO_THREAD_SAFETY_ANALYSIS {
ecm_ctx_t* ctx = cookie;
-
if (request->response.status != ZX_OK) {
- zxlogf(TRACE, "%s: usb_read_complete called with status %d\n",
- module_name, (int)request->response.status);
+ zxlogf(TRACE, "%s: usb_read_complete called with status %d\n", module_name,
+ (int)request->response.status);
}
if (request->response.status == ZX_ERR_IO_NOT_PRESENT) {
@@ -336,20 +362,37 @@
if (request->response.status == ZX_ERR_IO_REFUSED) {
zxlogf(TRACE, "%s: resetting receive endpoint\n", module_name);
- usb_reset_endpoint(&ctx->usb, ctx->rx_endpoint.addr);
+ request->reset = true;
+ request->reset_address = ctx->rx_endpoint.addr;
+ usb_request_complete_t complete = {
+ .callback = usb_read_complete,
+ .ctx = ctx,
+ };
+ usb_request_queue(&ctx->usb, request, &complete);
+ return;
} else if (request->response.status == ZX_ERR_IO_INVALID) {
if (ctx->rx_endpoint_delay < ETHMAC_MAX_RECV_DELAY) {
ctx->rx_endpoint_delay += ETHMAC_RECV_DELAY;
}
- zxlogf(TRACE, "%s: slowing down the requests by %d usec."
- "Resetting the recv endpoint\n",
+ zxlogf(TRACE,
+ "%s: slowing down the requests by %d usec."
+ "Resetting the recv endpoint\n",
module_name, ETHMAC_RECV_DELAY);
- usb_reset_endpoint(&ctx->usb, ctx->rx_endpoint.addr);
- } else if (request->response.status == ZX_OK) {
+ request->reset = true;
+ request->reset_address = ctx->rx_endpoint.addr;
+ usb_request_complete_t complete = {
+ .callback = usb_read_complete,
+ .ctx = ctx,
+ };
+ usb_request_queue(&ctx->usb, request, &complete);
+ return;
+ } else if (request->response.status == ZX_OK && !request->reset) {
usb_recv(ctx, request);
}
-
- zx_nanosleep(zx_deadline_after(ZX_USEC(ctx->rx_endpoint_delay)));
+ if (ctx->rx_endpoint_delay) {
+ zx_nanosleep(zx_deadline_after(ZX_USEC(ctx->rx_endpoint_delay)));
+ }
+ request->reset = false;
usb_request_complete_t complete = {
.callback = usb_read_complete,
.ctx = ctx,
@@ -722,8 +765,9 @@
size_t tx_buf_remain = MAX_TX_BUF_SZ;
while (tx_buf_remain >= tx_buf_sz) {
usb_request_t* tx_buf;
- zx_status_t alloc_result = usb_request_alloc(&tx_buf, tx_buf_sz,
- ecm_ctx->tx_endpoint.addr, req_size);
+ zx_status_t alloc_result =
+ usb_request_alloc(&tx_buf, tx_buf_sz, ecm_ctx->tx_endpoint.addr, req_size);
+ tx_buf->direct = true;
if (alloc_result != ZX_OK) {
result = alloc_result;
goto fail;
@@ -741,7 +785,7 @@
}
// Allocate rx transaction buffers
- uint16_t rx_buf_sz = ecm_ctx->mtu;
+ uint32_t rx_buf_sz = ecm_ctx->mtu;
#if MAX_TX_BUF_SZ < UINT16_MAX
if (rx_buf_sz > MAX_RX_BUF_SZ) {
zxlogf(ERROR, "%s: insufficient space for even a single rx buffer\n", module_name);
@@ -763,7 +807,7 @@
result = alloc_result;
goto fail;
}
-
+ rx_buf->direct = true;
usb_request_queue(&ecm_ctx->usb, rx_buf, &complete);
rx_buf_remain -= rx_buf_sz;
}
diff --git a/zircon/system/dev/lib/amlogic/aml-usb-phy-v2.c b/zircon/system/dev/lib/amlogic/aml-usb-phy-v2.c
index 8e02226..3356d889 100644
--- a/zircon/system/dev/lib/amlogic/aml-usb-phy-v2.c
+++ b/zircon/system/dev/lib/amlogic/aml-usb-phy-v2.c
@@ -10,9 +10,16 @@
#include <soc/aml-s905d2/s905d2-hw.h>
// from mesong12a.dtsi
-#define PLL_SETTING_0 0x09400414
-#define PLL_SETTING_1 0x927E0000
-#define PLL_SETTING_2 0xac5f49e5
+#define PLL_SETTING_0 0x09400414
+#define PLL_SETTING_1 0x927E0000
+#define PLL_SETTING_2 0xac5f49e5
+#define PLL_SETTING_3 0xfe18
+#define PLL_SETTING_4 0xfff
+#define PLL_SETTING_5 0x78000
+#define PLL_SETTING_6 0xe0004
+#define PLL_SETTING_7 0xe000c
+
+// Order PLL, tuning
// set_usb_pll() in phy_aml_new_usb2_v2.c
static zx_status_t set_usb_pll(zx_paddr_t reg_base) {
@@ -33,6 +40,24 @@
zx_nanosleep(zx_deadline_after(ZX_USEC(100)));
writel((0x10000000 | PLL_SETTING_0), reg + 0x40);
+ // PLL
+
+ zx_nanosleep(zx_deadline_after(ZX_USEC(100)));
+ writel((PLL_SETTING_3), reg + 0x50);
+ writel((PLL_SETTING_4), reg + 0x10);
+ // Recovery state
+ writel(0, reg + 0x38);
+ writel((PLL_SETTING_5), reg + 0x34);
+ // Disconnect threshold
+ writel(0x3c, reg + 0xc);
+ // Tuning
+
+ zx_nanosleep(zx_deadline_after(ZX_USEC(100)));
+ writel(PLL_SETTING_6, reg + 0x38);
+ writel(PLL_SETTING_5, reg + 0x34);
+
+ zx_nanosleep(zx_deadline_after(ZX_MSEC(100)));
+
mmio_buffer_release(&buf);
return ZX_OK;
}
diff --git a/zircon/system/dev/usb/usb-bus/usb-device.cpp b/zircon/system/dev/usb/usb-bus/usb-device.cpp
index 442f902..59b9b3f 100644
--- a/zircon/system/dev/usb/usb-bus/usb-device.cpp
+++ b/zircon/system/dev/usb/usb-bus/usb-device.cpp
@@ -62,6 +62,11 @@
// Call completion callbacks outside of the lock.
for (auto req = temp_queue.pop(); req; req = temp_queue.pop()) {
+ if (req->operation()->reset) {
+ req->Complete(hci_.ResetEndpoint(device_id_, req->operation()->reset_address), 0,
+ req->private_storage()->silent_completions_count);
+ continue;
+ }
const auto& response = req->request()->response;
req->Complete(response.status, response.actual,
req->private_storage()->silent_completions_count);
@@ -163,6 +168,10 @@
// usb request completion for the requests passed down to the HCI driver
void UsbDevice::RequestComplete(usb_request_t* req) {
+ if (req->reset) {
+ QueueCallback(req);
+ return;
+ }
auto* ep = GetEndpoint(req->header.ep_address);
if (!ep) {
zxlogf(ERROR, "could not find endpoint with address 0x%x\n", req->header.ep_address);
@@ -352,6 +361,19 @@
void UsbDevice::UsbRequestQueue(usb_request_t* req, const usb_request_complete_t* complete_cb) {
req->header.device_id = device_id_;
+ if (req->reset) {
+ // Save client's callback in private storage.
+ UnownedRequest request(req, *complete_cb, parent_req_size_);
+ *request.private_storage() = {.ready_for_client = false,
+ .require_callback = !req->cb_on_error_only,
+ .silent_completions_count = 0};
+ RequestComplete(request.take());
+ return;
+ }
+ if (req->direct) {
+ hci_.RequestQueue(req, complete_cb);
+ return;
+ }
// Queue to HCI driver with our own completion callback so we can call client's completion
// on our own thread, to avoid drivers from deadlocking the HCI driver's interrupt thread.
diff --git a/zircon/system/dev/usb/xhci/trb-sizes.h b/zircon/system/dev/usb/xhci/trb-sizes.h
new file mode 100644
index 0000000..accff8bf
--- /dev/null
+++ b/zircon/system/dev/usb/xhci/trb-sizes.h
@@ -0,0 +1,9 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// choose ring sizes to allow each ring to fit in a single page
+#define COMMAND_RING_SIZE (PAGE_SIZE / sizeof(xhci_trb_t))
+#define TRANSFER_RING_SIZE ((PAGE_SIZE * 16) / sizeof(xhci_trb_t))
+#define EVENT_RING_SIZE ((PAGE_SIZE * 16) / sizeof(xhci_trb_t))
+#define ERST_ARRAY_SIZE (static_cast<uint32_t>((EVENT_RING_SIZE * sizeof(xhci_trb_t)) / PAGE_SIZE))
diff --git a/zircon/system/dev/usb/xhci/xdc-transfer.cpp b/zircon/system/dev/usb/xhci/xdc-transfer.cpp
index f833b31b..981975e 100644
--- a/zircon/system/dev/usb/xhci/xdc-transfer.cpp
+++ b/zircon/system/dev/usb/xhci/xdc-transfer.cpp
@@ -57,7 +57,7 @@
// If we get here, then we are ready to ring the doorbell.
// Save the ring position so we can update the ring dequeue ptr once the transfer completes.
xdc_req_internal_t* req_int = USB_REQ_TO_XDC_INTERNAL(req, sizeof(usb_request_t));
- req_int->context = ring->current;
+ req_int->context = ring->current_trb;
xdc_ring_doorbell(xdc, ep);
return ZX_OK;
@@ -174,7 +174,7 @@
// Convert all pending TRBs on the transfer ring into NO-OPs TRBs.
// ring->current is just after our last queued TRB.
xhci_trb_t* last_trb = nullptr;
- while (trb != ring->current) {
+ while (trb != ring->current_trb) {
xhci_set_transfer_noop_trb(trb);
last_trb = trb;
trb = xhci_get_next_trb(ring, trb);
diff --git a/zircon/system/dev/usb/xhci/xdc.cpp b/zircon/system/dev/usb/xhci/xdc.cpp
index b71dfd1..a79bf3b 100644
--- a/zircon/system/dev/usb/xhci/xdc.cpp
+++ b/zircon/system/dev/usb/xhci/xdc.cpp
@@ -2,20 +2,22 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
+#include <assert.h>
#include <ddk/debug.h>
#include <fuchsia/usb/debug/c/fidl.h>
-#include <xdc-server-utils/msg.h>
-#include <xdc-server-utils/stream.h>
-#include <zircon/hw/usb.h>
-#include <assert.h>
#include <string.h>
#include <threads.h>
#include <unistd.h>
+#include <xdc-server-utils/msg.h>
+#include <xdc-server-utils/stream.h>
+#include <zircon/hw/usb.h>
-#include "xdc.h"
+#include "trb-sizes.h"
#include "xdc-transfer.h"
+#include "xdc.h"
#include "xhci-hw.h"
#include "xhci-util.h"
+#include <fbl/alloc_checker.h>
#ifndef MIN
#define MIN(a, b) ((a) < (b) ? (a) : (b))
@@ -29,11 +31,9 @@
#define XDC_SERIAL_NUMBER u""
#define XDC_VENDOR_ID 0x18D1
#define XDC_PRODUCT_ID 0xA0DC
-#define XDC_REVISION 0x1000
+#define XDC_REVISION 0x1000
// Multi-segment event rings are not currently supported.
-#define ERST_ARRAY_SIZE 1
-#define EVENT_RING_SIZE (PAGE_SIZE / sizeof(xhci_trb_t))
// The maximum duration to transition from connected to configured state.
#define TRANSITION_CONFIGURED_THRESHOLD ZX_SEC(5)
@@ -183,10 +183,10 @@
if (status != ZX_OK) {
return status;
}
- zx_paddr_t tr_dequeue = xhci_transfer_ring_start_phys(&ep->transfer_ring);
+ zx_paddr_t tr_dequeue = ep->transfer_ring.buffers.front()->phys_list()[0];
- uint32_t max_burst = XHCI_GET_BITS32(&xdc->debug_cap_regs->dcctrl,
- DCCTRL_MAX_BURST_START, DCCTRL_MAX_BURST_BITS);
+ uint32_t max_burst = XHCI_GET_BITS32(&xdc->debug_cap_regs->dcctrl, DCCTRL_MAX_BURST_START,
+ DCCTRL_MAX_BURST_BITS);
int avg_trb_length = EP_CTX_MAX_PACKET_SIZE * (max_burst + 1);
@@ -690,7 +690,7 @@
while ((req = xdc_req_list_remove_tail(&xdc->free_read_reqs, usb_req_size)) != nullptr) {
usb_request_release(req);
}
- free(xdc);
+ delete xdc;
}
static zx_status_t xdc_suspend(void* ctx, uint32_t flags) {
@@ -1286,8 +1286,9 @@
}
zx_status_t xdc_bind(zx_device_t* parent, zx_handle_t bti_handle, void* mmio) {
- auto* xdc = static_cast<xdc_t*>(calloc(1, sizeof(xdc_t)));
- if (!xdc) {
+ fbl::AllocChecker ac;
+ auto* xdc = new (&ac) xdc_t();
+ if (!ac.check()) {
return ZX_ERR_NO_MEMORY;
}
xdc->bti_handle = bti_handle;
@@ -1332,7 +1333,7 @@
error_return:
zxlogf(ERROR, "xdc_bind failed: %d\n", status);
- xdc_free(xdc);
+ delete xdc;
return status;
}
diff --git a/zircon/system/dev/usb/xhci/xdc.h b/zircon/system/dev/usb/xhci/xdc.h
index 84d4099..b39c149 100644
--- a/zircon/system/dev/usb/xhci/xdc.h
+++ b/zircon/system/dev/usb/xhci/xdc.h
@@ -17,10 +17,10 @@
namespace usb_xhci {
-#define TRANSFER_RING_SIZE (PAGE_SIZE / sizeof(xhci_trb_t))
+#define TRANSFER_RING_SIZE ((PAGE_SIZE * 16) / sizeof(xhci_trb_t))
// The type and length fields for a string descriptor are one byte each.
-#define STR_DESC_METADATA_LEN 2
+#define STR_DESC_METADATA_LEN 2
#define MAX_STR_LEN 64
// There are only two endpoints, one for bulk OUT and one for bulk IN.
@@ -88,49 +88,49 @@
} xdc_poll_state_t;
typedef struct {
- zx_device_t* zxdev;
+ zx_device_t* zxdev = nullptr;
// Shared from XHCI.
- zx_handle_t bti_handle;
- void* mmio;
+ zx_handle_t bti_handle = ZX_HANDLE_INVALID;
+ void* mmio = nullptr;
- xdc_debug_cap_regs_t* debug_cap_regs;
+ xdc_debug_cap_regs_t* debug_cap_regs = nullptr;
// Underlying buffer for the event ring segment table
- io_buffer_t erst_buffer;
- erst_entry_t* erst_array;
+ io_buffer_t erst_buffer = {};
+ erst_entry_t* erst_array = nullptr;
xhci_event_ring_t event_ring;
// Underlying buffer for the context data and string descriptors.
- io_buffer_t context_str_descs_buffer;
- xdc_context_data_t* context_data;
- xdc_str_descs_t* str_descs;
+ io_buffer_t context_str_descs_buffer = {};
+ xdc_context_data_t* context_data = nullptr;
+ xdc_str_descs_t* str_descs = nullptr;
- thrd_t start_thread;
+ thrd_t start_thread = 0;
// Whether to suspend all activity.
std::atomic<bool> suspended;
xdc_endpoint_t eps[NUM_EPS];
// Whether the Debug Device is in the Configured state.
- bool configured;
+ bool configured = false;
// Needs to be acquired before accessing the eps and configured members.
// TODO(jocelyndang): make these separate locks?
mtx_t lock;
- bool writable;
- usb_request_pool_t free_write_reqs;
+ bool writable = false;
+ usb_request_pool_t free_write_reqs = {};
mtx_t write_lock;
- list_node_t free_read_reqs;
- xdc_packet_state_t cur_read_packet;
+ list_node_t free_read_reqs = {};
+ xdc_packet_state_t cur_read_packet = {};
mtx_t read_lock;
- list_node_t instance_list;
+ list_node_t instance_list = {};
// Streams registered by the host.
- list_node_t host_streams;
- mtx_t instance_list_lock;
+ list_node_t host_streams = {};
+ mtx_t instance_list_lock = {};
// At least one xdc instance has been opened.
sync_completion_t has_instance_completion;
diff --git a/zircon/system/dev/usb/xhci/xhci-device-manager.cpp b/zircon/system/dev/usb/xhci/xhci-device-manager.cpp
index 5cea6b7..6be14d33 100644
--- a/zircon/system/dev/usb/xhci/xhci-device-manager.cpp
+++ b/zircon/system/dev/usb/xhci/xhci-device-manager.cpp
@@ -93,10 +93,10 @@
zxlogf(ERROR, "xhci_address_device: failed to allocate io_buffer for slot\n");
return status;
}
-
- status = xhci_transfer_ring_init(&ep->transfer_ring, xhci->bti_handle.get(),
- TRANSFER_RING_SIZE);
- if (status < 0) return status;
+ status =
+ xhci_transfer_ring_init(&ep->transfer_ring, xhci->bti_handle.get(), TRANSFER_RING_SIZE);
+ if (status < 0)
+ return status;
fbl::AllocChecker ac;
ep->transfer_state = new (&ac) xhci_transfer_state_t;
@@ -791,7 +791,7 @@
return status;
}
- zx_paddr_t tr_dequeue = xhci_transfer_ring_start_phys(&slot->eps[index].transfer_ring);
+ zx_paddr_t tr_dequeue = slot->eps[index].transfer_ring.buffers.front()->phys_list()[0];
XHCI_SET_BITS32(&epc->epc0, EP_CTX_INTERVAL_START, EP_CTX_INTERVAL_BITS,
compute_interval(ep_desc, speed));
diff --git a/zircon/system/dev/usb/xhci/xhci-transfer-common.cpp b/zircon/system/dev/usb/xhci/xhci-transfer-common.cpp
index 3708e67..0d7ed65 100644
--- a/zircon/system/dev/usb/xhci/xhci-transfer-common.cpp
+++ b/zircon/system/dev/usb/xhci/xhci-transfer-common.cpp
@@ -12,10 +12,14 @@
void xhci_print_trb(xhci_transfer_ring_t* ring, xhci_trb_t* trb) {
size_t index = trb - ring->start;
- uint32_t* ptr = (uint32_t *)trb;
- uint64_t paddr = ring->buffer.phys() + index * sizeof(xhci_trb_t);
+ uint32_t* ptr = (uint32_t*)trb;
+ auto physmap = ring->virt_to_phys_map.get(VirtualAddress(reinterpret_cast<size_t>(trb)));
- zxlogf(LSPEW, "trb[%03zu] %p: %08X %08X %08X %08X\n", index, (void *)paddr, ptr[0], ptr[1], ptr[2], ptr[3]);
+ uint64_t paddr =
+ physmap->second + (reinterpret_cast<size_t>(ring->current_trb) - physmap->first.virt_start);
+
+ zxlogf(LSPEW, "trb[%03zu] %p: %08X %08X %08X %08X\n", index, (void*)paddr, ptr[0], ptr[1],
+ ptr[2], ptr[3]);
}
void xhci_transfer_state_init(xhci_transfer_state_t* state, usb_request_t* req,
@@ -59,9 +63,10 @@
zx_paddr_t paddr;
size_t transfer_size = 0;
bool first_packet = (state->phys_iter.total_iterated == 0);
- while (free_trbs > 0 && (((transfer_size = usb_request_phys_iter_next(&state->phys_iter, &paddr)) > 0) ||
- state->needs_transfer_trb || state->needs_zlp)) {
- xhci_trb_t* trb = ring->current;
+ while (free_trbs > 0 &&
+ (((transfer_size = usb_request_phys_iter_next(&state->phys_iter, &paddr)) > 0) ||
+ state->needs_transfer_trb || state->needs_zlp)) {
+ xhci_trb_t* trb = ring->current_trb;
xhci_clear_trb(trb);
XHCI_WRITE64(&trb->ptr, paddr);
XHCI_SET_BITS32(&trb->status, XFER_TRB_XFER_LENGTH_START, XFER_TRB_XFER_LENGTH_BITS,
@@ -121,7 +126,7 @@
}
// Queue event data TRB
- xhci_trb_t* trb = ring->current;
+ xhci_trb_t* trb = ring->current_trb;
xhci_clear_trb(trb);
trb->ptr = reinterpret_cast<uint64_t>(req);
XHCI_SET_BITS32(&trb->status, XFER_TRB_INTR_TARGET_START, XFER_TRB_INTR_TARGET_BITS,
diff --git a/zircon/system/dev/usb/xhci/xhci-transfer.cpp b/zircon/system/dev/usb/xhci/xhci-transfer.cpp
index b7eb5f3..06ef856 100644
--- a/zircon/system/dev/usb/xhci/xhci-transfer.cpp
+++ b/zircon/system/dev/usb/xhci/xhci-transfer.cpp
@@ -13,6 +13,7 @@
#include <threads.h>
#include "xhci-transfer.h"
+#include "xhci-trb.h"
#include "xhci-util.h"
namespace usb_xhci {
@@ -45,7 +46,7 @@
zxlogf(ERROR, "TRB_CMD_SET_TR_DEQUEUE failed cc: %d\n", cc);
return ZX_ERR_INTERNAL;
}
- xhci_set_dequeue_ptr(transfer_ring, transfer_ring->current);
+ xhci_set_dequeue_ptr(transfer_ring, transfer_ring->current_trb);
return ZX_OK;
}
@@ -207,7 +208,7 @@
usb_setup_t* setup = (req->header.ep_address == 0 ? &req->setup : nullptr);
if (setup) {
// Setup Stage
- xhci_trb_t* trb = ring->current;
+ xhci_trb_t* trb = ring->current_trb;
xhci_clear_trb(trb);
XHCI_SET_BITS32(&trb->ptr_low, SETUP_TRB_REQ_TYPE_START, SETUP_TRB_REQ_TYPE_BITS,
@@ -293,7 +294,7 @@
}
// Status Stage
- xhci_trb_t* trb = ring->current;
+ xhci_trb_t* trb = ring->current_trb;
xhci_clear_trb(trb);
XHCI_SET_BITS32(&trb->status, XFER_TRB_INTR_TARGET_START, XFER_TRB_INTR_TARGET_BITS,
interrupter_target);
@@ -312,7 +313,7 @@
xhci_usb_request_internal_t* req_int = USB_REQ_TO_XHCI_INTERNAL(req);
// if we get here, then we are ready to ring the doorbell
// update dequeue_ptr to TRB following this transaction
- req_int->context = ring->current;
+ req_int->context = ring->current_trb;
XHCI_WRITE32(&xhci->doorbells[header->device_id], ep_index + 1);
// it seems we need to ring the doorbell a second time when transitioning from STOPPED
@@ -643,99 +644,98 @@
list_node_t completed_reqs = LIST_INITIAL_VALUE(completed_reqs);
{
fbl::AutoLock al(&ep->lock);
-
zx_status_t result;
switch (cc) {
- case TRB_CC_SUCCESS:
- case TRB_CC_SHORT_PACKET:
- result = length;
- break;
- case TRB_CC_BABBLE_DETECTED_ERROR:
- zxlogf(TRACE, "xhci_handle_transfer_event: TRB_CC_BABBLE_DETECTED_ERROR\n");
- result = ZX_ERR_IO_OVERRUN;
- break;
- case TRB_CC_TRB_ERROR:
- zxlogf(TRACE, "xhci_handle_transfer_event: TRB_CC_TRB_ERROR\n");
- int ep_ctx_state;
- ep_ctx_state = xhci_get_ep_ctx_state(slot, ep);
- /*
- * For usb-c ethernet adapters on Intel xhci controller, we receive this error
- * when a packet fails with NRDY token on the bus.see NET:97 for more details.
- * Slow down the requests in the client when this error is received.
- */
- if (ep_ctx_state == EP_CTX_STATE_ERROR) {
- result = ZX_ERR_IO_INVALID;
- } else {
- result = ZX_ERR_IO;
- }
- break;
- case TRB_CC_USB_TRANSACTION_ERROR:
- case TRB_CC_STALL_ERROR: {
- int ep_ctx_state = xhci_get_ep_ctx_state(slot, ep);
- zxlogf(TRACE, "xhci_handle_transfer_event: cc %d ep_ctx_state %d\n", cc,
- ep_ctx_state);
- if (ep_ctx_state == EP_CTX_STATE_HALTED) {
- result = ZX_ERR_IO_REFUSED;
- } else {
- result = ZX_ERR_IO;
- }
- break;
+ case TRB_CC_SUCCESS:
+ case TRB_CC_SHORT_PACKET:
+ result = length;
+ break;
+ case TRB_CC_BABBLE_DETECTED_ERROR:
+ zxlogf(TRACE, "xhci_handle_transfer_event: TRB_CC_BABBLE_DETECTED_ERROR\n");
+ result = ZX_ERR_IO_OVERRUN;
+ break;
+ case TRB_CC_TRB_ERROR:
+ zxlogf(TRACE, "xhci_handle_transfer_event: TRB_CC_TRB_ERROR\n");
+ int ep_ctx_state;
+ ep_ctx_state = xhci_get_ep_ctx_state(slot, ep);
+ /*
+ * For usb-c ethernet adapters on Intel xhci controller, we receive this error
+ * when a packet fails with NRDY token on the bus.see NET:97 for more details.
+ * Slow down the requests in the client when this error is received.
+ */
+ if (ep_ctx_state == EP_CTX_STATE_ERROR) {
+ result = ZX_ERR_IO_INVALID;
+ } else {
+ result = ZX_ERR_IO;
}
- case TRB_CC_RING_UNDERRUN:
- // non-fatal error that happens when no transfers are available for isochronous
- // endpoint
- zxlogf(TRACE, "xhci_handle_transfer_event: TRB_CC_RING_UNDERRUN\n");
- return;
- case TRB_CC_RING_OVERRUN:
- // non-fatal error that happens when no transfers are available for isochronous
- // endpoint
- zxlogf(TRACE, "xhci_handle_transfer_event: TRB_CC_RING_OVERRUN\n");
- return;
- case TRB_CC_MISSED_SERVICE_ERROR:
- zxlogf(TRACE, "xhci_handle_transfer_event: TRB_CC_MISSED_SERVICE_ERROR\n");
- result = ZX_ERR_IO_MISSED_DEADLINE;
- break;
- case TRB_CC_STOPPED:
- case TRB_CC_STOPPED_LENGTH_INVALID:
- case TRB_CC_STOPPED_SHORT_PACKET:
- case TRB_CC_ENDPOINT_NOT_ENABLED_ERROR:
- switch (ep->state) {
- case EP_STATE_PAUSED:
- result = ZX_ERR_CANCELED;
- break;
- case EP_STATE_DISABLED:
- result = ZX_ERR_BAD_STATE;
- break;
- case EP_STATE_DEAD:
- result = ZX_ERR_IO_NOT_PRESENT;
- break;
- default:
- zxlogf(ERROR, "xhci_handle_transfer_event: bad state for stopped req: %d\n",
- ep->state);
- result = ZX_ERR_INTERNAL;
- }
- break;
- default: {
- int ep_ctx_state = xhci_get_ep_ctx_state(slot, ep);
- zxlogf(ERROR, "xhci_handle_transfer_event: unhandled transfer event condition"
- " code %d ep_ctx_state %d: %08X %08X %08X %08X\n", cc, ep_ctx_state,
- ((uint32_t*)trb)[0], ((uint32_t*)trb)[1], ((uint32_t*)trb)[2],
- ((uint32_t*)trb)[3]);
- if (ep_ctx_state == EP_CTX_STATE_HALTED) {
- result = ZX_ERR_IO_REFUSED;
- } else if (ep_ctx_state == EP_CTX_STATE_ERROR) {
- result = ZX_ERR_IO_INVALID;
- } else {
- result = ZX_ERR_IO;
- }
- break;
+ break;
+ case TRB_CC_USB_TRANSACTION_ERROR:
+ case TRB_CC_STALL_ERROR: {
+ int ep_ctx_state = xhci_get_ep_ctx_state(slot, ep);
+ zxlogf(TRACE, "xhci_handle_transfer_event: cc %d ep_ctx_state %d\n", cc, ep_ctx_state);
+ if (ep_ctx_state == EP_CTX_STATE_HALTED) {
+ result = ZX_ERR_IO_REFUSED;
+ } else {
+ result = ZX_ERR_IO;
}
+ break;
+ }
+ case TRB_CC_RING_UNDERRUN:
+ // non-fatal error that happens when no transfers are available for isochronous
+ // endpoint
+ zxlogf(TRACE, "xhci_handle_transfer_event: TRB_CC_RING_UNDERRUN\n");
+ return;
+ case TRB_CC_RING_OVERRUN:
+ // non-fatal error that happens when no transfers are available for isochronous
+ // endpoint
+ zxlogf(TRACE, "xhci_handle_transfer_event: TRB_CC_RING_OVERRUN\n");
+ return;
+ case TRB_CC_MISSED_SERVICE_ERROR:
+ zxlogf(TRACE, "xhci_handle_transfer_event: TRB_CC_MISSED_SERVICE_ERROR\n");
+ result = ZX_ERR_IO_MISSED_DEADLINE;
+ break;
+ case TRB_CC_STOPPED:
+ case TRB_CC_STOPPED_LENGTH_INVALID:
+ case TRB_CC_STOPPED_SHORT_PACKET:
+ case TRB_CC_ENDPOINT_NOT_ENABLED_ERROR:
+ switch (ep->state) {
+ case EP_STATE_PAUSED:
+ result = ZX_ERR_CANCELED;
+ break;
+ case EP_STATE_DISABLED:
+ result = ZX_ERR_BAD_STATE;
+ break;
+ case EP_STATE_DEAD:
+ result = ZX_ERR_IO_NOT_PRESENT;
+ break;
+ default:
+ zxlogf(ERROR, "xhci_handle_transfer_event: bad state for stopped req: %d\n",
+ ep->state);
+ result = ZX_ERR_INTERNAL;
+ }
+ break;
+ default: {
+ int ep_ctx_state = xhci_get_ep_ctx_state(slot, ep);
+ zxlogf(ERROR,
+ "xhci_handle_transfer_event: unhandled transfer event condition"
+ " code %d ep_ctx_state %d: %08X %08X %08X %08X\n",
+ cc, ep_ctx_state, ((uint32_t*)trb)[0], ((uint32_t*)trb)[1], ((uint32_t*)trb)[2],
+ ((uint32_t*)trb)[3]);
+ if (ep_ctx_state == EP_CTX_STATE_HALTED) {
+ result = ZX_ERR_IO_REFUSED;
+ } else if (ep_ctx_state == EP_CTX_STATE_ERROR) {
+ result = ZX_ERR_IO_INVALID;
+ } else {
+ result = ZX_ERR_IO;
+ }
+ break;
+ }
}
bool req_status_set = false;
- if (trb->ptr && !list_is_empty(&ep->pending_reqs)
- && ep->state != EP_STATE_DISABLED && ep->state != EP_STATE_DEAD) {
+ if (trb->ptr && !list_is_empty(&ep->pending_reqs) && ep->state != EP_STATE_DISABLED &&
+ ep->state != EP_STATE_DEAD) {
if (control & EVT_TRB_ED) {
req = reinterpret_cast<usb_request_t*>(trb->ptr);
if (ep_index == 0) {
@@ -808,7 +808,7 @@
}
if (!found_req) {
zxlogf(TRACE, "xhci_handle_transfer_event: ignoring transfer event for completed "
- "transfer\n");
+ "transfer\n");
return;
}
diff --git a/zircon/system/dev/usb/xhci/xhci-trb.cpp b/zircon/system/dev/usb/xhci/xhci-trb.cpp
index e9db845..8c53050 100644
--- a/zircon/system/dev/usb/xhci/xhci-trb.cpp
+++ b/zircon/system/dev/usb/xhci/xhci-trb.cpp
@@ -3,38 +3,66 @@
// found in the LICENSE file.
#include <assert.h>
+#include <fbl/auto_lock.h>
#include <hw/arch_ops.h>
+#include <bits/limits.h>
#include "xhci.h"
namespace usb_xhci {
zx_status_t xhci_transfer_ring_init(xhci_transfer_ring_t* ring, zx_handle_t bti_handle, int count) {
- zx_status_t status = ring->buffer.Init(bti_handle, count * sizeof(xhci_trb_t),
- IO_BUFFER_RW | IO_BUFFER_CONTIG |
- XHCI_IO_BUFFER_UNCACHED);
- if (status != ZX_OK) return status;
-
- ring->start = static_cast<xhci_trb_t*>(ring->buffer.virt());
- ring->current = ring->start;
+ fbl::AllocChecker ac;
+ ddk::IoBuffer buffer;
+ zx_status_t status =
+ buffer.Init(bti_handle, count * sizeof(xhci_trb_t), IO_BUFFER_RW | XHCI_IO_BUFFER_UNCACHED);
+ if (status != ZX_OK)
+ return status;
+ status = buffer.PhysMap();
+ if (status != ZX_OK)
+ return status;
+ std::unique_ptr<IoBufferContainer> container(new (&ac) IoBufferContainer(std::move(buffer)));
+ if (!ac.check()) {
+ return ZX_ERR_NO_MEMORY;
+ }
+ size_t phys_count = (*container)->phys_count();
+ const zx_paddr_t* sg_list = (*container)->phys_list();
+ ring->start = static_cast<xhci_trb_t*>((*container)->virt());
+ ring->current_trb = ring->start;
ring->dequeue_ptr = ring->start;
ring->full = false;
- ring->size = count - 1; // subtract 1 for LINK TRB at the end
+ ring->size = count - phys_count; // subtract 1 for LINK TRB at the end
ring->pcs = TRB_C;
-
// set link TRB at end to point back to the beginning
- ring->start[count - 1].ptr = ring->buffer.phys();
+ ring->start[count - 1].ptr = sg_list[0];
trb_set_control(&ring->start[count - 1], TRB_LINK, TRB_TC);
+ for (size_t i = 0; i < phys_count; i++) {
+ if (i + 1 < phys_count) {
+ xhci_trb_t* trb =
+ reinterpret_cast<xhci_trb_t*>(reinterpret_cast<size_t>(ring->start) +
+ (i * PAGE_SIZE) + (PAGE_SIZE - sizeof(xhci_trb_t)));
+ XHCI_WRITE64(&trb->ptr, sg_list[i + 1]);
+ XHCI_WRITE32(&trb->control, TRB_LINK << TRB_TYPE_START);
+ }
+ VirtualAddress address_mapping(reinterpret_cast<size_t>((*container)->virt()) +
+ (PAGE_SIZE * i));
+ address_mapping.phys_start = sg_list[i];
+ ring->virt_to_phys_map[address_mapping] = address_mapping.phys_start;
+ ring->phys_to_virt_map[address_mapping.phys_start / PAGE_SIZE] = address_mapping.virt_start;
+ }
+ ring->buffers.push_back(std::move(container));
return ZX_OK;
}
void xhci_transfer_ring_free(xhci_transfer_ring_t* ring) {
- ring->buffer.release();
+ ring->buffers.clear();
+ ring->virt_to_phys_map.clear();
+ ring->phys_to_virt_map.clear();
}
// return the number of free TRBs in the ring
size_t xhci_transfer_ring_free_trbs(xhci_transfer_ring_t* ring) {
- xhci_trb_t* current = ring->current;
+ xhci_trb_t* current = ring->current_trb;
xhci_trb_t* dequeue_ptr = ring->dequeue_ptr;
if (ring->full) {
@@ -55,23 +83,41 @@
zx_status_t xhci_event_ring_init(xhci_event_ring_t* ring, zx_handle_t bti_handle,
erst_entry_t* erst_array, int count) {
// allocate a read-only buffer for TRBs
- zx_status_t status = ring->buffer.Init(bti_handle, count * sizeof(xhci_trb_t),
- IO_BUFFER_RO | IO_BUFFER_CONTIG |
- XHCI_IO_BUFFER_UNCACHED);
- if (status != ZX_OK) return status;
-
- ring->start = static_cast<xhci_trb_t*>(ring->buffer.virt());
- XHCI_WRITE64(&erst_array[0].ptr, ring->buffer.phys());
- XHCI_WRITE32(&erst_array[0].size, count);
-
+ fbl::AllocChecker ac;
+ ddk::IoBuffer buffer;
+ zx_status_t status =
+ buffer.Init(bti_handle, count * sizeof(xhci_trb_t), IO_BUFFER_RW | XHCI_IO_BUFFER_UNCACHED);
+ if (status != ZX_OK)
+ return status;
+ status = buffer.PhysMap();
+ if (status != ZX_OK)
+ return status;
+ size_t phys_count = buffer.phys_count();
+ const zx_paddr_t* sg_list = buffer.phys_list();
+ std::unique_ptr<IoBufferContainer> container(new (&ac) IoBufferContainer(std::move(buffer)));
+ if (!ac.check()) {
+ return ZX_ERR_NO_MEMORY;
+ }
+ ring->start = static_cast<xhci_trb_t*>((*container)->virt());
ring->current = ring->start;
ring->end = ring->start + count;
ring->ccs = TRB_C;
+ for (size_t i = 0; i < phys_count; i++) {
+ VirtualAddress address_mapping(reinterpret_cast<size_t>((*container)->virt()) +
+ (PAGE_SIZE * i));
+ address_mapping.phys_start = sg_list[i];
+ ring->virt_to_phys_map[address_mapping] = address_mapping.phys_start;
+ ring->phys_to_virt_map[address_mapping.phys_start / PAGE_SIZE] = address_mapping.virt_start;
+ XHCI_WRITE64(&erst_array[i].ptr, sg_list[i]);
+ XHCI_WRITE32(&erst_array[i].size, PAGE_SIZE / sizeof(xhci_trb_t));
+ }
+ ring->buffers.push_back(std::move(container));
return ZX_OK;
}
void xhci_event_ring_free(xhci_event_ring_t* ring) {
- ring->buffer.release();
+ ring->buffers.clear();
+ ring->virt_to_phys_map.clear();
}
void xhci_clear_trb(xhci_trb_t* trb) {
@@ -95,10 +141,12 @@
xhci_trb_t* xhci_read_trb_ptr(xhci_transfer_ring_t* ring, xhci_trb_t* trb) {
// convert physical address to virtual
uintptr_t ptr = trb->ptr;
- ptr += (reinterpret_cast<uintptr_t>(ring->buffer.virt()) - ring->buffer.phys());
- return reinterpret_cast<xhci_trb_t*>(ptr);
+ return reinterpret_cast<xhci_trb_t*>(ring->phys_to_virt_map[ptr / PAGE_SIZE] +
+ (ptr % PAGE_SIZE));
}
-
+xhci_trb_t* xhci_next_evt(xhci_event_ring_t* ring, xhci_trb_t* trb) {
+ return trb + 1;
+}
xhci_trb_t* xhci_get_next_trb(xhci_transfer_ring_t* ring, xhci_trb_t* trb) {
trb++;
uint32_t control = XHCI_READ32(&trb->control);
@@ -109,14 +157,13 @@
}
void xhci_increment_ring(xhci_transfer_ring_t* ring) {
- xhci_trb_t* trb = ring->current;
+ xhci_trb_t* trb = ring->current_trb;
uint32_t control = XHCI_READ32(&trb->control);
uint32_t chain = control & TRB_CHAIN;
if (ring->pcs) {
XHCI_WRITE32(&trb->control, control | ring->pcs);
}
- trb = ++ring->current;
-
+ trb = ++ring->current_trb;
// check for LINK TRB
control = XHCI_READ32(&trb->control);
if ((control & TRB_TYPE_MASK) == (TRB_LINK << TRB_TYPE_START)) {
@@ -127,10 +174,10 @@
if (control & TRB_TC) {
ring->pcs ^= TRB_C;
}
- ring->current = xhci_read_trb_ptr(ring, trb);
+ ring->current_trb = xhci_read_trb_ptr(ring, trb);
}
- if (ring->current == ring->dequeue_ptr) {
+ if (ring->current_trb == ring->dequeue_ptr) {
// We've just enqueued something, so if the pointers are equal,
// the ring must be full.
ring->full = true;
@@ -143,15 +190,8 @@
}
xhci_trb_t* xhci_transfer_ring_phys_to_trb(xhci_transfer_ring_t* ring, zx_paddr_t phys) {
- zx_paddr_t first_trb_phys = xhci_transfer_ring_start_phys(ring);
- // Get the physical address of the start of the last trb,
- // ring->size does not include the LINK TRB at the end of the ring.
- zx_paddr_t last_trb_phys = first_trb_phys + (ring->size * sizeof(xhci_trb_t));
-
- if (phys < first_trb_phys || phys > last_trb_phys) {
- return nullptr;
- }
- return ring->start + ((phys - first_trb_phys) / sizeof(xhci_trb_t));
+ return reinterpret_cast<xhci_trb_t*>(ring->phys_to_virt_map[phys / PAGE_SIZE] +
+ (phys % PAGE_SIZE));
}
} // namespace usb_xhci
diff --git a/zircon/system/dev/usb/xhci/xhci-trb.h b/zircon/system/dev/usb/xhci/xhci-trb.h
index e009067..f407837 100644
--- a/zircon/system/dev/usb/xhci/xhci-trb.h
+++ b/zircon/system/dev/usb/xhci/xhci-trb.h
@@ -4,18 +4,100 @@
#pragma once
-#include <ddk/io-buffer.h>
-#include <zircon/listnode.h>
-
+#include <bits/limits.h>
+#include "trb-sizes.h"
#include "xhci-hw.h"
+#include <ddk/io-buffer.h>
+#include <fbl/intrusive_double_list.h>
+#include <fbl/mutex.h>
+#include <memory>
+#include <zircon/listnode.h>
namespace usb_xhci {
+class IoBufferContainer;
+// IO Buffer Container to allow nesting an IoBuffer
+// in an intrusive doubly-linked list.
+class IoBufferContainer : public fbl::DoublyLinkedListable<std::unique_ptr<IoBufferContainer>> {
+public:
+ IoBufferContainer(ddk::IoBuffer buffer) : buffer_(std::move(buffer)) {}
+ const ddk::IoBuffer* operator->() const { return &buffer_; }
+ const ddk::IoBuffer& operator*() { return buffer_; }
+
+private:
+ ddk::IoBuffer buffer_;
+};
+
+// Represents a virtual memory address mapping.
+// Contains information about the virtual range,
+// and physical starting address for contiguous mappings.
+struct VirtualAddress {
+ VirtualAddress() {}
+ VirtualAddress(zx_vaddr_t virt_start, size_t virt_end)
+ : virt_start(virt_start), virt_end(virt_end) {}
+ VirtualAddress(size_t virt_start)
+ : virt_start(virt_start), virt_end((virt_start + PAGE_SIZE) - 1) {}
+ bool operator<(const VirtualAddress& other) const {
+ return (virt_start / PAGE_SIZE) < (other.virt_start / PAGE_SIZE);
+ }
+ bool operator==(const VirtualAddress& other) const {
+ return (virt_start / PAGE_SIZE) == (other.virt_start / PAGE_SIZE);
+ }
+ size_t GetKey() const { return virt_start / PAGE_SIZE; }
+ zx_vaddr_t virt_start = 0;
+ zx_vaddr_t virt_end = 0;
+ size_t phys_start = 0;
+};
+
+// Constant-size map implementation with O(n) lookup time,
+// and O(1) insertion time. Maintains a mapping of Keys to Values.
+template <typename Key, typename Value, size_t Count> class XhciMap {
+public:
+ XhciMap() {}
+ // Retrieves a given key, or creates one if it doesn't already exist
+ // Asserts if the number of keys exceeds the statically-allocated buffer
+ // size.
+ Value& operator[](const Key& key) {
+ for (size_t i = 0; i < len_; i++) {
+ if (data_[i].first == key) {
+ return data_[i].second;
+ }
+ }
+ assert(len_ < Count);
+ data_[len_].first = key;
+ return data_[len_++].second;
+ }
+ // Retrieves the std::pair<Key, Value> for a given Key.
+ // Returns nullptr if the key is not found in the map.
+ std::pair<Key, Value>* get(const Key& key) {
+ for (size_t i = 0; i < len_; i++) {
+ if (data_[i].first == key) {
+ return data_ + i;
+ }
+ }
+ return nullptr;
+ }
+ // Removes all entries from this map
+ void clear() {
+ for (size_t i = 0; i < len_; i++) {
+ data_[i] = {};
+ }
+ len_ = 0;
+ }
+
+private:
+ size_t len_ = 0;
+ std::pair<Key, Value> data_[Count];
+};
+
// used for both command ring and transfer rings
struct xhci_transfer_ring_t {
- ddk::IoBuffer buffer;
+ fbl::DoublyLinkedList<std::unique_ptr<IoBufferContainer>> buffers;
+ XhciMap<VirtualAddress, uint64_t, TRANSFER_RING_SIZE / sizeof(xhci_trb_t)> virt_to_phys_map;
+ // Map of physical page indices to virtual addresses
+ XhciMap<zx_paddr_t, zx_vaddr_t, TRANSFER_RING_SIZE / sizeof(xhci_trb_t)> phys_to_virt_map;
xhci_trb_t* start;
- xhci_trb_t* current; // next to be filled by producer
+ xhci_trb_t* current_trb; // next to be filled by producer
uint8_t pcs; // producer cycle status
xhci_trb_t* dequeue_ptr; // next to be processed by consumer
// (not used for command ring)
@@ -23,14 +105,25 @@
bool full; // true if there are no available TRBs,
// this is needed to differentiate between
// an empty and full ring state
+ fbl::Mutex xfer_lock;
+};
+
+struct EventMapping {
+ uint64_t phys = 0;
+ xhci_trb_t* next = nullptr;
+ EventMapping() {}
+ EventMapping(uint64_t phys) : phys(phys) {}
};
struct xhci_event_ring_t {
- ddk::IoBuffer buffer;
+ fbl::DoublyLinkedList<std::unique_ptr<IoBufferContainer>> buffers;
+ XhciMap<VirtualAddress, EventMapping, TRANSFER_RING_SIZE / sizeof(xhci_trb_t)> virt_to_phys_map;
+ XhciMap<zx_paddr_t, zx_vaddr_t, TRANSFER_RING_SIZE / sizeof(xhci_trb_t)> phys_to_virt_map;
xhci_trb_t* start;
xhci_trb_t* current;
xhci_trb_t* end;
uint8_t ccs; // consumer cycle status
+ fbl::Mutex xfer_lock;
};
zx_status_t xhci_transfer_ring_init(xhci_transfer_ring_t* tr, zx_handle_t bti_handle, int count);
@@ -43,27 +136,37 @@
// Converts a transfer trb into a NO-OP transfer TRB, does nothing if it is the LINK TRB.
void xhci_set_transfer_noop_trb(xhci_trb_t* trb);
xhci_trb_t* xhci_read_trb_ptr(xhci_transfer_ring_t* ring, xhci_trb_t* trb);
+xhci_trb_t* xhci_next_evt(xhci_event_ring_t* ring, xhci_trb_t* trb);
xhci_trb_t* xhci_get_next_trb(xhci_transfer_ring_t* ring, xhci_trb_t* trb);
void xhci_increment_ring(xhci_transfer_ring_t* ring);
void xhci_set_dequeue_ptr(xhci_transfer_ring_t* ring, xhci_trb_t* new_ptr);
-// Returns the TRB corresponding to the given physical address, or nullptr if the address is invalid.
+// Returns the TRB corresponding to the given physical address, or nullptr if the address is
+// invalid.
xhci_trb_t* xhci_transfer_ring_phys_to_trb(xhci_transfer_ring_t* ring, zx_paddr_t phys);
-static inline zx_paddr_t xhci_transfer_ring_start_phys(xhci_transfer_ring_t* ring) {
- return ring->buffer.phys();
-}
-
static inline zx_paddr_t xhci_transfer_ring_current_phys(xhci_transfer_ring_t* ring) {
- return ring->buffer.phys() + ((ring->current - ring->start) * sizeof(xhci_trb_t));
-}
-
-static inline zx_paddr_t xhci_event_ring_start_phys(xhci_event_ring_t* ring) {
- return ring->buffer.phys();
+ auto physmap =
+ ring->virt_to_phys_map.get(VirtualAddress(reinterpret_cast<size_t>(ring->current_trb)));
+ if (physmap == nullptr) {
+ abort();
+ }
+ return physmap->second +
+ (reinterpret_cast<size_t>(ring->current_trb) - physmap->first.virt_start);
}
static inline zx_paddr_t xhci_event_ring_current_phys(xhci_event_ring_t* ring) {
- return ring->buffer.phys() + ((ring->current - ring->start) * sizeof(xhci_trb_t));
+ auto physmap =
+ ring->virt_to_phys_map.get(VirtualAddress(reinterpret_cast<size_t>(ring->current)));
+ if (physmap == nullptr) {
+ abort();
+ }
+ return physmap->second.phys +
+ (reinterpret_cast<size_t>(ring->current) - physmap->first.virt_start);
}
+struct xhci_t;
+// Enlarges the XHCI rings. The caller must ensure exclusive ownership of the rings
+// before invoking this function. Refer to XHCI 4.9.2.3
+zx_status_t xhci_enlarge_ring(xhci_t* xhci, xhci_transfer_ring_t* transfer);
} // namespace usb_xhci
diff --git a/zircon/system/dev/usb/xhci/xhci.cpp b/zircon/system/dev/usb/xhci/xhci.cpp
index b3e859c..afaf33d 100644
--- a/zircon/system/dev/usb/xhci/xhci.cpp
+++ b/zircon/system/dev/usb/xhci/xhci.cpp
@@ -5,25 +5,28 @@
#include "xhci.h"
#include <ddk/debug.h>
-#include <hw/arch_ops.h>
#include <fbl/alloc_checker.h>
#include <fbl/auto_lock.h>
+#include <hw/arch_ops.h>
#include <hw/reg.h>
-#include <zircon/types.h>
-#include <zircon/syscalls.h>
-#include <zircon/process.h>
-#include <usb/usb-request.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <threads.h>
#include <unistd.h>
+#include <usb/usb-request.h>
+#include <zircon/process.h>
+#include <zircon/syscalls.h>
+#include <zircon/types.h>
+#include <bits/limits.h>
+#include <ddk/io-buffer.h>
#include "xhci-device-manager.h"
#include "xhci-root-hub.h"
#include "xhci-transfer.h"
#include "xhci-util.h"
+#include <zircon/errors.h>
namespace usb_xhci {
@@ -179,8 +182,15 @@
uint32_t scratch_pad_bufs = XHCI_GET_BITS32(hcsparams2, HCSPARAMS2_MAX_SBBUF_HI_START,
HCSPARAMS2_MAX_SBBUF_HI_BITS);
scratch_pad_bufs <<= HCSPARAMS2_MAX_SBBUF_LO_BITS;
- scratch_pad_bufs |= XHCI_GET_BITS32(hcsparams2, HCSPARAMS2_MAX_SBBUF_LO_START,
- HCSPARAMS2_MAX_SBBUF_LO_BITS);
+ scratch_pad_bufs |=
+ XHCI_GET_BITS32(hcsparams2, HCSPARAMS2_MAX_SBBUF_LO_START, HCSPARAMS2_MAX_SBBUF_LO_BITS);
+ uint32_t max_erst_count =
+ 1 << XHCI_GET_BITS32(hcsparams2, HCSPARAMS2_ERST_MAX_START, HCSPARAMS2_ERST_MAX_BITS);
+ if (max_erst_count * sizeof(erst_entry_t) > PAGE_SIZE) {
+ max_erst_count = PAGE_SIZE / sizeof(erst_entry_t);
+ // TODO(bbosak): Implement a mechanism to allocate many physically contiguous pages
+ // reliably.
+ }
xhci->page_size = XHCI_READ32(&xhci->op_regs->pagesize) << 12;
fbl::AllocChecker ac;
@@ -265,28 +275,21 @@
xhci->input_context = static_cast<uint8_t*>(xhci->input_context_buffer.virt());
xhci->input_context_phys = xhci->input_context_buffer.phys();
- // DCBAA can only be 256 * sizeof(uint64_t) = 2048 bytes, so we have room for ERST array after DCBAA
- zx_off_t erst_offset;
- erst_offset = 256 * sizeof(uint64_t);
-
- size_t array_bytes;
- array_bytes = ERST_ARRAY_SIZE * sizeof(erst_entry_t);
- // MSI only supports up to 32 interrupts, so the required ERST arrays will fit
- // within the page. Potentially more pages will need to be allocated for MSI-X.
+ // DCBAA can only be 256 * sizeof(uint64_t) = 2048 bytes, so we have room for ERST array after
+ // DCBAA MSI only supports up to 32 interrupts, so the required ERST arrays will fit within the
+ // page. Potentially more pages will need to be allocated for MSI-X. Max number of contiguous
+ // physical pages per interrupt: 8
for (uint32_t i = 0; i < xhci->num_interrupts; i++) {
- // Ran out of space in page.
- if (erst_offset + array_bytes > PAGE_SIZE) {
- zxlogf(ERROR, "only have space for %u ERST arrays, want %u\n", i,
- xhci->num_interrupts);
+ result = xhci->erst_buffers[i].Init(
+ xhci->bti_handle.get(), PAGE_ROUNDUP(max_erst_count * sizeof(erst_entry_t)),
+ IO_BUFFER_RW | IO_BUFFER_CONTIG | XHCI_IO_BUFFER_UNCACHED);
+ if (result != ZX_OK) {
+ zxlogf(ERROR, "io_buffer_init failed for xhci->erst_buffer\n");
goto fail;
}
- xhci->erst_arrays[i] = reinterpret_cast<erst_entry_t*>(
- reinterpret_cast<uintptr_t>(xhci->dcbaa) + erst_offset);
- xhci->erst_arrays_phys[i] = xhci->dcbaa_phys + erst_offset;
- // ERST arrays must be 64 byte aligned - see Table 54 in XHCI spec.
- // dcbaa_phys is already page (and hence 64 byte) aligned, so only
- // need to round the offset.
- erst_offset = ROUNDUP_TO(erst_offset + array_bytes, 64);
+ xhci->erst_sizes[i] = max_erst_count;
+ xhci->erst_arrays[i] = reinterpret_cast<erst_entry_t*>(xhci->erst_buffers[i].virt());
+ xhci->erst_arrays_phys[i] = xhci->erst_buffers[i].phys();
}
if (scratch_pad_bufs > 0) {
@@ -339,7 +342,6 @@
ep->current_req = nullptr;
}
}
-
// initialize virtual root hub devices
for (int i = 0; i < XHCI_RH_COUNT; i++) {
result = xhci_root_hub_init(xhci, i);
@@ -439,7 +441,7 @@
// setup operational registers
xhci_op_regs_t* op_regs = xhci->op_regs;
// initialize command ring
- uint64_t crcr = xhci_transfer_ring_start_phys(&xhci->command_ring);
+ uint64_t crcr = xhci->command_ring.buffers.front()->phys_list()[0];
if (xhci->command_ring.pcs) {
crcr |= CRCR_RCS;
}
@@ -533,7 +535,7 @@
return ZX_ERR_NO_RESOURCES;
}
- xhci_trb_t* trb = cr->current;
+ xhci_trb_t* trb = cr->current_trb;
auto index = trb - cr->start;
xhci->command_contexts[index] = context;
@@ -542,7 +544,7 @@
trb_set_control(trb, command, control_bits);
xhci_increment_ring(cr);
- context->next_trb = cr->current;
+ context->next_trb = cr->current_trb;
hw_mb();
XHCI_WRITE32(&xhci->doorbells[0], 0);
@@ -600,7 +602,6 @@
static void xhci_handle_events(xhci_t* xhci, int interrupter) {
xhci_event_ring_t* er = &xhci->event_rings[interrupter];
-
// process all TRBs with cycle bit matching our CCS
while ((XHCI_READ32(&er->current->control) & TRB_C) == er->ccs) {
uint32_t type = trb_get_type(er->current);
@@ -621,12 +622,17 @@
zxlogf(ERROR, "xhci_handle_events: unhandled event type %d\n", type);
break;
}
-
- er->current++;
- if (er->current == er->end) {
+ fbl::AutoLock l(&er->xfer_lock);
+ if (er->current + 1 == er->end) {
er->current = er->start;
er->ccs ^= TRB_C;
+ } else {
+ er->current = xhci_next_evt(er, er->current);
}
+ xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter];
+ // Update the dequeue pointer to allow other events to come in
+ uint64_t erdp = xhci_event_ring_current_phys(er);
+ XHCI_WRITE64(&intr_regs->erdp, erdp);
}
// update event ring dequeue pointer and clear event handler busy flag
@@ -637,7 +643,6 @@
// clear the interrupt pending flag
xhci_intr_regs_t* intr_regs = &xhci->runtime_regs->intr_regs[interrupter];
XHCI_WRITE32(&intr_regs->iman, IMAN_IE | IMAN_IP);
-
xhci_handle_events(xhci, interrupter);
}
diff --git a/zircon/system/dev/usb/xhci/xhci.h b/zircon/system/dev/usb/xhci/xhci.h
index 18c1154..4d25580 100644
--- a/zircon/system/dev/usb/xhci/xhci.h
+++ b/zircon/system/dev/usb/xhci/xhci.h
@@ -5,6 +5,7 @@
#pragma once
#include <ddk/device.h>
+#include <ddk/io-buffer.h>
#include <ddk/protocol/pci.h>
#include <ddk/protocol/usb/bus.h>
#include <ddk/protocol/usb/request.h>
@@ -24,6 +25,7 @@
#include <limits.h>
#include <threads.h>
+#include "trb-sizes.h"
#include "xhci-hw.h"
#include "xhci-root-hub.h"
#include "xhci-transfer-common.h"
@@ -31,12 +33,6 @@
namespace usb_xhci {
-// choose ring sizes to allow each ring to fit in a single page
-#define COMMAND_RING_SIZE (PAGE_SIZE / sizeof(xhci_trb_t))
-#define TRANSFER_RING_SIZE (PAGE_SIZE / sizeof(xhci_trb_t))
-#define EVENT_RING_SIZE (PAGE_SIZE / sizeof(xhci_trb_t))
-#define ERST_ARRAY_SIZE 1
-
#define XHCI_RH_USB_2 0 // index of USB 2.0 virtual root hub device
#define XHCI_RH_USB_3 1 // index of USB 2.0 virtual root hub device
#define XHCI_RH_COUNT 2 // number of virtual root hub devices
@@ -154,7 +150,7 @@
// Each interrupter has an event ring.
// Only indices up to num_interrupts will be populated.
- xhci_event_ring_t event_rings[INTERRUPTER_COUNT] = {};
+ xhci_event_ring_t event_rings[INTERRUPTER_COUNT];
erst_entry_t* erst_arrays[INTERRUPTER_COUNT] = {};
zx_paddr_t erst_arrays_phys[INTERRUPTER_COUNT] = {};
@@ -203,6 +199,8 @@
// VMO buffer for DCBAA and ERST array
ddk::IoBuffer dcbaa_erst_buffer;
+ ddk::IoBuffer erst_buffers[INTERRUPTER_COUNT];
+ size_t erst_sizes[INTERRUPTER_COUNT];
// VMO buffer for input context
ddk::IoBuffer input_context_buffer;
// VMO buffer for scratch pad pages
diff --git a/zircon/system/ulib/ddk/include/ddk/io-buffer.h b/zircon/system/ulib/ddk/include/ddk/io-buffer.h
index 9eeacf4..51f5287 100644
--- a/zircon/system/ulib/ddk/include/ddk/io-buffer.h
+++ b/zircon/system/ulib/ddk/include/ddk/io-buffer.h
@@ -122,14 +122,18 @@
class IoBuffer {
public:
IoBuffer() {}
-
- ~IoBuffer() {
- io_buffer_release(&io_buffer_);
+ IoBuffer(IoBuffer&& other) {
+ io_buffer_ = other.io_buffer_;
+ other.io_buffer_ = {};
}
-
- inline void release() {
- io_buffer_release(&io_buffer_);
+ IoBuffer& operator=(IoBuffer&& other) {
+ io_buffer_ = other.io_buffer_;
+ other.io_buffer_ = {};
+ return *this;
}
+ ~IoBuffer() { io_buffer_release(&io_buffer_); }
+
+ inline void release() { io_buffer_release(&io_buffer_); }
inline zx_status_t Init(zx_handle_t bti, size_t size, uint32_t flags) {
return io_buffer_init(&io_buffer_, bti, size, flags);
