| // Copyright 2017 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include <assert.h> |
| #include <lib/device-protocol/pci.h> |
| #include <lib/sync/completion.h> |
| #include <limits.h> |
| #include <stdio.h> |
| #include <stdlib.h> |
| #include <string.h> |
| #include <threads.h> |
| #include <zircon/device/block.h> |
| #include <zircon/listnode.h> |
| #include <zircon/syscalls.h> |
| #include <zircon/types.h> |
| |
| #include <ddk/binding.h> |
| #include <ddk/debug.h> |
| #include <ddk/device.h> |
| #include <ddk/driver.h> |
| #include <ddk/io-buffer.h> |
| #include <ddk/mmio-buffer.h> |
| #include <ddk/protocol/block.h> |
| #include <ddk/protocol/pci.h> |
| #include <hw/pci.h> |
| #include <hw/reg.h> |
| |
| #include "nvme-hw.h" |
| |
| #define TXN_FLAG_FAILED 1 |
| |
| typedef struct { |
| block_op_t op; |
| list_node_t node; |
| block_impl_queue_callback completion_cb; |
| void* cookie; |
| uint16_t pending_utxns; |
| uint8_t opcode; |
| uint8_t flags; |
| } nvme_txn_t; |
| |
| typedef struct { |
| zx_paddr_t phys; // io buffer phys base (1 page) |
| void* virt; // io buffer virt base |
| zx_handle_t pmt; // pinned memory |
| nvme_txn_t* txn; // related txn |
| uint16_t id; |
| uint16_t reserved0; |
| uint32_t reserved1; |
| } nvme_utxn_t; |
| |
| #define UTXN_COUNT 63 |
| |
| // There's no system constant for this. Ensure it matches reality. |
| #define PAGE_SHIFT (12ULL) |
| static_assert(PAGE_SIZE == (1ULL << PAGE_SHIFT), ""); |
| |
| #define PAGE_MASK (PAGE_SIZE - 1ULL) |
| |
| // Limit maximum transfer size to 1MB which fits comfortably |
| // within our single scatter gather page per utxn setup |
| #define MAX_XFER (1024 * 1024) |
| |
| // Maximum submission and completion queue item counts, for |
| // queues that are a single page in size. |
| #define SQMAX (PAGE_SIZE / sizeof(nvme_cmd_t)) |
| #define CQMAX (PAGE_SIZE / sizeof(nvme_cpl_t)) |
| |
| // global driver state bits |
| #define FLAG_IRQ_THREAD_STARTED 0x0001 |
| #define FLAG_IO_THREAD_STARTED 0x0002 |
| #define FLAG_SHUTDOWN 0x0004 |
| |
| #define FLAG_HAS_VWC 0x0100 |
| |
| typedef struct { |
| mmio_buffer_t mmio; |
| zx_handle_t irqh; |
| zx_handle_t bti; |
| uint32_t flags; |
| mtx_t lock; |
| |
| // io queue doorbell registers |
| void* io_sq_tail_db; |
| void* io_cq_head_db; |
| |
| nvme_cpl_t* io_cq; |
| nvme_cmd_t* io_sq; |
| uint32_t io_nsid; |
| uint16_t io_cq_head; |
| uint16_t io_cq_toggle; |
| uint16_t io_sq_tail; |
| uint16_t io_sq_head; |
| |
| uint64_t utxn_avail; // bitmask of available utxns |
| |
| // The pending list is txns that have been received |
| // via nvme_queue() and are waiting for io to start. |
| // The exception is the head of the pending list which may |
| // be partially started, waiting for more utxns to become |
| // available. |
| // The active list consists of txns where all utxns have |
| // been created and we're waiting for them to complete or |
| // error out. |
| list_node_t pending_txns; // inbound txns to process |
| list_node_t active_txns; // txns in flight |
| |
| // The io signal completion is signaled from nvme_queue() |
| // or from the irq thread, notifying the io thread that |
| // it has work to do. |
| sync_completion_t io_signal; |
| |
| uint32_t max_xfer; |
| block_info_t info; |
| |
| // admin queue doorbell registers |
| void* io_admin_sq_tail_db; |
| void* io_admin_cq_head_db; |
| |
| // admin queues and state |
| nvme_cpl_t* admin_cq; |
| nvme_cmd_t* admin_sq; |
| uint16_t admin_cq_head; |
| uint16_t admin_cq_toggle; |
| uint16_t admin_sq_tail; |
| uint16_t admin_sq_head; |
| |
| // context for admin transactions |
| // presently we serialize these under the admin_lock |
| mtx_t admin_lock; |
| sync_completion_t admin_signal; |
| nvme_cpl_t admin_result; |
| |
| pci_protocol_t pci; |
| zx_device_t* zxdev; |
| |
| size_t iosz; |
| |
| // source of physical pages for queues and admin commands |
| io_buffer_t iob; |
| |
| thrd_t irqthread; |
| thrd_t iothread; |
| |
| // pool of utxns |
| nvme_utxn_t utxn[UTXN_COUNT]; |
| } nvme_device_t; |
| |
| // We break IO transactions down into one or more "micro transactions" (utxn) |
| // based on the transfer limits of the controller, etc. Each utxn has an |
| // id associated with it, which is used as the command id for the command |
| // queued to the NVME device. This id is the same as its index into the |
| // pool of utxns and the bitmask of free txns, to simplify management. |
| // |
| // We maintain a pool of 63 of these, which is the number of commands |
| // that can be submitted to NVME via a single page submit queue. |
| // |
| // The utxns are not protected by locks. Instead, after initialization, |
| // they may only be touched by the io thread, which is responsible for |
| // queueing commands and dequeuing completion messages. |
| |
| static nvme_utxn_t* utxn_get(nvme_device_t* nvme) { |
| uint64_t n = __builtin_ffsll(nvme->utxn_avail); |
| if (n == 0) { |
| return NULL; |
| } |
| n--; |
| nvme->utxn_avail &= ~(1ULL << n); |
| return nvme->utxn + n; |
| } |
| |
| static void utxn_put(nvme_device_t* nvme, nvme_utxn_t* utxn) { |
| uint64_t n = utxn->id; |
| nvme->utxn_avail |= (1ULL << n); |
| } |
| |
| static zx_status_t nvme_admin_cq_get(nvme_device_t* nvme, nvme_cpl_t* cpl) { |
| if ((readw(&nvme->admin_cq[nvme->admin_cq_head].status) & 1) != nvme->admin_cq_toggle) { |
| return ZX_ERR_SHOULD_WAIT; |
| } |
| *cpl = nvme->admin_cq[nvme->admin_cq_head]; |
| |
| // advance the head pointer, wrapping and inverting toggle at max |
| uint16_t next = (nvme->admin_cq_head + 1) & (CQMAX - 1); |
| if ((nvme->admin_cq_head = next) == 0) { |
| nvme->admin_cq_toggle ^= 1; |
| } |
| |
| // note the new sq head reported by hw |
| nvme->admin_sq_head = cpl->sq_head; |
| |
| // ring the doorbell |
| writel(next, nvme->io_admin_cq_head_db); |
| return ZX_OK; |
| } |
| |
| static zx_status_t nvme_admin_sq_put(nvme_device_t* nvme, nvme_cmd_t* cmd) { |
| uint16_t next = (nvme->admin_sq_tail + 1) & (SQMAX - 1); |
| |
| // if head+1 == tail: queue is full |
| if (next == nvme->admin_sq_head) { |
| return ZX_ERR_SHOULD_WAIT; |
| } |
| |
| nvme->admin_sq[nvme->admin_sq_tail] = *cmd; |
| nvme->admin_sq_tail = next; |
| |
| // ring the doorbell |
| writel(next, nvme->io_admin_sq_tail_db); |
| return ZX_OK; |
| } |
| |
| static zx_status_t nvme_io_cq_get(nvme_device_t* nvme, nvme_cpl_t* cpl) { |
| if ((readw(&nvme->io_cq[nvme->io_cq_head].status) & 1) != nvme->io_cq_toggle) { |
| return ZX_ERR_SHOULD_WAIT; |
| } |
| *cpl = nvme->io_cq[nvme->io_cq_head]; |
| |
| // advance the head pointer, wrapping and inverting toggle at max |
| uint16_t next = (nvme->io_cq_head + 1) & (CQMAX - 1); |
| if ((nvme->io_cq_head = next) == 0) { |
| nvme->io_cq_toggle ^= 1; |
| } |
| |
| // note the new sq head reported by hw |
| nvme->io_sq_head = cpl->sq_head; |
| return ZX_OK; |
| } |
| |
| static void nvme_io_cq_ack(nvme_device_t* nvme) { |
| // ring the doorbell |
| writel(nvme->io_cq_head, nvme->io_cq_head_db); |
| } |
| |
| static zx_status_t nvme_io_sq_put(nvme_device_t* nvme, nvme_cmd_t* cmd) { |
| uint16_t next = (nvme->io_sq_tail + 1) & (SQMAX - 1); |
| |
| // if head+1 == tail: queue is full |
| if (next == nvme->io_sq_head) { |
| return ZX_ERR_SHOULD_WAIT; |
| } |
| |
| nvme->io_sq[nvme->io_sq_tail] = *cmd; |
| nvme->io_sq_tail = next; |
| |
| // ring the doorbell |
| writel(next, nvme->io_sq_tail_db); |
| return ZX_OK; |
| } |
| |
| static int irq_thread(void* arg) { |
| nvme_device_t* nvme = arg; |
| for (;;) { |
| zx_status_t r; |
| if ((r = zx_interrupt_wait(nvme->irqh, NULL)) != ZX_OK) { |
| zxlogf(ERROR, "nvme: irq wait failed: %d\n", r); |
| break; |
| } |
| |
| nvme_cpl_t cpl; |
| if (nvme_admin_cq_get(nvme, &cpl) == ZX_OK) { |
| nvme->admin_result = cpl; |
| sync_completion_signal(&nvme->admin_signal); |
| } |
| |
| sync_completion_signal(&nvme->io_signal); |
| } |
| return 0; |
| } |
| |
| static zx_status_t nvme_admin_txn(nvme_device_t* nvme, nvme_cmd_t* cmd, nvme_cpl_t* cpl) { |
| zx_status_t r; |
| mtx_lock(&nvme->admin_lock); |
| sync_completion_reset(&nvme->admin_signal); |
| if ((r = nvme_admin_sq_put(nvme, cmd)) != ZX_OK) { |
| goto done; |
| } |
| if ((r = sync_completion_wait(&nvme->admin_signal, ZX_SEC(1))) != ZX_OK) { |
| zxlogf(ERROR, "nvme: admin txn: timed out\n"); |
| goto done; |
| } |
| |
| unsigned code = NVME_CPL_STATUS_CODE(nvme->admin_result.status); |
| if (code != 0) { |
| zxlogf(ERROR, "nvme: admin txn: nvm error %03x\n", code); |
| r = ZX_ERR_IO; |
| } |
| if (cpl != NULL) { |
| *cpl = nvme->admin_result; |
| } |
| done: |
| mtx_unlock(&nvme->admin_lock); |
| return r; |
| } |
| |
| static inline void txn_complete(nvme_txn_t* txn, zx_status_t status) { |
| txn->completion_cb(txn->cookie, status, &txn->op); |
| } |
| |
| // Attempt to generate utxns and queue nvme commands for a txn |
| // Returns true if this could not be completed due to temporary |
| // lack of resources or false if either it succeeded or errored out. |
| static bool io_process_txn(nvme_device_t* nvme, nvme_txn_t* txn) { |
| zx_handle_t vmo = txn->op.rw.vmo; |
| nvme_utxn_t* utxn; |
| zx_paddr_t* pages; |
| zx_status_t r; |
| |
| for (;;) { |
| // If there are no available utxns, we can't proceed |
| // and we tell the caller to retain the txn (true) |
| if ((utxn = utxn_get(nvme)) == NULL) { |
| return true; |
| } |
| |
| uint32_t blocks = txn->op.rw.length; |
| if (blocks > nvme->max_xfer) { |
| blocks = nvme->max_xfer; |
| } |
| |
| // Total transfer size in bytes |
| size_t bytes = ((size_t)blocks) * ((size_t)nvme->info.block_size); |
| |
| // Page offset of first page of transfer |
| size_t pageoffset = txn->op.rw.offset_vmo & (~PAGE_MASK); |
| |
| // Byte offset into first page of transfer |
| size_t byteoffset = txn->op.rw.offset_vmo & PAGE_MASK; |
| |
| // Total pages mapped / touched |
| size_t pagecount = (byteoffset + bytes + PAGE_MASK) >> PAGE_SHIFT; |
| |
| // read disk (OP_READ) -> memory (PERM_WRITE) or |
| // write memory (PERM_READ) -> disk (OP_WRITE) |
| uint32_t opt = (txn->opcode == NVME_OP_READ) ? ZX_BTI_PERM_WRITE : ZX_BTI_PERM_READ; |
| |
| pages = utxn->virt; |
| |
| if ((r = zx_bti_pin(nvme->bti, opt, vmo, pageoffset, pagecount << PAGE_SHIFT, pages, pagecount, |
| &utxn->pmt)) != ZX_OK) { |
| zxlogf(ERROR, "nvme: could not pin pages: %d\n", r); |
| break; |
| } |
| |
| nvme_cmd_t cmd; |
| memset(&cmd, 0, sizeof(cmd)); |
| cmd.cmd = NVME_CMD_CID(utxn->id) | NVME_CMD_PRP | NVME_CMD_NORMAL | NVME_CMD_OPC(txn->opcode); |
| cmd.nsid = 1; |
| cmd.u.rw.start_lba = txn->op.rw.offset_dev; |
| cmd.u.rw.block_count = blocks - 1; |
| // The NVME command has room for two data pointers inline. |
| // The first is always the pointer to the first page where data is. |
| // The second is the second page if pagecount is 2. |
| // The second is the address of an array of page 2..n if pagecount > 2 |
| cmd.dptr.prp[0] = pages[0] | byteoffset; |
| if (pagecount == 2) { |
| cmd.dptr.prp[1] = pages[1]; |
| } else if (pagecount > 2) { |
| cmd.dptr.prp[1] = utxn->phys + sizeof(uint64_t); |
| } |
| |
| zxlogf(TRACE, "nvme: txn=%p utxn id=%u pages=%zu op=%s\n", txn, utxn->id, pagecount, |
| txn->opcode == NVME_OP_WRITE ? "WR" : "RD"); |
| zxlogf(SPEW, "nvme: prp[0]=%016zx prp[1]=%016zx\n", cmd.dptr.prp[0], cmd.dptr.prp[1]); |
| zxlogf(SPEW, "nvme: pages[] = { %016zx, %016zx, %016zx, %016zx, ... }\n", pages[0], pages[1], |
| pages[2], pages[3]); |
| |
| if ((r = nvme_io_sq_put(nvme, &cmd)) != ZX_OK) { |
| zxlogf(ERROR, "nvme: could not submit cmd (txn=%p id=%u)\n", txn, utxn->id); |
| break; |
| } |
| |
| utxn->txn = txn; |
| |
| // keep track of where we are |
| txn->op.rw.offset_dev += blocks; |
| txn->op.rw.offset_vmo += bytes; |
| txn->op.rw.length -= blocks; |
| txn->pending_utxns++; |
| |
| // If there's no more remaining, we're done, and we |
| // move this txn to the active list and tell the |
| // caller not to retain the txn (false) |
| if (txn->op.rw.length == 0) { |
| mtx_lock(&nvme->lock); |
| list_add_tail(&nvme->active_txns, &txn->node); |
| mtx_unlock(&nvme->lock); |
| return false; |
| } |
| } |
| |
| // failure |
| if ((r = zx_pmt_unpin(utxn->pmt)) != ZX_OK) { |
| zxlogf(ERROR, "nvme: cannot unpin io buffer: %d\n", r); |
| } |
| utxn_put(nvme, utxn); |
| |
| mtx_lock(&nvme->lock); |
| txn->flags |= TXN_FLAG_FAILED; |
| if (txn->pending_utxns) { |
| // if there are earlier uncompleted IOs we become active now |
| // and will finish erroring out when they complete |
| list_add_tail(&nvme->active_txns, &txn->node); |
| txn = NULL; |
| } |
| mtx_unlock(&nvme->lock); |
| |
| if (txn != NULL) { |
| txn_complete(txn, ZX_ERR_INTERNAL); |
| } |
| |
| // Either way we tell the caller not to retain the txn (false) |
| return false; |
| } |
| |
| static void io_process_txns(nvme_device_t* nvme) { |
| nvme_txn_t* txn; |
| |
| for (;;) { |
| mtx_lock(&nvme->lock); |
| txn = list_remove_head_type(&nvme->pending_txns, nvme_txn_t, node); |
| mtx_unlock(&nvme->lock); |
| |
| if (txn == NULL) { |
| return; |
| } |
| |
| if (io_process_txn(nvme, txn)) { |
| // put txn back at front of queue for further processing later |
| mtx_lock(&nvme->lock); |
| list_add_head(&nvme->pending_txns, &txn->node); |
| mtx_unlock(&nvme->lock); |
| return; |
| } |
| } |
| } |
| |
| static void io_process_cpls(nvme_device_t* nvme) { |
| bool ring_doorbell = false; |
| nvme_cpl_t cpl; |
| |
| while (nvme_io_cq_get(nvme, &cpl) == ZX_OK) { |
| ring_doorbell = true; |
| |
| if (cpl.cmd_id >= UTXN_COUNT) { |
| zxlogf(ERROR, "nvme: unexpected cmd id %u\n", cpl.cmd_id); |
| continue; |
| } |
| nvme_utxn_t* utxn = nvme->utxn + cpl.cmd_id; |
| nvme_txn_t* txn = utxn->txn; |
| |
| if (txn == NULL) { |
| zxlogf(ERROR, "nvme: inactive utxn #%u completed?!\n", cpl.cmd_id); |
| continue; |
| } |
| |
| uint32_t code = NVME_CPL_STATUS_CODE(cpl.status); |
| if (code != 0) { |
| zxlogf(ERROR, "nvme: utxn #%u txn %p failed: status=%03x\n", cpl.cmd_id, txn, code); |
| txn->flags |= TXN_FLAG_FAILED; |
| // discard any remaining bytes -- no reason to keep creating |
| // further utxns once one has failed |
| txn->op.rw.length = 0; |
| } else { |
| zxlogf(SPEW, "nvme: utxn #%u txn %p OKAY\n", cpl.cmd_id, txn); |
| } |
| |
| zx_status_t r; |
| if ((r = zx_pmt_unpin(utxn->pmt)) != ZX_OK) { |
| zxlogf(ERROR, "nvme: cannot unpin io buffer: %d\n", r); |
| } |
| |
| // release the microtransaction |
| utxn->txn = NULL; |
| utxn_put(nvme, utxn); |
| |
| txn->pending_utxns--; |
| if ((txn->pending_utxns == 0) && (txn->op.rw.length == 0)) { |
| // remove from either pending or active list |
| mtx_lock(&nvme->lock); |
| list_delete(&txn->node); |
| mtx_unlock(&nvme->lock); |
| zxlogf(TRACE, "nvme: txn %p %s\n", txn, txn->flags & TXN_FLAG_FAILED ? "error" : "okay"); |
| txn_complete(txn, txn->flags & TXN_FLAG_FAILED ? ZX_ERR_IO : ZX_OK); |
| } |
| } |
| |
| if (ring_doorbell) { |
| nvme_io_cq_ack(nvme); |
| } |
| } |
| |
| static int io_thread(void* arg) { |
| nvme_device_t* nvme = arg; |
| for (;;) { |
| if (sync_completion_wait(&nvme->io_signal, ZX_TIME_INFINITE)) { |
| break; |
| } |
| if (nvme->flags & FLAG_SHUTDOWN) { |
| // TODO: cancel out pending IO |
| zxlogf(INFO, "nvme: io thread exiting\n"); |
| break; |
| } |
| |
| sync_completion_reset(&nvme->io_signal); |
| |
| // process completion messages |
| io_process_cpls(nvme); |
| |
| // process work queue |
| io_process_txns(nvme); |
| } |
| return 0; |
| } |
| |
| static void nvme_queue(void* ctx, block_op_t* op, block_impl_queue_callback completion_cb, |
| void* cookie) { |
| nvme_device_t* nvme = ctx; |
| nvme_txn_t* txn = containerof(op, nvme_txn_t, op); |
| txn->completion_cb = completion_cb; |
| txn->cookie = cookie; |
| |
| switch (txn->op.command & BLOCK_OP_MASK) { |
| case BLOCK_OP_READ: |
| txn->opcode = NVME_OP_READ; |
| break; |
| case BLOCK_OP_WRITE: |
| txn->opcode = NVME_OP_WRITE; |
| break; |
| case BLOCK_OP_FLUSH: |
| // TODO |
| txn_complete(txn, ZX_OK); |
| return; |
| default: |
| txn_complete(txn, ZX_ERR_NOT_SUPPORTED); |
| return; |
| } |
| |
| if (txn->op.rw.length == 0) { |
| txn_complete(txn, ZX_ERR_INVALID_ARGS); |
| return; |
| } |
| // Transaction must fit within device |
| if ((txn->op.rw.offset_dev >= nvme->info.block_count) || |
| (nvme->info.block_count - txn->op.rw.offset_dev < txn->op.rw.length)) { |
| txn_complete(txn, ZX_ERR_OUT_OF_RANGE); |
| return; |
| } |
| |
| // convert vmo offset to a byte offset |
| txn->op.rw.offset_vmo *= nvme->info.block_size; |
| |
| txn->pending_utxns = 0; |
| txn->flags = 0; |
| |
| zxlogf(SPEW, "nvme: io: %s: %ublks @ blk#%zu\n", txn->opcode == NVME_OP_WRITE ? "wr" : "rd", |
| txn->op.rw.length + 1U, txn->op.rw.offset_dev); |
| |
| mtx_lock(&nvme->lock); |
| list_add_tail(&nvme->pending_txns, &txn->node); |
| mtx_unlock(&nvme->lock); |
| |
| sync_completion_signal(&nvme->io_signal); |
| } |
| |
| static void nvme_query(void* ctx, block_info_t* info_out, size_t* block_op_size_out) { |
| nvme_device_t* nvme = ctx; |
| *info_out = nvme->info; |
| *block_op_size_out = sizeof(nvme_txn_t); |
| } |
| |
| static zx_off_t nvme_get_size(void* ctx) { |
| nvme_device_t* nvme = ctx; |
| return nvme->info.block_count * nvme->info.block_size; |
| } |
| |
| static zx_status_t nvme_suspend(void* ctx, uint32_t flags) { return ZX_OK; } |
| |
| static zx_status_t nvme_resume(void* ctx, uint32_t flags) { return ZX_OK; } |
| |
| static void nvme_release(void* ctx) { |
| nvme_device_t* nvme = ctx; |
| int r; |
| |
| zxlogf(INFO, "nvme: release\n"); |
| nvme->flags |= FLAG_SHUTDOWN; |
| if (nvme->mmio.vmo != ZX_HANDLE_INVALID) { |
| pci_enable_bus_master(&nvme->pci, false); |
| zx_handle_close(nvme->bti); |
| mmio_buffer_release(&nvme->mmio); |
| // TODO: risks a handle use-after-close, will be resolved by IRQ api |
| // changes coming soon |
| zx_handle_close(nvme->irqh); |
| } |
| if (nvme->flags & FLAG_IRQ_THREAD_STARTED) { |
| thrd_join(nvme->irqthread, &r); |
| } |
| if (nvme->flags & FLAG_IO_THREAD_STARTED) { |
| sync_completion_signal(&nvme->io_signal); |
| thrd_join(nvme->iothread, &r); |
| } |
| |
| // error out any pending txns |
| mtx_lock(&nvme->lock); |
| nvme_txn_t* txn; |
| while ((txn = list_remove_head_type(&nvme->active_txns, nvme_txn_t, node)) != NULL) { |
| txn_complete(txn, ZX_ERR_PEER_CLOSED); |
| } |
| while ((txn = list_remove_head_type(&nvme->pending_txns, nvme_txn_t, node)) != NULL) { |
| txn_complete(txn, ZX_ERR_PEER_CLOSED); |
| } |
| mtx_unlock(&nvme->lock); |
| |
| io_buffer_release(&nvme->iob); |
| free(nvme); |
| } |
| |
| static zx_protocol_device_t device_ops = { |
| .version = DEVICE_OPS_VERSION, |
| |
| .get_size = nvme_get_size, |
| |
| .suspend = nvme_suspend, |
| .resume = nvme_resume, |
| .release = nvme_release, |
| }; |
| |
| static void infostring(const char* prefix, uint8_t* str, size_t len) { |
| char tmp[len + 1]; |
| size_t i; |
| for (i = 0; i < len; i++) { |
| uint8_t c = str[i]; |
| if (c == 0) { |
| break; |
| } |
| if ((c < ' ') || (c > 127)) { |
| c = ' '; |
| } |
| tmp[i] = c; |
| } |
| tmp[i] = 0; |
| while (i > 0) { |
| i--; |
| if (tmp[i] == ' ') { |
| tmp[i] = 0; |
| } else { |
| break; |
| } |
| } |
| zxlogf(INFO, "nvme: %s'%s'\n", prefix, tmp); |
| } |
| |
| // Convenience accessors for BAR0 registers |
| #define rd32(r) readl(nvme->mmio.vaddr + NVME_REG_##r) |
| #define rd64(r) readll(nvme->mmio.vaddr + NVME_REG_##r) |
| #define wr32(v, r) writel(v, nvme->mmio.vaddr + NVME_REG_##r) |
| #define wr64(v, r) writell(v, nvme->mmio.vaddr + NVME_REG_##r) |
| |
| // dedicated pages from the page pool |
| #define IDX_ADMIN_SQ 0 |
| #define IDX_ADMIN_CQ 1 |
| #define IDX_IO_SQ 2 |
| #define IDX_IO_CQ 3 |
| #define IDX_SCRATCH 4 |
| #define IDX_UTXN_POOL 5 // this must always be last |
| |
| #define IO_PAGE_COUNT (IDX_UTXN_POOL + UTXN_COUNT) |
| |
| static inline uint64_t U64(uint8_t* x) { return *((uint64_t*)(void*)x); } |
| static inline uint32_t U32(uint8_t* x) { return *((uint32_t*)(void*)x); } |
| static inline uint32_t U16(uint8_t* x) { return *((uint16_t*)(void*)x); } |
| |
| #define WAIT_MS 5000 |
| |
| static zx_status_t nvme_init(nvme_device_t* nvme) { |
| uint32_t n = rd32(VS); |
| uint64_t cap = rd64(CAP); |
| |
| zxlogf(INFO, "nvme: version %d.%d.%d\n", n >> 16, (n >> 8) & 0xFF, n & 0xFF); |
| zxlogf(INFO, "nvme: page size: (MPSMIN): %u (MPSMAX): %u\n", |
| (unsigned)(1 << NVME_CAP_MPSMIN(cap)), (unsigned)(1 << NVME_CAP_MPSMAX(cap))); |
| zxlogf(INFO, "nvme: doorbell stride: %u\n", (unsigned)(1 << NVME_CAP_DSTRD(cap))); |
| zxlogf(INFO, "nvme: timeout: %u ms\n", (unsigned)(1 << NVME_CAP_TO(cap))); |
| zxlogf(INFO, "nvme: boot partition support (BPS): %c\n", NVME_CAP_BPS(cap) ? 'Y' : 'N'); |
| zxlogf(INFO, "nvme: supports NVM command set (CSS:NVM): %c\n", NVME_CAP_CSS_NVM(cap) ? 'Y' : 'N'); |
| zxlogf(INFO, "nvme: subsystem reset supported (NSSRS): %c\n", NVME_CAP_NSSRS(cap) ? 'Y' : 'N'); |
| zxlogf(INFO, "nvme: weighted-round-robin (AMS:WRR): %c\n", NVME_CAP_AMS_WRR(cap) ? 'Y' : 'N'); |
| zxlogf(INFO, "nvme: vendor-specific arbitration (AMS:VS): %c\n", |
| NVME_CAP_AMS_VS(cap) ? 'Y' : 'N'); |
| zxlogf(INFO, "nvme: contiquous queues required (CQR): %c\n", NVME_CAP_CQR(cap) ? 'Y' : 'N'); |
| zxlogf(INFO, "nvme: maximum queue entries supported (MQES): %u\n", |
| ((unsigned)NVME_CAP_MQES(cap)) + 1); |
| |
| if ((1 << NVME_CAP_MPSMIN(cap)) > PAGE_SIZE) { |
| zxlogf(ERROR, "nvme: minimum page size larger than platform page size\n"); |
| return ZX_ERR_NOT_SUPPORTED; |
| } |
| // allocate pages for various queues and the utxn scatter lists |
| // TODO: these should all be RO to hardware apart from the scratch io page(s) |
| if (io_buffer_init(&nvme->iob, nvme->bti, PAGE_SIZE * IO_PAGE_COUNT, IO_BUFFER_RW) || |
| io_buffer_physmap(&nvme->iob)) { |
| zxlogf(ERROR, "nvme: could not allocate io buffers\n"); |
| return ZX_ERR_NO_MEMORY; |
| } |
| |
| // initialize the microtransaction pool |
| nvme->utxn_avail = 0x7FFFFFFFFFFFFFFFULL; |
| for (unsigned n = 0; n < UTXN_COUNT; n++) { |
| nvme->utxn[n].id = n; |
| nvme->utxn[n].phys = nvme->iob.phys_list[IDX_UTXN_POOL + n]; |
| nvme->utxn[n].virt = nvme->iob.virt + (IDX_UTXN_POOL + n) * PAGE_SIZE; |
| } |
| |
| if (rd32(CSTS) & NVME_CSTS_RDY) { |
| zxlogf(INFO, "nvme: controller is active. resetting...\n"); |
| wr32(rd32(CC) & ~NVME_CC_EN, CC); // disable |
| } |
| |
| // ensure previous shutdown (by us or bootloader) has completed |
| unsigned ms_remain = WAIT_MS; |
| while (rd32(CSTS) & NVME_CSTS_RDY) { |
| if (--ms_remain == 0) { |
| zxlogf(ERROR, "nvme: timed out waiting for CSTS ~RDY\n"); |
| return ZX_ERR_INTERNAL; |
| } |
| zx_nanosleep(zx_deadline_after(ZX_MSEC(1))); |
| } |
| |
| zxlogf(INFO, "nvme: controller inactive. (after %u ms)\n", WAIT_MS - ms_remain); |
| |
| // configure admin submission and completion queues |
| wr64(nvme->iob.phys_list[IDX_ADMIN_SQ], ASQ); |
| wr64(nvme->iob.phys_list[IDX_ADMIN_CQ], ACQ); |
| wr32(NVME_AQA_ASQS(SQMAX - 1) | NVME_AQA_ACQS(CQMAX - 1), AQA); |
| |
| zxlogf(INFO, "nvme: enabling\n"); |
| wr32(NVME_CC_EN | NVME_CC_AMS_RR | NVME_CC_MPS(0) | NVME_CC_IOCQES(NVME_CPL_SHIFT) | |
| NVME_CC_IOSQES(NVME_CMD_SHIFT), |
| CC); |
| |
| ms_remain = WAIT_MS; |
| while (!(rd32(CSTS) & NVME_CSTS_RDY)) { |
| if (--ms_remain == 0) { |
| zxlogf(ERROR, "nvme: timed out waiting for CSTS RDY\n"); |
| return ZX_ERR_INTERNAL; |
| } |
| zx_nanosleep(zx_deadline_after(ZX_MSEC(1))); |
| } |
| zxlogf(INFO, "nvme: controller ready. (after %u ms)\n", WAIT_MS - ms_remain); |
| |
| // registers and buffers for admin queues |
| nvme->io_admin_sq_tail_db = nvme->mmio.vaddr + NVME_REG_SQnTDBL(0, cap); |
| nvme->io_admin_cq_head_db = nvme->mmio.vaddr + NVME_REG_CQnHDBL(0, cap); |
| |
| nvme->admin_sq = nvme->iob.virt + PAGE_SIZE * IDX_ADMIN_SQ; |
| nvme->admin_sq_head = 0; |
| nvme->admin_sq_tail = 0; |
| |
| nvme->admin_cq = nvme->iob.virt + PAGE_SIZE * IDX_ADMIN_CQ; |
| nvme->admin_cq_head = 0; |
| nvme->admin_cq_toggle = 1; |
| |
| // registers and buffers for IO queues |
| nvme->io_sq_tail_db = nvme->mmio.vaddr + NVME_REG_SQnTDBL(1, cap); |
| nvme->io_cq_head_db = nvme->mmio.vaddr + NVME_REG_CQnHDBL(1, cap); |
| |
| nvme->io_sq = nvme->iob.virt + PAGE_SIZE * IDX_IO_SQ; |
| nvme->io_sq_head = 0; |
| nvme->io_sq_tail = 0; |
| |
| nvme->io_cq = nvme->iob.virt + PAGE_SIZE * IDX_IO_CQ; |
| nvme->io_cq_head = 0; |
| nvme->io_cq_toggle = 1; |
| |
| // scratch page for admin ops |
| void* scratch = nvme->iob.virt + PAGE_SIZE * IDX_SCRATCH; |
| |
| if (thrd_create_with_name(&nvme->irqthread, irq_thread, nvme, "nvme-irq-thread")) { |
| zxlogf(ERROR, "nvme; cannot create irq thread\n"); |
| return ZX_ERR_INTERNAL; |
| } |
| nvme->flags |= FLAG_IRQ_THREAD_STARTED; |
| |
| if (thrd_create_with_name(&nvme->iothread, io_thread, nvme, "nvme-io-thread")) { |
| zxlogf(ERROR, "nvme; cannot create io thread\n"); |
| return ZX_ERR_INTERNAL; |
| } |
| nvme->flags |= FLAG_IO_THREAD_STARTED; |
| |
| nvme_cmd_t cmd; |
| |
| // identify device |
| cmd.cmd = NVME_CMD_CID(0) | NVME_CMD_PRP | NVME_CMD_NORMAL | NVME_CMD_OPC(NVME_ADMIN_OP_IDENTIFY); |
| cmd.nsid = 0; |
| cmd.reserved = 0; |
| cmd.mptr = 0; |
| cmd.dptr.prp[0] = nvme->iob.phys_list[IDX_SCRATCH]; |
| cmd.dptr.prp[1] = 0; |
| cmd.u.raw[0] = 1; // CNS 01 |
| |
| if (nvme_admin_txn(nvme, &cmd, NULL) != ZX_OK) { |
| zxlogf(ERROR, "nvme: device identify op failed\n"); |
| return ZX_ERR_INTERNAL; |
| } |
| |
| nvme_identify_t* ci = scratch; |
| infostring("model: ", ci->MN, sizeof(ci->MN)); |
| infostring("serial number: ", ci->SN, sizeof(ci->SN)); |
| infostring("firmware: ", ci->FR, sizeof(ci->FR)); |
| |
| if ((ci->SQES & 0xF) != NVME_CMD_SHIFT) { |
| zxlogf(ERROR, "nvme: SQES minimum is not %ub\n", NVME_CMD_SIZE); |
| return ZX_ERR_NOT_SUPPORTED; |
| } |
| if ((ci->CQES & 0xF) != NVME_CPL_SHIFT) { |
| zxlogf(ERROR, "nvme: CQES minimum is not %ub\n", NVME_CPL_SIZE); |
| return ZX_ERR_NOT_SUPPORTED; |
| } |
| zxlogf(INFO, "nvme: max outstanding commands: %u\n", ci->MAXCMD); |
| |
| uint32_t nscount = ci->NN; |
| zxlogf(INFO, "nvme: max namespaces: %u\n", nscount); |
| zxlogf(INFO, "nvme: scatter gather lists (SGL): %c %08x\n", (ci->SGLS & 3) ? 'Y' : 'N', ci->SGLS); |
| |
| // Maximum transfer is in units of 2^n * PAGESIZE, n == 0 means "infinite" |
| nvme->max_xfer = 0xFFFFFFFF; |
| if ((ci->MDTS != 0) && (ci->MDTS < (31 - PAGE_SHIFT))) { |
| nvme->max_xfer = (1 << ci->MDTS) * PAGE_SIZE; |
| } |
| |
| zxlogf(INFO, "nvme: max data transfer: %u bytes\n", nvme->max_xfer); |
| zxlogf(INFO, "nvme: sanitize caps: %u\n", ci->SANICAP & 3); |
| |
| zxlogf(INFO, "nvme: abort command limit (ACL): %u\n", ci->ACL + 1); |
| zxlogf(INFO, "nvme: asynch event req limit (AERL): %u\n", ci->AERL + 1); |
| zxlogf(INFO, "nvme: firmware: slots: %u reset: %c slot1ro: %c\n", (ci->FRMW >> 1) & 3, |
| (ci->FRMW & (1 << 4)) ? 'N' : 'Y', (ci->FRMW & 1) ? 'Y' : 'N'); |
| zxlogf(INFO, "nvme: host buffer: min/preferred: %u/%u pages\n", ci->HMMIN, ci->HMPRE); |
| zxlogf(INFO, "nvme: capacity: total/unalloc: %zu/%zu\n", ci->TNVMCAP_LO, ci->UNVMCAP_LO); |
| |
| if (ci->VWC & 1) { |
| nvme->flags |= FLAG_HAS_VWC; |
| } |
| uint32_t awun = ci->AWUN + 1; |
| uint32_t awupf = ci->AWUPF + 1; |
| zxlogf(INFO, "nvme: volatile write cache (VWC): %s\n", nvme->flags & FLAG_HAS_VWC ? "Y" : "N"); |
| zxlogf(INFO, "nvme: atomic write unit (AWUN)/(AWUPF): %u/%u blks\n", awun, awupf); |
| |
| #define FEATURE(a, b) \ |
| if (ci->a & a##_##b) \ |
| zxlogf(INFO, "nvme: feature: %s\n", #b) |
| FEATURE(OACS, DOORBELL_BUFFER_CONFIG); |
| FEATURE(OACS, VIRTUALIZATION_MANAGEMENT); |
| FEATURE(OACS, NVME_MI_SEND_RECV); |
| FEATURE(OACS, DIRECTIVE_SEND_RECV); |
| FEATURE(OACS, DEVICE_SELF_TEST); |
| FEATURE(OACS, NAMESPACE_MANAGEMENT); |
| FEATURE(OACS, FIRMWARE_DOWNLOAD_COMMIT); |
| FEATURE(OACS, FORMAT_NVM); |
| FEATURE(OACS, SECURITY_SEND_RECV); |
| FEATURE(ONCS, TIMESTAMP); |
| FEATURE(ONCS, RESERVATIONS); |
| FEATURE(ONCS, SAVE_SELECT_NONZERO); |
| FEATURE(ONCS, WRITE_UNCORRECTABLE); |
| FEATURE(ONCS, COMPARE); |
| |
| // set feature (number of queues) to 1 iosq and 1 iocq |
| memset(&cmd, 0, sizeof(cmd)); |
| cmd.cmd = |
| NVME_CMD_CID(0) | NVME_CMD_PRP | NVME_CMD_NORMAL | NVME_CMD_OPC(NVME_ADMIN_OP_SET_FEATURE); |
| cmd.u.raw[0] = NVME_FEATURE_NUMBER_OF_QUEUES; |
| cmd.u.raw[1] = 0; |
| |
| nvme_cpl_t cpl; |
| if (nvme_admin_txn(nvme, &cmd, &cpl) != ZX_OK) { |
| zxlogf(ERROR, "nvme: set feature (number queues) op failed\n"); |
| return ZX_ERR_INTERNAL; |
| } |
| zxlogf(INFO, "cpl.cmd %08x\n", cpl.cmd); |
| |
| // create the IO completion queue |
| memset(&cmd, 0, sizeof(cmd)); |
| cmd.cmd = |
| NVME_CMD_CID(0) | NVME_CMD_PRP | NVME_CMD_NORMAL | NVME_CMD_OPC(NVME_ADMIN_OP_CREATE_IOCQ); |
| cmd.dptr.prp[0] = nvme->iob.phys_list[IDX_IO_CQ]; |
| cmd.u.raw[0] = ((CQMAX - 1) << 16) | 1; // queue size, queue id |
| cmd.u.raw[1] = (0 << 16) | 2 | 1; // irq vector, irq enable, phys contig |
| |
| if (nvme_admin_txn(nvme, &cmd, NULL) != ZX_OK) { |
| zxlogf(ERROR, "nvme: completion queue creation op failed\n"); |
| return ZX_ERR_INTERNAL; |
| } |
| |
| // create the IO submit queue |
| memset(&cmd, 0, sizeof(cmd)); |
| cmd.cmd = |
| NVME_CMD_CID(0) | NVME_CMD_PRP | NVME_CMD_NORMAL | NVME_CMD_OPC(NVME_ADMIN_OP_CREATE_IOSQ); |
| cmd.dptr.prp[0] = nvme->iob.phys_list[IDX_IO_SQ]; |
| cmd.u.raw[0] = ((SQMAX - 1) << 16) | 1; // queue size, queue id |
| cmd.u.raw[1] = (1 << 16) | 0 | 1; // cqid, qprio, phys contig |
| |
| if (nvme_admin_txn(nvme, &cmd, NULL) != ZX_OK) { |
| zxlogf(ERROR, "nvme: submit queue creation op failed\n"); |
| return ZX_ERR_INTERNAL; |
| } |
| |
| // identify namespace 1 |
| memset(&cmd, 0, sizeof(cmd)); |
| cmd.cmd = NVME_CMD_CID(0) | NVME_CMD_PRP | NVME_CMD_NORMAL | NVME_CMD_OPC(NVME_ADMIN_OP_IDENTIFY); |
| cmd.nsid = 1; |
| cmd.dptr.prp[0] = nvme->iob.phys_list[IDX_SCRATCH]; |
| |
| if (nvme_admin_txn(nvme, &cmd, NULL) != ZX_OK) { |
| zxlogf(ERROR, "nvme: namespace identify op failed\n"); |
| return ZX_ERR_INTERNAL; |
| } |
| |
| nvme_identify_ns_t* ni = scratch; |
| |
| uint32_t nawun = (ni->NSFEAT & NSFEAT_LOCAL_ATOMIC_SIZES) ? (ni->NAWUN + 1U) : awun; |
| uint32_t nawupf = (ni->NSFEAT & NSFEAT_LOCAL_ATOMIC_SIZES) ? (ni->NAWUPF + 1U) : awupf; |
| zxlogf(INFO, "nvme: ns: atomic write unit (AWUN)/(AWUPF): %u/%u blks\n", nawun, nawupf); |
| zxlogf(INFO, "nvme: ns: NABSN/NABO/NABSPF/NOIOB: %u/%u/%u/%u\n", ni->NABSN, ni->NABO, ni->NABSPF, |
| ni->NOIOB); |
| |
| // table of block formats |
| for (unsigned i = 0; i < 16; i++) { |
| if (ni->LBAF[i]) { |
| zxlogf(INFO, "nvme: ns: LBA FMT %02d: RP=%u LBADS=2^%ub MS=%ub\n", i, |
| NVME_LBAFMT_RP(ni->LBAF[i]), NVME_LBAFMT_LBADS(ni->LBAF[i]), |
| NVME_LBAFMT_MS(ni->LBAF[i])); |
| } |
| } |
| |
| zxlogf(INFO, "nvme: ns: LBA FMT #%u active\n", ni->FLBAS & 0xF); |
| zxlogf(INFO, "nvme: ns: data protection: caps/set: 0x%02x/%u\n", ni->DPC & 0x3F, ni->DPS & 3); |
| |
| uint32_t fmt = ni->LBAF[ni->FLBAS & 0xF]; |
| |
| zxlogf(INFO, "nvme: ns: size/cap/util: %zu/%zu/%zu blks\n", ni->NSSZ, ni->NCAP, ni->NUSE); |
| |
| nvme->info.block_count = ni->NSSZ; |
| nvme->info.block_size = 1 << NVME_LBAFMT_LBADS(fmt); |
| nvme->info.max_transfer_size = BLOCK_MAX_TRANSFER_UNBOUNDED; |
| |
| if (NVME_LBAFMT_MS(fmt)) { |
| zxlogf(ERROR, "nvme: cannot handle LBA format with metadata\n"); |
| return ZX_ERR_NOT_SUPPORTED; |
| } |
| if ((nvme->info.block_size < 512) || (nvme->info.block_size > 32768)) { |
| zxlogf(ERROR, "nvme: cannot handle LBA size of %u\n", nvme->info.block_size); |
| return ZX_ERR_NOT_SUPPORTED; |
| } |
| |
| // NVME r/w commands operate in block units, maximum of 64K: |
| size_t max_bytes_per_cmd = ((size_t)nvme->info.block_size) * ((size_t)65536); |
| |
| if (nvme->max_xfer > max_bytes_per_cmd) { |
| nvme->max_xfer = max_bytes_per_cmd; |
| } |
| |
| // The device may allow transfers larger than we are prepared |
| // to handle. Clip to our limit. |
| if (nvme->max_xfer > MAX_XFER) { |
| nvme->max_xfer = MAX_XFER; |
| } |
| |
| // convert to block units |
| nvme->max_xfer /= nvme->info.block_size; |
| zxlogf(INFO, "nvme: max transfer per r/w op: %u blocks (%u bytes)\n", nvme->max_xfer, |
| nvme->max_xfer * nvme->info.block_size); |
| |
| device_make_visible(nvme->zxdev, NULL); |
| return ZX_OK; |
| } |
| |
| block_impl_protocol_ops_t block_ops = { |
| .query = nvme_query, |
| .queue = nvme_queue, |
| }; |
| |
| static zx_status_t nvme_bind(void* ctx, zx_device_t* dev) { |
| nvme_device_t* nvme; |
| if ((nvme = calloc(1, sizeof(nvme_device_t))) == NULL) { |
| return ZX_ERR_NO_MEMORY; |
| } |
| list_initialize(&nvme->pending_txns); |
| list_initialize(&nvme->active_txns); |
| mtx_init(&nvme->lock, mtx_plain); |
| mtx_init(&nvme->admin_lock, mtx_plain); |
| |
| if (device_get_protocol(dev, ZX_PROTOCOL_PCI, &nvme->pci)) { |
| goto fail; |
| } |
| |
| if (pci_map_bar_buffer(&nvme->pci, 0u, ZX_CACHE_POLICY_UNCACHED_DEVICE, &nvme->mmio)) { |
| zxlogf(ERROR, "nvme: cannot map registers\n"); |
| goto fail; |
| } |
| |
| uint32_t modes[3] = { |
| ZX_PCIE_IRQ_MODE_MSI_X, |
| ZX_PCIE_IRQ_MODE_MSI, |
| ZX_PCIE_IRQ_MODE_LEGACY, |
| }; |
| uint32_t nirq = 0; |
| for (unsigned n = 0; n < countof(modes); n++) { |
| if ((pci_query_irq_mode(&nvme->pci, modes[n], &nirq) == ZX_OK) && |
| (pci_set_irq_mode(&nvme->pci, modes[n], 1) == ZX_OK)) { |
| zxlogf(INFO, "nvme: irq mode %u, irq count %u (#%u)\n", modes[n], nirq, n); |
| goto irq_configured; |
| } |
| } |
| zxlogf(ERROR, "nvme: could not configure irqs\n"); |
| goto fail; |
| |
| irq_configured: |
| if (pci_map_interrupt(&nvme->pci, 0, &nvme->irqh) != ZX_OK) { |
| zxlogf(ERROR, "nvme: could not map irq\n"); |
| goto fail; |
| } |
| if (pci_enable_bus_master(&nvme->pci, true)) { |
| zxlogf(ERROR, "nvme: cannot enable bus mastering\n"); |
| goto fail; |
| } |
| if (pci_get_bti(&nvme->pci, 0, &nvme->bti) != ZX_OK) { |
| zxlogf(ERROR, "nvme: cannot obtain bti handle\n"); |
| goto fail; |
| } |
| |
| device_add_args_t args = { |
| .version = DEVICE_ADD_ARGS_VERSION, |
| .name = "nvme", |
| .ctx = nvme, |
| .ops = &device_ops, |
| .flags = DEVICE_ADD_INVISIBLE, |
| .proto_id = ZX_PROTOCOL_BLOCK_IMPL, |
| .proto_ops = &block_ops, |
| }; |
| |
| if (device_add(dev, &args, &nvme->zxdev)) { |
| goto fail; |
| } |
| |
| if (nvme_init(nvme) != ZX_OK) { |
| zxlogf(ERROR, "nvme: init failed\n"); |
| device_async_remove(nvme->zxdev); |
| return ZX_ERR_INTERNAL; |
| } |
| |
| return ZX_OK; |
| |
| fail: |
| nvme_release(nvme); |
| return ZX_ERR_NOT_SUPPORTED; |
| } |
| |
| static zx_driver_ops_t driver_ops = { |
| .version = DRIVER_OPS_VERSION, |
| .bind = nvme_bind, |
| }; |
| |
| ZIRCON_DRIVER_BEGIN(nvme, driver_ops, "zircon", "0.1", 4) |
| BI_ABORT_IF(NE, BIND_PROTOCOL, ZX_PROTOCOL_PCI), |
| BI_ABORT_IF(NE, BIND_PCI_CLASS, 1), // Mass Storage |
| BI_ABORT_IF(NE, BIND_PCI_SUBCLASS, 8), // NVM |
| BI_MATCH_IF(EQ, BIND_PCI_INTERFACE, 2), // NVMHCI |
| ZIRCON_DRIVER_END(nvme) |