| // Copyright 2019 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "port.h" |
| |
| #include <inttypes.h> |
| #include <lib/zx/clock.h> |
| #include <unistd.h> |
| |
| #include <algorithm> |
| |
| #include <ddk/debug.h> |
| #include <ddk/phys-iter.h> |
| #include <fbl/auto_lock.h> |
| |
| #include "controller.h" |
| |
| #define PAGE_MASK (PAGE_SIZE - 1ull) |
| |
| namespace ahci { |
| |
| constexpr zx::duration kTransactionTimeout(ZX_SEC(5)); |
| |
| constexpr uint32_t hi32(uint64_t val) { return static_cast<uint32_t>(val >> 32); } |
| constexpr uint32_t lo32(uint64_t val) { return static_cast<uint32_t>(val); } |
| |
| // Calculate the physical base of a virtual address. |
| zx_paddr_t vtop(zx_paddr_t phys_base, void* virt_base, void* virt_addr) { |
| uintptr_t addr = reinterpret_cast<uintptr_t>(virt_addr); |
| uintptr_t base = reinterpret_cast<uintptr_t>(virt_base); |
| return phys_base + (addr - base); |
| } |
| |
| bool cmd_is_read(uint8_t cmd) { |
| if (cmd == SATA_CMD_READ_DMA || cmd == SATA_CMD_READ_DMA_EXT || |
| cmd == SATA_CMD_READ_FPDMA_QUEUED) { |
| return true; |
| } else { |
| return false; |
| } |
| } |
| |
| bool cmd_is_write(uint8_t cmd) { |
| if (cmd == SATA_CMD_WRITE_DMA || cmd == SATA_CMD_WRITE_DMA_EXT || |
| cmd == SATA_CMD_WRITE_FPDMA_QUEUED) { |
| return true; |
| } else { |
| return false; |
| } |
| } |
| |
| bool cmd_is_queued(uint8_t cmd) { |
| return (cmd == SATA_CMD_READ_FPDMA_QUEUED) || (cmd == SATA_CMD_WRITE_FPDMA_QUEUED); |
| } |
| |
| Port::Port() { list_initialize(&txn_list_); } |
| |
| Port::~Port() { ZX_DEBUG_ASSERT(list_is_empty(&txn_list_)); } |
| |
| uint32_t Port::RegRead(size_t offset) { |
| uint32_t val = 0; |
| bus_->RegRead(reg_base_ + offset, &val); |
| return val; |
| } |
| |
| void Port::RegWrite(size_t offset, uint32_t val) { bus_->RegWrite(reg_base_ + offset, val); } |
| |
| bool Port::SlotBusyLocked(uint32_t slot) { |
| // a command slot is busy if a transaction is in flight or pending to be completed |
| return ((RegRead(kPortSataActive) | RegRead(kPortCommandIssue)) & (1u << slot)) || |
| (commands_[slot] != nullptr) || (running_ & (1u << slot)) || (completed_ & (1u << slot)); |
| } |
| |
| zx_status_t Port::Configure(uint32_t num, Bus* bus, size_t reg_base, uint32_t capabilities) { |
| fbl::AutoLock lock(&lock_); |
| num_ = num; |
| cap_ = capabilities; |
| bus_ = bus; |
| reg_base_ = reg_base + (num * sizeof(ahci_port_reg_t)); |
| flags_ = kPortFlagImplemented; |
| uint32_t cmd = RegRead(kPortCommand); |
| if (cmd & (AHCI_PORT_CMD_ST | AHCI_PORT_CMD_FRE | AHCI_PORT_CMD_CR | AHCI_PORT_CMD_FR)) { |
| zxlogf(ERROR, "ahci.%u: port busy", num_); |
| return ZX_ERR_UNAVAILABLE; |
| } |
| |
| // Allocate memory for the command list, FIS receive area, command table and PRDT. |
| zx_paddr_t phys_base; |
| void* virt_base; |
| zx_status_t status = bus_->IoBufferInit(&buffer_, sizeof(ahci_port_mem_t), |
| IO_BUFFER_RW | IO_BUFFER_CONTIG, &phys_base, &virt_base); |
| if (status != ZX_OK) { |
| zxlogf(ERROR, "ahci.%u: error %d allocating dma memory", num_, status); |
| return status; |
| } |
| mem_ = static_cast<ahci_port_mem_t*>(virt_base); |
| |
| // clear memory area |
| // order is command list (1024-byte aligned) |
| // FIS receive area (256-byte aligned) |
| // command table + PRDT (128-byte aligned) |
| memset(mem_, 0, sizeof(*mem_)); |
| |
| // command list. |
| zx_paddr_t paddr = vtop(phys_base, mem_, &mem_->cl); |
| RegWrite(kPortCommandListBase, lo32(paddr)); |
| RegWrite(kPortCommandListBaseUpper, hi32(paddr)); |
| |
| // FIS receive area. |
| paddr = vtop(phys_base, mem_, &mem_->fis); |
| RegWrite(kPortFISBase, lo32(paddr)); |
| RegWrite(kPortFISBaseUpper, hi32(paddr)); |
| |
| // command table, followed by PRDT. |
| for (int i = 0; i < AHCI_MAX_COMMANDS; i++) { |
| paddr = vtop(phys_base, mem_, &mem_->tab[i].ct); |
| mem_->cl[i].ctba = lo32(paddr); |
| mem_->cl[i].ctbau = hi32(paddr); |
| } |
| |
| // clear port interrupts |
| RegWrite(kPortInterruptStatus, RegRead(kPortInterruptStatus)); |
| |
| // clear error |
| RegWrite(kPortSataError, RegRead(kPortSataError)); |
| |
| // spin up |
| cmd |= AHCI_PORT_CMD_SUD; |
| RegWrite(kPortCommand, cmd); |
| |
| // activate link |
| cmd &= ~AHCI_PORT_CMD_ICC_MASK; |
| cmd |= AHCI_PORT_CMD_ICC_ACTIVE; |
| RegWrite(kPortCommand, cmd); |
| |
| // enable FIS receive |
| cmd |= AHCI_PORT_CMD_FRE; |
| RegWrite(kPortCommand, cmd); |
| |
| return ZX_OK; |
| } |
| |
| zx_status_t Port::Enable() { |
| uint32_t cmd = RegRead(kPortCommand); |
| if (cmd & AHCI_PORT_CMD_ST) |
| return ZX_OK; |
| if (!(cmd & AHCI_PORT_CMD_FRE)) { |
| zxlogf(ERROR, "ahci.%u: cannot enable port without FRE enabled", num_); |
| return ZX_ERR_BAD_STATE; |
| } |
| zx_status_t status = |
| bus_->WaitForClear(reg_base_ + kPortCommand, AHCI_PORT_CMD_CR, zx::msec(500)); |
| if (status) { |
| zxlogf(ERROR, "ahci.%u: dma engine still running when enabling port", num_); |
| return ZX_ERR_BAD_STATE; |
| } |
| cmd |= AHCI_PORT_CMD_ST; |
| RegWrite(kPortCommand, cmd); |
| return ZX_OK; |
| } |
| |
| void Port::Disable() { |
| uint32_t cmd = RegRead(kPortCommand); |
| if (!(cmd & AHCI_PORT_CMD_ST)) |
| return; |
| cmd &= ~AHCI_PORT_CMD_ST; |
| RegWrite(kPortCommand, cmd); |
| zx_status_t status = |
| bus_->WaitForClear(reg_base_ + kPortCommand, AHCI_PORT_CMD_CR, zx::msec(500)); |
| if (status) { |
| zxlogf(ERROR, "ahci.%u: port disable timed out", num_); |
| } |
| } |
| |
| void Port::Reset() { |
| // disable port |
| Disable(); |
| |
| // clear error |
| RegWrite(kPortSataError, RegRead(kPortSataError)); |
| |
| // wait for device idle |
| zx_status_t status = bus_->WaitForClear( |
| reg_base_ + kPortTaskFileData, AHCI_PORT_TFD_BUSY | AHCI_PORT_TFD_DATA_REQUEST, zx::sec(1)); |
| if (status != ZX_OK) { |
| // if busy is not cleared, do a full comreset |
| zxlogf(TRACE, "ahci.%u: timed out waiting for port idle, resetting", num_); |
| // v1.3.1, 10.4.2 port reset |
| uint32_t sctl = |
| AHCI_PORT_SCTL_IPM_ACTIVE | AHCI_PORT_SCTL_IPM_PARTIAL | AHCI_PORT_SCTL_DET_INIT; |
| RegWrite(kPortSataControl, sctl); |
| usleep(1000); |
| sctl = RegRead(kPortSataControl); |
| sctl &= ~AHCI_PORT_SCTL_DET_MASK; |
| RegWrite(kPortSataControl, sctl); |
| } |
| |
| // enable port |
| Enable(); |
| |
| // wait for device detect |
| status = bus_->WaitForSet(reg_base_ + kPortSataStatus, AHCI_PORT_SSTS_DET_PRESENT, zx::sec(1)); |
| if (status < 0) { |
| zxlogf(TRACE, "ahci.%u: no device detected", num_); |
| } |
| |
| // clear error |
| RegWrite(kPortSataError, RegRead(kPortSataError)); |
| } |
| |
| void Port::SetDevInfo(const sata_devinfo_t* devinfo) { |
| fbl::AutoLock lock(&lock_); |
| memcpy(&devinfo_, devinfo, sizeof(devinfo_)); |
| } |
| |
| zx_status_t Port::Queue(sata_txn_t* txn) { |
| fbl::AutoLock lock(&lock_); |
| if (!is_valid()) { |
| return ZX_ERR_BAD_STATE; |
| } |
| |
| // reset the physical address |
| txn->pmt = ZX_HANDLE_INVALID; |
| |
| // put the cmd on the queue |
| list_add_tail(&txn_list_, &txn->node); |
| |
| return ZX_OK; |
| } |
| |
| bool Port::Complete() { |
| fbl::AutoLock lock(&lock_); |
| if (!is_valid()) { |
| return false; |
| } |
| |
| sata_txn_t* txn_complete[AHCI_MAX_COMMANDS]; |
| size_t complete_count = 0; |
| bool active_txns = false; |
| |
| for (uint32_t slot = 0; slot < AHCI_MAX_COMMANDS; slot++) { |
| sata_txn_t* txn = commands_[slot]; |
| if (txn == nullptr) { |
| continue; // No transaction in this slot. |
| } |
| uint32_t slot_bit = (1u << slot); |
| if ((completed_ & slot_bit) == 0) { |
| // Not complete, check if timeout expired. |
| zx::time now = zx::clock::get_monotonic(); |
| if (txn->timeout > now) { |
| active_txns = true; |
| continue; // Still in progress. |
| } |
| // Timed out. |
| zx::duration delta = now - txn->timeout; |
| zxlogf(ERROR, "ahci: txn time out on port %d txn %p (%ld ms)", num_, txn, delta.to_msecs()); |
| txn->timeout = zx::time::infinite_past(); // Signal that timeout occured. |
| } |
| // Completed or timed out. |
| commands_[slot] = nullptr; |
| running_ &= ~slot_bit; |
| completed_ &= ~slot_bit; |
| txn_complete[complete_count] = txn; |
| complete_count++; |
| } |
| |
| sata_txn_t* sync_op = nullptr; |
| // resume the port if paused for sync and no outstanding transactions |
| if ((is_paused()) && !running_) { |
| flags_ &= ~kPortFlagSyncPaused; |
| if (sync_) { |
| sync_op = sync_; |
| sync_ = nullptr; |
| } |
| } |
| lock.release(); |
| |
| for (size_t i = 0; i < complete_count; i++) { |
| sata_txn_t* txn = txn_complete[i]; |
| if (txn->pmt != ZX_HANDLE_INVALID) { |
| zx_pmt_unpin(txn->pmt); |
| } |
| if (txn->timeout == zx::time::infinite_past()) { |
| block_complete(txn, ZX_ERR_TIMED_OUT); |
| } else { |
| zxlogf(TRACE, "ahci.%u: complete txn %p", num_, txn); |
| block_complete(txn, ZX_OK); |
| } |
| } |
| |
| if (sync_op != nullptr) { |
| block_complete(sync_op, ZX_OK); |
| } |
| return active_txns; |
| } |
| |
| bool Port::ProcessQueued() { |
| lock_.Acquire(); |
| if ((!is_valid()) || is_paused()) { |
| lock_.Release(); |
| return false; |
| } |
| |
| bool added_txns = false; |
| for (;;) { |
| sata_txn_t* txn = list_peek_head_type(&txn_list_, sata_txn_t, node); |
| if (!txn) { |
| break; |
| } |
| |
| // find a free command tag |
| uint32_t max = std::min(devinfo_.max_cmd, MaxCommands()); |
| uint32_t i = 0; |
| for (i = 0; i <= max; i++) { |
| if (!SlotBusyLocked(i)) |
| break; |
| } |
| if (i > max) { |
| break; |
| } |
| |
| list_delete(&txn->node); |
| |
| if (BLOCK_OP(txn->bop.command) == BLOCK_OP_FLUSH) { |
| if (running_) { |
| ZX_DEBUG_ASSERT(sync_ == nullptr); |
| // pause the port if FLUSH command |
| flags_ |= kPortFlagSyncPaused; |
| sync_ = txn; |
| added_txns = true; |
| } else { |
| // complete immediately if nothing in flight |
| lock_.Release(); |
| block_complete(txn, ZX_OK); |
| lock_.Acquire(); |
| } |
| } else { |
| // run the transaction |
| zx_status_t st = TxnBeginLocked(i, txn); |
| // complete the transaction with if it failed during processing |
| if (st != ZX_OK) { |
| lock_.Release(); |
| block_complete(txn, st); |
| lock_.Acquire(); |
| continue; |
| } |
| added_txns = true; |
| } |
| } |
| lock_.Release(); |
| return added_txns; |
| } |
| |
| void Port::TxnComplete(zx_status_t status) { |
| fbl::AutoLock lock(&lock_); |
| uint32_t active = RegRead(kPortSataActive); // Transactions active in hardware. |
| uint32_t running = running_; // Transactions tagged as running. |
| // Transactions active in hardware but not tagged as running. |
| uint32_t unaccounted = active & ~running; |
| // Remove transactions that have been completed by the watchdog. |
| unaccounted &= ~completed_; |
| // assert if a command slot without an outstanding transaction is active. |
| ZX_DEBUG_ASSERT(unaccounted == 0); |
| |
| // Transactions tagged as running but completed by hardware. |
| uint32_t done = running & ~active; |
| completed_ |= done; |
| } |
| |
| zx_status_t Port::TxnBeginLocked(uint32_t slot, sata_txn_t* txn) { |
| ZX_DEBUG_ASSERT(slot < AHCI_MAX_COMMANDS); |
| ZX_DEBUG_ASSERT(!SlotBusyLocked(slot)); |
| |
| uint64_t offset_vmo = txn->bop.rw.offset_vmo * devinfo_.block_size; |
| uint64_t bytes = txn->bop.rw.length * devinfo_.block_size; |
| size_t pagecount = ((offset_vmo & (PAGE_SIZE - 1)) + bytes + (PAGE_SIZE - 1)) / PAGE_SIZE; |
| zx_paddr_t pages[AHCI_MAX_PAGES]; |
| if (pagecount > AHCI_MAX_PAGES) { |
| zxlogf(TRACE, "ahci.%u: txn %p too many pages (%zu)", num_, txn, pagecount); |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| zx::unowned_vmo vmo(txn->bop.rw.vmo); |
| bool is_write = cmd_is_write(txn->cmd); |
| uint32_t options = is_write ? ZX_BTI_PERM_READ : ZX_BTI_PERM_WRITE; |
| zx::pmt pmt; |
| zx_status_t st = bus_->BtiPin(options, vmo, offset_vmo & ~PAGE_MASK, pagecount * PAGE_SIZE, pages, |
| pagecount, &pmt); |
| if (st != ZX_OK) { |
| zxlogf(TRACE, "ahci.%u: failed to pin pages, err = %d", num_, st); |
| return st; |
| } |
| txn->pmt = pmt.release(); |
| |
| phys_iter_buffer_t physbuf = {}; |
| physbuf.phys = pages; |
| physbuf.phys_count = pagecount; |
| physbuf.length = bytes; |
| physbuf.vmo_offset = offset_vmo; |
| |
| phys_iter_t iter; |
| phys_iter_init(&iter, &physbuf, AHCI_PRD_MAX_SIZE); |
| |
| uint8_t cmd = txn->cmd; |
| uint8_t device = txn->device; |
| uint64_t lba = txn->bop.rw.offset_dev; |
| uint64_t count = txn->bop.rw.length; |
| |
| // use queued command if available |
| if (HasCommandQueue()) { |
| if (cmd == SATA_CMD_READ_DMA_EXT) { |
| cmd = SATA_CMD_READ_FPDMA_QUEUED; |
| } else if (cmd == SATA_CMD_WRITE_DMA_EXT) { |
| cmd = SATA_CMD_WRITE_FPDMA_QUEUED; |
| } |
| } |
| |
| // build the command |
| ahci_cl_t* cl = &mem_->cl[slot]; |
| // don't clear the cl since we set up ctba/ctbau at init |
| cl->prdtl_flags_cfl = 0; |
| cl->cfl = 5; // 20 bytes |
| cl->w = is_write ? 1 : 0; |
| cl->prdbc = 0; |
| memset(&mem_->tab[slot].ct, 0, sizeof(ahci_ct_t)); |
| |
| uint8_t* cfis = mem_->tab[slot].ct.cfis; |
| cfis[0] = 0x27; // host-to-device |
| cfis[1] = 0x80; // command |
| cfis[2] = cmd; |
| cfis[7] = device; |
| |
| // some commands have lba/count fields |
| if (cmd == SATA_CMD_READ_DMA_EXT || cmd == SATA_CMD_WRITE_DMA_EXT) { |
| cfis[4] = lba & 0xff; |
| cfis[5] = (lba >> 8) & 0xff; |
| cfis[6] = (lba >> 16) & 0xff; |
| cfis[8] = (lba >> 24) & 0xff; |
| cfis[9] = (lba >> 32) & 0xff; |
| cfis[10] = (lba >> 40) & 0xff; |
| cfis[12] = count & 0xff; |
| cfis[13] = (count >> 8) & 0xff; |
| } else if (cmd_is_queued(cmd)) { |
| cfis[4] = lba & 0xff; |
| cfis[5] = (lba >> 8) & 0xff; |
| cfis[6] = (lba >> 16) & 0xff; |
| cfis[8] = (lba >> 24) & 0xff; |
| cfis[9] = (lba >> 32) & 0xff; |
| cfis[10] = (lba >> 40) & 0xff; |
| cfis[3] = count & 0xff; |
| cfis[11] = (count >> 8) & 0xff; |
| cfis[12] = (slot << 3) & 0xff; // tag |
| cfis[13] = 0; // normal priority |
| } |
| |
| cl->prdtl = 0; |
| size_t length; |
| zx_paddr_t paddr; |
| for (uint32_t i = 0; i < AHCI_MAX_PRDS; i++) { |
| length = phys_iter_next(&iter, &paddr); |
| if (length == 0) { |
| break; |
| } else if (length > AHCI_PRD_MAX_SIZE) { |
| zxlogf(ERROR, "ahci.%u: chunk size > %zu is unsupported", num_, length); |
| return ZX_ERR_NOT_SUPPORTED; |
| } else if (cl->prdtl == AHCI_MAX_PRDS) { |
| zxlogf(ERROR, "ahci.%u: txn with more than %d chunks is unsupported", num_, cl->prdtl); |
| return ZX_ERR_NOT_SUPPORTED; |
| } |
| |
| ahci_prd_t* prd = &mem_->tab[slot].prd[i]; |
| prd->dba = lo32(paddr); |
| prd->dbau = hi32(paddr); |
| prd->dbc = ((length - 1) & (AHCI_PRD_MAX_SIZE - 1)); // 0-based byte count |
| cl->prdtl++; |
| } |
| |
| running_ |= (1u << slot); |
| commands_[slot] = txn; |
| |
| zxlogf(TRACE, |
| "ahci.%u: do_txn txn %p (%c) offset 0x%" PRIx64 " length 0x%" PRIx64 " slot %d prdtl %u\n", |
| num_, txn, cl->w ? 'w' : 'r', lba, count, slot, cl->prdtl); |
| if (zxlog_level_enabled(TRACE)) { |
| for (uint32_t i = 0; i < cl->prdtl; i++) { |
| ahci_prd_t* prd = &mem_->tab[slot].prd[i]; |
| zxlogf(TRACE, "%04u: dbau=0x%08x dba=0x%08x dbc=0x%x", i, prd->dbau, prd->dba, prd->dbc); |
| } |
| } |
| |
| // start command |
| if (cmd_is_queued(cmd)) { |
| RegWrite(kPortSataActive, (1u << slot)); |
| } |
| RegWrite(kPortCommandIssue, (1u << slot)); |
| |
| txn->timeout = zx::clock::get_monotonic() + kTransactionTimeout; |
| return ZX_OK; |
| } |
| |
| // HandleIrq does not lock the port or check whether it is valid. |
| // It is assumed that an invalid port can not have scheduled operations that trigger an interrupt. |
| // The port may have been disabled or unplugged, but is still valid. |
| bool Port::HandleIrq() { |
| // Clear interrupt status. |
| uint32_t int_status = RegRead(kPortInterruptStatus); |
| RegWrite(kPortInterruptStatus, int_status); |
| |
| if (int_status & AHCI_PORT_INT_PRC) { // PhyRdy change |
| uint32_t serr = RegRead(kPortSataError); |
| RegWrite(kPortSataError, serr & ~0x1); |
| } |
| if (int_status & AHCI_PORT_INT_ERROR) { // error |
| zxlogf(ERROR, "ahci.%u: error is=0x%08x", num_, int_status); |
| TxnComplete(ZX_ERR_INTERNAL); |
| return true; |
| } else if (int_status) { |
| TxnComplete(ZX_OK); |
| return true; |
| } |
| return false; |
| } |
| |
| // Set up the running state for testing Complete() |
| void Port::TestSetRunning(sata_txn_t* txn, uint32_t slot) { |
| ZX_DEBUG_ASSERT(slot < AHCI_MAX_COMMANDS); |
| commands_[slot] = txn; |
| running_ |= (1u << slot); |
| completed_ &= ~(1u << slot); |
| } |
| |
| } // namespace ahci |