blob: 340305bd606e90ff3be1b4781c70b2e43afc5058 [file] [log] [blame]
// Copyright 2019 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "port.h"
#include <inttypes.h>
#include <lib/ddk/debug.h>
#include <lib/ddk/phys-iter.h>
#include <lib/zx/clock.h>
#include <unistd.h>
#include <algorithm>
#include <fbl/auto_lock.h>
#include "controller.h"
#define PAGE_MASK (PAGE_SIZE - 1ull)
namespace ahci {
constexpr zx::duration kTransactionTimeout(ZX_SEC(5));
constexpr uint32_t hi32(uint64_t val) { return static_cast<uint32_t>(val >> 32); }
constexpr uint32_t lo32(uint64_t val) { return static_cast<uint32_t>(val); }
// Calculate the physical base of a virtual address.
zx_paddr_t vtop(zx_paddr_t phys_base, void* virt_base, void* virt_addr) {
uintptr_t addr = reinterpret_cast<uintptr_t>(virt_addr);
uintptr_t base = reinterpret_cast<uintptr_t>(virt_base);
return phys_base + (addr - base);
}
bool cmd_is_read(uint8_t cmd) {
if (cmd == SATA_CMD_READ_DMA || cmd == SATA_CMD_READ_DMA_EXT ||
cmd == SATA_CMD_READ_FPDMA_QUEUED) {
return true;
} else {
return false;
}
}
bool cmd_is_write(uint8_t cmd) {
if (cmd == SATA_CMD_WRITE_DMA || cmd == SATA_CMD_WRITE_DMA_EXT ||
cmd == SATA_CMD_WRITE_FPDMA_QUEUED) {
return true;
} else {
return false;
}
}
bool cmd_is_queued(uint8_t cmd) {
return (cmd == SATA_CMD_READ_FPDMA_QUEUED) || (cmd == SATA_CMD_WRITE_FPDMA_QUEUED);
}
Port::Port() { list_initialize(&txn_list_); }
Port::~Port() { ZX_DEBUG_ASSERT(list_is_empty(&txn_list_)); }
uint32_t Port::RegRead(size_t offset) {
uint32_t val = 0;
bus_->RegRead(reg_base_ + offset, &val);
return val;
}
void Port::RegWrite(size_t offset, uint32_t val) { bus_->RegWrite(reg_base_ + offset, val); }
bool Port::SlotBusyLocked(uint32_t slot) {
// a command slot is busy if a transaction is in flight or pending to be completed
return ((RegRead(kPortSataActive) | RegRead(kPortCommandIssue)) & (1u << slot)) ||
(commands_[slot] != nullptr) || (running_ & (1u << slot)) || (completed_ & (1u << slot));
}
zx_status_t Port::Configure(uint32_t num, Bus* bus, size_t reg_base, uint32_t capabilities) {
fbl::AutoLock lock(&lock_);
num_ = num;
cap_ = capabilities;
bus_ = bus;
reg_base_ = reg_base + (num * sizeof(ahci_port_reg_t));
flags_ = kPortFlagImplemented;
uint32_t cmd = RegRead(kPortCommand);
if (cmd & (AHCI_PORT_CMD_ST | AHCI_PORT_CMD_FRE | AHCI_PORT_CMD_CR | AHCI_PORT_CMD_FR)) {
zxlogf(ERROR, "ahci.%u: port busy", num_);
return ZX_ERR_UNAVAILABLE;
}
// Allocate memory for the command list, FIS receive area, command table and PRDT.
zx_paddr_t phys_base;
void* virt_base;
zx_status_t status = bus_->IoBufferInit(&buffer_, sizeof(ahci_port_mem_t),
IO_BUFFER_RW | IO_BUFFER_CONTIG, &phys_base, &virt_base);
if (status != ZX_OK) {
zxlogf(ERROR, "ahci.%u: error %d allocating dma memory", num_, status);
return status;
}
mem_ = static_cast<ahci_port_mem_t*>(virt_base);
// clear memory area
// order is command list (1024-byte aligned)
// FIS receive area (256-byte aligned)
// command table + PRDT (128-byte aligned)
memset(mem_, 0, sizeof(*mem_));
// command list.
zx_paddr_t paddr = vtop(phys_base, mem_, &mem_->cl);
RegWrite(kPortCommandListBase, lo32(paddr));
RegWrite(kPortCommandListBaseUpper, hi32(paddr));
// FIS receive area.
paddr = vtop(phys_base, mem_, &mem_->fis);
RegWrite(kPortFISBase, lo32(paddr));
RegWrite(kPortFISBaseUpper, hi32(paddr));
// command table, followed by PRDT.
for (int i = 0; i < AHCI_MAX_COMMANDS; i++) {
paddr = vtop(phys_base, mem_, &mem_->tab[i].ct);
mem_->cl[i].ctba = lo32(paddr);
mem_->cl[i].ctbau = hi32(paddr);
}
// clear port interrupts
RegWrite(kPortInterruptStatus, RegRead(kPortInterruptStatus));
// clear error
RegWrite(kPortSataError, RegRead(kPortSataError));
// spin up
cmd |= AHCI_PORT_CMD_SUD;
RegWrite(kPortCommand, cmd);
// activate link
cmd &= ~AHCI_PORT_CMD_ICC_MASK;
cmd |= AHCI_PORT_CMD_ICC_ACTIVE;
RegWrite(kPortCommand, cmd);
// enable FIS receive
cmd |= AHCI_PORT_CMD_FRE;
RegWrite(kPortCommand, cmd);
return ZX_OK;
}
zx_status_t Port::Enable() {
uint32_t cmd = RegRead(kPortCommand);
if (cmd & AHCI_PORT_CMD_ST)
return ZX_OK;
if (!(cmd & AHCI_PORT_CMD_FRE)) {
zxlogf(ERROR, "ahci.%u: cannot enable port without FRE enabled", num_);
return ZX_ERR_BAD_STATE;
}
zx_status_t status =
bus_->WaitForClear(reg_base_ + kPortCommand, AHCI_PORT_CMD_CR, zx::msec(500));
if (status) {
zxlogf(ERROR, "ahci.%u: dma engine still running when enabling port", num_);
return ZX_ERR_BAD_STATE;
}
cmd |= AHCI_PORT_CMD_ST;
RegWrite(kPortCommand, cmd);
return ZX_OK;
}
void Port::Disable() {
uint32_t cmd = RegRead(kPortCommand);
if (!(cmd & AHCI_PORT_CMD_ST))
return;
cmd &= ~AHCI_PORT_CMD_ST;
RegWrite(kPortCommand, cmd);
zx_status_t status =
bus_->WaitForClear(reg_base_ + kPortCommand, AHCI_PORT_CMD_CR, zx::msec(500));
if (status) {
zxlogf(ERROR, "ahci.%u: port disable timed out", num_);
}
}
void Port::Reset() {
// disable port
Disable();
// clear error
RegWrite(kPortSataError, RegRead(kPortSataError));
// wait for device idle
zx_status_t status = bus_->WaitForClear(
reg_base_ + kPortTaskFileData, AHCI_PORT_TFD_BUSY | AHCI_PORT_TFD_DATA_REQUEST, zx::sec(1));
if (status != ZX_OK) {
// if busy is not cleared, do a full comreset
zxlogf(TRACE, "ahci.%u: timed out waiting for port idle, resetting", num_);
// v1.3.1, 10.4.2 port reset
uint32_t sctl =
AHCI_PORT_SCTL_IPM_ACTIVE | AHCI_PORT_SCTL_IPM_PARTIAL | AHCI_PORT_SCTL_DET_INIT;
RegWrite(kPortSataControl, sctl);
usleep(1000);
sctl = RegRead(kPortSataControl);
sctl &= ~AHCI_PORT_SCTL_DET_MASK;
RegWrite(kPortSataControl, sctl);
}
// enable port
Enable();
// wait for device detect
status = bus_->WaitForSet(reg_base_ + kPortSataStatus, AHCI_PORT_SSTS_DET_PRESENT, zx::sec(1));
if (status < 0) {
zxlogf(TRACE, "ahci.%u: no device detected", num_);
}
// clear error
RegWrite(kPortSataError, RegRead(kPortSataError));
}
void Port::SetDevInfo(const sata_devinfo_t* devinfo) {
fbl::AutoLock lock(&lock_);
memcpy(&devinfo_, devinfo, sizeof(devinfo_));
}
zx_status_t Port::Queue(sata_txn_t* txn) {
fbl::AutoLock lock(&lock_);
if (!is_valid()) {
return ZX_ERR_BAD_STATE;
}
// reset the physical address
txn->pmt = ZX_HANDLE_INVALID;
// put the cmd on the queue
list_add_tail(&txn_list_, &txn->node);
return ZX_OK;
}
bool Port::Complete() {
fbl::AutoLock lock(&lock_);
if (!is_valid()) {
return false;
}
sata_txn_t* txn_complete[AHCI_MAX_COMMANDS];
size_t complete_count = 0;
bool active_txns = false;
for (uint32_t slot = 0; slot < AHCI_MAX_COMMANDS; slot++) {
sata_txn_t* txn = commands_[slot];
if (txn == nullptr) {
continue; // No transaction in this slot.
}
uint32_t slot_bit = (1u << slot);
if ((completed_ & slot_bit) == 0) {
// Not complete, check if timeout expired.
zx::time now = zx::clock::get_monotonic();
if (txn->timeout > now) {
active_txns = true;
continue; // Still in progress.
}
// Timed out.
zx::duration delta = now - txn->timeout;
zxlogf(ERROR, "ahci: txn time out on port %d txn %p (%ld ms)", num_, txn, delta.to_msecs());
txn->timeout = zx::time::infinite_past(); // Signal that timeout occured.
}
// Completed or timed out.
commands_[slot] = nullptr;
running_ &= ~slot_bit;
completed_ &= ~slot_bit;
txn_complete[complete_count] = txn;
complete_count++;
}
sata_txn_t* sync_op = nullptr;
// resume the port if paused for sync and no outstanding transactions
if ((is_paused()) && !running_) {
flags_ &= ~kPortFlagSyncPaused;
if (sync_) {
sync_op = sync_;
sync_ = nullptr;
}
}
lock.release();
for (size_t i = 0; i < complete_count; i++) {
sata_txn_t* txn = txn_complete[i];
if (txn->pmt != ZX_HANDLE_INVALID) {
zx_pmt_unpin(txn->pmt);
}
if (txn->timeout == zx::time::infinite_past()) {
block_complete(txn, ZX_ERR_TIMED_OUT);
} else {
zxlogf(TRACE, "ahci.%u: complete txn %p", num_, txn);
block_complete(txn, ZX_OK);
}
}
if (sync_op != nullptr) {
block_complete(sync_op, ZX_OK);
}
return active_txns;
}
bool Port::ProcessQueued() {
lock_.Acquire();
if ((!is_valid()) || is_paused()) {
lock_.Release();
return false;
}
bool added_txns = false;
for (;;) {
sata_txn_t* txn = list_peek_head_type(&txn_list_, sata_txn_t, node);
if (!txn) {
break;
}
// find a free command tag
uint32_t max = std::min(devinfo_.max_cmd, MaxCommands());
uint32_t i = 0;
for (i = 0; i <= max; i++) {
if (!SlotBusyLocked(i))
break;
}
if (i > max) {
break;
}
list_delete(&txn->node);
if (BLOCK_OP(txn->bop.command) == BLOCK_OP_FLUSH) {
if (running_) {
ZX_DEBUG_ASSERT(sync_ == nullptr);
// pause the port if FLUSH command
flags_ |= kPortFlagSyncPaused;
sync_ = txn;
added_txns = true;
} else {
// complete immediately if nothing in flight
lock_.Release();
block_complete(txn, ZX_OK);
lock_.Acquire();
}
} else {
// run the transaction
zx_status_t st = TxnBeginLocked(i, txn);
// complete the transaction with if it failed during processing
if (st != ZX_OK) {
lock_.Release();
block_complete(txn, st);
lock_.Acquire();
continue;
}
added_txns = true;
}
}
lock_.Release();
return added_txns;
}
void Port::TxnComplete(zx_status_t status) {
fbl::AutoLock lock(&lock_);
uint32_t active = RegRead(kPortSataActive); // Transactions active in hardware.
uint32_t running = running_; // Transactions tagged as running.
// Transactions active in hardware but not tagged as running.
uint32_t unaccounted = active & ~running;
// Remove transactions that have been completed by the watchdog.
unaccounted &= ~completed_;
// assert if a command slot without an outstanding transaction is active.
ZX_DEBUG_ASSERT(unaccounted == 0);
// Transactions tagged as running but completed by hardware.
uint32_t done = running & ~active;
completed_ |= done;
}
zx_status_t Port::TxnBeginLocked(uint32_t slot, sata_txn_t* txn) {
ZX_DEBUG_ASSERT(slot < AHCI_MAX_COMMANDS);
ZX_DEBUG_ASSERT(!SlotBusyLocked(slot));
uint64_t offset_vmo = txn->bop.rw.offset_vmo * devinfo_.block_size;
uint64_t bytes = txn->bop.rw.length * devinfo_.block_size;
size_t pagecount = ((offset_vmo & (PAGE_SIZE - 1)) + bytes + (PAGE_SIZE - 1)) / PAGE_SIZE;
zx_paddr_t pages[AHCI_MAX_PAGES];
if (pagecount > AHCI_MAX_PAGES) {
zxlogf(TRACE, "ahci.%u: txn %p too many pages (%zu)", num_, txn, pagecount);
return ZX_ERR_INVALID_ARGS;
}
zx::unowned_vmo vmo(txn->bop.rw.vmo);
bool is_write = cmd_is_write(txn->cmd);
uint32_t options = is_write ? ZX_BTI_PERM_READ : ZX_BTI_PERM_WRITE;
zx::pmt pmt;
zx_status_t st = bus_->BtiPin(options, vmo, offset_vmo & ~PAGE_MASK, pagecount * PAGE_SIZE, pages,
pagecount, &pmt);
if (st != ZX_OK) {
zxlogf(TRACE, "ahci.%u: failed to pin pages, err = %d", num_, st);
return st;
}
txn->pmt = pmt.release();
phys_iter_buffer_t physbuf = {};
physbuf.phys = pages;
physbuf.phys_count = pagecount;
physbuf.length = bytes;
physbuf.vmo_offset = offset_vmo;
phys_iter_t iter;
phys_iter_init(&iter, &physbuf, AHCI_PRD_MAX_SIZE);
uint8_t cmd = txn->cmd;
uint8_t device = txn->device;
uint64_t lba = txn->bop.rw.offset_dev;
uint64_t count = txn->bop.rw.length;
// use queued command if available
if (HasCommandQueue()) {
if (cmd == SATA_CMD_READ_DMA_EXT) {
cmd = SATA_CMD_READ_FPDMA_QUEUED;
} else if (cmd == SATA_CMD_WRITE_DMA_EXT) {
cmd = SATA_CMD_WRITE_FPDMA_QUEUED;
}
}
// build the command
ahci_cl_t* cl = &mem_->cl[slot];
// don't clear the cl since we set up ctba/ctbau at init
cl->prdtl_flags_cfl = 0;
cl->cfl = 5; // 20 bytes
cl->w = is_write ? 1 : 0;
cl->prdbc = 0;
memset(&mem_->tab[slot].ct, 0, sizeof(ahci_ct_t));
uint8_t* cfis = mem_->tab[slot].ct.cfis;
cfis[0] = 0x27; // host-to-device
cfis[1] = 0x80; // command
cfis[2] = cmd;
cfis[7] = device;
// some commands have lba/count fields
if (cmd == SATA_CMD_READ_DMA_EXT || cmd == SATA_CMD_WRITE_DMA_EXT) {
cfis[4] = lba & 0xff;
cfis[5] = (lba >> 8) & 0xff;
cfis[6] = (lba >> 16) & 0xff;
cfis[8] = (lba >> 24) & 0xff;
cfis[9] = (lba >> 32) & 0xff;
cfis[10] = (lba >> 40) & 0xff;
cfis[12] = count & 0xff;
cfis[13] = (count >> 8) & 0xff;
} else if (cmd_is_queued(cmd)) {
cfis[4] = lba & 0xff;
cfis[5] = (lba >> 8) & 0xff;
cfis[6] = (lba >> 16) & 0xff;
cfis[8] = (lba >> 24) & 0xff;
cfis[9] = (lba >> 32) & 0xff;
cfis[10] = (lba >> 40) & 0xff;
cfis[3] = count & 0xff;
cfis[11] = (count >> 8) & 0xff;
cfis[12] = (slot << 3) & 0xff; // tag
cfis[13] = 0; // normal priority
}
cl->prdtl = 0;
size_t length;
zx_paddr_t paddr;
for (uint32_t i = 0; i < AHCI_MAX_PRDS; i++) {
length = phys_iter_next(&iter, &paddr);
if (length == 0) {
break;
} else if (length > AHCI_PRD_MAX_SIZE) {
zxlogf(ERROR, "ahci.%u: chunk size > %zu is unsupported", num_, length);
return ZX_ERR_NOT_SUPPORTED;
} else if (cl->prdtl == AHCI_MAX_PRDS) {
zxlogf(ERROR, "ahci.%u: txn with more than %d chunks is unsupported", num_, cl->prdtl);
return ZX_ERR_NOT_SUPPORTED;
}
ahci_prd_t* prd = &mem_->tab[slot].prd[i];
prd->dba = lo32(paddr);
prd->dbau = hi32(paddr);
prd->dbc = ((length - 1) & (AHCI_PRD_MAX_SIZE - 1)); // 0-based byte count
cl->prdtl++;
}
running_ |= (1u << slot);
commands_[slot] = txn;
zxlogf(TRACE,
"ahci.%u: do_txn txn %p (%c) offset 0x%" PRIx64 " length 0x%" PRIx64 " slot %d prdtl %u\n",
num_, txn, cl->w ? 'w' : 'r', lba, count, slot, cl->prdtl);
if (zxlog_level_enabled(TRACE)) {
for (uint32_t i = 0; i < cl->prdtl; i++) {
ahci_prd_t* prd = &mem_->tab[slot].prd[i];
zxlogf(TRACE, "%04u: dbau=0x%08x dba=0x%08x dbc=0x%x", i, prd->dbau, prd->dba, prd->dbc);
}
}
// start command
if (cmd_is_queued(cmd)) {
RegWrite(kPortSataActive, (1u << slot));
}
RegWrite(kPortCommandIssue, (1u << slot));
txn->timeout = zx::clock::get_monotonic() + kTransactionTimeout;
return ZX_OK;
}
// HandleIrq does not lock the port or check whether it is valid.
// It is assumed that an invalid port can not have scheduled operations that trigger an interrupt.
// The port may have been disabled or unplugged, but is still valid.
bool Port::HandleIrq() {
// Clear interrupt status.
uint32_t int_status = RegRead(kPortInterruptStatus);
RegWrite(kPortInterruptStatus, int_status);
if (int_status & AHCI_PORT_INT_PRC) { // PhyRdy change
uint32_t serr = RegRead(kPortSataError);
RegWrite(kPortSataError, serr & ~0x1);
}
if (int_status & AHCI_PORT_INT_ERROR) { // error
zxlogf(ERROR, "ahci.%u: error is=0x%08x", num_, int_status);
TxnComplete(ZX_ERR_INTERNAL);
return true;
} else if (int_status) {
TxnComplete(ZX_OK);
return true;
}
return false;
}
// Set up the running state for testing Complete()
void Port::TestSetRunning(sata_txn_t* txn, uint32_t slot) {
ZX_DEBUG_ASSERT(slot < AHCI_MAX_COMMANDS);
commands_[slot] = txn;
running_ |= (1u << slot);
completed_ &= ~(1u << slot);
}
} // namespace ahci