| // Copyright 2016 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include <assert.h> |
| #include <ddk/iotxn.h> |
| #include <ddk/device.h> |
| #include <zircon/assert.h> |
| #include <zircon/process.h> |
| #include <zircon/syscalls.h> |
| #include <sys/param.h> |
| #include <stdatomic.h> |
| #include <stdlib.h> |
| #include <stdio.h> |
| #include <string.h> |
| #include <inttypes.h> |
| #include <threads.h> |
| |
| #define TRACE 0 |
| |
| #if TRACE |
| #define xprintf(fmt...) printf(fmt) |
| #else |
| #define xprintf(fmt...) \ |
| do { \ |
| } while (0) |
| #endif |
| |
| // Warn if free list length exceeds a multiple of FREE_LIST_MONITOR_LIMIT |
| #define FREE_LIST_MONITOR_LIMIT 100 |
| |
| #define IOTXN_PFLAG_CONTIGUOUS (1 << 0) // the vmo is contiguous |
| #define IOTXN_PFLAG_ALLOC (1 << 1) // the vmo is allocated by us |
| #define IOTXN_PFLAG_PHYSMAP (1 << 2) // we performed physmap() on this vmo and allocated memory |
| #define IOTXN_PFLAG_MMAP (1 << 3) // we performed mmap() on this vmo |
| #define IOTXN_PFLAG_FREE (1 << 4) // this txn has been released |
| #define IOTXN_PFLAG_QUEUED (1 << 5) // transaction has been queued and not yet released |
| |
| #define IOTXN_STATE_MASK (IOTXN_PFLAG_FREE | IOTXN_PFLAG_QUEUED) |
| |
| static list_node_t free_list = LIST_INITIAL_VALUE(free_list); |
| static mtx_t free_list_mutex = MTX_INIT; |
| #if FREE_LIST_MONITOR_LIMIT |
| static size_t free_list_length = 0; |
| static size_t free_list_monitor_warned = 0; |
| #endif |
| |
| // This assert will fail if we attempt to access the buffer of a cloned txn after it has been completed |
| #define ASSERT_BUFFER_VALID(priv) ZX_DEBUG_ASSERT(!(priv->flags & IOTXN_FLAG_DEAD)) |
| |
| static uint32_t alloc_flags_to_pflags(uint32_t alloc_flags) { |
| if (alloc_flags & IOTXN_ALLOC_CONTIGUOUS) { |
| return IOTXN_PFLAG_CONTIGUOUS; |
| } |
| return 0; |
| } |
| |
| static bool do_free_phys(uint32_t pflags) { |
| // only free phys if we called physmap and allocated memory |
| return (pflags & IOTXN_PFLAG_PHYSMAP); |
| } |
| |
| static iotxn_t* find_in_free_list(uint32_t pflags, uint64_t data_size) { |
| bool found = false; |
| iotxn_t* txn = NULL; |
| //xprintf("find_in_free_list pflags 0x%x data_size 0x%" PRIx64 "\n", pflags, data_size); |
| mtx_lock(&free_list_mutex); |
| list_for_every_entry (&free_list, txn, iotxn_t, node) { |
| // txn->pflags has IOTXN_ALLOC_CONTIGUOUS set if the txn has a contiguous VMO we allocated, |
| // or zero otherwise. And the pflags passed into this function is either zero or |
| // IOTXN_ALLOC_CONTIGUOUS. So here we mask txn->pflags with IOTXN_ALLOC_CONTIGUOUS |
| // to compare just this bit and not get confused by IOTXN_PFLAG_FREE or other flags. |
| if ((txn->vmo_length == data_size) && (((txn->pflags & IOTXN_ALLOC_CONTIGUOUS) == pflags) || data_size == 0)) { |
| found = true; |
| break; |
| } |
| } |
| if (found) { |
| txn->pflags &= ~IOTXN_PFLAG_FREE; |
| list_delete(&txn->node); |
| #if FREE_LIST_MONITOR_LIMIT |
| free_list_length--; |
| #endif |
| } |
| mtx_unlock(&free_list_mutex); |
| //xprintf("find_in_free_list found %d txn %p\n", found, txn); |
| return found ? txn : NULL; |
| } |
| |
| // return the iotxn into the free list |
| static void iotxn_release_free_list(iotxn_t* txn) { |
| zx_handle_t vmo_handle = txn->vmo_handle; |
| uint64_t vmo_offset = txn->vmo_offset; |
| uint64_t vmo_length = txn->vmo_length; |
| void* virt = txn->virt; |
| zx_paddr_t* phys = txn->phys; |
| uint64_t phys_count = txn->phys_count; |
| uint32_t pflags = txn->pflags; |
| zx_paddr_t phys_inline[3]; |
| memcpy(phys_inline, txn->phys_inline, sizeof(txn->phys_inline)); |
| |
| memset(txn, 0, sizeof(iotxn_t)); |
| |
| if (pflags & IOTXN_PFLAG_ALLOC) { |
| // if we allocated the vmo, keep it around |
| txn->vmo_handle = vmo_handle; |
| txn->vmo_offset = vmo_offset; |
| txn->vmo_length = vmo_length; |
| txn->virt = virt; |
| txn->phys = phys; |
| txn->phys_count = phys_count; |
| txn->pflags = pflags; |
| memcpy(txn->phys_inline, phys_inline, sizeof(txn->phys_inline)); |
| } else { |
| if (do_free_phys(pflags)) { |
| if (phys != NULL) { |
| free(phys); |
| } |
| } |
| if (pflags & IOTXN_PFLAG_MMAP) { |
| // only unmap if we called mmap() |
| if (virt) { |
| zx_vmar_unmap(zx_vmar_root_self(), (uintptr_t)virt, vmo_length); |
| } |
| } |
| } |
| |
| txn->pflags |= IOTXN_PFLAG_FREE; |
| txn->release_cb = iotxn_release_free_list; |
| |
| mtx_lock(&free_list_mutex); |
| list_add_head(&free_list, &txn->node); |
| #if FREE_LIST_MONITOR_LIMIT |
| free_list_length++; |
| if (free_list_length % FREE_LIST_MONITOR_LIMIT == 0 |
| && free_list_length > free_list_monitor_warned) { |
| printf("WARNING: iotxn free_list_length is %zu\n", free_list_length); |
| free_list_monitor_warned = free_list_length; |
| } |
| #endif |
| mtx_unlock(&free_list_mutex); |
| |
| xprintf("iotxn_release_free_list released txn %p\n", txn); |
| } |
| |
| // free the iotxn |
| static void iotxn_release_free(iotxn_t* txn) { |
| if (do_free_phys(txn->pflags)) { |
| if (txn->phys != NULL) { |
| free(txn->phys); |
| } |
| } |
| if (txn->pflags & IOTXN_PFLAG_MMAP) { |
| if (txn->virt) { |
| zx_vmar_unmap(zx_vmar_root_self(), (uintptr_t)txn->virt, txn->vmo_length); |
| } |
| } |
| if (txn->pflags & IOTXN_PFLAG_ALLOC) { |
| zx_handle_close(txn->vmo_handle); |
| } |
| free(txn); |
| } |
| |
| // releases data for a statically allocated iotxn |
| static void iotxn_release_static(iotxn_t* txn) { |
| uint32_t pflags = txn->pflags; |
| |
| if (do_free_phys(txn->pflags)) { |
| // only free the scatter list if we called physmap() |
| if (txn->phys != NULL) { |
| free(txn->phys); |
| txn->phys = NULL; |
| txn->phys_count = 0; |
| } |
| } |
| if (pflags & IOTXN_PFLAG_MMAP) { |
| // only unmap if we called mmap() |
| if (txn->virt) { |
| zx_vmar_unmap(zx_vmar_root_self(), (uintptr_t)txn->virt, txn->vmo_length); |
| txn->virt = NULL; |
| } |
| } |
| } |
| |
| void iotxn_complete(iotxn_t* txn, zx_status_t status, zx_off_t actual) { |
| xprintf("iotxn_complete txn %p\n", txn); |
| |
| ZX_DEBUG_ASSERT((txn->pflags & IOTXN_STATE_MASK) == IOTXN_PFLAG_QUEUED); |
| txn->pflags &= ~IOTXN_PFLAG_QUEUED; |
| |
| txn->actual = actual; |
| txn->status = status; |
| if (txn->complete_cb) { |
| txn->complete_cb(txn, txn->cookie); |
| } |
| } |
| |
| ssize_t iotxn_copyfrom(iotxn_t* txn, void* data, size_t length, size_t offset) { |
| length = MIN(txn->vmo_length - offset, length); |
| size_t actual; |
| zx_status_t status = zx_vmo_read(txn->vmo_handle, data, txn->vmo_offset + offset, length, &actual); |
| xprintf("iotxn_copyfrom: txn %p vmo_offset 0x%" PRIx64 " offset 0x%zx length 0x%zx actual 0x%zx status %d\n", txn, txn->vmo_offset, offset, length, actual, status); |
| return (status == ZX_OK) ? (ssize_t)actual : status; |
| } |
| |
| ssize_t iotxn_copyto(iotxn_t* txn, const void* data, size_t length, size_t offset) { |
| length = MIN(txn->vmo_length - offset, length); |
| size_t actual; |
| zx_status_t status = zx_vmo_write(txn->vmo_handle, data, txn->vmo_offset + offset, length, &actual); |
| xprintf("iotxn_copyto: txn %p vmo_offset 0x%" PRIx64 " offset 0x%zx length 0x%zx actual 0x%zx status %d\n", txn, txn->vmo_offset, offset, length, actual, status); |
| return (status == ZX_OK) ? (ssize_t)actual : status; |
| } |
| |
| static zx_status_t iotxn_physmap_contiguous(iotxn_t* txn) { |
| txn->phys = txn->phys_inline; |
| |
| // for contiguous buffers, commit the whole range but just map the first |
| // page |
| uint64_t page_offset = ROUNDDOWN(txn->vmo_offset, PAGE_SIZE); |
| zx_status_t status = zx_vmo_op_range(txn->vmo_handle, ZX_VMO_OP_COMMIT, page_offset, txn->vmo_length, NULL, 0); |
| if (status != ZX_OK) { |
| goto fail; |
| } |
| |
| status = zx_vmo_op_range(txn->vmo_handle, ZX_VMO_OP_LOOKUP, page_offset, PAGE_SIZE, txn->phys, sizeof(zx_paddr_t)); |
| if (status != ZX_OK) { |
| goto fail; |
| } |
| |
| txn->phys_count = 1; |
| return ZX_OK; |
| fail: |
| txn->phys = NULL; |
| return status; |
| } |
| |
| static zx_status_t iotxn_physmap_paged(iotxn_t* txn) { |
| // ZX_VMO_OP_LOOKUP returns whole pages, so take into account unaligned vmo |
| // offset and length when calculating the amount of pages returned |
| uint64_t page_offset = ROUNDDOWN(txn->vmo_offset, PAGE_SIZE); |
| uint64_t page_length = txn->vmo_length + (txn->vmo_offset - page_offset); |
| uint64_t pages = ROUNDUP(page_length, PAGE_SIZE) / PAGE_SIZE; |
| |
| bool use_inline = pages <= 3; |
| zx_paddr_t* paddrs = use_inline ? txn->phys_inline : malloc(sizeof(zx_paddr_t) * pages); |
| if (paddrs == NULL) { |
| xprintf("iotxn_physmap_paged: out of memory\n"); |
| return ZX_ERR_NO_MEMORY; |
| } |
| |
| // commit pages and lookup physical addresses |
| // assume that commited pages will never be auto-decommitted |
| zx_status_t status = zx_vmo_op_range(txn->vmo_handle, ZX_VMO_OP_COMMIT, txn->vmo_offset, txn->vmo_length, NULL, 0); |
| if (status != ZX_OK) { |
| xprintf("iotxn_physmap_paged: error %d in commit\n", status); |
| if (!use_inline) { |
| free(paddrs); |
| } |
| return status; |
| } |
| |
| status = zx_vmo_op_range(txn->vmo_handle, ZX_VMO_OP_LOOKUP, page_offset, page_length, paddrs, sizeof(zx_paddr_t) * pages); |
| if (status != ZX_OK) { |
| xprintf("iotxn_physmap_paged: error %d in lookup\n", status); |
| if (!use_inline) { |
| free(paddrs); |
| } |
| return status; |
| } |
| |
| if (!use_inline) { |
| txn->pflags |= IOTXN_PFLAG_PHYSMAP; |
| } |
| txn->phys = paddrs; |
| txn->phys_count = pages; |
| return ZX_OK; |
| } |
| |
| zx_status_t iotxn_physmap(iotxn_t* txn) { |
| if (txn->phys_count > 0) { |
| return ZX_OK; |
| } |
| if (txn->vmo_length == 0) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| zx_status_t status; |
| if (txn->pflags & IOTXN_PFLAG_CONTIGUOUS) { |
| status = iotxn_physmap_contiguous(txn); |
| } else { |
| status = iotxn_physmap_paged(txn); |
| } |
| return status; |
| } |
| |
| zx_status_t iotxn_mmap(iotxn_t* txn, void** data) { |
| xprintf("iotxn_mmap: txn %p\n", txn); |
| if (txn->virt) { |
| *data = txn->virt; |
| return ZX_OK; |
| } |
| zx_status_t status = zx_vmar_map(zx_vmar_root_self(), 0, txn->vmo_handle, txn->vmo_offset, txn->vmo_length, ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE, (uintptr_t*)(&txn->virt)); |
| if (status == ZX_OK) { |
| txn->pflags |= IOTXN_PFLAG_MMAP; |
| *data = txn->virt; |
| } |
| return status; |
| } |
| |
| zx_status_t iotxn_clone(iotxn_t* txn, iotxn_t** out) { |
| xprintf("iotxn_clone txn %p\n", txn); |
| iotxn_t* clone = NULL; |
| if (*out != NULL) { |
| clone = *out; |
| } else { |
| clone = find_in_free_list(0, 0); |
| if (clone == NULL) { |
| clone = calloc(1, sizeof(iotxn_t)); |
| if (clone == NULL) { |
| return ZX_ERR_NO_MEMORY; |
| } |
| } |
| } |
| |
| memcpy(clone, txn, sizeof(iotxn_t)); |
| // the only relevant pflag for a clone is the contiguous bit |
| clone->pflags = txn->pflags & IOTXN_PFLAG_CONTIGUOUS; |
| clone->complete_cb = NULL; |
| // clones are always freelisted on release |
| clone->release_cb = iotxn_release_free_list; |
| |
| *out = clone; |
| return ZX_OK; |
| } |
| |
| zx_status_t iotxn_clone_partial(iotxn_t* txn, uint64_t vmo_offset, zx_off_t length, iotxn_t** out) { |
| xprintf("iotxn_clone_partial txn %p\n", txn); |
| if (vmo_offset < txn->vmo_offset) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| if (length > txn->length) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| if ((vmo_offset - txn->vmo_offset) > (txn->length - length)) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| zx_status_t status = iotxn_clone(txn, out); |
| if (status != ZX_OK) { |
| return status; |
| } |
| |
| iotxn_t* clone = *out; |
| uint64_t offd = vmo_offset - txn->vmo_offset; |
| clone->vmo_offset = vmo_offset; |
| clone->vmo_length -= offd; |
| clone->length = length; |
| if (clone->virt) { |
| clone->virt += offd; |
| } |
| if (clone->phys) { |
| uint64_t page_offset = ROUNDDOWN(txn->vmo_offset, PAGE_SIZE); |
| uint64_t new_page_offset = ROUNDDOWN(clone->vmo_offset, PAGE_SIZE); |
| if (page_offset != new_page_offset) { |
| if (txn->pflags & IOTXN_PFLAG_CONTIGUOUS) { |
| clone->phys = clone->phys_inline; |
| clone->phys[0] = txn->phys[0] + new_page_offset - page_offset; |
| } else { |
| uint64_t pages = (new_page_offset - page_offset) / PAGE_SIZE; |
| if (pages >= clone->phys_count) { |
| iotxn_release(clone); |
| return ZX_ERR_INVALID_ARGS; |
| } |
| clone->phys += pages; |
| clone->phys_count -= pages; |
| ZX_DEBUG_ASSERT(clone->phys_count > 0); |
| } |
| } |
| } |
| ZX_DEBUG_ASSERT(clone->release_cb == iotxn_release_free_list); |
| return ZX_OK; |
| } |
| |
| void iotxn_release(iotxn_t* txn) { |
| // should not release a queued transaction |
| ZX_DEBUG_ASSERT((txn->pflags & IOTXN_STATE_MASK) == 0); |
| |
| if (txn->release_cb) { |
| txn->release_cb(txn); |
| } |
| } |
| |
| void iotxn_cacheop(iotxn_t* txn, uint32_t op, size_t offset, size_t length) { |
| #ifdef __aarch64__ |
| // Bail out if the syscall has nothing to do. |
| if (length == 0 || txn->vmo_length == 0) |
| return; |
| |
| zx_vmo_op_range(txn->vmo_handle, op, txn->vmo_offset + offset, length, NULL, 0); |
| #endif |
| } |
| |
| zx_status_t iotxn_alloc(iotxn_t** out, uint32_t alloc_flags, uint64_t data_size) { |
| //xprintf("iotxn_alloc: alloc_flags 0x%x data_size 0x%" PRIx64 "\n", alloc_flags, data_size); |
| |
| // look in free list first for a iotxn with data_size |
| iotxn_t* txn = find_in_free_list(alloc_flags_to_pflags(alloc_flags), data_size); |
| if (txn != NULL) { |
| //xprintf("iotxn_alloc: found iotxn with size 0x%" PRIx64 " in free list\n", data_size); |
| goto out; |
| } |
| |
| // didn't find one that fits, allocate a new one |
| txn = calloc(1, sizeof(iotxn_t)); |
| if (!txn) { |
| return ZX_ERR_NO_MEMORY; |
| } |
| if (data_size > 0) { |
| zx_status_t status; |
| if (alloc_flags & IOTXN_ALLOC_CONTIGUOUS) { |
| status = zx_vmo_create_contiguous(get_root_resource(), data_size, 0, &txn->vmo_handle); |
| txn->pflags |= IOTXN_PFLAG_CONTIGUOUS; |
| } else { |
| status = zx_vmo_create(data_size, 0, &txn->vmo_handle); |
| } |
| zx_object_set_property(txn->vmo_handle, ZX_PROP_NAME, "iotxn", 5); |
| if (status != ZX_OK) { |
| xprintf("iotxn_alloc: error %d in zx_vmo_create, flags 0x%x\n", status, alloc_flags); |
| free(txn); |
| return status; |
| } |
| txn->vmo_offset = 0; |
| txn->vmo_length = data_size; |
| txn->pflags |= IOTXN_PFLAG_ALLOC; |
| } |
| |
| out: |
| ZX_DEBUG_ASSERT(txn != NULL); |
| ZX_DEBUG_ASSERT(!(txn->pflags & IOTXN_PFLAG_FREE)); |
| if (alloc_flags & IOTXN_ALLOC_POOL) { |
| txn->release_cb = iotxn_release_free_list; |
| } else { |
| txn->release_cb = iotxn_release_free; |
| } |
| *out = txn; |
| return ZX_OK; |
| } |
| |
| void iotxn_queue(zx_device_t* dev, iotxn_t* txn) { |
| // don't assert not queued here, since iotxns are allowed to be requeued |
| txn->pflags |= IOTXN_PFLAG_QUEUED; |
| |
| // This can only fail if iotxn_queue() is not implemented by the |
| // device, in which case we fall back to calling the read or write op |
| if (device_iotxn_queue(dev, txn) != ZX_OK) { |
| zx_status_t status; |
| size_t actual = 0; |
| void* buf; |
| iotxn_mmap(txn, &buf); |
| if (txn->opcode == IOTXN_OP_READ) { |
| status = device_read(dev, buf, txn->length, txn->offset, &actual); |
| } else if (txn->opcode == IOTXN_OP_WRITE) { |
| status = device_write(dev, buf, txn->length, txn->offset, &actual); |
| } else { |
| status = ZX_ERR_NOT_SUPPORTED; |
| } |
| iotxn_complete(txn, status, actual); |
| } |
| } |
| |
| void iotxn_init(iotxn_t* txn, zx_handle_t vmo_handle, uint64_t vmo_offset, uint64_t length) { |
| memset(txn, 0, sizeof(*txn)); |
| txn->vmo_handle = vmo_handle; |
| txn->vmo_offset = vmo_offset; |
| txn->vmo_length = length; |
| txn->length = length; |
| txn->release_cb = iotxn_release_static; |
| } |
| |
| zx_status_t iotxn_alloc_vmo(iotxn_t** out, uint32_t alloc_flags, zx_handle_t vmo_handle, |
| uint64_t vmo_offset, uint64_t length) { |
| iotxn_t* txn; |
| zx_status_t status = iotxn_alloc(&txn, alloc_flags, 0); |
| if (status != ZX_OK) { |
| return status; |
| } |
| txn->vmo_handle = vmo_handle; |
| txn->vmo_offset = vmo_offset; |
| txn->vmo_length = length; |
| txn->length = length; |
| *out = txn; |
| return ZX_OK; |
| } |
| |
| void iotxn_phys_iter_init(iotxn_phys_iter_t* iter, iotxn_t* txn, size_t max_length) { |
| phys_iter_buffer_t buf = { |
| .length = txn->length, |
| .vmo_offset = txn->vmo_offset, |
| .phys = txn->phys, |
| .phys_count = txn->phys_count |
| }; |
| phys_iter_init(iter, &buf, max_length); |
| } |
| |
| size_t iotxn_phys_iter_next(iotxn_phys_iter_t* iter, zx_paddr_t* out_paddr) { |
| return phys_iter_next(iter, out_paddr); |
| } |