| // Copyright 2016 The Fuchsia Authors |
| // Copyright (c) 2014 Google Inc. All rights reserved |
| // |
| // Use of this source code is governed by a MIT-style |
| // license that can be found in the LICENSE file or at |
| // https://opensource.org/licenses/MIT |
| #include "arch/arm64/mmu.h" |
| |
| #include <align.h> |
| #include <assert.h> |
| #include <bits.h> |
| #include <debug.h> |
| #include <inttypes.h> |
| #include <lib/arch/intrin.h> |
| #include <lib/counters.h> |
| #include <lib/fit/defer.h> |
| #include <lib/heap.h> |
| #include <lib/instrumentation/asan.h> |
| #include <lib/ktrace.h> |
| #include <lib/lazy_init/lazy_init.h> |
| #include <stdlib.h> |
| #include <string.h> |
| #include <sys/types.h> |
| #include <trace.h> |
| #include <zircon/errors.h> |
| #include <zircon/types.h> |
| |
| #include <arch/arm64/hypervisor/el2_state.h> |
| #include <arch/aspace.h> |
| #include <fbl/auto_lock.h> |
| #include <fbl/bits.h> |
| #include <kernel/auto_preempt_disabler.h> |
| #include <kernel/mutex.h> |
| #include <ktl/algorithm.h> |
| #include <vm/arch_vm_aspace.h> |
| #include <vm/physmap.h> |
| #include <vm/pmm.h> |
| #include <vm/vm.h> |
| |
| #include "asid_allocator.h" |
| |
| #include <ktl/enforce.h> |
| |
| #define LOCAL_TRACE 0 |
| #define TRACE_CONTEXT_SWITCH 0 |
| |
| /* ktraces just local to this file */ |
| #define LOCAL_KTRACE_ENABLE 0 |
| |
| #define LOCAL_KTRACE(string, args...) \ |
| ktrace_probe(LocalTrace<LOCAL_KTRACE_ENABLE>, TraceContext::Cpu, KTRACE_STRING_REF(string), \ |
| ##args) |
| |
| using LocalTraceDuration = |
| TraceDuration<TraceEnabled<LOCAL_KTRACE_ENABLE>, KTRACE_GRP_VM, TraceContext::Thread>; |
| |
| // Use one of the ignored bits for a software simulated accessed flag for non-terminal entries. |
| // TODO: Once the hardware setting of the terminal AF is supported usage of this for non-terminal AF |
| // will have to become optional as we rely on the software terminal fault to set the non-terminal |
| // bits. |
| #define MMU_PTE_ATTR_RES_SOFTWARE_AF BM(55, 1, 1) |
| // Ensure we picked a bit that is actually part of the software controlled bits. |
| static_assert((MMU_PTE_ATTR_RES_SOFTWARE & MMU_PTE_ATTR_RES_SOFTWARE_AF) == |
| MMU_PTE_ATTR_RES_SOFTWARE_AF); |
| |
| static_assert(((long)KERNEL_BASE >> MMU_KERNEL_SIZE_SHIFT) == -1, ""); |
| static_assert(((long)KERNEL_ASPACE_BASE >> MMU_KERNEL_SIZE_SHIFT) == -1, ""); |
| static_assert(MMU_KERNEL_SIZE_SHIFT <= 48, ""); |
| static_assert(MMU_KERNEL_SIZE_SHIFT >= 25, ""); |
| |
| // Static relocated base to prepare for KASLR. Used at early boot and by gdb |
| // script to know the target relocated address. |
| // TODO(fxbug.dev/24762): Choose it randomly. |
| #if DISABLE_KASLR |
| uint64_t kernel_relocated_base = KERNEL_BASE; |
| #else |
| uint64_t kernel_relocated_base = 0xffffffff10000000; |
| #endif |
| |
| // The main translation table for the kernel. Globally declared because it's reached |
| // from assembly. |
| pte_t arm64_kernel_translation_table[MMU_KERNEL_PAGE_TABLE_ENTRIES_TOP] __ALIGNED( |
| MMU_KERNEL_PAGE_TABLE_ENTRIES_TOP * 8); |
| // Physical address of the above table, saved in start.S. |
| paddr_t arm64_kernel_translation_table_phys; |
| |
| // Global accessor for the kernel page table |
| pte_t* arm64_get_kernel_ptable() { return arm64_kernel_translation_table; } |
| |
| namespace { |
| |
| KCOUNTER(cm_flush_all, "mmu.consistency_manager.flush_all") |
| KCOUNTER(cm_flush_all_replacing, "mmu.consistency_manager.flush_all_replacing") |
| KCOUNTER(cm_single_tlb_invalidates, "mmu.consistency_manager.single_tlb_invalidate") |
| KCOUNTER(cm_flush, "mmu.consistency_manager.flush") |
| |
| lazy_init::LazyInit<AsidAllocator> asid; |
| |
| KCOUNTER(vm_mmu_protect_make_execute_calls, "vm.mmu.protect.make_execute_calls") |
| KCOUNTER(vm_mmu_protect_make_execute_pages, "vm.mmu.protect.make_execute_pages") |
| |
| // Convert user level mmu flags to flags that go in L1 descriptors. |
| // Hypervisor flag modifies behavior to work for single translation regimes |
| // such as the mapping of kernel pages with ArmAspaceType::kHypervisor in EL2. |
| pte_t mmu_flags_to_s1_pte_attr(uint flags, bool hypervisor = false) { |
| pte_t attr = MMU_PTE_ATTR_AF; |
| |
| switch (flags & ARCH_MMU_FLAG_CACHE_MASK) { |
| case ARCH_MMU_FLAG_CACHED: |
| attr |= MMU_PTE_ATTR_NORMAL_MEMORY | MMU_PTE_ATTR_SH_INNER_SHAREABLE; |
| break; |
| case ARCH_MMU_FLAG_WRITE_COMBINING: |
| attr |= MMU_PTE_ATTR_NORMAL_UNCACHED | MMU_PTE_ATTR_SH_INNER_SHAREABLE; |
| break; |
| case ARCH_MMU_FLAG_UNCACHED: |
| attr |= MMU_PTE_ATTR_STRONGLY_ORDERED; |
| break; |
| case ARCH_MMU_FLAG_UNCACHED_DEVICE: |
| attr |= MMU_PTE_ATTR_DEVICE; |
| break; |
| default: |
| PANIC_UNIMPLEMENTED; |
| } |
| |
| switch (flags & (ARCH_MMU_FLAG_PERM_USER | ARCH_MMU_FLAG_PERM_WRITE)) { |
| case 0: |
| attr |= MMU_PTE_ATTR_AP_P_RO_U_NA; |
| break; |
| case ARCH_MMU_FLAG_PERM_WRITE: |
| attr |= MMU_PTE_ATTR_AP_P_RW_U_NA; |
| break; |
| case ARCH_MMU_FLAG_PERM_USER: |
| attr |= MMU_PTE_ATTR_AP_P_RO_U_RO; |
| break; |
| case ARCH_MMU_FLAG_PERM_USER | ARCH_MMU_FLAG_PERM_WRITE: |
| attr |= MMU_PTE_ATTR_AP_P_RW_U_RW; |
| break; |
| } |
| |
| if (hypervisor) { |
| // For single translation regimes such as the hypervisor pages, only |
| // the XN bit applies. |
| if ((flags & ARCH_MMU_FLAG_PERM_EXECUTE) == 0) { |
| attr |= MMU_PTE_ATTR_XN; |
| } |
| } else { |
| if (flags & ARCH_MMU_FLAG_PERM_EXECUTE) { |
| if (flags & ARCH_MMU_FLAG_PERM_USER) { |
| // User executable page, marked privileged execute never. |
| attr |= MMU_PTE_ATTR_PXN; |
| } else { |
| // Privileged executable page, marked user execute never. |
| attr |= MMU_PTE_ATTR_UXN; |
| } |
| } else { |
| // All non executable pages are marked both privileged and user execute never. |
| attr |= MMU_PTE_ATTR_UXN | MMU_PTE_ATTR_PXN; |
| } |
| } |
| |
| if (flags & ARCH_MMU_FLAG_NS) { |
| attr |= MMU_PTE_ATTR_NON_SECURE; |
| } |
| |
| return attr; |
| } |
| |
| uint s1_pte_attr_to_mmu_flags(pte_t pte, bool hypervisor = false) { |
| uint mmu_flags = 0; |
| switch (pte & MMU_PTE_ATTR_ATTR_INDEX_MASK) { |
| case MMU_PTE_ATTR_STRONGLY_ORDERED: |
| mmu_flags |= ARCH_MMU_FLAG_UNCACHED; |
| break; |
| case MMU_PTE_ATTR_DEVICE: |
| mmu_flags |= ARCH_MMU_FLAG_UNCACHED_DEVICE; |
| break; |
| case MMU_PTE_ATTR_NORMAL_UNCACHED: |
| mmu_flags |= ARCH_MMU_FLAG_WRITE_COMBINING; |
| break; |
| case MMU_PTE_ATTR_NORMAL_MEMORY: |
| mmu_flags |= ARCH_MMU_FLAG_CACHED; |
| break; |
| default: |
| PANIC_UNIMPLEMENTED; |
| } |
| |
| mmu_flags |= ARCH_MMU_FLAG_PERM_READ; |
| switch (pte & MMU_PTE_ATTR_AP_MASK) { |
| case MMU_PTE_ATTR_AP_P_RW_U_NA: |
| mmu_flags |= ARCH_MMU_FLAG_PERM_WRITE; |
| break; |
| case MMU_PTE_ATTR_AP_P_RW_U_RW: |
| mmu_flags |= ARCH_MMU_FLAG_PERM_USER | ARCH_MMU_FLAG_PERM_WRITE; |
| break; |
| case MMU_PTE_ATTR_AP_P_RO_U_NA: |
| break; |
| case MMU_PTE_ATTR_AP_P_RO_U_RO: |
| mmu_flags |= ARCH_MMU_FLAG_PERM_USER; |
| break; |
| } |
| |
| if (hypervisor) { |
| // Single translation regimes such as the hypervisor only support the XN bit. |
| if ((pte & MMU_PTE_ATTR_XN) == 0) { |
| mmu_flags |= ARCH_MMU_FLAG_PERM_EXECUTE; |
| } |
| } else { |
| // Based on whether or not this is a user page, check UXN or PXN bit to determine |
| // if it's an executable page. |
| if (mmu_flags & ARCH_MMU_FLAG_PERM_USER) { |
| if ((pte & MMU_PTE_ATTR_UXN) == 0) { |
| mmu_flags |= ARCH_MMU_FLAG_PERM_EXECUTE; |
| } |
| } else if ((pte & MMU_PTE_ATTR_PXN) == 0) { |
| // Privileged page, check the PXN bit. |
| mmu_flags |= ARCH_MMU_FLAG_PERM_EXECUTE; |
| } |
| |
| // TODO: fxbug.dev/88451 |
| // Add additional asserts here that the translation table entries are correctly formed |
| // with regards to UXN and PXN bits and possibly other unhandled and/or ambiguous bits. |
| } |
| |
| if (pte & MMU_PTE_ATTR_NON_SECURE) { |
| mmu_flags |= ARCH_MMU_FLAG_NS; |
| } |
| |
| return mmu_flags; |
| } |
| |
| pte_t mmu_flags_to_s2_pte_attr(uint flags) { |
| pte_t attr = MMU_PTE_ATTR_AF; |
| |
| switch (flags & ARCH_MMU_FLAG_CACHE_MASK) { |
| case ARCH_MMU_FLAG_CACHED: |
| attr |= MMU_S2_PTE_ATTR_NORMAL_MEMORY | MMU_PTE_ATTR_SH_INNER_SHAREABLE; |
| break; |
| case ARCH_MMU_FLAG_WRITE_COMBINING: |
| attr |= MMU_S2_PTE_ATTR_NORMAL_UNCACHED | MMU_PTE_ATTR_SH_INNER_SHAREABLE; |
| break; |
| case ARCH_MMU_FLAG_UNCACHED: |
| attr |= MMU_S2_PTE_ATTR_STRONGLY_ORDERED; |
| break; |
| case ARCH_MMU_FLAG_UNCACHED_DEVICE: |
| attr |= MMU_S2_PTE_ATTR_DEVICE; |
| break; |
| default: |
| PANIC_UNIMPLEMENTED; |
| } |
| |
| if (flags & ARCH_MMU_FLAG_PERM_WRITE) { |
| attr |= MMU_S2_PTE_ATTR_S2AP_RW; |
| } else { |
| attr |= MMU_S2_PTE_ATTR_S2AP_RO; |
| } |
| if (!(flags & ARCH_MMU_FLAG_PERM_EXECUTE)) { |
| attr |= MMU_S2_PTE_ATTR_XN; |
| } |
| |
| return attr; |
| } |
| |
| uint s2_pte_attr_to_mmu_flags(pte_t pte) { |
| uint mmu_flags = 0; |
| |
| switch (pte & MMU_S2_PTE_ATTR_ATTR_INDEX_MASK) { |
| case MMU_S2_PTE_ATTR_STRONGLY_ORDERED: |
| mmu_flags |= ARCH_MMU_FLAG_UNCACHED; |
| break; |
| case MMU_S2_PTE_ATTR_DEVICE: |
| mmu_flags |= ARCH_MMU_FLAG_UNCACHED_DEVICE; |
| break; |
| case MMU_S2_PTE_ATTR_NORMAL_UNCACHED: |
| mmu_flags |= ARCH_MMU_FLAG_WRITE_COMBINING; |
| break; |
| case MMU_S2_PTE_ATTR_NORMAL_MEMORY: |
| mmu_flags |= ARCH_MMU_FLAG_CACHED; |
| break; |
| default: |
| PANIC_UNIMPLEMENTED; |
| } |
| |
| mmu_flags |= ARCH_MMU_FLAG_PERM_READ; |
| switch (pte & MMU_PTE_ATTR_AP_MASK) { |
| case MMU_S2_PTE_ATTR_S2AP_RO: |
| break; |
| case MMU_S2_PTE_ATTR_S2AP_RW: |
| mmu_flags |= ARCH_MMU_FLAG_PERM_WRITE; |
| break; |
| default: |
| PANIC_UNIMPLEMENTED; |
| } |
| |
| if (pte & MMU_S2_PTE_ATTR_XN) { |
| mmu_flags |= ARCH_MMU_FLAG_PERM_EXECUTE; |
| } |
| |
| return mmu_flags; |
| } |
| |
| bool is_pte_valid(pte_t pte) { |
| return (pte & MMU_PTE_DESCRIPTOR_MASK) != MMU_PTE_DESCRIPTOR_INVALID; |
| } |
| |
| void update_pte(volatile pte_t* pte, pte_t newval) { *pte = newval; } |
| |
| int first_used_page_table_entry(const volatile pte_t* page_table, uint page_size_shift) { |
| const unsigned int count = 1U << (page_size_shift - 3); |
| |
| for (unsigned int i = 0; i < count; i++) { |
| pte_t pte = page_table[i]; |
| if (pte != MMU_PTE_DESCRIPTOR_INVALID) { |
| return i; |
| } |
| } |
| return -1; |
| } |
| |
| bool page_table_is_clear(const volatile pte_t* page_table, uint page_size_shift) { |
| const int index = first_used_page_table_entry(page_table, page_size_shift); |
| const bool clear = index == -1; |
| if (clear) { |
| LTRACEF("page table at %p is clear\n", page_table); |
| } else { |
| LTRACEF("page_table at %p still in use, index %d is %#" PRIx64 "\n", page_table, index, |
| page_table[index]); |
| } |
| return clear; |
| } |
| |
| ArmAspaceType AspaceTypeFromFlags(uint mmu_flags) { |
| // Kernel/Guest flags are mutually exclusive. Ensure at most 1 is set. |
| DEBUG_ASSERT(((mmu_flags & ARCH_ASPACE_FLAG_KERNEL) != 0) + |
| ((mmu_flags & ARCH_ASPACE_FLAG_GUEST) != 0) <= |
| 1); |
| if (mmu_flags & ARCH_ASPACE_FLAG_KERNEL) { |
| return ArmAspaceType::kKernel; |
| } |
| if (mmu_flags & ARCH_ASPACE_FLAG_GUEST) { |
| return ArmAspaceType::kGuest; |
| } |
| return ArmAspaceType::kUser; |
| } |
| |
| ktl::string_view ArmAspaceTypeName(ArmAspaceType type) { |
| switch (type) { |
| case ArmAspaceType::kKernel: |
| return "kernel"; |
| case ArmAspaceType::kUser: |
| return "user"; |
| case ArmAspaceType::kGuest: |
| return "guest"; |
| case ArmAspaceType::kHypervisor: |
| return "hypervisor"; |
| } |
| __UNREACHABLE; |
| } |
| |
| } // namespace |
| |
| // A consistency manager that tracks TLB updates, walker syncs and free pages in an effort to |
| // minimize DSBs (by delaying and coalescing TLB invalidations) and switching to full ASID |
| // invalidations if too many TLB invalidations are requested. |
| class ArmArchVmAspace::ConsistencyManager { |
| public: |
| ConsistencyManager(ArmArchVmAspace& aspace) TA_REQ(aspace.lock_) : aspace_(aspace) {} |
| ~ConsistencyManager() { |
| Flush(); |
| if (!list_is_empty(&to_free_)) { |
| pmm_free(&to_free_); |
| } |
| } |
| |
| // Queue a TLB entry for flushing. This may get turned into a complete ASID flush. |
| void FlushEntry(vaddr_t va, bool terminal) { |
| AssertHeld(aspace_.lock_); |
| // Check we have queued too many entries already. |
| if (num_pending_tlbs_ >= kMaxPendingTlbs) { |
| // Most of the time we will now prefer to invalidate the entire ASID, the exception is if |
| // this aspace is using the global ASID. |
| if (aspace_.asid_ != MMU_ARM64_GLOBAL_ASID) { |
| // Keep counting entries so that we can track how many TLB invalidates we saved by grouping. |
| num_pending_tlbs_++; |
| return; |
| } |
| // Flush what pages we've cached up until now and reset counter to zero. |
| Flush(); |
| } |
| |
| // va must be page aligned so we can safely throw away the bottom bit. |
| DEBUG_ASSERT(IS_PAGE_ALIGNED(va)); |
| DEBUG_ASSERT(aspace_.IsValidVaddr(va)); |
| |
| pending_tlbs_[num_pending_tlbs_].terminal = terminal; |
| pending_tlbs_[num_pending_tlbs_].va_shifted = va >> 1; |
| num_pending_tlbs_++; |
| } |
| |
| // Performs any pending synchronization of TLBs and page table walkers. Includes the DSB to ensure |
| // TLB flushes have completed prior to returning to user. |
| void Flush() TA_REQ(aspace_.lock_) { |
| cm_flush.Add(1); |
| if (num_pending_tlbs_ == 0) { |
| return; |
| } |
| // Need a DSB to synchronize any page table updates prior to flushing the TLBs. |
| __dsb(ARM_MB_ISH); |
| |
| // Check if we should just be performing a full ASID invalidation. |
| if (num_pending_tlbs_ > kMaxPendingTlbs || aspace_.type_ == ArmAspaceType::kHypervisor) { |
| cm_flush_all.Add(1); |
| cm_flush_all_replacing.Add(num_pending_tlbs_); |
| aspace_.FlushAsid(); |
| } else { |
| for (size_t i = 0; i < num_pending_tlbs_; i++) { |
| const vaddr_t va = pending_tlbs_[i].va_shifted << 1; |
| DEBUG_ASSERT(aspace_.IsValidVaddr(va)); |
| aspace_.FlushTLBEntry(va, pending_tlbs_[i].terminal); |
| } |
| cm_single_tlb_invalidates.Add(num_pending_tlbs_); |
| } |
| |
| // DSB to ensure TLB flushes happen prior to returning to user. |
| __dsb(ARM_MB_ISH); |
| |
| // Local flushes that the kernel may observe prior to Context Synchronization Event |
| // should go ahead and get an ISB to force it. |
| if (aspace_.type_ == ArmAspaceType::kKernel) { |
| __isb(ARM_MB_SY); |
| } |
| |
| num_pending_tlbs_ = 0; |
| } |
| |
| // Queue a page for freeing that is dependent on TLB flushing. This is for pages that were |
| // previously installed as page tables and they should not be reused until the non-terminal TLB |
| // flush has occurred. |
| void FreePage(vm_page_t* page) { list_add_tail(&to_free_, &page->queue_node); } |
| |
| private: |
| // Maximum number of TLB entries we will queue before switching to ASID invalidation. |
| static constexpr size_t kMaxPendingTlbs = 16; |
| |
| // Pending TLBs to flush are stored as 63 bits, with the bottom bit stolen to store the terminal |
| // flag. 63 bits is more than enough as these entries are page aligned at the minimum. |
| FBL_BITFIELD_DEF_START(PendingTlbs, uint64_t) |
| FBL_BITFIELD_MEMBER(terminal, 0, 1); |
| FBL_BITFIELD_MEMBER(va_shifted, 1, 63); |
| FBL_BITFIELD_DEF_END(); |
| |
| PendingTlbs pending_tlbs_[kMaxPendingTlbs]; |
| |
| size_t num_pending_tlbs_ = 0; |
| |
| // vm_page_t's to release to the PMM after the TLB invalidation occurs. |
| list_node to_free_ = LIST_INITIAL_VALUE(to_free_); |
| |
| // The aspace we are invalidating TLBs for. |
| const ArmArchVmAspace& aspace_; |
| }; |
| |
| uint ArmArchVmAspace::MmuFlagsFromPte(pte_t pte) { |
| switch (type_) { |
| case ArmAspaceType::kUser: |
| case ArmAspaceType::kKernel: |
| return s1_pte_attr_to_mmu_flags(pte); |
| case ArmAspaceType::kHypervisor: |
| return s1_pte_attr_to_mmu_flags(pte, true); |
| case ArmAspaceType::kGuest: |
| return s2_pte_attr_to_mmu_flags(pte); |
| } |
| __UNREACHABLE; |
| } |
| |
| zx_status_t ArmArchVmAspace::Query(vaddr_t vaddr, paddr_t* paddr, uint* mmu_flags) { |
| Guard<Mutex> al{&lock_}; |
| return QueryLocked(vaddr, paddr, mmu_flags); |
| } |
| |
| zx_status_t ArmArchVmAspace::QueryLocked(vaddr_t vaddr, paddr_t* paddr, uint* mmu_flags) { |
| vaddr_t vaddr_rem; |
| |
| canary_.Assert(); |
| LTRACEF("aspace %p, vaddr 0x%lx\n", this, vaddr); |
| |
| DEBUG_ASSERT(tt_virt_); |
| |
| DEBUG_ASSERT(IsValidVaddr(vaddr)); |
| if (!IsValidVaddr(vaddr)) { |
| return ZX_ERR_OUT_OF_RANGE; |
| } |
| |
| const volatile pte_t* page_table = tt_virt_; |
| uint32_t index_shift = top_index_shift_; |
| vaddr_rem = vaddr - vaddr_base_; |
| while (true) { |
| const ulong index = vaddr_rem >> index_shift; |
| vaddr_rem -= (vaddr_t)index << index_shift; |
| const pte_t pte = page_table[index]; |
| const uint descriptor_type = pte & MMU_PTE_DESCRIPTOR_MASK; |
| const paddr_t pte_addr = pte & MMU_PTE_OUTPUT_ADDR_MASK; |
| |
| LTRACEF("va %#" PRIxPTR ", index %lu, index_shift %u, rem %#" PRIxPTR ", pte %#" PRIx64 "\n", |
| vaddr, index, index_shift, vaddr_rem, pte); |
| |
| if (descriptor_type == MMU_PTE_DESCRIPTOR_INVALID) { |
| return ZX_ERR_NOT_FOUND; |
| } |
| |
| if (descriptor_type == ((index_shift > page_size_shift_) ? MMU_PTE_L012_DESCRIPTOR_BLOCK |
| : MMU_PTE_L3_DESCRIPTOR_PAGE)) { |
| if (paddr) { |
| *paddr = pte_addr + vaddr_rem; |
| } |
| if (mmu_flags) { |
| *mmu_flags = MmuFlagsFromPte(pte); |
| } |
| LTRACEF("va 0x%lx, paddr 0x%lx, flags 0x%x\n", vaddr, paddr ? *paddr : ~0UL, |
| mmu_flags ? *mmu_flags : ~0U); |
| return ZX_OK; |
| } |
| |
| if (index_shift <= page_size_shift_ || descriptor_type != MMU_PTE_L012_DESCRIPTOR_TABLE) { |
| PANIC_UNIMPLEMENTED; |
| } |
| |
| page_table = static_cast<const volatile pte_t*>(paddr_to_physmap(pte_addr)); |
| index_shift -= page_size_shift_ - 3; |
| } |
| } |
| |
| zx_status_t ArmArchVmAspace::AllocPageTable(paddr_t* paddrp) { |
| LTRACEF("page_size_shift %u\n", page_size_shift_); |
| |
| // currently we only support allocating a single page |
| DEBUG_ASSERT(page_size_shift_ == PAGE_SIZE_SHIFT); |
| |
| // Allocate a page from the pmm via function pointer passed to us in Init(). |
| // The default is pmm_alloc_page so test and explicitly call it to avoid any unnecessary |
| // virtual functions. |
| vm_page_t* page; |
| zx_status_t status; |
| if (likely(!test_page_alloc_func_)) { |
| status = pmm_alloc_page(0, &page, paddrp); |
| } else { |
| status = test_page_alloc_func_(0, &page, paddrp); |
| } |
| if (status != ZX_OK) { |
| return status; |
| } |
| |
| page->set_state(vm_page_state::MMU); |
| pt_pages_++; |
| |
| LOCAL_KTRACE("page table alloc"); |
| |
| LTRACEF("allocated %#lx\n", *paddrp); |
| return ZX_OK; |
| } |
| |
| void ArmArchVmAspace::FreePageTable(void* vaddr, paddr_t paddr, ConsistencyManager& cm) { |
| LTRACEF("vaddr %p paddr %#lx page_size_shift %u\n", vaddr, paddr, page_size_shift_); |
| |
| // currently we only support freeing a single page |
| DEBUG_ASSERT(page_size_shift_ == PAGE_SIZE_SHIFT); |
| |
| LOCAL_KTRACE("page table free"); |
| |
| vm_page_t* page = paddr_to_vm_page(paddr); |
| if (!page) { |
| panic("bad page table paddr %#lx\n", paddr); |
| } |
| DEBUG_ASSERT(page->state() == vm_page_state::MMU); |
| cm.FreePage(page); |
| |
| pt_pages_--; |
| } |
| |
| zx_status_t ArmArchVmAspace::SplitLargePage(vaddr_t vaddr, const uint index_shift, vaddr_t pt_index, |
| volatile pte_t* page_table, ConsistencyManager& cm) { |
| DEBUG_ASSERT(index_shift > page_size_shift_); |
| |
| const pte_t pte = page_table[pt_index]; |
| DEBUG_ASSERT((pte & MMU_PTE_DESCRIPTOR_MASK) == MMU_PTE_L012_DESCRIPTOR_BLOCK); |
| |
| paddr_t paddr; |
| zx_status_t ret = AllocPageTable(&paddr); |
| if (ret) { |
| TRACEF("failed to allocate page table\n"); |
| return ret; |
| } |
| |
| const uint next_shift = (index_shift - (page_size_shift_ - 3)); |
| |
| const auto new_page_table = static_cast<volatile pte_t*>(paddr_to_physmap(paddr)); |
| const auto new_desc_type = |
| (next_shift == page_size_shift_) ? MMU_PTE_L3_DESCRIPTOR_PAGE : MMU_PTE_L012_DESCRIPTOR_BLOCK; |
| const auto attrs = (pte & ~(MMU_PTE_OUTPUT_ADDR_MASK | MMU_PTE_DESCRIPTOR_MASK)) | new_desc_type; |
| |
| const uint next_size = 1U << next_shift; |
| for (uint64_t i = 0, mapped_paddr = pte & MMU_PTE_OUTPUT_ADDR_MASK; |
| i < MMU_KERNEL_PAGE_TABLE_ENTRIES; i++, mapped_paddr += next_size) { |
| // directly write to the pte, no need to update since this is |
| // a completely new table |
| new_page_table[i] = mapped_paddr | attrs; |
| } |
| |
| // Ensure all zeroing becomes visible prior to page table installation. |
| __dmb(ARM_MB_ISHST); |
| |
| update_pte(&page_table[pt_index], paddr | MMU_PTE_L012_DESCRIPTOR_TABLE); |
| LTRACEF("pte %p[%#" PRIxPTR "] = %#" PRIx64 "\n", page_table, pt_index, page_table[pt_index]); |
| |
| // no need to update the page table count here since we're replacing a block entry with a table |
| // entry. |
| |
| cm.FlushEntry(vaddr, false); |
| |
| return ZX_OK; |
| } |
| |
| // use the appropriate TLB flush instruction to globally flush the modified entry |
| // terminal is set when flushing at the final level of the page table. |
| void ArmArchVmAspace::FlushTLBEntry(vaddr_t vaddr, bool terminal) const { |
| switch (type_) { |
| case ArmAspaceType::kUser: { |
| // flush this address for the specific asid |
| if (terminal) { |
| ARM64_TLBI(vale1is, vaddr >> 12 | (vaddr_t)asid_ << 48); |
| } else { |
| ARM64_TLBI(vae1is, vaddr >> 12 | (vaddr_t)asid_ << 48); |
| } |
| return; |
| } |
| case ArmAspaceType::kKernel: { |
| DEBUG_ASSERT(asid_ == MMU_ARM64_GLOBAL_ASID); |
| // flush this address on all ASIDs |
| if (terminal) { |
| ARM64_TLBI(vaale1is, vaddr >> 12); |
| } else { |
| ARM64_TLBI(vaae1is, vaddr >> 12); |
| } |
| return; |
| } |
| case ArmAspaceType::kGuest: { |
| paddr_t vttbr = arm64_vttbr(asid_, tt_phys_); |
| __UNUSED zx_status_t status = arm64_el2_tlbi_ipa(vttbr, vaddr, terminal); |
| DEBUG_ASSERT(status == ZX_OK); |
| return; |
| } |
| case ArmAspaceType::kHypervisor: |
| PANIC("Unsupported."); |
| return; |
| } |
| __UNREACHABLE; |
| } |
| |
| void ArmArchVmAspace::FlushAsid() const { |
| switch (type_) { |
| case ArmAspaceType::kUser: { |
| DEBUG_ASSERT(asid_ != MMU_ARM64_GLOBAL_ASID); |
| ARM64_TLBI_ASID(aside1is, asid_); |
| return; |
| } |
| case ArmAspaceType::kKernel: { |
| DEBUG_ASSERT(asid_ == MMU_ARM64_GLOBAL_ASID); |
| ARM64_TLBI_NOADDR(alle1is); |
| return; |
| } |
| case ArmAspaceType::kGuest: { |
| paddr_t vttbr = arm64_vttbr(asid_, tt_phys_); |
| zx_status_t status = arm64_el2_tlbi_vmid(vttbr); |
| DEBUG_ASSERT(status == ZX_OK); |
| return; |
| } |
| case ArmAspaceType::kHypervisor: { |
| // Flush all TLB entries in EL2. |
| zx_status_t status = arm64_el2_tlbi_el2(); |
| DEBUG_ASSERT(status == ZX_OK); |
| return; |
| } |
| } |
| __UNREACHABLE; |
| } |
| |
| ssize_t ArmArchVmAspace::UnmapPageTable(vaddr_t vaddr, vaddr_t vaddr_rel, size_t size, |
| EnlargeOperation enlarge, const uint index_shift, |
| volatile pte_t* page_table, ConsistencyManager& cm) { |
| const vaddr_t block_size = 1UL << index_shift; |
| const vaddr_t block_mask = block_size - 1; |
| |
| LTRACEF( |
| "vaddr 0x%lx, vaddr_rel 0x%lx, size 0x%lx, index shift %u, page_size_shift %u, page_table " |
| "%p\n", |
| vaddr, vaddr_rel, size, index_shift, page_size_shift_, page_table); |
| |
| size_t unmap_size = 0; |
| while (size) { |
| const vaddr_t vaddr_rem = vaddr_rel & block_mask; |
| const size_t chunk_size = ktl::min(size, block_size - vaddr_rem); |
| const vaddr_t index = vaddr_rel >> index_shift; |
| |
| pte_t pte = page_table[index]; |
| |
| // If the input range partially covers a large page, attempt to split. |
| if (index_shift > page_size_shift_ && |
| (pte & MMU_PTE_DESCRIPTOR_MASK) == MMU_PTE_L012_DESCRIPTOR_BLOCK && |
| chunk_size != block_size) { |
| zx_status_t s = SplitLargePage(vaddr, index_shift, index, page_table, cm); |
| // If the split failed then check if we are allowed to unmap extra, and if so just fall |
| // through and unmap the entire large page. |
| if (likely(s == ZX_OK)) { |
| pte = page_table[index]; |
| } else if (enlarge == EnlargeOperation::No) { |
| return s; |
| } |
| } |
| if (index_shift > page_size_shift_ && |
| (pte & MMU_PTE_DESCRIPTOR_MASK) == MMU_PTE_L012_DESCRIPTOR_TABLE) { |
| const paddr_t page_table_paddr = pte & MMU_PTE_OUTPUT_ADDR_MASK; |
| volatile pte_t* next_page_table = |
| static_cast<volatile pte_t*>(paddr_to_physmap(page_table_paddr)); |
| |
| // Recurse a level. |
| ssize_t result = UnmapPageTable(vaddr, vaddr_rem, chunk_size, enlarge, |
| index_shift - (page_size_shift_ - 3), next_page_table, cm); |
| if (result < 0) { |
| return result; |
| } |
| |
| // if we unmapped an entire page table leaf and/or the unmap made the level below us empty, |
| // free the page table |
| if (chunk_size == block_size || page_table_is_clear(next_page_table, page_size_shift_)) { |
| LTRACEF("pte %p[0x%lx] = 0 (was page table phys %#lx)\n", page_table, index, |
| page_table_paddr); |
| update_pte(&page_table[index], MMU_PTE_DESCRIPTOR_INVALID); |
| |
| // We can safely defer TLB flushing as the consistency manager will not return the backing |
| // page to the PMM until after the tlb is flushed. |
| cm.FlushEntry(vaddr, false); |
| FreePageTable(const_cast<pte_t*>(next_page_table), page_table_paddr, cm); |
| } |
| } else if (is_pte_valid(pte)) { |
| LTRACEF("pte %p[0x%lx] = 0 (was phys %#lx)\n", page_table, index, |
| page_table[index] & MMU_PTE_OUTPUT_ADDR_MASK); |
| update_pte(&page_table[index], MMU_PTE_DESCRIPTOR_INVALID); |
| cm.FlushEntry(vaddr, true); |
| } else { |
| LTRACEF("pte %p[0x%lx] already clear\n", page_table, index); |
| } |
| vaddr += chunk_size; |
| vaddr_rel += chunk_size; |
| size -= chunk_size; |
| unmap_size += chunk_size; |
| } |
| |
| return unmap_size; |
| } |
| |
| ssize_t ArmArchVmAspace::MapPageTable(vaddr_t vaddr_in, vaddr_t vaddr_rel_in, paddr_t paddr_in, |
| size_t size_in, pte_t attrs, const uint index_shift, |
| volatile pte_t* page_table, ConsistencyManager& cm) { |
| vaddr_t vaddr = vaddr_in; |
| vaddr_t vaddr_rel = vaddr_rel_in; |
| paddr_t paddr = paddr_in; |
| size_t size = size_in; |
| |
| const vaddr_t block_size = 1UL << index_shift; |
| const vaddr_t block_mask = block_size - 1; |
| LTRACEF("vaddr %#" PRIxPTR ", vaddr_rel %#" PRIxPTR ", paddr %#" PRIxPTR |
| ", size %#zx, attrs %#" PRIx64 ", index shift %u, page_size_shift %u, page_table %p\n", |
| vaddr, vaddr_rel, paddr, size, attrs, index_shift, page_size_shift_, page_table); |
| |
| if ((vaddr_rel | paddr | size) & ((1UL << page_size_shift_) - 1)) { |
| TRACEF("not page aligned\n"); |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| auto cleanup = fit::defer([&]() { |
| AssertHeld(lock_); |
| // Unmapping what we have just mapped in should never fail, and we should not have to enlarge |
| // the unmap for it to succeed. |
| ssize_t result = UnmapPageTable(vaddr_in, vaddr_rel_in, size_in - size, EnlargeOperation::No, |
| index_shift, page_table, cm); |
| ASSERT(result >= 0); |
| }); |
| |
| size_t mapped_size = 0; |
| while (size) { |
| const vaddr_t vaddr_rem = vaddr_rel & block_mask; |
| const size_t chunk_size = ktl::min(size, block_size - vaddr_rem); |
| const vaddr_t index = vaddr_rel >> index_shift; |
| pte_t pte = page_table[index]; |
| |
| // if we're at an unaligned address, not trying to map a block, and not at the terminal level, |
| // recurse one more level of the page table tree |
| if (((vaddr_rel | paddr) & block_mask) || (chunk_size != block_size) || |
| (index_shift > MMU_PTE_DESCRIPTOR_BLOCK_MAX_SHIFT)) { |
| // Lookup the next level page table, allocating if required. |
| bool allocated_page_table = false; |
| paddr_t page_table_paddr = 0; |
| volatile pte_t* next_page_table = nullptr; |
| |
| switch (pte & MMU_PTE_DESCRIPTOR_MASK) { |
| case MMU_PTE_DESCRIPTOR_INVALID: { |
| zx_status_t ret = AllocPageTable(&page_table_paddr); |
| if (ret) { |
| TRACEF("failed to allocate page table\n"); |
| return ret; |
| } |
| allocated_page_table = true; |
| void* pt_vaddr = paddr_to_physmap(page_table_paddr); |
| |
| LTRACEF("allocated page table, vaddr %p, paddr 0x%lx\n", pt_vaddr, page_table_paddr); |
| arch_zero_page(pt_vaddr); |
| |
| // ensure that the zeroing is observable from hardware page table walkers, as we need to |
| // do this prior to writing the pte we cannot defer it using the consistency manager. |
| __dmb(ARM_MB_ISHST); |
| |
| // When new pages are mapped they they have their AF set, under the assumption they are |
| // being mapped due to being accessed, and this lets us avoid an accessed fault. Since new |
| // terminal mappings start with the AF flag set, we then also need to start non-terminal |
| // mappings as having the AF set. |
| pte = page_table_paddr | MMU_PTE_L012_DESCRIPTOR_TABLE | MMU_PTE_ATTR_RES_SOFTWARE_AF; |
| update_pte(&page_table[index], pte); |
| // We do not need to sync the walker, despite writing a new entry, as this is a |
| // non-terminal entry and so is irrelevant to the walker anyway. |
| LTRACEF("pte %p[%#" PRIxPTR "] = %#" PRIx64 "\n", page_table, index, pte); |
| next_page_table = static_cast<volatile pte_t*>(pt_vaddr); |
| break; |
| } |
| case MMU_PTE_L012_DESCRIPTOR_TABLE: |
| // Similar to creating a page table, if we end up mapping a page lower down in this |
| // hierarchy then it will start off as accessed. As such we set the accessed flag on the |
| // way down. |
| pte |= MMU_PTE_ATTR_RES_SOFTWARE_AF; |
| update_pte(&page_table[index], pte); |
| page_table_paddr = pte & MMU_PTE_OUTPUT_ADDR_MASK; |
| LTRACEF("found page table %#" PRIxPTR "\n", page_table_paddr); |
| next_page_table = static_cast<volatile pte_t*>(paddr_to_physmap(page_table_paddr)); |
| break; |
| case MMU_PTE_L012_DESCRIPTOR_BLOCK: |
| return ZX_ERR_ALREADY_EXISTS; |
| |
| default: |
| PANIC_UNIMPLEMENTED; |
| } |
| DEBUG_ASSERT(next_page_table); |
| |
| ssize_t ret = MapPageTable(vaddr, vaddr_rem, paddr, chunk_size, attrs, |
| index_shift - (page_size_shift_ - 3), next_page_table, cm); |
| if (ret < 0) { |
| if (allocated_page_table) { |
| // We just allocated this page table. The unmap in err will not clean it up as the size |
| // we pass in will not cause us to look at this page table. This is reasonable as if we |
| // didn't allocate the page table then we shouldn't look into and potentially unmap |
| // anything from that page table. |
| // Since we just allocated it there should be nothing in it, otherwise the MapPageTable |
| // call would not have failed. |
| DEBUG_ASSERT(page_table_is_clear(next_page_table, page_size_shift_)); |
| update_pte(&page_table[index], MMU_PTE_DESCRIPTOR_INVALID); |
| |
| // We can safely defer TLB flushing as the consistency manager will not return the backing |
| // page to the PMM until after the tlb is flushed. |
| cm.FlushEntry(vaddr, false); |
| FreePageTable(const_cast<pte_t*>(next_page_table), page_table_paddr, cm); |
| } |
| return ret; |
| } |
| DEBUG_ASSERT(static_cast<size_t>(ret) == chunk_size); |
| } else { |
| if (is_pte_valid(pte)) { |
| LTRACEF("page table entry already in use, index %#" PRIxPTR ", %#" PRIx64 "\n", index, pte); |
| return ZX_ERR_ALREADY_EXISTS; |
| } |
| |
| pte = paddr | attrs; |
| if (index_shift > page_size_shift_) { |
| pte |= MMU_PTE_L012_DESCRIPTOR_BLOCK; |
| } else { |
| pte |= MMU_PTE_L3_DESCRIPTOR_PAGE; |
| } |
| LTRACEF("pte %p[%#" PRIxPTR "] = %#" PRIx64 " (paddr %#lx)\n", page_table, index, pte, paddr); |
| update_pte(&page_table[index], pte); |
| } |
| vaddr += chunk_size; |
| vaddr_rel += chunk_size; |
| paddr += chunk_size; |
| size -= chunk_size; |
| mapped_size += chunk_size; |
| } |
| |
| cleanup.cancel(); |
| return mapped_size; |
| } |
| |
| zx_status_t ArmArchVmAspace::ProtectPageTable(vaddr_t vaddr_in, vaddr_t vaddr_rel_in, |
| size_t size_in, pte_t attrs, const uint index_shift, |
| volatile pte_t* page_table, ConsistencyManager& cm) { |
| vaddr_t vaddr = vaddr_in; |
| vaddr_t vaddr_rel = vaddr_rel_in; |
| size_t size = size_in; |
| |
| const vaddr_t block_size = 1UL << index_shift; |
| const vaddr_t block_mask = block_size - 1; |
| |
| LTRACEF("vaddr %#" PRIxPTR ", vaddr_rel %#" PRIxPTR ", size %#" PRIxPTR ", attrs %#" PRIx64 |
| ", index shift %u, page_size_shift %u, page_table %p\n", |
| vaddr, vaddr_rel, size, attrs, index_shift, page_size_shift_, page_table); |
| |
| // vaddr_rel and size must be page aligned |
| DEBUG_ASSERT(((vaddr_rel | size) & ((1UL << page_size_shift_) - 1)) == 0); |
| |
| while (size) { |
| const vaddr_t vaddr_rem = vaddr_rel & block_mask; |
| const size_t chunk_size = ktl::min(size, block_size - vaddr_rem); |
| const vaddr_t index = vaddr_rel >> index_shift; |
| |
| pte_t pte = page_table[index]; |
| |
| // If the input range partially covers a large page, split the page. |
| if (index_shift > page_size_shift_ && |
| (pte & MMU_PTE_DESCRIPTOR_MASK) == MMU_PTE_L012_DESCRIPTOR_BLOCK && |
| chunk_size != block_size) { |
| zx_status_t s = SplitLargePage(vaddr, index_shift, index, page_table, cm); |
| if (unlikely(s != ZX_OK)) { |
| return s; |
| } |
| pte = page_table[index]; |
| } |
| |
| if (index_shift > page_size_shift_ && |
| (pte & MMU_PTE_DESCRIPTOR_MASK) == MMU_PTE_L012_DESCRIPTOR_TABLE) { |
| const paddr_t page_table_paddr = pte & MMU_PTE_OUTPUT_ADDR_MASK; |
| volatile pte_t* next_page_table = |
| static_cast<volatile pte_t*>(paddr_to_physmap(page_table_paddr)); |
| |
| // Recurse a level. |
| zx_status_t status = |
| ProtectPageTable(vaddr, vaddr_rem, chunk_size, attrs, |
| index_shift - (page_size_shift_ - 3), next_page_table, cm); |
| if (unlikely(status != ZX_OK)) { |
| return status; |
| } |
| } else if (is_pte_valid(pte)) { |
| pte = (pte & ~MMU_PTE_PERMISSION_MASK) | attrs; |
| LTRACEF("pte %p[%#" PRIxPTR "] = %#" PRIx64 "\n", page_table, index, pte); |
| update_pte(&page_table[index], pte); |
| cm.FlushEntry(vaddr, true); |
| } else { |
| LTRACEF("page table entry does not exist, index %#" PRIxPTR ", %#" PRIx64 "\n", index, pte); |
| } |
| vaddr += chunk_size; |
| vaddr_rel += chunk_size; |
| size -= chunk_size; |
| } |
| |
| return ZX_OK; |
| } |
| |
| size_t ArmArchVmAspace::HarvestAccessedPageTable( |
| size_t* entry_limit, vaddr_t vaddr, vaddr_t vaddr_rel_in, size_t size, const uint index_shift, |
| NonTerminalAction non_terminal_action, TerminalAction terminal_action, |
| volatile pte_t* page_table, ConsistencyManager& cm, bool* unmapped_out) { |
| const vaddr_t block_size = 1UL << index_shift; |
| const vaddr_t block_mask = block_size - 1; |
| |
| vaddr_t vaddr_rel = vaddr_rel_in; |
| |
| // vaddr_rel and size must be page aligned |
| DEBUG_ASSERT(((vaddr_rel | size) & ((1UL << page_size_shift_) - 1)) == 0); |
| |
| size_t harvested_size = 0; |
| |
| while (size > 0 && *entry_limit > 0) { |
| LocalTraceDuration trace{"page_table_loop"_stringref}; |
| |
| const vaddr_t vaddr_rem = vaddr_rel & block_mask; |
| const vaddr_t index = vaddr_rel >> index_shift; |
| |
| size_t chunk_size = ktl::min(size, block_size - vaddr_rem); |
| |
| pte_t pte = page_table[index]; |
| |
| if (index_shift > page_size_shift_ && |
| (pte & MMU_PTE_DESCRIPTOR_MASK) == MMU_PTE_L012_DESCRIPTOR_BLOCK && |
| chunk_size != block_size) { |
| // Ignore large pages, we do not support harvesting accessed bits from them. Having this empty |
| // if block simplifies the overall logic. |
| } else if (index_shift > page_size_shift_ && |
| (pte & MMU_PTE_DESCRIPTOR_MASK) == MMU_PTE_L012_DESCRIPTOR_TABLE) { |
| const paddr_t page_table_paddr = pte & MMU_PTE_OUTPUT_ADDR_MASK; |
| volatile pte_t* next_page_table = |
| static_cast<volatile pte_t*>(paddr_to_physmap(page_table_paddr)); |
| |
| // Start with the assumption that we will unmap if we can. |
| bool do_unmap = non_terminal_action == NonTerminalAction::FreeUnaccessed; |
| // Check for our emulated non-terminal AF so we can potentially skip the recursion. |
| // TODO: make this optional when hardware AF is supported (see todo on |
| // MMU_PTE_ATTR_RES_SOFTWARE_AF for details) |
| if (pte & MMU_PTE_ATTR_RES_SOFTWARE_AF) { |
| bool unmapped = false; |
| chunk_size = HarvestAccessedPageTable( |
| entry_limit, vaddr, vaddr_rem, chunk_size, index_shift - (page_size_shift_ - 3), |
| non_terminal_action, terminal_action, next_page_table, cm, &unmapped); |
| // This was accessed so we don't necessarily want to unmap it, unless our recursive call |
| // caused the page table to be empty, in which case we are obligated to. |
| do_unmap = (unmapped && page_table_is_clear(next_page_table, page_size_shift_)); |
| // If we processed till the end of sub page table, and we are not retaining page tables, |
| // then we can clear the AF as we know we will not have to process entries from this one |
| // again. |
| if (!do_unmap && (vaddr_rel + chunk_size) >> index_shift != index && |
| non_terminal_action != NonTerminalAction::Retain) { |
| pte &= ~MMU_PTE_ATTR_RES_SOFTWARE_AF; |
| update_pte(&page_table[index], pte); |
| } |
| } |
| if (do_unmap) { |
| // Unmapping an exact block, which should not need enlarging and hence should never be able |
| // to fail. |
| ssize_t result = UnmapPageTable(vaddr, vaddr_rem, chunk_size, EnlargeOperation::No, |
| index_shift - (page_size_shift_ - 3), next_page_table, cm); |
| ASSERT(result >= 0); |
| DEBUG_ASSERT(page_table_is_clear(next_page_table, page_size_shift_)); |
| update_pte(&page_table[index], MMU_PTE_DESCRIPTOR_INVALID); |
| |
| // We can safely defer TLB flushing as the consistency manager will not return the backing |
| // page to the PMM until after the tlb is flushed. |
| cm.FlushEntry(vaddr, false); |
| FreePageTable(const_cast<pte_t*>(next_page_table), page_table_paddr, cm); |
| if (unmapped_out) { |
| *unmapped_out = true; |
| } |
| } |
| } else if (is_pte_valid(pte) && (pte & MMU_PTE_ATTR_AF)) { |
| const paddr_t pte_addr = pte & MMU_PTE_OUTPUT_ADDR_MASK; |
| const paddr_t paddr = pte_addr + vaddr_rem; |
| |
| vm_page_t* page = paddr_to_vm_page(paddr); |
| // Mappings for physical VMOs do not have pages associated with them and so there's no state |
| // to update on an access. |
| if (likely(page)) { |
| pmm_page_queues()->MarkAccessedDeferredCount(page); |
| |
| if (terminal_action == TerminalAction::UpdateAgeAndHarvest) { |
| // Modifying the access flag does not require break-before-make for correctness and as we |
| // do not support hardware access flag setting at the moment we do not have to deal with |
| // potential concurrent modifications. |
| pte = (pte & ~MMU_PTE_ATTR_AF); |
| LTRACEF("pte %p[%#" PRIxPTR "] = %#" PRIx64 "\n", page_table, index, pte); |
| update_pte(&page_table[index], pte); |
| |
| cm.FlushEntry(vaddr, true); |
| } |
| } |
| } |
| vaddr += chunk_size; |
| vaddr_rel += chunk_size; |
| size -= chunk_size; |
| |
| harvested_size += chunk_size; |
| |
| // Each iteration of this loop examines a PTE at the current level. The |
| // total number of PTEs examined is limited to avoid holding the aspace lock |
| // for too long. However, the remaining limit balance is updated at the end |
| // of the loop to ensure that harvesting makes progress, even if the initial |
| // limit is too small to reach a terminal PTE. |
| if (*entry_limit > 0) { |
| *entry_limit -= 1; |
| } |
| } |
| |
| return harvested_size; |
| } |
| |
| void ArmArchVmAspace::MarkAccessedPageTable(vaddr_t vaddr, vaddr_t vaddr_rel_in, size_t size, |
| const uint index_shift, volatile pte_t* page_table, |
| ConsistencyManager& cm) { |
| const vaddr_t block_size = 1UL << index_shift; |
| const vaddr_t block_mask = block_size - 1; |
| |
| vaddr_t vaddr_rel = vaddr_rel_in; |
| |
| // vaddr_rel and size must be page aligned |
| DEBUG_ASSERT(((vaddr_rel | size) & ((1UL << page_size_shift_) - 1)) == 0); |
| |
| while (size) { |
| const vaddr_t vaddr_rem = vaddr_rel & block_mask; |
| const size_t chunk_size = ktl::min(size, block_size - vaddr_rem); |
| const vaddr_t index = vaddr_rel >> index_shift; |
| |
| pte_t pte = page_table[index]; |
| |
| if (index_shift > page_size_shift_ && |
| (pte & MMU_PTE_DESCRIPTOR_MASK) == MMU_PTE_L012_DESCRIPTOR_BLOCK && |
| chunk_size != block_size) { |
| // Ignore large pages as we don't support modifying their access flags. Having this empty if |
| // block simplifies the overall logic. |
| } else if (index_shift > page_size_shift_ && |
| (pte & MMU_PTE_DESCRIPTOR_MASK) == MMU_PTE_L012_DESCRIPTOR_TABLE) { |
| // Set the software bit we use to represent that this page table has been accessed. |
| pte |= MMU_PTE_ATTR_RES_SOFTWARE_AF; |
| update_pte(&page_table[index], pte); |
| const paddr_t page_table_paddr = pte & MMU_PTE_OUTPUT_ADDR_MASK; |
| volatile pte_t* next_page_table = |
| static_cast<volatile pte_t*>(paddr_to_physmap(page_table_paddr)); |
| MarkAccessedPageTable(vaddr, vaddr_rem, chunk_size, index_shift - (page_size_shift_ - 3), |
| next_page_table, cm); |
| } else if (is_pte_valid(pte) && (pte & MMU_PTE_ATTR_AF) == 0) { |
| pte |= MMU_PTE_ATTR_AF; |
| update_pte(&page_table[index], pte); |
| } |
| vaddr += chunk_size; |
| vaddr_rel += chunk_size; |
| size -= chunk_size; |
| } |
| } |
| |
| ssize_t ArmArchVmAspace::MapPages(vaddr_t vaddr, paddr_t paddr, size_t size, pte_t attrs, |
| vaddr_t vaddr_base, ConsistencyManager& cm) { |
| vaddr_t vaddr_rel = vaddr - vaddr_base; |
| vaddr_t vaddr_rel_max = 1UL << top_size_shift_; |
| |
| LTRACEF("vaddr %#" PRIxPTR ", paddr %#" PRIxPTR ", size %#" PRIxPTR ", attrs %#" PRIx64 |
| ", asid %#x\n", |
| vaddr, paddr, size, attrs, asid_); |
| |
| if (vaddr_rel > vaddr_rel_max - size || size > vaddr_rel_max) { |
| TRACEF("vaddr %#" PRIxPTR ", size %#" PRIxPTR " out of range vaddr %#" PRIxPTR |
| ", size %#" PRIxPTR "\n", |
| vaddr, size, vaddr_base, vaddr_rel_max); |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| LOCAL_KTRACE("mmu map", (vaddr & ~PAGE_MASK) | ((size >> PAGE_SIZE_SHIFT) & PAGE_MASK)); |
| ssize_t ret = MapPageTable(vaddr, vaddr_rel, paddr, size, attrs, top_index_shift_, tt_virt_, cm); |
| return ret; |
| } |
| |
| ssize_t ArmArchVmAspace::UnmapPages(vaddr_t vaddr, size_t size, EnlargeOperation enlarge, |
| vaddr_t vaddr_base, ConsistencyManager& cm) { |
| vaddr_t vaddr_rel = vaddr - vaddr_base; |
| vaddr_t vaddr_rel_max = 1UL << top_size_shift_; |
| |
| LTRACEF("vaddr 0x%lx, size 0x%lx, asid 0x%x\n", vaddr, size, asid_); |
| |
| if (vaddr_rel > vaddr_rel_max - size || size > vaddr_rel_max) { |
| TRACEF("vaddr 0x%lx, size 0x%lx out of range vaddr 0x%lx, size 0x%lx\n", vaddr, size, |
| vaddr_base, vaddr_rel_max); |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| LOCAL_KTRACE("mmu unmap", (vaddr & ~PAGE_MASK) | ((size >> PAGE_SIZE_SHIFT) & PAGE_MASK)); |
| |
| ssize_t ret = UnmapPageTable(vaddr, vaddr_rel, size, enlarge, top_index_shift_, tt_virt_, cm); |
| return ret; |
| } |
| |
| zx_status_t ArmArchVmAspace::ProtectPages(vaddr_t vaddr, size_t size, pte_t attrs, |
| vaddr_t vaddr_base, ConsistencyManager& cm) { |
| vaddr_t vaddr_rel = vaddr - vaddr_base; |
| vaddr_t vaddr_rel_max = 1UL << top_size_shift_; |
| |
| LTRACEF("vaddr %#" PRIxPTR ", size %#" PRIxPTR ", attrs %#" PRIx64 ", asid %#x\n", vaddr, size, |
| attrs, asid_); |
| |
| if (vaddr_rel > vaddr_rel_max - size || size > vaddr_rel_max) { |
| TRACEF("vaddr %#" PRIxPTR ", size %#" PRIxPTR " out of range vaddr %#" PRIxPTR |
| ", size %#" PRIxPTR "\n", |
| vaddr, size, vaddr_base, vaddr_rel_max); |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| LOCAL_KTRACE("mmu protect", (vaddr & ~PAGE_MASK) | ((size >> PAGE_SIZE_SHIFT) & PAGE_MASK)); |
| |
| zx_status_t ret = ProtectPageTable(vaddr, vaddr_rel, size, attrs, top_index_shift_, tt_virt_, cm); |
| return ret; |
| } |
| |
| pte_t ArmArchVmAspace::MmuParamsFromFlags(uint mmu_flags) { |
| pte_t attrs = 0; |
| switch (type_) { |
| case ArmAspaceType::kUser: |
| attrs = mmu_flags_to_s1_pte_attr(mmu_flags); |
| // User pages are marked non global |
| attrs |= MMU_PTE_ATTR_NON_GLOBAL; |
| break; |
| case ArmAspaceType::kKernel: |
| attrs = mmu_flags_to_s1_pte_attr(mmu_flags); |
| break; |
| case ArmAspaceType::kGuest: |
| attrs = mmu_flags_to_s2_pte_attr(mmu_flags); |
| break; |
| case ArmAspaceType::kHypervisor: |
| attrs = mmu_flags_to_s1_pte_attr(mmu_flags, true); |
| break; |
| } |
| return attrs; |
| } |
| |
| zx_status_t ArmArchVmAspace::MapContiguous(vaddr_t vaddr, paddr_t paddr, size_t count, |
| uint mmu_flags, size_t* mapped) { |
| canary_.Assert(); |
| LTRACEF("vaddr %#" PRIxPTR " paddr %#" PRIxPTR " count %zu flags %#x\n", vaddr, paddr, count, |
| mmu_flags); |
| |
| DEBUG_ASSERT(tt_virt_); |
| |
| DEBUG_ASSERT(IsValidVaddr(vaddr)); |
| if (!IsValidVaddr(vaddr)) { |
| return ZX_ERR_OUT_OF_RANGE; |
| } |
| |
| if (!(mmu_flags & ARCH_MMU_FLAG_PERM_READ)) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| // paddr and vaddr must be aligned. |
| DEBUG_ASSERT(IS_PAGE_ALIGNED(vaddr)); |
| DEBUG_ASSERT(IS_PAGE_ALIGNED(paddr)); |
| if (!IS_PAGE_ALIGNED(vaddr) || !IS_PAGE_ALIGNED(paddr)) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| if (count == 0) { |
| return ZX_OK; |
| } |
| |
| ssize_t ret; |
| { |
| Guard<Mutex> a{&lock_}; |
| ASSERT(updates_enabled_); |
| if (mmu_flags & ARCH_MMU_FLAG_PERM_EXECUTE) { |
| ArmVmICacheConsistencyManager cache_cm; |
| cache_cm.SyncAddr(reinterpret_cast<vaddr_t>(paddr_to_physmap(paddr)), count * PAGE_SIZE); |
| } |
| pte_t attrs = MmuParamsFromFlags(mmu_flags); |
| |
| ConsistencyManager cm(*this); |
| ret = MapPages(vaddr, paddr, count * PAGE_SIZE, attrs, vaddr_base_, cm); |
| MarkAspaceModified(); |
| } |
| |
| if (mapped) { |
| *mapped = (ret > 0) ? (ret / PAGE_SIZE) : 0u; |
| DEBUG_ASSERT(*mapped <= count); |
| } |
| |
| #if __has_feature(address_sanitizer) |
| if (ret >= 0 && type_ == ArmAspaceType::kKernel) { |
| asan_map_shadow_for(vaddr, ret); |
| } |
| #endif // __has_feature(address_sanitizer) |
| |
| return (ret < 0) ? (zx_status_t)ret : ZX_OK; |
| } |
| |
| zx_status_t ArmArchVmAspace::Map(vaddr_t vaddr, paddr_t* phys, size_t count, uint mmu_flags, |
| ExistingEntryAction existing_action, size_t* mapped) { |
| canary_.Assert(); |
| LTRACEF("vaddr %#" PRIxPTR " count %zu flags %#x\n", vaddr, count, mmu_flags); |
| |
| DEBUG_ASSERT(tt_virt_); |
| |
| DEBUG_ASSERT(IsValidVaddr(vaddr)); |
| if (!IsValidVaddr(vaddr)) { |
| return ZX_ERR_OUT_OF_RANGE; |
| } |
| for (size_t i = 0; i < count; ++i) { |
| DEBUG_ASSERT(IS_PAGE_ALIGNED(phys[i])); |
| if (!IS_PAGE_ALIGNED(phys[i])) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| } |
| |
| if (!(mmu_flags & ARCH_MMU_FLAG_PERM_READ)) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| // vaddr must be aligned. |
| DEBUG_ASSERT(IS_PAGE_ALIGNED(vaddr)); |
| if (!IS_PAGE_ALIGNED(vaddr)) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| if (count == 0) { |
| return ZX_OK; |
| } |
| |
| size_t total_mapped = 0; |
| { |
| Guard<Mutex> a{&lock_}; |
| ASSERT(updates_enabled_); |
| if (mmu_flags & ARCH_MMU_FLAG_PERM_EXECUTE) { |
| ArmVmICacheConsistencyManager cache_cm; |
| for (size_t idx = 0; idx < count; ++idx) { |
| cache_cm.SyncAddr(reinterpret_cast<vaddr_t>(paddr_to_physmap(phys[idx])), PAGE_SIZE); |
| } |
| } |
| pte_t attrs = MmuParamsFromFlags(mmu_flags); |
| |
| ssize_t ret; |
| size_t idx = 0; |
| ConsistencyManager cm(*this); |
| auto undo = fit::defer([&]() TA_NO_THREAD_SAFETY_ANALYSIS { |
| if (idx > 0) { |
| UnmapPages(vaddr, idx * PAGE_SIZE, EnlargeOperation::No, vaddr_base_, cm); |
| } |
| }); |
| |
| vaddr_t v = vaddr; |
| for (; idx < count; ++idx) { |
| paddr_t paddr = phys[idx]; |
| DEBUG_ASSERT(IS_PAGE_ALIGNED(paddr)); |
| ret = MapPages(v, paddr, PAGE_SIZE, attrs, vaddr_base_, cm); |
| MarkAspaceModified(); |
| if (ret < 0) { |
| zx_status_t status = static_cast<zx_status_t>(ret); |
| if (status != ZX_ERR_ALREADY_EXISTS || existing_action == ExistingEntryAction::Error) { |
| return status; |
| } |
| } |
| |
| v += PAGE_SIZE; |
| if (ret > 0) { |
| total_mapped += ret / PAGE_SIZE; |
| } |
| } |
| undo.cancel(); |
| } |
| DEBUG_ASSERT(total_mapped <= count); |
| DEBUG_ASSERT(existing_action != ExistingEntryAction::Error || total_mapped == count); |
| |
| if (mapped) { |
| // For ExistingEntryAction::Error, we should have mapped all the addresses we were asked to. |
| // For ExistingEntryAction::Skip, we might have mapped less if we encountered existing entries, |
| // but skipped entries contribute towards the total as well. |
| *mapped = count; |
| } |
| |
| #if __has_feature(address_sanitizer) |
| if (type_ == ArmAspaceType::kKernel) { |
| asan_map_shadow_for(vaddr, total_mapped * PAGE_SIZE); |
| } |
| #endif // __has_feature(address_sanitizer) |
| |
| return ZX_OK; |
| } |
| |
| zx_status_t ArmArchVmAspace::Unmap(vaddr_t vaddr, size_t count, EnlargeOperation enlarge, |
| size_t* unmapped) { |
| canary_.Assert(); |
| LTRACEF("vaddr %#" PRIxPTR " count %zu\n", vaddr, count); |
| |
| DEBUG_ASSERT(tt_virt_); |
| |
| DEBUG_ASSERT(IsValidVaddr(vaddr)); |
| |
| if (!IsValidVaddr(vaddr)) { |
| return ZX_ERR_OUT_OF_RANGE; |
| } |
| |
| DEBUG_ASSERT(IS_PAGE_ALIGNED(vaddr)); |
| if (!IS_PAGE_ALIGNED(vaddr)) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| Guard<Mutex> a{&lock_}; |
| |
| ASSERT(updates_enabled_); |
| ssize_t ret; |
| { |
| ConsistencyManager cm(*this); |
| ret = UnmapPages(vaddr, count * PAGE_SIZE, enlarge, vaddr_base_, cm); |
| MarkAspaceModified(); |
| } |
| |
| if (unmapped) { |
| *unmapped = (ret > 0) ? (ret / PAGE_SIZE) : 0u; |
| DEBUG_ASSERT(*unmapped <= count); |
| } |
| |
| return (ret < 0) ? (zx_status_t)ret : 0; |
| } |
| |
| zx_status_t ArmArchVmAspace::Protect(vaddr_t vaddr, size_t count, uint mmu_flags) { |
| canary_.Assert(); |
| |
| if (!IsValidVaddr(vaddr)) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| if (!IS_PAGE_ALIGNED(vaddr)) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| if (!(mmu_flags & ARCH_MMU_FLAG_PERM_READ)) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| Guard<Mutex> a{&lock_}; |
| ASSERT(updates_enabled_); |
| if (mmu_flags & ARCH_MMU_FLAG_PERM_EXECUTE) { |
| // If mappings are going to become executable then we first need to sync their caches. |
| // Unfortunately this needs to be done on kernel virtual addresses to avoid taking translation |
| // faults, and so we need to first query for the physical address to then get the kernel virtual |
| // address in the physmap. |
| // This sync could be more deeply integrated into ProtectPages, but making existing regions |
| // executable is very uncommon operation and so we keep it simple. |
| vm_mmu_protect_make_execute_calls.Add(1); |
| ArmVmICacheConsistencyManager cache_cm; |
| size_t pages_synced = 0; |
| for (size_t idx = 0; idx < count; idx++) { |
| paddr_t paddr; |
| uint flags; |
| if (QueryLocked(vaddr + idx * PAGE_SIZE, &paddr, &flags) == ZX_OK && |
| (flags & ARCH_MMU_FLAG_PERM_EXECUTE)) { |
| cache_cm.SyncAddr(reinterpret_cast<vaddr_t>(paddr_to_physmap(paddr)), PAGE_SIZE); |
| pages_synced++; |
| } |
| } |
| vm_mmu_protect_make_execute_pages.Add(pages_synced); |
| } |
| |
| int ret; |
| { |
| pte_t attrs = MmuParamsFromFlags(mmu_flags); |
| |
| ConsistencyManager cm(*this); |
| ret = ProtectPages(vaddr, count * PAGE_SIZE, attrs, vaddr_base_, cm); |
| MarkAspaceModified(); |
| } |
| |
| return ret; |
| } |
| |
| zx_status_t ArmArchVmAspace::HarvestAccessed(vaddr_t vaddr, size_t count, |
| NonTerminalAction non_terminal_action, |
| TerminalAction terminal_action) { |
| VM_KTRACE_DURATION(2, "ArmArchVmAspace::HarvestAccessed", vaddr, count); |
| canary_.Assert(); |
| |
| if (!IS_PAGE_ALIGNED(vaddr) || !IsValidVaddr(vaddr)) { |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| // Avoid preemption while "involuntarily" holding the arch aspace lock during |
| // access harvesting. The harvest loop below is O(n), however, the amount of |
| // work performed with the lock held and preemption disabled is limited. Other |
| // O(n) operations under this lock are opt-in by the user (e.g. Map, Protect) |
| // and are performed with preemption enabled. |
| AutoPreemptDisabler preempt_disable; |
| Guard<Mutex> guard{&lock_}; |
| |
| const vaddr_t vaddr_rel = vaddr - vaddr_base_; |
| const vaddr_t vaddr_rel_max = 1UL << top_size_shift_; |
| const size_t size = count * PAGE_SIZE; |
| |
| if (vaddr_rel > vaddr_rel_max - size || size > vaddr_rel_max) { |
| TRACEF("vaddr %#" PRIxPTR ", size %#" PRIxPTR " out of range vaddr %#" PRIxPTR |
| ", size %#" PRIxPTR "\n", |
| vaddr, size, vaddr_base_, vaddr_rel_max); |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| LOCAL_KTRACE("mmu harvest accessed", |
| (vaddr & ~PAGE_MASK) | ((size >> PAGE_SIZE_SHIFT) & PAGE_MASK)); |
| |
| // Limit harvesting to 32 entries per iteration with the arch aspace lock held |
| // to avoid delays in accessed faults in the same aspace running in parallel. |
| // |
| // This limit is derived from the following observations: |
| // 1. Worst case runtime to harvest a terminal PTE on a low-end A53 is ~780ns. |
| // 2. Real workloads can result in harvesting thousands of terminal PTEs in a |
| // single aspace. |
| // 3. An access fault handler will spin up to 150us on the aspace adaptive |
| // mutex before blocking. |
| // 4. Unnecessarily blocking is costly when the system is heavily loaded, |
| // especially during accessed faults, which tend to occur multiple times in |
| // quick succession within and across threads in the same process. |
| // |
| // To achieve optimal contention between access harvesting and access faults, |
| // it is important to avoid exhausting the 150us mutex spin phase by holding |
| // the aspace mutex for too long. The selected entry limit results in a worst |
| // case harvest time of about 1/6 of the mutex spin phase. |
| // |
| // Ti = worst case runtime per top-level harvest iteration. |
| // Te = worst case runtime per terminal entry harvest. |
| // L = max entries per top-level harvest iteration. |
| // |
| // Ti = Te * L = 780ns * 32 = 24.96us |
| // |
| const size_t kMaxEntriesPerIteration = 32; |
| |
| ConsistencyManager cm(*this); |
| size_t remaining_size = size; |
| vaddr_t current_vaddr = vaddr; |
| vaddr_t current_vaddr_rel = vaddr_rel; |
| |
| while (remaining_size) { |
| LocalTraceDuration trace{"harvest_loop"_stringref}; |
| size_t entry_limit = kMaxEntriesPerIteration; |
| const size_t harvested_size = HarvestAccessedPageTable( |
| &entry_limit, current_vaddr, current_vaddr_rel, remaining_size, top_index_shift_, |
| non_terminal_action, terminal_action, tt_virt_, cm, nullptr); |
| DEBUG_ASSERT(harvested_size > 0); |
| DEBUG_ASSERT(harvested_size <= remaining_size); |
| |
| remaining_size -= harvested_size; |
| current_vaddr += harvested_size; |
| current_vaddr_rel += harvested_size; |
| |
| // Release and re-acquire the lock to let contending threads have a chance |
| // to acquire the arch aspace lock between iterations. Use arch::Yield() to |
| // give other CPUs spinning on the aspace mutex a slight edge in acquiring |
| // the mutex. Reenable preemption to flush any pending preemptions that may |
| // have pended during the critical section. |
| guard.CallUnlocked([] { |
| PreemptionState& preemption_state = Thread::Current::preemption_state(); |
| DEBUG_ASSERT(!preemption_state.PreemptIsEnabled()); |
| preemption_state.PreemptReenable(); |
| |
| arch::Yield(); |
| |
| preemption_state.PreemptDisable(); |
| }); |
| } |
| |
| return ZX_OK; |
| } |
| |
| zx_status_t ArmArchVmAspace::MarkAccessed(vaddr_t vaddr, size_t count) { |
| VM_KTRACE_DURATION(2, "ArmArchVmAspace::MarkAccessed", vaddr, count); |
| canary_.Assert(); |
| |
| if (!IS_PAGE_ALIGNED(vaddr) || !IsValidVaddr(vaddr)) { |
| return ZX_ERR_OUT_OF_RANGE; |
| } |
| |
| Guard<Mutex> a{&lock_}; |
| |
| const vaddr_t vaddr_rel = vaddr - vaddr_base_; |
| const vaddr_t vaddr_rel_max = 1UL << top_size_shift_; |
| const size_t size = count * PAGE_SIZE; |
| |
| if (vaddr_rel > vaddr_rel_max - size || size > vaddr_rel_max) { |
| TRACEF("vaddr %#" PRIxPTR ", size %#" PRIxPTR " out of range vaddr %#" PRIxPTR |
| ", size %#" PRIxPTR "\n", |
| vaddr, size, vaddr_base_, vaddr_rel_max); |
| return ZX_ERR_OUT_OF_RANGE; |
| } |
| |
| LOCAL_KTRACE("mmu mark accessed", (vaddr & ~PAGE_MASK) | ((size >> PAGE_SIZE_SHIFT) & PAGE_MASK)); |
| |
| ConsistencyManager cm(*this); |
| |
| MarkAccessedPageTable(vaddr, vaddr_rel, size, top_index_shift_, tt_virt_, cm); |
| MarkAspaceModified(); |
| |
| return ZX_OK; |
| } |
| |
| bool ArmArchVmAspace::ActiveSinceLastCheck(bool clear) { |
| // Read whether any CPUs are presently executing. |
| bool currently_active = num_active_cpus_.load(ktl::memory_order_relaxed) != 0; |
| // Exchange the current notion of active, with the previously active information. This is the only |
| // time a |false| value can potentially be written to active_since_last_check_, and doing an |
| // exchange means we can never 'lose' a |true| value. |
| bool previously_active = |
| clear ? active_since_last_check_.exchange(currently_active, ktl::memory_order_relaxed) |
| : active_since_last_check_.load(ktl::memory_order_relaxed); |
| // Return whether we had previously been active. It is not necessary to also consider whether we |
| // are currently active, since activating would also have active_since_last_check_ to true. In the |
| // scenario where we race and currently_active is true, but we observe previously_active to be |
| // false, this means that as of the start of this function ::ContextSwitch had not completed, and |
| // so this aspace is still not actually active. |
| return previously_active; |
| } |
| |
| zx_status_t ArmArchVmAspace::Init() { |
| canary_.Assert(); |
| LTRACEF("aspace %p, base %#" PRIxPTR ", size 0x%zx, type %*s\n", this, base_, size_, |
| static_cast<int>(ArmAspaceTypeName(type_).size()), ArmAspaceTypeName(type_).data()); |
| |
| Guard<Mutex> a{&lock_}; |
| |
| // Validate that the base + size is sane and doesn't wrap. |
| DEBUG_ASSERT(size_ > PAGE_SIZE); |
| DEBUG_ASSERT(base_ + size_ - 1 > base_); |
| |
| if (type_ == ArmAspaceType::kKernel) { |
| // At the moment we can only deal with address spaces as globally defined. |
| DEBUG_ASSERT(base_ == ~0UL << MMU_KERNEL_SIZE_SHIFT); |
| DEBUG_ASSERT(size_ == 1UL << MMU_KERNEL_SIZE_SHIFT); |
| |
| vaddr_base_ = ~0UL << MMU_KERNEL_SIZE_SHIFT; |
| top_size_shift_ = MMU_KERNEL_SIZE_SHIFT; |
| top_index_shift_ = MMU_KERNEL_TOP_SHIFT; |
| page_size_shift_ = MMU_KERNEL_PAGE_SIZE_SHIFT; |
| |
| tt_virt_ = arm64_kernel_translation_table; |
| tt_phys_ = arm64_kernel_translation_table_phys; |
| asid_ = (uint16_t)MMU_ARM64_GLOBAL_ASID; |
| } else { |
| if (type_ == ArmAspaceType::kUser) { |
| DEBUG_ASSERT(base_ + size_ <= 1UL << MMU_USER_SIZE_SHIFT); |
| |
| vaddr_base_ = 0; |
| top_size_shift_ = MMU_USER_SIZE_SHIFT; |
| top_index_shift_ = MMU_USER_TOP_SHIFT; |
| page_size_shift_ = MMU_USER_PAGE_SIZE_SHIFT; |
| |
| auto status = asid->Alloc(); |
| if (status.is_error()) { |
| printf("ARM: out of ASIDs!\n"); |
| return status.status_value(); |
| } |
| asid_ = status.value(); |
| } else if (type_ == ArmAspaceType::kGuest) { |
| DEBUG_ASSERT(base_ + size_ <= 1UL << MMU_GUEST_SIZE_SHIFT); |
| |
| vaddr_base_ = 0; |
| top_size_shift_ = MMU_GUEST_SIZE_SHIFT; |
| top_index_shift_ = MMU_GUEST_TOP_SHIFT; |
| page_size_shift_ = MMU_GUEST_PAGE_SIZE_SHIFT; |
| } else { |
| DEBUG_ASSERT(type_ == ArmAspaceType::kHypervisor); |
| DEBUG_ASSERT(base_ + size_ <= 1UL << MMU_IDENT_SIZE_SHIFT); |
| |
| vaddr_base_ = 0; |
| top_size_shift_ = MMU_IDENT_SIZE_SHIFT; |
| top_index_shift_ = MMU_IDENT_TOP_SHIFT; |
| page_size_shift_ = MMU_IDENT_PAGE_SIZE_SHIFT; |
| } |
| |
| // allocate a top level page table to serve as the translation table |
| paddr_t pa; |
| zx_status_t status = AllocPageTable(&pa); |
| if (status != ZX_OK) { |
| return status; |
| } |
| |
| volatile pte_t* va = static_cast<volatile pte_t*>(paddr_to_physmap(pa)); |
| |
| tt_virt_ = va; |
| tt_phys_ = pa; |
| |
| // zero the top level translation table. |
| arch_zero_page(const_cast<pte_t*>(tt_virt_)); |
| __dmb(ARM_MB_ISHST); |
| } |
| pt_pages_ = 1; |
| |
| LTRACEF("tt_phys %#" PRIxPTR " tt_virt %p\n", tt_phys_, tt_virt_); |
| |
| return ZX_OK; |
| } |
| |
| void ArmArchVmAspace::AssertEmptyLocked() const { |
| // Check to see if the top level page table is empty. If not the user didn't |
| // properly unmap everything before destroying the aspace |
| if (const int index = first_used_page_table_entry(tt_virt_, page_size_shift_); index != -1) { |
| panic( |
| "top level page table still in use! aspace %p pt_pages_ %zu tt_virt %p index %d entry " |
| "%" PRIx64 "\n", |
| this, pt_pages_, tt_virt_, index, tt_virt_[index]); |
| } |
| |
| if (pt_pages_ != 1) { |
| panic("allocated page table count is wrong, aspace %p count %zu (should be 1)\n", this, |
| pt_pages_); |
| } |
| } |
| |
| void ArmArchVmAspace::DisableUpdates() { |
| canary_.Assert(); |
| |
| Guard<Mutex> a{&lock_}; |
| updates_enabled_ = false; |
| AssertEmptyLocked(); |
| } |
| |
| zx_status_t ArmArchVmAspace::Destroy() { |
| canary_.Assert(); |
| LTRACEF("aspace %p\n", this); |
| |
| Guard<Mutex> a{&lock_}; |
| |
| // Not okay to destroy the kernel address space |
| DEBUG_ASSERT(type_ != ArmAspaceType::kKernel); |
| |
| if (!tt_virt_) { |
| // Initialization must not have succeeded. |
| DEBUG_ASSERT(!tt_phys_); |
| return ZX_OK; |
| } |
| |
| AssertEmptyLocked(); |
| |
| // Need a DSB to synchronize any page table updates prior to flushing the TLBs. |
| __dsb(ARM_MB_ISH); |
| |
| // Flush the ASID or VMID associated with this aspace |
| FlushAsid(); |
| |
| // Need a DSB to ensure all other cpus have fully processed the TLB flush. |
| __dsb(ARM_MB_ISH); |
| |
| // Free any ASID. |
| if (type_ == ArmAspaceType::kUser) { |
| auto status = asid->Free(asid_); |
| ASSERT(status.is_ok()); |
| asid_ = MMU_ARM64_UNUSED_ASID; |
| } |
| |
| // Free the top level page table. |
| vm_page_t* page = paddr_to_vm_page(tt_phys_); |
| DEBUG_ASSERT(page); |
| pmm_free_page(page); |
| pt_pages_--; |
| |
| tt_phys_ = 0; |
| tt_virt_ = nullptr; |
| |
| return ZX_OK; |
| } |
| |
| // Called during context switches between threads with different address spaces. Swaps the |
| // mmu context on hardware. Assumes old_aspace != aspace and optimizes as such. |
| void ArmArchVmAspace::ContextSwitch(ArmArchVmAspace* old_aspace, ArmArchVmAspace* aspace) { |
| uint64_t tcr; |
| uint64_t ttbr; |
| if (likely(aspace)) { |
| aspace->canary_.Assert(); |
| DEBUG_ASSERT(aspace->type_ == ArmAspaceType::kUser); |
| |
| // Load the user space TTBR with the translation table and user space ASID. |
| ttbr = ((uint64_t)aspace->asid_ << 48) | aspace->tt_phys_; |
| __arm_wsr64("ttbr0_el1", ttbr); |
| __isb(ARM_MB_SY); |
| |
| // If we're switching away from the kernel aspace, load TCR with the user flags. |
| tcr = MMU_TCR_FLAGS_USER; |
| if (unlikely(!old_aspace)) { |
| __arm_wsr64("tcr_el1", tcr); |
| __isb(ARM_MB_SY); |
| } else { |
| __UNUSED uint32_t prev = old_aspace->num_active_cpus_.fetch_sub(1, ktl::memory_order_relaxed); |
| DEBUG_ASSERT(prev > 0); |
| } |
| __UNUSED uint32_t prev = aspace->num_active_cpus_.fetch_add(1, ktl::memory_order_relaxed); |
| DEBUG_ASSERT(prev < SMP_MAX_CPUS); |
| aspace->active_since_last_check_.store(true, ktl::memory_order_relaxed); |
| |
| } else { |
| // Switching to the null aspace, which means kernel address space only. |
| // Load a null TTBR0 and disable page table walking for user space. |
| tcr = MMU_TCR_FLAGS_KERNEL; |
| __arm_wsr64("tcr_el1", tcr); |
| __isb(ARM_MB_SY); |
| |
| ttbr = 0; // MMU_ARM64_UNUSED_ASID |
| __arm_wsr64("ttbr0_el1", ttbr); |
| __isb(ARM_MB_SY); |
| |
| if (likely(old_aspace != nullptr)) { |
| __UNUSED uint32_t prev = old_aspace->num_active_cpus_.fetch_sub(1, ktl::memory_order_relaxed); |
| DEBUG_ASSERT(prev > 0); |
| } |
| } |
| if (TRACE_CONTEXT_SWITCH) { |
| TRACEF("old aspace %p aspace %p ttbr %#" PRIx64 ", tcr %#" PRIx64 "\n", old_aspace, aspace, |
| ttbr, tcr); |
| } |
| } |
| |
| void arch_zero_page(void* _ptr) { |
| uintptr_t ptr = (uintptr_t)_ptr; |
| |
| uint32_t zva_size = arm64_zva_size; |
| uintptr_t end_ptr = ptr + PAGE_SIZE; |
| do { |
| __asm volatile("dc zva, %0" ::"r"(ptr)); |
| ptr += zva_size; |
| } while (ptr != end_ptr); |
| } |
| |
| zx_status_t arm64_mmu_translate(vaddr_t va, paddr_t* pa, bool user, bool write) { |
| // disable interrupts around this operation to make the at/par instruction combination atomic |
| uint64_t par; |
| { |
| InterruptDisableGuard irqd; |
| |
| if (user) { |
| if (write) { |
| __asm__ volatile("at s1e0w, %0" ::"r"(va) : "memory"); |
| } else { |
| __asm__ volatile("at s1e0r, %0" ::"r"(va) : "memory"); |
| } |
| } else { |
| if (write) { |
| __asm__ volatile("at s1e1w, %0" ::"r"(va) : "memory"); |
| } else { |
| __asm__ volatile("at s1e1r, %0" ::"r"(va) : "memory"); |
| } |
| } |
| |
| par = __arm_rsr64("par_el1"); |
| } |
| |
| // if bit 0 is clear, the translation succeeded |
| if (BIT(par, 0)) { |
| return ZX_ERR_NOT_FOUND; |
| } |
| |
| // physical address is stored in bits [51..12], naturally aligned |
| *pa = BITS(par, 51, 12) | (va & (PAGE_SIZE - 1)); |
| |
| return ZX_OK; |
| } |
| |
| ArmArchVmAspace::ArmArchVmAspace(vaddr_t base, size_t size, ArmAspaceType type, page_alloc_fn_t paf) |
| : test_page_alloc_func_(paf), type_(type), base_(base), size_(size) {} |
| |
| ArmArchVmAspace::ArmArchVmAspace(vaddr_t base, size_t size, uint mmu_flags, page_alloc_fn_t paf) |
| : ArmArchVmAspace(base, size, AspaceTypeFromFlags(mmu_flags), paf) {} |
| |
| ArmArchVmAspace::~ArmArchVmAspace() { |
| // Destroy() will have freed the final page table if it ran correctly, and further validated that |
| // everything else was freed. |
| DEBUG_ASSERT(pt_pages_ == 0); |
| } |
| |
| vaddr_t ArmArchVmAspace::PickSpot(vaddr_t base, vaddr_t end, vaddr_t align, size_t size, |
| uint mmu_flags) { |
| canary_.Assert(); |
| return PAGE_ALIGN(base); |
| } |
| |
| void ArmVmICacheConsistencyManager::SyncAddr(vaddr_t start, size_t len) { |
| // Validate we are operating on a kernel address range. |
| DEBUG_ASSERT(is_kernel_address(start)); |
| // use the physmap to clean the range to PoU, which is the point of where the instruction cache |
| // pulls from. Cleaning to PoU is potentially cheaper than cleaning to PoC, which is the default |
| // of arch_clean_cache_range. |
| arm64_clean_cache_range_pou(start, len); |
| // We can batch the icache invalidate and just perform it once at the end. |
| need_invalidate_ = true; |
| } |
| void ArmVmICacheConsistencyManager::Finish() { |
| if (!need_invalidate_) { |
| return; |
| } |
| // Under the assumption our icache is VIPT then as we do not know all the virtual aliases of the |
| // sections we cleaned our only option is to dump the entire icache. |
| asm volatile("ic ialluis" ::: "memory"); |
| __isb(ARM_MB_SY); |
| need_invalidate_ = false; |
| } |
| |
| void arm64_mmu_early_init() { |
| // after we've probed the feature set, initialize the asid allocator |
| asid.Initialize(); |
| } |
| |
| uint32_t arch_address_tagging_features() { |
| static_assert(MMU_TCR_FLAGS_USER & MMU_TCR_TBI0, "Expected TBI enabled."); |
| return ZX_ARM64_FEATURE_ADDRESS_TAGGING_TBI; |
| } |