| // Copyright 2016 The Fuchsia Authors |
| // |
| // Use of this source code is governed by a MIT-style |
| // license that can be found in the LICENSE file or at |
| // https://opensource.org/licenses/MIT |
| |
| #include <assert.h> |
| #include <err.h> |
| #include <string.h> |
| #include <trace.h> |
| |
| #include <arch/arch_ops.h> |
| #include <arch/mmu.h> |
| #include <arch/x86.h> |
| #include <arch/x86/descriptor.h> |
| #include <arch/x86/feature.h> |
| #include <arch/x86/mmu.h> |
| #include <arch/x86/mmu_mem_types.h> |
| #include <kernel/mp.h> |
| #include <new> |
| #include <vm/arch_vm_aspace.h> |
| #include <vm/physmap.h> |
| #include <vm/pmm.h> |
| #include <vm/vm.h> |
| #include <zircon/types.h> |
| |
| #define LOCAL_TRACE 0 |
| |
| /* Default address width including virtual/physical address. |
| * newer versions fetched below */ |
| uint8_t g_vaddr_width = 48; |
| uint8_t g_paddr_width = 32; |
| |
| /* True if the system supports 1GB pages */ |
| static bool supports_huge_pages = false; |
| |
| /* top level kernel page tables, initialized in start.S */ |
| volatile pt_entry_t pml4[NO_OF_PT_ENTRIES] __ALIGNED(PAGE_SIZE); |
| volatile pt_entry_t pdp[NO_OF_PT_ENTRIES] __ALIGNED(PAGE_SIZE); /* temporary */ |
| volatile pt_entry_t pte[NO_OF_PT_ENTRIES] __ALIGNED(PAGE_SIZE); |
| |
| /* top level pdp needed to map the -512GB..0 space */ |
| volatile pt_entry_t pdp_high[NO_OF_PT_ENTRIES] __ALIGNED(PAGE_SIZE); |
| |
| /* a big pile of page tables needed to map 64GB of memory into kernel space using 2MB pages */ |
| volatile pt_entry_t linear_map_pdp[(64ULL * GB) / (2 * MB)] __ALIGNED(PAGE_SIZE); |
| |
| /* which of the above variables is the top level page table */ |
| #define KERNEL_PT pml4 |
| |
| // Static relocated base to prepare for KASLR. Used at early boot and by gdb |
| // script to know the target relocated address. |
| // TODO(thgarnie): Move to a dynamicly generated base address |
| #if DISABLE_KASLR |
| uint64_t kernel_relocated_base = KERNEL_BASE - KERNEL_LOAD_OFFSET; |
| #else |
| uint64_t kernel_relocated_base = 0xffffffff00000000; |
| #endif |
| |
| /* kernel base top level page table in physical space */ |
| static const paddr_t kernel_pt_phys = |
| (vaddr_t)KERNEL_PT - (vaddr_t)__code_start + KERNEL_LOAD_OFFSET; |
| |
| // Valid EPT MMU flags. |
| static const uint kValidEptFlags = |
| ARCH_MMU_FLAG_PERM_READ | ARCH_MMU_FLAG_PERM_WRITE | ARCH_MMU_FLAG_PERM_EXECUTE; |
| |
| paddr_t x86_kernel_cr3(void) { |
| return kernel_pt_phys; |
| } |
| |
| /** |
| * @brief check if the virtual address is canonical |
| */ |
| bool x86_is_vaddr_canonical(vaddr_t vaddr) { |
| uint64_t max_vaddr_lohalf, min_vaddr_hihalf; |
| |
| /* get max address in lower-half canonical addr space */ |
| /* e.g. if width is 48, then 0x00007FFF_FFFFFFFF */ |
| max_vaddr_lohalf = ((uint64_t)1ull << (g_vaddr_width - 1)) - 1; |
| |
| /* get min address in higher-half canonical addr space */ |
| /* e.g. if width is 48, then 0xFFFF8000_00000000*/ |
| min_vaddr_hihalf = ~max_vaddr_lohalf; |
| |
| /* Check to see if the address in a canonical address */ |
| if ((vaddr > max_vaddr_lohalf) && (vaddr < min_vaddr_hihalf)) |
| return false; |
| |
| return true; |
| } |
| |
| /** |
| * @brief check if the virtual address is aligned and canonical |
| */ |
| static bool x86_mmu_check_vaddr(vaddr_t vaddr) { |
| /* Check to see if the address is PAGE aligned */ |
| if (!IS_ALIGNED(vaddr, PAGE_SIZE)) |
| return false; |
| |
| return x86_is_vaddr_canonical(vaddr); |
| } |
| |
| /** |
| * @brief check if the physical address is valid and aligned |
| */ |
| bool x86_mmu_check_paddr(paddr_t paddr) { |
| uint64_t max_paddr; |
| |
| /* Check to see if the address is PAGE aligned */ |
| if (!IS_ALIGNED(paddr, PAGE_SIZE)) |
| return false; |
| |
| max_paddr = ((uint64_t)1ull << g_paddr_width) - 1; |
| |
| return paddr <= max_paddr; |
| } |
| |
| /** |
| * @brief invalidate all TLB entries, including global entries |
| */ |
| static void x86_tlb_global_invalidate() { |
| /* See Intel 3A section 4.10.4.1 */ |
| ulong cr4 = x86_get_cr4(); |
| if (likely(cr4 & X86_CR4_PGE)) { |
| x86_set_cr4(cr4 & ~X86_CR4_PGE); |
| x86_set_cr4(cr4); |
| } else { |
| x86_set_cr3(x86_get_cr3()); |
| } |
| } |
| |
| /** |
| * @brief invalidate all TLB entries, excluding global entries |
| */ |
| static void x86_tlb_nonglobal_invalidate() { |
| x86_set_cr3(x86_get_cr3()); |
| } |
| |
| /* Task used for invalidating a TLB entry on each CPU */ |
| struct TlbInvalidatePage_context { |
| ulong target_cr3; |
| const PendingTlbInvalidation* pending; |
| }; |
| static void TlbInvalidatePage_task(void* raw_context) { |
| DEBUG_ASSERT(arch_ints_disabled()); |
| TlbInvalidatePage_context* context = (TlbInvalidatePage_context*)raw_context; |
| |
| ulong cr3 = x86_get_cr3(); |
| if (context->target_cr3 != cr3 && !context->pending->contains_global) { |
| /* This invalidation doesn't apply to this CPU, ignore it */ |
| return; |
| } |
| |
| if (context->pending->full_shootdown) { |
| if (context->pending->contains_global) { |
| x86_tlb_global_invalidate(); |
| } else { |
| x86_tlb_nonglobal_invalidate(); |
| } |
| return; |
| } |
| |
| for (uint i = 0; i < context->pending->count; ++i) { |
| const auto& item = context->pending->item[i]; |
| switch (item.page_level()) { |
| case PML4_L: |
| panic("PML4_L invld found; should not be here\n"); |
| case PDP_L: |
| case PD_L: |
| case PT_L: |
| __asm__ volatile("invlpg %0" ::"m"(*(uint8_t*)item.addr())); |
| break; |
| } |
| } |
| } |
| |
| /** |
| * @brief Execute a queued TLB invalidation |
| * |
| * @param pt The page table we're invalidating for (if nullptr, assume for current one) |
| * @param pending The planned invalidation |
| */ |
| static void x86_tlb_invalidate_page(const X86PageTableBase* pt, PendingTlbInvalidation* pending) { |
| if (pending->count == 0) { |
| return; |
| } |
| |
| ulong cr3 = pt ? pt->phys() : x86_get_cr3(); |
| struct TlbInvalidatePage_context task_context = { |
| .target_cr3 = cr3, .pending = pending, |
| }; |
| |
| /* Target only CPUs this aspace is active on. It may be the case that some |
| * other CPU will become active in it after this load, or will have left it |
| * just before this load. In the former case, it is becoming active after |
| * the write to the page table, so it will see the change. In the latter |
| * case, it will get a spurious request to flush. */ |
| mp_ipi_target_t target; |
| cpu_mask_t target_mask = 0; |
| if (pending->contains_global || pt == nullptr) { |
| target = MP_IPI_TARGET_ALL; |
| } else { |
| target = MP_IPI_TARGET_MASK; |
| target_mask = static_cast<X86ArchVmAspace*>(pt->ctx())->active_cpus(); |
| } |
| |
| mp_sync_exec(target, target_mask, TlbInvalidatePage_task, &task_context); |
| pending->clear(); |
| } |
| |
| bool X86PageTableMmu::check_paddr(paddr_t paddr) { |
| return x86_mmu_check_paddr(paddr); |
| } |
| |
| bool X86PageTableMmu::check_vaddr(vaddr_t vaddr) { |
| return x86_mmu_check_vaddr(vaddr); |
| } |
| |
| bool X86PageTableMmu::supports_page_size(PageTableLevel level) { |
| DEBUG_ASSERT(level != PT_L); |
| switch (level) { |
| case PD_L: |
| return true; |
| case PDP_L: |
| return supports_huge_pages; |
| case PML4_L: |
| return false; |
| default: |
| panic("Unreachable case in supports_page_size\n"); |
| } |
| } |
| |
| X86PageTableBase::IntermediatePtFlags X86PageTableMmu::intermediate_flags() { |
| return X86_MMU_PG_RW | X86_MMU_PG_U; |
| } |
| |
| X86PageTableBase::PtFlags X86PageTableMmu::terminal_flags(PageTableLevel level, |
| uint flags) { |
| X86PageTableBase::PtFlags terminal_flags = 0; |
| |
| if (flags & ARCH_MMU_FLAG_PERM_WRITE) { |
| terminal_flags |= X86_MMU_PG_RW; |
| } |
| if (flags & ARCH_MMU_FLAG_PERM_USER) { |
| terminal_flags |= X86_MMU_PG_U; |
| } |
| if (use_global_mappings_) { |
| terminal_flags |= X86_MMU_PG_G; |
| } |
| if (!(flags & ARCH_MMU_FLAG_PERM_EXECUTE)) { |
| terminal_flags |= X86_MMU_PG_NX; |
| } |
| |
| if (level > 0) { |
| switch (flags & ARCH_MMU_FLAG_CACHE_MASK) { |
| case ARCH_MMU_FLAG_CACHED: |
| terminal_flags |= X86_MMU_LARGE_PAT_WRITEBACK; |
| break; |
| case ARCH_MMU_FLAG_UNCACHED_DEVICE: |
| case ARCH_MMU_FLAG_UNCACHED: |
| terminal_flags |= X86_MMU_LARGE_PAT_UNCACHABLE; |
| break; |
| case ARCH_MMU_FLAG_WRITE_COMBINING: |
| terminal_flags |= X86_MMU_LARGE_PAT_WRITE_COMBINING; |
| break; |
| default: |
| PANIC_UNIMPLEMENTED; |
| } |
| } else { |
| switch (flags & ARCH_MMU_FLAG_CACHE_MASK) { |
| case ARCH_MMU_FLAG_CACHED: |
| terminal_flags |= X86_MMU_PTE_PAT_WRITEBACK; |
| break; |
| case ARCH_MMU_FLAG_UNCACHED_DEVICE: |
| case ARCH_MMU_FLAG_UNCACHED: |
| terminal_flags |= X86_MMU_PTE_PAT_UNCACHABLE; |
| break; |
| case ARCH_MMU_FLAG_WRITE_COMBINING: |
| terminal_flags |= X86_MMU_PTE_PAT_WRITE_COMBINING; |
| break; |
| default: |
| PANIC_UNIMPLEMENTED; |
| } |
| } |
| |
| return terminal_flags; |
| } |
| |
| X86PageTableBase::PtFlags X86PageTableMmu::split_flags(PageTableLevel level, |
| X86PageTableBase::PtFlags flags) { |
| DEBUG_ASSERT(level != PML4_L && level != PT_L); |
| DEBUG_ASSERT(flags & X86_MMU_PG_PS); |
| if (level == PD_L) { |
| // Note: Clear PS before the check below; the PAT bit for a PTE is the |
| // the same as the PS bit for a higher table entry. |
| flags &= ~X86_MMU_PG_PS; |
| |
| /* If the larger page had the PAT flag set, make sure it's |
| * transferred to the different index for a PTE */ |
| if (flags & X86_MMU_PG_LARGE_PAT) { |
| flags &= ~X86_MMU_PG_LARGE_PAT; |
| flags |= X86_MMU_PG_PTE_PAT; |
| } |
| } |
| return flags; |
| } |
| |
| void X86PageTableMmu::TlbInvalidate(PendingTlbInvalidation* pending) { |
| x86_tlb_invalidate_page(this, pending); |
| } |
| |
| uint X86PageTableMmu::pt_flags_to_mmu_flags(PtFlags flags, PageTableLevel level) { |
| uint mmu_flags = ARCH_MMU_FLAG_PERM_READ; |
| |
| if (flags & X86_MMU_PG_RW) { |
| mmu_flags |= ARCH_MMU_FLAG_PERM_WRITE; |
| } |
| if (flags & X86_MMU_PG_U) { |
| mmu_flags |= ARCH_MMU_FLAG_PERM_USER; |
| } |
| if (!(flags & X86_MMU_PG_NX)) { |
| mmu_flags |= ARCH_MMU_FLAG_PERM_EXECUTE; |
| } |
| |
| if (level > 0) { |
| switch (flags & X86_MMU_LARGE_PAT_MASK) { |
| case X86_MMU_LARGE_PAT_WRITEBACK: |
| mmu_flags |= ARCH_MMU_FLAG_CACHED; |
| break; |
| case X86_MMU_LARGE_PAT_UNCACHABLE: |
| mmu_flags |= ARCH_MMU_FLAG_UNCACHED; |
| break; |
| case X86_MMU_LARGE_PAT_WRITE_COMBINING: |
| mmu_flags |= ARCH_MMU_FLAG_WRITE_COMBINING; |
| break; |
| default: |
| PANIC_UNIMPLEMENTED; |
| } |
| } else { |
| switch (flags & X86_MMU_PTE_PAT_MASK) { |
| case X86_MMU_PTE_PAT_WRITEBACK: |
| mmu_flags |= ARCH_MMU_FLAG_CACHED; |
| break; |
| case X86_MMU_PTE_PAT_UNCACHABLE: |
| mmu_flags |= ARCH_MMU_FLAG_UNCACHED; |
| break; |
| case X86_MMU_PTE_PAT_WRITE_COMBINING: |
| mmu_flags |= ARCH_MMU_FLAG_WRITE_COMBINING; |
| break; |
| default: |
| PANIC_UNIMPLEMENTED; |
| } |
| } |
| return mmu_flags; |
| } |
| |
| bool X86PageTableEpt::allowed_flags(uint flags) { |
| if (!(flags & ARCH_MMU_FLAG_PERM_READ)) { |
| return false; |
| } |
| if (flags & ~kValidEptFlags) { |
| return false; |
| } |
| return true; |
| } |
| |
| bool X86PageTableEpt::check_paddr(paddr_t paddr) { |
| return x86_mmu_check_paddr(paddr); |
| } |
| |
| bool X86PageTableEpt::check_vaddr(vaddr_t vaddr) { |
| return x86_mmu_check_vaddr(vaddr); |
| } |
| |
| bool X86PageTableEpt::supports_page_size(PageTableLevel level) { |
| DEBUG_ASSERT(level != PT_L); |
| switch (level) { |
| case PD_L: |
| return true; |
| case PDP_L: |
| return supports_huge_pages; |
| case PML4_L: |
| return false; |
| default: |
| panic("Unreachable case in supports_page_size\n"); |
| } |
| } |
| |
| X86PageTableBase::PtFlags X86PageTableEpt::intermediate_flags() { |
| return X86_EPT_R | X86_EPT_W | X86_EPT_X; |
| } |
| |
| X86PageTableBase::PtFlags X86PageTableEpt::terminal_flags(PageTableLevel level, |
| uint flags) { |
| X86PageTableBase::PtFlags terminal_flags = 0; |
| |
| if (flags & ARCH_MMU_FLAG_PERM_READ) { |
| terminal_flags |= X86_EPT_R; |
| } |
| if (flags & ARCH_MMU_FLAG_PERM_WRITE) { |
| terminal_flags |= X86_EPT_W; |
| } |
| if (flags & ARCH_MMU_FLAG_PERM_EXECUTE) { |
| terminal_flags |= X86_EPT_X; |
| } |
| |
| switch (flags & ARCH_MMU_FLAG_CACHE_MASK) { |
| case ARCH_MMU_FLAG_CACHED: |
| terminal_flags |= X86_EPT_WB; |
| break; |
| case ARCH_MMU_FLAG_UNCACHED_DEVICE: |
| case ARCH_MMU_FLAG_UNCACHED: |
| terminal_flags |= X86_EPT_UC; |
| break; |
| case ARCH_MMU_FLAG_WRITE_COMBINING: |
| terminal_flags |= X86_EPT_WC; |
| break; |
| default: |
| PANIC_UNIMPLEMENTED; |
| } |
| |
| return terminal_flags; |
| } |
| |
| X86PageTableBase::PtFlags X86PageTableEpt::split_flags(PageTableLevel level, |
| X86PageTableBase::PtFlags flags) { |
| DEBUG_ASSERT(level != PML4_L && level != PT_L); |
| // We don't need to relocate any flags on split for EPT. |
| return flags; |
| } |
| |
| |
| void X86PageTableEpt::TlbInvalidate(PendingTlbInvalidation* pending) { |
| // TODO(ZX-981): Implement this. |
| pending->clear(); |
| } |
| |
| uint X86PageTableEpt::pt_flags_to_mmu_flags(PtFlags flags, PageTableLevel level) { |
| uint mmu_flags = 0; |
| |
| if (flags & X86_EPT_R) { |
| mmu_flags |= ARCH_MMU_FLAG_PERM_READ; |
| } |
| if (flags & X86_EPT_W) { |
| mmu_flags |= ARCH_MMU_FLAG_PERM_WRITE; |
| } |
| if (flags & X86_EPT_X) { |
| mmu_flags |= ARCH_MMU_FLAG_PERM_EXECUTE; |
| } |
| |
| switch (flags & X86_EPT_MEMORY_TYPE_MASK) { |
| case X86_EPT_WB: |
| mmu_flags |= ARCH_MMU_FLAG_CACHED; |
| break; |
| case X86_EPT_UC: |
| mmu_flags |= ARCH_MMU_FLAG_UNCACHED; |
| break; |
| case X86_EPT_WC: |
| mmu_flags |= ARCH_MMU_FLAG_WRITE_COMBINING; |
| break; |
| default: |
| PANIC_UNIMPLEMENTED; |
| } |
| |
| return mmu_flags; |
| } |
| |
| void x86_mmu_early_init() { |
| x86_mmu_percpu_init(); |
| |
| x86_mmu_mem_type_init(); |
| |
| // Unmap the lower identity mapping. |
| pml4[0] = 0; |
| PendingTlbInvalidation tlb; |
| tlb.enqueue(0, PML4_L, /* global */ false, /* terminal */ false); |
| x86_tlb_invalidate_page(nullptr, &tlb); |
| |
| /* get the address width from the CPU */ |
| uint8_t vaddr_width = x86_linear_address_width(); |
| uint8_t paddr_width = x86_physical_address_width(); |
| |
| supports_huge_pages = x86_feature_test(X86_FEATURE_HUGE_PAGE); |
| |
| /* if we got something meaningful, override the defaults. |
| * some combinations of cpu on certain emulators seems to return |
| * nonsense paddr widths (1), so trim it. */ |
| if (paddr_width > g_paddr_width) |
| g_paddr_width = paddr_width; |
| if (vaddr_width > g_vaddr_width) |
| g_vaddr_width = vaddr_width; |
| |
| LTRACEF("paddr_width %u vaddr_width %u\n", g_paddr_width, g_vaddr_width); |
| } |
| |
| void x86_mmu_init(void) {} |
| |
| X86PageTableBase::X86PageTableBase() { |
| } |
| |
| X86PageTableBase::~X86PageTableBase() { |
| DEBUG_ASSERT_MSG(!phys_, "page table dtor called before Destroy()"); |
| } |
| |
| // We disable analysis due to the write to |pages_| tripping it up. It is safe |
| // to write to |pages_| since this is part of object construction. |
| zx_status_t X86PageTableBase::Init(void* ctx) TA_NO_THREAD_SAFETY_ANALYSIS { |
| /* allocate a top level page table for the new address space */ |
| vm_page* p; |
| paddr_t pa; |
| zx_status_t status = pmm_alloc_page(0, &p, &pa); |
| if (status != ZX_OK) { |
| TRACEF("error allocating top level page directory\n"); |
| return ZX_ERR_NO_MEMORY; |
| } |
| virt_ = reinterpret_cast<pt_entry_t*>(paddr_to_physmap(pa)); |
| phys_ = pa; |
| p->state = VM_PAGE_STATE_MMU; |
| |
| // TODO(abdulla): Remove when PMM returns pre-zeroed pages. |
| arch_zero_page(virt_); |
| |
| ctx_ = ctx; |
| pages_ = 1; |
| return ZX_OK; |
| } |
| |
| // We disable analysis due to the write to |pages_| tripping it up. It is safe |
| // to write to |pages_| since this is part of object construction. |
| zx_status_t X86PageTableMmu::InitKernel(void* ctx) TA_NO_THREAD_SAFETY_ANALYSIS { |
| phys_ = kernel_pt_phys; |
| virt_ = (pt_entry_t*)X86_PHYS_TO_VIRT(phys_); |
| ctx_ = ctx; |
| pages_ = 1; |
| use_global_mappings_ = true; |
| return ZX_OK; |
| } |
| |
| zx_status_t X86PageTableMmu::AliasKernelMappings() { |
| // Copy the kernel portion of it from the master kernel pt. |
| memcpy(virt_ + NO_OF_PT_ENTRIES / 2, |
| const_cast<pt_entry_t*>(&KERNEL_PT[NO_OF_PT_ENTRIES / 2]), |
| sizeof(pt_entry_t) * NO_OF_PT_ENTRIES / 2); |
| return ZX_OK; |
| } |
| |
| X86ArchVmAspace::X86ArchVmAspace() {} |
| |
| /* |
| * Fill in the high level x86 arch aspace structure and allocating a top level page table. |
| */ |
| zx_status_t X86ArchVmAspace::Init(vaddr_t base, size_t size, uint mmu_flags) { |
| static_assert(sizeof(cpu_mask_t) == sizeof(active_cpus_), "err"); |
| canary_.Assert(); |
| |
| LTRACEF("aspace %p, base %#" PRIxPTR ", size 0x%zx, mmu_flags 0x%x\n", this, base, size, |
| mmu_flags); |
| |
| flags_ = mmu_flags; |
| base_ = base; |
| size_ = size; |
| if (mmu_flags & ARCH_ASPACE_FLAG_KERNEL) { |
| X86PageTableMmu* mmu = new (page_table_storage_) X86PageTableMmu(); |
| pt_ = mmu; |
| |
| zx_status_t status = mmu->InitKernel(this); |
| if (status != ZX_OK) { |
| return status; |
| } |
| LTRACEF("kernel aspace: pt phys %#" PRIxPTR ", virt %p\n", pt_->phys(), pt_->virt()); |
| } else if (mmu_flags & ARCH_ASPACE_FLAG_GUEST) { |
| X86PageTableEpt* ept = new (page_table_storage_) X86PageTableEpt(); |
| pt_ = ept; |
| |
| zx_status_t status = ept->Init(this); |
| if (status != ZX_OK) { |
| return status; |
| } |
| LTRACEF("guest paspace: pt phys %#" PRIxPTR ", virt %p\n", pt_->phys(), pt_->virt()); |
| } else { |
| X86PageTableMmu* mmu = new (page_table_storage_) X86PageTableMmu; |
| pt_ = mmu; |
| |
| zx_status_t status = mmu->Init(this); |
| if (status != ZX_OK) { |
| return status; |
| } |
| |
| status = mmu->AliasKernelMappings(); |
| if (status != ZX_OK) { |
| return status; |
| } |
| |
| LTRACEF("user aspace: pt phys %#" PRIxPTR ", virt %p\n", pt_->phys(), pt_->virt()); |
| } |
| fbl::atomic_init(&active_cpus_, 0); |
| |
| return ZX_OK; |
| } |
| |
| zx_status_t X86ArchVmAspace::Destroy() { |
| canary_.Assert(); |
| DEBUG_ASSERT(active_cpus_.load() == 0); |
| |
| if (flags_ & ARCH_ASPACE_FLAG_GUEST) { |
| static_cast<X86PageTableEpt*>(pt_)->Destroy(base_, size_); |
| } else { |
| static_cast<X86PageTableMmu*>(pt_)->Destroy(base_, size_); |
| } |
| return ZX_OK; |
| } |
| |
| zx_status_t X86ArchVmAspace::Unmap(vaddr_t vaddr, size_t count, size_t* unmapped) { |
| if (!IsValidVaddr(vaddr)) |
| return ZX_ERR_INVALID_ARGS; |
| |
| return pt_->UnmapPages(vaddr, count, unmapped); |
| } |
| |
| zx_status_t X86ArchVmAspace::MapContiguous(vaddr_t vaddr, paddr_t paddr, size_t count, |
| uint mmu_flags, size_t* mapped) { |
| if (!IsValidVaddr(vaddr)) |
| return ZX_ERR_INVALID_ARGS; |
| |
| return pt_->MapPagesContiguous(vaddr, paddr, count, mmu_flags, mapped); |
| } |
| |
| zx_status_t X86ArchVmAspace::Map(vaddr_t vaddr, paddr_t* phys, size_t count, |
| uint mmu_flags, size_t* mapped) { |
| if (!IsValidVaddr(vaddr)) |
| return ZX_ERR_INVALID_ARGS; |
| |
| return pt_->MapPages(vaddr, phys, count, mmu_flags, mapped); |
| } |
| |
| zx_status_t X86ArchVmAspace::Protect(vaddr_t vaddr, size_t count, uint mmu_flags) { |
| if (!IsValidVaddr(vaddr)) |
| return ZX_ERR_INVALID_ARGS; |
| |
| return pt_->ProtectPages(vaddr, count, mmu_flags); |
| } |
| |
| void X86ArchVmAspace::ContextSwitch(X86ArchVmAspace* old_aspace, X86ArchVmAspace* aspace) { |
| cpu_mask_t cpu_bit = cpu_num_to_mask(arch_curr_cpu_num()); |
| if (aspace != nullptr) { |
| aspace->canary_.Assert(); |
| paddr_t phys = aspace->pt_phys(); |
| LTRACEF_LEVEL(3, "switching to aspace %p, pt %#" PRIXPTR "\n", aspace, phys); |
| x86_set_cr3(phys); |
| |
| if (old_aspace != nullptr) { |
| old_aspace->active_cpus_.fetch_and(~cpu_bit); |
| } |
| aspace->active_cpus_.fetch_or(cpu_bit); |
| } else { |
| LTRACEF_LEVEL(3, "switching to kernel aspace, pt %#" PRIxPTR "\n", kernel_pt_phys); |
| x86_set_cr3(kernel_pt_phys); |
| if (old_aspace != nullptr) { |
| old_aspace->active_cpus_.fetch_and(~cpu_bit); |
| } |
| } |
| |
| // Cleanup io bitmap entries from previous thread. |
| if (old_aspace) |
| x86_clear_tss_io_bitmap(old_aspace->io_bitmap()); |
| |
| // Set the io bitmap for this thread. |
| if (aspace) |
| x86_set_tss_io_bitmap(aspace->io_bitmap()); |
| } |
| |
| zx_status_t X86ArchVmAspace::Query(vaddr_t vaddr, paddr_t* paddr, uint* mmu_flags) { |
| if (!IsValidVaddr(vaddr)) |
| return ZX_ERR_INVALID_ARGS; |
| |
| return pt_->QueryVaddr(vaddr, paddr, mmu_flags); |
| } |
| |
| void x86_mmu_percpu_init(void) { |
| ulong cr0 = x86_get_cr0(); |
| /* Set write protect bit in CR0*/ |
| cr0 |= X86_CR0_WP; |
| // Clear Cache disable/not write-through bits |
| cr0 &= ~(X86_CR0_NW | X86_CR0_CD); |
| x86_set_cr0(cr0); |
| |
| /* Setting the SMEP & SMAP bit in CR4 */ |
| ulong cr4 = x86_get_cr4(); |
| if (x86_feature_test(X86_FEATURE_SMEP)) |
| cr4 |= X86_CR4_SMEP; |
| if (x86_feature_test(X86_FEATURE_SMAP)) |
| cr4 |= X86_CR4_SMAP; |
| x86_set_cr4(cr4); |
| |
| // Set NXE bit in X86_MSR_IA32_EFER. |
| uint64_t efer_msr = read_msr(X86_MSR_IA32_EFER); |
| efer_msr |= X86_EFER_NXE; |
| write_msr(X86_MSR_IA32_EFER, efer_msr); |
| } |
| |
| X86ArchVmAspace::~X86ArchVmAspace() { |
| if (pt_) { |
| pt_->~X86PageTableBase(); |
| } |
| // TODO(ZX-980): check that we've destroyed the aspace. |
| } |
| |
| vaddr_t X86ArchVmAspace::PickSpot(vaddr_t base, uint prev_region_mmu_flags, |
| vaddr_t end, uint next_region_mmu_flags, |
| vaddr_t align, size_t size, uint mmu_flags) { |
| canary_.Assert(); |
| return PAGE_ALIGN(base); |
| } |