| // Copyright 2017 The Fuchsia Authors |
| // |
| // Use of this source code is governed by a MIT-style |
| // license that can be found in the LICENSE file or at |
| // https://opensource.org/licenses/MIT |
| |
| #include <bits.h> |
| |
| #include <arch/x86/descriptor.h> |
| #include <arch/x86/feature.h> |
| #include <arch/x86/pvclock.h> |
| #include <fbl/auto_call.h> |
| #include <hypervisor/cpu.h> |
| #include <hypervisor/ktrace.h> |
| #include <kernel/mp.h> |
| #include <lib/ktrace.h> |
| #include <vm/fault.h> |
| #include <vm/pmm.h> |
| #include <vm/vm_object.h> |
| #include <zircon/syscalls/hypervisor.h> |
| |
| #include "pvclock_priv.h" |
| #include "vcpu_priv.h" |
| #include "vmexit_priv.h" |
| #include "vmx_cpu_state_priv.h" |
| |
| static constexpr uint32_t kInterruptInfoValid = 1u << 31; |
| static constexpr uint32_t kInterruptInfoDeliverErrorCode = 1u << 11; |
| static constexpr uint32_t kInterruptTypeNmi = 2u << 8; |
| static constexpr uint32_t kInterruptTypeHardwareException = 3u << 8; |
| static constexpr uint32_t kInterruptTypeSoftwareException = 6u << 8; |
| static constexpr uint16_t kBaseProcessorVpid = 1; |
| |
| static zx_status_t invept(InvEpt invalidation, uint64_t eptp) { |
| uint8_t err; |
| uint64_t descriptor[] = {eptp, 0}; |
| |
| __asm__ volatile( |
| "invept %[descriptor], %[invalidation];" VMX_ERR_CHECK(err) |
| : [err] "=r"(err) |
| : [descriptor] "m"(descriptor), [invalidation] "r"(invalidation) |
| : "cc"); |
| |
| return err ? ZX_ERR_INTERNAL : ZX_OK; |
| } |
| |
| static zx_status_t vmptrld(paddr_t pa) { |
| uint8_t err; |
| |
| __asm__ volatile( |
| "vmptrld %[pa];" VMX_ERR_CHECK(err) |
| : [err] "=r"(err) |
| : [pa] "m"(pa) |
| : "cc", "memory"); |
| |
| return err ? ZX_ERR_INTERNAL : ZX_OK; |
| } |
| |
| static zx_status_t vmclear(paddr_t pa) { |
| uint8_t err; |
| |
| __asm__ volatile( |
| "vmclear %[pa];" VMX_ERR_CHECK(err) |
| : [err] "=r"(err) |
| : [pa] "m"(pa) |
| : "cc", "memory"); |
| |
| return err ? ZX_ERR_INTERNAL : ZX_OK; |
| } |
| |
| static uint64_t vmread(uint64_t field) { |
| uint8_t err; |
| uint64_t val; |
| |
| __asm__ volatile( |
| "vmread %[field], %[val];" VMX_ERR_CHECK(err) |
| : [err] "=r"(err), [val] "=m"(val) |
| : [field] "r"(field) |
| : "cc"); |
| |
| DEBUG_ASSERT(err == ZX_OK); |
| return val; |
| } |
| |
| static void vmwrite(uint64_t field, uint64_t val) { |
| uint8_t err; |
| |
| __asm__ volatile( |
| "vmwrite %[val], %[field];" VMX_ERR_CHECK(err) |
| : [err] "=r"(err) |
| : [val] "r"(val), [field] "r"(field) |
| : "cc"); |
| |
| DEBUG_ASSERT(err == ZX_OK); |
| } |
| |
| AutoVmcs::AutoVmcs(paddr_t vmcs_address) |
| : vmcs_address_(vmcs_address) { |
| DEBUG_ASSERT(!arch_ints_disabled()); |
| arch_disable_ints(); |
| __UNUSED zx_status_t status = vmptrld(vmcs_address_); |
| DEBUG_ASSERT(status == ZX_OK); |
| } |
| |
| AutoVmcs::~AutoVmcs() { |
| DEBUG_ASSERT(arch_ints_disabled()); |
| arch_enable_ints(); |
| } |
| |
| void AutoVmcs::Invalidate() { |
| #if LK_DEBUGLEVEL > 0 |
| vmcs_address_ = 0; |
| #endif |
| } |
| |
| void AutoVmcs::InterruptWindowExiting(bool enable) { |
| DEBUG_ASSERT(vmcs_address_ != 0); |
| uint32_t controls = Read(VmcsField32::PROCBASED_CTLS); |
| if (enable) { |
| controls |= kProcbasedCtlsIntWindowExiting; |
| } else { |
| controls &= ~kProcbasedCtlsIntWindowExiting; |
| } |
| Write(VmcsField32::PROCBASED_CTLS, controls); |
| } |
| |
| static bool has_error_code(uint32_t vector) { |
| switch (vector) { |
| case X86_INT_DOUBLE_FAULT: |
| case X86_INT_INVALID_TSS: |
| case X86_INT_SEGMENT_NOT_PRESENT: |
| case X86_INT_STACK_FAULT: |
| case X86_INT_GP_FAULT: |
| case X86_INT_PAGE_FAULT: |
| case X86_INT_ALIGNMENT_CHECK: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| void AutoVmcs::IssueInterrupt(uint32_t vector) { |
| DEBUG_ASSERT(vmcs_address_ != 0); |
| uint32_t interrupt_info = kInterruptInfoValid | (vector & UINT8_MAX); |
| if (vector == X86_INT_BREAKPOINT || vector == X86_INT_OVERFLOW) { |
| // From Volume 3, Section 24.8.3. A VMM should use type hardware exception for all |
| // exceptions other than breakpoints and overflows, which should be software exceptions. |
| interrupt_info |= kInterruptTypeSoftwareException; |
| } else if (vector == X86_INT_NMI) { |
| interrupt_info |= kInterruptTypeNmi; |
| } else if (vector <= X86_INT_VIRT) { |
| // From Volume 3, Section 6.15. All other vectors from 0 to X86_INT_VIRT are exceptions. |
| interrupt_info |= kInterruptTypeHardwareException; |
| } |
| if (has_error_code(vector)) { |
| interrupt_info |= kInterruptInfoDeliverErrorCode; |
| Write(VmcsField32::ENTRY_EXCEPTION_ERROR_CODE, 0); |
| } |
| |
| DEBUG_ASSERT((Read(VmcsField32::ENTRY_INTERRUPTION_INFORMATION) & kInterruptInfoValid) == 0); |
| Write(VmcsField32::ENTRY_INTERRUPTION_INFORMATION, interrupt_info); |
| } |
| |
| uint16_t AutoVmcs::Read(VmcsField16 field) const { |
| DEBUG_ASSERT(vmcs_address_ != 0); |
| return static_cast<uint16_t>(vmread(static_cast<uint64_t>(field))); |
| } |
| |
| uint32_t AutoVmcs::Read(VmcsField32 field) const { |
| DEBUG_ASSERT(vmcs_address_ != 0); |
| return static_cast<uint32_t>(vmread(static_cast<uint64_t>(field))); |
| } |
| |
| uint64_t AutoVmcs::Read(VmcsField64 field) const { |
| DEBUG_ASSERT(vmcs_address_ != 0); |
| return vmread(static_cast<uint64_t>(field)); |
| } |
| |
| uint64_t AutoVmcs::Read(VmcsFieldXX field) const { |
| DEBUG_ASSERT(vmcs_address_ != 0); |
| return vmread(static_cast<uint64_t>(field)); |
| } |
| |
| void AutoVmcs::Write(VmcsField16 field, uint16_t val) { |
| DEBUG_ASSERT(vmcs_address_ != 0); |
| vmwrite(static_cast<uint64_t>(field), val); |
| } |
| |
| void AutoVmcs::Write(VmcsField32 field, uint32_t val) { |
| DEBUG_ASSERT(vmcs_address_ != 0); |
| vmwrite(static_cast<uint64_t>(field), val); |
| } |
| |
| void AutoVmcs::Write(VmcsField64 field, uint64_t val) { |
| DEBUG_ASSERT(vmcs_address_ != 0); |
| vmwrite(static_cast<uint64_t>(field), val); |
| } |
| |
| void AutoVmcs::Write(VmcsFieldXX field, uint64_t val) { |
| DEBUG_ASSERT(vmcs_address_ != 0); |
| vmwrite(static_cast<uint64_t>(field), val); |
| } |
| |
| zx_status_t AutoVmcs::SetControl(VmcsField32 controls, uint64_t true_msr, uint64_t old_msr, |
| uint32_t set, uint32_t clear) { |
| DEBUG_ASSERT(vmcs_address_ != 0); |
| uint32_t allowed_0 = static_cast<uint32_t>(BITS(true_msr, 31, 0)); |
| uint32_t allowed_1 = static_cast<uint32_t>(BITS_SHIFT(true_msr, 63, 32)); |
| if ((allowed_1 & set) != set) { |
| dprintf(INFO, "can not set vmcs controls %#x\n", static_cast<uint>(controls)); |
| return ZX_ERR_NOT_SUPPORTED; |
| } |
| if ((~allowed_0 & clear) != clear) { |
| dprintf(INFO, "can not clear vmcs controls %#x\n", static_cast<uint>(controls)); |
| return ZX_ERR_NOT_SUPPORTED; |
| } |
| if ((set & clear) != 0) { |
| dprintf(INFO, "can not set and clear the same vmcs controls %#x\n", |
| static_cast<uint>(controls)); |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| // See Volume 3, Section 31.5.1, Algorithm 3, Part C. If the control can be |
| // either 0 or 1 (flexible), and the control is unknown, then refer to the |
| // old MSR to find the default value. |
| uint32_t flexible = allowed_0 ^ allowed_1; |
| uint32_t unknown = flexible & ~(set | clear); |
| uint32_t defaults = unknown & BITS(old_msr, 31, 0); |
| Write(controls, allowed_0 | defaults | set); |
| return ZX_OK; |
| } |
| |
| AutoPin::AutoPin(uint16_t vpid) |
| : prev_cpu_mask_(get_current_thread()->cpu_affinity), thread_(hypervisor::pin_thread(vpid)) {} |
| |
| AutoPin::~AutoPin() { |
| thread_set_cpu_affinity(thread_, prev_cpu_mask_); |
| } |
| |
| static uint64_t ept_pointer(paddr_t pml4_address) { |
| return |
| // Physical address of the PML4 page, page aligned. |
| pml4_address | |
| // Use write-back memory type for paging structures. |
| VMX_MEMORY_TYPE_WRITE_BACK << 0 | |
| // Page walk length of 4 (defined as N minus 1). |
| 3u << 3; |
| } |
| |
| struct MsrListEntry { |
| uint32_t msr; |
| uint32_t reserved; |
| uint64_t value; |
| } __PACKED; |
| |
| static void edit_msr_list(VmxPage* msr_list_page, size_t index, uint32_t msr, uint64_t value) { |
| // From Volume 3, Section 24.7.2. |
| |
| // From Volume 3, Appendix A.6: Specifically, if the value bits 27:25 of |
| // IA32_VMX_MISC is N, then 512 * (N + 1) is the recommended maximum number |
| // of MSRs to be included in each list. |
| // |
| // From Volume 3, Section 24.7.2: This field specifies the number of MSRs to |
| // be stored on VM exit. It is recommended that this count not exceed 512 |
| // bytes. |
| // |
| // Since these two statements conflict, we are taking the conservative |
| // minimum and asserting that: index < (512 bytes / size of MsrListEntry). |
| ASSERT(index < (512 / sizeof(MsrListEntry))); |
| |
| MsrListEntry* entry = msr_list_page->VirtualAddress<MsrListEntry>() + index; |
| entry->msr = msr; |
| entry->value = value; |
| } |
| |
| static zx_status_t vmcs_init(paddr_t vmcs_address, uint16_t vpid, uintptr_t entry, |
| paddr_t msr_bitmaps_address, paddr_t pml4_address, VmxState* vmx_state, |
| VmxPage* host_msr_page, VmxPage* guest_msr_page) { |
| zx_status_t status = vmclear(vmcs_address); |
| if (status != ZX_OK) |
| return status; |
| |
| AutoVmcs vmcs(vmcs_address); |
| // Setup secondary processor-based VMCS controls. |
| status = vmcs.SetControl(VmcsField32::PROCBASED_CTLS2, |
| read_msr(X86_MSR_IA32_VMX_PROCBASED_CTLS2), |
| 0, |
| // Enable use of extended page tables. |
| kProcbasedCtls2Ept | |
| // Enable use of RDTSCP instruction. |
| kProcbasedCtls2Rdtscp | |
| // Enable X2APIC. |
| kProcbasedCtls2x2Apic | |
| // Associate cached translations of linear |
| // addresses with a virtual processor ID. |
| kProcbasedCtls2Vpid | |
| // Enable unrestricted guest. |
| kProcbasedCtls2UnrestrictedGuest, |
| 0); |
| if (status != ZX_OK) |
| return status; |
| |
| // Enable use of INVPCID instruction if available. |
| vmcs.SetControl(VmcsField32::PROCBASED_CTLS2, |
| read_msr(X86_MSR_IA32_VMX_PROCBASED_CTLS2), |
| vmcs.Read(VmcsField32::PROCBASED_CTLS2), |
| kProcbasedCtls2Invpcid, |
| 0); |
| |
| // Setup pin-based VMCS controls. |
| status = vmcs.SetControl(VmcsField32::PINBASED_CTLS, |
| read_msr(X86_MSR_IA32_VMX_TRUE_PINBASED_CTLS), |
| read_msr(X86_MSR_IA32_VMX_PINBASED_CTLS), |
| // External interrupts cause a VM exit. |
| kPinbasedCtlsExtIntExiting | |
| // Non-maskable interrupts cause a VM exit. |
| kPinbasedCtlsNmiExiting, |
| 0); |
| if (status != ZX_OK) |
| return status; |
| |
| // Setup primary processor-based VMCS controls. |
| status = vmcs.SetControl(VmcsField32::PROCBASED_CTLS, |
| read_msr(X86_MSR_IA32_VMX_TRUE_PROCBASED_CTLS), |
| read_msr(X86_MSR_IA32_VMX_PROCBASED_CTLS), |
| // Enable VM exit when interrupts are enabled. |
| kProcbasedCtlsIntWindowExiting | |
| // Enable VM exit on HLT instruction. |
| kProcbasedCtlsHltExiting | |
| // Enable TPR virtualization. |
| kProcbasedCtlsTprShadow | |
| // Enable VM exit on IO instructions. |
| kProcbasedCtlsIoExiting | |
| // Enable use of MSR bitmaps. |
| kProcbasedCtlsMsrBitmaps | |
| // Enable VM exit on pause instruction. |
| kProcbasedCtlsPauseExiting | |
| // Enable secondary processor-based controls. |
| kProcbasedCtlsProcbasedCtls2, |
| // Disable VM exit on CR3 load. |
| kProcbasedCtlsCr3LoadExiting | |
| // Disable VM exit on CR3 store. |
| kProcbasedCtlsCr3StoreExiting | |
| // Disable VM exit on CR8 load. |
| kProcbasedCtlsCr8LoadExiting | |
| // Disable VM exit on CR8 store. |
| kProcbasedCtlsCr8StoreExiting); |
| if (status != ZX_OK) |
| return status; |
| |
| // We only enable interrupt-window exiting above to ensure that the |
| // processor supports it for later use. So disable it for now. |
| vmcs.InterruptWindowExiting(false); |
| |
| // Setup VM-exit VMCS controls. |
| status = vmcs.SetControl(VmcsField32::EXIT_CTLS, |
| read_msr(X86_MSR_IA32_VMX_TRUE_EXIT_CTLS), |
| read_msr(X86_MSR_IA32_VMX_EXIT_CTLS), |
| // Logical processor is in 64-bit mode after VM |
| // exit. On VM exit CS.L, IA32_EFER.LME, and |
| // IA32_EFER.LMA is set to true. |
| kExitCtls64bitMode | |
| // Save the guest IA32_PAT MSR on exit. |
| kExitCtlsSaveIa32Pat | |
| // Load the host IA32_PAT MSR on exit. |
| kExitCtlsLoadIa32Pat | |
| // Save the guest IA32_EFER MSR on exit. |
| kExitCtlsSaveIa32Efer | |
| // Load the host IA32_EFER MSR on exit. |
| kExitCtlsLoadIa32Efer | |
| // Acknowledge external interrupt on exit. |
| kExitCtlsAckIntOnExit, |
| 0); |
| if (status != ZX_OK) |
| return status; |
| |
| // Setup VM-entry VMCS controls. |
| // Load the guest IA32_PAT MSR and IA32_EFER MSR on entry. |
| uint32_t entry_ctls = kEntryCtlsLoadIa32Pat | kEntryCtlsLoadIa32Efer; |
| if (vpid == kBaseProcessorVpid) { |
| // On the BSP, go straight to IA32E mode on entry. |
| entry_ctls |= kEntryCtlsIa32eMode; |
| } |
| status = vmcs.SetControl(VmcsField32::ENTRY_CTLS, |
| read_msr(X86_MSR_IA32_VMX_TRUE_ENTRY_CTLS), |
| read_msr(X86_MSR_IA32_VMX_ENTRY_CTLS), |
| entry_ctls, 0); |
| if (status != ZX_OK) |
| return status; |
| |
| // From Volume 3, Section 24.6.3: The exception bitmap is a 32-bit field |
| // that contains one bit for each exception. When an exception occurs, |
| // its vector is used to select a bit in this field. If the bit is 1, |
| // the exception causes a VM exit. If the bit is 0, the exception is |
| // delivered normally through the IDT, using the descriptor |
| // corresponding to the exception’s vector. |
| // |
| // From Volume 3, Section 25.2: If software desires VM exits on all page |
| // faults, it can set bit 14 in the exception bitmap to 1 and set the |
| // page-fault error-code mask and match fields each to 00000000H. |
| vmcs.Write(VmcsField32::EXCEPTION_BITMAP, 0); |
| vmcs.Write(VmcsField32::PAGEFAULT_ERRORCODE_MASK, 0); |
| vmcs.Write(VmcsField32::PAGEFAULT_ERRORCODE_MATCH, 0); |
| |
| // From Volume 3, Section 28.1: Virtual-processor identifiers (VPIDs) |
| // introduce to VMX operation a facility by which a logical processor may |
| // cache information for multiple linear-address spaces. When VPIDs are |
| // used, VMX transitions may retain cached information and the logical |
| // processor switches to a different linear-address space. |
| // |
| // From Volume 3, Section 26.2.1.1: If the “enable VPID” VM-execution |
| // control is 1, the value of the VPID VM-execution control field must not |
| // be 0000H. |
| // |
| // From Volume 3, Section 28.3.3.3: If EPT is in use, the logical processor |
| // associates all mappings it creates with the value of bits 51:12 of |
| // current EPTP. If a VMM uses different EPTP values for different guests, |
| // it may use the same VPID for those guests. |
| // |
| // From Volume 3, Section 28.3.3.1: Operations that architecturally |
| // invalidate entries in the TLBs or paging-structure caches independent of |
| // VMX operation (e.g., the INVLPG and INVPCID instructions) invalidate |
| // linear mappings and combined mappings. They are required to do so only |
| // for the current VPID (but, for combined mappings, all EP4TAs). Linear |
| // mappings for the current VPID are invalidated even if EPT is in use. |
| // Combined mappings for the current VPID are invalidated even if EPT is |
| // not in use. |
| vmcs.Write(VmcsField16::VPID, vpid); |
| |
| // From Volume 3, Section 28.2: The extended page-table mechanism (EPT) is a |
| // feature that can be used to support the virtualization of physical |
| // memory. When EPT is in use, certain addresses that would normally be |
| // treated as physical addresses (and used to access memory) are instead |
| // treated as guest-physical addresses. Guest-physical addresses are |
| // translated by traversing a set of EPT paging structures to produce |
| // physical addresses that are used to access memory. |
| const auto eptp = ept_pointer(pml4_address); |
| vmcs.Write(VmcsField64::EPT_POINTER, eptp); |
| |
| // From Volume 3, Section 28.3.3.4: Software can use an INVEPT with type all |
| // ALL_CONTEXT to prevent undesired retention of cached EPT information. Here, |
| // we only care about invalidating information associated with this EPTP. |
| invept(InvEpt::SINGLE_CONTEXT, eptp); |
| |
| // Setup MSR handling. |
| vmcs.Write(VmcsField64::MSR_BITMAPS_ADDRESS, msr_bitmaps_address); |
| |
| edit_msr_list(host_msr_page, 0, X86_MSR_IA32_KERNEL_GS_BASE, |
| read_msr(X86_MSR_IA32_KERNEL_GS_BASE)); |
| edit_msr_list(host_msr_page, 1, X86_MSR_IA32_STAR, read_msr(X86_MSR_IA32_STAR)); |
| edit_msr_list(host_msr_page, 2, X86_MSR_IA32_LSTAR, read_msr(X86_MSR_IA32_LSTAR)); |
| edit_msr_list(host_msr_page, 3, X86_MSR_IA32_FMASK, read_msr(X86_MSR_IA32_FMASK)); |
| edit_msr_list(host_msr_page, 4, X86_MSR_IA32_TSC_ADJUST, read_msr(X86_MSR_IA32_TSC_ADJUST)); |
| edit_msr_list(host_msr_page, 5, X86_MSR_IA32_TSC_AUX, read_msr(X86_MSR_IA32_TSC_AUX)); |
| |
| vmcs.Write(VmcsField64::EXIT_MSR_LOAD_ADDRESS, host_msr_page->PhysicalAddress()); |
| vmcs.Write(VmcsField32::EXIT_MSR_LOAD_COUNT, 6); |
| |
| edit_msr_list(guest_msr_page, 0, X86_MSR_IA32_KERNEL_GS_BASE, 0); |
| edit_msr_list(guest_msr_page, 1, X86_MSR_IA32_STAR, 0); |
| edit_msr_list(guest_msr_page, 2, X86_MSR_IA32_LSTAR, 0); |
| edit_msr_list(guest_msr_page, 3, X86_MSR_IA32_FMASK, 0); |
| edit_msr_list(guest_msr_page, 4, X86_MSR_IA32_TSC_ADJUST, 0); |
| edit_msr_list(guest_msr_page, 5, X86_MSR_IA32_TSC_AUX, 0); |
| vmcs.Write(VmcsField64::EXIT_MSR_STORE_ADDRESS, guest_msr_page->PhysicalAddress()); |
| vmcs.Write(VmcsField32::EXIT_MSR_STORE_COUNT, 6); |
| vmcs.Write(VmcsField64::ENTRY_MSR_LOAD_ADDRESS, guest_msr_page->PhysicalAddress()); |
| vmcs.Write(VmcsField32::ENTRY_MSR_LOAD_COUNT, 6); |
| |
| // Setup VMCS host state. |
| // |
| // NOTE: We are pinned to a thread when executing this function, therefore |
| // it is acceptable to use per-CPU state. |
| x86_percpu* percpu = x86_get_percpu(); |
| vmcs.Write(VmcsField64::HOST_IA32_PAT, read_msr(X86_MSR_IA32_PAT)); |
| vmcs.Write(VmcsField64::HOST_IA32_EFER, read_msr(X86_MSR_IA32_EFER)); |
| vmcs.Write(VmcsFieldXX::HOST_CR0, x86_get_cr0()); |
| vmcs.Write(VmcsFieldXX::HOST_CR3, x86_get_cr3()); |
| vmcs.Write(VmcsFieldXX::HOST_CR4, x86_get_cr4()); |
| vmcs.Write(VmcsField16::HOST_ES_SELECTOR, 0); |
| vmcs.Write(VmcsField16::HOST_CS_SELECTOR, CODE_64_SELECTOR); |
| vmcs.Write(VmcsField16::HOST_SS_SELECTOR, DATA_SELECTOR); |
| vmcs.Write(VmcsField16::HOST_DS_SELECTOR, 0); |
| vmcs.Write(VmcsField16::HOST_FS_SELECTOR, 0); |
| vmcs.Write(VmcsField16::HOST_GS_SELECTOR, 0); |
| vmcs.Write(VmcsField16::HOST_TR_SELECTOR, TSS_SELECTOR(percpu->cpu_num)); |
| vmcs.Write(VmcsFieldXX::HOST_FS_BASE, read_msr(X86_MSR_IA32_FS_BASE)); |
| vmcs.Write(VmcsFieldXX::HOST_GS_BASE, read_msr(X86_MSR_IA32_GS_BASE)); |
| vmcs.Write(VmcsFieldXX::HOST_TR_BASE, reinterpret_cast<uint64_t>(&percpu->default_tss)); |
| vmcs.Write(VmcsFieldXX::HOST_GDTR_BASE, reinterpret_cast<uint64_t>(gdt_get())); |
| vmcs.Write(VmcsFieldXX::HOST_IDTR_BASE, reinterpret_cast<uint64_t>(idt_get_readonly())); |
| vmcs.Write(VmcsFieldXX::HOST_IA32_SYSENTER_ESP, 0); |
| vmcs.Write(VmcsFieldXX::HOST_IA32_SYSENTER_EIP, 0); |
| vmcs.Write(VmcsField32::HOST_IA32_SYSENTER_CS, 0); |
| vmcs.Write(VmcsFieldXX::HOST_RSP, reinterpret_cast<uint64_t>(vmx_state)); |
| vmcs.Write(VmcsFieldXX::HOST_RIP, reinterpret_cast<uint64_t>(vmx_exit_entry)); |
| |
| // Setup VMCS guest state. |
| uint64_t cr0 = X86_CR0_PE | // Enable protected mode |
| X86_CR0_PG | // Enable paging |
| X86_CR0_NE; // Enable internal x87 exception handling |
| if (vpid != kBaseProcessorVpid) { |
| // Disable protected mode and paging on secondary VCPUs. |
| cr0 &= ~(X86_CR0_PE | X86_CR0_PG); |
| } |
| if (cr0_is_invalid(&vmcs, cr0)) { |
| return ZX_ERR_BAD_STATE; |
| } |
| vmcs.Write(VmcsFieldXX::GUEST_CR0, cr0); |
| |
| // Ensure that CR0.NE remains set by masking and manually handling writes to CR0 that unset it. |
| vmcs.Write(VmcsFieldXX::CR0_GUEST_HOST_MASK, X86_CR0_NE); |
| vmcs.Write(VmcsFieldXX::CR0_READ_SHADOW, X86_CR0_NE); |
| |
| uint64_t cr4 = X86_CR4_VMXE; // Enable VMX |
| if (vpid == kBaseProcessorVpid) { |
| // Enable the PAE bit on the BSP for 64-bit paging. |
| cr4 |= X86_CR4_PAE; |
| } |
| if (cr_is_invalid(cr4, X86_MSR_IA32_VMX_CR4_FIXED0, X86_MSR_IA32_VMX_CR4_FIXED1)) { |
| return ZX_ERR_BAD_STATE; |
| } |
| vmcs.Write(VmcsFieldXX::GUEST_CR4, cr4); |
| |
| // For now, the guest can own all of the CR4 bits except VMXE, which it shouldn't touch. |
| // TODO(andymutton): Implement proper CR4 handling. |
| vmcs.Write(VmcsFieldXX::CR4_GUEST_HOST_MASK, X86_CR4_VMXE); |
| vmcs.Write(VmcsFieldXX::CR4_READ_SHADOW, 0); |
| |
| vmcs.Write(VmcsField64::GUEST_IA32_PAT, read_msr(X86_MSR_IA32_PAT)); |
| |
| uint64_t guest_efer = read_msr(X86_MSR_IA32_EFER); |
| if (vpid != kBaseProcessorVpid) { |
| // Disable LME and LMA on all but the BSP. |
| guest_efer &= ~(X86_EFER_LME | X86_EFER_LMA); |
| } |
| vmcs.Write(VmcsField64::GUEST_IA32_EFER, guest_efer); |
| |
| uint32_t cs_access_rights = kGuestXxAccessRightsDefault | |
| kGuestXxAccessRightsTypeE | |
| kGuestXxAccessRightsTypeCode; |
| if (vpid == kBaseProcessorVpid) { |
| // Ensure that the BSP starts with a 64-bit code segment. |
| cs_access_rights |= kGuestXxAccessRightsL; |
| } |
| vmcs.Write(VmcsField32::GUEST_CS_ACCESS_RIGHTS, cs_access_rights); |
| |
| vmcs.Write(VmcsField32::GUEST_TR_ACCESS_RIGHTS, |
| kGuestTrAccessRightsTssBusy | kGuestXxAccessRightsP); |
| |
| vmcs.Write(VmcsField32::GUEST_SS_ACCESS_RIGHTS, kGuestXxAccessRightsDefault); |
| vmcs.Write(VmcsField32::GUEST_DS_ACCESS_RIGHTS, kGuestXxAccessRightsDefault); |
| vmcs.Write(VmcsField32::GUEST_ES_ACCESS_RIGHTS, kGuestXxAccessRightsDefault); |
| vmcs.Write(VmcsField32::GUEST_FS_ACCESS_RIGHTS, kGuestXxAccessRightsDefault); |
| vmcs.Write(VmcsField32::GUEST_GS_ACCESS_RIGHTS, kGuestXxAccessRightsDefault); |
| |
| vmcs.Write(VmcsField32::GUEST_LDTR_ACCESS_RIGHTS, |
| kGuestXxAccessRightsTypeW | kGuestXxAccessRightsP); |
| |
| if (vpid == kBaseProcessorVpid) { |
| // Use GUEST_RIP to set the entry point on the BSP. |
| vmcs.Write(VmcsFieldXX::GUEST_CS_BASE, 0); |
| vmcs.Write(VmcsField16::GUEST_CS_SELECTOR, 0); |
| vmcs.Write(VmcsFieldXX::GUEST_RIP, entry); |
| } else { |
| // Use CS to set the entry point on APs. |
| vmcs.Write(VmcsFieldXX::GUEST_CS_BASE, entry); |
| vmcs.Write(VmcsField16::GUEST_CS_SELECTOR, static_cast<uint16_t>(entry >> 4)); |
| vmcs.Write(VmcsFieldXX::GUEST_RIP, 0); |
| } |
| vmcs.Write(VmcsField32::GUEST_CS_LIMIT, 0xffff); |
| vmcs.Write(VmcsFieldXX::GUEST_TR_BASE, 0); |
| vmcs.Write(VmcsField16::GUEST_TR_SELECTOR, 0); |
| vmcs.Write(VmcsField32::GUEST_TR_LIMIT, 0xffff); |
| vmcs.Write(VmcsFieldXX::GUEST_DS_BASE, 0); |
| vmcs.Write(VmcsField32::GUEST_DS_LIMIT, 0xffff); |
| vmcs.Write(VmcsFieldXX::GUEST_SS_BASE, 0); |
| vmcs.Write(VmcsField32::GUEST_SS_LIMIT, 0xffff); |
| vmcs.Write(VmcsFieldXX::GUEST_ES_BASE, 0); |
| vmcs.Write(VmcsField32::GUEST_ES_LIMIT, 0xffff); |
| vmcs.Write(VmcsFieldXX::GUEST_FS_BASE, 0); |
| vmcs.Write(VmcsField32::GUEST_FS_LIMIT, 0xffff); |
| vmcs.Write(VmcsFieldXX::GUEST_GS_BASE, 0); |
| vmcs.Write(VmcsField32::GUEST_GS_LIMIT, 0xffff); |
| vmcs.Write(VmcsField32::GUEST_LDTR_LIMIT, 0xffff); |
| vmcs.Write(VmcsFieldXX::GUEST_GDTR_BASE, 0); |
| vmcs.Write(VmcsField32::GUEST_GDTR_LIMIT, 0xffff); |
| vmcs.Write(VmcsFieldXX::GUEST_IDTR_BASE, 0); |
| vmcs.Write(VmcsField32::GUEST_IDTR_LIMIT, 0xffff); |
| |
| // Set all reserved RFLAGS bits to their correct values |
| vmcs.Write(VmcsFieldXX::GUEST_RFLAGS, X86_FLAGS_RESERVED_ONES); |
| |
| vmcs.Write(VmcsField32::GUEST_ACTIVITY_STATE, 0); |
| vmcs.Write(VmcsField32::GUEST_INTERRUPTIBILITY_STATE, 0); |
| vmcs.Write(VmcsFieldXX::GUEST_PENDING_DEBUG_EXCEPTIONS, 0); |
| |
| // From Volume 3, Section 26.3.1.1: The IA32_SYSENTER_ESP field and the |
| // IA32_SYSENTER_EIP field must each contain a canonical address. |
| vmcs.Write(VmcsFieldXX::GUEST_IA32_SYSENTER_ESP, 0); |
| vmcs.Write(VmcsFieldXX::GUEST_IA32_SYSENTER_EIP, 0); |
| vmcs.Write(VmcsField32::GUEST_IA32_SYSENTER_CS, 0); |
| |
| vmcs.Write(VmcsFieldXX::GUEST_RSP, 0); |
| vmcs.Write(VmcsFieldXX::GUEST_CR3, 0); |
| |
| // From Volume 3, Section 24.4.2: If the “VMCS shadowing” VM-execution |
| // control is 1, the VMREAD and VMWRITE instructions access the VMCS |
| // referenced by this pointer (see Section 24.10). Otherwise, software |
| // should set this field to FFFFFFFF_FFFFFFFFH to avoid VM-entry |
| // failures (see Section 26.3.1.5). |
| vmcs.Write(VmcsField64::LINK_POINTER, kLinkPointerInvalidate); |
| |
| if (x86_feature_test(X86_FEATURE_XSAVE)) { |
| // Enable x87 state in guest XCR0. |
| vmx_state->guest_state.xcr0 = X86_XSAVE_STATE_BIT_X87; |
| } |
| |
| return ZX_OK; |
| } |
| |
| // static |
| zx_status_t Vcpu::Create(Guest* guest, zx_vaddr_t entry, ktl::unique_ptr<Vcpu>* out) { |
| hypervisor::GuestPhysicalAddressSpace* gpas = guest->AddressSpace(); |
| if (entry >= gpas->size()) |
| return ZX_ERR_INVALID_ARGS; |
| |
| uint16_t vpid; |
| zx_status_t status = guest->AllocVpid(&vpid); |
| if (status != ZX_OK) { |
| return status; |
| } |
| |
| auto auto_call = fbl::MakeAutoCall([guest, vpid]() { |
| guest->FreeVpid(vpid); |
| }); |
| |
| // When we create a VCPU, we bind it to the current thread and a CPU based |
| // on the VPID. The VCPU must always be run on the current thread and the |
| // given CPU, unless an explicit migration is performed. |
| // |
| // The reason we do this is that: |
| // 1. The state of the current thread is stored within the VMCS, to be |
| // restored upon a guest-to-host transition. |
| // 2. The state of the VMCS associated with the VCPU is cached within the |
| // CPU. To move to a different CPU, we must perform an explicit migration |
| // which will cost us performance. |
| thread_t* thread = hypervisor::pin_thread(vpid); |
| |
| fbl::AllocChecker ac; |
| ktl::unique_ptr<Vcpu> vcpu(new (&ac) Vcpu(guest, vpid, thread)); |
| if (!ac.check()) |
| return ZX_ERR_NO_MEMORY; |
| |
| timer_init(&vcpu->local_apic_state_.timer); |
| status = vcpu->local_apic_state_.interrupt_tracker.Init(); |
| if (status != ZX_OK) |
| return status; |
| |
| vcpu->pvclock_state_.is_stable = |
| pvclock_is_present() ? pvclock_is_stable() : x86_feature_test(X86_FEATURE_INVAR_TSC); |
| |
| VmxInfo vmx_info; |
| status = vcpu->host_msr_page_.Alloc(vmx_info, 0); |
| if (status != ZX_OK) |
| return status; |
| |
| status = vcpu->guest_msr_page_.Alloc(vmx_info, 0); |
| if (status != ZX_OK) |
| return status; |
| |
| status = vcpu->vmcs_page_.Alloc(vmx_info, 0); |
| if (status != ZX_OK) |
| return status; |
| auto_call.cancel(); |
| |
| VmxRegion* region = vcpu->vmcs_page_.VirtualAddress<VmxRegion>(); |
| region->revision_id = vmx_info.revision_id; |
| zx_paddr_t table = gpas->arch_aspace()->arch_table_phys(); |
| status = vmcs_init(vcpu->vmcs_page_.PhysicalAddress(), vpid, entry, guest->MsrBitmapsAddress(), |
| table, &vcpu->vmx_state_, &vcpu->host_msr_page_, &vcpu->guest_msr_page_); |
| if (status != ZX_OK) |
| return status; |
| |
| *out = ktl::move(vcpu); |
| return ZX_OK; |
| } |
| |
| Vcpu::Vcpu(Guest* guest, uint16_t vpid, const thread_t* thread) |
| : guest_(guest), vpid_(vpid), thread_(thread), running_(false), vmx_state_(/* zero-init */) {} |
| |
| Vcpu::~Vcpu() { |
| if (!vmcs_page_.IsAllocated()) { |
| return; |
| } |
| timer_cancel(&local_apic_state_.timer); |
| // The destructor may be called from a different thread, therefore we must |
| // pin the current thread to the same CPU as the VCPU. |
| AutoPin pin(vpid_); |
| vmclear(vmcs_page_.PhysicalAddress()); |
| __UNUSED zx_status_t status = guest_->FreeVpid(vpid_); |
| DEBUG_ASSERT(status == ZX_OK); |
| } |
| |
| // Injects an interrupt into the guest, if there is one pending. |
| static zx_status_t local_apic_maybe_interrupt(AutoVmcs* vmcs, LocalApicState* local_apic_state) { |
| // Since hardware generated exceptions are delivered to the guest directly, the only exceptions |
| // we see here are those we generate in the VMM, e.g. GP faults in vmexit handlers. Therefore |
| // we simplify interrupt priority to 1) NMIs, 2) interrupts, and 3) generated exceptions. See |
| // Volume 3, Section 6.9, Table 6-2. |
| uint32_t vector; |
| hypervisor::InterruptType type = local_apic_state->interrupt_tracker.TryPop(X86_INT_NMI); |
| if (type != hypervisor::InterruptType::INACTIVE) { |
| vector = X86_INT_NMI; |
| } else { |
| // Pop scans vectors from highest to lowest, which will correctly pop interrupts before |
| // exceptions. All vectors <= X86_INT_VIRT except the NMI vector are exceptions. |
| type = local_apic_state->interrupt_tracker.Pop(&vector); |
| if (type == hypervisor::InterruptType::INACTIVE) { |
| return ZX_OK; |
| } |
| } |
| |
| if (vector > X86_INT_VIRT && vector < X86_INT_PLATFORM_BASE) { |
| dprintf(INFO, "Invalid interrupt vector: %u\n", vector); |
| return ZX_ERR_NOT_SUPPORTED; |
| } else if (vector >= X86_INT_PLATFORM_BASE && |
| !(vmcs->Read(VmcsFieldXX::GUEST_RFLAGS) & X86_FLAGS_IF)) { |
| // Volume 3, Section 6.8.1: The IF flag does not affect non-maskable interrupts (NMIs), |
| // [...] nor does it affect processor generated exceptions. |
| local_apic_state->interrupt_tracker.Track(vector, type); |
| // If interrupts are disabled, we set VM exit on interrupt enable. |
| vmcs->InterruptWindowExiting(true); |
| return ZX_OK; |
| } |
| |
| // If the vector is non-maskable or interrupts are enabled, we inject an interrupt. |
| vmcs->IssueInterrupt(vector); |
| |
| // Volume 3, Section 6.9: Lower priority exceptions are discarded; lower priority interrupts are |
| // held pending. Discarded exceptions are re-generated when the interrupt handler returns |
| // execution to the point in the program or task where the exceptions and/or interrupts |
| // occurred. |
| local_apic_state->interrupt_tracker.Clear(0, X86_INT_NMI); |
| local_apic_state->interrupt_tracker.Clear(X86_INT_NMI + 1, X86_INT_VIRT + 1); |
| |
| return ZX_OK; |
| } |
| |
| zx_status_t Vcpu::Resume(zx_port_packet_t* packet) { |
| if (!hypervisor::check_pinned_cpu_invariant(vpid_, thread_)) |
| return ZX_ERR_BAD_STATE; |
| zx_status_t status; |
| do { |
| AutoVmcs vmcs(vmcs_page_.PhysicalAddress()); |
| status = local_apic_maybe_interrupt(&vmcs, &local_apic_state_); |
| if (status != ZX_OK) { |
| return status; |
| } |
| if (x86_feature_test(X86_FEATURE_XSAVE)) { |
| // Save the host XCR0, and load the guest XCR0. |
| vmx_state_.host_state.xcr0 = x86_xgetbv(0); |
| x86_xsetbv(0, vmx_state_.guest_state.xcr0); |
| } |
| |
| // Updates guest system time if the guest subscribed to updates. |
| pvclock_update_system_time(&pvclock_state_, guest_->AddressSpace()); |
| |
| ktrace(TAG_VCPU_ENTER, 0, 0, 0, 0); |
| running_.store(true); |
| status = vmx_enter(&vmx_state_); |
| running_.store(false); |
| if (x86_feature_test(X86_FEATURE_XSAVE)) { |
| // Save the guest XCR0, and load the host XCR0. |
| vmx_state_.guest_state.xcr0 = x86_xgetbv(0); |
| x86_xsetbv(0, vmx_state_.host_state.xcr0); |
| } |
| |
| if (status != ZX_OK) { |
| ktrace_vcpu_exit(VCPU_FAILURE, vmcs.Read(VmcsFieldXX::GUEST_RIP)); |
| uint64_t error = vmcs.Read(VmcsField32::INSTRUCTION_ERROR); |
| dprintf(INFO, "VCPU resume failed: %#lx\n", error); |
| } else { |
| vmx_state_.resume = true; |
| status = vmexit_handler(&vmcs, &vmx_state_.guest_state, &local_apic_state_, |
| &pvclock_state_, guest_->AddressSpace(), guest_->Traps(), |
| packet); |
| } |
| } while (status == ZX_OK); |
| return status == ZX_ERR_NEXT ? ZX_OK : status; |
| } |
| |
| void vmx_exit(VmxState* vmx_state) { |
| DEBUG_ASSERT(arch_ints_disabled()); |
| |
| // Reload the task segment in order to restore its limit. VMX always |
| // restores it with a limit of 0x67, which excludes the IO bitmap. |
| seg_sel_t selector = TSS_SELECTOR(arch_curr_cpu_num()); |
| x86_clear_tss_busy(selector); |
| x86_ltr(selector); |
| } |
| |
| cpu_mask_t Vcpu::Interrupt(uint32_t vector, hypervisor::InterruptType type) { |
| bool signaled = false; |
| local_apic_state_.interrupt_tracker.Interrupt(vector, type, &signaled); |
| if (signaled || !running_.load()) { |
| return 0; |
| } |
| return cpu_num_to_mask(hypervisor::cpu_of(vpid_)); |
| } |
| |
| void Vcpu::VirtualInterrupt(uint32_t vector) { |
| cpu_mask_t mask = Interrupt(vector, hypervisor::InterruptType::VIRTUAL); |
| if (mask != 0) { |
| mp_interrupt(MP_IPI_TARGET_MASK, mask); |
| } |
| } |
| |
| template <typename Out, typename In> |
| static void register_copy(Out* out, const In& in) { |
| out->rax = in.rax; |
| out->rcx = in.rcx; |
| out->rdx = in.rdx; |
| out->rbx = in.rbx; |
| out->rbp = in.rbp; |
| out->rsi = in.rsi; |
| out->rdi = in.rdi; |
| out->r8 = in.r8; |
| out->r9 = in.r9; |
| out->r10 = in.r10; |
| out->r11 = in.r11; |
| out->r12 = in.r12; |
| out->r13 = in.r13; |
| out->r14 = in.r14; |
| out->r15 = in.r15; |
| } |
| |
| zx_status_t Vcpu::ReadState(uint32_t kind, void* buf, size_t len) const { |
| if (!hypervisor::check_pinned_cpu_invariant(vpid_, thread_)) |
| return ZX_ERR_BAD_STATE; |
| switch (kind) { |
| case ZX_VCPU_STATE: { |
| if (len != sizeof(zx_vcpu_state_t)) |
| break; |
| auto state = static_cast<zx_vcpu_state_t*>(buf); |
| register_copy(state, vmx_state_.guest_state); |
| AutoVmcs vmcs(vmcs_page_.PhysicalAddress()); |
| state->rsp = vmcs.Read(VmcsFieldXX::GUEST_RSP); |
| state->rflags = vmcs.Read(VmcsFieldXX::GUEST_RFLAGS) & X86_FLAGS_USER; |
| return ZX_OK; |
| } |
| } |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| zx_status_t Vcpu::WriteState(uint32_t kind, const void* buf, size_t len) { |
| if (!hypervisor::check_pinned_cpu_invariant(vpid_, thread_)) |
| return ZX_ERR_BAD_STATE; |
| switch (kind) { |
| case ZX_VCPU_STATE: { |
| if (len != sizeof(zx_vcpu_state_t)) |
| break; |
| auto state = static_cast<const zx_vcpu_state_t*>(buf); |
| register_copy(&vmx_state_.guest_state, *state); |
| AutoVmcs vmcs(vmcs_page_.PhysicalAddress()); |
| vmcs.Write(VmcsFieldXX::GUEST_RSP, state->rsp); |
| if (state->rflags & X86_FLAGS_RESERVED_ONES) { |
| const uint64_t rflags = vmcs.Read(VmcsFieldXX::GUEST_RFLAGS); |
| const uint64_t user_flags = (rflags & ~X86_FLAGS_USER) | |
| (state->rflags & X86_FLAGS_USER); |
| vmcs.Write(VmcsFieldXX::GUEST_RFLAGS, user_flags); |
| } |
| return ZX_OK; |
| } |
| case ZX_VCPU_IO: { |
| if (len != sizeof(zx_vcpu_io_t)) |
| break; |
| auto io = static_cast<const zx_vcpu_io_t*>(buf); |
| memcpy(&vmx_state_.guest_state.rax, io->data, io->access_size); |
| return ZX_OK; |
| } |
| } |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| bool cr0_is_invalid(AutoVmcs* vmcs, uint64_t cr0_value) { |
| uint64_t check_value = cr0_value; |
| // From Volume 3, Section 26.3.1.1: PE and PG bits of CR0 are not checked when unrestricted |
| // guest is enabled. Set both here to avoid clashing with X86_MSR_IA32_VMX_CR0_FIXED1. |
| if (vmcs->Read(VmcsField32::PROCBASED_CTLS2) & kProcbasedCtls2UnrestrictedGuest) { |
| check_value |= X86_CR0_PE | X86_CR0_PG; |
| } |
| return cr_is_invalid(check_value, X86_MSR_IA32_VMX_CR0_FIXED0, X86_MSR_IA32_VMX_CR0_FIXED1); |
| } |
