// Copyright 2017 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT

#include <bits.h>
#include <inttypes.h>
#include <lib/ktrace.h>
#include <platform.h>
#include <string.h>
#include <trace.h>
#include <zircon/syscalls/hypervisor.h>
#include <zircon/time.h>
#include <zircon/types.h>

#include <arch/hypervisor.h>
#include <arch/x86/apic.h>
#include <arch/x86/feature.h>
#include <arch/x86/mmu.h>
#include <arch/x86/pv.h>
#include <explicit-memory/bytes.h>
#include <fbl/canary.h>
#include <hypervisor/interrupt_tracker.h>
#include <hypervisor/ktrace.h>
#include <kernel/auto_lock.h>
#include <kernel/percpu.h>
#include <kernel/stats.h>
#include <platform/pc/timer.h>
#include <vm/fault.h>
#include <vm/physmap.h>
#include <vm/pmm.h>

#include "pv_priv.h"
#include "vcpu_priv.h"
#include "vmexit_priv.h"

#define LOCAL_TRACE 0

static constexpr uint64_t kLocalApicPhysBase =
    APIC_PHYS_BASE | IA32_APIC_BASE_XAPIC_ENABLE | IA32_APIC_BASE_X2APIC_ENABLE;

static constexpr uint64_t kX2ApicMsrBase = 0x800;
static constexpr uint64_t kX2ApicMsrMax = 0x83f;

static constexpr uint64_t kMiscEnableFastStrings = 1u << 0;

static constexpr uint32_t kFirstExtendedStateComponent = 2;
static constexpr uint32_t kLastExtendedStateComponent = 9;
// From Volume 1, Section 13.4.
static constexpr uint32_t kXsaveLegacyRegionSize = 512;
static constexpr uint32_t kXsaveHeaderSize = 64;

static constexpr char kHypVendorId[] = "KVMKVMKVM\0\0\0";
static constexpr size_t kHypVendorIdLength = 12;
static_assert(sizeof(kHypVendorId) - 1 == kHypVendorIdLength, "");

static constexpr uint64_t kKvmFeatureNoIoDelay = 1u << 1;

static void dump_guest_state(const GuestState& guest_state, const ExitInfo& exit_info) {
  dprintf(CRITICAL, " RAX: %#18lx  RCX: %#18lx  RDX: %#18lx  RBX: %#18lx\n", guest_state.rax,
          guest_state.rcx, guest_state.rdx, guest_state.rbx);
  dprintf(CRITICAL, " RBP: %#18lx  RSI: %#18lx  RDI: %#18lx\n", guest_state.rbp, guest_state.rsi,
          guest_state.rdi);
  dprintf(CRITICAL, "  R8: %#18lx   R9: %#18lx  R10: %#18lx  R11: %#18lx\n", guest_state.r8,
          guest_state.r9, guest_state.r10, guest_state.r11);
  dprintf(CRITICAL, " R12: %#18lx  R13: %#18lx  R14: %#18lx  R15: %#18lx\n", guest_state.r12,
          guest_state.r13, guest_state.r14, guest_state.r15);
  dprintf(CRITICAL, " RIP: %#18lx  CR2: %#18lx XCR0: %#18lx\n", exit_info.guest_rip,
          guest_state.cr2, guest_state.xcr0);

  dprintf(CRITICAL, "entry failure: %d\n", exit_info.entry_failure);
  dprintf(CRITICAL, "exit qualification: %#lx\n", exit_info.exit_qualification);
  dprintf(CRITICAL, "exit instruction length: %#x\n", exit_info.exit_instruction_length);
  dprintf(CRITICAL, "guest physical address: %#lx\n", exit_info.guest_physical_address);
}

extern "C" void x86_call_external_interrupt_handler(uint64_t vector);

ExitInfo::ExitInfo(const AutoVmcs& vmcs) {
  // From Volume 3, Section 26.7.
  uint32_t full_exit_reason = vmcs.Read(VmcsField32::EXIT_REASON);
  entry_failure = BIT(full_exit_reason, 31);
  exit_reason = static_cast<ExitReason>(BITS(full_exit_reason, 15, 0));

  exit_qualification = vmcs.Read(VmcsFieldXX::EXIT_QUALIFICATION);
  exit_instruction_length = vmcs.Read(VmcsField32::EXIT_INSTRUCTION_LENGTH);
  guest_physical_address = vmcs.Read(VmcsField64::GUEST_PHYSICAL_ADDRESS);
  guest_rip = vmcs.Read(VmcsFieldXX::GUEST_RIP);

  if (exit_reason == ExitReason::EXTERNAL_INTERRUPT || exit_reason == ExitReason::IO_INSTRUCTION)
    return;

  LTRACEF("entry failure: %d\n", entry_failure);
  LTRACEF("exit reason: %#x (%s)\n", static_cast<uint32_t>(exit_reason),
          exit_reason_name(exit_reason));
  LTRACEF("exit qualification: %#lx\n", exit_qualification);
  LTRACEF("exit instruction length: %#x\n", exit_instruction_length);
  LTRACEF("guest activity state: %#x\n", vmcs.Read(VmcsField32::GUEST_ACTIVITY_STATE));
  LTRACEF("guest interruptibility state: %#x\n",
          vmcs.Read(VmcsField32::GUEST_INTERRUPTIBILITY_STATE));
  LTRACEF("guest physical address: %#lx\n", guest_physical_address);
  LTRACEF("guest linear address: %#lx\n", vmcs.Read(VmcsFieldXX::GUEST_LINEAR_ADDRESS));
  LTRACEF("guest rip: %#lx\n", guest_rip);
}

ExitInterruptionInformation::ExitInterruptionInformation(const AutoVmcs& vmcs) {
  uint32_t int_info = vmcs.Read(VmcsField32::EXIT_INTERRUPTION_INFORMATION);
  vector = static_cast<uint8_t>(BITS(int_info, 7, 0));
  interruption_type = static_cast<InterruptionType>(BITS_SHIFT(int_info, 10, 8));
  valid = BIT(int_info, 31);
}

CrAccessInfo::CrAccessInfo(uint64_t qualification) {
  // From Volume 3, Table 27-3.
  cr_number = static_cast<uint8_t>(BITS(qualification, 3, 0));
  access_type = static_cast<CrAccessType>(BITS_SHIFT(qualification, 5, 4));
  reg = static_cast<uint8_t>(BITS_SHIFT(qualification, 11, 8));
}

IoInfo::IoInfo(uint64_t qualification) {
  access_size = static_cast<uint8_t>(BITS(qualification, 2, 0) + 1);
  input = BIT_SHIFT(qualification, 3);
  string = BIT_SHIFT(qualification, 4);
  repeat = BIT_SHIFT(qualification, 5);
  port = static_cast<uint16_t>(BITS_SHIFT(qualification, 31, 16));
}

EptViolationInfo::EptViolationInfo(uint64_t qualification) {
  // From Volume 3C, Table 27-7.
  read = BIT(qualification, 0);
  write = BIT(qualification, 1);
  instruction = BIT(qualification, 2);
}

InterruptCommandRegister::InterruptCommandRegister(uint32_t hi, uint32_t lo) {
  destination = hi;
  destination_mode = static_cast<InterruptDestinationMode>(BIT_SHIFT(lo, 11));
  delivery_mode = static_cast<InterruptDeliveryMode>(BITS_SHIFT(lo, 10, 8));
  destination_shorthand = static_cast<InterruptDestinationShorthand>(BITS_SHIFT(lo, 19, 18));
  vector = static_cast<uint8_t>(BITS(lo, 7, 0));
}

VmCallInfo::VmCallInfo(const GuestState* guest_state) {
  // ABI is documented in Linux kernel documentation, see
  // Documents/virtual/kvm/hypercalls.txt
  type = static_cast<VmCallType>(guest_state->rax);
  arg[0] = guest_state->rbx;
  arg[1] = guest_state->rcx;
  arg[2] = guest_state->rdx;
  arg[3] = guest_state->rsi;
}

static void next_rip(const ExitInfo& exit_info, AutoVmcs* vmcs) {
  vmcs->Write(VmcsFieldXX::GUEST_RIP, exit_info.guest_rip + exit_info.exit_instruction_length);

  // Clear any flags blocking interrupt injection for a single instruction.
  uint32_t guest_interruptibility = vmcs->Read(VmcsField32::GUEST_INTERRUPTIBILITY_STATE);
  uint32_t new_interruptibility =
      guest_interruptibility & ~(kInterruptibilityStiBlocking | kInterruptibilityMovSsBlocking);
  if (new_interruptibility != guest_interruptibility) {
    vmcs->Write(VmcsField32::GUEST_INTERRUPTIBILITY_STATE, new_interruptibility);
  }
}

static zx_status_t handle_external_interrupt(AutoVmcs* vmcs) {
  ExitInterruptionInformation int_info(*vmcs);
  DEBUG_ASSERT(int_info.valid);
  DEBUG_ASSERT(int_info.interruption_type == InterruptionType::EXTERNAL_INTERRUPT);
  vmcs->Invalidate();
  x86_call_external_interrupt_handler(int_info.vector);

  // If we are receiving an external interrupt because the thread is being
  // killed, we should exit with an error.
  return Thread::Current::Get()->signals_ & THREAD_SIGNAL_KILL ? ZX_ERR_CANCELED : ZX_OK;
}

static zx_status_t handle_interrupt_window(AutoVmcs* vmcs, LocalApicState* local_apic_state) {
  vmcs->InterruptWindowExiting(false);
  return ZX_OK;
}

// From Volume 2, Section 3.2, Table 3-8  "Processor Extended State Enumeration
// Main Leaf (EAX = 0DH, ECX = 0)".
//
// Bits 31-00: Maximum size (bytes, from the beginning of the XSAVE/XRSTOR save
// area) required by enabled features in XCR0. May be different than ECX if some
// features at the end of the XSAVE save area are not enabled.
static zx_status_t compute_xsave_size(uint64_t guest_xcr0, uint32_t* xsave_size) {
  *xsave_size = kXsaveLegacyRegionSize + kXsaveHeaderSize;
  for (uint32_t i = kFirstExtendedStateComponent; i <= kLastExtendedStateComponent; ++i) {
    cpuid_leaf leaf;
    if (!(guest_xcr0 & (1 << i)))
      continue;
    if (!x86_get_cpuid_subleaf(X86_CPUID_XSAVE, i, &leaf))
      return ZX_ERR_INTERNAL;
    if (leaf.a == 0 && leaf.b == 0 && leaf.c == 0 && leaf.d == 0)
      continue;
    const uint32_t component_offset = leaf.b;
    const uint32_t component_size = leaf.a;
    *xsave_size = component_offset + component_size;
  }
  return ZX_OK;
}

static zx_status_t handle_cpuid(const ExitInfo& exit_info, AutoVmcs* vmcs,
                                GuestState* guest_state) {
  const uint32_t leaf = static_cast<uint32_t>(guest_state->rax);
  const uint32_t subleaf = static_cast<uint32_t>(guest_state->rcx);

  next_rip(exit_info, vmcs);
  switch (leaf) {
    case X86_CPUID_BASE:
    case X86_CPUID_EXT_BASE:
      cpuid(leaf, reinterpret_cast<uint32_t*>(&guest_state->rax),
            reinterpret_cast<uint32_t*>(&guest_state->rbx),
            reinterpret_cast<uint32_t*>(&guest_state->rcx),
            reinterpret_cast<uint32_t*>(&guest_state->rdx));
      return ZX_OK;
    case X86_CPUID_BASE + 1 ... MAX_SUPPORTED_CPUID:
    case X86_CPUID_EXT_BASE + 1 ... MAX_SUPPORTED_CPUID_EXT:
      cpuid_c(leaf, subleaf, reinterpret_cast<uint32_t*>(&guest_state->rax),
              reinterpret_cast<uint32_t*>(&guest_state->rbx),
              reinterpret_cast<uint32_t*>(&guest_state->rcx),
              reinterpret_cast<uint32_t*>(&guest_state->rdx));
      switch (leaf) {
        case X86_CPUID_MODEL_FEATURES:
          // Override the initial local APIC ID. From Vol 2, Table 3-8.
          guest_state->rbx &= ~(0xff << 24);
          guest_state->rbx |= (vmcs->Read(VmcsField16::VPID) - 1) << 24;
          // Enable the hypervisor bit.
          guest_state->rcx |= 1u << X86_FEATURE_HYPERVISOR.bit;
          // Enable the x2APIC bit.
          guest_state->rcx |= 1u << X86_FEATURE_X2APIC.bit;
          // Disable the VMX bit.
          guest_state->rcx &= ~(1u << X86_FEATURE_VMX.bit);
          // Disable the PDCM bit.
          guest_state->rcx &= ~(1u << X86_FEATURE_PDCM.bit);
          // Disable MONITOR/MWAIT.
          guest_state->rcx &= ~(1u << X86_FEATURE_MON.bit);
          // Disable THERM_INTERRUPT and THERM_STATUS MSRs
          guest_state->rcx &= ~(1u << X86_FEATURE_TM2.bit);
          // Enable the SEP (SYSENTER support).
          guest_state->rdx |= 1u << X86_FEATURE_SEP.bit;
          // Disable the Thermal Monitor bit.
          guest_state->rdx &= ~(1u << X86_FEATURE_TM.bit);
          // Disable the THERM_CONTROL_MSR bit.
          guest_state->rdx &= ~(1u << X86_FEATURE_ACPI.bit);
          break;
        case X86_CPUID_TOPOLOGY:
          guest_state->rax = 0;
          guest_state->rbx = 0;
          guest_state->rcx = 0;
          guest_state->rdx = vmcs->Read(VmcsField16::VPID) - 1;
          break;
        case X86_CPUID_XSAVE:
          if (subleaf == 0) {
            uint32_t xsave_size = 0;
            zx_status_t status = compute_xsave_size(guest_state->xcr0, &xsave_size);
            if (status != ZX_OK)
              return status;
            guest_state->rbx = xsave_size;
          } else if (subleaf == 1) {
            guest_state->rax &= ~(1u << 3);
          }
          break;
        case X86_CPUID_THERMAL_AND_POWER:
          // Disable the performance energy bias bit.
          guest_state->rcx &= ~(1u << X86_FEATURE_PERF_BIAS.bit);
          // Disable the hardware coordination feedback bit.
          guest_state->rcx &= ~(1u << X86_FEATURE_HW_FEEDBACK.bit);
          guest_state->rax &= ~(
              // Disable Digital Thermal Sensor
              1u << X86_FEATURE_DTS.bit |
              // Disable Package Thermal Status MSR.
              1u << X86_FEATURE_PTM.bit |
              // Disable THERM_STATUS MSR bits 10/11 & THERM_INTERRUPT MSR bit 24
              1u << X86_FEATURE_PTM.bit |
              // Disable HWP MSRs.
              1u << X86_FEATURE_HWP.bit | 1u << X86_FEATURE_HWP_NOT.bit |
              1u << X86_FEATURE_HWP_ACT.bit | 1u << X86_FEATURE_HWP_PREF.bit);
          break;
        case X86_CPUID_PERFORMANCE_MONITORING: {
          // Disable all performance monitoring.
          // 31-07 = Reserved 0, 06-00 = 1 if event is not available.
          const uint32_t performance_monitoring_no_events = 0b1111111;
          guest_state->rax = 0;
          guest_state->rbx = performance_monitoring_no_events;
          guest_state->rcx = 0;
          guest_state->rdx = 0;
          break;
        }
        case X86_CPUID_MON:
          // MONITOR/MWAIT are not implemented.
          guest_state->rax = 0;
          guest_state->rbx = 0;
          guest_state->rcx = 0;
          guest_state->rdx = 0;
          break;
        case X86_CPUID_EXTENDED_FEATURE_FLAGS:
          // It's possible when running under KVM in nVMX mode, that host
          // CPUID indicates that invpcid is supported but VMX doesn't allow
          // to enable INVPCID bit in secondary processor based controls.
          // Therefore explicitly clear INVPCID bit in CPUID if the VMX flag
          // wasn't set.
          if ((vmcs->Read(VmcsField32::PROCBASED_CTLS2) & kProcbasedCtls2Invpcid) == 0)
            guest_state->rbx &= ~(1u << X86_FEATURE_INVPCID.bit);
          // Disable the Processor Trace bit.
          guest_state->rbx &= ~(1u << X86_FEATURE_PT.bit);
          // Disable:
          //  * Indirect Branch Prediction Barrier bit
          //  * Single Thread Indirect Branch Predictors bit
          //  * Speculative Store Bypass Disable bit
          // These imply support for the IA32_SPEC_CTRL and IA32_PRED_CMD
          // MSRs, which are not implemented.
          guest_state->rdx &= ~(1u << X86_FEATURE_IBRS_IBPB.bit | 1u << X86_FEATURE_STIBP.bit |
                                1u << X86_FEATURE_SSBD.bit);
          // Disable support of IA32_ARCH_CAPABILITIES MSR.
          guest_state->rdx &= ~(1u << X86_FEATURE_ARCH_CAPABILITIES.bit);
          break;
      }
      return ZX_OK;
    case X86_CPUID_HYP_VENDOR: {
      // This leaf is commonly used to identify a hypervisor via ebx:ecx:edx.
      static const uint32_t* regs = reinterpret_cast<const uint32_t*>(kHypVendorId);
      // Since Zircon hypervisor disguises itself as KVM, it needs to return
      // in EAX max CPUID function supported by hypervisor. Zero in EAX
      // should be interpreted as 0x40000001. Details are available in the
      // Linux kernel documentation (Documentation/virtual/kvm/cpuid.txt).
      guest_state->rax = X86_CPUID_KVM_FEATURES;
      guest_state->rbx = regs[0];
      guest_state->rcx = regs[1];
      guest_state->rdx = regs[2];
      return ZX_OK;
    }
    case X86_CPUID_KVM_FEATURES:
      // We support KVM clock.
      guest_state->rax = kKvmFeatureClockSourceOld | kKvmFeatureClockSource | kKvmFeatureNoIoDelay;
      guest_state->rbx = 0;
      guest_state->rcx = 0;
      guest_state->rdx = 0;
      return ZX_OK;
    // From Volume 2A, CPUID instruction reference. If the EAX value is outside
    // the range recognized by CPUID then the information for the highest
    // supported base information leaf is returned. Any value in ECX is
    // honored.
    default:
      cpuid_c(MAX_SUPPORTED_CPUID, subleaf, reinterpret_cast<uint32_t*>(&guest_state->rax),
              reinterpret_cast<uint32_t*>(&guest_state->rbx),
              reinterpret_cast<uint32_t*>(&guest_state->rcx),
              reinterpret_cast<uint32_t*>(&guest_state->rdx));
      return ZX_OK;
  }
}

static zx_status_t handle_hlt(const ExitInfo& exit_info, AutoVmcs* vmcs,
                              LocalApicState* local_apic_state) {
  next_rip(exit_info, vmcs);
  return local_apic_state->interrupt_tracker.Wait(ZX_TIME_INFINITE, vmcs);
}

static zx_status_t handle_cr0_write(AutoVmcs* vmcs, GuestState* guest_state, uint64_t val,
                                    LocalApicState* local_apic_state) {
  // X86_CR0_NE is masked so that guests may write to it, but depending on
  // IA32_VMX_CR0_FIXED0 it might be unsupported in VMX operation to set it to
  // zero. Allow the guest to control its value in CR0_READ_SHADOW but not in
  // GUEST_CR0 so that GUEST_CR0 stays valid.
  uint64_t cr0 = val | X86_CR0_NE;
  if (cr0_is_invalid(vmcs, cr0)) {
    return ZX_ERR_INVALID_ARGS;
  }

  // From Volume 3, Table 11-5: CD=0 and NW=1 is an invalid setting and should
  // generate a GP fault.
  if (!(val & X86_CR0_CD) && (val & X86_CR0_NW)) {
    local_apic_state->interrupt_tracker.VirtualInterrupt(X86_INT_GP_FAULT);
    return ZX_OK;
  }

  // From Volume 3, Section 26.3.2.1: CR0 is loaded from the CR0 field with the
  // exception of the following bits, which are never modified on VM entry: ET
  // (bit 4); reserved bits ...; NW (bit 29) and CD (bit 30). The values of
  // these bits in the CR0 field are ignored.
  //
  // Even though these bits will be ignored on VM entry, to ensure that
  // GUEST_CR0 matches the actual value of CR0 while the guest is running set
  // those bits to match the host values. This is done only to make debugging
  // simpler.
  cr0 &= ~(X86_CR0_NW | X86_CR0_CD);
  cr0 |= X86_CR0_ET;
  vmcs->Write(VmcsFieldXX::GUEST_CR0, cr0);

  // From Volume 3, Section 25.3: For each position corresponding to a bit clear
  // in the CR0 guest/host mask, the destination operand is loaded with the
  // value of the corresponding bit in CR0. For each position corresponding to a
  // bit set in the CR0 guest/host mask, the destination operand is loaded with
  // the value of the corresponding bit in the CR0 read shadow.
  //
  // Allow the guest to control the shadow.
  vmcs->Write(VmcsFieldXX::CR0_READ_SHADOW, val);

  // From Volume 3, Section 26.3.1.1: If CR0.PG and EFER.LME are set then
  // EFER.LMA and the IA-32e mode guest entry control must also be set.
  uint64_t efer = vmcs->Read(VmcsField64::GUEST_IA32_EFER);
  if (!(efer & X86_EFER_LME && cr0 & X86_CR0_PG)) {
    return ZX_OK;
  }
  vmcs->Write(VmcsField64::GUEST_IA32_EFER, efer | X86_EFER_LMA);
  return vmcs->SetControl(VmcsField32::ENTRY_CTLS, read_msr(X86_MSR_IA32_VMX_TRUE_ENTRY_CTLS),
                          read_msr(X86_MSR_IA32_VMX_ENTRY_CTLS), kEntryCtlsIa32eMode, 0);
}

static zx_status_t register_value(AutoVmcs* vmcs, GuestState* guest_state, uint8_t register_id,
                                  uint64_t* out) {
  switch (register_id) {
    // From Intel Volume 3, Table 27-3.
    case 0:
      *out = guest_state->rax;
      return ZX_OK;
    case 1:
      *out = guest_state->rcx;
      return ZX_OK;
    case 2:
      *out = guest_state->rdx;
      return ZX_OK;
    case 3:
      *out = guest_state->rbx;
      return ZX_OK;
    case 4:
      *out = vmcs->Read(VmcsFieldXX::GUEST_RSP);
      return ZX_OK;
    case 5:
      *out = guest_state->rbp;
      return ZX_OK;
    case 6:
      *out = guest_state->rsi;
      return ZX_OK;
    case 7:
      *out = guest_state->rdi;
      return ZX_OK;
    case 8:
      *out = guest_state->r8;
      return ZX_OK;
    case 9:
      *out = guest_state->r9;
      return ZX_OK;
    case 10:
      *out = guest_state->r10;
      return ZX_OK;
    case 11:
      *out = guest_state->r11;
      return ZX_OK;
    case 12:
      *out = guest_state->r12;
      return ZX_OK;
    case 13:
      *out = guest_state->r13;
      return ZX_OK;
    case 14:
      *out = guest_state->r14;
      return ZX_OK;
    case 15:
      *out = guest_state->r15;
      return ZX_OK;
    default:
      return ZX_ERR_INVALID_ARGS;
  }
}

static zx_status_t handle_control_register_access(const ExitInfo& exit_info, AutoVmcs* vmcs,
                                                  GuestState* guest_state,
                                                  LocalApicState* local_apic_state) {
  CrAccessInfo cr_access_info(exit_info.exit_qualification);
  switch (cr_access_info.access_type) {
    case CrAccessType::MOV_TO_CR: {
      // Handle CR0 only.
      if (cr_access_info.cr_number != 0) {
        return ZX_ERR_NOT_SUPPORTED;
      }
      uint64_t val;
      zx_status_t status = register_value(vmcs, guest_state, cr_access_info.reg, &val);
      if (status != ZX_OK) {
        return status;
      }
      status = handle_cr0_write(vmcs, guest_state, val, local_apic_state);
      if (status != ZX_OK) {
        return status;
      }
      next_rip(exit_info, vmcs);
      return ZX_OK;
    }
    default:
      return ZX_ERR_NOT_SUPPORTED;
  }
}

static zx_status_t handle_io_instruction(const ExitInfo& exit_info, AutoVmcs* vmcs,
                                         GuestState* guest_state, hypervisor::TrapMap* traps,
                                         zx_port_packet_t* packet) {
  IoInfo io_info(exit_info.exit_qualification);
  if (io_info.string || io_info.repeat) {
    dprintf(CRITICAL, "Unsupported IO instruction\n");
    return ZX_ERR_NOT_SUPPORTED;
  }

  hypervisor::Trap* trap;
  zx_status_t status = traps->FindTrap(ZX_GUEST_TRAP_IO, io_info.port, &trap);
  if (status != ZX_OK) {
    dprintf(CRITICAL, "Unhandled IO port %s %#x\n", io_info.input ? "in" : "out", io_info.port);
    return status;
  }
  next_rip(exit_info, vmcs);

  memset(packet, 0, sizeof(*packet));
  packet->key = trap->key();
  packet->type = ZX_PKT_TYPE_GUEST_IO;
  packet->guest_io.port = io_info.port;
  packet->guest_io.access_size = io_info.access_size;
  packet->guest_io.input = io_info.input;
  if (io_info.input) {
    // From Volume 1, Section 3.4.1.1: 32-bit operands generate a 32-bit
    // result, zero-extended to a 64-bit result in the destination general-
    // purpose register.
    if (io_info.access_size == 4)
      guest_state->rax = 0;
  } else {
    memcpy(packet->guest_io.data, &guest_state->rax, io_info.access_size);
    if (trap->HasPort())
      return trap->Queue(*packet, vmcs);
    // If there was no port for the range, then return to user-space.
  }

  return ZX_ERR_NEXT;
}

static zx_status_t handle_apic_rdmsr(const ExitInfo& exit_info, AutoVmcs* vmcs,
                                     GuestState* guest_state, LocalApicState* local_apic_state) {
  switch (static_cast<X2ApicMsr>(guest_state->rcx)) {
    case X2ApicMsr::ID:
      next_rip(exit_info, vmcs);
      guest_state->rax = vmcs->Read(VmcsField16::VPID) - 1;
      return ZX_OK;
    case X2ApicMsr::VERSION: {
      next_rip(exit_info, vmcs);
      // We choose 15H as it causes us to be seen as a modern APIC by Linux,
      // and is the highest non-reserved value. See Volume 3 Section 10.4.8.
      const uint32_t version = 0x15;
      const uint32_t max_lvt_entry = 0x6;  // LVT entries minus 1.
      const uint32_t eoi_suppression = 0;  // Disable support for EOI-broadcast suppression.
      guest_state->rax = version | (max_lvt_entry << 16) | (eoi_suppression << 24);
      return ZX_OK;
    }
    case X2ApicMsr::SVR:
      // Spurious interrupt vector resets to 0xff. See Volume 3 Section 10.12.5.1.
      next_rip(exit_info, vmcs);
      guest_state->rax = 0xff;
      return ZX_OK;
    case X2ApicMsr::TPR:
    case X2ApicMsr::LDR:
    case X2ApicMsr::ISR_31_0... X2ApicMsr::ISR_255_224:
    case X2ApicMsr::TMR_31_0... X2ApicMsr::TMR_255_224:
    case X2ApicMsr::IRR_31_0... X2ApicMsr::IRR_255_224:
    case X2ApicMsr::ESR:
    case X2ApicMsr::LVT_MONITOR:
      // These registers reset to 0. See Volume 3 Section 10.12.5.1.
      next_rip(exit_info, vmcs);
      guest_state->rax = 0;
      return ZX_OK;
    case X2ApicMsr::LVT_LINT0:
    case X2ApicMsr::LVT_LINT1:
    case X2ApicMsr::LVT_THERMAL_SENSOR:
    case X2ApicMsr::LVT_CMCI:
      // LVT registers reset with the mask bit set. See Volume 3 Section 10.12.5.1.
      next_rip(exit_info, vmcs);
      guest_state->rax = LVT_MASKED;
      return ZX_OK;
    case X2ApicMsr::LVT_TIMER:
      next_rip(exit_info, vmcs);
      guest_state->rax = local_apic_state->lvt_timer;
      return ZX_OK;
    default:
      // Issue a general protection fault for write only and unimplemented
      // registers.
      dprintf(INFO, "Unhandled x2APIC rdmsr %#lx\n", guest_state->rcx);
      local_apic_state->interrupt_tracker.VirtualInterrupt(X86_INT_GP_FAULT);
      return ZX_OK;
  }
}

static zx_status_t handle_rdmsr(const ExitInfo& exit_info, AutoVmcs* vmcs, GuestState* guest_state,
                                LocalApicState* local_apic_state) {
  // On execution of rdmsr, rcx specifies the MSR and the value is loaded into edx:eax.
  switch (static_cast<uint32_t>(guest_state->rcx)) {
    case X86_MSR_IA32_APIC_BASE:
      next_rip(exit_info, vmcs);
      guest_state->rax = kLocalApicPhysBase;
      if (vmcs->Read(VmcsField16::VPID) == 1)
        guest_state->rax |= IA32_APIC_BASE_BSP;
      guest_state->rdx = 0;
      return ZX_OK;
    // From Volume 4, Section 2.1, Table 2-2: For now, only enable fast strings.
    case X86_MSR_IA32_MISC_ENABLE:
      next_rip(exit_info, vmcs);
      guest_state->rax = read_msr(X86_MSR_IA32_MISC_ENABLE) & kMiscEnableFastStrings;
      guest_state->rdx = 0;
      return ZX_OK;
    case X86_MSR_DRAM_ENERGY_STATUS:
    case X86_MSR_DRAM_POWER_LIMIT:
    // From Volume 3, Section 28.2.6.2: The MTRRs have no effect on the memory
    // type used for an access to a guest-physical address.
    case X86_MSR_IA32_MTRRCAP:
    case X86_MSR_IA32_MTRR_DEF_TYPE:
    case X86_MSR_IA32_MTRR_FIX64K_00000:
    case X86_MSR_IA32_MTRR_FIX16K_80000 ... X86_MSR_IA32_MTRR_FIX16K_A0000:
    case X86_MSR_IA32_MTRR_FIX4K_C0000 ... X86_MSR_IA32_MTRR_FIX4K_F8000:
    case X86_MSR_IA32_MTRR_PHYSBASE0 ... X86_MSR_IA32_MTRR_PHYSMASK9:
    // From Volume 3, Section 9.11.4: For now, 0.
    case X86_MSR_IA32_PLATFORM_ID:
    // From Volume 3, Section 9.11.7: 0 indicates no microcode update is loaded.
    case X86_MSR_IA32_BIOS_SIGN_ID:
    // From Volume 3, Section 15.3.1: 0 indicates that our machine has no
    // checking capabilities.
    case X86_MSR_IA32_MCG_CAP:
    case X86_MSR_IA32_MCG_STATUS:
    case X86_MSR_IA32_TEMPERATURE_TARGET:
    case X86_MSR_PKG_ENERGY_STATUS:
    case X86_MSR_PLATFORM_ENERGY_COUNTER:
    case X86_MSR_PLATFORM_POWER_LIMIT:
    case X86_MSR_PP0_ENERGY_STATUS:
    case X86_MSR_PP0_POWER_LIMIT:
    case X86_MSR_PP1_ENERGY_STATUS:
    case X86_MSR_PP1_POWER_LIMIT:
    case X86_MSR_RAPL_POWER_UNIT:
      next_rip(exit_info, vmcs);
      guest_state->rax = 0;
      guest_state->rdx = 0;
      return ZX_OK;
    case kX2ApicMsrBase ... kX2ApicMsrMax:
      return handle_apic_rdmsr(exit_info, vmcs, guest_state, local_apic_state);
    default:
      dprintf(INFO, "Unhandled rdmsr %#lx\n", guest_state->rcx);
      local_apic_state->interrupt_tracker.VirtualInterrupt(X86_INT_GP_FAULT);
      return ZX_OK;
  }
}

zx_time_t lvt_deadline(LocalApicState* local_apic_state) {
  if ((local_apic_state->lvt_timer & LVT_TIMER_MODE_MASK) != LVT_TIMER_MODE_ONESHOT &&
      (local_apic_state->lvt_timer & LVT_TIMER_MODE_MASK) != LVT_TIMER_MODE_PERIODIC) {
    return 0;
  }
  uint32_t shift = BITS_SHIFT(local_apic_state->lvt_divide_config, 1, 0) |
                   (BIT_SHIFT(local_apic_state->lvt_divide_config, 3) << 2);
  uint32_t divisor_shift = (shift + 1) & 7;
  zx_duration_t duration = rdtsc_to_nanos().Scale(
      static_cast<int64_t>(local_apic_state->lvt_initial_count << divisor_shift));
  return zx_time_add_duration(current_time(), duration);
}

static void update_timer(LocalApicState* local_apic_state, zx_time_t deadline);

static void deadline_callback(Timer* timer, zx_time_t now, void* arg) {
  LocalApicState* local_apic_state = static_cast<LocalApicState*>(arg);
  if (local_apic_state->lvt_timer & LVT_MASKED) {
    return;
  }
  if ((local_apic_state->lvt_timer & LVT_TIMER_MODE_MASK) == LVT_TIMER_MODE_PERIODIC) {
    update_timer(local_apic_state, lvt_deadline(local_apic_state));
  }
  uint8_t vector = local_apic_state->lvt_timer & LVT_TIMER_VECTOR_MASK;
  local_apic_state->interrupt_tracker.VirtualInterrupt(vector);
}

static void update_timer(LocalApicState* local_apic_state, zx_time_t deadline) {
  local_apic_state->timer.Cancel();
  if (deadline > 0) {
    local_apic_state->timer.SetOneshot(deadline, deadline_callback, local_apic_state);
  }
}

static uint64_t ipi_target_mask(const InterruptCommandRegister& icr, uint16_t self) {
  switch (icr.destination_shorthand) {
    case InterruptDestinationShorthand::NO_SHORTHAND:
      return 1u << icr.destination;
    case InterruptDestinationShorthand::SELF:
      return 1u << (self - 1);
    case InterruptDestinationShorthand::ALL_INCLUDING_SELF:
      return UINT64_MAX;
    case InterruptDestinationShorthand::ALL_EXCLUDING_SELF:
      return ~(1u << (self - 1));
  }
  return 0;
}

static zx_status_t handle_ipi(const ExitInfo& exit_info, AutoVmcs* vmcs, GuestState* guest_state,
                              zx_port_packet* packet) {
  if (guest_state->rax > UINT32_MAX || guest_state->rdx > UINT32_MAX)
    return ZX_ERR_INVALID_ARGS;
  InterruptCommandRegister icr(static_cast<uint32_t>(guest_state->rdx),
                               static_cast<uint32_t>(guest_state->rax));
  if (icr.destination_mode == InterruptDestinationMode::LOGICAL) {
    dprintf(CRITICAL, "Logical IPI destination mode is not supported\n");
    return ZX_ERR_NOT_SUPPORTED;
  }
  switch (icr.delivery_mode) {
    case InterruptDeliveryMode::FIXED: {
      uint16_t self = static_cast<uint16_t>(vmcs->Read(VmcsField16::VPID) - 1);
      memset(packet, 0, sizeof(*packet));
      packet->type = ZX_PKT_TYPE_GUEST_VCPU;
      packet->guest_vcpu.type = ZX_PKT_GUEST_VCPU_INTERRUPT;
      packet->guest_vcpu.interrupt.mask = ipi_target_mask(icr, self);
      packet->guest_vcpu.interrupt.vector = icr.vector;
      next_rip(exit_info, vmcs);
      return ZX_ERR_NEXT;
    }
    case InterruptDeliveryMode::INIT:
      // Ignore INIT IPIs, we only need STARTUP to bring up a VCPU.
      next_rip(exit_info, vmcs);
      return ZX_OK;
    case InterruptDeliveryMode::STARTUP:
      memset(packet, 0, sizeof(*packet));
      packet->type = ZX_PKT_TYPE_GUEST_VCPU;
      packet->guest_vcpu.type = ZX_PKT_GUEST_VCPU_STARTUP;
      packet->guest_vcpu.startup.id = icr.destination;
      packet->guest_vcpu.startup.entry = icr.vector << 12;
      next_rip(exit_info, vmcs);
      return ZX_ERR_NEXT;
    default:
      dprintf(CRITICAL, "Unsupported IPI delivery mode %#x\n",
              static_cast<uint8_t>(icr.delivery_mode));
      return ZX_ERR_NOT_SUPPORTED;
  }
}

static zx_status_t handle_apic_wrmsr(const ExitInfo& exit_info, AutoVmcs* vmcs,
                                     GuestState* guest_state, LocalApicState* local_apic_state,
                                     zx_port_packet* packet) {
  switch (static_cast<X2ApicMsr>(guest_state->rcx)) {
    case X2ApicMsr::EOI:
    case X2ApicMsr::ESR:
      if (guest_state->rax != 0) {
        // Non-zero writes to EOI and ESR cause GP fault. See Volume 3 Section 10.12.1.2.
        local_apic_state->interrupt_tracker.VirtualInterrupt(X86_INT_GP_FAULT);
        return ZX_OK;
      }
      __FALLTHROUGH;
    case X2ApicMsr::TPR:
    case X2ApicMsr::SVR:
    case X2ApicMsr::LVT_MONITOR:
    case X2ApicMsr::LVT_ERROR:
    case X2ApicMsr::LVT_LINT0:
    case X2ApicMsr::LVT_LINT1:
    case X2ApicMsr::LVT_THERMAL_SENSOR:
    case X2ApicMsr::LVT_CMCI:
      if (guest_state->rdx != 0 || guest_state->rax > UINT32_MAX)
        return ZX_ERR_INVALID_ARGS;
      next_rip(exit_info, vmcs);
      return ZX_OK;
    case X2ApicMsr::LVT_TIMER:
      if (guest_state->rax > UINT32_MAX)
        return ZX_ERR_INVALID_ARGS;
      if ((guest_state->rax & LVT_TIMER_MODE_MASK) == LVT_TIMER_MODE_RESERVED)
        return ZX_ERR_INVALID_ARGS;
      next_rip(exit_info, vmcs);
      local_apic_state->lvt_timer = static_cast<uint32_t>(guest_state->rax);
      update_timer(local_apic_state, lvt_deadline(local_apic_state));
      return ZX_OK;
    case X2ApicMsr::INITIAL_COUNT:
      if (guest_state->rax > UINT32_MAX)
        return ZX_ERR_INVALID_ARGS;
      next_rip(exit_info, vmcs);
      local_apic_state->lvt_initial_count = static_cast<uint32_t>(guest_state->rax);
      update_timer(local_apic_state, lvt_deadline(local_apic_state));
      return ZX_OK;
    case X2ApicMsr::DCR:
      if (guest_state->rax > UINT32_MAX)
        return ZX_ERR_INVALID_ARGS;
      next_rip(exit_info, vmcs);
      local_apic_state->lvt_divide_config = static_cast<uint32_t>(guest_state->rax);
      update_timer(local_apic_state, lvt_deadline(local_apic_state));
      return ZX_OK;
    case X2ApicMsr::SELF_IPI: {
      next_rip(exit_info, vmcs);
      uint32_t vector = static_cast<uint32_t>(guest_state->rax) & UINT8_MAX;
      local_apic_state->interrupt_tracker.VirtualInterrupt(vector);
      return ZX_OK;
    }
    case X2ApicMsr::ICR:
      return handle_ipi(exit_info, vmcs, guest_state, packet);
    default:
      // Issue a general protection fault for read only and unimplemented
      // registers.
      dprintf(INFO, "Unhandled x2APIC wrmsr %#lx\n", guest_state->rcx);
      local_apic_state->interrupt_tracker.VirtualInterrupt(X86_INT_GP_FAULT);
      return ZX_OK;
  }
}

static zx_status_t handle_kvm_wrmsr(const ExitInfo& exit_info, AutoVmcs* vmcs,
                                    GuestState* guest_state, LocalApicState* local_apic_state,
                                    PvClockState* pv_clock,
                                    hypervisor::GuestPhysicalAddressSpace* gpas) {
  zx_paddr_t guest_paddr = BITS(guest_state->rax, 31, 0) | (BITS(guest_state->rdx, 31, 0) << 32);

  next_rip(exit_info, vmcs);
  switch (static_cast<uint32_t>(guest_state->rcx)) {
    case kKvmSystemTimeMsrOld:
    case kKvmSystemTimeMsr:
      vmcs->Invalidate();
      if ((guest_paddr & 1) != 0)
        return pv_clock_reset_clock(pv_clock, gpas, guest_paddr & ~static_cast<zx_paddr_t>(1));
      else
        pv_clock_stop_clock(pv_clock);
      return ZX_OK;
    case kKvmBootTimeOld:
    case kKvmBootTime:
      vmcs->Invalidate();
      return pv_clock_update_boot_time(gpas, guest_paddr);
    default:
      local_apic_state->interrupt_tracker.VirtualInterrupt(X86_INT_GP_FAULT);
      return ZX_OK;
  }
}

static zx_status_t handle_wrmsr(const ExitInfo& exit_info, AutoVmcs* vmcs, GuestState* guest_state,
                                LocalApicState* local_apic_state, PvClockState* pv_clock,
                                hypervisor::GuestPhysicalAddressSpace* gpas,
                                zx_port_packet* packet) {
  // On execution of wrmsr, rcx specifies the MSR and edx:eax contains the value to be written.
  switch (static_cast<uint32_t>(guest_state->rcx)) {
    case X86_MSR_IA32_APIC_BASE:
      if (guest_state->rdx != 0)
        return ZX_ERR_INVALID_ARGS;
      if ((guest_state->rax & ~IA32_APIC_BASE_BSP) != kLocalApicPhysBase)
        return ZX_ERR_INVALID_ARGS;
      next_rip(exit_info, vmcs);
      return ZX_OK;
    // See note in handle_rdmsr.
    case X86_MSR_IA32_MTRRCAP:
    case X86_MSR_IA32_MTRR_DEF_TYPE:
    case X86_MSR_IA32_MTRR_FIX64K_00000:
    case X86_MSR_IA32_MTRR_FIX16K_80000 ... X86_MSR_IA32_MTRR_FIX16K_A0000:
    case X86_MSR_IA32_MTRR_FIX4K_C0000 ... X86_MSR_IA32_MTRR_FIX4K_F8000:
    case X86_MSR_IA32_MTRR_PHYSBASE0 ... X86_MSR_IA32_MTRR_PHYSMASK9:
    case X86_MSR_IA32_BIOS_SIGN_ID:
    case X86_MSR_DRAM_POWER_LIMIT:
    case X86_MSR_PP0_POWER_LIMIT:
    case X86_MSR_PP1_POWER_LIMIT:
    case X86_MSR_PLATFORM_POWER_LIMIT:
    // From AMD64 Volume 2, Section 6.1.1: CSTAR is unused, but Linux likes to
    // set a null handler, even when not in compatibility mode. Just ignore it.
    case X86_MSR_IA32_CSTAR:
      next_rip(exit_info, vmcs);
      return ZX_OK;
    case X86_MSR_IA32_TSC_DEADLINE: {
      if ((local_apic_state->lvt_timer & LVT_TIMER_MODE_MASK) != LVT_TIMER_MODE_TSC_DEADLINE)
        return ZX_ERR_INVALID_ARGS;
      next_rip(exit_info, vmcs);
      int64_t tsc_deadline =
          static_cast<int64_t>((guest_state->rdx << 32) | (guest_state->rax & UINT32_MAX));
      update_timer(local_apic_state, rdtsc_to_nanos().Scale(tsc_deadline));
      return ZX_OK;
    }
    case kX2ApicMsrBase ... kX2ApicMsrMax:
      return handle_apic_wrmsr(exit_info, vmcs, guest_state, local_apic_state, packet);
    case kKvmSystemTimeMsrOld:
    case kKvmSystemTimeMsr:
    case kKvmBootTimeOld:
    case kKvmBootTime:
      return handle_kvm_wrmsr(exit_info, vmcs, guest_state, local_apic_state, pv_clock, gpas);
    default:
      dprintf(INFO, "Unhandled wrmsr %#lx\n", guest_state->rcx);
      local_apic_state->interrupt_tracker.VirtualInterrupt(X86_INT_GP_FAULT);
      return ZX_OK;
  }
}

/* Returns the page address for a given page table entry.
 *
 * If the page address is for a large page, we additionally calculate the offset
 * to the correct guest physical page that backs the large page.
 */
static zx_paddr_t page_addr(zx_paddr_t pt_addr, size_t level, zx_vaddr_t guest_vaddr) {
  zx_paddr_t off = 0;
  if (IS_LARGE_PAGE(pt_addr)) {
    if (level == 1) {
      off = guest_vaddr & PAGE_OFFSET_MASK_HUGE;
    } else if (level == 2) {
      off = guest_vaddr & PAGE_OFFSET_MASK_LARGE;
    }
  }
  return (pt_addr & X86_PG_FRAME) + (off & X86_PG_FRAME);
}

static zx_status_t get_page(hypervisor::GuestPhysicalAddressSpace* gpas, zx_vaddr_t guest_vaddr,
                            zx_paddr_t pt_addr, zx_paddr_t* host_paddr) {
  size_t indices[X86_PAGING_LEVELS] = {
      VADDR_TO_PML4_INDEX(guest_vaddr),
      VADDR_TO_PDP_INDEX(guest_vaddr),
      VADDR_TO_PD_INDEX(guest_vaddr),
      VADDR_TO_PT_INDEX(guest_vaddr),
  };
  zx_paddr_t pa;
  for (size_t level = 0; level <= X86_PAGING_LEVELS; level++) {
    zx_status_t status = gpas->GetPage(page_addr(pt_addr, level - 1, guest_vaddr), &pa);
    if (status != ZX_OK)
      return status;
    if (level == X86_PAGING_LEVELS || IS_LARGE_PAGE(pt_addr))
      break;
    pt_entry_t* pt = static_cast<pt_entry_t*>(paddr_to_physmap(pa));
    pt_addr = pt[indices[level]];
    if (!IS_PAGE_PRESENT(pt_addr))
      return ZX_ERR_NOT_FOUND;
  }
  *host_paddr = pa;
  return ZX_OK;
}

static zx_status_t fetch_data(hypervisor::GuestPhysicalAddressSpace* gpas, zx_vaddr_t guest_vaddr,
                              uint8_t* data, size_t size, zx_paddr_t pt_addr) {
  // TODO(abdulla): Make this handle a fetch that crosses more than two pages.
  if (size > PAGE_SIZE)
    return ZX_ERR_OUT_OF_RANGE;

  zx_paddr_t pa;
  zx_status_t status = get_page(gpas, guest_vaddr, pt_addr, &pa);
  if (status != ZX_OK)
    return status;

  size_t page_offset = guest_vaddr & PAGE_OFFSET_MASK_4KB;
  uint8_t* page = static_cast<uint8_t*>(paddr_to_physmap(pa));
  size_t from_page = fbl::min(size, PAGE_SIZE - page_offset);
  mandatory_memcpy(data, page + page_offset, from_page);

  // If the fetch is not split across pages, return.
  if (from_page == size)
    return ZX_OK;

  status = get_page(gpas, guest_vaddr + size, pt_addr, &pa);
  if (status != ZX_OK)
    return status;

  page = static_cast<uint8_t*>(paddr_to_physmap(pa));
  mandatory_memcpy(data + from_page, page, size - from_page);
  return ZX_OK;
}

static zx_status_t handle_trap(const ExitInfo& exit_info, AutoVmcs* vmcs, bool read,
                               zx_vaddr_t guest_paddr, hypervisor::GuestPhysicalAddressSpace* gpas,
                               hypervisor::TrapMap* traps, zx_port_packet_t* packet) {
  if (exit_info.exit_instruction_length > X86_MAX_INST_LEN)
    return ZX_ERR_INTERNAL;

  hypervisor::Trap* trap;
  zx_status_t status = traps->FindTrap(ZX_GUEST_TRAP_BELL, guest_paddr, &trap);
  if (status != ZX_OK)
    return status;
  next_rip(exit_info, vmcs);

  switch (trap->kind()) {
    case ZX_GUEST_TRAP_BELL:
      if (read)
        return ZX_ERR_NOT_SUPPORTED;
      *packet = {};
      packet->key = trap->key();
      packet->type = ZX_PKT_TYPE_GUEST_BELL;
      packet->guest_bell.addr = guest_paddr;
      if (!trap->HasPort())
        return ZX_ERR_BAD_STATE;
      return trap->Queue(*packet, vmcs);
    case ZX_GUEST_TRAP_MEM: {
      *packet = {};
      packet->key = trap->key();
      packet->type = ZX_PKT_TYPE_GUEST_MEM;
      packet->guest_mem.addr = guest_paddr;
      packet->guest_mem.inst_len = exit_info.exit_instruction_length & UINT8_MAX;
      // See Volume 3, Section 5.2.1.
      uint64_t efer = vmcs->Read(VmcsField64::GUEST_IA32_EFER);
      uint32_t cs_access_rights = vmcs->Read(VmcsField32::GUEST_CS_ACCESS_RIGHTS);
      if ((efer & X86_EFER_LMA) && (cs_access_rights & kGuestXxAccessRightsL)) {
        // IA32-e 64 bit mode.
        packet->guest_mem.default_operand_size = 4;
      } else if (cs_access_rights & kGuestXxAccessRightsD) {
        // CS.D set (and not 64 bit mode).
        packet->guest_mem.default_operand_size = 4;
      } else {
        // CS.D clear (and not 64 bit mode).
        packet->guest_mem.default_operand_size = 2;
      }
      zx_paddr_t pt_addr = vmcs->Read(VmcsFieldXX::GUEST_CR3);
      // Done with the vmcs, can now invalidate in case we block.
      vmcs->Invalidate();
      status = fetch_data(gpas, exit_info.guest_rip, packet->guest_mem.inst_buf,
                          packet->guest_mem.inst_len, pt_addr);
      return status == ZX_OK ? ZX_ERR_NEXT : status;
    }
    default:
      return ZX_ERR_BAD_STATE;
  }
}

static zx_status_t handle_ept_violation(const ExitInfo& exit_info, AutoVmcs* vmcs,
                                        hypervisor::GuestPhysicalAddressSpace* gpas,
                                        hypervisor::TrapMap* traps, zx_port_packet_t* packet) {
  EptViolationInfo ept_violation_info(exit_info.exit_qualification);
  zx_vaddr_t guest_paddr = exit_info.guest_physical_address;
  zx_status_t status =
      handle_trap(exit_info, vmcs, ept_violation_info.read, guest_paddr, gpas, traps, packet);
  switch (status) {
    case ZX_ERR_NOT_FOUND:
      break;
    case ZX_OK:
    default:
      return status;
  }
  // We may have to block when handling the page fault.
  vmcs->Invalidate();

  // If there was no trap associated with this address and it is outside of
  // guest physical address space, return failure.
  if (guest_paddr >= gpas->size())
    return ZX_ERR_OUT_OF_RANGE;

  status = gpas->PageFault(guest_paddr);
  if (status != ZX_OK) {
    dprintf(CRITICAL, "Unhandled EPT violation %#lx\n", exit_info.guest_physical_address);
  }
  return status;
}

static zx_status_t handle_xsetbv(const ExitInfo& exit_info, AutoVmcs* vmcs,
                                 GuestState* guest_state) {
  uint64_t guest_cr4 = vmcs->Read(VmcsFieldXX::GUEST_CR4);
  if (!(guest_cr4 & X86_CR4_OSXSAVE))
    return ZX_ERR_INVALID_ARGS;

  // We only support XCR0.
  if (guest_state->rcx != 0)
    return ZX_ERR_INVALID_ARGS;

  cpuid_leaf leaf;
  if (!x86_get_cpuid_subleaf(X86_CPUID_XSAVE, 0, &leaf))
    return ZX_ERR_INTERNAL;

  // Check that XCR0 is valid.
  uint64_t xcr0_bitmap = ((uint64_t)leaf.d << 32) | leaf.a;
  uint64_t xcr0 = (guest_state->rdx << 32) | (guest_state->rax & UINT32_MAX);
  if (~xcr0_bitmap & xcr0 ||
      // x87 state must be enabled.
      (xcr0 & X86_XSAVE_STATE_BIT_X87) != X86_XSAVE_STATE_BIT_X87 ||
      // If AVX state is enabled, SSE state must be enabled.
      (xcr0 & (X86_XSAVE_STATE_BIT_AVX | X86_XSAVE_STATE_BIT_SSE)) == X86_XSAVE_STATE_BIT_AVX)
    return ZX_ERR_INVALID_ARGS;

  guest_state->xcr0 = xcr0;
  next_rip(exit_info, vmcs);
  return ZX_OK;
}

static zx_status_t handle_pause(const ExitInfo& exit_info, AutoVmcs* vmcs) {
  next_rip(exit_info, vmcs);
  return ZX_OK;
}

static zx_status_t handle_vmcall(const ExitInfo& exit_info, AutoVmcs* vmcs,
                                 hypervisor::GuestPhysicalAddressSpace* gpas,
                                 GuestState* guest_state) {
  next_rip(exit_info, vmcs);

  const uint32_t access_rights = vmcs->Read(VmcsField32::GUEST_SS_ACCESS_RIGHTS);
  if ((access_rights & kGuestXxAccessRightsDplUser) != 0) {
    // We only accept a VMCALL if CPL is 0.
    guest_state->rax = VmCallStatus::NOT_PERMITTED;
    return ZX_OK;
  }

  vmcs->Invalidate();
  VmCallInfo info(guest_state);
  switch (info.type) {
    case VmCallType::CLOCK_PAIRING: {
      if (info.arg[1] != 0) {
        dprintf(INFO, "CLOCK_PAIRING hypercall doesn't support clock type %lu\n", info.arg[1]);
        guest_state->rax = VmCallStatus::NOT_SUPPORTED;
        break;
      }
      zx_status_t status = pv_clock_populate_offset(gpas, info.arg[0]);
      if (status != ZX_OK) {
        dprintf(INFO, "Populating lock offset failed with %d\n", status);
        guest_state->rax = VmCallStatus::FAULT;
        break;
      }
      guest_state->rax = VmCallStatus::OK;
      break;
    }
    default:
      dprintf(INFO, "Unknown hypercall %lu (arg0=%#lx, arg1=%#lx, arg2=%#lx, arg3=%#lx)\n",
              static_cast<unsigned long>(info.type), info.arg[0], info.arg[1], info.arg[2],
              info.arg[3]);
      guest_state->rax = VmCallStatus::UNKNOWN_HYPERCALL;
      break;
  }
  // We never fail in case of hypercalls, we just return/propagate errors to the caller.
  return ZX_OK;
}

zx_status_t vmexit_handler(AutoVmcs* vmcs, GuestState* guest_state,
                           LocalApicState* local_apic_state, PvClockState* pv_clock,
                           hypervisor::GuestPhysicalAddressSpace* gpas, hypervisor::TrapMap* traps,
                           zx_port_packet_t* packet) {
  zx_status_t status;
  ExitInfo exit_info(*vmcs);
  switch (exit_info.exit_reason) {
    case ExitReason::EXTERNAL_INTERRUPT:
      ktrace_vcpu_exit(VCPU_EXTERNAL_INTERRUPT, exit_info.guest_rip);
      GUEST_STATS_INC(interrupts);
      status = handle_external_interrupt(vmcs);
      break;
    case ExitReason::INTERRUPT_WINDOW:
      LTRACEF("handling interrupt window\n\n");
      ktrace_vcpu_exit(VCPU_INTERRUPT_WINDOW, exit_info.guest_rip);
      GUEST_STATS_INC(interrupt_windows);
      status = handle_interrupt_window(vmcs, local_apic_state);
      break;
    case ExitReason::CPUID:
      LTRACEF("handling CPUID\n\n");
      ktrace_vcpu_exit(VCPU_CPUID, exit_info.guest_rip);
      GUEST_STATS_INC(cpuid_instructions);
      status = handle_cpuid(exit_info, vmcs, guest_state);
      break;
    case ExitReason::HLT:
      LTRACEF("handling HLT\n\n");
      ktrace_vcpu_exit(VCPU_HLT, exit_info.guest_rip);
      GUEST_STATS_INC(hlt_instructions);
      status = handle_hlt(exit_info, vmcs, local_apic_state);
      break;
    case ExitReason::CONTROL_REGISTER_ACCESS:
      LTRACEF("handling control-register access\n\n");
      ktrace_vcpu_exit(VCPU_CONTROL_REGISTER_ACCESS, exit_info.guest_rip);
      GUEST_STATS_INC(control_register_accesses);
      status = handle_control_register_access(exit_info, vmcs, guest_state, local_apic_state);
      break;
    case ExitReason::IO_INSTRUCTION:
      ktrace_vcpu_exit(VCPU_IO_INSTRUCTION, exit_info.guest_rip);
      GUEST_STATS_INC(io_instructions);
      status = handle_io_instruction(exit_info, vmcs, guest_state, traps, packet);
      break;
    case ExitReason::RDMSR:
      LTRACEF("handling RDMSR %#lx\n\n", guest_state->rcx);
      ktrace_vcpu_exit(VCPU_RDMSR, exit_info.guest_rip);
      GUEST_STATS_INC(rdmsr_instructions);
      status = handle_rdmsr(exit_info, vmcs, guest_state, local_apic_state);
      break;
    case ExitReason::WRMSR:
      LTRACEF("handling WRMSR %#lx\n\n", guest_state->rcx);
      ktrace_vcpu_exit(VCPU_WRMSR, exit_info.guest_rip);
      GUEST_STATS_INC(wrmsr_instructions);
      status = handle_wrmsr(exit_info, vmcs, guest_state, local_apic_state, pv_clock, gpas, packet);
      break;
    case ExitReason::ENTRY_FAILURE_GUEST_STATE:
    case ExitReason::ENTRY_FAILURE_MSR_LOADING:
      LTRACEF("handling VM entry failure\n\n");
      ktrace_vcpu_exit(VCPU_VM_ENTRY_FAILURE, exit_info.guest_rip);
      status = ZX_ERR_BAD_STATE;
      break;
    case ExitReason::EPT_VIOLATION:
      LTRACEF("handling EPT violation\n\n");
      ktrace_vcpu_exit(VCPU_EPT_VIOLATION, exit_info.guest_rip);
      GUEST_STATS_INC(ept_violations);
      status = handle_ept_violation(exit_info, vmcs, gpas, traps, packet);
      break;
    case ExitReason::XSETBV:
      LTRACEF("handling XSETBV\n\n");
      ktrace_vcpu_exit(VCPU_XSETBV, exit_info.guest_rip);
      GUEST_STATS_INC(xsetbv_instructions);
      status = handle_xsetbv(exit_info, vmcs, guest_state);
      break;
    case ExitReason::PAUSE:
      LTRACEF("handling PAUSE\n\n");
      ktrace_vcpu_exit(VCPU_PAUSE, exit_info.guest_rip);
      GUEST_STATS_INC(pause_instructions);
      status = handle_pause(exit_info, vmcs);
      break;
    case ExitReason::VMCALL:
      LTRACEF("handling VMCALL\n\n");
      ktrace_vcpu_exit(VCPU_VMCALL, exit_info.guest_rip);
      GUEST_STATS_INC(vmcall_instructions);
      status = handle_vmcall(exit_info, vmcs, gpas, guest_state);
      break;
    // Currently all exceptions except NMI delivered to guest directly. NMI causes vmexit
    // and handled by host via IDT as any other interrupt/exception.
    case ExitReason::EXCEPTION:
    default:
      ktrace_vcpu_exit(VCPU_UNKNOWN, exit_info.guest_rip);
      status = ZX_ERR_NOT_SUPPORTED;
      break;
  }
  if (status != ZX_OK && status != ZX_ERR_NEXT && status != ZX_ERR_CANCELED) {
    dprintf(CRITICAL, "VM exit handler for %s (%u) returned %d\n",
            exit_reason_name(exit_info.exit_reason), static_cast<uint32_t>(exit_info.exit_reason),
            status);
    dump_guest_state(*guest_state, exit_info);
  }
  return status;
}