// Copyright 2016 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT

// TODO(ZX-992): Need to be able to r/w MSRs.
// The thought is to use resources (as in ResourceDispatcher), at which point
// this will all get rewritten. Until such time, the goal here is KISS.
// This file contains the lower part of Intel Processor Trace support that must
// be done in the kernel (so that we can read/write msrs).
// The userspace driver is in system/dev/misc/cpu-trace/intel-pt.c.
//
// We currently only support Table of Physical Addresses mode:
// it supports discontiguous buffers and supports stop-on-full behavior
// in addition to wrap-around.
//
// IPT tracing has two "modes":
// - per-cpu tracing
// - thread-specific tracing
// Tracing can only be done in one mode at a time. This is because saving/
// restoring thread PT state via the xsaves/xrstors instructions is a global
// flag in the XSS msr.
// Plus once a trace has been done with IPT_TRACE_THREADS one cannot go back
// to IPT_TRACE_CPUS: supporting this requires flushing trace state from all
// threads which is a bit of work. For now it's easy enough to just require
// the user to reboot. ZX-892

#include <arch/arch_ops.h>
#include <arch/mmu.h>
#include <arch/x86.h>
#include <arch/x86/feature.h>
#include <arch/x86/mmu.h>
#include <arch/x86/proc_trace.h>
#include <err.h>
#include <fbl/auto_lock.h>
#include <fbl/macros.h>
#include <fbl/mutex.h>
#include <fbl/unique_ptr.h>
#include <kernel/mp.h>
#include <kernel/thread.h>
#include <lib/ktrace.h>
#include <pow2.h>
#include <string.h>
#include <trace.h>
#include <vm/vm.h>
#include <vm/vm_aspace.h>
#include <lib/zircon-internal/device/cpu-trace/intel-pt.h>
#include <lib/zircon-internal/ktrace.h>
#include <lib/zircon-internal/mtrace.h>
#include <zircon/thread_annotations.h>
#include <zircon/types.h>

using fbl::AutoLock;

#define LOCAL_TRACE 0

// Control MSRs
#define IA32_RTIT_OUTPUT_BASE 0x560
#define IA32_RTIT_OUTPUT_MASK_PTRS 0x561
#define IA32_RTIT_CTL 0x570
#define IA32_RTIT_STATUS 0x571
#define IA32_RTIT_CR3_MATCH 0x572
#define IA32_RTIT_ADDR0_A 0x580
#define IA32_RTIT_ADDR0_B 0x581
#define IA32_RTIT_ADDR1_A 0x582
#define IA32_RTIT_ADDR1_B 0x583
#define IA32_RTIT_ADDR2_A 0x584
#define IA32_RTIT_ADDR2_B 0x585
#define IA32_RTIT_ADDR3_A 0x586
#define IA32_RTIT_ADDR3_B 0x587

// We need bits[15:8] to get the "maximum non-turbo ratio".
// See libipt:intel-pt.h:pt_config, and Intel Vol. 3 chapter 35.5.
#define IA32_PLATFORM_INFO 0xce

// Our own copy of what h/w supports, mostly for sanity checking.
static bool supports_pt = false;
static bool supports_cr3_filtering = false;
static bool supports_psb = false;
static bool supports_ip_filtering = false;
static bool supports_mtc = false;
static bool supports_ptwrite = false;
static bool supports_power_events = false;
static bool supports_output_topa = false;
static bool supports_output_topa_multi = false;
static bool supports_output_single = false;
static bool supports_output_transport = false;

struct ipt_trace_state_t {
    uint64_t ctl;
    uint64_t status;
    uint64_t output_base;
    uint64_t output_mask_ptrs;
    uint64_t cr3_match;
    struct {
        uint64_t a, b;
    } addr_ranges[IPT_MAX_NUM_ADDR_RANGES];
};

static fbl::Mutex ipt_lock;

static ipt_trace_state_t* ipt_trace_state TA_GUARDED(ipt_lock);

static bool active TA_GUARDED(ipt_lock) = false;

static ipt_trace_mode_t trace_mode TA_GUARDED(ipt_lock) = IPT_TRACE_CPUS;

// In cpu mode this arch_max_num_cpus.
// In thread mode this is provided by the user.
static uint32_t ipt_num_traces TA_GUARDED(ipt_lock);

void x86_processor_trace_init(void) {
    if (!x86_feature_test(X86_FEATURE_PT)) {
        return;
    }

    struct cpuid_leaf leaf;
    if (!x86_get_cpuid_subleaf(X86_CPUID_PT, 0, &leaf)) {
        return;
    }

    supports_pt = true;

    // Keep our own copy of these flags, mostly for potential sanity checks.
    supports_cr3_filtering = !!(leaf.b & (1 << 0));
    supports_psb = !!(leaf.b & (1 << 1));
    supports_ip_filtering = !!(leaf.b & (1 << 2));
    supports_mtc = !!(leaf.b & (1 << 3));
    supports_ptwrite = !!(leaf.b & (1 << 4));
    supports_power_events = !!(leaf.b & (1 << 5));

    supports_output_topa = !!(leaf.c & (1 << 0));
    supports_output_topa_multi = !!(leaf.c & (1 << 1));
    supports_output_single = !!(leaf.c & (1 << 2));
    supports_output_transport = !!(leaf.c & (1 << 3));
}

// Intel Processor Trace support needs to be able to map cr3 values that
// appear in the trace to pids that ld.so uses to dump memory maps.
void arch_trace_process_create(uint64_t pid, paddr_t pt_phys) {
    // The cr3 value that appears in Intel PT h/w tracing.
    uint64_t cr3 = pt_phys;
    ktrace(TAG_IPT_PROCESS_CREATE, (uint32_t)pid, (uint32_t)(pid >> 32),
           (uint32_t)cr3, (uint32_t)(cr3 >> 32));
}

// Worker for x86_ipt_alloc_trace to be executed on all cpus.
// This is invoked via mp_sync_exec which thread safety analysis cannot follow.
static void x86_ipt_set_mode_task(void* raw_context) TA_NO_THREAD_SAFETY_ANALYSIS {
    DEBUG_ASSERT(arch_ints_disabled());
    DEBUG_ASSERT(!active);

    // When changing modes make sure all PT MSRs are in the init state.
    // We don't want a value to appear in the xsave buffer and have xrstors
    // #gp because XCOMP_BV has the PT bit set that's not set in XSS.
    // We still need to do this, even with ZX-892, when transitioning
    // from IPT_TRACE_CPUS to IPT_TRACE_THREADS.
    write_msr(IA32_RTIT_CTL, 0);
    write_msr(IA32_RTIT_STATUS, 0);
    write_msr(IA32_RTIT_OUTPUT_BASE, 0);
    write_msr(IA32_RTIT_OUTPUT_MASK_PTRS, 0);
    if (supports_cr3_filtering)
        write_msr(IA32_RTIT_CR3_MATCH, 0);
    // TODO(dje): addr range msrs

    ipt_trace_mode_t new_mode = static_cast<ipt_trace_mode_t>(reinterpret_cast<uintptr_t>(raw_context));

    // PT state saving, if supported, was enabled during boot so there's no
    // need to recalculate the xsave space needed.
    x86_set_extended_register_pt_state(new_mode == IPT_TRACE_THREADS);
}

zx_status_t x86_ipt_alloc_trace(ipt_trace_mode_t mode, uint32_t num_traces) {
    AutoLock al(&ipt_lock);

    DEBUG_ASSERT(mode == IPT_TRACE_CPUS || mode == IPT_TRACE_THREADS);
    if (mode == IPT_TRACE_CPUS) {
        if (num_traces != arch_max_num_cpus())
            return ZX_ERR_INVALID_ARGS;
    } else {
        return ZX_ERR_NOT_SUPPORTED;
    }

    if (!supports_pt)
        return ZX_ERR_NOT_SUPPORTED;
    if (active)
        return ZX_ERR_BAD_STATE;
    if (ipt_trace_state)
        return ZX_ERR_BAD_STATE;

    // ZX-892: We don't support changing the mode from IPT_TRACE_THREADS to
    // IPT_TRACE_CPUS: We can't turn off XSS.PT until we're sure all threads
    // have no PT state, and that's too tricky to do right now. Instead,
    // require the developer to reboot.
    if (trace_mode == IPT_TRACE_THREADS && mode == IPT_TRACE_CPUS)
        return ZX_ERR_NOT_SUPPORTED;

    ipt_trace_state =
        reinterpret_cast<ipt_trace_state_t*>(calloc(num_traces,
                                                    sizeof(*ipt_trace_state)));
    if (!ipt_trace_state)
        return ZX_ERR_NO_MEMORY;

    mp_sync_exec(MP_IPI_TARGET_ALL, 0, x86_ipt_set_mode_task,
                 reinterpret_cast<void*>(static_cast<uintptr_t>(mode)));

    trace_mode = mode;
    ipt_num_traces = num_traces;
    return ZX_OK;
}

// Free resources obtained by x86_ipt_alloc_trace().
// This doesn't care if resources have already been freed to save callers
// from having to care during any cleanup.

zx_status_t x86_ipt_free_trace() {
    AutoLock al(&ipt_lock);

    if (!supports_pt)
        return ZX_ERR_NOT_SUPPORTED;
    if (trace_mode == IPT_TRACE_THREADS)
        return ZX_ERR_BAD_STATE;
    if (active)
        return ZX_ERR_BAD_STATE;

    free(ipt_trace_state);
    ipt_trace_state = nullptr;
    return ZX_OK;
}

// This is invoked via mp_sync_exec which thread safety analysis cannot follow.
static void x86_ipt_start_cpu_task(void* raw_context) TA_NO_THREAD_SAFETY_ANALYSIS {
    DEBUG_ASSERT(arch_ints_disabled());
    DEBUG_ASSERT(active && raw_context);

    ipt_trace_state_t* context = reinterpret_cast<ipt_trace_state_t*>(raw_context);
    uint32_t cpu = arch_curr_cpu_num();
    ipt_trace_state_t* state = &context[cpu];

    DEBUG_ASSERT(!(read_msr(IA32_RTIT_CTL) & IPT_CTL_TRACE_EN_MASK));

    // Load the ToPA configuration
    write_msr(IA32_RTIT_OUTPUT_BASE, state->output_base);
    write_msr(IA32_RTIT_OUTPUT_MASK_PTRS, state->output_mask_ptrs);

    // Load all other msrs, prior to enabling tracing.
    write_msr(IA32_RTIT_STATUS, state->status);
    if (supports_cr3_filtering)
        write_msr(IA32_RTIT_CR3_MATCH, state->cr3_match);

    // Enable the trace
    write_msr(IA32_RTIT_CTL, state->ctl);
}

// Begin the trace.

zx_status_t x86_ipt_start() {
    AutoLock al(&ipt_lock);

    if (!supports_pt)
        return ZX_ERR_NOT_SUPPORTED;
    if (trace_mode == IPT_TRACE_THREADS)
        return ZX_ERR_BAD_STATE;
    if (active)
        return ZX_ERR_BAD_STATE;
    if (!ipt_trace_state)
        return ZX_ERR_BAD_STATE;

    uint64_t kernel_cr3 = x86_kernel_cr3();
    TRACEF("Starting processor trace, kernel cr3: 0x%" PRIxPTR "\n",
           kernel_cr3);

    if (LOCAL_TRACE && trace_mode == IPT_TRACE_CPUS) {
        uint32_t num_cpus = ipt_num_traces;
        for (uint32_t cpu = 0; cpu < num_cpus; ++cpu) {
            TRACEF("Cpu %u: ctl 0x%" PRIx64 ", status 0x%" PRIx64 ", base 0x%" PRIx64 ", mask 0x%" PRIx64 "\n",
                   cpu, ipt_trace_state[cpu].ctl, ipt_trace_state[cpu].status,
                   ipt_trace_state[cpu].output_base,
                   ipt_trace_state[cpu].output_mask_ptrs);
        }
    }

    active = true;

    // Sideband info needed by the trace reader.
    uint64_t platform_msr = read_msr(IA32_PLATFORM_INFO);
    unsigned nom_freq = (platform_msr >> 8) & 0xff;
    ktrace(TAG_IPT_START, (uint32_t)nom_freq, 0,
           (uint32_t)kernel_cr3, (uint32_t)(kernel_cr3 >> 32));
    const struct x86_model_info* model_info = x86_get_model();
    ktrace(TAG_IPT_CPU_INFO, model_info->processor_type,
           model_info->display_family, model_info->display_model,
           model_info->stepping);

    if (trace_mode == IPT_TRACE_CPUS) {
        mp_sync_exec(MP_IPI_TARGET_ALL, 0, x86_ipt_start_cpu_task, ipt_trace_state);
    }

    return ZX_OK;
}

// This is invoked via mp_sync_exec which thread safety analysis cannot follow.
static void x86_ipt_stop_cpu_task(void* raw_context) TA_NO_THREAD_SAFETY_ANALYSIS {
    DEBUG_ASSERT(arch_ints_disabled());
    DEBUG_ASSERT(raw_context);

    ipt_trace_state_t* context = reinterpret_cast<ipt_trace_state_t*>(raw_context);
    uint32_t cpu = arch_curr_cpu_num();
    ipt_trace_state_t* state = &context[cpu];

    // Disable the trace
    write_msr(IA32_RTIT_CTL, 0);

    // Retrieve msr values for later providing to userspace
    state->ctl = 0;
    state->status = read_msr(IA32_RTIT_STATUS);
    state->output_base = read_msr(IA32_RTIT_OUTPUT_BASE);
    state->output_mask_ptrs = read_msr(IA32_RTIT_OUTPUT_MASK_PTRS);

    // Zero all MSRs so that we are in the XSAVE initial configuration.
    // This allows h/w to do some optimizations regarding the state.
    write_msr(IA32_RTIT_STATUS, 0);
    write_msr(IA32_RTIT_OUTPUT_BASE, 0);
    write_msr(IA32_RTIT_OUTPUT_MASK_PTRS, 0);
    if (supports_cr3_filtering)
        write_msr(IA32_RTIT_CR3_MATCH, 0);

    // TODO(dje): Make it explicit that packets have been completely written.
    // See Intel Vol 3 chapter 36.2.4.

    // TODO(teisenbe): Clear ADDR* MSRs depending on leaf 1
}

// This can be called while not active, so the caller doesn't have to care
// during any cleanup.

zx_status_t x86_ipt_stop() {
    AutoLock al(&ipt_lock);

    if (!supports_pt)
        return ZX_ERR_NOT_SUPPORTED;
    if (trace_mode == IPT_TRACE_THREADS)
        return ZX_ERR_BAD_STATE;
    if (!ipt_trace_state)
        return ZX_ERR_BAD_STATE;

    TRACEF("Stopping processor trace\n");

    if (trace_mode == IPT_TRACE_CPUS) {
        mp_sync_exec(MP_IPI_TARGET_ALL, 0, x86_ipt_stop_cpu_task, ipt_trace_state);
    }

    ktrace(TAG_IPT_STOP, 0, 0, 0, 0);
    active = false;

    if (LOCAL_TRACE && trace_mode == IPT_TRACE_CPUS) {
        uint32_t num_cpus = ipt_num_traces;
        for (uint32_t cpu = 0; cpu < num_cpus; ++cpu) {
            TRACEF("Cpu %u: ctl 0x%" PRIx64 ", status 0x%" PRIx64 ", base 0x%" PRIx64 ", mask 0x%" PRIx64 "\n",
                   cpu, ipt_trace_state[cpu].ctl, ipt_trace_state[cpu].status,
                   ipt_trace_state[cpu].output_base,
                   ipt_trace_state[cpu].output_mask_ptrs);
        }
    }

    return ZX_OK;
}

zx_status_t x86_ipt_stage_trace_data(zx_itrace_buffer_descriptor_t descriptor,
                                     const zx_x86_pt_regs_t* regs) {
    AutoLock al(&ipt_lock);

    if (!supports_pt)
        return ZX_ERR_NOT_SUPPORTED;
    if (trace_mode == IPT_TRACE_CPUS && active)
        return ZX_ERR_BAD_STATE;
    if (!ipt_trace_state)
        return ZX_ERR_BAD_STATE;
    if (descriptor >= ipt_num_traces)
        return ZX_ERR_INVALID_ARGS;

    ipt_trace_state[descriptor].ctl = regs->ctl;
    ipt_trace_state[descriptor].status = regs->status;
    ipt_trace_state[descriptor].output_base = regs->output_base;
    ipt_trace_state[descriptor].output_mask_ptrs = regs->output_mask_ptrs;
    ipt_trace_state[descriptor].cr3_match = regs->cr3_match;
    static_assert(sizeof(ipt_trace_state[descriptor].addr_ranges) == sizeof(regs->addr_ranges), "addr_ranges size mismatch");
    memcpy(ipt_trace_state[descriptor].addr_ranges, regs->addr_ranges, sizeof(regs->addr_ranges));

    return ZX_OK;
}

zx_status_t x86_ipt_get_trace_data(zx_itrace_buffer_descriptor_t descriptor,
                                   zx_x86_pt_regs_t* regs) {
    AutoLock al(&ipt_lock);

    if (!supports_pt)
        return ZX_ERR_NOT_SUPPORTED;
    if (trace_mode == IPT_TRACE_CPUS && active)
        return ZX_ERR_BAD_STATE;
    if (!ipt_trace_state)
        return ZX_ERR_BAD_STATE;
    if (descriptor >= ipt_num_traces)
        return ZX_ERR_INVALID_ARGS;

    regs->ctl = ipt_trace_state[descriptor].ctl;
    regs->status = ipt_trace_state[descriptor].status;
    regs->output_base = ipt_trace_state[descriptor].output_base;
    regs->output_mask_ptrs = ipt_trace_state[descriptor].output_mask_ptrs;
    regs->cr3_match = ipt_trace_state[descriptor].cr3_match;
    static_assert(sizeof(regs->addr_ranges) == sizeof(ipt_trace_state[descriptor].addr_ranges), "addr_ranges size mismatch");
    memcpy(regs->addr_ranges, ipt_trace_state[descriptor].addr_ranges, sizeof(regs->addr_ranges));

    return ZX_OK;
}