src/devices/misc/drivers/cpu-trace/intel-pm.cc

// Copyright 2017 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// See the README.md in this directory for documentation.

#include <assert.h>
#include <cpuid.h>

#include <ddk/debug.h>
#include <ddk/protocol/platform/device.h>
#include <fbl/alloc_checker.h>

#include "perf-mon.h"

namespace perfmon {

// TODO(dje): Having trouble getting this working, so just punt for now.
#define TRY_FREEZE_ON_PMI 0

// Individual bits in the fixed counter enable field.
// See Intel Volume 3, Figure 18-2 "Layout of IA32_FIXED_CTR_CTRL MSR".
#define FIXED_CTR_ENABLE_OS 1
#define FIXED_CTR_ENABLE_USR 2

// This table is sorted at startup.
static EventId misc_event_table_contents[IPM_NUM_MISC_EVENTS] = {
#define DEF_MISC_SKL_EVENT(symbol, event_name, id, offset, size, flags, readable_name, \
                           description)                                                \
  MakeEventId(kGroupMisc, id),
#include <lib/zircon-internal/device/cpu-trace/skylake-misc-events.inc>
};

// Const accessor to give the illusion of the table being const.
static const EventId* misc_event_table = &misc_event_table_contents[0];

enum ArchEvent : uint16_t {
#define DEF_ARCH_EVENT(symbol, event_name, id, ebx_bit, event, umask, flags, readable_name, \
                       description)                                                         \
  symbol,
#include <lib/zircon-internal/device/cpu-trace/intel-pm-events.inc>
};

enum ModelEvent : uint16_t {
#define DEF_SKL_EVENT(symbol, event_name, id, event, umask, flags, readable_name, description) \
  symbol,
#include <lib/zircon-internal/device/cpu-trace/skylake-pm-events.inc>
};

static const EventDetails kArchEvents[] = {
#define DEF_ARCH_EVENT(symbol, event_name, id, ebx_bit, event, umask, flags, readable_name, \
                       description)                                                         \
  {id, event, umask, flags},
#include <lib/zircon-internal/device/cpu-trace/intel-pm-events.inc>
};

static const EventDetails kModelEvents[] = {
#define DEF_SKL_EVENT(symbol, event_name, id, event, umask, flags, readable_name, description) \
  {id, event, umask, flags},
#include <lib/zircon-internal/device/cpu-trace/skylake-pm-events.inc>
};

// A table to map event id to index in |kArchEvents|.
// We use the kConstant naming style as once computed it is constant.
static const uint16_t* kArchEventMap;
static size_t kArchEventMapSize;

// A table to map event id to index in |kModelEvents|.
// We use the kConstant naming style as once computed it is constant.
static const uint16_t* kModelEventMap;
static size_t kModelEventMapSize;

// Map a fixed counter event id to its h/w register number.
// Returns IPM_MAX_FIXED_COUNTERS if |id| is unknown.
static unsigned PmuFixedCounterNumber(EventId id) {
  enum {
#define DEF_FIXED_EVENT(symbol, event_name, id, regnum, flags, readable_name, description) \
  symbol##_NUMBER = regnum,
#include <lib/zircon-internal/device/cpu-trace/intel-pm-events.inc>
  };
  switch (id) {
    case FIXED_INSTRUCTIONS_RETIRED_ID:
      return FIXED_INSTRUCTIONS_RETIRED_NUMBER;
    case FIXED_UNHALTED_CORE_CYCLES_ID:
      return FIXED_UNHALTED_CORE_CYCLES_NUMBER;
    case FIXED_UNHALTED_REFERENCE_CYCLES_ID:
      return FIXED_UNHALTED_REFERENCE_CYCLES_NUMBER;
    default:
      return IPM_MAX_FIXED_COUNTERS;
  }
}

static void PmuInitMiscEventTable() {
  qsort(misc_event_table_contents, countof(misc_event_table_contents),
        sizeof(misc_event_table_contents[0]), ComparePerfmonEventId);
}

// Map a misc event id to its ordinal (unique number in range
// 0 ... IPM_NUM_MISC_EVENTS - 1).
// Returns -1 if |id| is unknown.
static int PmuLookupMiscEvent(EventId id) {
  auto p =
      reinterpret_cast<EventId*>(bsearch(&id, misc_event_table, countof(misc_event_table_contents),
                                         sizeof(id), ComparePerfmonEventId));
  if (!p) {
    return -1;
  }
  ptrdiff_t result = p - misc_event_table;
  assert(result < IPM_NUM_MISC_EVENTS);
  return (int)result;
}

// Initialize the event maps.
// If there's a problem with the database just flag the error but don't crash.

static zx_status_t InitializeEventMaps() {
  PmuInitMiscEventTable();

  zx_status_t status =
      BuildEventMap(kArchEvents, countof(kArchEvents), &kArchEventMap, &kArchEventMapSize);
  if (status != ZX_OK) {
    return status;
  }

  status = BuildEventMap(kModelEvents, countof(kModelEvents), &kModelEventMap, &kModelEventMapSize);
  if (status != ZX_OK) {
    return status;
  }

  return ZX_OK;
}

// Each arch provides its own |InitOnce()| method.

zx_status_t PerfmonDevice::InitOnce() {
  zx_status_t status = InitializeEventMaps();
  if (status != ZX_OK) {
    return status;
  }

  return ZX_OK;
}

// Architecture-provided helpers for |PmuStageConfig()|.

static bool LbrSupported(const perfmon::PmuHwProperties& props) { return props.lbr_stack_size > 0; }

void PerfmonDevice::InitializeStagingState(StagingState* ss) {
  ss->max_num_fixed = pmu_hw_properties_.common.max_num_fixed_events;
  ss->max_num_programmable = pmu_hw_properties_.common.max_num_programmable_events;
  ss->max_num_misc = pmu_hw_properties_.common.max_num_misc_events;
  ss->max_fixed_value = (pmu_hw_properties_.common.max_fixed_counter_width < 64
                             ? (1ul << pmu_hw_properties_.common.max_fixed_counter_width) - 1
                             : ~0ul);
  ss->max_programmable_value =
      (pmu_hw_properties_.common.max_programmable_counter_width < 64
           ? (1ul << pmu_hw_properties_.common.max_programmable_counter_width) - 1
           : ~0ul);
}

zx_status_t PerfmonDevice::StageFixedConfig(const FidlPerfmonConfig* icfg, StagingState* ss,
                                            unsigned input_index, PmuConfig* ocfg) {
  const unsigned ii = input_index;
  const EventId id = icfg->events[ii].event;
  unsigned counter = PmuFixedCounterNumber(id);
  EventRate rate = icfg->events[ii].rate;
  fidl_perfmon::EventConfigFlags flags = icfg->events[ii].flags;
  bool uses_timebase = ocfg->timebase_event != kEventIdNone && rate == 0;

  if (counter == IPM_MAX_FIXED_COUNTERS || counter >= countof(ocfg->fixed_events) ||
      counter >= ss->max_num_fixed) {
    zxlogf(ERROR, "%s: Invalid fixed event [%u]", __func__, ii);
    return ZX_ERR_INVALID_ARGS;
  }
  if (ss->have_fixed[counter]) {
    zxlogf(ERROR, "%s: Fixed event [%u] already provided", __func__, counter);
    return ZX_ERR_INVALID_ARGS;
  }
  ss->have_fixed[counter] = true;
  ocfg->fixed_events[ss->num_fixed] = id;

  if (rate == 0) {
    ocfg->fixed_initial_value[ss->num_fixed] = 0;
  } else {
    if (rate > ss->max_fixed_value) {
      zxlogf(ERROR, "%s: Rate too large, event [%u]", __func__, ii);
      return ZX_ERR_INVALID_ARGS;
    }
    ocfg->fixed_initial_value[ss->num_fixed] = ss->max_fixed_value - rate + 1;
  }
  // Don't generate PMI's for counters that use another as the timebase.
  // We still generate interrupts in "tally mode" in case the counter overflows.
  if (!uses_timebase) {
    ocfg->fixed_ctrl |= IA32_FIXED_CTR_CTRL_PMI_MASK(counter);
  }

  unsigned enable = 0;
  if (flags & fidl_perfmon::EventConfigFlags::COLLECT_OS) {
    enable |= FIXED_CTR_ENABLE_OS;
  }
  if (flags & fidl_perfmon::EventConfigFlags::COLLECT_USER) {
    enable |= FIXED_CTR_ENABLE_USR;
  }
  ocfg->fixed_ctrl |= enable << IA32_FIXED_CTR_CTRL_EN_SHIFT(counter);
  ocfg->global_ctrl |= IA32_PERF_GLOBAL_CTRL_FIXED_EN_MASK(counter);

  if (uses_timebase) {
    ocfg->fixed_flags[ss->num_fixed] |= kPmuConfigFlagUsesTimebase;
  }
  if (flags & fidl_perfmon::EventConfigFlags::COLLECT_PC) {
    ocfg->fixed_flags[ss->num_fixed] |= kPmuConfigFlagPc;
  }
  if (flags & fidl_perfmon::EventConfigFlags::COLLECT_LAST_BRANCH) {
    if (!LbrSupported(pmu_hw_properties())) {
      zxlogf(ERROR, "%s: Last branch not supported, event [%u]", __func__, ii);
      return ZX_ERR_INVALID_ARGS;
    }
    ocfg->fixed_flags[ss->num_fixed] |= kPmuConfigFlagLastBranch;
    ocfg->debug_ctrl |= IA32_DEBUGCTL_LBR_MASK;
  }

  ++ss->num_fixed;
  return ZX_OK;
}

zx_status_t PerfmonDevice::StageProgrammableConfig(const FidlPerfmonConfig* icfg, StagingState* ss,
                                                   unsigned input_index, PmuConfig* ocfg) {
  const unsigned ii = input_index;
  EventId id = icfg->events[ii].event;
  unsigned group = GetEventIdGroup(id);
  unsigned event = GetEventIdEvent(id);
  EventRate rate = icfg->events[ii].rate;
  fidl_perfmon::EventConfigFlags flags = icfg->events[ii].flags;
  bool uses_timebase = ocfg->timebase_event != kEventIdNone && rate == 0;

  // TODO(dje): Verify no duplicates.
  if (ss->num_programmable == ss->max_num_programmable) {
    zxlogf(ERROR, "%s: Too many programmable counters provided", __func__);
    return ZX_ERR_INVALID_ARGS;
  }

  ocfg->programmable_events[ss->num_programmable] = id;

  if (rate == 0) {
    ocfg->programmable_initial_value[ss->num_programmable] = 0;
  } else {
    if (rate > ss->max_programmable_value) {
      zxlogf(ERROR, "%s: Rate too large, event [%u]", __func__, ii);
      return ZX_ERR_INVALID_ARGS;
    }
    ocfg->programmable_initial_value[ss->num_programmable] = ss->max_programmable_value - rate + 1;
  }

  const EventDetails* details = nullptr;
  switch (group) {
    case kGroupArch:
      if (event >= kArchEventMapSize) {
        zxlogf(ERROR, "%s: Invalid event id, event [%u]", __func__, ii);
        return ZX_ERR_INVALID_ARGS;
      }
      details = &kArchEvents[kArchEventMap[event]];
      break;
    case kGroupModel:
      if (event >= kModelEventMapSize) {
        zxlogf(ERROR, "%s: Invalid event id, event [%u]", __func__, ii);
        return ZX_ERR_INVALID_ARGS;
      }
      details = &kModelEvents[kModelEventMap[event]];
      break;
    default:
      zxlogf(ERROR, "%s: Invalid event id, event [%u]", __func__, ii);
      return ZX_ERR_INVALID_ARGS;
  }
  if (details->event == 0 && details->umask == 0) {
    zxlogf(ERROR, "%s: Invalid event id, event [%u]", __func__, ii);
    return ZX_ERR_INVALID_ARGS;
  }

  uint64_t evtsel = 0;
  evtsel |= details->event << IA32_PERFEVTSEL_EVENT_SELECT_SHIFT;
  evtsel |= details->umask << IA32_PERFEVTSEL_UMASK_SHIFT;
  if (flags & fidl_perfmon::EventConfigFlags::COLLECT_OS) {
    evtsel |= IA32_PERFEVTSEL_OS_MASK;
  }
  if (flags & fidl_perfmon::EventConfigFlags::COLLECT_USER) {
    evtsel |= IA32_PERFEVTSEL_USR_MASK;
  }
  if (details->flags & IPM_REG_FLAG_EDG) {
    evtsel |= IA32_PERFEVTSEL_E_MASK;
  }
  if (details->flags & IPM_REG_FLAG_ANYT) {
    evtsel |= IA32_PERFEVTSEL_ANY_MASK;
  }
  if (details->flags & IPM_REG_FLAG_INV) {
    evtsel |= IA32_PERFEVTSEL_INV_MASK;
  }
  evtsel |= (details->flags & IPM_REG_FLAG_CMSK_MASK) << IA32_PERFEVTSEL_CMASK_SHIFT;
  // Don't generate PMI's for counters that use another as the timebase.
  // We still generate interrupts in "tally mode" in case the counter overflows.
  if (!uses_timebase) {
    evtsel |= IA32_PERFEVTSEL_INT_MASK;
  }
  evtsel |= IA32_PERFEVTSEL_EN_MASK;
  ocfg->programmable_hw_events[ss->num_programmable] = evtsel;
  ocfg->global_ctrl |= IA32_PERF_GLOBAL_CTRL_PMC_EN_MASK(ss->num_programmable);

  if (uses_timebase) {
    ocfg->programmable_flags[ss->num_programmable] |= kPmuConfigFlagUsesTimebase;
  }
  if (flags & fidl_perfmon::EventConfigFlags::COLLECT_PC) {
    ocfg->programmable_flags[ss->num_programmable] |= kPmuConfigFlagPc;
  }
  if (flags & fidl_perfmon::EventConfigFlags::COLLECT_LAST_BRANCH) {
    if (!LbrSupported(pmu_hw_properties())) {
      zxlogf(ERROR, "%s: Last branch not supported, event [%u]", __func__, ii);
      return ZX_ERR_INVALID_ARGS;
    }
    ocfg->programmable_flags[ss->num_programmable] |= kPmuConfigFlagLastBranch;
    ocfg->debug_ctrl |= IA32_DEBUGCTL_LBR_MASK;
  }

  ++ss->num_programmable;
  return ZX_OK;
}

zx_status_t PerfmonDevice::StageMiscConfig(const FidlPerfmonConfig* icfg, StagingState* ss,
                                           unsigned input_index, PmuConfig* ocfg) {
  const unsigned ii = input_index;
  EventId id = icfg->events[ii].event;
  int event = PmuLookupMiscEvent(id);
  EventRate rate = icfg->events[ii].rate;
  bool uses_timebase = ocfg->timebase_event != kEventIdNone && rate == 0;

  if (event < 0) {
    zxlogf(ERROR, "%s: Invalid misc event [%u]", __func__, ii);
    return ZX_ERR_INVALID_ARGS;
  }
  if (ss->num_misc == ss->max_num_misc) {
    zxlogf(ERROR, "%s: Too many misc counters provided", __func__);
    return ZX_ERR_INVALID_ARGS;
  }
  if (ss->have_misc[event / 64] & (1ul << (event % 64))) {
    zxlogf(ERROR, "%s: Misc event [%u] already provided", __func__, ii);
    return ZX_ERR_INVALID_ARGS;
  }
  if (rate != 0) {
    zxlogf(ERROR, "%s: Misc event [%u] cannot be own timebase", __func__, ii);
    return ZX_ERR_INVALID_ARGS;
  }

  ss->have_misc[event / 64] |= 1ul << (event % 64);
  ocfg->misc_events[ss->num_misc] = id;

  if (uses_timebase) {
    ocfg->misc_flags[ss->num_misc] |= kPmuConfigFlagUsesTimebase;
  }

  ++ss->num_misc;
  return ZX_OK;
}

zx_status_t PerfmonDevice::VerifyStaging(StagingState* ss, PmuConfig* ocfg) {
  PmuPerTraceState* per_trace = per_trace_state_.get();

  // Require something to be enabled in order to start tracing.
  // This is mostly a sanity check.
  if (per_trace->config.global_ctrl == 0) {
    zxlogf(ERROR, "%s: Requested config doesn't collect any data", __func__);
    return ZX_ERR_INVALID_ARGS;
  }

#if TRY_FREEZE_ON_PMI
  ocfg->debug_ctrl |= IA32_DEBUGCTL_FREEZE_PERFMON_ON_PMI_MASK;
#endif

  return ZX_OK;
}

}  // namespace perfmon