src/intel/vulkan/anv_perf.c - third_party/mesa - Git at Google

 /*
  * Copyright © 2018 Intel Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
  * to deal in the Software without restriction, including without limitation
  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  * and/or sell copies of the Software, and to permit persons to whom the
  * Software is furnished to do so, subject to the following conditions:
  *
  * The above copyright notice and this permission notice (including the next
  * paragraph) shall be included in all copies or substantial portions of the
  * Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  * DEALINGS IN THE SOFTWARE.
  */

 #include <assert.h>
 #include <stdbool.h>
 #include <stdint.h>

 #include "anv_private.h"
 #include "vk_util.h"

 #include "perf/intel_perf.h"
 #include "perf/intel_perf_mdapi.h"

 #include "util/mesa-sha1.h"

 void
 anv_physical_device_init_perf(struct anv_physical_device *device, int fd)
 {
    struct intel_perf_config *perf = intel_perf_new(NULL);

    intel_perf_init_metrics(perf, &device->info, fd,
                            false /* pipeline statistics */,
                            true /* register snapshots */);

    if (!perf->n_queries)
       goto err;

    /* We need DRM_I915_PERF_PROP_HOLD_PREEMPTION support, only available in
     * perf revision 2.
     */
    if (!INTEL_DEBUG(DEBUG_NO_OACONFIG)) {
       if (!intel_perf_has_hold_preemption(perf))
          goto err;
    }

    device->perf = perf;

    /* Compute the number of commands we need to implement a performance
     * query.
     */
    const struct intel_perf_query_field_layout *layout = &perf->query_layout;
    device->n_perf_query_commands = 0;
    for (uint32_t f = 0; f < layout->n_fields; f++) {
       struct intel_perf_query_field *field = &layout->fields[f];

       switch (field->type) {
       case INTEL_PERF_QUERY_FIELD_TYPE_MI_RPC:
          device->n_perf_query_commands++;
          break;
       case INTEL_PERF_QUERY_FIELD_TYPE_SRM_PERFCNT:
       case INTEL_PERF_QUERY_FIELD_TYPE_SRM_RPSTAT:
       case INTEL_PERF_QUERY_FIELD_TYPE_SRM_OA_A:
       case INTEL_PERF_QUERY_FIELD_TYPE_SRM_OA_B:
       case INTEL_PERF_QUERY_FIELD_TYPE_SRM_OA_C:
       case INTEL_PERF_QUERY_FIELD_TYPE_SRM_OA_PEC:
          device->n_perf_query_commands += field->size / 4;
          break;
       default:
          unreachable("Unhandled register type");
       }
    }
    device->n_perf_query_commands *= 2; /* Begin & End */
    device->n_perf_query_commands += 1; /* availability */

    return;

 err:
    intel_perf_free(perf);
 }

 void
 anv_device_perf_init(struct anv_device *device)
 {
    device->perf_fd = -1;
    device->perf_queue = NULL;
 }

 void
 anv_device_perf_close(struct anv_device *device)
 {
    if (device->perf_fd == -1)
       return;

    if (intel_bind_timeline_get_syncobj(&device->perf_timeline))
       intel_bind_timeline_finish(&device->perf_timeline, device->fd);
    close(device->perf_fd);
    device->perf_fd = -1;
 }

 static uint32_t
 anv_device_perf_get_queue_context_or_exec_queue_id(struct anv_queue *queue)
 {
    struct anv_device *device = queue->device;
    uint32_t context_or_exec_queue_id;

    switch (device->physical->info.kmd_type) {
    case INTEL_KMD_TYPE_I915:
       context_or_exec_queue_id = device->physical->has_vm_control ?
                                  queue->context_id : device->context_id;
       break;
    case INTEL_KMD_TYPE_XE:
       context_or_exec_queue_id = queue->exec_queue_id;
       break;
    default:
       unreachable("missing");
       context_or_exec_queue_id = 0;
    }

    return context_or_exec_queue_id;
 }

 static int
 anv_device_perf_open(struct anv_device *device, struct anv_queue *queue, uint64_t metric_id)
 {
    uint64_t period_exponent = 31; /* slowest sampling period */
    int ret;

    if (intel_perf_has_metric_sync(device->physical->perf)) {
       if (!intel_bind_timeline_init(&device->perf_timeline, device->fd))
          return -1;
    }

    ret = intel_perf_stream_open(device->physical->perf, device->fd,
                                 anv_device_perf_get_queue_context_or_exec_queue_id(queue),
                                 metric_id, period_exponent, true, true,
                                 &device->perf_timeline);
    if (ret >= 0)
       device->perf_queue = queue;
    else
       intel_bind_timeline_finish(&device->perf_timeline, device->fd);

    return ret;
 }

 /* VK_INTEL_performance_query */
 VkResult anv_InitializePerformanceApiINTEL(
     VkDevice                                    _device,
     const VkInitializePerformanceApiInfoINTEL*  pInitializeInfo)
 {
    ANV_FROM_HANDLE(anv_device, device, _device);

    if (!device->physical->perf)
       return VK_ERROR_EXTENSION_NOT_PRESENT;

    /* Not much to do here */
    return VK_SUCCESS;
 }

 VkResult anv_GetPerformanceParameterINTEL(
     VkDevice                                    _device,
     VkPerformanceParameterTypeINTEL             parameter,
     VkPerformanceValueINTEL*                    pValue)
 {
       ANV_FROM_HANDLE(anv_device, device, _device);

       if (!device->physical->perf)
          return VK_ERROR_EXTENSION_NOT_PRESENT;

       VkResult result = VK_SUCCESS;
       switch (parameter) {
       case VK_PERFORMANCE_PARAMETER_TYPE_HW_COUNTERS_SUPPORTED_INTEL:
          pValue->type = VK_PERFORMANCE_VALUE_TYPE_BOOL_INTEL;
          pValue->data.valueBool = VK_TRUE;
          break;

       case VK_PERFORMANCE_PARAMETER_TYPE_STREAM_MARKER_VALID_BITS_INTEL:
          pValue->type = VK_PERFORMANCE_VALUE_TYPE_UINT32_INTEL;
          pValue->data.value32 = 25;
          break;

       default:
          result = VK_ERROR_FEATURE_NOT_PRESENT;
          break;
       }

       return result;
 }

 VkResult anv_CmdSetPerformanceMarkerINTEL(
     VkCommandBuffer                             commandBuffer,
     const VkPerformanceMarkerInfoINTEL*         pMarkerInfo)
 {
    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);

    cmd_buffer->intel_perf_marker = pMarkerInfo->marker;

    return VK_SUCCESS;
 }

 VkResult anv_AcquirePerformanceConfigurationINTEL(
     VkDevice                                    _device,
     const VkPerformanceConfigurationAcquireInfoINTEL* pAcquireInfo,
     VkPerformanceConfigurationINTEL*            pConfiguration)
 {
    ANV_FROM_HANDLE(anv_device, device, _device);
    struct anv_performance_configuration_intel *config;

    config = vk_object_alloc(&device->vk, NULL, sizeof(*config),
                             VK_OBJECT_TYPE_PERFORMANCE_CONFIGURATION_INTEL);
    if (!config)
       return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);

    if (!INTEL_DEBUG(DEBUG_NO_OACONFIG)) {
       config->register_config =
          intel_perf_load_configuration(device->physical->perf, device->fd,
                                      INTEL_PERF_QUERY_GUID_MDAPI);
       if (!config->register_config) {
          vk_object_free(&device->vk, NULL, config);
          return VK_INCOMPLETE;
       }

       uint64_t ret =
          intel_perf_store_configuration(device->physical->perf, device->fd,
                                       config->register_config, NULL /* guid */);
       if (ret == 0) {
          ralloc_free(config->register_config);
          vk_object_free(&device->vk, NULL, config);
          return VK_INCOMPLETE;
       }

       config->config_id = ret;
    }

    *pConfiguration = anv_performance_configuration_intel_to_handle(config);

    return VK_SUCCESS;
 }

 VkResult anv_ReleasePerformanceConfigurationINTEL(
     VkDevice                                    _device,
     VkPerformanceConfigurationINTEL             _configuration)
 {
    ANV_FROM_HANDLE(anv_device, device, _device);
    ANV_FROM_HANDLE(anv_performance_configuration_intel, config, _configuration);

    if (!INTEL_DEBUG(DEBUG_NO_OACONFIG))
       intel_perf_remove_configuration(device->physical->perf, device->fd, config->config_id);

    ralloc_free(config->register_config);

    vk_object_free(&device->vk, NULL, config);

    return VK_SUCCESS;
 }

 static struct anv_queue *
 anv_device_get_perf_queue(struct anv_device *device)
 {
    for (uint32_t i = 0; i < device->queue_count; i++) {
       struct anv_queue *queue = &device->queues[i];
       const struct anv_queue_family *family = queue->family;

       if (family->supports_perf)
          return queue;
    }

    return NULL;
 }

 VkResult anv_QueueSetPerformanceConfigurationINTEL(
     VkQueue                                     _queue,
     VkPerformanceConfigurationINTEL             _configuration)
 {
    ANV_FROM_HANDLE(anv_queue, queue, _queue);
    ANV_FROM_HANDLE(anv_performance_configuration_intel, config, _configuration);
    struct anv_device *device = queue->device;

    if (queue != anv_device_get_perf_queue(device))
       return VK_ERROR_UNKNOWN;

    if (!INTEL_DEBUG(DEBUG_NO_OACONFIG)) {
       if (device->perf_fd < 0) {
          device->perf_fd = anv_device_perf_open(device, queue, config->config_id);
          if (device->perf_fd < 0)
             return VK_ERROR_INITIALIZATION_FAILED;
       } else {
          uint32_t context_or_exec_queue = anv_device_perf_get_queue_context_or_exec_queue_id(device->perf_queue);
          int ret = intel_perf_stream_set_metrics_id(device->physical->perf,
                                                     device->fd,
                                                     device->perf_fd,
                                                     context_or_exec_queue,
                                                     config->config_id,
                                                     &device->perf_timeline);
          if (ret < 0)
             return vk_device_set_lost(&device->vk, "i915-perf config failed: %m");
       }
    }

    return VK_SUCCESS;
 }

 void anv_UninitializePerformanceApiINTEL(
     VkDevice                                    _device)
 {
    ANV_FROM_HANDLE(anv_device, device, _device);

    anv_device_perf_close(device);
 }

 /* VK_KHR_performance_query */
 static const VkPerformanceCounterUnitKHR
 intel_perf_counter_unit_to_vk_unit[] = {
    [INTEL_PERF_COUNTER_UNITS_BYTES]                                = VK_PERFORMANCE_COUNTER_UNIT_BYTES_KHR,
    [INTEL_PERF_COUNTER_UNITS_HZ]                                   = VK_PERFORMANCE_COUNTER_UNIT_HERTZ_KHR,
    [INTEL_PERF_COUNTER_UNITS_NS]                                   = VK_PERFORMANCE_COUNTER_UNIT_NANOSECONDS_KHR,
    [INTEL_PERF_COUNTER_UNITS_US]                                   = VK_PERFORMANCE_COUNTER_UNIT_NANOSECONDS_KHR, /* todo */
    [INTEL_PERF_COUNTER_UNITS_PIXELS]                               = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
    [INTEL_PERF_COUNTER_UNITS_TEXELS]                               = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
    [INTEL_PERF_COUNTER_UNITS_THREADS]                              = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
    [INTEL_PERF_COUNTER_UNITS_PERCENT]                              = VK_PERFORMANCE_COUNTER_UNIT_PERCENTAGE_KHR,
    [INTEL_PERF_COUNTER_UNITS_MESSAGES]                             = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
    [INTEL_PERF_COUNTER_UNITS_NUMBER]                               = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
    [INTEL_PERF_COUNTER_UNITS_CYCLES]                               = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
    [INTEL_PERF_COUNTER_UNITS_EVENTS]                               = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
    [INTEL_PERF_COUNTER_UNITS_UTILIZATION]                          = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
    [INTEL_PERF_COUNTER_UNITS_EU_SENDS_TO_L3_CACHE_LINES]           = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
    [INTEL_PERF_COUNTER_UNITS_EU_ATOMIC_REQUESTS_TO_L3_CACHE_LINES] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
    [INTEL_PERF_COUNTER_UNITS_EU_REQUESTS_TO_L3_CACHE_LINES]        = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
    [INTEL_PERF_COUNTER_UNITS_EU_BYTES_PER_L3_CACHE_LINE]           = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
 };

 static const VkPerformanceCounterStorageKHR
 intel_perf_counter_data_type_to_vk_storage[] = {
    [INTEL_PERF_COUNTER_DATA_TYPE_BOOL32] = VK_PERFORMANCE_COUNTER_STORAGE_UINT32_KHR,
    [INTEL_PERF_COUNTER_DATA_TYPE_UINT32] = VK_PERFORMANCE_COUNTER_STORAGE_UINT32_KHR,
    [INTEL_PERF_COUNTER_DATA_TYPE_UINT64] = VK_PERFORMANCE_COUNTER_STORAGE_UINT64_KHR,
    [INTEL_PERF_COUNTER_DATA_TYPE_FLOAT]  = VK_PERFORMANCE_COUNTER_STORAGE_FLOAT32_KHR,
    [INTEL_PERF_COUNTER_DATA_TYPE_DOUBLE] = VK_PERFORMANCE_COUNTER_STORAGE_FLOAT64_KHR,
 };

 VkResult anv_EnumeratePhysicalDeviceQueueFamilyPerformanceQueryCountersKHR(
     VkPhysicalDevice                            physicalDevice,
     uint32_t                                    queueFamilyIndex,
     uint32_t*                                   pCounterCount,
     VkPerformanceCounterKHR*                    pCounters,
     VkPerformanceCounterDescriptionKHR*         pCounterDescriptions)
 {
    ANV_FROM_HANDLE(anv_physical_device, pdevice, physicalDevice);
    struct intel_perf_config *perf = pdevice->perf;

    uint32_t desc_count = *pCounterCount;

    VK_OUTARRAY_MAKE_TYPED(VkPerformanceCounterKHR, out, pCounters, pCounterCount);
    VK_OUTARRAY_MAKE_TYPED(VkPerformanceCounterDescriptionKHR, out_desc,
                           pCounterDescriptions, &desc_count);

    /* We cannot support performance queries on anything other than RCS,
     * because the MI_REPORT_PERF_COUNT command is not available on other
     * engines.
     */
    struct anv_queue_family *queue_family =
       &pdevice->queue.families[queueFamilyIndex];
    if (queue_family->engine_class != INTEL_ENGINE_CLASS_RENDER)
       return vk_outarray_status(&out);

    for (int c = 0; c < (perf ? perf->n_counters : 0); c++) {
       const struct intel_perf_query_counter *intel_counter = perf->counter_infos[c].counter;

       vk_outarray_append_typed(VkPerformanceCounterKHR, &out, counter) {
          counter->unit = intel_perf_counter_unit_to_vk_unit[intel_counter->units];
          counter->scope = VK_PERFORMANCE_COUNTER_SCOPE_COMMAND_KHR;
          counter->storage = intel_perf_counter_data_type_to_vk_storage[intel_counter->data_type];

          unsigned char sha1_result[20];
          _mesa_sha1_compute(intel_counter->symbol_name,
                             strlen(intel_counter->symbol_name),
                             sha1_result);
          memcpy(counter->uuid, sha1_result, sizeof(counter->uuid));
       }

       vk_outarray_append_typed(VkPerformanceCounterDescriptionKHR, &out_desc, desc) {
          desc->flags = 0; /* None so far. */
          snprintf(desc->name, sizeof(desc->name), "%s",
                   INTEL_DEBUG(DEBUG_PERF_SYMBOL_NAMES) ?
                   intel_counter->symbol_name :
                   intel_counter->name);
          snprintf(desc->category, sizeof(desc->category), "%s", intel_counter->category);
          snprintf(desc->description, sizeof(desc->description), "%s", intel_counter->desc);
       }
    }

    return vk_outarray_status(&out);
 }

 void anv_GetPhysicalDeviceQueueFamilyPerformanceQueryPassesKHR(
     VkPhysicalDevice                            physicalDevice,
     const VkQueryPoolPerformanceCreateInfoKHR*  pPerformanceQueryCreateInfo,
     uint32_t*                                   pNumPasses)
 {
    ANV_FROM_HANDLE(anv_physical_device, pdevice, physicalDevice);
    struct intel_perf_config *perf = pdevice->perf;

    if (!perf) {
       *pNumPasses = 0;
       return;
    }

    *pNumPasses = intel_perf_get_n_passes(perf,
                                        pPerformanceQueryCreateInfo->pCounterIndices,
                                        pPerformanceQueryCreateInfo->counterIndexCount,
                                        NULL);
 }

 VkResult anv_AcquireProfilingLockKHR(
     VkDevice                                    _device,
     const VkAcquireProfilingLockInfoKHR*        pInfo)
 {
    ANV_FROM_HANDLE(anv_device, device, _device);
    struct intel_perf_config *perf = device->physical->perf;
    struct intel_perf_query_info *first_metric_set = &perf->queries[0];
    int fd = -1;

    assert(device->perf_fd == -1);

    if (!INTEL_DEBUG(DEBUG_NO_OACONFIG)) {
       struct anv_queue *queue = anv_device_get_perf_queue(device);

       if (queue == NULL)
          return VK_ERROR_UNKNOWN;
       fd = anv_device_perf_open(device, queue, first_metric_set->oa_metrics_set_id);
       if (fd < 0)
          return VK_TIMEOUT;
    }

    device->perf_fd = fd;
    return VK_SUCCESS;
 }

 void anv_ReleaseProfilingLockKHR(
     VkDevice                                    _device)
 {
    ANV_FROM_HANDLE(anv_device, device, _device);

    anv_device_perf_close(device);
 }

 void
 anv_perf_write_pass_results(struct intel_perf_config *perf,
                             struct anv_query_pool *pool, uint32_t pass,
                             const struct intel_perf_query_result *accumulated_results,
                             union VkPerformanceCounterResultKHR *results)
 {
    const struct intel_perf_query_info *query = pool->pass_query[pass];

    for (uint32_t c = 0; c < pool->n_counters; c++) {
       const struct intel_perf_counter_pass *counter_pass = &pool->counter_pass[c];

       if (counter_pass->query != query)
          continue;

       switch (pool->pass_query[pass]->kind) {
       case INTEL_PERF_QUERY_TYPE_PIPELINE: {
          assert(counter_pass->counter->data_type == INTEL_PERF_COUNTER_DATA_TYPE_UINT64);
          uint32_t accu_offset = counter_pass->counter->offset / sizeof(uint64_t);
          results[c].uint64 = accumulated_results->accumulator[accu_offset];
          break;
       }

       case INTEL_PERF_QUERY_TYPE_OA:
       case INTEL_PERF_QUERY_TYPE_RAW:
          switch (counter_pass->counter->data_type) {
          case INTEL_PERF_COUNTER_DATA_TYPE_UINT64:
             results[c].uint64 =
                counter_pass->counter->oa_counter_read_uint64(perf,
                                                              counter_pass->query,
                                                              accumulated_results);
             break;
          case INTEL_PERF_COUNTER_DATA_TYPE_FLOAT:
             results[c].float32 =
                counter_pass->counter->oa_counter_read_float(perf,
                                                             counter_pass->query,
                                                             accumulated_results);
             break;
          default:
             /* So far we aren't using uint32, double or bool32... */
             unreachable("unexpected counter data type");
          }
          break;

       default:
          unreachable("invalid query type");
       }

       /* The Vulkan extension only has nanoseconds as a unit */
       if (counter_pass->counter->units == INTEL_PERF_COUNTER_UNITS_US) {
          assert(counter_pass->counter->data_type == INTEL_PERF_COUNTER_DATA_TYPE_UINT64);
          results[c].uint64 *= 1000;
       }
    }
 }
	/*
	* Copyright © 2018 Intel Corporation
	*
	* Permission is hereby granted, free of charge, to any person obtaining a
	* copy of this software and associated documentation files (the "Software"),
	* to deal in the Software without restriction, including without limitation
	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	* and/or sell copies of the Software, and to permit persons to whom the
	* Software is furnished to do so, subject to the following conditions:
	*
	* The above copyright notice and this permission notice (including the next
	* paragraph) shall be included in all copies or substantial portions of the
	* Software.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
	* DEALINGS IN THE SOFTWARE.
	*/

	#include <assert.h>
	#include <stdbool.h>
	#include <stdint.h>

	#include "anv_private.h"
	#include "vk_util.h"

	#include "perf/intel_perf.h"
	#include "perf/intel_perf_mdapi.h"

	#include "util/mesa-sha1.h"

	void
	anv_physical_device_init_perf(struct anv_physical_device *device, int fd)
	{
	struct intel_perf_config *perf = intel_perf_new(NULL);

	intel_perf_init_metrics(perf, &device->info, fd,
	false /* pipeline statistics */,
	true /* register snapshots */);

	if (!perf->n_queries)
	goto err;

	/* We need DRM_I915_PERF_PROP_HOLD_PREEMPTION support, only available in
	* perf revision 2.
	*/
	if (!INTEL_DEBUG(DEBUG_NO_OACONFIG)) {
	if (!intel_perf_has_hold_preemption(perf))
	goto err;
	}

	device->perf = perf;

	/* Compute the number of commands we need to implement a performance
	* query.
	*/
	const struct intel_perf_query_field_layout *layout = &perf->query_layout;
	device->n_perf_query_commands = 0;
	for (uint32_t f = 0; f < layout->n_fields; f++) {
	struct intel_perf_query_field *field = &layout->fields[f];

	switch (field->type) {
	case INTEL_PERF_QUERY_FIELD_TYPE_MI_RPC:
	device->n_perf_query_commands++;
	break;
	case INTEL_PERF_QUERY_FIELD_TYPE_SRM_PERFCNT:
	case INTEL_PERF_QUERY_FIELD_TYPE_SRM_RPSTAT:
	case INTEL_PERF_QUERY_FIELD_TYPE_SRM_OA_A:
	case INTEL_PERF_QUERY_FIELD_TYPE_SRM_OA_B:
	case INTEL_PERF_QUERY_FIELD_TYPE_SRM_OA_C:
	case INTEL_PERF_QUERY_FIELD_TYPE_SRM_OA_PEC:
	device->n_perf_query_commands += field->size / 4;
	break;
	default:
	unreachable("Unhandled register type");
	}
	}
	device->n_perf_query_commands = 2; / Begin & End */
	device->n_perf_query_commands += 1; /* availability */

	return;

	err:
	intel_perf_free(perf);
	}

	void
	anv_device_perf_init(struct anv_device *device)
	{
	device->perf_fd = -1;
	device->perf_queue = NULL;
	}

	void
	anv_device_perf_close(struct anv_device *device)
	{
	if (device->perf_fd == -1)
	return;

	if (intel_bind_timeline_get_syncobj(&device->perf_timeline))
	intel_bind_timeline_finish(&device->perf_timeline, device->fd);
	close(device->perf_fd);
	device->perf_fd = -1;
	}

	static uint32_t
	anv_device_perf_get_queue_context_or_exec_queue_id(struct anv_queue *queue)
	{
	struct anv_device *device = queue->device;
	uint32_t context_or_exec_queue_id;

	switch (device->physical->info.kmd_type) {
	case INTEL_KMD_TYPE_I915:
	context_or_exec_queue_id = device->physical->has_vm_control ?
	queue->context_id : device->context_id;
	break;
	case INTEL_KMD_TYPE_XE:
	context_or_exec_queue_id = queue->exec_queue_id;
	break;
	default:
	unreachable("missing");
	context_or_exec_queue_id = 0;
	}

	return context_or_exec_queue_id;
	}

	static int
	anv_device_perf_open(struct anv_device device, struct anv_queue queue, uint64_t metric_id)
	{
	uint64_t period_exponent = 31; /* slowest sampling period */
	int ret;

	if (intel_perf_has_metric_sync(device->physical->perf)) {
	if (!intel_bind_timeline_init(&device->perf_timeline, device->fd))
	return -1;
	}

	ret = intel_perf_stream_open(device->physical->perf, device->fd,
	anv_device_perf_get_queue_context_or_exec_queue_id(queue),
	metric_id, period_exponent, true, true,
	&device->perf_timeline);
	if (ret >= 0)
	device->perf_queue = queue;
	else
	intel_bind_timeline_finish(&device->perf_timeline, device->fd);

	return ret;
	}

	/* VK_INTEL_performance_query */
	VkResult anv_InitializePerformanceApiINTEL(
	VkDevice _device,
	const VkInitializePerformanceApiInfoINTEL* pInitializeInfo)
	{
	ANV_FROM_HANDLE(anv_device, device, _device);

	if (!device->physical->perf)
	return VK_ERROR_EXTENSION_NOT_PRESENT;

	/* Not much to do here */
	return VK_SUCCESS;
	}

	VkResult anv_GetPerformanceParameterINTEL(
	VkDevice _device,
	VkPerformanceParameterTypeINTEL parameter,
	VkPerformanceValueINTEL* pValue)
	{
	ANV_FROM_HANDLE(anv_device, device, _device);

	if (!device->physical->perf)
	return VK_ERROR_EXTENSION_NOT_PRESENT;

	VkResult result = VK_SUCCESS;
	switch (parameter) {
	case VK_PERFORMANCE_PARAMETER_TYPE_HW_COUNTERS_SUPPORTED_INTEL:
	pValue->type = VK_PERFORMANCE_VALUE_TYPE_BOOL_INTEL;
	pValue->data.valueBool = VK_TRUE;
	break;

	case VK_PERFORMANCE_PARAMETER_TYPE_STREAM_MARKER_VALID_BITS_INTEL:
	pValue->type = VK_PERFORMANCE_VALUE_TYPE_UINT32_INTEL;
	pValue->data.value32 = 25;
	break;

	default:
	result = VK_ERROR_FEATURE_NOT_PRESENT;
	break;
	}

	return result;
	}

	VkResult anv_CmdSetPerformanceMarkerINTEL(
	VkCommandBuffer commandBuffer,
	const VkPerformanceMarkerInfoINTEL* pMarkerInfo)
	{
	ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);

	cmd_buffer->intel_perf_marker = pMarkerInfo->marker;

	return VK_SUCCESS;
	}

	VkResult anv_AcquirePerformanceConfigurationINTEL(
	VkDevice _device,
	const VkPerformanceConfigurationAcquireInfoINTEL* pAcquireInfo,
	VkPerformanceConfigurationINTEL* pConfiguration)
	{
	ANV_FROM_HANDLE(anv_device, device, _device);
	struct anv_performance_configuration_intel *config;

	config = vk_object_alloc(&device->vk, NULL, sizeof(*config),
	VK_OBJECT_TYPE_PERFORMANCE_CONFIGURATION_INTEL);
	if (!config)
	return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);

	if (!INTEL_DEBUG(DEBUG_NO_OACONFIG)) {
	config->register_config =
	intel_perf_load_configuration(device->physical->perf, device->fd,
	INTEL_PERF_QUERY_GUID_MDAPI);
	if (!config->register_config) {
	vk_object_free(&device->vk, NULL, config);
	return VK_INCOMPLETE;
	}

	uint64_t ret =
	intel_perf_store_configuration(device->physical->perf, device->fd,
	config->register_config, NULL /* guid */);
	if (ret == 0) {
	ralloc_free(config->register_config);
	vk_object_free(&device->vk, NULL, config);
	return VK_INCOMPLETE;
	}

	config->config_id = ret;
	}

	*pConfiguration = anv_performance_configuration_intel_to_handle(config);

	return VK_SUCCESS;
	}

	VkResult anv_ReleasePerformanceConfigurationINTEL(
	VkDevice _device,
	VkPerformanceConfigurationINTEL _configuration)
	{
	ANV_FROM_HANDLE(anv_device, device, _device);
	ANV_FROM_HANDLE(anv_performance_configuration_intel, config, _configuration);

	if (!INTEL_DEBUG(DEBUG_NO_OACONFIG))
	intel_perf_remove_configuration(device->physical->perf, device->fd, config->config_id);

	ralloc_free(config->register_config);

	vk_object_free(&device->vk, NULL, config);

	return VK_SUCCESS;
	}

	static struct anv_queue *
	anv_device_get_perf_queue(struct anv_device *device)
	{
	for (uint32_t i = 0; i < device->queue_count; i++) {
	struct anv_queue *queue = &device->queues[i];
	const struct anv_queue_family *family = queue->family;

	if (family->supports_perf)
	return queue;
	}

	return NULL;
	}

	VkResult anv_QueueSetPerformanceConfigurationINTEL(
	VkQueue _queue,
	VkPerformanceConfigurationINTEL _configuration)
	{
	ANV_FROM_HANDLE(anv_queue, queue, _queue);
	ANV_FROM_HANDLE(anv_performance_configuration_intel, config, _configuration);
	struct anv_device *device = queue->device;

	if (queue != anv_device_get_perf_queue(device))
	return VK_ERROR_UNKNOWN;

	if (!INTEL_DEBUG(DEBUG_NO_OACONFIG)) {
	if (device->perf_fd < 0) {
	device->perf_fd = anv_device_perf_open(device, queue, config->config_id);
	if (device->perf_fd < 0)
	return VK_ERROR_INITIALIZATION_FAILED;
	} else {
	uint32_t context_or_exec_queue = anv_device_perf_get_queue_context_or_exec_queue_id(device->perf_queue);
	int ret = intel_perf_stream_set_metrics_id(device->physical->perf,
	device->fd,
	device->perf_fd,
	context_or_exec_queue,
	config->config_id,
	&device->perf_timeline);
	if (ret < 0)
	return vk_device_set_lost(&device->vk, "i915-perf config failed: %m");
	}
	}

	return VK_SUCCESS;
	}

	void anv_UninitializePerformanceApiINTEL(
	VkDevice _device)
	{
	ANV_FROM_HANDLE(anv_device, device, _device);

	anv_device_perf_close(device);
	}

	/* VK_KHR_performance_query */
	static const VkPerformanceCounterUnitKHR
	intel_perf_counter_unit_to_vk_unit[] = {
	[INTEL_PERF_COUNTER_UNITS_BYTES] = VK_PERFORMANCE_COUNTER_UNIT_BYTES_KHR,
	[INTEL_PERF_COUNTER_UNITS_HZ] = VK_PERFORMANCE_COUNTER_UNIT_HERTZ_KHR,
	[INTEL_PERF_COUNTER_UNITS_NS] = VK_PERFORMANCE_COUNTER_UNIT_NANOSECONDS_KHR,
	[INTEL_PERF_COUNTER_UNITS_US] = VK_PERFORMANCE_COUNTER_UNIT_NANOSECONDS_KHR, /* todo */
	[INTEL_PERF_COUNTER_UNITS_PIXELS] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
	[INTEL_PERF_COUNTER_UNITS_TEXELS] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
	[INTEL_PERF_COUNTER_UNITS_THREADS] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
	[INTEL_PERF_COUNTER_UNITS_PERCENT] = VK_PERFORMANCE_COUNTER_UNIT_PERCENTAGE_KHR,
	[INTEL_PERF_COUNTER_UNITS_MESSAGES] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
	[INTEL_PERF_COUNTER_UNITS_NUMBER] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
	[INTEL_PERF_COUNTER_UNITS_CYCLES] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
	[INTEL_PERF_COUNTER_UNITS_EVENTS] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
	[INTEL_PERF_COUNTER_UNITS_UTILIZATION] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
	[INTEL_PERF_COUNTER_UNITS_EU_SENDS_TO_L3_CACHE_LINES] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
	[INTEL_PERF_COUNTER_UNITS_EU_ATOMIC_REQUESTS_TO_L3_CACHE_LINES] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
	[INTEL_PERF_COUNTER_UNITS_EU_REQUESTS_TO_L3_CACHE_LINES] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
	[INTEL_PERF_COUNTER_UNITS_EU_BYTES_PER_L3_CACHE_LINE] = VK_PERFORMANCE_COUNTER_UNIT_GENERIC_KHR,
	};

	static const VkPerformanceCounterStorageKHR
	intel_perf_counter_data_type_to_vk_storage[] = {
	[INTEL_PERF_COUNTER_DATA_TYPE_BOOL32] = VK_PERFORMANCE_COUNTER_STORAGE_UINT32_KHR,
	[INTEL_PERF_COUNTER_DATA_TYPE_UINT32] = VK_PERFORMANCE_COUNTER_STORAGE_UINT32_KHR,
	[INTEL_PERF_COUNTER_DATA_TYPE_UINT64] = VK_PERFORMANCE_COUNTER_STORAGE_UINT64_KHR,
	[INTEL_PERF_COUNTER_DATA_TYPE_FLOAT] = VK_PERFORMANCE_COUNTER_STORAGE_FLOAT32_KHR,
	[INTEL_PERF_COUNTER_DATA_TYPE_DOUBLE] = VK_PERFORMANCE_COUNTER_STORAGE_FLOAT64_KHR,
	};

	VkResult anv_EnumeratePhysicalDeviceQueueFamilyPerformanceQueryCountersKHR(
	VkPhysicalDevice physicalDevice,
	uint32_t queueFamilyIndex,
	uint32_t* pCounterCount,
	VkPerformanceCounterKHR* pCounters,
	VkPerformanceCounterDescriptionKHR* pCounterDescriptions)
	{
	ANV_FROM_HANDLE(anv_physical_device, pdevice, physicalDevice);
	struct intel_perf_config *perf = pdevice->perf;

	uint32_t desc_count = *pCounterCount;

	VK_OUTARRAY_MAKE_TYPED(VkPerformanceCounterKHR, out, pCounters, pCounterCount);
	VK_OUTARRAY_MAKE_TYPED(VkPerformanceCounterDescriptionKHR, out_desc,
	pCounterDescriptions, &desc_count);

	/* We cannot support performance queries on anything other than RCS,
	* because the MI_REPORT_PERF_COUNT command is not available on other
	* engines.
	*/
	struct anv_queue_family *queue_family =
	&pdevice->queue.families[queueFamilyIndex];
	if (queue_family->engine_class != INTEL_ENGINE_CLASS_RENDER)
	return vk_outarray_status(&out);

	for (int c = 0; c < (perf ? perf->n_counters : 0); c++) {
	const struct intel_perf_query_counter *intel_counter = perf->counter_infos[c].counter;

	vk_outarray_append_typed(VkPerformanceCounterKHR, &out, counter) {
	counter->unit = intel_perf_counter_unit_to_vk_unit[intel_counter->units];
	counter->scope = VK_PERFORMANCE_COUNTER_SCOPE_COMMAND_KHR;
	counter->storage = intel_perf_counter_data_type_to_vk_storage[intel_counter->data_type];

	unsigned char sha1_result[20];
	_mesa_sha1_compute(intel_counter->symbol_name,
	strlen(intel_counter->symbol_name),
	sha1_result);
	memcpy(counter->uuid, sha1_result, sizeof(counter->uuid));
	}

	vk_outarray_append_typed(VkPerformanceCounterDescriptionKHR, &out_desc, desc) {
	desc->flags = 0; /* None so far. */
	snprintf(desc->name, sizeof(desc->name), "%s",
	INTEL_DEBUG(DEBUG_PERF_SYMBOL_NAMES) ?
	intel_counter->symbol_name :
	intel_counter->name);
	snprintf(desc->category, sizeof(desc->category), "%s", intel_counter->category);
	snprintf(desc->description, sizeof(desc->description), "%s", intel_counter->desc);
	}
	}

	return vk_outarray_status(&out);
	}

	void anv_GetPhysicalDeviceQueueFamilyPerformanceQueryPassesKHR(
	VkPhysicalDevice physicalDevice,
	const VkQueryPoolPerformanceCreateInfoKHR* pPerformanceQueryCreateInfo,
	uint32_t* pNumPasses)
	{
	ANV_FROM_HANDLE(anv_physical_device, pdevice, physicalDevice);
	struct intel_perf_config *perf = pdevice->perf;

	if (!perf) {
	*pNumPasses = 0;
	return;
	}

	*pNumPasses = intel_perf_get_n_passes(perf,
	pPerformanceQueryCreateInfo->pCounterIndices,
	pPerformanceQueryCreateInfo->counterIndexCount,
	NULL);
	}

	VkResult anv_AcquireProfilingLockKHR(
	VkDevice _device,
	const VkAcquireProfilingLockInfoKHR* pInfo)
	{
	ANV_FROM_HANDLE(anv_device, device, _device);
	struct intel_perf_config *perf = device->physical->perf;
	struct intel_perf_query_info *first_metric_set = &perf->queries[0];
	int fd = -1;

	assert(device->perf_fd == -1);

	if (!INTEL_DEBUG(DEBUG_NO_OACONFIG)) {
	struct anv_queue *queue = anv_device_get_perf_queue(device);

	if (queue == NULL)
	return VK_ERROR_UNKNOWN;
	fd = anv_device_perf_open(device, queue, first_metric_set->oa_metrics_set_id);
	if (fd < 0)
	return VK_TIMEOUT;
	}

	device->perf_fd = fd;
	return VK_SUCCESS;
	}

	void anv_ReleaseProfilingLockKHR(
	VkDevice _device)
	{
	ANV_FROM_HANDLE(anv_device, device, _device);

	anv_device_perf_close(device);
	}

	void
	anv_perf_write_pass_results(struct intel_perf_config *perf,
	struct anv_query_pool *pool, uint32_t pass,
	const struct intel_perf_query_result *accumulated_results,
	union VkPerformanceCounterResultKHR *results)
	{
	const struct intel_perf_query_info *query = pool->pass_query[pass];

	for (uint32_t c = 0; c < pool->n_counters; c++) {
	const struct intel_perf_counter_pass *counter_pass = &pool->counter_pass[c];

	if (counter_pass->query != query)
	continue;

	switch (pool->pass_query[pass]->kind) {
	case INTEL_PERF_QUERY_TYPE_PIPELINE: {
	assert(counter_pass->counter->data_type == INTEL_PERF_COUNTER_DATA_TYPE_UINT64);
	uint32_t accu_offset = counter_pass->counter->offset / sizeof(uint64_t);
	results[c].uint64 = accumulated_results->accumulator[accu_offset];
	break;
	}

	case INTEL_PERF_QUERY_TYPE_OA:
	case INTEL_PERF_QUERY_TYPE_RAW:
	switch (counter_pass->counter->data_type) {
	case INTEL_PERF_COUNTER_DATA_TYPE_UINT64:
	results[c].uint64 =
	counter_pass->counter->oa_counter_read_uint64(perf,
	counter_pass->query,
	accumulated_results);
	break;
	case INTEL_PERF_COUNTER_DATA_TYPE_FLOAT:
	results[c].float32 =
	counter_pass->counter->oa_counter_read_float(perf,
	counter_pass->query,
	accumulated_results);
	break;
	default:
	/* So far we aren't using uint32, double or bool32... */
	unreachable("unexpected counter data type");
	}
	break;

	default:
	unreachable("invalid query type");
	}

	/* The Vulkan extension only has nanoseconds as a unit */
	if (counter_pass->counter->units == INTEL_PERF_COUNTER_UNITS_US) {
	assert(counter_pass->counter->data_type == INTEL_PERF_COUNTER_DATA_TYPE_UINT64);
	results[c].uint64 *= 1000;
	}
	}
	}