src/amd/vulkan/radv_pipeline_binary.c - third_party/mesa - Git at Google

 /*
  * Copyright © 2024 Valve Corporation
  *
  * SPDX-License-Identifier: MIT
  */

 #include "radv_pipeline_binary.h"
 #include "util/blob.h"
 #include "util/disk_cache.h"
 #include "util/macros.h"
 #include "util/mesa-blake3.h"
 #include "util/mesa-sha1.h"
 #include "util/u_atomic.h"
 #include "util/u_debug.h"
 #include "radv_debug.h"
 #include "radv_device.h"
 #include "radv_entrypoints.h"
 #include "radv_pipeline_cache.h"
 #include "radv_pipeline_graphics.h"
 #include "radv_pipeline_rt.h"
 #include "radv_shader.h"
 #include "vk_log.h"
 #include "vk_pipeline.h"
 #include "vk_util.h"

 static VkResult
 radv_get_pipeline_key(struct radv_device *device, const VkPipelineCreateInfoKHR *pPipelineCreateInfo,
                       unsigned char *key)
 {
    VkResult result = VK_SUCCESS;

    switch (((VkBaseInStructure *)pPipelineCreateInfo->pNext)->sType) {
    case VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO: {
       const VkGraphicsPipelineCreateInfo *graphics_create_info =
          (VkGraphicsPipelineCreateInfo *)pPipelineCreateInfo->pNext;
       struct radv_graphics_pipeline_state gfx_state;

       result = radv_generate_graphics_pipeline_state(device, graphics_create_info, &gfx_state);
       if (result != VK_SUCCESS)
          return result;

       radv_graphics_pipeline_hash(device, &gfx_state, key);
       radv_graphics_pipeline_state_finish(device, &gfx_state);
       break;
    }
    case VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO: {
       const VkComputePipelineCreateInfo *compute_create_info =
          (VkComputePipelineCreateInfo *)pPipelineCreateInfo->pNext;

       radv_compute_pipeline_hash(device, compute_create_info, key);
       break;
    }
    case VK_STRUCTURE_TYPE_RAY_TRACING_PIPELINE_CREATE_INFO_KHR: {
       const VkRayTracingPipelineCreateInfoKHR *rt_create_info =
          (VkRayTracingPipelineCreateInfoKHR *)pPipelineCreateInfo->pNext;
       struct radv_ray_tracing_state_key rt_state;

       result = radv_generate_ray_tracing_state_key(device, rt_create_info, &rt_state);
       if (result != VK_SUCCESS)
          return result;

       radv_ray_tracing_pipeline_hash(device, rt_create_info, &rt_state, key);
       radv_ray_tracing_state_key_finish(&rt_state);
       break;
    }
    default:
       unreachable("unsupported pipeline create info struct");
    }

    return result;
 }

 VKAPI_ATTR VkResult VKAPI_CALL
 radv_GetPipelineKeyKHR(VkDevice _device, const VkPipelineCreateInfoKHR *pPipelineCreateInfo,
                        VkPipelineBinaryKeyKHR *pPipelineKey)
 {
    VK_FROM_HANDLE(radv_device, device, _device);
    const struct radv_physical_device *pdev = radv_device_physical(device);
    VkResult result;

    memset(pPipelineKey->key, 0, sizeof(pPipelineKey->key));

    /* Return the global key that applies to all pipelines. */
    if (!pPipelineCreateInfo) {
       struct mesa_blake3 ctx;

       static_assert(sizeof(blake3_hash) <= sizeof(pPipelineKey->key), "mismatch pipeline binary key size");

       _mesa_blake3_init(&ctx);
       _mesa_blake3_update(&ctx, pdev->cache_uuid, sizeof(pdev->cache_uuid));
       _mesa_blake3_update(&ctx, device->cache_hash, sizeof(device->cache_hash));
       _mesa_blake3_final(&ctx, pPipelineKey->key);

       pPipelineKey->keySize = sizeof(blake3_hash);

       return VK_SUCCESS;
    }

    result = radv_get_pipeline_key(device, pPipelineCreateInfo, pPipelineKey->key);
    if (result != VK_SUCCESS)
       return result;

    pPipelineKey->keySize = SHA1_DIGEST_LENGTH;

    return VK_SUCCESS;
 }

 static VkResult
 radv_create_pipeline_binary(struct radv_device *device, const VkAllocationCallbacks *pAllocator, const blake3_hash key,
                             const void *data, size_t data_size, struct radv_pipeline_binary **pipeline_binary_out)
 {
    struct radv_pipeline_binary *pipeline_binary;

    pipeline_binary =
       vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*pipeline_binary), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
    if (pipeline_binary == NULL)
       return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);

    vk_object_base_init(&device->vk, &pipeline_binary->base, VK_OBJECT_TYPE_PIPELINE_BINARY_KHR);

    pipeline_binary->data = (void *)data;
    pipeline_binary->size = data_size;

    memcpy(pipeline_binary->key, key, BLAKE3_OUT_LEN);

    *pipeline_binary_out = pipeline_binary;
    return VK_SUCCESS;
 }

 static VkResult
 radv_create_pipeline_binary_from_data(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
                                       const VkPipelineBinaryDataKHR *pData, const VkPipelineBinaryKeyKHR *pKey,
                                       struct util_dynarray *pipeline_binaries, uint32_t *num_binaries)
 {
    struct radv_pipeline_binary *pipeline_binary;
    VkResult result;
    void *data;

    if (!pipeline_binaries) {
       (*num_binaries)++;
       return VK_SUCCESS;
    }

    data = malloc(pData->dataSize);
    if (!data)
       return VK_ERROR_OUT_OF_HOST_MEMORY;

    memcpy(data, pData->pData, pData->dataSize);

    result = radv_create_pipeline_binary(device, pAllocator, pKey->key, data, pData->dataSize, &pipeline_binary);
    if (result != VK_SUCCESS) {
       free(data);
       return result;
    }

    util_dynarray_append(pipeline_binaries, struct radv_pipeline_binary *, pipeline_binary);
    return result;
 }

 VkResult
 radv_create_pipeline_binary_from_shader(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
                                         struct radv_shader *shader, struct util_dynarray *pipeline_binaries,
                                         uint32_t *num_binaries)
 {
    struct radv_pipeline_binary *pipeline_binary;
    struct blob blob;
    size_t data_size;
    VkResult result;
    void *data;

    if (!pipeline_binaries) {
       (*num_binaries)++;
       return VK_SUCCESS;
    }

    blob_init(&blob);
    radv_shader_serialize(shader, &blob);
    blob_finish_get_buffer(&blob, &data, &data_size);

    result = radv_create_pipeline_binary(device, pAllocator, shader->hash, data, data_size, &pipeline_binary);
    if (result != VK_SUCCESS) {
       free(data);
       return result;
    }

    util_dynarray_append(pipeline_binaries, struct radv_pipeline_binary *, pipeline_binary);
    return result;
 }

 VkResult
 radv_create_pipeline_binary_from_rt_shader(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
                                            struct radv_shader *shader, bool is_traversal_shader,
                                            const uint8_t stage_sha1[SHA1_DIGEST_LENGTH],
                                            const struct radv_ray_tracing_stage_info *rt_stage_info, uint32_t stack_size,
                                            struct vk_pipeline_cache_object *nir,
                                            struct util_dynarray *pipeline_binaries, uint32_t *num_binaries)
 {
    struct radv_pipeline_binary *pipeline_binary;
    struct mesa_blake3 ctx;
    struct blob blob;
    size_t data_size;
    blake3_hash key;
    VkResult result;
    void *data;

    if (!pipeline_binaries) {
       (*num_binaries)++;
       return VK_SUCCESS;
    }

    _mesa_blake3_init(&ctx);
    _mesa_blake3_update(&ctx, stage_sha1, sizeof(*stage_sha1));
    _mesa_blake3_final(&ctx, key);

    struct radv_ray_tracing_binary_header header = {
       .is_traversal_shader = is_traversal_shader,
       .has_shader = !!shader,
       .has_nir = !!nir,
       .stack_size = stack_size,
    };

    memcpy(header.stage_sha1, stage_sha1, sizeof(header.stage_sha1));
    if (rt_stage_info)
       memcpy(&header.stage_info, rt_stage_info, sizeof(header.stage_info));

    blob_init(&blob);
    blob_write_bytes(&blob, &header, sizeof(header));
    if (header.has_shader)
       radv_shader_serialize(shader, &blob);
    if (header.has_nir) {
       struct vk_raw_data_cache_object *nir_object = container_of(nir, struct vk_raw_data_cache_object, base);
       blob_write_bytes(&blob, nir_object->data, nir_object->data_size);
    }
    blob_finish_get_buffer(&blob, &data, &data_size);

    result = radv_create_pipeline_binary(device, pAllocator, key, data, data_size, &pipeline_binary);
    if (result != VK_SUCCESS) {
       free(data);
       return result;
    }

    util_dynarray_append(pipeline_binaries, struct radv_pipeline_binary *, pipeline_binary);
    return result;
 }

 static VkResult
 radv_create_pipeline_binary_from_pipeline(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
                                           struct radv_pipeline *pipeline, struct util_dynarray *pipeline_binaries,
                                           uint32_t *num_binaries)
 {
    VkResult result = VK_SUCCESS;

    if (pipeline->type == RADV_PIPELINE_RAY_TRACING) {
       struct radv_ray_tracing_pipeline *rt_pipeline = radv_pipeline_to_ray_tracing(pipeline);

       for (uint32_t i = 0; i < rt_pipeline->non_imported_stage_count; i++) {
          struct radv_ray_tracing_stage *rt_stage = &rt_pipeline->stages[i];

          result = radv_create_pipeline_binary_from_rt_shader(device, pAllocator, rt_stage->shader, false,
                                                              rt_stage->sha1, &rt_stage->info, rt_stage->stack_size,
                                                              rt_stage->nir, pipeline_binaries, num_binaries);
          if (result != VK_SUCCESS)
             return result;
       }

       struct radv_shader *traversal_shader = rt_pipeline->base.base.shaders[MESA_SHADER_INTERSECTION];
       if (traversal_shader) {
          result = radv_create_pipeline_binary_from_rt_shader(device, pAllocator, traversal_shader, true,
                                                              traversal_shader->hash, NULL, 0, NULL, pipeline_binaries,
                                                              num_binaries);
          if (result != VK_SUCCESS)
             return result;
       }
    } else {
       for (uint32_t i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) {
          if (!pipeline->shaders[i])
             continue;

          result = radv_create_pipeline_binary_from_shader(device, pAllocator, pipeline->shaders[i], pipeline_binaries,
                                                           num_binaries);
          if (result != VK_SUCCESS)
             return result;
       }

       if (pipeline->gs_copy_shader) {
          result = radv_create_pipeline_binary_from_shader(device, pAllocator, pipeline->gs_copy_shader,
                                                           pipeline_binaries, num_binaries);
          if (result != VK_SUCCESS)
             return result;
       }
    }

    return result;
 }

 static VkResult
 radv_create_pipeline_binary_from_cache(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
                                        const VkPipelineCreateInfoKHR *pPipelineCreateInfo,
                                        struct util_dynarray *pipeline_binaries, uint32_t *num_binaries)
 {
    unsigned char key[SHA1_DIGEST_LENGTH];
    bool found_in_internal_cache;
    VkResult result;

    assert(pPipelineCreateInfo);

    result = radv_get_pipeline_key(device, pPipelineCreateInfo, key);
    if (result != VK_SUCCESS)
       return result;

    result = radv_pipeline_cache_get_binaries(device, pAllocator, key, pipeline_binaries, num_binaries,
                                              &found_in_internal_cache);
    if (result != VK_SUCCESS)
       return result;

    return found_in_internal_cache ? VK_SUCCESS : VK_PIPELINE_BINARY_MISSING_KHR;
 }

 static VkResult
 radv_create_pipeline_binaries(struct radv_device *device, const VkPipelineBinaryCreateInfoKHR *pCreateInfo,
                               const VkAllocationCallbacks *pAllocator, struct util_dynarray *pipeline_binaries,
                               uint32_t *num_binaries)
 {
    VkResult result = VK_SUCCESS;

    if (pCreateInfo->pKeysAndDataInfo) {
       const VkPipelineBinaryKeysAndDataKHR *pKeysAndDataInfo = pCreateInfo->pKeysAndDataInfo;

       for (uint32_t i = 0; i < pKeysAndDataInfo->binaryCount; i++) {
          const VkPipelineBinaryDataKHR *pData = &pKeysAndDataInfo->pPipelineBinaryData[i];
          const VkPipelineBinaryKeyKHR *pKey = &pKeysAndDataInfo->pPipelineBinaryKeys[i];

          result =
             radv_create_pipeline_binary_from_data(device, pAllocator, pData, pKey, pipeline_binaries, num_binaries);
          if (result != VK_SUCCESS)
             return result;
       }
    } else if (pCreateInfo->pipeline) {
       VK_FROM_HANDLE(radv_pipeline, pipeline, pCreateInfo->pipeline);

       result = radv_create_pipeline_binary_from_pipeline(device, pAllocator, pipeline, pipeline_binaries, num_binaries);
    } else {
       result = radv_create_pipeline_binary_from_cache(device, pAllocator, pCreateInfo->pPipelineCreateInfo,
                                                       pipeline_binaries, num_binaries);
    }

    return result;
 }

 static void
 radv_destroy_pipeline_binary(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
                              struct radv_pipeline_binary *pipeline_binary)
 {
    if (!pipeline_binary)
       return;

    free(pipeline_binary->data);

    vk_object_base_finish(&pipeline_binary->base);
    vk_free2(&device->vk.alloc, pAllocator, pipeline_binary);
 }

 VKAPI_ATTR VkResult VKAPI_CALL
 radv_CreatePipelineBinariesKHR(VkDevice _device, const VkPipelineBinaryCreateInfoKHR *pCreateInfo,
                                const VkAllocationCallbacks *pAllocator, VkPipelineBinaryHandlesInfoKHR *pBinaries)
 {
    VK_FROM_HANDLE(radv_device, device, _device);
    struct util_dynarray pipeline_binaries;
    VkResult result;

    if (!pBinaries->pPipelineBinaries) {
       result = radv_create_pipeline_binaries(device, pCreateInfo, pAllocator, NULL, &pBinaries->pipelineBinaryCount);
       return result;
    }

    for (uint32_t i = 0; i < pBinaries->pipelineBinaryCount; i++)
       pBinaries->pPipelineBinaries[i] = VK_NULL_HANDLE;

    util_dynarray_init(&pipeline_binaries, NULL);

    /* Get all pipeline binaries from the pCreateInfo first to simplify the creation. */
    result = radv_create_pipeline_binaries(device, pCreateInfo, pAllocator, &pipeline_binaries, NULL);
    if (result != VK_SUCCESS) {
       util_dynarray_foreach (&pipeline_binaries, struct radv_pipeline_binary *, pipeline_binary)
          radv_destroy_pipeline_binary(device, pAllocator, *pipeline_binary);
       util_dynarray_fini(&pipeline_binaries);
       return result;
    }

    const uint32_t num_binaries = util_dynarray_num_elements(&pipeline_binaries, struct radv_pipeline_binary *);

    for (uint32_t i = 0; i < num_binaries; i++) {
       struct radv_pipeline_binary **pipeline_binary =
          util_dynarray_element(&pipeline_binaries, struct radv_pipeline_binary *, i);

       if (i < pBinaries->pipelineBinaryCount) {
          pBinaries->pPipelineBinaries[i] = radv_pipeline_binary_to_handle(*pipeline_binary);
       } else {
          /* Free the pipeline binary that couldn't be returned. */
          radv_destroy_pipeline_binary(device, pAllocator, *pipeline_binary);
       }
    }

    result = pBinaries->pipelineBinaryCount < num_binaries ? VK_INCOMPLETE : result;
    pBinaries->pipelineBinaryCount = MIN2(num_binaries, pBinaries->pipelineBinaryCount);

    util_dynarray_fini(&pipeline_binaries);
    return result;
 }

 VKAPI_ATTR void VKAPI_CALL
 radv_DestroyPipelineBinaryKHR(VkDevice _device, VkPipelineBinaryKHR pipelineBinary,
                               const VkAllocationCallbacks *pAllocator)
 {
    VK_FROM_HANDLE(radv_pipeline_binary, pipeline_binary, pipelineBinary);
    VK_FROM_HANDLE(radv_device, device, _device);

    radv_destroy_pipeline_binary(device, pAllocator, pipeline_binary);
 }

 VKAPI_ATTR VkResult VKAPI_CALL
 radv_GetPipelineBinaryDataKHR(VkDevice _device, const VkPipelineBinaryDataInfoKHR *pInfo,
                               VkPipelineBinaryKeyKHR *pPipelineBinaryKey, size_t *pPipelineBinaryDataSize,
                               void *pPipelineBinaryData)
 {
    VK_FROM_HANDLE(radv_pipeline_binary, pipeline_binary, pInfo->pipelineBinary);
    const size_t size = pipeline_binary->size;

    memcpy(pPipelineBinaryKey->key, pipeline_binary->key, sizeof(pipeline_binary->key));
    pPipelineBinaryKey->keySize = sizeof(pipeline_binary->key);

    if (!pPipelineBinaryData) {
       *pPipelineBinaryDataSize = size;
       return VK_SUCCESS;
    }

    if (*pPipelineBinaryDataSize < size) {
       *pPipelineBinaryDataSize = size;
       return VK_ERROR_NOT_ENOUGH_SPACE_KHR;
    }

    memcpy(pPipelineBinaryData, pipeline_binary->data, size);
    *pPipelineBinaryDataSize = size;

    return VK_SUCCESS;
 }

 VKAPI_ATTR VkResult VKAPI_CALL
 radv_ReleaseCapturedPipelineDataKHR(VkDevice _device, const VkReleaseCapturedPipelineDataInfoKHR *pInfo,
                                     const VkAllocationCallbacks *pAllocator)
 {
    /* no-op */
    return VK_SUCCESS;
 }
	/*
	* Copyright © 2024 Valve Corporation
	*
	* SPDX-License-Identifier: MIT
	*/

	#include "radv_pipeline_binary.h"
	#include "util/blob.h"
	#include "util/disk_cache.h"
	#include "util/macros.h"
	#include "util/mesa-blake3.h"
	#include "util/mesa-sha1.h"
	#include "util/u_atomic.h"
	#include "util/u_debug.h"
	#include "radv_debug.h"
	#include "radv_device.h"
	#include "radv_entrypoints.h"
	#include "radv_pipeline_cache.h"
	#include "radv_pipeline_graphics.h"
	#include "radv_pipeline_rt.h"
	#include "radv_shader.h"
	#include "vk_log.h"
	#include "vk_pipeline.h"
	#include "vk_util.h"

	static VkResult
	radv_get_pipeline_key(struct radv_device device, const VkPipelineCreateInfoKHR pPipelineCreateInfo,
	unsigned char *key)
	{
	VkResult result = VK_SUCCESS;

	switch (((VkBaseInStructure *)pPipelineCreateInfo->pNext)->sType) {
	case VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO: {
	const VkGraphicsPipelineCreateInfo *graphics_create_info =
	(VkGraphicsPipelineCreateInfo *)pPipelineCreateInfo->pNext;
	struct radv_graphics_pipeline_state gfx_state;

	result = radv_generate_graphics_pipeline_state(device, graphics_create_info, &gfx_state);
	if (result != VK_SUCCESS)
	return result;

	radv_graphics_pipeline_hash(device, &gfx_state, key);
	radv_graphics_pipeline_state_finish(device, &gfx_state);
	break;
	}
	case VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO: {
	const VkComputePipelineCreateInfo *compute_create_info =
	(VkComputePipelineCreateInfo *)pPipelineCreateInfo->pNext;

	radv_compute_pipeline_hash(device, compute_create_info, key);
	break;
	}
	case VK_STRUCTURE_TYPE_RAY_TRACING_PIPELINE_CREATE_INFO_KHR: {
	const VkRayTracingPipelineCreateInfoKHR *rt_create_info =
	(VkRayTracingPipelineCreateInfoKHR *)pPipelineCreateInfo->pNext;
	struct radv_ray_tracing_state_key rt_state;

	result = radv_generate_ray_tracing_state_key(device, rt_create_info, &rt_state);
	if (result != VK_SUCCESS)
	return result;

	radv_ray_tracing_pipeline_hash(device, rt_create_info, &rt_state, key);
	radv_ray_tracing_state_key_finish(&rt_state);
	break;
	}
	default:
	unreachable("unsupported pipeline create info struct");
	}

	return result;
	}

	VKAPI_ATTR VkResult VKAPI_CALL
	radv_GetPipelineKeyKHR(VkDevice _device, const VkPipelineCreateInfoKHR *pPipelineCreateInfo,
	VkPipelineBinaryKeyKHR *pPipelineKey)
	{
	VK_FROM_HANDLE(radv_device, device, _device);
	const struct radv_physical_device *pdev = radv_device_physical(device);
	VkResult result;

	memset(pPipelineKey->key, 0, sizeof(pPipelineKey->key));

	/* Return the global key that applies to all pipelines. */
	if (!pPipelineCreateInfo) {
	struct mesa_blake3 ctx;

	static_assert(sizeof(blake3_hash) <= sizeof(pPipelineKey->key), "mismatch pipeline binary key size");

	_mesa_blake3_init(&ctx);
	_mesa_blake3_update(&ctx, pdev->cache_uuid, sizeof(pdev->cache_uuid));
	_mesa_blake3_update(&ctx, device->cache_hash, sizeof(device->cache_hash));
	_mesa_blake3_final(&ctx, pPipelineKey->key);

	pPipelineKey->keySize = sizeof(blake3_hash);

	return VK_SUCCESS;
	}

	result = radv_get_pipeline_key(device, pPipelineCreateInfo, pPipelineKey->key);
	if (result != VK_SUCCESS)
	return result;

	pPipelineKey->keySize = SHA1_DIGEST_LENGTH;

	return VK_SUCCESS;
	}

	static VkResult
	radv_create_pipeline_binary(struct radv_device device, const VkAllocationCallbacks pAllocator, const blake3_hash key,
	const void data, size_t data_size, struct radv_pipeline_binary *pipeline_binary_out)
	{
	struct radv_pipeline_binary *pipeline_binary;

	pipeline_binary =
	vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*pipeline_binary), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
	if (pipeline_binary == NULL)
	return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);

	vk_object_base_init(&device->vk, &pipeline_binary->base, VK_OBJECT_TYPE_PIPELINE_BINARY_KHR);

	pipeline_binary->data = (void *)data;
	pipeline_binary->size = data_size;

	memcpy(pipeline_binary->key, key, BLAKE3_OUT_LEN);

	*pipeline_binary_out = pipeline_binary;
	return VK_SUCCESS;
	}

	static VkResult
	radv_create_pipeline_binary_from_data(struct radv_device device, const VkAllocationCallbacks pAllocator,
	const VkPipelineBinaryDataKHR pData, const VkPipelineBinaryKeyKHR pKey,
	struct util_dynarray pipeline_binaries, uint32_t num_binaries)
	{
	struct radv_pipeline_binary *pipeline_binary;
	VkResult result;
	void *data;

	if (!pipeline_binaries) {
	(*num_binaries)++;
	return VK_SUCCESS;
	}

	data = malloc(pData->dataSize);
	if (!data)
	return VK_ERROR_OUT_OF_HOST_MEMORY;

	memcpy(data, pData->pData, pData->dataSize);

	result = radv_create_pipeline_binary(device, pAllocator, pKey->key, data, pData->dataSize, &pipeline_binary);
	if (result != VK_SUCCESS) {
	free(data);
	return result;
	}

	util_dynarray_append(pipeline_binaries, struct radv_pipeline_binary *, pipeline_binary);
	return result;
	}

	VkResult
	radv_create_pipeline_binary_from_shader(struct radv_device device, const VkAllocationCallbacks pAllocator,
	struct radv_shader shader, struct util_dynarray pipeline_binaries,
	uint32_t *num_binaries)
	{
	struct radv_pipeline_binary *pipeline_binary;
	struct blob blob;
	size_t data_size;
	VkResult result;
	void *data;

	if (!pipeline_binaries) {
	(*num_binaries)++;
	return VK_SUCCESS;
	}

	blob_init(&blob);
	radv_shader_serialize(shader, &blob);
	blob_finish_get_buffer(&blob, &data, &data_size);

	result = radv_create_pipeline_binary(device, pAllocator, shader->hash, data, data_size, &pipeline_binary);
	if (result != VK_SUCCESS) {
	free(data);
	return result;
	}

	util_dynarray_append(pipeline_binaries, struct radv_pipeline_binary *, pipeline_binary);
	return result;
	}

	VkResult
	radv_create_pipeline_binary_from_rt_shader(struct radv_device device, const VkAllocationCallbacks pAllocator,
	struct radv_shader *shader, bool is_traversal_shader,
	const uint8_t stage_sha1[SHA1_DIGEST_LENGTH],
	const struct radv_ray_tracing_stage_info *rt_stage_info, uint32_t stack_size,
	struct vk_pipeline_cache_object *nir,
	struct util_dynarray pipeline_binaries, uint32_t num_binaries)
	{
	struct radv_pipeline_binary *pipeline_binary;
	struct mesa_blake3 ctx;
	struct blob blob;
	size_t data_size;
	blake3_hash key;
	VkResult result;
	void *data;

	if (!pipeline_binaries) {
	(*num_binaries)++;
	return VK_SUCCESS;
	}

	_mesa_blake3_init(&ctx);
	_mesa_blake3_update(&ctx, stage_sha1, sizeof(*stage_sha1));
	_mesa_blake3_final(&ctx, key);

	struct radv_ray_tracing_binary_header header = {
	.is_traversal_shader = is_traversal_shader,
	.has_shader = !!shader,
	.has_nir = !!nir,
	.stack_size = stack_size,
	};

	memcpy(header.stage_sha1, stage_sha1, sizeof(header.stage_sha1));
	if (rt_stage_info)
	memcpy(&header.stage_info, rt_stage_info, sizeof(header.stage_info));

	blob_init(&blob);
	blob_write_bytes(&blob, &header, sizeof(header));
	if (header.has_shader)
	radv_shader_serialize(shader, &blob);
	if (header.has_nir) {
	struct vk_raw_data_cache_object *nir_object = container_of(nir, struct vk_raw_data_cache_object, base);
	blob_write_bytes(&blob, nir_object->data, nir_object->data_size);
	}
	blob_finish_get_buffer(&blob, &data, &data_size);

	result = radv_create_pipeline_binary(device, pAllocator, key, data, data_size, &pipeline_binary);
	if (result != VK_SUCCESS) {
	free(data);
	return result;
	}

	util_dynarray_append(pipeline_binaries, struct radv_pipeline_binary *, pipeline_binary);
	return result;
	}

	static VkResult
	radv_create_pipeline_binary_from_pipeline(struct radv_device device, const VkAllocationCallbacks pAllocator,
	struct radv_pipeline pipeline, struct util_dynarray pipeline_binaries,
	uint32_t *num_binaries)
	{
	VkResult result = VK_SUCCESS;

	if (pipeline->type == RADV_PIPELINE_RAY_TRACING) {
	struct radv_ray_tracing_pipeline *rt_pipeline = radv_pipeline_to_ray_tracing(pipeline);

	for (uint32_t i = 0; i < rt_pipeline->non_imported_stage_count; i++) {
	struct radv_ray_tracing_stage *rt_stage = &rt_pipeline->stages[i];

	result = radv_create_pipeline_binary_from_rt_shader(device, pAllocator, rt_stage->shader, false,
	rt_stage->sha1, &rt_stage->info, rt_stage->stack_size,
	rt_stage->nir, pipeline_binaries, num_binaries);
	if (result != VK_SUCCESS)
	return result;
	}

	struct radv_shader *traversal_shader = rt_pipeline->base.base.shaders[MESA_SHADER_INTERSECTION];
	if (traversal_shader) {
	result = radv_create_pipeline_binary_from_rt_shader(device, pAllocator, traversal_shader, true,
	traversal_shader->hash, NULL, 0, NULL, pipeline_binaries,
	num_binaries);
	if (result != VK_SUCCESS)
	return result;
	}
	} else {
	for (uint32_t i = 0; i < MESA_VULKAN_SHADER_STAGES; i++) {
	if (!pipeline->shaders[i])
	continue;

	result = radv_create_pipeline_binary_from_shader(device, pAllocator, pipeline->shaders[i], pipeline_binaries,
	num_binaries);
	if (result != VK_SUCCESS)
	return result;
	}

	if (pipeline->gs_copy_shader) {
	result = radv_create_pipeline_binary_from_shader(device, pAllocator, pipeline->gs_copy_shader,
	pipeline_binaries, num_binaries);
	if (result != VK_SUCCESS)
	return result;
	}
	}

	return result;
	}

	static VkResult
	radv_create_pipeline_binary_from_cache(struct radv_device device, const VkAllocationCallbacks pAllocator,
	const VkPipelineCreateInfoKHR *pPipelineCreateInfo,
	struct util_dynarray pipeline_binaries, uint32_t num_binaries)
	{
	unsigned char key[SHA1_DIGEST_LENGTH];
	bool found_in_internal_cache;
	VkResult result;

	assert(pPipelineCreateInfo);

	result = radv_get_pipeline_key(device, pPipelineCreateInfo, key);
	if (result != VK_SUCCESS)
	return result;

	result = radv_pipeline_cache_get_binaries(device, pAllocator, key, pipeline_binaries, num_binaries,
	&found_in_internal_cache);
	if (result != VK_SUCCESS)
	return result;

	return found_in_internal_cache ? VK_SUCCESS : VK_PIPELINE_BINARY_MISSING_KHR;
	}

	static VkResult
	radv_create_pipeline_binaries(struct radv_device device, const VkPipelineBinaryCreateInfoKHR pCreateInfo,
	const VkAllocationCallbacks pAllocator, struct util_dynarray pipeline_binaries,
	uint32_t *num_binaries)
	{
	VkResult result = VK_SUCCESS;

	if (pCreateInfo->pKeysAndDataInfo) {
	const VkPipelineBinaryKeysAndDataKHR *pKeysAndDataInfo = pCreateInfo->pKeysAndDataInfo;

	for (uint32_t i = 0; i < pKeysAndDataInfo->binaryCount; i++) {
	const VkPipelineBinaryDataKHR *pData = &pKeysAndDataInfo->pPipelineBinaryData[i];
	const VkPipelineBinaryKeyKHR *pKey = &pKeysAndDataInfo->pPipelineBinaryKeys[i];

	result =
	radv_create_pipeline_binary_from_data(device, pAllocator, pData, pKey, pipeline_binaries, num_binaries);
	if (result != VK_SUCCESS)
	return result;
	}
	} else if (pCreateInfo->pipeline) {
	VK_FROM_HANDLE(radv_pipeline, pipeline, pCreateInfo->pipeline);

	result = radv_create_pipeline_binary_from_pipeline(device, pAllocator, pipeline, pipeline_binaries, num_binaries);
	} else {
	result = radv_create_pipeline_binary_from_cache(device, pAllocator, pCreateInfo->pPipelineCreateInfo,
	pipeline_binaries, num_binaries);
	}

	return result;
	}

	static void
	radv_destroy_pipeline_binary(struct radv_device device, const VkAllocationCallbacks pAllocator,
	struct radv_pipeline_binary *pipeline_binary)
	{
	if (!pipeline_binary)
	return;

	free(pipeline_binary->data);

	vk_object_base_finish(&pipeline_binary->base);
	vk_free2(&device->vk.alloc, pAllocator, pipeline_binary);
	}

	VKAPI_ATTR VkResult VKAPI_CALL
	radv_CreatePipelineBinariesKHR(VkDevice _device, const VkPipelineBinaryCreateInfoKHR *pCreateInfo,
	const VkAllocationCallbacks pAllocator, VkPipelineBinaryHandlesInfoKHR pBinaries)
	{
	VK_FROM_HANDLE(radv_device, device, _device);
	struct util_dynarray pipeline_binaries;
	VkResult result;

	if (!pBinaries->pPipelineBinaries) {
	result = radv_create_pipeline_binaries(device, pCreateInfo, pAllocator, NULL, &pBinaries->pipelineBinaryCount);
	return result;
	}

	for (uint32_t i = 0; i < pBinaries->pipelineBinaryCount; i++)
	pBinaries->pPipelineBinaries[i] = VK_NULL_HANDLE;

	util_dynarray_init(&pipeline_binaries, NULL);

	/* Get all pipeline binaries from the pCreateInfo first to simplify the creation. */
	result = radv_create_pipeline_binaries(device, pCreateInfo, pAllocator, &pipeline_binaries, NULL);
	if (result != VK_SUCCESS) {
	util_dynarray_foreach (&pipeline_binaries, struct radv_pipeline_binary *, pipeline_binary)
	radv_destroy_pipeline_binary(device, pAllocator, *pipeline_binary);
	util_dynarray_fini(&pipeline_binaries);
	return result;
	}

	const uint32_t num_binaries = util_dynarray_num_elements(&pipeline_binaries, struct radv_pipeline_binary *);

	for (uint32_t i = 0; i < num_binaries; i++) {
	struct radv_pipeline_binary **pipeline_binary =
	util_dynarray_element(&pipeline_binaries, struct radv_pipeline_binary *, i);

	if (i < pBinaries->pipelineBinaryCount) {
	pBinaries->pPipelineBinaries[i] = radv_pipeline_binary_to_handle(*pipeline_binary);
	} else {
	/* Free the pipeline binary that couldn't be returned. */
	radv_destroy_pipeline_binary(device, pAllocator, *pipeline_binary);
	}
	}

	result = pBinaries->pipelineBinaryCount < num_binaries ? VK_INCOMPLETE : result;
	pBinaries->pipelineBinaryCount = MIN2(num_binaries, pBinaries->pipelineBinaryCount);

	util_dynarray_fini(&pipeline_binaries);
	return result;
	}

	VKAPI_ATTR void VKAPI_CALL
	radv_DestroyPipelineBinaryKHR(VkDevice _device, VkPipelineBinaryKHR pipelineBinary,
	const VkAllocationCallbacks *pAllocator)
	{
	VK_FROM_HANDLE(radv_pipeline_binary, pipeline_binary, pipelineBinary);
	VK_FROM_HANDLE(radv_device, device, _device);

	radv_destroy_pipeline_binary(device, pAllocator, pipeline_binary);
	}

	VKAPI_ATTR VkResult VKAPI_CALL
	radv_GetPipelineBinaryDataKHR(VkDevice _device, const VkPipelineBinaryDataInfoKHR *pInfo,
	VkPipelineBinaryKeyKHR pPipelineBinaryKey, size_t pPipelineBinaryDataSize,
	void *pPipelineBinaryData)
	{
	VK_FROM_HANDLE(radv_pipeline_binary, pipeline_binary, pInfo->pipelineBinary);
	const size_t size = pipeline_binary->size;

	memcpy(pPipelineBinaryKey->key, pipeline_binary->key, sizeof(pipeline_binary->key));
	pPipelineBinaryKey->keySize = sizeof(pipeline_binary->key);

	if (!pPipelineBinaryData) {
	*pPipelineBinaryDataSize = size;
	return VK_SUCCESS;
	}

	if (*pPipelineBinaryDataSize < size) {
	*pPipelineBinaryDataSize = size;
	return VK_ERROR_NOT_ENOUGH_SPACE_KHR;
	}

	memcpy(pPipelineBinaryData, pipeline_binary->data, size);
	*pPipelineBinaryDataSize = size;

	return VK_SUCCESS;
	}

	VKAPI_ATTR VkResult VKAPI_CALL
	radv_ReleaseCapturedPipelineDataKHR(VkDevice _device, const VkReleaseCapturedPipelineDataInfoKHR *pInfo,
	const VkAllocationCallbacks *pAllocator)
	{
	/* no-op */
	return VK_SUCCESS;
	}