| /* Copyright © 2024 Intel Corporation | |
| * SPDX-License-Identifier: MIT | |
| */ | |
| #include "anv_private.h" | |
| static void | |
| anv_bind_buffer_memory(struct anv_device *device, | |
| const VkBindBufferMemoryInfo *pBindInfo) | |
| { | |
| ANV_FROM_HANDLE(anv_device_memory, mem, pBindInfo->memory); | |
| ANV_FROM_HANDLE(anv_buffer, buffer, pBindInfo->buffer); | |
| assert(pBindInfo->sType == VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO); | |
| assert(!anv_buffer_is_sparse(buffer)); | |
| const VkBindMemoryStatus *bind_status = | |
| vk_find_struct_const(pBindInfo->pNext, BIND_MEMORY_STATUS); | |
| if (mem) { | |
| assert(pBindInfo->memoryOffset < mem->vk.size); | |
| assert(mem->vk.size - pBindInfo->memoryOffset >= buffer->vk.size); | |
| buffer->address = (struct anv_address) { | |
| .bo = mem->bo, | |
| .offset = pBindInfo->memoryOffset, | |
| .protected = anv_buffer_is_protected(buffer), | |
| }; | |
| } else { | |
| buffer->address = ANV_NULL_ADDRESS; | |
| } | |
| buffer->vk.device_address = anv_address_physical(buffer->address); | |
| ANV_RMV(buffer_bind, device, buffer); | |
| ANV_ADDR_BINDING_REPORT_ADDR_BIND(device, | |
| &buffer->vk.base, | |
| buffer->vk.device_address, | |
| buffer->vk.size); | |
| if (bind_status) | |
| *bind_status->pResult = VK_SUCCESS; | |
| } | |
| VkResult anv_BindBufferMemory2( | |
| VkDevice _device, | |
| uint32_t bindInfoCount, | |
| const VkBindBufferMemoryInfo* pBindInfos) | |
| { | |
| ANV_FROM_HANDLE(anv_device, device, _device); | |
| for (uint32_t i = 0; i < bindInfoCount; i++) | |
| anv_bind_buffer_memory(device, &pBindInfos[i]); | |
| return VK_SUCCESS; | |
| } | |
| // Buffer functions | |
| static void | |
| anv_get_buffer_memory_requirements(struct anv_device *device, | |
| VkBufferCreateFlags flags, | |
| VkDeviceSize size, | |
| VkBufferUsageFlags2 usage, | |
| bool is_sparse, | |
| VkMemoryRequirements2* pMemoryRequirements) | |
| { | |
| /* The Vulkan spec (git aaed022) says: | |
| * | |
| * memoryTypeBits is a bitfield and contains one bit set for every | |
| * supported memory type for the resource. The bit `1<<i` is set if and | |
| * only if the memory type `i` in the VkPhysicalDeviceMemoryProperties | |
| * structure for the physical device is supported. | |
| * | |
| * Descriptors buffers need to be in the dynamic visible heap (for the | |
| * SAMPLER_STATE). | |
| * | |
| * Preprocess buffers containing INTERFACE_DESCRIPTOR_DATA structures also | |
| * need to be there (those structures are indexed from the dynamic state | |
| * heap). | |
| */ | |
| bool need_dynamic_visible_buffer = | |
| (usage & (VK_BUFFER_USAGE_2_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT | | |
| VK_BUFFER_USAGE_2_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT | | |
| VK_BUFFER_USAGE_2_DESCRIPTOR_HEAP_BIT_EXT)) || | |
| (device->info->verx10 <= 120 && | |
| (usage & VK_BUFFER_USAGE_2_PREPROCESS_BUFFER_BIT_EXT)); | |
| uint32_t memory_types; | |
| if (flags & VK_BUFFER_CREATE_PROTECTED_BIT) | |
| memory_types = device->physical->memory.protected_mem_types; | |
| else if (need_dynamic_visible_buffer) | |
| memory_types = device->physical->memory.dynamic_visible_mem_types; | |
| else | |
| memory_types = device->physical->memory.default_buffer_mem_types | | |
| device->physical->memory.compressed_mem_types; | |
| /* The GPU appears to write back to main memory in cachelines. Writes to a | |
| * buffers should not clobber with writes to another buffers so make sure | |
| * those are in different cachelines. | |
| */ | |
| uint32_t alignment = 64; | |
| /* From the spec, section "Sparse Buffer and Fully-Resident Image Block | |
| * Size": | |
| * "The sparse block size in bytes for sparse buffers and fully-resident | |
| * images is reported as VkMemoryRequirements::alignment. alignment | |
| * represents both the memory alignment requirement and the binding | |
| * granularity (in bytes) for sparse resources." | |
| */ | |
| if (is_sparse) { | |
| alignment = ANV_SPARSE_BLOCK_SIZE; | |
| size = align64(size, alignment); | |
| } | |
| pMemoryRequirements->memoryRequirements.size = size; | |
| pMemoryRequirements->memoryRequirements.alignment = alignment; | |
| /* Storage and Uniform buffers should have their size aligned to | |
| * 32-bits to avoid boundary checks when last DWord is not complete. | |
| * This would ensure that not internal padding would be needed for | |
| * 16-bit types. | |
| */ | |
| if (device->robust_buffer_access && | |
| (usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT || | |
| usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT)) | |
| pMemoryRequirements->memoryRequirements.size = align64(size, 4); | |
| pMemoryRequirements->memoryRequirements.memoryTypeBits = memory_types; | |
| vk_foreach_struct(sType, ext, pMemoryRequirements->pNext) { | |
| switch (sType) { | |
| case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: { | |
| VkMemoryDedicatedRequirements *requirements = ext; | |
| requirements->prefersDedicatedAllocation = false; | |
| requirements->requiresDedicatedAllocation = false; | |
| break; | |
| } | |
| default: | |
| vk_debug_ignored_stype(sType); | |
| break; | |
| } | |
| } | |
| } | |
| void anv_GetDeviceBufferMemoryRequirements( | |
| VkDevice _device, | |
| const VkDeviceBufferMemoryRequirements* pInfo, | |
| VkMemoryRequirements2* pMemoryRequirements) | |
| { | |
| ANV_FROM_HANDLE(anv_device, device, _device); | |
| const bool is_sparse = | |
| pInfo->pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT; | |
| VkBufferUsageFlags2 usages = vk_buffer_usage_flags(pInfo->pCreateInfo); | |
| if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) && | |
| INTEL_DEBUG(DEBUG_SPARSE) && | |
| pInfo->pCreateInfo->flags & (VK_BUFFER_CREATE_SPARSE_BINDING_BIT | | |
| VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT | | |
| VK_BUFFER_CREATE_SPARSE_ALIASED_BIT)) | |
| mesa_logi("=== %s %s:%d flags:0x%08x\n", __func__, __FILE__, | |
| __LINE__, pInfo->pCreateInfo->flags); | |
| anv_get_buffer_memory_requirements(device, | |
| pInfo->pCreateInfo->flags, | |
| pInfo->pCreateInfo->size, | |
| usages, | |
| is_sparse, | |
| pMemoryRequirements); | |
| } | |
| VkResult anv_CreateBuffer( | |
| VkDevice _device, | |
| const VkBufferCreateInfo* pCreateInfo, | |
| const VkAllocationCallbacks* pAllocator, | |
| VkBuffer* pBuffer) | |
| { | |
| ANV_FROM_HANDLE(anv_device, device, _device); | |
| struct anv_buffer *buffer; | |
| if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) && | |
| INTEL_DEBUG(DEBUG_SPARSE) && | |
| pCreateInfo->flags & (VK_BUFFER_CREATE_SPARSE_BINDING_BIT | | |
| VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT | | |
| VK_BUFFER_CREATE_SPARSE_ALIASED_BIT)) | |
| mesa_logi("=== %s %s:%d flags:0x%08x\n", __func__, __FILE__, | |
| __LINE__, pCreateInfo->flags); | |
| if ((pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) && | |
| device->physical->sparse_type == ANV_SPARSE_TYPE_TRTT) { | |
| VkBufferUsageFlags2 usages = vk_buffer_usage_flags(pCreateInfo); | |
| if (usages & (VK_BUFFER_USAGE_2_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT | | |
| VK_BUFFER_USAGE_2_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT)) { | |
| return vk_errorf(device, VK_ERROR_UNKNOWN, | |
| "Cannot support sparse descriptor buffers with TRTT."); | |
| } | |
| } | |
| /* Don't allow creating buffers bigger than our address space. The real | |
| * issue here is that we may align up the buffer size and we don't want | |
| * doing so to cause roll-over. However, no one has any business | |
| * allocating a buffer larger than our GTT size. | |
| */ | |
| if (pCreateInfo->size > device->physical->gtt_size) | |
| return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); | |
| buffer = vk_buffer_create(&device->vk, pCreateInfo, | |
| pAllocator, sizeof(*buffer)); | |
| if (buffer == NULL) | |
| return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); | |
| buffer->address = ANV_NULL_ADDRESS; | |
| if (anv_buffer_is_sparse(buffer)) { | |
| enum anv_bo_alloc_flags alloc_flags = 0; | |
| uint64_t client_address = 0; | |
| if (buffer->vk.create_flags & VK_BUFFER_CREATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT) { | |
| alloc_flags = ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS; | |
| const VkBufferOpaqueCaptureAddressCreateInfo *opaque_addr_info = | |
| vk_find_struct_const(pCreateInfo->pNext, | |
| BUFFER_OPAQUE_CAPTURE_ADDRESS_CREATE_INFO); | |
| if (opaque_addr_info) | |
| client_address = opaque_addr_info->opaqueCaptureAddress; | |
| } | |
| if (buffer->vk.create_flags & VK_BUFFER_CREATE_DESCRIPTOR_BUFFER_CAPTURE_REPLAY_BIT_EXT) { | |
| alloc_flags = ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS; | |
| const VkOpaqueCaptureDescriptorDataCreateInfoEXT *opaque_info = | |
| vk_find_struct_const(pCreateInfo->pNext, | |
| OPAQUE_CAPTURE_DESCRIPTOR_DATA_CREATE_INFO_EXT); | |
| if (opaque_info) | |
| client_address = *((const uint64_t *)opaque_info->opaqueCaptureDescriptorData); | |
| } | |
| /* If this buffer will be used as a descriptor buffer, make sure we | |
| * allocate it on the correct heap. | |
| */ | |
| if (buffer->vk.usage & (VK_BUFFER_USAGE_2_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT | | |
| VK_BUFFER_USAGE_2_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT | | |
| VK_BUFFER_USAGE_2_DESCRIPTOR_HEAP_BIT_EXT)) { | |
| alloc_flags |= device->physical->uses_efficient_64bit ? ANV_BO_ALLOC_DESCRIPTOR_POOL : ANV_BO_ALLOC_DYNAMIC_VISIBLE_POOL; | |
| } | |
| VkResult result = anv_init_sparse_bindings(device, buffer->vk.size, | |
| &buffer->sparse_data, | |
| alloc_flags, client_address, | |
| &buffer->address); | |
| if (result != VK_SUCCESS) { | |
| vk_buffer_destroy(&device->vk, pAllocator, &buffer->vk); | |
| return result; | |
| } | |
| buffer->vk.device_address = anv_address_physical(buffer->address); | |
| ANV_ADDR_BINDING_REPORT_ADDR_BIND(device, &buffer->vk.base, | |
| buffer->vk.device_address, | |
| buffer->sparse_data.size); | |
| } | |
| ANV_RMV(buffer_create, device, false, buffer); | |
| *pBuffer = anv_buffer_to_handle(buffer); | |
| return VK_SUCCESS; | |
| } | |
| void anv_DestroyBuffer( | |
| VkDevice _device, | |
| VkBuffer _buffer, | |
| const VkAllocationCallbacks* pAllocator) | |
| { | |
| ANV_FROM_HANDLE(anv_device, device, _device); | |
| ANV_FROM_HANDLE(anv_buffer, buffer, _buffer); | |
| if (!buffer) | |
| return; | |
| ANV_RMV(buffer_destroy, device, buffer); | |
| if (anv_buffer_is_sparse(buffer)) { | |
| assert(buffer->address.offset == buffer->sparse_data.address); | |
| anv_free_sparse_bindings(device, &buffer->sparse_data); | |
| ANV_ADDR_BINDING_REPORT_ADDR_UNBIND(device, &buffer->vk.base, | |
| buffer->vk.device_address, | |
| buffer->sparse_data.size); | |
| } else { | |
| ANV_ADDR_BINDING_REPORT_ADDR_UNBIND(device, &buffer->vk.base, | |
| buffer->vk.device_address, | |
| buffer->vk.size); | |
| } | |
| vk_buffer_destroy(&device->vk, pAllocator, &buffer->vk); | |
| } | |
| VkDeviceAddress anv_GetBufferDeviceAddress( | |
| VkDevice device, | |
| const VkBufferDeviceAddressInfo* pInfo) | |
| { | |
| ANV_FROM_HANDLE(anv_buffer, buffer, pInfo->buffer); | |
| assert(!anv_address_is_null(buffer->address)); | |
| return anv_address_physical(buffer->address); | |
| } | |
| uint64_t anv_GetBufferOpaqueCaptureAddress( | |
| VkDevice device, | |
| const VkBufferDeviceAddressInfo* pInfo) | |
| { | |
| ANV_FROM_HANDLE(anv_buffer, buffer, pInfo->buffer); | |
| if (buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) | |
| return anv_address_physical(buffer->address); | |
| else | |
| return 0; | |
| } | |
| VkResult anv_GetBufferOpaqueCaptureDescriptorDataEXT( | |
| VkDevice device, | |
| const VkBufferCaptureDescriptorDataInfoEXT* pInfo, | |
| void* pData) | |
| { | |
| ANV_FROM_HANDLE(anv_buffer, buffer, pInfo->buffer); | |
| uint64_t address = anv_address_physical(buffer->address); | |
| if (buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) | |
| memcpy(pData, &address, sizeof(address)); | |
| else | |
| memset(pData, 0, sizeof(address)); | |
| return VK_SUCCESS; | |
| } | |
| uint64_t anv_GetDeviceMemoryOpaqueCaptureAddress( | |
| VkDevice device, | |
| const VkDeviceMemoryOpaqueCaptureAddressInfo* pInfo) | |
| { | |
| ANV_FROM_HANDLE(anv_device_memory, memory, pInfo->memory); | |
| assert(memory->bo->alloc_flags & ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS); | |
| return intel_48b_address(memory->bo->offset); | |
| } | |
| void | |
| anv_fill_buffer_surface_state(struct anv_device *device, | |
| void *surface_state_ptr, | |
| enum isl_format format, | |
| struct isl_swizzle swizzle, | |
| isl_surf_usage_flags_t usage, | |
| struct anv_address address, | |
| uint64_t range, uint32_t stride) | |
| { | |
| if (address.bo && address.bo->alloc_flags & ANV_BO_ALLOC_PROTECTED) | |
| usage |= ISL_SURF_USAGE_PROTECTED_BIT; | |
| isl_buffer_fill_state(&device->isl_dev, surface_state_ptr, | |
| .address = anv_address_physical(address), | |
| .mocs = isl_mocs(&device->isl_dev, usage, | |
| address.bo && anv_bo_is_external(address.bo)), | |
| .size_B = range, | |
| .format = format, | |
| .swizzle = swizzle, | |
| .stride_B = stride, | |
| .usage = usage); | |
| } |