| /* |
| * Copyright © 2020 Intel Corporation |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice (including the next |
| * paragraph) shall be included in all copies or substantial portions of the |
| * Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
| * IN THE SOFTWARE. |
| */ |
| |
| #include "vk_device.h" |
| |
| #include "vk_alloc.h" |
| #include "vk_common_entrypoints.h" |
| #include "vk_fence.h" |
| #include "vk_instance.h" |
| #include "vk_log.h" |
| #include "vk_physical_device.h" |
| #include "vk_queue.h" |
| #include "vk_semaphore.h" |
| #include "vk_sync.h" |
| #include "vk_sync_timeline.h" |
| #include "vk_util.h" |
| #include "util/compiler.h" |
| #include "util/detect_os.h" |
| #include "util/u_debug.h" |
| #include "util/hash_table.h" |
| #include "util/perf/cpu_trace.h" |
| #include "util/ralloc.h" |
| #include "util/timespec.h" |
| |
| /* Breaks linking cycles since WSI common depends on runtime, |
| * and Meson does not allow runtime to depend on WSI common. */ |
| #include "vulkan/wsi/wsi_common_private.h" |
| |
| static enum vk_device_timeline_mode |
| get_timeline_mode(struct vk_physical_device *physical_device) |
| { |
| if (physical_device->supported_sync_types == NULL) |
| return VK_DEVICE_TIMELINE_MODE_NONE; |
| |
| const struct vk_sync_type *timeline_type = NULL; |
| for (const struct vk_sync_type *const *t = |
| physical_device->supported_sync_types; *t; t++) { |
| if ((*t)->features & VK_SYNC_FEATURE_TIMELINE) { |
| /* We can only have one timeline mode */ |
| assert(timeline_type == NULL); |
| timeline_type = *t; |
| } |
| } |
| |
| if (timeline_type == NULL) |
| return VK_DEVICE_TIMELINE_MODE_NONE; |
| |
| if (vk_sync_type_is_vk_sync_timeline(timeline_type)) |
| return VK_DEVICE_TIMELINE_MODE_EMULATED; |
| |
| if (timeline_type->features & VK_SYNC_FEATURE_WAIT_BEFORE_SIGNAL) |
| return VK_DEVICE_TIMELINE_MODE_NATIVE; |
| |
| /* For assisted mode, we require a few additional things of all sync types |
| * which may be used as semaphores. |
| */ |
| for (const struct vk_sync_type *const *t = |
| physical_device->supported_sync_types; *t; t++) { |
| if ((*t)->features & VK_SYNC_FEATURE_GPU_WAIT) { |
| assert((*t)->features & VK_SYNC_FEATURE_WAIT_PENDING); |
| if ((*t)->features & VK_SYNC_FEATURE_BINARY) |
| assert((*t)->features & VK_SYNC_FEATURE_CPU_RESET); |
| } |
| } |
| |
| return VK_DEVICE_TIMELINE_MODE_ASSISTED; |
| } |
| |
| static void |
| collect_enabled_features(struct vk_device *device, |
| const VkDeviceCreateInfo *pCreateInfo) |
| { |
| if (pCreateInfo->pEnabledFeatures) |
| vk_set_physical_device_features_1_0(&device->enabled_features, pCreateInfo->pEnabledFeatures); |
| vk_set_physical_device_features(&device->enabled_features, pCreateInfo->pNext); |
| } |
| |
| static VkResult |
| vk_device_memory_report_init(struct vk_device *device, |
| const VkDeviceCreateInfo *pCreateInfo) |
| { |
| struct vk_device_memory_report *mem_reports = NULL; |
| uint32_t count = 0; |
| |
| vk_foreach_struct_const(pnext, pCreateInfo->pNext) { |
| if (pnext->sType == VK_STRUCTURE_TYPE_DEVICE_DEVICE_MEMORY_REPORT_CREATE_INFO_EXT) |
| count++; |
| } |
| |
| if (!count) |
| return VK_SUCCESS; |
| |
| mem_reports = vk_alloc(&device->alloc, sizeof(*mem_reports) * count, |
| 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); |
| if (!mem_reports) |
| return VK_ERROR_OUT_OF_HOST_MEMORY; |
| |
| count = 0; |
| vk_foreach_struct_const(pnext, pCreateInfo->pNext) { |
| if (pnext->sType == VK_STRUCTURE_TYPE_DEVICE_DEVICE_MEMORY_REPORT_CREATE_INFO_EXT) { |
| const struct VkDeviceDeviceMemoryReportCreateInfoEXT *report = (void *)pnext; |
| mem_reports[count].callback = report->pfnUserCallback; |
| mem_reports[count].data = report->pUserData; |
| count++; |
| } |
| } |
| |
| device->memory_report_count = count; |
| device->memory_reports = mem_reports; |
| |
| return VK_SUCCESS; |
| } |
| |
| static VkPipelineRobustnessBufferBehaviorEXT |
| vk_device_default_robust_buffer_behavior(const struct vk_device *device) |
| { |
| if (device->enabled_features.robustBufferAccess2) { |
| return VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_ROBUST_BUFFER_ACCESS_2_EXT; |
| } else if (device->enabled_features.robustBufferAccess) { |
| return VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_ROBUST_BUFFER_ACCESS_EXT; |
| } else { |
| return VK_PIPELINE_ROBUSTNESS_BUFFER_BEHAVIOR_DISABLED_EXT; |
| } |
| } |
| |
| static VkPipelineRobustnessImageBehaviorEXT |
| vk_device_default_robust_image_behavior(const struct vk_device *device) |
| { |
| if (device->enabled_features.robustImageAccess2) { |
| return VK_PIPELINE_ROBUSTNESS_IMAGE_BEHAVIOR_ROBUST_IMAGE_ACCESS_2_EXT; |
| } else if (device->enabled_features.robustImageAccess) { |
| return VK_PIPELINE_ROBUSTNESS_IMAGE_BEHAVIOR_ROBUST_IMAGE_ACCESS_EXT; |
| } else { |
| return VK_PIPELINE_ROBUSTNESS_IMAGE_BEHAVIOR_DISABLED_EXT; |
| } |
| } |
| |
| VkResult |
| vk_device_init(struct vk_device *device, |
| struct vk_physical_device *physical_device, |
| const struct vk_device_dispatch_table *dispatch_table, |
| const VkDeviceCreateInfo *pCreateInfo, |
| const VkAllocationCallbacks *alloc) |
| { |
| memset(device, 0, sizeof(*device)); |
| vk_object_base_init(device, &device->base, VK_OBJECT_TYPE_DEVICE); |
| if (alloc != NULL) |
| device->alloc = *alloc; |
| else |
| device->alloc = physical_device->instance->alloc; |
| |
| device->physical = physical_device; |
| |
| if (dispatch_table) { |
| device->dispatch_table = *dispatch_table; |
| |
| /* Add common entrypoints without overwriting driver-provided ones. */ |
| vk_device_dispatch_table_from_entrypoints( |
| &device->dispatch_table, &vk_common_device_entrypoints, false); |
| } |
| |
| for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) { |
| int idx; |
| for (idx = 0; idx < VK_DEVICE_EXTENSION_COUNT; idx++) { |
| if (strcmp(pCreateInfo->ppEnabledExtensionNames[i], |
| vk_device_extensions[idx].extensionName) == 0) |
| break; |
| } |
| |
| if (idx >= VK_DEVICE_EXTENSION_COUNT) |
| return vk_errorf(physical_device, VK_ERROR_EXTENSION_NOT_PRESENT, |
| "%s not supported", |
| pCreateInfo->ppEnabledExtensionNames[i]); |
| |
| if (!physical_device->supported_extensions.extensions[idx]) |
| return vk_errorf(physical_device, VK_ERROR_EXTENSION_NOT_PRESENT, |
| "%s not supported", |
| pCreateInfo->ppEnabledExtensionNames[i]); |
| |
| #ifdef ANDROID_STRICT |
| if (!vk_android_allowed_device_extensions.extensions[idx]) |
| return vk_errorf(physical_device, VK_ERROR_EXTENSION_NOT_PRESENT, |
| "%s not supported", |
| pCreateInfo->ppEnabledExtensionNames[i]); |
| #endif |
| |
| device->enabled_extensions.extensions[idx] = true; |
| } |
| |
| VkResult result = |
| vk_physical_device_check_device_features(physical_device, |
| pCreateInfo); |
| if (result != VK_SUCCESS) |
| return result; |
| |
| collect_enabled_features(device, pCreateInfo); |
| |
| p_atomic_set(&device->private_data_next_index, 0); |
| |
| list_inithead(&device->queues); |
| |
| device->mem_cache = NULL; |
| |
| device->timeline_mode = get_timeline_mode(physical_device); |
| |
| switch (device->timeline_mode) { |
| case VK_DEVICE_TIMELINE_MODE_NONE: |
| case VK_DEVICE_TIMELINE_MODE_NATIVE: |
| device->submit_mode = VK_QUEUE_SUBMIT_MODE_IMMEDIATE; |
| break; |
| |
| case VK_DEVICE_TIMELINE_MODE_EMULATED: |
| device->submit_mode = VK_QUEUE_SUBMIT_MODE_DEFERRED; |
| break; |
| |
| case VK_DEVICE_TIMELINE_MODE_ASSISTED: |
| if (os_get_option("MESA_VK_ENABLE_SUBMIT_THREAD")) { |
| if (debug_get_bool_option("MESA_VK_ENABLE_SUBMIT_THREAD", false)) { |
| device->submit_mode = VK_QUEUE_SUBMIT_MODE_THREADED; |
| } else { |
| device->submit_mode = VK_QUEUE_SUBMIT_MODE_IMMEDIATE; |
| } |
| } else { |
| device->submit_mode = VK_QUEUE_SUBMIT_MODE_THREADED_ON_DEMAND; |
| } |
| break; |
| |
| default: |
| UNREACHABLE("Invalid timeline mode"); |
| } |
| |
| #ifdef VK_USE_PLATFORM_ANDROID_KHR |
| mtx_init(&device->swapchain_private_mtx, mtx_plain); |
| device->swapchain_private = NULL; |
| #endif /* VK_USE_PLATFORM_ANDROID_KHR */ |
| |
| simple_mtx_init(&device->trace_mtx, mtx_plain); |
| |
| vk_foreach_struct_const (ext, pCreateInfo->pNext) { |
| switch (ext->sType) { |
| case VK_STRUCTURE_TYPE_DEVICE_PIPELINE_BINARY_INTERNAL_CACHE_CONTROL_KHR: { |
| const VkDevicePipelineBinaryInternalCacheControlKHR *cache_control = (const void *)ext; |
| if (cache_control->disableInternalCache) |
| device->disable_internal_cache = true; |
| break; |
| } |
| default: |
| break; |
| } |
| } |
| |
| if (device->enabled_extensions.KHR_calibrated_timestamps || |
| device->enabled_extensions.EXT_calibrated_timestamps) { |
| /* sorted by preference */ |
| const VkTimeDomainKHR calibrate_domains[] = { |
| VK_TIME_DOMAIN_CLOCK_MONOTONIC_RAW_KHR, |
| VK_TIME_DOMAIN_CLOCK_MONOTONIC_KHR, |
| VK_TIME_DOMAIN_QUERY_PERFORMANCE_COUNTER_KHR, |
| }; |
| for (uint32_t i = 0; i < ARRAY_SIZE(calibrate_domains); i++) { |
| const VkTimeDomainKHR domain = calibrate_domains[i]; |
| uint64_t ts; |
| if (vk_device_get_timestamp(NULL, domain, &ts) == VK_SUCCESS) { |
| device->calibrate_time_domain = domain; |
| break; |
| } |
| } |
| |
| assert(device->calibrate_time_domain != VK_TIME_DOMAIN_DEVICE_KHR); |
| device->device_time_domain_period = |
| (uint64_t)ceilf(device->physical->properties.timestampPeriod); |
| } |
| |
| result = vk_device_memory_report_init(device, pCreateInfo); |
| if (result != VK_SUCCESS) |
| return result; |
| |
| device->robustness_state = (struct vk_pipeline_robustness_state) { |
| .uniform_buffers = vk_device_default_robust_buffer_behavior(device), |
| .storage_buffers = vk_device_default_robust_buffer_behavior(device), |
| .vertex_inputs = vk_device_default_robust_buffer_behavior(device), |
| .images = vk_device_default_robust_image_behavior(device), |
| .null_uniform_buffer_descriptor = device->enabled_features.nullDescriptor, |
| .null_storage_buffer_descriptor = device->enabled_features.nullDescriptor, |
| }; |
| |
| device->disable_lto = false; |
| |
| return VK_SUCCESS; |
| } |
| |
| static void |
| vk_device_memory_report_finish(struct vk_device *device) |
| { |
| vk_free(&device->alloc, device->memory_reports); |
| } |
| |
| void |
| vk_device_finish(struct vk_device *device) |
| { |
| /* Drivers should tear down their own queues */ |
| assert(list_is_empty(&device->queues)); |
| |
| if (device->sync) |
| device->sync->finalize(device->sync); |
| |
| vk_device_memory_report_finish(device); |
| vk_memory_trace_finish(device); |
| |
| #ifdef VK_USE_PLATFORM_ANDROID_KHR |
| if (device->swapchain_private) { |
| hash_table_foreach(device->swapchain_private, entry) |
| util_sparse_array_finish(entry->data); |
| ralloc_free(device->swapchain_private); |
| } |
| #endif /* VK_USE_PLATFORM_ANDROID_KHR */ |
| |
| simple_mtx_destroy(&device->trace_mtx); |
| |
| vk_object_base_finish(&device->base); |
| } |
| |
| void |
| vk_device_enable_threaded_submit(struct vk_device *device) |
| { |
| /* This must be called before any queues are created */ |
| assert(list_is_empty(&device->queues)); |
| |
| /* In order to use threaded submit, we need every sync type that can be |
| * used as a wait fence for vkQueueSubmit() to support WAIT_PENDING. |
| * It's required for cross-thread/process submit re-ordering. |
| */ |
| for (const struct vk_sync_type *const *t = |
| device->physical->supported_sync_types; *t; t++) { |
| if ((*t)->features & VK_SYNC_FEATURE_GPU_WAIT) |
| assert((*t)->features & VK_SYNC_FEATURE_WAIT_PENDING); |
| } |
| |
| /* Any binary vk_sync types which will be used as permanent semaphore |
| * payloads also need to support vk_sync_type::move, but that's a lot |
| * harder to assert since it only applies to permanent semaphore payloads. |
| */ |
| |
| if (device->submit_mode != VK_QUEUE_SUBMIT_MODE_THREADED) |
| device->submit_mode = VK_QUEUE_SUBMIT_MODE_THREADED_ON_DEMAND; |
| } |
| |
| VkResult |
| vk_device_flush(struct vk_device *device) |
| { |
| if (device->submit_mode != VK_QUEUE_SUBMIT_MODE_DEFERRED) |
| return VK_SUCCESS; |
| |
| bool progress; |
| do { |
| progress = false; |
| |
| vk_foreach_queue(queue, device) { |
| uint32_t queue_submit_count; |
| VkResult result = vk_queue_flush(queue, &queue_submit_count); |
| if (unlikely(result != VK_SUCCESS)) |
| return result; |
| |
| if (queue_submit_count) |
| progress = true; |
| } |
| } while (progress); |
| |
| return VK_SUCCESS; |
| } |
| |
| static const char * |
| timeline_mode_str(struct vk_device *device) |
| { |
| switch (device->timeline_mode) { |
| #define CASE(X) case VK_DEVICE_TIMELINE_MODE_##X: return #X; |
| CASE(NONE) |
| CASE(EMULATED) |
| CASE(ASSISTED) |
| CASE(NATIVE) |
| #undef CASE |
| default: return "UNKNOWN"; |
| } |
| } |
| |
| void |
| _vk_device_report_lost(struct vk_device *device) |
| { |
| assert(p_atomic_read(&device->_lost.lost) > 0); |
| |
| device->_lost.reported = true; |
| |
| vk_foreach_queue(queue, device) { |
| if (queue->_lost.lost) { |
| __vk_errorf(queue, VK_ERROR_DEVICE_LOST, |
| queue->_lost.error_file, queue->_lost.error_line, |
| "%s", queue->_lost.error_msg); |
| } |
| } |
| |
| vk_logd(VK_LOG_OBJS(device), "Timeline mode is %s.", |
| timeline_mode_str(device)); |
| } |
| |
| VkResult |
| _vk_device_set_lost(struct vk_device *device, |
| const char *file, int line, |
| const char *msg, ...) |
| { |
| /* This flushes out any per-queue device lost messages */ |
| if (vk_device_is_lost(device)) |
| return VK_ERROR_DEVICE_LOST; |
| |
| p_atomic_inc(&device->_lost.lost); |
| device->_lost.reported = true; |
| |
| va_list ap; |
| va_start(ap, msg); |
| __vk_errorv(device, VK_ERROR_DEVICE_LOST, file, line, msg, ap); |
| va_end(ap); |
| |
| vk_logd(VK_LOG_OBJS(device), "Timeline mode is %s.", |
| timeline_mode_str(device)); |
| |
| if (debug_get_bool_option("MESA_VK_ABORT_ON_DEVICE_LOSS", false)) |
| abort(); |
| |
| return VK_ERROR_DEVICE_LOST; |
| } |
| |
| PFN_vkVoidFunction |
| vk_device_get_proc_addr(const struct vk_device *device, |
| const char *name) |
| { |
| if (device == NULL || name == NULL) |
| return NULL; |
| |
| struct vk_instance *instance = device->physical->instance; |
| return vk_device_dispatch_table_get_if_supported(&device->dispatch_table, |
| name, |
| instance->app_info.api_version, |
| &instance->enabled_extensions, |
| &device->enabled_extensions); |
| } |
| |
| VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL |
| vk_common_GetDeviceProcAddr(VkDevice _device, |
| const char *pName) |
| { |
| VK_FROM_HANDLE(vk_device, device, _device); |
| return vk_device_get_proc_addr(device, pName); |
| } |
| |
| VKAPI_ATTR void VKAPI_CALL |
| vk_common_GetDeviceQueue(VkDevice _device, |
| uint32_t queueFamilyIndex, |
| uint32_t queueIndex, |
| VkQueue *pQueue) |
| { |
| VK_FROM_HANDLE(vk_device, device, _device); |
| |
| const VkDeviceQueueInfo2 info = { |
| .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_INFO_2, |
| .pNext = NULL, |
| /* flags = 0 because (Vulkan spec 1.2.170 - vkGetDeviceQueue): |
| * |
| * "vkGetDeviceQueue must only be used to get queues that were |
| * created with the flags parameter of VkDeviceQueueCreateInfo set |
| * to zero. To get queues that were created with a non-zero flags |
| * parameter use vkGetDeviceQueue2." |
| */ |
| .flags = 0, |
| .queueFamilyIndex = queueFamilyIndex, |
| .queueIndex = queueIndex, |
| }; |
| |
| device->dispatch_table.GetDeviceQueue2(_device, &info, pQueue); |
| } |
| |
| VKAPI_ATTR void VKAPI_CALL |
| vk_common_GetDeviceQueue2(VkDevice _device, |
| const VkDeviceQueueInfo2 *pQueueInfo, |
| VkQueue *pQueue) |
| { |
| VK_FROM_HANDLE(vk_device, device, _device); |
| |
| /* From the Vulkan 1.1.70 spec: |
| * |
| * "The queue returned by vkGetDeviceQueue2 must have the same flags |
| * value from this structure as that used at device creation time in a |
| * VkDeviceQueueCreateInfo instance. If no matching flags were specified |
| * at device creation time then pQueue will return VK_NULL_HANDLE." |
| */ |
| struct vk_queue *queue = NULL; |
| vk_foreach_queue(iter, device) { |
| if (iter->queue_family_index == pQueueInfo->queueFamilyIndex && |
| iter->index_in_family == pQueueInfo->queueIndex && |
| iter->flags == pQueueInfo->flags) { |
| queue = iter; |
| break; |
| } |
| } |
| |
| *pQueue = queue ? vk_queue_to_handle(queue) : VK_NULL_HANDLE; |
| } |
| |
| VKAPI_ATTR VkResult VKAPI_CALL |
| vk_common_MapMemory(VkDevice _device, |
| VkDeviceMemory memory, |
| VkDeviceSize offset, |
| VkDeviceSize size, |
| VkMemoryMapFlags flags, |
| void **ppData) |
| { |
| VK_FROM_HANDLE(vk_device, device, _device); |
| |
| const VkMemoryMapInfoKHR info = { |
| .sType = VK_STRUCTURE_TYPE_MEMORY_MAP_INFO_KHR, |
| .flags = flags, |
| .memory = memory, |
| .offset = offset, |
| .size = size, |
| }; |
| |
| return device->dispatch_table.MapMemory2KHR(_device, &info, ppData); |
| } |
| |
| VKAPI_ATTR void VKAPI_CALL |
| vk_common_UnmapMemory(VkDevice _device, |
| VkDeviceMemory memory) |
| { |
| VK_FROM_HANDLE(vk_device, device, _device); |
| ASSERTED VkResult result; |
| |
| const VkMemoryUnmapInfoKHR info = { |
| .sType = VK_STRUCTURE_TYPE_MEMORY_UNMAP_INFO_KHR, |
| .memory = memory, |
| }; |
| |
| result = device->dispatch_table.UnmapMemory2KHR(_device, &info); |
| assert(result == VK_SUCCESS); |
| } |
| |
| VKAPI_ATTR void VKAPI_CALL |
| vk_common_GetDeviceGroupPeerMemoryFeatures( |
| VkDevice device, |
| uint32_t heapIndex, |
| uint32_t localDeviceIndex, |
| uint32_t remoteDeviceIndex, |
| VkPeerMemoryFeatureFlags *pPeerMemoryFeatures) |
| { |
| assert(localDeviceIndex == 0 && remoteDeviceIndex == 0); |
| *pPeerMemoryFeatures = VK_PEER_MEMORY_FEATURE_COPY_SRC_BIT | |
| VK_PEER_MEMORY_FEATURE_COPY_DST_BIT | |
| VK_PEER_MEMORY_FEATURE_GENERIC_SRC_BIT | |
| VK_PEER_MEMORY_FEATURE_GENERIC_DST_BIT; |
| } |
| |
| VKAPI_ATTR void VKAPI_CALL |
| vk_common_GetImageMemoryRequirements(VkDevice _device, |
| VkImage image, |
| VkMemoryRequirements *pMemoryRequirements) |
| { |
| VK_FROM_HANDLE(vk_device, device, _device); |
| |
| VkImageMemoryRequirementsInfo2 info = { |
| .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2, |
| .image = image, |
| }; |
| VkMemoryRequirements2 reqs = { |
| .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, |
| }; |
| device->dispatch_table.GetImageMemoryRequirements2(_device, &info, &reqs); |
| |
| *pMemoryRequirements = reqs.memoryRequirements; |
| } |
| |
| VKAPI_ATTR VkResult VKAPI_CALL |
| vk_common_BindImageMemory(VkDevice _device, |
| VkImage image, |
| VkDeviceMemory memory, |
| VkDeviceSize memoryOffset) |
| { |
| VK_FROM_HANDLE(vk_device, device, _device); |
| |
| VkBindImageMemoryInfo bind = { |
| .sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO, |
| .image = image, |
| .memory = memory, |
| .memoryOffset = memoryOffset, |
| }; |
| |
| return device->dispatch_table.BindImageMemory2(_device, 1, &bind); |
| } |
| |
| VKAPI_ATTR void VKAPI_CALL |
| vk_common_GetImageSparseMemoryRequirements(VkDevice _device, |
| VkImage image, |
| uint32_t *pSparseMemoryRequirementCount, |
| VkSparseImageMemoryRequirements *pSparseMemoryRequirements) |
| { |
| VK_FROM_HANDLE(vk_device, device, _device); |
| |
| VkImageSparseMemoryRequirementsInfo2 info = { |
| .sType = VK_STRUCTURE_TYPE_IMAGE_SPARSE_MEMORY_REQUIREMENTS_INFO_2, |
| .image = image, |
| }; |
| |
| if (!pSparseMemoryRequirements) { |
| device->dispatch_table.GetImageSparseMemoryRequirements2(_device, |
| &info, |
| pSparseMemoryRequirementCount, |
| NULL); |
| return; |
| } |
| |
| STACK_ARRAY(VkSparseImageMemoryRequirements2, mem_reqs2, *pSparseMemoryRequirementCount); |
| |
| for (unsigned i = 0; i < *pSparseMemoryRequirementCount; ++i) { |
| mem_reqs2[i].sType = VK_STRUCTURE_TYPE_SPARSE_IMAGE_MEMORY_REQUIREMENTS_2; |
| mem_reqs2[i].pNext = NULL; |
| } |
| |
| device->dispatch_table.GetImageSparseMemoryRequirements2(_device, |
| &info, |
| pSparseMemoryRequirementCount, |
| mem_reqs2); |
| |
| for (unsigned i = 0; i < *pSparseMemoryRequirementCount; ++i) |
| pSparseMemoryRequirements[i] = mem_reqs2[i].memoryRequirements; |
| |
| STACK_ARRAY_FINISH(mem_reqs2); |
| } |
| |
| VKAPI_ATTR VkResult VKAPI_CALL |
| vk_common_DeviceWaitIdle(VkDevice _device) |
| { |
| MESA_TRACE_FUNC(); |
| |
| VK_FROM_HANDLE(vk_device, device, _device); |
| const struct vk_device_dispatch_table *disp = &device->dispatch_table; |
| |
| vk_foreach_queue(queue, device) { |
| VkResult result = disp->QueueWaitIdle(vk_queue_to_handle(queue)); |
| if (result != VK_SUCCESS) |
| return result; |
| } |
| |
| return VK_SUCCESS; |
| } |
| |
| VKAPI_ATTR VkResult VKAPI_CALL |
| vk_common_RegisterCustomBorderColorEXT(VkDevice device, |
| const VkSamplerCustomBorderColorCreateInfoEXT* pBorderColor, |
| VkBool32 requestIndex, |
| uint32_t *pIndex) |
| { |
| if (requestIndex) |
| *pIndex = 0; |
| |
| return VK_SUCCESS; |
| } |
| |
| VKAPI_ATTR void VKAPI_CALL |
| vk_common_UnregisterCustomBorderColorEXT(VkDevice device, |
| uint32_t index) |
| { } |
| |
| VkResult |
| vk_device_copy_semaphore_payloads(struct vk_device *device, |
| uint32_t wait_semaphore_count, |
| const VkSemaphoreSubmitInfo *wait_semaphores, |
| uint32_t signal_semaphore_count, |
| const VkSemaphoreSubmitInfo *signal_semaphores, |
| uint32_t fence_count, |
| const VkFence *fences) |
| { |
| if (device->copy_sync_payloads == NULL) |
| return VK_ERROR_FEATURE_NOT_PRESENT; |
| |
| STACK_ARRAY(struct vk_sync_wait, waits, wait_semaphore_count); |
| STACK_ARRAY(struct vk_sync_timeline_point *, wait_points, |
| wait_semaphore_count); |
| STACK_ARRAY(struct vk_sync *, resets, wait_semaphore_count); |
| STACK_ARRAY(struct vk_sync_signal, signals, |
| signal_semaphore_count + fence_count); |
| STACK_ARRAY(struct vk_sync_timeline_point *, signal_points, |
| signal_semaphore_count + fence_count); |
| uint32_t wait_count = 0, reset_count = 0, signal_count = 0; |
| VkResult result = VK_SUCCESS; |
| |
| for (uint32_t i = 0; i < wait_semaphore_count; i++) { |
| VK_FROM_HANDLE(vk_semaphore, semaphore, wait_semaphores[i].semaphore); |
| |
| struct vk_sync_wait wait = { |
| .sync = vk_semaphore_get_active_sync(semaphore), |
| .stage_mask = wait_semaphores[i].stageMask, |
| .wait_value = semaphore->type == VK_SEMAPHORE_TYPE_TIMELINE ? |
| wait_semaphores[i].value : 0, |
| }; |
| struct vk_sync_timeline_point *wait_point = NULL; |
| VkResult result = vk_sync_wait_unwrap(device, &wait, &wait_point); |
| if (unlikely(result != VK_SUCCESS)) |
| goto fail; |
| |
| if (wait.sync == NULL) |
| continue; |
| |
| wait_points[wait_count] = wait_point; |
| waits[wait_count] = wait; |
| wait_count++; |
| } |
| |
| for (uint32_t i = 0; i < signal_semaphore_count; i++) { |
| VK_FROM_HANDLE(vk_semaphore, semaphore, signal_semaphores[i].semaphore); |
| |
| if (semaphore->type == VK_SEMAPHORE_TYPE_TIMELINE && |
| signal_semaphores[i].value == 0) { |
| result = vk_errorf(device, VK_ERROR_UNKNOWN, |
| "Tried to signal a timeline with value 0"); |
| goto fail; |
| } |
| |
| struct vk_sync_signal signal = { |
| .sync = vk_semaphore_get_active_sync(semaphore), |
| .stage_mask = signal_semaphores[i].stageMask, |
| .signal_value = semaphore->type == VK_SEMAPHORE_TYPE_TIMELINE ? |
| signal_semaphores[i].value : 0, |
| }; |
| struct vk_sync_timeline_point *signal_point = NULL; |
| VkResult result = vk_sync_signal_unwrap(device, &signal, &signal_point); |
| if (unlikely(result != VK_SUCCESS)) |
| goto fail; |
| |
| signal_points[signal_count] = signal_point; |
| signals[signal_count] = signal; |
| signal_count++; |
| } |
| |
| for (uint32_t i = 0; i < fence_count; i++) { |
| VK_FROM_HANDLE(vk_fence, fence, fences[i]); |
| |
| struct vk_sync_signal signal = { |
| .sync = vk_fence_get_active_sync(fence), |
| .stage_mask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, |
| }; |
| struct vk_sync_timeline_point *signal_point = NULL; |
| VkResult result = vk_sync_signal_unwrap(device, &signal, &signal_point); |
| if (unlikely(result != VK_SUCCESS)) |
| goto fail; |
| |
| /* Timeline fences aren't a thing */ |
| assert(signal_point == NULL); |
| |
| signal_points[signal_count] = signal_point; |
| signals[signal_count] = signal; |
| signal_count++; |
| } |
| |
| if (wait_count == 0) { |
| /* Nothing to wait on. Just signal everything */ |
| result = vk_sync_signal_many(device, signal_count, signals); |
| if (result != VK_SUCCESS) |
| goto fail; |
| } else if (signal_count > 0) { |
| /* Wait for time points to materialize */ |
| result = vk_sync_wait_many(device, wait_count, waits, |
| VK_SYNC_WAIT_PENDING, UINT64_MAX); |
| if (result != VK_SUCCESS) |
| goto fail; |
| |
| /* Now do the copy */ |
| result = device->copy_sync_payloads(device, wait_count, waits, |
| signal_count, signals); |
| if (result != VK_SUCCESS) |
| goto fail; |
| } |
| |
| /* Reset any syncs which were waited on but not signaled */ |
| for (uint32_t i = 0; i < wait_count; i++) { |
| bool was_signaled = false; |
| for (uint32_t j = 0; j < signal_count; j++) { |
| if (signals[j].sync == waits[i].sync) { |
| was_signaled = true; |
| break; |
| } |
| } |
| |
| if (!was_signaled) |
| resets[reset_count++] = waits[i].sync; |
| } |
| |
| if (reset_count > 0) { |
| result = vk_sync_reset_many(device, reset_count, resets); |
| if (result != VK_SUCCESS) |
| goto fail; |
| } |
| |
| /* Reset any temporary semaphores we waited on */ |
| for (uint32_t i = 0; i < wait_semaphore_count; i++) { |
| VK_FROM_HANDLE(vk_semaphore, semaphore, wait_semaphores[i].semaphore); |
| |
| vk_semaphore_reset_temporary(device, semaphore); |
| } |
| |
| /* Install time points */ |
| for (uint32_t i = 0; i < signal_count; i++) { |
| if (signal_points[i] == NULL) |
| continue; |
| |
| vk_sync_timeline_point_install(device, signal_points[i]); |
| |
| /* Installing the point consumes our reference */ |
| signal_points[i] = NULL; |
| } |
| |
| fail: |
| |
| for (uint32_t i = 0; i < signal_count; i++) { |
| if (signal_points[i] != NULL) |
| vk_sync_timeline_point_unref(device, signal_points[i]); |
| } |
| |
| STACK_ARRAY_FINISH(waits); |
| STACK_ARRAY_FINISH(wait_points); |
| STACK_ARRAY_FINISH(resets); |
| STACK_ARRAY_FINISH(signals); |
| STACK_ARRAY_FINISH(signal_points); |
| |
| return result; |
| } |
| |
| VkResult |
| vk_device_get_timestamp(struct vk_device *device, VkTimeDomainKHR domain, |
| uint64_t *timestamp) |
| { |
| if (domain == VK_TIME_DOMAIN_DEVICE_KHR) { |
| assert(device && device->get_timestamp); |
| return device->get_timestamp(device, timestamp); |
| } |
| |
| /* device is not used for host time domains */ |
| #if DETECT_OS_WINDOWS |
| if (domain == VK_TIME_DOMAIN_QUERY_PERFORMANCE_COUNTER_KHR) { |
| LARGE_INTEGER ts; |
| if (QueryPerformanceCounter(&ts)) { |
| *timestamp = ts.QuadPart; |
| return VK_SUCCESS; |
| } |
| } |
| #else /* !DETECT_OS_WINDOWS */ |
| clockid_t clockid; |
| struct timespec ts; |
| |
| switch (domain) { |
| case VK_TIME_DOMAIN_CLOCK_MONOTONIC_KHR: |
| clockid = CLOCK_MONOTONIC; |
| break; |
| case VK_TIME_DOMAIN_CLOCK_MONOTONIC_RAW_KHR: |
| /* The "RAW" clocks on Linux are called "FAST" on FreeBSD */ |
| #if defined(CLOCK_MONOTONIC_RAW) |
| clockid = CLOCK_MONOTONIC_RAW; |
| break; |
| #elif defined(CLOCK_MONOTONIC_FAST) |
| clockid = CLOCK_MONOTONIC_FAST; |
| break; |
| #else |
| FALLTHROUGH; |
| #endif |
| default: |
| goto fail; |
| } |
| |
| if (clock_gettime(clockid, &ts) < 0) |
| goto fail; |
| |
| *timestamp = (uint64_t)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; |
| return VK_SUCCESS; |
| |
| fail: |
| #endif /* DETECT_OS_WINDOWS */ |
| return VK_ERROR_FEATURE_NOT_PRESENT; |
| } |
| |
| VKAPI_ATTR VkResult VKAPI_CALL |
| vk_common_GetCalibratedTimestampsKHR( |
| VkDevice _device, uint32_t timestampCount, |
| const VkCalibratedTimestampInfoKHR *pTimestampInfos, uint64_t *pTimestamps, |
| uint64_t *pMaxDeviation) |
| { |
| VK_FROM_HANDLE(vk_device, device, _device); |
| uint64_t begin, end; |
| VkResult result; |
| |
| /* collect timestamps as tight as possible */ |
| result = |
| vk_device_get_timestamp(device, device->calibrate_time_domain, &begin); |
| for (uint32_t i = 0; i < timestampCount; i++) { |
| VkTimeDomainKHR domain = pTimestampInfos[i].timeDomain; |
| if (domain == VK_TIME_DOMAIN_PRESENT_STAGE_LOCAL_EXT) { |
| const VkSwapchainCalibratedTimestampInfoEXT *swap = |
| vk_find_struct_const(pTimestampInfos[i].pNext, SWAPCHAIN_CALIBRATED_TIMESTAMP_INFO_EXT); |
| domain = wsi_common_get_time_domain(swap->swapchain, swap->presentStage, swap->timeDomainId); |
| } |
| if (domain == device->calibrate_time_domain) |
| pTimestamps[i] = begin; |
| else |
| result |= vk_device_get_timestamp(device, domain, &pTimestamps[i]); |
| } |
| result |= |
| vk_device_get_timestamp(device, device->calibrate_time_domain, &end); |
| |
| if (result != VK_SUCCESS) |
| return VK_ERROR_OUT_OF_HOST_MEMORY; |
| |
| uint64_t max_clock_period = 0; |
| for (uint32_t i = 0; i < timestampCount; i++) { |
| VkTimeDomainKHR domain = pTimestampInfos[i].timeDomain; |
| |
| bool domain_is_device_derived = domain == VK_TIME_DOMAIN_DEVICE_KHR; |
| |
| if (domain == VK_TIME_DOMAIN_PRESENT_STAGE_LOCAL_EXT) { |
| /* Need to rescale device timestamps to nanoseconds. */ |
| const VkSwapchainCalibratedTimestampInfoEXT *swap = |
| vk_find_struct_const(pTimestampInfos[i].pNext, SWAPCHAIN_CALIBRATED_TIMESTAMP_INFO_EXT); |
| if (wsi_common_get_time_domain(swap->swapchain, swap->presentStage, swap->timeDomainId) == |
| VK_TIME_DOMAIN_DEVICE_KHR) { |
| pTimestamps[i] = (uint64_t)((double)pTimestamps[i] * (double)device->physical->properties.timestampPeriod); |
| } |
| |
| /* Timestamps in QueueOperationsEnd are always derived from a device timestamp, |
| * even if the reported time domain is not. */ |
| if (swap->presentStage == VK_PRESENT_STAGE_QUEUE_OPERATIONS_END_BIT_EXT) |
| domain_is_device_derived = true; |
| } |
| |
| const uint64_t period = domain_is_device_derived |
| ? device->device_time_domain_period |
| : domain != device->calibrate_time_domain ? 1 : 0; |
| max_clock_period = MAX2(max_clock_period, period); |
| } |
| |
| *pMaxDeviation = vk_time_max_deviation(begin, end, max_clock_period); |
| |
| return VK_SUCCESS; |
| } |
