| /* | |
| * Copyright © 2021 Intel Corporation | |
| * | |
| * Permission is hereby granted, free of charge, to any person obtaining a | |
| * copy of this software and associated documentation files (the "Software"), | |
| * to deal in the Software without restriction, including without limitation | |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| * and/or sell copies of the Software, and to permit persons to whom the | |
| * Software is furnished to do so, subject to the following conditions: | |
| * | |
| * The above copyright notice and this permission notice (including the next | |
| * paragraph) shall be included in all copies or substantial portions of the | |
| * Software. | |
| * | |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
| * IN THE SOFTWARE. | |
| */ | |
| #include "vk_sync.h" | |
| #include <assert.h> | |
| #include <string.h> | |
| #include "util/u_debug.h" | |
| #include "util/macros.h" | |
| #include "util/os_time.h" | |
| #include "vk_alloc.h" | |
| #include "vk_device.h" | |
| #include "vk_log.h" | |
| #include "vk_physical_device.h" | |
| #include "vk_sync_binary.h" | |
| #include "vk_sync_dummy.h" | |
| #include "vk_sync_timeline.h" | |
| static void | |
| vk_sync_type_validate(const struct vk_sync_type *type) | |
| { | |
| assert(type->init); | |
| assert(type->finish); | |
| assert(type->features & (VK_SYNC_FEATURE_BINARY | | |
| VK_SYNC_FEATURE_TIMELINE)); | |
| if (type->features & VK_SYNC_FEATURE_TIMELINE) { | |
| assert(type->features & VK_SYNC_FEATURE_GPU_WAIT); | |
| assert(type->features & VK_SYNC_FEATURE_CPU_WAIT); | |
| assert(type->features & VK_SYNC_FEATURE_CPU_SIGNAL); | |
| assert(type->features & (VK_SYNC_FEATURE_WAIT_BEFORE_SIGNAL | | |
| VK_SYNC_FEATURE_WAIT_PENDING)); | |
| assert(type->signal); | |
| assert(type->get_value); | |
| } | |
| if (!(type->features & VK_SYNC_FEATURE_BINARY)) { | |
| assert(!(type->features & (VK_SYNC_FEATURE_GPU_MULTI_WAIT | | |
| VK_SYNC_FEATURE_CPU_RESET))); | |
| assert(!type->import_sync_file); | |
| assert(!type->export_sync_file); | |
| } | |
| if (type->features & VK_SYNC_FEATURE_CPU_WAIT) { | |
| assert(type->wait || type->wait_many); | |
| } else { | |
| assert(!(type->features & (VK_SYNC_FEATURE_WAIT_ANY | | |
| VK_SYNC_FEATURE_WAIT_PENDING))); | |
| } | |
| if (type->features & VK_SYNC_FEATURE_GPU_MULTI_WAIT) | |
| assert(type->features & VK_SYNC_FEATURE_GPU_WAIT); | |
| if (type->features & VK_SYNC_FEATURE_CPU_RESET) | |
| assert(type->reset); | |
| if (type->features & VK_SYNC_FEATURE_CPU_SIGNAL) | |
| assert(type->signal); | |
| } | |
| VkResult | |
| vk_sync_init(struct vk_device *device, | |
| struct vk_sync *sync, | |
| const struct vk_sync_type *type, | |
| enum vk_sync_flags flags, | |
| uint64_t initial_value) | |
| { | |
| vk_sync_type_validate(type); | |
| if (flags & VK_SYNC_IS_TIMELINE) | |
| assert(type->features & VK_SYNC_FEATURE_TIMELINE); | |
| else | |
| assert(type->features & VK_SYNC_FEATURE_BINARY); | |
| assert(type->size >= sizeof(*sync)); | |
| memset(sync, 0, type->size); | |
| sync->type = type; | |
| sync->flags = flags; | |
| return type->init(device, sync, initial_value); | |
| } | |
| void | |
| vk_sync_finish(struct vk_device *device, | |
| struct vk_sync *sync) | |
| { | |
| sync->type->finish(device, sync); | |
| } | |
| VkResult | |
| vk_sync_create(struct vk_device *device, | |
| const struct vk_sync_type *type, | |
| enum vk_sync_flags flags, | |
| uint64_t initial_value, | |
| struct vk_sync **sync_out) | |
| { | |
| struct vk_sync *sync; | |
| sync = vk_alloc(&device->alloc, type->size, 8, | |
| VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); | |
| if (sync == NULL) | |
| return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); | |
| VkResult result = vk_sync_init(device, sync, type, flags, initial_value); | |
| if (result != VK_SUCCESS) { | |
| vk_free(&device->alloc, sync); | |
| return result; | |
| } | |
| *sync_out = sync; | |
| return VK_SUCCESS; | |
| } | |
| void | |
| vk_sync_destroy(struct vk_device *device, | |
| struct vk_sync *sync) | |
| { | |
| vk_sync_finish(device, sync); | |
| vk_free(&device->alloc, sync); | |
| } | |
| static void | |
| assert_signal_valid(struct vk_sync *sync, uint64_t value) | |
| { | |
| assert(sync->type->features & VK_SYNC_FEATURE_CPU_SIGNAL); | |
| if (sync->flags & VK_SYNC_IS_TIMELINE) | |
| assert(value > 0); | |
| else | |
| assert(value == 0); | |
| } | |
| VkResult | |
| vk_sync_signal(struct vk_device *device, | |
| struct vk_sync *sync, | |
| uint64_t value) | |
| { | |
| assert_signal_valid(sync, value); | |
| return sync->type->signal(device, sync, value); | |
| } | |
| static bool | |
| can_signal_many(struct vk_device *device, | |
| uint32_t signal_count, | |
| const struct vk_sync_signal *signals) | |
| { | |
| if (signals[0].sync->type->signal_many == NULL) | |
| return false; | |
| /* If we only have one sync type, there's no need to check everything */ | |
| if (device->physical->supported_sync_types[1] == NULL) { | |
| assert(signals[0].sync->type == device->physical->supported_sync_types[0]); | |
| return true; | |
| } | |
| for (uint32_t i = 1; i < signal_count; i++) { | |
| if (signals[i].sync->type != signals[0].sync->type) | |
| return false; | |
| } | |
| return true; | |
| } | |
| VkResult | |
| vk_sync_signal_many(struct vk_device *device, | |
| uint32_t signal_count, | |
| const struct vk_sync_signal *signals) | |
| { | |
| if (signal_count == 0) | |
| return VK_SUCCESS; | |
| for (uint32_t i = 0; i < signal_count; i++) | |
| assert_signal_valid(signals[i].sync, signals[i].signal_value); | |
| if (can_signal_many(device, signal_count, signals)) | |
| return signals[0].sync->type->signal_many(device, signal_count, signals); | |
| for (uint32_t i = 0; i < signal_count; i++) { | |
| struct vk_sync *sync = signals[i].sync; | |
| uint64_t value = signals[i].signal_value; | |
| VkResult result = sync->type->signal(device, sync, value); | |
| if (unlikely(result != VK_SUCCESS)) | |
| return result; | |
| } | |
| return VK_SUCCESS; | |
| } | |
| VkResult | |
| vk_sync_get_value(struct vk_device *device, | |
| struct vk_sync *sync, | |
| uint64_t *value) | |
| { | |
| assert(sync->flags & VK_SYNC_IS_TIMELINE); | |
| return sync->type->get_value(device, sync, value); | |
| } | |
| static void | |
| assert_reset_valid(struct vk_sync *sync) | |
| { | |
| assert(sync->type->features & VK_SYNC_FEATURE_CPU_RESET); | |
| assert(!(sync->flags & VK_SYNC_IS_TIMELINE)); | |
| } | |
| VkResult | |
| vk_sync_reset(struct vk_device *device, | |
| struct vk_sync *sync) | |
| { | |
| assert_reset_valid(sync); | |
| return sync->type->reset(device, sync); | |
| } | |
| static bool | |
| can_reset_many(struct vk_device *device, | |
| uint32_t sync_count, | |
| struct vk_sync *const *syncs) | |
| { | |
| if (syncs[0]->type->reset_many == NULL) | |
| return false; | |
| if (device->physical->supported_sync_types[1] == NULL) { | |
| assert(syncs[0]->type == device->physical->supported_sync_types[0]); | |
| return true; | |
| } | |
| for (uint32_t i = 1; i < sync_count; i++) { | |
| if (syncs[i]->type != syncs[0]->type) | |
| return false; | |
| } | |
| return true; | |
| } | |
| VkResult | |
| vk_sync_reset_many(struct vk_device *device, | |
| uint32_t sync_count, | |
| struct vk_sync *const *syncs) | |
| { | |
| if (sync_count == 0) | |
| return VK_SUCCESS; | |
| for (uint32_t i = 0; i < sync_count; i++) | |
| assert_reset_valid(syncs[i]); | |
| if (can_reset_many(device, sync_count, syncs)) | |
| return syncs[0]->type->reset_many(device, sync_count, syncs); | |
| for (uint32_t i = 0; i < sync_count; i++) { | |
| VkResult result = syncs[i]->type->reset(device, syncs[i]); | |
| if (unlikely(result != VK_SUCCESS)) | |
| return result; | |
| } | |
| return VK_SUCCESS; | |
| } | |
| VkResult vk_sync_move(struct vk_device *device, | |
| struct vk_sync *dst, | |
| struct vk_sync *src) | |
| { | |
| assert(!(dst->flags & VK_SYNC_IS_TIMELINE)); | |
| assert(!(src->flags & VK_SYNC_IS_TIMELINE)); | |
| assert(dst->type == src->type); | |
| return src->type->move(device, dst, src); | |
| } | |
| static void | |
| assert_valid_wait(struct vk_sync *sync, | |
| uint64_t wait_value, | |
| enum vk_sync_wait_flags wait_flags) | |
| { | |
| assert(sync->type->features & VK_SYNC_FEATURE_CPU_WAIT); | |
| if (!(sync->flags & VK_SYNC_IS_TIMELINE)) | |
| assert(wait_value == 0); | |
| if (wait_flags & VK_SYNC_WAIT_PENDING) | |
| assert(sync->type->features & VK_SYNC_FEATURE_WAIT_PENDING); | |
| } | |
| static uint64_t | |
| get_max_abs_timeout_ns(void) | |
| { | |
| static int max_timeout_ms = -1; | |
| if (max_timeout_ms < 0) | |
| max_timeout_ms = debug_get_num_option("MESA_VK_MAX_TIMEOUT", 0); | |
| if (max_timeout_ms == 0) | |
| return UINT64_MAX; | |
| else | |
| return os_time_get_absolute_timeout(max_timeout_ms * 1000000ull); | |
| } | |
| static VkResult | |
| __vk_sync_wait(struct vk_device *device, | |
| struct vk_sync *sync, | |
| uint64_t wait_value, | |
| enum vk_sync_wait_flags wait_flags, | |
| uint64_t abs_timeout_ns) | |
| { | |
| assert_valid_wait(sync, wait_value, wait_flags); | |
| /* This doesn't make sense for a single wait */ | |
| assert(!(wait_flags & VK_SYNC_WAIT_ANY)); | |
| if (sync->type->wait) { | |
| return sync->type->wait(device, sync, wait_value, | |
| wait_flags, abs_timeout_ns); | |
| } else { | |
| struct vk_sync_wait wait = { | |
| .sync = sync, | |
| .stage_mask = ~(VkPipelineStageFlags2)0, | |
| .wait_value = wait_value, | |
| }; | |
| return sync->type->wait_many(device, 1, &wait, wait_flags, | |
| abs_timeout_ns); | |
| } | |
| } | |
| VkResult | |
| vk_sync_wait(struct vk_device *device, | |
| struct vk_sync *sync, | |
| uint64_t wait_value, | |
| enum vk_sync_wait_flags wait_flags, | |
| uint64_t abs_timeout_ns) | |
| { | |
| uint64_t max_abs_timeout_ns = get_max_abs_timeout_ns(); | |
| if (abs_timeout_ns > max_abs_timeout_ns) { | |
| VkResult result = | |
| __vk_sync_wait(device, sync, wait_value, wait_flags, | |
| max_abs_timeout_ns); | |
| if (unlikely(result == VK_TIMEOUT)) | |
| return vk_device_set_lost(device, "Maximum timeout exceeded!"); | |
| return result; | |
| } else { | |
| return __vk_sync_wait(device, sync, wait_value, wait_flags, | |
| abs_timeout_ns); | |
| } | |
| } | |
| static bool | |
| can_wait_many(struct vk_device *device, | |
| uint32_t wait_count, | |
| const struct vk_sync_wait *waits, | |
| enum vk_sync_wait_flags wait_flags) | |
| { | |
| if (waits[0].sync->type->wait_many == NULL) | |
| return false; | |
| if ((wait_flags & VK_SYNC_WAIT_ANY) && | |
| !(waits[0].sync->type->features & VK_SYNC_FEATURE_WAIT_ANY)) | |
| return false; | |
| /* If we only have one sync type, there's no need to check everything */ | |
| if (device->physical->supported_sync_types[1] == NULL) { | |
| assert(waits[0].sync->type == device->physical->supported_sync_types[0]); | |
| return true; | |
| } | |
| for (uint32_t i = 0; i < wait_count; i++) { | |
| assert_valid_wait(waits[i].sync, waits[i].wait_value, wait_flags); | |
| if (waits[i].sync->type != waits[0].sync->type) | |
| return false; | |
| } | |
| return true; | |
| } | |
| static VkResult | |
| __vk_sync_wait_many(struct vk_device *device, | |
| uint32_t wait_count, | |
| const struct vk_sync_wait *waits, | |
| enum vk_sync_wait_flags wait_flags, | |
| uint64_t abs_timeout_ns) | |
| { | |
| if (wait_count == 0) | |
| return VK_SUCCESS; | |
| if (wait_count == 1) { | |
| return __vk_sync_wait(device, waits[0].sync, waits[0].wait_value, | |
| wait_flags & ~VK_SYNC_WAIT_ANY, abs_timeout_ns); | |
| } | |
| if (can_wait_many(device, wait_count, waits, wait_flags)) { | |
| return waits[0].sync->type->wait_many(device, wait_count, waits, | |
| wait_flags, abs_timeout_ns); | |
| } else if (wait_flags & VK_SYNC_WAIT_ANY) { | |
| /* If we have multiple syncs and they don't support wait_any or they're | |
| * not all the same type, there's nothing better we can do than spin. | |
| */ | |
| do { | |
| for (uint32_t i = 0; i < wait_count; i++) { | |
| VkResult result = __vk_sync_wait(device, waits[i].sync, | |
| waits[i].wait_value, | |
| wait_flags & ~VK_SYNC_WAIT_ANY, | |
| 0 /* abs_timeout_ns */); | |
| if (result != VK_TIMEOUT) | |
| return result; | |
| } | |
| } while (os_time_get_nano() < abs_timeout_ns); | |
| return VK_TIMEOUT; | |
| } else { | |
| for (uint32_t i = 0; i < wait_count; i++) { | |
| VkResult result = __vk_sync_wait(device, waits[i].sync, | |
| waits[i].wait_value, | |
| wait_flags, abs_timeout_ns); | |
| if (result != VK_SUCCESS) | |
| return result; | |
| } | |
| return VK_SUCCESS; | |
| } | |
| } | |
| VkResult | |
| vk_sync_wait_many(struct vk_device *device, | |
| uint32_t wait_count, | |
| const struct vk_sync_wait *waits, | |
| enum vk_sync_wait_flags wait_flags, | |
| uint64_t abs_timeout_ns) | |
| { | |
| uint64_t max_abs_timeout_ns = get_max_abs_timeout_ns(); | |
| if (abs_timeout_ns > max_abs_timeout_ns) { | |
| VkResult result = | |
| __vk_sync_wait_many(device, wait_count, waits, wait_flags, | |
| max_abs_timeout_ns); | |
| if (unlikely(result == VK_TIMEOUT)) | |
| return vk_device_set_lost(device, "Maximum timeout exceeded!"); | |
| return result; | |
| } else { | |
| return __vk_sync_wait_many(device, wait_count, waits, wait_flags, | |
| abs_timeout_ns); | |
| } | |
| } | |
| VkResult | |
| vk_sync_import_opaque_fd(struct vk_device *device, | |
| struct vk_sync *sync, | |
| int fd) | |
| { | |
| VkResult result = sync->type->import_opaque_fd(device, sync, fd); | |
| if (unlikely(result != VK_SUCCESS)) | |
| return result; | |
| sync->flags |= VK_SYNC_IS_SHAREABLE | | |
| VK_SYNC_IS_SHARED; | |
| return VK_SUCCESS; | |
| } | |
| VkResult | |
| vk_sync_export_opaque_fd(struct vk_device *device, | |
| struct vk_sync *sync, | |
| int *fd) | |
| { | |
| assert(sync->flags & VK_SYNC_IS_SHAREABLE); | |
| VkResult result = sync->type->export_opaque_fd(device, sync, fd); | |
| if (unlikely(result != VK_SUCCESS)) | |
| return result; | |
| sync->flags |= VK_SYNC_IS_SHARED; | |
| return VK_SUCCESS; | |
| } | |
| VkResult | |
| vk_sync_import_sync_file(struct vk_device *device, | |
| struct vk_sync *sync, | |
| int sync_file) | |
| { | |
| assert(!(sync->flags & VK_SYNC_IS_TIMELINE)); | |
| /* Silently handle negative file descriptors in case the driver doesn't | |
| * want to bother. | |
| */ | |
| if (sync_file < 0 && sync->type->signal) | |
| return sync->type->signal(device, sync, 0); | |
| return sync->type->import_sync_file(device, sync, sync_file); | |
| } | |
| VkResult | |
| vk_sync_export_sync_file(struct vk_device *device, | |
| struct vk_sync *sync, | |
| int *sync_file) | |
| { | |
| assert(!(sync->flags & VK_SYNC_IS_TIMELINE)); | |
| return sync->type->export_sync_file(device, sync, sync_file); | |
| } | |
| VkResult | |
| vk_sync_import_win32_handle(struct vk_device *device, | |
| struct vk_sync *sync, | |
| void *handle, | |
| const wchar_t *name) | |
| { | |
| VkResult result = sync->type->import_win32_handle(device, sync, handle, name); | |
| if (unlikely(result != VK_SUCCESS)) | |
| return result; | |
| sync->flags |= VK_SYNC_IS_SHAREABLE | | |
| VK_SYNC_IS_SHARED; | |
| return VK_SUCCESS; | |
| } | |
| VkResult | |
| vk_sync_export_win32_handle(struct vk_device *device, | |
| struct vk_sync *sync, | |
| void **handle) | |
| { | |
| assert(sync->flags & VK_SYNC_IS_SHAREABLE); | |
| VkResult result = sync->type->export_win32_handle(device, sync, handle); | |
| if (unlikely(result != VK_SUCCESS)) | |
| return result; | |
| sync->flags |= VK_SYNC_IS_SHARED; | |
| return VK_SUCCESS; | |
| } | |
| VkResult | |
| vk_sync_set_win32_export_params(struct vk_device *device, | |
| struct vk_sync *sync, | |
| const void *security_attributes, | |
| uint32_t access, | |
| const wchar_t *name) | |
| { | |
| assert(sync->flags & VK_SYNC_IS_SHARED); | |
| return sync->type->set_win32_export_params(device, sync, security_attributes, access, name); | |
| } | |
| /** | |
| * Unwraps a vk_sync_wait, removing any vk_sync_timeline or vk_sync_binary | |
| * and replacing the sync with the actual driver primitive. | |
| * | |
| * After this is returns, wait->sync may be NULL, indicating no actual wait | |
| * is needed and this wait can be discarded. | |
| * | |
| * If the sync is a timeline, the point will be returned in point_out. | |
| * Otherwise point_out will be set to NULL. | |
| */ | |
| VkResult | |
| vk_sync_wait_unwrap(struct vk_device *device, | |
| struct vk_sync_wait *wait, | |
| struct vk_sync_timeline_point **point_out) | |
| { | |
| *point_out = NULL; | |
| if (wait->sync->flags & VK_SYNC_IS_TIMELINE) { | |
| if (wait->wait_value == 0) { | |
| *wait = (struct vk_sync_wait) { .sync = NULL }; | |
| return VK_SUCCESS; | |
| } | |
| } else { | |
| assert(wait->wait_value == 0); | |
| } | |
| struct vk_sync_timeline *timeline = vk_sync_as_timeline(wait->sync); | |
| if (timeline) { | |
| assert(device->timeline_mode == VK_DEVICE_TIMELINE_MODE_EMULATED); | |
| VkResult result = vk_sync_timeline_get_point(device, timeline, | |
| wait->wait_value, | |
| point_out); | |
| if (unlikely(result != VK_SUCCESS)) { | |
| /* vk_sync_timeline_get_point() returns VK_NOT_READY if no time | |
| * point can be found. Turn that into an actual error. | |
| */ | |
| return vk_errorf(device, VK_ERROR_UNKNOWN, | |
| "Time point >= %"PRIu64" not found", | |
| wait->wait_value); | |
| } | |
| /* This can happen if the point is long past */ | |
| if (*point_out == NULL) { | |
| *wait = (struct vk_sync_wait) { .sync = NULL }; | |
| return VK_SUCCESS; | |
| } | |
| wait->sync = &(*point_out)->sync; | |
| wait->wait_value = 0; | |
| } | |
| struct vk_sync_binary *binary = vk_sync_as_binary(wait->sync); | |
| if (binary) { | |
| wait->sync = &binary->timeline; | |
| wait->wait_value = binary->next_point; | |
| } | |
| if (vk_sync_type_is_dummy(wait->sync->type)) { | |
| if (*point_out != NULL) { | |
| vk_sync_timeline_point_unref(device, *point_out); | |
| *point_out = NULL; | |
| } | |
| *wait = (struct vk_sync_wait) { .sync = NULL }; | |
| return VK_SUCCESS; | |
| } | |
| return VK_SUCCESS; | |
| } | |
| /** | |
| * Unwraps a vk_sync_signal, removing any vk_sync_timeline or vk_sync_binary | |
| * and replacing the sync with the actual driver primitive. | |
| * | |
| * If the sync is a timeline, the point will be returned in point_out. This | |
| * point must be installed in the timeline after the actual signal has been | |
| * submitted to the kernel driver. Otherwise point_out will be set to NULL. | |
| */ | |
| VkResult | |
| vk_sync_signal_unwrap(struct vk_device *device, | |
| struct vk_sync_signal *signal, | |
| struct vk_sync_timeline_point **point_out) | |
| { | |
| if (signal->sync->flags & VK_SYNC_IS_TIMELINE) | |
| assert(signal->signal_value > 0); | |
| else | |
| assert(signal->signal_value == 0); | |
| *point_out = NULL; | |
| struct vk_sync_timeline *timeline = vk_sync_as_timeline(signal->sync); | |
| if (timeline) { | |
| assert(device->timeline_mode == VK_DEVICE_TIMELINE_MODE_EMULATED); | |
| VkResult result = vk_sync_timeline_alloc_point(device, timeline, | |
| signal->signal_value, | |
| point_out); | |
| if (unlikely(result != VK_SUCCESS)) | |
| return result; | |
| signal->sync = &(*point_out)->sync; | |
| signal->signal_value = 0; | |
| } | |
| struct vk_sync_binary *binary = vk_sync_as_binary(signal->sync); | |
| if (binary) { | |
| signal->sync = &binary->timeline; | |
| signal->signal_value = ++binary->next_point; | |
| } | |
| assert(!vk_sync_type_is_dummy(signal->sync->type)); | |
| return VK_SUCCESS; | |
| } |