Google Git
Sign in
fuchsia / third_party / mesa / refs/heads/gitlab/explicit-sync / . / src / virtio / vulkan / vn_queue.c
blob: 7a74743ac23625cb464bd2b435a2858bf36a10da [file] [log] [blame] [edit]
/*
* Copyright 2019 Google LLC
* SPDX-License-Identifier: MIT
*
* based in part on anv and radv which are:
* Copyright © 2015 Intel Corporation
* Copyright © 2016 Red Hat.
* Copyright © 2016 Bas Nieuwenhuizen
*/
#include "vn_queue.h"
#include "util/libsync.h"
#include "venus-protocol/vn_protocol_driver_event.h"
#include "venus-protocol/vn_protocol_driver_fence.h"
#include "venus-protocol/vn_protocol_driver_queue.h"
#include "venus-protocol/vn_protocol_driver_semaphore.h"
#include "venus-protocol/vn_protocol_driver_transport.h"
#include "vn_command_buffer.h"
#include "vn_device.h"
#include "vn_device_memory.h"
#include "vn_feedback.h"
#include "vn_instance.h"
#include "vn_physical_device.h"
#include "vn_query_pool.h"
#include "vn_renderer.h"
#include "vn_wsi.h"
/* queue commands */
struct vn_submit_info_pnext_fix {
VkDeviceGroupSubmitInfo group;
VkProtectedSubmitInfo protected;
VkTimelineSemaphoreSubmitInfo timeline;
};
struct vn_queue_submission {
VkStructureType batch_type;
VkQueue queue_handle;
uint32_t batch_count;
union {
const void *batches;
const VkSubmitInfo *submit_batches;
const VkSubmitInfo2 *submit2_batches;
const VkBindSparseInfo *sparse_batches;
};
VkFence fence_handle;
uint32_t cmd_count;
uint32_t feedback_types;
uint32_t pnext_count;
uint32_t dev_mask_count;
bool has_zink_sync_batch;
const struct vn_device_memory *wsi_mem;
struct vn_sync_payload_external external_payload;
/* Temporary storage allocation for submission
*
* A single alloc for storage is performed and the offsets inside storage
* are set as below:
*
* batches
* - non-empty submission: copy of original batches
* - empty submission: a single batch for fence feedback (ffb)
* cmds
* - for each batch:
* - copy of original batch cmds
* - a single cmd for query feedback (qfb)
* - one cmd for each signal semaphore that has feedback (sfb)
* - if last batch, a single cmd for ffb
*/
struct {
void *storage;
union {
void *batches;
VkSubmitInfo *submit_batches;
VkSubmitInfo2 *submit2_batches;
};
union {
void *cmds;
VkCommandBuffer *cmd_handles;
VkCommandBufferSubmitInfo *cmd_infos;
};
struct vn_submit_info_pnext_fix *pnexts;
uint32_t *dev_masks;
} temp;
};
static inline uint32_t
vn_get_wait_semaphore_count(struct vn_queue_submission *submit,
uint32_t batch_index)
{
switch (submit->batch_type) {
case VK_STRUCTURE_TYPE_SUBMIT_INFO:
return submit->submit_batches[batch_index].waitSemaphoreCount;
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
return submit->submit2_batches[batch_index].waitSemaphoreInfoCount;
case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
return submit->sparse_batches[batch_index].waitSemaphoreCount;
default:
unreachable("unexpected batch type");
}
}
static inline uint32_t
vn_get_signal_semaphore_count(struct vn_queue_submission *submit,
uint32_t batch_index)
{
switch (submit->batch_type) {
case VK_STRUCTURE_TYPE_SUBMIT_INFO:
return submit->submit_batches[batch_index].signalSemaphoreCount;
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
return submit->submit2_batches[batch_index].signalSemaphoreInfoCount;
case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
return submit->sparse_batches[batch_index].signalSemaphoreCount;
default:
unreachable("unexpected batch type");
}
}
static inline VkSemaphore
vn_get_wait_semaphore(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t semaphore_index)
{
switch (submit->batch_type) {
case VK_STRUCTURE_TYPE_SUBMIT_INFO:
return submit->submit_batches[batch_index]
.pWaitSemaphores[semaphore_index];
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
return submit->submit2_batches[batch_index]
.pWaitSemaphoreInfos[semaphore_index]
.semaphore;
case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
return submit->sparse_batches[batch_index]
.pWaitSemaphores[semaphore_index];
default:
unreachable("unexpected batch type");
}
}
static inline VkSemaphore
vn_get_signal_semaphore(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t semaphore_index)
{
switch (submit->batch_type) {
case VK_STRUCTURE_TYPE_SUBMIT_INFO:
return submit->submit_batches[batch_index]
.pSignalSemaphores[semaphore_index];
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
return submit->submit2_batches[batch_index]
.pSignalSemaphoreInfos[semaphore_index]
.semaphore;
case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
return submit->sparse_batches[batch_index]
.pSignalSemaphores[semaphore_index];
default:
unreachable("unexpected batch type");
}
}
static inline size_t
vn_get_batch_size(struct vn_queue_submission *submit)
{
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
? sizeof(VkSubmitInfo)
: sizeof(VkSubmitInfo2);
}
static inline size_t
vn_get_cmd_size(struct vn_queue_submission *submit)
{
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
? sizeof(VkCommandBuffer)
: sizeof(VkCommandBufferSubmitInfo);
}
static inline uint32_t
vn_get_cmd_count(struct vn_queue_submission *submit, uint32_t batch_index)
{
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
? submit->submit_batches[batch_index].commandBufferCount
: submit->submit2_batches[batch_index].commandBufferInfoCount;
}
static inline const void *
vn_get_cmds(struct vn_queue_submission *submit, uint32_t batch_index)
{
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
return submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
? (const void *)submit->submit_batches[batch_index]
.pCommandBuffers
: (const void *)submit->submit2_batches[batch_index]
.pCommandBufferInfos;
}
static inline struct vn_command_buffer *
vn_get_cmd(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t cmd_index)
{
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
return vn_command_buffer_from_handle(
submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO
? submit->submit_batches[batch_index].pCommandBuffers[cmd_index]
: submit->submit2_batches[batch_index]
.pCommandBufferInfos[cmd_index]
.commandBuffer);
}
static inline void
vn_set_temp_cmd(struct vn_queue_submission *submit,
uint32_t cmd_index,
VkCommandBuffer cmd_handle)
{
assert((submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO) ||
(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2));
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
submit->temp.cmd_infos[cmd_index] = (VkCommandBufferSubmitInfo){
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO,
.commandBuffer = cmd_handle,
};
} else {
submit->temp.cmd_handles[cmd_index] = cmd_handle;
}
}
static uint64_t
vn_get_signal_semaphore_counter(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t sem_index)
{
switch (submit->batch_type) {
case VK_STRUCTURE_TYPE_SUBMIT_INFO: {
const struct VkTimelineSemaphoreSubmitInfo *timeline_sem_info =
vk_find_struct_const(submit->submit_batches[batch_index].pNext,
TIMELINE_SEMAPHORE_SUBMIT_INFO);
return timeline_sem_info->pSignalSemaphoreValues[sem_index];
}
case VK_STRUCTURE_TYPE_SUBMIT_INFO_2:
return submit->submit2_batches[batch_index]
.pSignalSemaphoreInfos[sem_index]
.value;
default:
unreachable("unexpected batch type");
}
}
static bool
vn_has_zink_sync_batch(struct vn_queue_submission *submit)
{
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
struct vn_device *dev = (void *)queue->base.base.base.device;
struct vn_instance *instance = dev->instance;
const uint32_t last_batch_index = submit->batch_count - 1;
if (!instance->engine_is_zink)
return false;
if (!submit->batch_count || !last_batch_index ||
vn_get_cmd_count(submit, last_batch_index))
return false;
if (vn_get_wait_semaphore_count(submit, last_batch_index))
return false;
const uint32_t signal_count =
vn_get_signal_semaphore_count(submit, last_batch_index);
for (uint32_t i = 0; i < signal_count; i++) {
struct vn_semaphore *sem = vn_semaphore_from_handle(
vn_get_signal_semaphore(submit, last_batch_index, i));
if (sem->feedback.slot) {
return true;
}
}
return false;
}
static bool
vn_fix_batch_cmd_count_for_zink_sync(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t new_cmd_count)
{
/* If the last batch is a zink sync batch which is empty but contains
* feedback, append the feedback to the previous batch instead so that
* the last batch remains empty for perf.
*/
if (batch_index == submit->batch_count - 1 &&
submit->has_zink_sync_batch) {
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
VkSubmitInfo2 *batch =
&submit->temp.submit2_batches[batch_index - 1];
assert(batch->pCommandBufferInfos);
batch->commandBufferInfoCount += new_cmd_count;
} else {
VkSubmitInfo *batch = &submit->temp.submit_batches[batch_index - 1];
assert(batch->pCommandBuffers);
batch->commandBufferCount += new_cmd_count;
}
return true;
}
return false;
}
static void
vn_fix_device_group_cmd_count(struct vn_queue_submission *submit,
uint32_t batch_index)
{
const VkSubmitInfo *src_batch = &submit->submit_batches[batch_index];
struct vn_submit_info_pnext_fix *pnext_fix = submit->temp.pnexts;
VkBaseOutStructure *dst =
(void *)&submit->temp.submit_batches[batch_index];
uint32_t new_cmd_count =
submit->temp.submit_batches[batch_index].commandBufferCount;
vk_foreach_struct_const(src, src_batch->pNext) {
void *pnext = NULL;
switch (src->sType) {
case VK_STRUCTURE_TYPE_DEVICE_GROUP_SUBMIT_INFO: {
uint32_t orig_cmd_count = 0;
memcpy(&pnext_fix->group, src, sizeof(pnext_fix->group));
VkDeviceGroupSubmitInfo *src_device_group =
(VkDeviceGroupSubmitInfo *)src;
if (src_device_group->commandBufferCount) {
orig_cmd_count = src_device_group->commandBufferCount;
memcpy(submit->temp.dev_masks,
src_device_group->pCommandBufferDeviceMasks,
sizeof(uint32_t) * orig_cmd_count);
}
/* Set feedback cmd device masks to 0 */
for (uint32_t i = orig_cmd_count; i < new_cmd_count; i++) {
submit->temp.dev_masks[i] = 0;
}
pnext_fix->group.commandBufferCount = new_cmd_count;
pnext_fix->group.pCommandBufferDeviceMasks = submit->temp.dev_masks;
pnext = &pnext_fix->group;
break;
}
case VK_STRUCTURE_TYPE_PROTECTED_SUBMIT_INFO:
memcpy(&pnext_fix->protected, src, sizeof(pnext_fix->protected));
pnext = &pnext_fix->protected;
break;
case VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO:
memcpy(&pnext_fix->timeline, src, sizeof(pnext_fix->timeline));
pnext = &pnext_fix->timeline;
break;
default:
/* The following structs are not supported by venus so are not
* handled here. VkAmigoProfilingSubmitInfoSEC,
* VkD3D12FenceSubmitInfoKHR, VkFrameBoundaryEXT,
* VkLatencySubmissionPresentIdNV, VkPerformanceQuerySubmitInfoKHR,
* VkWin32KeyedMutexAcquireReleaseInfoKHR,
* VkWin32KeyedMutexAcquireReleaseInfoNV
*/
break;
}
if (pnext) {
dst->pNext = pnext;
dst = pnext;
}
}
submit->temp.pnexts++;
submit->temp.dev_masks += new_cmd_count;
}
static bool
vn_semaphore_wait_external(struct vn_device *dev, struct vn_semaphore *sem);
static VkResult
vn_queue_submission_fix_batch_semaphores(struct vn_queue_submission *submit,
uint32_t batch_index)
{
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
VkDevice dev_handle = vk_device_to_handle(queue_vk->base.device);
struct vn_device *dev = vn_device_from_handle(dev_handle);
const uint32_t wait_count =
vn_get_wait_semaphore_count(submit, batch_index);
for (uint32_t i = 0; i < wait_count; i++) {
VkSemaphore sem_handle = vn_get_wait_semaphore(submit, batch_index, i);
struct vn_semaphore *sem = vn_semaphore_from_handle(sem_handle);
const struct vn_sync_payload *payload = sem->payload;
if (payload->type != VN_SYNC_TYPE_IMPORTED_SYNC_FD)
continue;
if (!vn_semaphore_wait_external(dev, sem))
return VK_ERROR_DEVICE_LOST;
assert(dev->physical_device->renderer_sync_fd.semaphore_importable);
const VkImportSemaphoreResourceInfoMESA res_info = {
.sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_RESOURCE_INFO_MESA,
.semaphore = sem_handle,
.resourceId = 0,
};
vn_async_vkImportSemaphoreResourceMESA(dev->primary_ring, dev_handle,
&res_info);
}
return VK_SUCCESS;
}
static void
vn_queue_submission_count_batch_feedback(struct vn_queue_submission *submit,
uint32_t batch_index)
{
const uint32_t signal_count =
vn_get_signal_semaphore_count(submit, batch_index);
uint32_t extra_cmd_count = 0;
uint32_t feedback_types = 0;
for (uint32_t i = 0; i < signal_count; i++) {
struct vn_semaphore *sem = vn_semaphore_from_handle(
vn_get_signal_semaphore(submit, batch_index, i));
if (sem->feedback.slot) {
feedback_types |= VN_FEEDBACK_TYPE_SEMAPHORE;
extra_cmd_count++;
}
}
if (submit->batch_type != VK_STRUCTURE_TYPE_BIND_SPARSE_INFO) {
const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index);
for (uint32_t i = 0; i < cmd_count; i++) {
struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i);
if (!list_is_empty(&cmd->builder.query_records))
feedback_types |= VN_FEEDBACK_TYPE_QUERY;
/* If a cmd that was submitted previously and already has a feedback
* cmd linked, as long as
* VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT was not set we can
* assume it has completed execution and is no longer in the pending
* state so its safe to recycle the old feedback command.
*/
if (cmd->linked_qfb_cmd) {
assert(!cmd->builder.is_simultaneous);
vn_query_feedback_cmd_free(cmd->linked_qfb_cmd);
cmd->linked_qfb_cmd = NULL;
}
}
if (feedback_types & VN_FEEDBACK_TYPE_QUERY)
extra_cmd_count++;
if (submit->feedback_types & VN_FEEDBACK_TYPE_FENCE &&
batch_index == submit->batch_count - 1) {
feedback_types |= VN_FEEDBACK_TYPE_FENCE;
extra_cmd_count++;
}
/* Space to copy the original cmds to append feedback to it.
* If the last batch is a zink sync batch which is an empty batch with
* sem feedback, feedback will be appended to the second to last batch
* so also need to copy the second to last batch's original cmds even
* if it doesn't have feedback itself.
*/
if (feedback_types || (batch_index == submit->batch_count - 2 &&
submit->has_zink_sync_batch)) {
extra_cmd_count += cmd_count;
}
}
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO &&
extra_cmd_count) {
const VkDeviceGroupSubmitInfo *device_group = vk_find_struct_const(
submit->submit_batches[batch_index].pNext, DEVICE_GROUP_SUBMIT_INFO);
if (device_group) {
submit->pnext_count++;
submit->dev_mask_count += extra_cmd_count;
}
}
submit->feedback_types |= feedback_types;
submit->cmd_count += extra_cmd_count;
}
static VkResult
vn_queue_submission_prepare(struct vn_queue_submission *submit)
{
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
struct vn_fence *fence = vn_fence_from_handle(submit->fence_handle);
assert(!fence || !fence->is_external || !fence->feedback.slot);
if (fence && fence->feedback.slot)
submit->feedback_types |= VN_FEEDBACK_TYPE_FENCE;
if (submit->batch_type != VK_STRUCTURE_TYPE_BIND_SPARSE_INFO)
submit->has_zink_sync_batch = vn_has_zink_sync_batch(submit);
submit->external_payload.ring_idx = queue->ring_idx;
submit->wsi_mem = NULL;
if (submit->batch_count == 1 &&
submit->batch_type != VK_STRUCTURE_TYPE_BIND_SPARSE_INFO) {
const struct wsi_memory_signal_submit_info *info = vk_find_struct_const(
submit->submit_batches[0].pNext, WSI_MEMORY_SIGNAL_SUBMIT_INFO_MESA);
if (info) {
submit->wsi_mem = vn_device_memory_from_handle(info->memory);
assert(!submit->wsi_mem->base_memory && submit->wsi_mem->base_bo);
}
}
for (uint32_t i = 0; i < submit->batch_count; i++) {
VkResult result = vn_queue_submission_fix_batch_semaphores(submit, i);
if (result != VK_SUCCESS)
return result;
vn_queue_submission_count_batch_feedback(submit, i);
}
return VK_SUCCESS;
}
static VkResult
vn_queue_submission_alloc_storage(struct vn_queue_submission *submit)
{
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
if (!submit->feedback_types)
return VK_SUCCESS;
/* for original batches or a new batch to hold feedback fence cmd */
const size_t total_batch_size =
vn_get_batch_size(submit) * MAX2(submit->batch_count, 1);
/* for fence, timeline semaphore and query feedback cmds */
const size_t total_cmd_size =
vn_get_cmd_size(submit) * MAX2(submit->cmd_count, 1);
/* for fixing command buffer counts in device group info, if it exists */
const size_t total_pnext_size =
submit->pnext_count * sizeof(struct vn_submit_info_pnext_fix);
const size_t total_dev_mask_size =
submit->dev_mask_count * sizeof(uint32_t);
submit->temp.storage = vn_cached_storage_get(
&queue->storage, total_batch_size + total_cmd_size + total_pnext_size +
total_dev_mask_size);
if (!submit->temp.storage)
return VK_ERROR_OUT_OF_HOST_MEMORY;
submit->temp.batches = submit->temp.storage;
submit->temp.cmds = submit->temp.storage + total_batch_size;
submit->temp.pnexts =
submit->temp.storage + total_batch_size + total_cmd_size;
submit->temp.dev_masks = submit->temp.storage + total_batch_size +
total_cmd_size + total_pnext_size;
return VK_SUCCESS;
}
static VkResult
vn_queue_submission_get_resolved_query_records(
struct vn_queue_submission *submit,
uint32_t batch_index,
struct vn_feedback_cmd_pool *fb_cmd_pool,
struct list_head *resolved_records)
{
struct vn_command_pool *cmd_pool =
vn_command_pool_from_handle(fb_cmd_pool->pool_handle);
struct list_head dropped_records;
VkResult result = VK_SUCCESS;
list_inithead(resolved_records);
list_inithead(&dropped_records);
const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index);
for (uint32_t i = 0; i < cmd_count; i++) {
struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i);
list_for_each_entry(struct vn_cmd_query_record, record,
&cmd->builder.query_records, head) {
if (!record->copy) {
list_for_each_entry_safe(struct vn_cmd_query_record, prev,
resolved_records, head) {
/* If we previously added a query feedback that is now getting
* reset, remove it since it is now a no-op and the deferred
* feedback copy will cause a hang waiting for the reset query
* to become available.
*/
if (prev->copy && prev->query_pool == record->query_pool &&
prev->query >= record->query &&
prev->query < record->query + record->query_count)
list_move_to(&prev->head, &dropped_records);
}
}
simple_mtx_lock(&fb_cmd_pool->mutex);
struct vn_cmd_query_record *curr = vn_cmd_pool_alloc_query_record(
cmd_pool, record->query_pool, record->query, record->query_count,
record->copy);
simple_mtx_unlock(&fb_cmd_pool->mutex);
if (!curr) {
list_splicetail(resolved_records, &dropped_records);
result = VK_ERROR_OUT_OF_HOST_MEMORY;
goto out_free_dropped_records;
}
list_addtail(&curr->head, resolved_records);
}
}
/* further resolve to batch sequential queries */
struct vn_cmd_query_record *curr =
list_first_entry(resolved_records, struct vn_cmd_query_record, head);
list_for_each_entry_safe(struct vn_cmd_query_record, next,
resolved_records, head) {
if (curr->query_pool == next->query_pool && curr->copy == next->copy) {
if (curr->query + curr->query_count == next->query) {
curr->query_count += next->query_count;
list_move_to(&next->head, &dropped_records);
} else if (curr->query == next->query + next->query_count) {
curr->query = next->query;
curr->query_count += next->query_count;
list_move_to(&next->head, &dropped_records);
} else {
curr = next;
}
} else {
curr = next;
}
}
out_free_dropped_records:
simple_mtx_lock(&fb_cmd_pool->mutex);
vn_cmd_pool_free_query_records(cmd_pool, &dropped_records);
simple_mtx_unlock(&fb_cmd_pool->mutex);
return result;
}
static VkResult
vn_queue_submission_add_query_feedback(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t *new_cmd_count)
{
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
struct vn_device *dev = (void *)queue_vk->base.device;
VkResult result;
struct vn_feedback_cmd_pool *fb_cmd_pool = NULL;
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
if (dev->queue_families[i] == queue_vk->queue_family_index) {
fb_cmd_pool = &dev->fb_cmd_pools[i];
break;
}
}
assert(fb_cmd_pool);
struct list_head resolved_records;
result = vn_queue_submission_get_resolved_query_records(
submit, batch_index, fb_cmd_pool, &resolved_records);
if (result != VK_SUCCESS)
return result;
/* currently the reset query is always recorded */
assert(!list_is_empty(&resolved_records));
struct vn_query_feedback_cmd *qfb_cmd;
result = vn_query_feedback_cmd_alloc(vn_device_to_handle(dev), fb_cmd_pool,
&resolved_records, &qfb_cmd);
if (result == VK_SUCCESS) {
/* link query feedback cmd lifecycle with a cmd in the original batch so
* that the feedback cmd can be reset and recycled when that cmd gets
* reset/freed.
*
* Avoid cmd buffers with VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT
* since we don't know if all its instances have completed execution.
* Should be rare enough to just log and leak the feedback cmd.
*/
bool found_companion_cmd = false;
const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index);
for (uint32_t i = 0; i < cmd_count; i++) {
struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i);
if (!cmd->builder.is_simultaneous) {
cmd->linked_qfb_cmd = qfb_cmd;
found_companion_cmd = true;
break;
}
}
if (!found_companion_cmd)
vn_log(dev->instance, "WARN: qfb cmd has leaked!");
vn_set_temp_cmd(submit, (*new_cmd_count)++, qfb_cmd->cmd_handle);
}
simple_mtx_lock(&fb_cmd_pool->mutex);
vn_cmd_pool_free_query_records(
vn_command_pool_from_handle(fb_cmd_pool->pool_handle),
&resolved_records);
simple_mtx_unlock(&fb_cmd_pool->mutex);
return result;
}
struct vn_semaphore_feedback_cmd *
vn_semaphore_get_feedback_cmd(struct vn_device *dev,
struct vn_semaphore *sem);
static VkResult
vn_queue_submission_add_semaphore_feedback(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t signal_index,
uint32_t *new_cmd_count)
{
struct vn_semaphore *sem = vn_semaphore_from_handle(
vn_get_signal_semaphore(submit, batch_index, signal_index));
if (!sem->feedback.slot)
return VK_SUCCESS;
VK_FROM_HANDLE(vk_queue, queue_vk, submit->queue_handle);
struct vn_device *dev = (void *)queue_vk->base.device;
struct vn_semaphore_feedback_cmd *sfb_cmd =
vn_semaphore_get_feedback_cmd(dev, sem);
if (!sfb_cmd)
return VK_ERROR_OUT_OF_HOST_MEMORY;
const uint64_t counter =
vn_get_signal_semaphore_counter(submit, batch_index, signal_index);
vn_feedback_set_counter(sfb_cmd->src_slot, counter);
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
if (dev->queue_families[i] == queue_vk->queue_family_index) {
vn_set_temp_cmd(submit, (*new_cmd_count)++, sfb_cmd->cmd_handles[i]);
return VK_SUCCESS;
}
}
unreachable("bad feedback sem");
}
static void
vn_queue_submission_add_fence_feedback(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t *new_cmd_count)
{
VK_FROM_HANDLE(vk_queue, queue_vk, submit->queue_handle);
struct vn_device *dev = (void *)queue_vk->base.device;
struct vn_fence *fence = vn_fence_from_handle(submit->fence_handle);
VkCommandBuffer ffb_cmd_handle = VK_NULL_HANDLE;
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
if (dev->queue_families[i] == queue_vk->queue_family_index) {
ffb_cmd_handle = fence->feedback.commands[i];
}
}
assert(ffb_cmd_handle != VK_NULL_HANDLE);
vn_set_temp_cmd(submit, (*new_cmd_count)++, ffb_cmd_handle);
}
static VkResult
vn_queue_submission_add_feedback_cmds(struct vn_queue_submission *submit,
uint32_t batch_index,
uint32_t feedback_types)
{
VkResult result;
uint32_t new_cmd_count = vn_get_cmd_count(submit, batch_index);
if (feedback_types & VN_FEEDBACK_TYPE_QUERY) {
result = vn_queue_submission_add_query_feedback(submit, batch_index,
&new_cmd_count);
if (result != VK_SUCCESS)
return result;
}
if (feedback_types & VN_FEEDBACK_TYPE_SEMAPHORE) {
const uint32_t signal_count =
vn_get_signal_semaphore_count(submit, batch_index);
for (uint32_t i = 0; i < signal_count; i++) {
result = vn_queue_submission_add_semaphore_feedback(
submit, batch_index, i, &new_cmd_count);
if (result != VK_SUCCESS)
return result;
}
if (vn_fix_batch_cmd_count_for_zink_sync(submit, batch_index,
new_cmd_count))
return VK_SUCCESS;
}
if (feedback_types & VN_FEEDBACK_TYPE_FENCE) {
vn_queue_submission_add_fence_feedback(submit, batch_index,
&new_cmd_count);
}
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
VkSubmitInfo2 *batch = &submit->temp.submit2_batches[batch_index];
batch->pCommandBufferInfos = submit->temp.cmd_infos;
batch->commandBufferInfoCount = new_cmd_count;
} else {
VkSubmitInfo *batch = &submit->temp.submit_batches[batch_index];
batch->pCommandBuffers = submit->temp.cmd_handles;
batch->commandBufferCount = new_cmd_count;
const VkDeviceGroupSubmitInfo *device_group = vk_find_struct_const(
submit->submit_batches[batch_index].pNext, DEVICE_GROUP_SUBMIT_INFO);
if (device_group)
vn_fix_device_group_cmd_count(submit, batch_index);
}
return VK_SUCCESS;
}
static VkResult
vn_queue_submission_setup_batch(struct vn_queue_submission *submit,
uint32_t batch_index)
{
uint32_t feedback_types = 0;
uint32_t extra_cmd_count = 0;
const uint32_t signal_count =
vn_get_signal_semaphore_count(submit, batch_index);
for (uint32_t i = 0; i < signal_count; i++) {
struct vn_semaphore *sem = vn_semaphore_from_handle(
vn_get_signal_semaphore(submit, batch_index, i));
if (sem->feedback.slot) {
feedback_types |= VN_FEEDBACK_TYPE_SEMAPHORE;
extra_cmd_count++;
}
}
const uint32_t cmd_count = vn_get_cmd_count(submit, batch_index);
for (uint32_t i = 0; i < cmd_count; i++) {
struct vn_command_buffer *cmd = vn_get_cmd(submit, batch_index, i);
if (!list_is_empty(&cmd->builder.query_records)) {
feedback_types |= VN_FEEDBACK_TYPE_QUERY;
extra_cmd_count++;
break;
}
}
if (submit->feedback_types & VN_FEEDBACK_TYPE_FENCE &&
batch_index == submit->batch_count - 1) {
feedback_types |= VN_FEEDBACK_TYPE_FENCE;
extra_cmd_count++;
}
/* If the batch has qfb, sfb or ffb, copy the original commands and append
* feedback cmds.
* If this is the second to last batch and the last batch a zink sync batch
* which is empty but has feedback, also copy the original commands for
* this batch so that the last batch's feedback can be appended to it.
*/
if (feedback_types || (batch_index == submit->batch_count - 2 &&
submit->has_zink_sync_batch)) {
const size_t cmd_size = vn_get_cmd_size(submit);
const size_t total_cmd_size = cmd_count * cmd_size;
/* copy only needed for non-empty batches */
if (total_cmd_size) {
memcpy(submit->temp.cmds, vn_get_cmds(submit, batch_index),
total_cmd_size);
}
VkResult result = vn_queue_submission_add_feedback_cmds(
submit, batch_index, feedback_types);
if (result != VK_SUCCESS)
return result;
/* advance the temp cmds for working on next batch cmds */
submit->temp.cmds += total_cmd_size + (extra_cmd_count * cmd_size);
}
return VK_SUCCESS;
}
static VkResult
vn_queue_submission_setup_batches(struct vn_queue_submission *submit)
{
assert(submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2 ||
submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO);
if (!submit->feedback_types)
return VK_SUCCESS;
/* For a submission that is:
* - non-empty: copy batches for adding feedbacks
* - empty: initialize a batch for fence feedback
*/
if (submit->batch_count) {
memcpy(submit->temp.batches, submit->batches,
vn_get_batch_size(submit) * submit->batch_count);
} else {
assert(submit->feedback_types & VN_FEEDBACK_TYPE_FENCE);
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
submit->temp.submit2_batches[0] = (VkSubmitInfo2){
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
};
} else {
submit->temp.submit_batches[0] = (VkSubmitInfo){
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
};
}
submit->batch_count = 1;
submit->batches = submit->temp.batches;
}
for (uint32_t i = 0; i < submit->batch_count; i++) {
VkResult result = vn_queue_submission_setup_batch(submit, i);
if (result != VK_SUCCESS)
return result;
}
submit->batches = submit->temp.batches;
return VK_SUCCESS;
}
static void
vn_queue_submission_cleanup_semaphore_feedback(
struct vn_queue_submission *submit)
{
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
VkDevice dev_handle = vk_device_to_handle(queue_vk->base.device);
for (uint32_t i = 0; i < submit->batch_count; i++) {
const uint32_t wait_count = vn_get_wait_semaphore_count(submit, i);
for (uint32_t j = 0; j < wait_count; j++) {
VkSemaphore sem_handle = vn_get_wait_semaphore(submit, i, j);
struct vn_semaphore *sem = vn_semaphore_from_handle(sem_handle);
if (!sem->feedback.slot)
continue;
/* sfb pending cmds are recycled when signaled counter is updated */
uint64_t counter = 0;
vn_GetSemaphoreCounterValue(dev_handle, sem_handle, &counter);
}
}
}
static void
vn_queue_submission_cleanup(struct vn_queue_submission *submit)
{
/* TODO clean up pending src feedbacks on failure? */
if (submit->feedback_types & VN_FEEDBACK_TYPE_SEMAPHORE)
vn_queue_submission_cleanup_semaphore_feedback(submit);
}
static VkResult
vn_queue_submission_prepare_submit(struct vn_queue_submission *submit)
{
VkResult result = vn_queue_submission_prepare(submit);
if (result != VK_SUCCESS)
return result;
result = vn_queue_submission_alloc_storage(submit);
if (result != VK_SUCCESS)
return result;
result = vn_queue_submission_setup_batches(submit);
if (result != VK_SUCCESS) {
vn_queue_submission_cleanup(submit);
return result;
}
return VK_SUCCESS;
}
static void
vn_queue_wsi_present(struct vn_queue_submission *submit)
{
struct vk_queue *queue_vk = vk_queue_from_handle(submit->queue_handle);
struct vn_device *dev = (void *)queue_vk->base.device;
if (!submit->wsi_mem)
return;
if (dev->renderer->info.has_implicit_fencing) {
struct vn_renderer_submit_batch batch = {
.ring_idx = submit->external_payload.ring_idx,
};
uint32_t local_data[8];
struct vn_cs_encoder local_enc =
VN_CS_ENCODER_INITIALIZER_LOCAL(local_data, sizeof(local_data));
if (submit->external_payload.ring_seqno_valid) {
const uint64_t ring_id = vn_ring_get_id(dev->primary_ring);
vn_encode_vkWaitRingSeqnoMESA(&local_enc, 0, ring_id,
submit->external_payload.ring_seqno);
batch.cs_data = local_data;
batch.cs_size = vn_cs_encoder_get_len(&local_enc);
}
const struct vn_renderer_submit renderer_submit = {
.bos = &submit->wsi_mem->base_bo,
.bo_count = 1,
.batches = &batch,
.batch_count = 1,
};
vn_renderer_submit(dev->renderer, &renderer_submit);
} else {
if (VN_DEBUG(WSI)) {
static uint32_t num_rate_limit_warning = 0;
if (num_rate_limit_warning++ < 10)
vn_log(dev->instance,
"forcing vkQueueWaitIdle before presenting");
}
vn_QueueWaitIdle(submit->queue_handle);
}
}
static VkResult
vn_queue_submit(struct vn_queue_submission *submit)
{
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
struct vn_device *dev = (void *)queue->base.base.base.device;
struct vn_instance *instance = dev->instance;
VkResult result;
/* To ensure external components waiting on the correct fence payload,
* below sync primitives must be installed after the submission:
* - explicit fencing: sync file export
* - implicit fencing: dma-fence attached to the wsi bo
*
* We enforce above via an asynchronous vkQueueSubmit(2) via ring followed
* by an asynchronous renderer submission to wait for the ring submission:
* - struct wsi_memory_signal_submit_info
* - fence is an external fence
* - has an external signal semaphore
*/
result = vn_queue_submission_prepare_submit(submit);
if (result != VK_SUCCESS)
return vn_error(instance, result);
/* skip no-op submit */
if (!submit->batch_count && submit->fence_handle == VK_NULL_HANDLE)
return VK_SUCCESS;
if (VN_PERF(NO_ASYNC_QUEUE_SUBMIT)) {
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
result = vn_call_vkQueueSubmit2(
dev->primary_ring, submit->queue_handle, submit->batch_count,
submit->submit2_batches, submit->fence_handle);
} else {
result = vn_call_vkQueueSubmit(
dev->primary_ring, submit->queue_handle, submit->batch_count,
submit->submit_batches, submit->fence_handle);
}
if (result != VK_SUCCESS) {
vn_queue_submission_cleanup(submit);
return vn_error(instance, result);
}
} else {
struct vn_ring_submit_command ring_submit;
if (submit->batch_type == VK_STRUCTURE_TYPE_SUBMIT_INFO_2) {
vn_submit_vkQueueSubmit2(
dev->primary_ring, 0, submit->queue_handle, submit->batch_count,
submit->submit2_batches, submit->fence_handle, &ring_submit);
} else {
vn_submit_vkQueueSubmit(dev->primary_ring, 0, submit->queue_handle,
submit->batch_count, submit->submit_batches,
submit->fence_handle, &ring_submit);
}
if (!ring_submit.ring_seqno_valid) {
vn_queue_submission_cleanup(submit);
return vn_error(instance, VK_ERROR_DEVICE_LOST);
}
submit->external_payload.ring_seqno_valid = true;
submit->external_payload.ring_seqno = ring_submit.ring_seqno;
}
/* If external fence, track the submission's ring_idx to facilitate
* sync_file export.
*
* Imported syncs don't need a proxy renderer sync on subsequent export,
* because an fd is already available.
*/
struct vn_fence *fence = vn_fence_from_handle(submit->fence_handle);
if (fence && fence->is_external) {
assert(fence->payload->type == VN_SYNC_TYPE_DEVICE_ONLY);
fence->external_payload = submit->external_payload;
}
for (uint32_t i = 0; i < submit->batch_count; i++) {
const uint32_t signal_count = vn_get_signal_semaphore_count(submit, i);
for (uint32_t j = 0; j < signal_count; j++) {
struct vn_semaphore *sem =
vn_semaphore_from_handle(vn_get_signal_semaphore(submit, i, j));
if (sem->is_external) {
assert(sem->payload->type == VN_SYNC_TYPE_DEVICE_ONLY);
sem->external_payload = submit->external_payload;
}
}
}
vn_queue_wsi_present(submit);
vn_queue_submission_cleanup(submit);
return VK_SUCCESS;
}
VkResult
vn_QueueSubmit(VkQueue queue,
uint32_t submitCount,
const VkSubmitInfo *pSubmits,
VkFence fence)
{
VN_TRACE_FUNC();
struct vn_queue_submission submit = {
.batch_type = VK_STRUCTURE_TYPE_SUBMIT_INFO,
.queue_handle = queue,
.batch_count = submitCount,
.submit_batches = pSubmits,
.fence_handle = fence,
};
return vn_queue_submit(&submit);
}
VkResult
vn_QueueSubmit2(VkQueue queue,
uint32_t submitCount,
const VkSubmitInfo2 *pSubmits,
VkFence fence)
{
VN_TRACE_FUNC();
struct vn_queue_submission submit = {
.batch_type = VK_STRUCTURE_TYPE_SUBMIT_INFO_2,
.queue_handle = queue,
.batch_count = submitCount,
.submit2_batches = pSubmits,
.fence_handle = fence,
};
return vn_queue_submit(&submit);
}
static VkResult
vn_queue_bind_sparse_submit(struct vn_queue_submission *submit)
{
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
struct vn_device *dev = (void *)queue->base.base.base.device;
struct vn_instance *instance = dev->instance;
VkResult result;
if (VN_PERF(NO_ASYNC_QUEUE_SUBMIT)) {
result = vn_call_vkQueueBindSparse(
dev->primary_ring, submit->queue_handle, submit->batch_count,
submit->sparse_batches, submit->fence_handle);
if (result != VK_SUCCESS)
return vn_error(instance, result);
} else {
struct vn_ring_submit_command ring_submit;
vn_submit_vkQueueBindSparse(dev->primary_ring, 0, submit->queue_handle,
submit->batch_count, submit->sparse_batches,
submit->fence_handle, &ring_submit);
if (!ring_submit.ring_seqno_valid)
return vn_error(instance, VK_ERROR_DEVICE_LOST);
}
return VK_SUCCESS;
}
static VkResult
vn_queue_bind_sparse_submit_batch(struct vn_queue_submission *submit,
uint32_t batch_index)
{
struct vn_queue *queue = vn_queue_from_handle(submit->queue_handle);
VkDevice dev_handle = vk_device_to_handle(queue->base.base.base.device);
const VkBindSparseInfo *sparse_info = &submit->sparse_batches[batch_index];
const VkSemaphore *signal_sem = sparse_info->pSignalSemaphores;
uint32_t signal_sem_count = sparse_info->signalSemaphoreCount;
VkResult result;
struct vn_queue_submission sparse_batch = {
.batch_type = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO,
.queue_handle = submit->queue_handle,
.batch_count = 1,
.fence_handle = VK_NULL_HANDLE,
};
/* lazily create sparse semaphore */
if (queue->sparse_semaphore == VK_NULL_HANDLE) {
queue->sparse_semaphore_counter = 1;
const VkSemaphoreTypeCreateInfo sem_type_create_info = {
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
.pNext = NULL,
/* This must be timeline type to adhere to mesa's requirement
* not to mix binary semaphores with wait-before-signal.
*/
.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
.initialValue = 1,
};
const VkSemaphoreCreateInfo create_info = {
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
.pNext = &sem_type_create_info,
.flags = 0,
};
result = vn_CreateSemaphore(dev_handle, &create_info, NULL,
&queue->sparse_semaphore);
if (result != VK_SUCCESS)
return result;
}
/* Setup VkTimelineSemaphoreSubmitInfo's for our queue sparse semaphore
* so that the vkQueueSubmit waits on the vkQueueBindSparse signal.
*/
queue->sparse_semaphore_counter++;
struct VkTimelineSemaphoreSubmitInfo wait_timeline_sem_info = { 0 };
wait_timeline_sem_info.sType =
VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO;
wait_timeline_sem_info.signalSemaphoreValueCount = 1;
wait_timeline_sem_info.pSignalSemaphoreValues =
&queue->sparse_semaphore_counter;
struct VkTimelineSemaphoreSubmitInfo signal_timeline_sem_info = { 0 };
signal_timeline_sem_info.sType =
VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO;
signal_timeline_sem_info.waitSemaphoreValueCount = 1;
signal_timeline_sem_info.pWaitSemaphoreValues =
&queue->sparse_semaphore_counter;
/* Split up the original wait and signal semaphores into its respective
* vkTimelineSemaphoreSubmitInfo
*/
const struct VkTimelineSemaphoreSubmitInfo *timeline_sem_info =
vk_find_struct_const(sparse_info->pNext,
TIMELINE_SEMAPHORE_SUBMIT_INFO);
if (timeline_sem_info) {
if (timeline_sem_info->waitSemaphoreValueCount) {
wait_timeline_sem_info.waitSemaphoreValueCount =
timeline_sem_info->waitSemaphoreValueCount;
wait_timeline_sem_info.pWaitSemaphoreValues =
timeline_sem_info->pWaitSemaphoreValues;
}
if (timeline_sem_info->signalSemaphoreValueCount) {
signal_timeline_sem_info.signalSemaphoreValueCount =
timeline_sem_info->signalSemaphoreValueCount;
signal_timeline_sem_info.pSignalSemaphoreValues =
timeline_sem_info->pSignalSemaphoreValues;
}
}
/* Attach the original VkDeviceGroupBindSparseInfo if it exists */
struct VkDeviceGroupBindSparseInfo batch_device_group_info;
const struct VkDeviceGroupBindSparseInfo *device_group_info =
vk_find_struct_const(sparse_info->pNext, DEVICE_GROUP_BIND_SPARSE_INFO);
if (device_group_info) {
memcpy(&batch_device_group_info, device_group_info,
sizeof(*device_group_info));
batch_device_group_info.pNext = NULL;
wait_timeline_sem_info.pNext = &batch_device_group_info;
}
/* Copy the original batch VkBindSparseInfo modified to signal
* our sparse semaphore.
*/
VkBindSparseInfo batch_sparse_info;
memcpy(&batch_sparse_info, sparse_info, sizeof(*sparse_info));
batch_sparse_info.pNext = &wait_timeline_sem_info;
batch_sparse_info.signalSemaphoreCount = 1;
batch_sparse_info.pSignalSemaphores = &queue->sparse_semaphore;
/* Set up the SubmitInfo to wait on our sparse semaphore before sending
* feedback and signaling the original semaphores/fence
*
* Even if this VkBindSparse batch does not have feedback semaphores,
* we still glue all the batches together to ensure the feedback
* fence occurs after.
*/
VkPipelineStageFlags stage_masks = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
VkSubmitInfo batch_submit_info = {
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
.pNext = &signal_timeline_sem_info,
.waitSemaphoreCount = 1,
.pWaitSemaphores = &queue->sparse_semaphore,
.pWaitDstStageMask = &stage_masks,
.signalSemaphoreCount = signal_sem_count,
.pSignalSemaphores = signal_sem,
};
/* Set the possible fence if on the last batch */
VkFence fence_handle = VK_NULL_HANDLE;
if ((submit->feedback_types & VN_FEEDBACK_TYPE_FENCE) &&
batch_index == (submit->batch_count - 1)) {
fence_handle = submit->fence_handle;
}
sparse_batch.sparse_batches = &batch_sparse_info;
result = vn_queue_bind_sparse_submit(&sparse_batch);
if (result != VK_SUCCESS)
return result;
result = vn_QueueSubmit(submit->queue_handle, 1, &batch_submit_info,
fence_handle);
if (result != VK_SUCCESS)
return result;
return VK_SUCCESS;
}
VkResult
vn_QueueBindSparse(VkQueue queue,
uint32_t bindInfoCount,
const VkBindSparseInfo *pBindInfo,
VkFence fence)
{
VN_TRACE_FUNC();
VkResult result;
struct vn_queue_submission submit = {
.batch_type = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO,
.queue_handle = queue,
.batch_count = bindInfoCount,
.sparse_batches = pBindInfo,
.fence_handle = fence,
};
result = vn_queue_submission_prepare(&submit);
if (result != VK_SUCCESS)
return result;
if (!submit.batch_count) {
/* skip no-op submit */
if (submit.fence_handle == VK_NULL_HANDLE)
return VK_SUCCESS;
/* if empty batch, just send a vkQueueSubmit with the fence */
result =
vn_QueueSubmit(submit.queue_handle, 0, NULL, submit.fence_handle);
if (result != VK_SUCCESS)
return result;
}
/* if feedback isn't used in the batch, can directly submit */
if (!submit.feedback_types)
return vn_queue_bind_sparse_submit(&submit);
for (uint32_t i = 0; i < submit.batch_count; i++) {
result = vn_queue_bind_sparse_submit_batch(&submit, i);
if (result != VK_SUCCESS)
return result;
}
return VK_SUCCESS;
}
VkResult
vn_QueueWaitIdle(VkQueue _queue)
{
VN_TRACE_FUNC();
struct vn_queue *queue = vn_queue_from_handle(_queue);
VkDevice dev_handle = vk_device_to_handle(queue->base.base.base.device);
struct vn_device *dev = vn_device_from_handle(dev_handle);
VkResult result;
/* lazily create queue wait fence for queue idle waiting */
if (queue->wait_fence == VK_NULL_HANDLE) {
const VkFenceCreateInfo create_info = {
.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO,
.flags = 0,
};
result =
vn_CreateFence(dev_handle, &create_info, NULL, &queue->wait_fence);
if (result != VK_SUCCESS)
return result;
}
result = vn_QueueSubmit(_queue, 0, NULL, queue->wait_fence);
if (result != VK_SUCCESS)
return result;
result =
vn_WaitForFences(dev_handle, 1, &queue->wait_fence, true, UINT64_MAX);
vn_ResetFences(dev_handle, 1, &queue->wait_fence);
return vn_result(dev->instance, result);
}
/* fence commands */
static void
vn_sync_payload_release(UNUSED struct vn_device *dev,
struct vn_sync_payload *payload)
{
if (payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD && payload->fd >= 0)
close(payload->fd);
payload->type = VN_SYNC_TYPE_INVALID;
}
static VkResult
vn_fence_init_payloads(struct vn_device *dev,
struct vn_fence *fence,
bool signaled,
const VkAllocationCallbacks *alloc)
{
fence->permanent.type = VN_SYNC_TYPE_DEVICE_ONLY;
fence->temporary.type = VN_SYNC_TYPE_INVALID;
fence->payload = &fence->permanent;
return VK_SUCCESS;
}
void
vn_fence_signal_wsi(struct vn_device *dev, struct vn_fence *fence)
{
struct vn_sync_payload *temp = &fence->temporary;
vn_sync_payload_release(dev, temp);
temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD;
temp->fd = -1;
fence->payload = temp;
}
static VkResult
vn_fence_feedback_init(struct vn_device *dev,
struct vn_fence *fence,
bool signaled,
const VkAllocationCallbacks *alloc)
{
VkDevice dev_handle = vn_device_to_handle(dev);
struct vn_feedback_slot *slot;
VkCommandBuffer *cmd_handles;
VkResult result;
if (fence->is_external)
return VK_SUCCESS;
if (VN_PERF(NO_FENCE_FEEDBACK))
return VK_SUCCESS;
slot = vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_FENCE);
if (!slot)
return VK_ERROR_OUT_OF_HOST_MEMORY;
vn_feedback_set_status(slot, signaled ? VK_SUCCESS : VK_NOT_READY);
cmd_handles =
vk_zalloc(alloc, sizeof(*cmd_handles) * dev->queue_family_count,
VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!cmd_handles) {
vn_feedback_pool_free(&dev->feedback_pool, slot);
return VK_ERROR_OUT_OF_HOST_MEMORY;
}
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
result = vn_feedback_cmd_alloc(dev_handle, &dev->fb_cmd_pools[i], slot,
NULL, &cmd_handles[i]);
if (result != VK_SUCCESS) {
for (uint32_t j = 0; j < i; j++) {
vn_feedback_cmd_free(dev_handle, &dev->fb_cmd_pools[j],
cmd_handles[j]);
}
break;
}
}
if (result != VK_SUCCESS) {
vk_free(alloc, cmd_handles);
vn_feedback_pool_free(&dev->feedback_pool, slot);
return result;
}
fence->feedback.slot = slot;
fence->feedback.commands = cmd_handles;
return VK_SUCCESS;
}
static void
vn_fence_feedback_fini(struct vn_device *dev,
struct vn_fence *fence,
const VkAllocationCallbacks *alloc)
{
VkDevice dev_handle = vn_device_to_handle(dev);
if (!fence->feedback.slot)
return;
for (uint32_t i = 0; i < dev->queue_family_count; i++) {
vn_feedback_cmd_free(dev_handle, &dev->fb_cmd_pools[i],
fence->feedback.commands[i]);
}
vn_feedback_pool_free(&dev->feedback_pool, fence->feedback.slot);
vk_free(alloc, fence->feedback.commands);
}
VkResult
vn_CreateFence(VkDevice device,
const VkFenceCreateInfo *pCreateInfo,
const VkAllocationCallbacks *pAllocator,
VkFence *pFence)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
const VkAllocationCallbacks *alloc =
pAllocator ? pAllocator : &dev->base.base.alloc;
const bool signaled = pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT;
VkResult result;
struct vn_fence *fence = vk_zalloc(alloc, sizeof(*fence), VN_DEFAULT_ALIGN,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!fence)
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
vn_object_base_init(&fence->base, VK_OBJECT_TYPE_FENCE, &dev->base);
const struct VkExportFenceCreateInfo *export_info =
vk_find_struct_const(pCreateInfo->pNext, EXPORT_FENCE_CREATE_INFO);
fence->is_external = export_info && export_info->handleTypes;
result = vn_fence_init_payloads(dev, fence, signaled, alloc);
if (result != VK_SUCCESS)
goto out_object_base_fini;
result = vn_fence_feedback_init(dev, fence, signaled, alloc);
if (result != VK_SUCCESS)
goto out_payloads_fini;
*pFence = vn_fence_to_handle(fence);
vn_async_vkCreateFence(dev->primary_ring, device, pCreateInfo, NULL,
pFence);
return VK_SUCCESS;
out_payloads_fini:
vn_sync_payload_release(dev, &fence->permanent);
vn_sync_payload_release(dev, &fence->temporary);
out_object_base_fini:
vn_object_base_fini(&fence->base);
vk_free(alloc, fence);
return vn_error(dev->instance, result);
}
void
vn_DestroyFence(VkDevice device,
VkFence _fence,
const VkAllocationCallbacks *pAllocator)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_fence *fence = vn_fence_from_handle(_fence);
const VkAllocationCallbacks *alloc =
pAllocator ? pAllocator : &dev->base.base.alloc;
if (!fence)
return;
vn_async_vkDestroyFence(dev->primary_ring, device, _fence, NULL);
vn_fence_feedback_fini(dev, fence, alloc);
vn_sync_payload_release(dev, &fence->permanent);
vn_sync_payload_release(dev, &fence->temporary);
vn_object_base_fini(&fence->base);
vk_free(alloc, fence);
}
VkResult
vn_ResetFences(VkDevice device, uint32_t fenceCount, const VkFence *pFences)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
/* TODO if the fence is shared-by-ref, this needs to be synchronous */
if (false)
vn_call_vkResetFences(dev->primary_ring, device, fenceCount, pFences);
else
vn_async_vkResetFences(dev->primary_ring, device, fenceCount, pFences);
for (uint32_t i = 0; i < fenceCount; i++) {
struct vn_fence *fence = vn_fence_from_handle(pFences[i]);
struct vn_sync_payload *perm = &fence->permanent;
vn_sync_payload_release(dev, &fence->temporary);
assert(perm->type == VN_SYNC_TYPE_DEVICE_ONLY);
fence->payload = perm;
if (fence->feedback.slot)
vn_feedback_reset_status(fence->feedback.slot);
}
return VK_SUCCESS;
}
VkResult
vn_GetFenceStatus(VkDevice device, VkFence _fence)
{
struct vn_device *dev = vn_device_from_handle(device);
struct vn_fence *fence = vn_fence_from_handle(_fence);
struct vn_sync_payload *payload = fence->payload;
VkResult result;
switch (payload->type) {
case VN_SYNC_TYPE_DEVICE_ONLY:
if (fence->feedback.slot) {
result = vn_feedback_get_status(fence->feedback.slot);
if (result == VK_SUCCESS) {
/* When fence feedback slot gets signaled, the real fence
* signal operation follows after but the signaling isr can be
* deferred or preempted. To avoid racing, we let the
* renderer wait for the fence. This also helps resolve
* synchronization validation errors, because the layer no
* longer sees any fence status checks and falsely believes the
* caller does not sync.
*/
vn_async_vkWaitForFences(dev->primary_ring, device, 1, &_fence,
VK_TRUE, UINT64_MAX);
}
} else {
result = vn_call_vkGetFenceStatus(dev->primary_ring, device, _fence);
}
break;
case VN_SYNC_TYPE_IMPORTED_SYNC_FD:
if (payload->fd < 0 || sync_wait(payload->fd, 0) == 0)
result = VK_SUCCESS;
else
result = errno == ETIME ? VK_NOT_READY : VK_ERROR_DEVICE_LOST;
break;
default:
unreachable("unexpected fence payload type");
break;
}
return vn_result(dev->instance, result);
}
static VkResult
vn_find_first_signaled_fence(VkDevice device,
const VkFence *fences,
uint32_t count)
{
for (uint32_t i = 0; i < count; i++) {
VkResult result = vn_GetFenceStatus(device, fences[i]);
if (result == VK_SUCCESS || result < 0)
return result;
}
return VK_NOT_READY;
}
static VkResult
vn_remove_signaled_fences(VkDevice device, VkFence *fences, uint32_t *count)
{
uint32_t cur = 0;
for (uint32_t i = 0; i < *count; i++) {
VkResult result = vn_GetFenceStatus(device, fences[i]);
if (result != VK_SUCCESS) {
if (result < 0)
return result;
fences[cur++] = fences[i];
}
}
*count = cur;
return cur ? VK_NOT_READY : VK_SUCCESS;
}
static VkResult
vn_update_sync_result(struct vn_device *dev,
VkResult result,
int64_t abs_timeout,
struct vn_relax_state *relax_state)
{
switch (result) {
case VK_NOT_READY:
if (abs_timeout != OS_TIMEOUT_INFINITE &&
os_time_get_nano() >= abs_timeout)
result = VK_TIMEOUT;
else
vn_relax(relax_state);
break;
default:
assert(result == VK_SUCCESS || result < 0);
break;
}
return result;
}
VkResult
vn_WaitForFences(VkDevice device,
uint32_t fenceCount,
const VkFence *pFences,
VkBool32 waitAll,
uint64_t timeout)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
const int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
VkResult result = VK_NOT_READY;
if (fenceCount > 1 && waitAll) {
VkFence local_fences[8];
VkFence *fences = local_fences;
if (fenceCount > ARRAY_SIZE(local_fences)) {
fences =
vk_alloc(alloc, sizeof(*fences) * fenceCount, VN_DEFAULT_ALIGN,
VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
if (!fences)
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
}
memcpy(fences, pFences, sizeof(*fences) * fenceCount);
struct vn_relax_state relax_state =
vn_relax_init(dev->instance, "client");
while (result == VK_NOT_READY) {
result = vn_remove_signaled_fences(device, fences, &fenceCount);
result =
vn_update_sync_result(dev, result, abs_timeout, &relax_state);
}
vn_relax_fini(&relax_state);
if (fences != local_fences)
vk_free(alloc, fences);
} else {
struct vn_relax_state relax_state =
vn_relax_init(dev->instance, "client");
while (result == VK_NOT_READY) {
result = vn_find_first_signaled_fence(device, pFences, fenceCount);
result =
vn_update_sync_result(dev, result, abs_timeout, &relax_state);
}
vn_relax_fini(&relax_state);
}
return vn_result(dev->instance, result);
}
static VkResult
vn_create_sync_file(struct vn_device *dev,
struct vn_sync_payload_external *external_payload,
int *out_fd)
{
struct vn_renderer_sync *sync;
VkResult result = vn_renderer_sync_create(dev->renderer, 0,
VN_RENDERER_SYNC_BINARY, &sync);
if (result != VK_SUCCESS)
return vn_error(dev->instance, result);
struct vn_renderer_submit_batch batch = {
.syncs = &sync,
.sync_values = &(const uint64_t){ 1 },
.sync_count = 1,
.ring_idx = external_payload->ring_idx,
};
uint32_t local_data[8];
struct vn_cs_encoder local_enc =
VN_CS_ENCODER_INITIALIZER_LOCAL(local_data, sizeof(local_data));
if (external_payload->ring_seqno_valid) {
const uint64_t ring_id = vn_ring_get_id(dev->primary_ring);
vn_encode_vkWaitRingSeqnoMESA(&local_enc, 0, ring_id,
external_payload->ring_seqno);
batch.cs_data = local_data;
batch.cs_size = vn_cs_encoder_get_len(&local_enc);
}
const struct vn_renderer_submit submit = {
.batches = &batch,
.batch_count = 1,
};
result = vn_renderer_submit(dev->renderer, &submit);
if (result != VK_SUCCESS) {
vn_renderer_sync_destroy(dev->renderer, sync);
return vn_error(dev->instance, result);
}
*out_fd = vn_renderer_sync_export_syncobj(dev->renderer, sync, true);
vn_renderer_sync_destroy(dev->renderer, sync);
return *out_fd >= 0 ? VK_SUCCESS : VK_ERROR_TOO_MANY_OBJECTS;
}
static inline bool
vn_sync_valid_fd(int fd)
{
/* the special value -1 for fd is treated like a valid sync file descriptor
* referring to an object that has already signaled
*/
return (fd >= 0 && sync_valid_fd(fd)) || fd == -1;
}
VkResult
vn_ImportFenceFdKHR(VkDevice device,
const VkImportFenceFdInfoKHR *pImportFenceFdInfo)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_fence *fence = vn_fence_from_handle(pImportFenceFdInfo->fence);
ASSERTED const bool sync_file = pImportFenceFdInfo->handleType ==
VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
const int fd = pImportFenceFdInfo->fd;
assert(sync_file);
if (!vn_sync_valid_fd(fd))
return vn_error(dev->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
struct vn_sync_payload *temp = &fence->temporary;
vn_sync_payload_release(dev, temp);
temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD;
temp->fd = fd;
fence->payload = temp;
return VK_SUCCESS;
}
VkResult
vn_GetFenceFdKHR(VkDevice device,
const VkFenceGetFdInfoKHR *pGetFdInfo,
int *pFd)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_fence *fence = vn_fence_from_handle(pGetFdInfo->fence);
const bool sync_file =
pGetFdInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
struct vn_sync_payload *payload = fence->payload;
VkResult result;
assert(sync_file);
assert(dev->physical_device->renderer_sync_fd.fence_exportable);
int fd = -1;
if (payload->type == VN_SYNC_TYPE_DEVICE_ONLY) {
result = vn_create_sync_file(dev, &fence->external_payload, &fd);
if (result != VK_SUCCESS)
return vn_error(dev->instance, result);
vn_async_vkResetFenceResourceMESA(dev->primary_ring, device,
pGetFdInfo->fence);
vn_sync_payload_release(dev, &fence->temporary);
fence->payload = &fence->permanent;
#ifdef VN_USE_WSI_PLATFORM
if (!dev->renderer->info.has_implicit_fencing)
sync_wait(fd, -1);
#endif
} else {
assert(payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD);
/* transfer ownership of imported sync fd to save a dup */
fd = payload->fd;
payload->fd = -1;
/* reset host fence in case in signaled state before import */
result = vn_ResetFences(device, 1, &pGetFdInfo->fence);
if (result != VK_SUCCESS) {
/* transfer sync fd ownership back on error */
payload->fd = fd;
return result;
}
}
*pFd = fd;
return VK_SUCCESS;
}
/* semaphore commands */
static VkResult
vn_semaphore_init_payloads(struct vn_device *dev,
struct vn_semaphore *sem,
uint64_t initial_val,
const VkAllocationCallbacks *alloc)
{
sem->permanent.type = VN_SYNC_TYPE_DEVICE_ONLY;
sem->temporary.type = VN_SYNC_TYPE_INVALID;
sem->payload = &sem->permanent;
return VK_SUCCESS;
}
static bool
vn_semaphore_wait_external(struct vn_device *dev, struct vn_semaphore *sem)
{
struct vn_sync_payload *temp = &sem->temporary;
assert(temp->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD);
if (temp->fd >= 0) {
if (sync_wait(temp->fd, -1))
return false;
}
vn_sync_payload_release(dev, &sem->temporary);
sem->payload = &sem->permanent;
return true;
}
void
vn_semaphore_signal_wsi(struct vn_device *dev, struct vn_semaphore *sem)
{
struct vn_sync_payload *temp = &sem->temporary;
vn_sync_payload_release(dev, temp);
temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD;
temp->fd = -1;
sem->payload = temp;
}
struct vn_semaphore_feedback_cmd *
vn_semaphore_get_feedback_cmd(struct vn_device *dev, struct vn_semaphore *sem)
{
struct vn_semaphore_feedback_cmd *sfb_cmd = NULL;
simple_mtx_lock(&sem->feedback.cmd_mtx);
if (!list_is_empty(&sem->feedback.free_cmds)) {
sfb_cmd = list_first_entry(&sem->feedback.free_cmds,
struct vn_semaphore_feedback_cmd, head);
list_move_to(&sfb_cmd->head, &sem->feedback.pending_cmds);
}
simple_mtx_unlock(&sem->feedback.cmd_mtx);
if (!sfb_cmd) {
sfb_cmd = vn_semaphore_feedback_cmd_alloc(dev, sem->feedback.slot);
simple_mtx_lock(&sem->feedback.cmd_mtx);
list_add(&sfb_cmd->head, &sem->feedback.pending_cmds);
simple_mtx_unlock(&sem->feedback.cmd_mtx);
}
return sfb_cmd;
}
static VkResult
vn_semaphore_feedback_init(struct vn_device *dev,
struct vn_semaphore *sem,
uint64_t initial_value,
const VkAllocationCallbacks *alloc)
{
struct vn_feedback_slot *slot;
assert(sem->type == VK_SEMAPHORE_TYPE_TIMELINE);
if (sem->is_external)
return VK_SUCCESS;
if (VN_PERF(NO_SEMAPHORE_FEEDBACK))
return VK_SUCCESS;
slot =
vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_SEMAPHORE);
if (!slot)
return VK_ERROR_OUT_OF_HOST_MEMORY;
list_inithead(&sem->feedback.pending_cmds);
list_inithead(&sem->feedback.free_cmds);
vn_feedback_set_counter(slot, initial_value);
simple_mtx_init(&sem->feedback.cmd_mtx, mtx_plain);
simple_mtx_init(&sem->feedback.async_wait_mtx, mtx_plain);
sem->feedback.signaled_counter = initial_value;
sem->feedback.slot = slot;
return VK_SUCCESS;
}
static void
vn_semaphore_feedback_fini(struct vn_device *dev, struct vn_semaphore *sem)
{
if (!sem->feedback.slot)
return;
list_for_each_entry_safe(struct vn_semaphore_feedback_cmd, sfb_cmd,
&sem->feedback.free_cmds, head)
vn_semaphore_feedback_cmd_free(dev, sfb_cmd);
list_for_each_entry_safe(struct vn_semaphore_feedback_cmd, sfb_cmd,
&sem->feedback.pending_cmds, head)
vn_semaphore_feedback_cmd_free(dev, sfb_cmd);
simple_mtx_destroy(&sem->feedback.cmd_mtx);
simple_mtx_destroy(&sem->feedback.async_wait_mtx);
vn_feedback_pool_free(&dev->feedback_pool, sem->feedback.slot);
}
VkResult
vn_CreateSemaphore(VkDevice device,
const VkSemaphoreCreateInfo *pCreateInfo,
const VkAllocationCallbacks *pAllocator,
VkSemaphore *pSemaphore)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
const VkAllocationCallbacks *alloc =
pAllocator ? pAllocator : &dev->base.base.alloc;
struct vn_semaphore *sem = vk_zalloc(alloc, sizeof(*sem), VN_DEFAULT_ALIGN,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!sem)
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
vn_object_base_init(&sem->base, VK_OBJECT_TYPE_SEMAPHORE, &dev->base);
const VkSemaphoreTypeCreateInfo *type_info =
vk_find_struct_const(pCreateInfo->pNext, SEMAPHORE_TYPE_CREATE_INFO);
uint64_t initial_val = 0;
if (type_info && type_info->semaphoreType == VK_SEMAPHORE_TYPE_TIMELINE) {
sem->type = VK_SEMAPHORE_TYPE_TIMELINE;
initial_val = type_info->initialValue;
} else {
sem->type = VK_SEMAPHORE_TYPE_BINARY;
}
const struct VkExportSemaphoreCreateInfo *export_info =
vk_find_struct_const(pCreateInfo->pNext, EXPORT_SEMAPHORE_CREATE_INFO);
sem->is_external = export_info && export_info->handleTypes;
VkResult result = vn_semaphore_init_payloads(dev, sem, initial_val, alloc);
if (result != VK_SUCCESS)
goto out_object_base_fini;
if (sem->type == VK_SEMAPHORE_TYPE_TIMELINE) {
result = vn_semaphore_feedback_init(dev, sem, initial_val, alloc);
if (result != VK_SUCCESS)
goto out_payloads_fini;
}
VkSemaphore sem_handle = vn_semaphore_to_handle(sem);
vn_async_vkCreateSemaphore(dev->primary_ring, device, pCreateInfo, NULL,
&sem_handle);
*pSemaphore = sem_handle;
return VK_SUCCESS;
out_payloads_fini:
vn_sync_payload_release(dev, &sem->permanent);
vn_sync_payload_release(dev, &sem->temporary);
out_object_base_fini:
vn_object_base_fini(&sem->base);
vk_free(alloc, sem);
return vn_error(dev->instance, result);
}
void
vn_DestroySemaphore(VkDevice device,
VkSemaphore semaphore,
const VkAllocationCallbacks *pAllocator)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_semaphore *sem = vn_semaphore_from_handle(semaphore);
const VkAllocationCallbacks *alloc =
pAllocator ? pAllocator : &dev->base.base.alloc;
if (!sem)
return;
vn_async_vkDestroySemaphore(dev->primary_ring, device, semaphore, NULL);
if (sem->type == VK_SEMAPHORE_TYPE_TIMELINE)
vn_semaphore_feedback_fini(dev, sem);
vn_sync_payload_release(dev, &sem->permanent);
vn_sync_payload_release(dev, &sem->temporary);
vn_object_base_fini(&sem->base);
vk_free(alloc, sem);
}
VkResult
vn_GetSemaphoreCounterValue(VkDevice device,
VkSemaphore semaphore,
uint64_t *pValue)
{
struct vn_device *dev = vn_device_from_handle(device);
struct vn_semaphore *sem = vn_semaphore_from_handle(semaphore);
ASSERTED struct vn_sync_payload *payload = sem->payload;
assert(payload->type == VN_SYNC_TYPE_DEVICE_ONLY);
if (sem->feedback.slot) {
simple_mtx_lock(&sem->feedback.async_wait_mtx);
const uint64_t counter = vn_feedback_get_counter(sem->feedback.slot);
if (sem->feedback.signaled_counter < counter) {
/* When the timeline semaphore feedback slot gets signaled, the real
* semaphore signal operation follows after but the signaling isr can
* be deferred or preempted. To avoid racing, we let the renderer
* wait for the semaphore by sending an asynchronous wait call for
* the feedback value.
* We also cache the counter value to only send the async call once
* per counter value to prevent spamming redundant async wait calls.
* The cached counter value requires a lock to ensure multiple
* threads querying for the same value are guaranteed to encode after
* the async wait call.
*
* This also helps resolve synchronization validation errors, because
* the layer no longer sees any semaphore status checks and falsely
* believes the caller does not sync.
*/
VkSemaphoreWaitInfo wait_info = {
.sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
.pNext = NULL,
.flags = 0,
.semaphoreCount = 1,
.pSemaphores = &semaphore,
.pValues = &counter,
};
vn_async_vkWaitSemaphores(dev->primary_ring, device, &wait_info,
UINT64_MAX);
/* search pending cmds for already signaled values */
simple_mtx_lock(&sem->feedback.cmd_mtx);
list_for_each_entry_safe(struct vn_semaphore_feedback_cmd, sfb_cmd,
&sem->feedback.pending_cmds, head) {
if (counter >= vn_feedback_get_counter(sfb_cmd->src_slot))
list_move_to(&sfb_cmd->head, &sem->feedback.free_cmds);
}
simple_mtx_unlock(&sem->feedback.cmd_mtx);
sem->feedback.signaled_counter = counter;
}
simple_mtx_unlock(&sem->feedback.async_wait_mtx);
*pValue = counter;
return VK_SUCCESS;
} else {
return vn_call_vkGetSemaphoreCounterValue(dev->primary_ring, device,
semaphore, pValue);
}
}
VkResult
vn_SignalSemaphore(VkDevice device, const VkSemaphoreSignalInfo *pSignalInfo)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_semaphore *sem =
vn_semaphore_from_handle(pSignalInfo->semaphore);
/* TODO if the semaphore is shared-by-ref, this needs to be synchronous */
if (false)
vn_call_vkSignalSemaphore(dev->primary_ring, device, pSignalInfo);
else
vn_async_vkSignalSemaphore(dev->primary_ring, device, pSignalInfo);
if (sem->feedback.slot) {
simple_mtx_lock(&sem->feedback.async_wait_mtx);
vn_feedback_set_counter(sem->feedback.slot, pSignalInfo->value);
/* Update async counters. Since we're signaling, we're aligned with
* the renderer.
*/
sem->feedback.signaled_counter = pSignalInfo->value;
simple_mtx_unlock(&sem->feedback.async_wait_mtx);
}
return VK_SUCCESS;
}
static VkResult
vn_find_first_signaled_semaphore(VkDevice device,
const VkSemaphore *semaphores,
const uint64_t *values,
uint32_t count)
{
for (uint32_t i = 0; i < count; i++) {
uint64_t val = 0;
VkResult result =
vn_GetSemaphoreCounterValue(device, semaphores[i], &val);
if (result != VK_SUCCESS || val >= values[i])
return result;
}
return VK_NOT_READY;
}
static VkResult
vn_remove_signaled_semaphores(VkDevice device,
VkSemaphore *semaphores,
uint64_t *values,
uint32_t *count)
{
uint32_t cur = 0;
for (uint32_t i = 0; i < *count; i++) {
uint64_t val = 0;
VkResult result =
vn_GetSemaphoreCounterValue(device, semaphores[i], &val);
if (result != VK_SUCCESS)
return result;
if (val < values[i])
semaphores[cur++] = semaphores[i];
}
*count = cur;
return cur ? VK_NOT_READY : VK_SUCCESS;
}
VkResult
vn_WaitSemaphores(VkDevice device,
const VkSemaphoreWaitInfo *pWaitInfo,
uint64_t timeout)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
const int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
VkResult result = VK_NOT_READY;
if (pWaitInfo->semaphoreCount > 1 &&
!(pWaitInfo->flags & VK_SEMAPHORE_WAIT_ANY_BIT)) {
uint32_t semaphore_count = pWaitInfo->semaphoreCount;
VkSemaphore local_semaphores[8];
uint64_t local_values[8];
VkSemaphore *semaphores = local_semaphores;
uint64_t *values = local_values;
if (semaphore_count > ARRAY_SIZE(local_semaphores)) {
semaphores = vk_alloc(
alloc, (sizeof(*semaphores) + sizeof(*values)) * semaphore_count,
VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
if (!semaphores)
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
values = (uint64_t *)&semaphores[semaphore_count];
}
memcpy(semaphores, pWaitInfo->pSemaphores,
sizeof(*semaphores) * semaphore_count);
memcpy(values, pWaitInfo->pValues, sizeof(*values) * semaphore_count);
struct vn_relax_state relax_state =
vn_relax_init(dev->instance, "client");
while (result == VK_NOT_READY) {
result = vn_remove_signaled_semaphores(device, semaphores, values,
&semaphore_count);
result =
vn_update_sync_result(dev, result, abs_timeout, &relax_state);
}
vn_relax_fini(&relax_state);
if (semaphores != local_semaphores)
vk_free(alloc, semaphores);
} else {
struct vn_relax_state relax_state =
vn_relax_init(dev->instance, "client");
while (result == VK_NOT_READY) {
result = vn_find_first_signaled_semaphore(
device, pWaitInfo->pSemaphores, pWaitInfo->pValues,
pWaitInfo->semaphoreCount);
result =
vn_update_sync_result(dev, result, abs_timeout, &relax_state);
}
vn_relax_fini(&relax_state);
}
return vn_result(dev->instance, result);
}
VkResult
vn_ImportSemaphoreFdKHR(
VkDevice device, const VkImportSemaphoreFdInfoKHR *pImportSemaphoreFdInfo)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_semaphore *sem =
vn_semaphore_from_handle(pImportSemaphoreFdInfo->semaphore);
ASSERTED const bool sync_file =
pImportSemaphoreFdInfo->handleType ==
VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
const int fd = pImportSemaphoreFdInfo->fd;
assert(sync_file);
if (!vn_sync_valid_fd(fd))
return vn_error(dev->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
struct vn_sync_payload *temp = &sem->temporary;
vn_sync_payload_release(dev, temp);
temp->type = VN_SYNC_TYPE_IMPORTED_SYNC_FD;
temp->fd = fd;
sem->payload = temp;
return VK_SUCCESS;
}
VkResult
vn_GetSemaphoreFdKHR(VkDevice device,
const VkSemaphoreGetFdInfoKHR *pGetFdInfo,
int *pFd)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_semaphore *sem = vn_semaphore_from_handle(pGetFdInfo->semaphore);
const bool sync_file =
pGetFdInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
struct vn_sync_payload *payload = sem->payload;
assert(sync_file);
assert(dev->physical_device->renderer_sync_fd.semaphore_exportable);
assert(dev->physical_device->renderer_sync_fd.semaphore_importable);
int fd = -1;
if (payload->type == VN_SYNC_TYPE_DEVICE_ONLY) {
VkResult result = vn_create_sync_file(dev, &sem->external_payload, &fd);
if (result != VK_SUCCESS)
return vn_error(dev->instance, result);
#ifdef VN_USE_WSI_PLATFORM
if (!dev->renderer->info.has_implicit_fencing)
sync_wait(fd, -1);
#endif
} else {
assert(payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD);
/* transfer ownership of imported sync fd to save a dup */
fd = payload->fd;
payload->fd = -1;
}
/* When payload->type is VN_SYNC_TYPE_IMPORTED_SYNC_FD, the current
* payload is from a prior temporary sync_fd import. The permanent
* payload of the sempahore might be in signaled state. So we do an
* import here to ensure later wait operation is legit. With resourceId
* 0, renderer does a signaled sync_fd -1 payload import on the host
* semaphore.
*/
if (payload->type == VN_SYNC_TYPE_IMPORTED_SYNC_FD) {
const VkImportSemaphoreResourceInfoMESA res_info = {
.sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_RESOURCE_INFO_MESA,
.semaphore = pGetFdInfo->semaphore,
.resourceId = 0,
};
vn_async_vkImportSemaphoreResourceMESA(dev->primary_ring, device,
&res_info);
}
/* perform wait operation on the host semaphore */
vn_async_vkWaitSemaphoreResourceMESA(dev->primary_ring, device,
pGetFdInfo->semaphore);
vn_sync_payload_release(dev, &sem->temporary);
sem->payload = &sem->permanent;
*pFd = fd;
return VK_SUCCESS;
}
/* event commands */
static VkResult
vn_event_feedback_init(struct vn_device *dev, struct vn_event *ev)
{
struct vn_feedback_slot *slot;
if (VN_PERF(NO_EVENT_FEEDBACK))
return VK_SUCCESS;
slot = vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_EVENT);
if (!slot)
return VK_ERROR_OUT_OF_HOST_MEMORY;
/* newly created event object is in the unsignaled state */
vn_feedback_set_status(slot, VK_EVENT_RESET);
ev->feedback_slot = slot;
return VK_SUCCESS;
}
static inline void
vn_event_feedback_fini(struct vn_device *dev, struct vn_event *ev)
{
if (ev->feedback_slot)
vn_feedback_pool_free(&dev->feedback_pool, ev->feedback_slot);
}
VkResult
vn_CreateEvent(VkDevice device,
const VkEventCreateInfo *pCreateInfo,
const VkAllocationCallbacks *pAllocator,
VkEvent *pEvent)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
const VkAllocationCallbacks *alloc =
pAllocator ? pAllocator : &dev->base.base.alloc;
struct vn_event *ev = vk_zalloc(alloc, sizeof(*ev), VN_DEFAULT_ALIGN,
VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
if (!ev)
return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
vn_object_base_init(&ev->base, VK_OBJECT_TYPE_EVENT, &dev->base);
/* feedback is only needed to speed up host operations */
if (!(pCreateInfo->flags & VK_EVENT_CREATE_DEVICE_ONLY_BIT)) {
VkResult result = vn_event_feedback_init(dev, ev);
if (result != VK_SUCCESS)
return vn_error(dev->instance, result);
}
VkEvent ev_handle = vn_event_to_handle(ev);
vn_async_vkCreateEvent(dev->primary_ring, device, pCreateInfo, NULL,
&ev_handle);
*pEvent = ev_handle;
return VK_SUCCESS;
}
void
vn_DestroyEvent(VkDevice device,
VkEvent event,
const VkAllocationCallbacks *pAllocator)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_event *ev = vn_event_from_handle(event);
const VkAllocationCallbacks *alloc =
pAllocator ? pAllocator : &dev->base.base.alloc;
if (!ev)
return;
vn_async_vkDestroyEvent(dev->primary_ring, device, event, NULL);
vn_event_feedback_fini(dev, ev);
vn_object_base_fini(&ev->base);
vk_free(alloc, ev);
}
VkResult
vn_GetEventStatus(VkDevice device, VkEvent event)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_event *ev = vn_event_from_handle(event);
VkResult result;
if (ev->feedback_slot)
result = vn_feedback_get_status(ev->feedback_slot);
else
result = vn_call_vkGetEventStatus(dev->primary_ring, device, event);
return vn_result(dev->instance, result);
}
VkResult
vn_SetEvent(VkDevice device, VkEvent event)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_event *ev = vn_event_from_handle(event);
if (ev->feedback_slot) {
vn_feedback_set_status(ev->feedback_slot, VK_EVENT_SET);
vn_async_vkSetEvent(dev->primary_ring, device, event);
} else {
VkResult result = vn_call_vkSetEvent(dev->primary_ring, device, event);
if (result != VK_SUCCESS)
return vn_error(dev->instance, result);
}
return VK_SUCCESS;
}
VkResult
vn_ResetEvent(VkDevice device, VkEvent event)
{
VN_TRACE_FUNC();
struct vn_device *dev = vn_device_from_handle(device);
struct vn_event *ev = vn_event_from_handle(event);
if (ev->feedback_slot) {
vn_feedback_reset_status(ev->feedback_slot);
vn_async_vkResetEvent(dev->primary_ring, device, event);
} else {
VkResult result =
vn_call_vkResetEvent(dev->primary_ring, device, event);
if (result != VK_SUCCESS)
return vn_error(dev->instance, result);
}
return VK_SUCCESS;
}