blob: 5d375c466737a7528020487a30189f34210378cb [file] [edit]
/*
* Copyright © 2019 Raspberry Pi Ltd
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice (including the next
* paragraph) shall be included in all copies or substantial portions of the
* Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include "drm.h"
#include "v3dv_device.h"
#include "v3dv_cmd_buffer.h"
#include "v3dv_image.h"
#include "v3dv_entrypoints.h"
#include "v3dv_tracepoints.h"
#include "v3dv_version_dispatch.h"
#include <xf86drm.h>
#include "broadcom/clif/clif_dump.h"
#include "broadcom/common/v3d_submit_util.h"
#include "util/libsync.h"
#include "util/perf/cpu_trace.h"
#include "util/perf/u_trace.h"
#include "vulkan/vulkan_core.h"
#include "vk_drm_syncobj.h"
#include <time.h>
static void
v3dv_clif_dump(struct v3dv_device *device,
struct v3dv_job *job,
struct drm_v3d_submit_cl *submit)
{
if (!(V3D_DBG(CL) ||
V3D_DBG(CL_NO_BIN) ||
V3D_DBG(CLIF)))
return;
struct log_stream *stream = mesa_log_streami();
struct clif_dump *clif = clif_dump_init(&device->devinfo,
stream,
V3D_DBG(CL) ||
V3D_DBG(CL_NO_BIN),
V3D_DBG(CL_NO_BIN));
set_foreach(job->bos, entry) {
struct v3dv_bo *bo = (void *)entry->key;
char *name = ralloc_asprintf(NULL, "%s_0x%x",
bo->name, bo->offset);
bool ok = v3dv_bo_map(device, bo, bo->size);
if (!ok) {
mesa_loge("failed to map BO for clif_dump.\n");
ralloc_free(name);
goto free_clif;
}
clif_dump_add_bo(clif, name, bo->offset, bo->size, bo->map);
ralloc_free(name);
}
clif_dump(clif, submit);
free_clif:
clif_dump_destroy(clif);
mesa_log_stream_destroy(stream);
}
static uint32_t
gather_in_syncs(struct v3dv_queue *queue,
struct v3dv_job *job,
enum v3dv_queue_type queue_sync,
struct vk_sync_wait *waits,
unsigned wait_count,
struct v3dv_submit_sync_info *sync_info,
uint32_t *handles)
{
uint32_t n_syncs = 0;
uint32_t idx = 0;
/* If this is the first job submitted to a given GPU queue in this cmd buf
* batch, it has to wait on wait semaphores (if any) before running.
*/
if (queue->last_job_syncs.first[queue_sync])
n_syncs = sync_info->wait_count;
/* If the serialize flag is set the job needs to be serialized in the
* corresponding queues. Notice that we may implement transfer operations
* as both CL or TFU jobs.
*
* FIXME: maybe we could track more precisely if the source of a transfer
* barrier is a CL and/or a TFU job.
*/
bool sync_csd = job->serialize & V3DV_BARRIER_COMPUTE_BIT;
bool sync_tfu = job->serialize & V3DV_BARRIER_TRANSFER_BIT;
bool sync_cl = job->serialize & (V3DV_BARRIER_GRAPHICS_BIT |
V3DV_BARRIER_TRANSFER_BIT);
bool sync_cpu = job->serialize & V3DV_BARRIER_CPU_BIT;
/* first job waits */
for (uint32_t i = 0; i < n_syncs; i++)
handles[idx++] = vk_sync_as_drm_syncobj(sync_info->waits[i].sync)->syncobj;
/* explicit call-site waits */
for (unsigned i = 0; i < wait_count; i++)
handles[idx++] = vk_sync_as_drm_syncobj(waits[i].sync)->syncobj;
/* internal serialization barriers */
if (sync_cl)
handles[idx++] = queue->last_job_syncs.syncs[V3DV_QUEUE_CL];
if (sync_csd)
handles[idx++] = queue->last_job_syncs.syncs[V3DV_QUEUE_CSD];
if (sync_tfu)
handles[idx++] = queue->last_job_syncs.syncs[V3DV_QUEUE_TFU];
if (sync_cpu)
handles[idx++] = queue->last_job_syncs.syncs[V3DV_QUEUE_CPU];
return idx;
}
/* helpers for multisync common code */
static void *
multisync_zalloc(void *mem_ctx, size_t size)
{
struct v3dv_device *device = mem_ctx;
return vk_zalloc(&device->vk.alloc, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
}
static void
multisync_free(void *mem_ctx, void *ptr)
{
struct v3dv_device *device = mem_ctx;
vk_free(&device->vk.alloc, ptr);
}
/* This function sets the extension for multiple in/out syncobjs. When it is
* successful, it sets the extension id to DRM_V3D_EXT_ID_MULTI_SYNC.
* Otherwise, the extension id is 0, which means an out-of-memory error.
*/
static bool
set_multisync(struct v3d_multisync *ms,
struct v3dv_submit_sync_info *sync_info,
struct vk_sync_wait *waits,
unsigned wait_count,
struct drm_v3d_extension *next,
struct v3dv_queue *queue,
struct v3dv_job *job,
enum v3dv_queue_type in_queue_sync,
enum v3dv_queue_type out_queue_sync,
enum v3d_queue wait_stage)
{
struct v3dv_device *device = queue->device;
bool ret = false;
/* Max input handles includes:
* - All API waits that apply on the first job we submit to each queue.
* - Any additional waits (i.e. pipeline barriers or dependencies between jobs)
* - All the last_syncs we track per queue (if we need to drain all queues for a barrier)
*/
uint32_t max_in_syncs = (sync_info ? sync_info->wait_count : 0) + wait_count +
V3DV_QUEUE_COUNT;
assert(ms);
ms->ops.zalloc = multisync_zalloc;
ms->ops.free = multisync_free;
ms->ops.mem_ctx = device;
STACK_ARRAY(uint32_t, in_sync_handles, max_in_syncs);
if (!in_sync_handles)
return false;
uint32_t total_in_syncs = gather_in_syncs(queue, job, in_queue_sync,
waits, wait_count, sync_info, in_sync_handles);
assert(total_in_syncs <= max_in_syncs);
/* We always signal the syncobj from `device->last_job_syncs` related to
* this v3dv_queue_type to track the last job submitted to this queue.
*/
uint32_t out_sync_handle = queue->last_job_syncs.syncs[out_queue_sync];
ret = v3d_multisync_init(ms, wait_stage,
in_sync_handles, total_in_syncs,
&out_sync_handle, 1, next);
if (!ret)
mesa_loge("Multisync Set Failed");
STACK_ARRAY_FINISH(in_sync_handles);
return ret;
}
static VkResult
handle_reset_query_cpu_job(struct v3dv_queue *queue,
struct v3dv_job *job,
struct v3dv_submit_sync_info *sync_info)
{
MESA_TRACE_FUNC();
struct v3dv_device *device = queue->device;
struct v3dv_reset_query_cpu_job_info *info = &job->cpu.query_reset;
assert(info->pool);
assert(info->pool->query_type != VK_QUERY_TYPE_OCCLUSION);
assert(info->first + info->count <= info->pool->query_count);
struct drm_v3d_submit_cpu submit = {0};
struct v3d_multisync ms = {0};
uint32_t *syncs = (uint32_t *) malloc(sizeof(uint32_t) * info->count);
uintptr_t *kperfmon_ids = NULL;
if (info->pool->query_type == VK_QUERY_TYPE_TIMESTAMP) {
submit.bo_handle_count = 1;
submit.bo_handles = (uintptr_t)(void *)&info->pool->timestamp.bo->handle;
struct drm_v3d_reset_timestamp_query reset = {0};
v3d_submit_ext_set(&reset.base, NULL, DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY, 0);
reset.count = info->count;
reset.offset = info->pool->queries[info->first].timestamp.offset;
for (uint32_t i = 0; i < info->count; i++) {
struct v3dv_query *query = &info->pool->queries[info->first + i];
syncs[i] = vk_sync_as_drm_syncobj(query->timestamp.sync)->syncobj;
}
reset.syncs = (uintptr_t)(void *)syncs;
if (!set_multisync(&ms, sync_info, NULL, 0, (void *)&reset, queue, job,
V3DV_QUEUE_CPU, V3DV_QUEUE_CPU, V3D_CPU)) {
free(syncs);
return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
}
} else {
assert(info->pool->query_type == VK_QUERY_TYPE_PERFORMANCE_QUERY_KHR);
struct drm_v3d_reset_performance_query reset = {0};
v3d_submit_ext_set(&reset.base, NULL, DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY, 0);
struct vk_sync_wait waits[info->count];
unsigned wait_count = 0;
for (int i = 0; i < info->count; i++) {
struct v3dv_query *query = &info->pool->queries[info->first + i];
/* Only wait for a query if we've used it otherwise we will be
* waiting forever for the fence to become signaled.
*/
if (query->maybe_available) {
waits[wait_count] = (struct vk_sync_wait){
.sync = query->perf.last_job_sync
};
wait_count++;
};
}
reset.count = info->count;
reset.nperfmons = info->pool->perfmon.nperfmons;
kperfmon_ids = (uintptr_t *) malloc(sizeof(uintptr_t) * info->count);
for (uint32_t i = 0; i < info->count; i++) {
struct v3dv_query *query = &info->pool->queries[info->first + i];
syncs[i] = vk_sync_as_drm_syncobj(query->perf.last_job_sync)->syncobj;
kperfmon_ids[i] = (uintptr_t)(void *)query->perf.kperfmon_ids;
}
reset.syncs = (uintptr_t)(void *)syncs;
reset.kperfmon_ids = (uintptr_t)(void *)kperfmon_ids;
if (!set_multisync(&ms, sync_info, waits, wait_count, (void *)&reset, queue, job,
V3DV_QUEUE_CPU, V3DV_QUEUE_CPU, V3D_CPU)) {
free(syncs);
free(kperfmon_ids);
return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
}
}
submit.flags |= DRM_V3D_SUBMIT_EXTENSION;
submit.extensions = (uintptr_t)(void *)&ms;
/* From the Vulkan spec for vkCmdResetQueryPool:
*
* "This command defines an execution dependency between other query commands
* that reference the same query.
* ...
* The second synchronization scope includes all commands which reference the
* queries in queryPool indicated by firstQuery and queryCount that occur later
* in submission order."
*
* This means we should ensure that any timestamps after a reset don't execute before
* the reset, however, for timestamps queries in particular we don't have to do
* anything special because timestamp queries have to wait for all previously
* submitted work to complete before executing (which we accomplish by using
* V3DV_BARRIER_ALL on them) and that includes reset jobs submitted to the CPU queue.
*/
int ret = v3d_ioctl(device->pdevice->render_fd,
DRM_IOCTL_V3D_SUBMIT_CPU, &submit);
free(syncs);
free(kperfmon_ids);
v3d_multisync_free(&ms);
queue->last_job_syncs.first[V3DV_QUEUE_CPU] = false;
if (ret)
return vk_queue_set_lost(&queue->vk, "V3D_SUBMIT_CPU failed: %m");
return VK_SUCCESS;
}
static VkResult
export_perfmon_last_job_sync(struct v3dv_queue *queue, struct v3dv_job *job, int *fd)
{
int err;
static const enum v3dv_queue_type queues_to_sync[] = {
V3DV_QUEUE_CL,
V3DV_QUEUE_CSD,
};
for (uint32_t i = 0; i < ARRAY_SIZE(queues_to_sync); i++) {
enum v3dv_queue_type queue_type = queues_to_sync[i];
int tmp_fd = -1;
err = drmSyncobjExportSyncFile(job->device->pdevice->render_fd,
queue->last_job_syncs.syncs[queue_type],
&tmp_fd);
if (err) {
close(*fd);
return vk_errorf(queue, VK_ERROR_UNKNOWN,
"sync file export failed: %m");
}
err = sync_accumulate("v3dv", fd, tmp_fd);
if (err) {
close(tmp_fd);
close(*fd);
return vk_errorf(queue, VK_ERROR_UNKNOWN,
"failed to accumulate sync files: %m");
}
}
return VK_SUCCESS;
}
static VkResult
handle_end_query_cpu_job(struct v3dv_queue *queue, struct v3dv_job *job, uint32_t counter_pass_idx)
{
MESA_TRACE_FUNC();
VkResult result = VK_SUCCESS;
mtx_lock(&job->device->query_mutex);
struct v3dv_end_query_info *info = &job->cpu.query_end;
int err = 0;
int fd = -1;
assert(info->pool->query_type == VK_QUERY_TYPE_PERFORMANCE_QUERY_KHR);
if (info->pool->query_type == VK_QUERY_TYPE_PERFORMANCE_QUERY_KHR) {
result = export_perfmon_last_job_sync(queue, job, &fd);
if (result != VK_SUCCESS)
goto fail;
assert(fd >= 0);
}
for (uint32_t i = 0; i < info->count; i++) {
assert(info->query + i < info->pool->query_count);
struct v3dv_query *query = &info->pool->queries[info->query + i];
if (info->pool->query_type == VK_QUERY_TYPE_PERFORMANCE_QUERY_KHR) {
uint32_t syncobj = vk_sync_as_drm_syncobj(query->perf.last_job_sync)->syncobj;
err = drmSyncobjImportSyncFile(job->device->pdevice->render_fd,
syncobj, fd);
if (err) {
result = vk_errorf(queue, VK_ERROR_UNKNOWN,
"sync file import failed: %m");
goto fail;
}
}
query->maybe_available = true;
}
fail:
if (info->pool->query_type == VK_QUERY_TYPE_PERFORMANCE_QUERY_KHR)
close(fd);
cnd_broadcast(&job->device->query_ended);
mtx_unlock(&job->device->query_mutex);
return result;
}
static VkResult
handle_copy_query_results_cpu_job(struct v3dv_queue *queue,
struct v3dv_job *job,
struct v3dv_submit_sync_info *sync_info)
{
MESA_TRACE_FUNC();
struct v3dv_device *device = queue->device;
struct v3dv_copy_query_results_cpu_job_info *info =
&job->cpu.query_copy_results;
assert(info->pool->query_type == VK_QUERY_TYPE_PERFORMANCE_QUERY_KHR ||
info->pool->query_type == VK_QUERY_TYPE_TIMESTAMP);
assert(info->dst && info->dst->mem && info->dst->mem->bo);
struct v3dv_bo *bo = info->dst->mem->bo;
struct drm_v3d_submit_cpu submit = {0};
struct v3d_multisync ms = {0};
uint32_t *offsets = (uint32_t *) malloc(sizeof(uint32_t) * info->count);
uint32_t *syncs = (uint32_t *) malloc(sizeof(uint32_t) * info->count);
uint32_t *bo_handles = NULL;
uintptr_t *kperfmon_ids = NULL;
if (info->pool->query_type == VK_QUERY_TYPE_TIMESTAMP) {
/* timestamp pool BO is V3DV-internal, never aliased by user BO. If
* that could happen we would need to dedupe them
*/
assert(bo->handle != info->pool->timestamp.bo->handle);
submit.bo_handle_count = 2;
bo_handles = (uint32_t *)
malloc(sizeof(uint32_t) * submit.bo_handle_count);
bo_handles[0] = bo->handle;
bo_handles[1] = info->pool->timestamp.bo->handle;
submit.bo_handles = (uintptr_t)(void *)bo_handles;
struct drm_v3d_copy_timestamp_query copy = {0};
v3d_submit_ext_set(&copy.base, NULL, DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY, 0);
copy.do_64bit = info->flags & VK_QUERY_RESULT_64_BIT;
copy.do_partial = info->flags & VK_QUERY_RESULT_PARTIAL_BIT;
copy.availability_bit = info->flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT;
copy.offset = info->offset + info->dst->mem_offset;
copy.stride = info->stride;
copy.count = info->count;
for (uint32_t i = 0; i < info->count; i++) {
assert(info->first < info->pool->query_count);
assert(info->first + info->count <= info->pool->query_count);
struct v3dv_query *query = &info->pool->queries[info->first + i];
offsets[i] = query->timestamp.offset;
syncs[i] = vk_sync_as_drm_syncobj(query->timestamp.sync)->syncobj;
}
copy.offsets = (uintptr_t)(void *)offsets;
copy.syncs = (uintptr_t)(void *)syncs;
if (!set_multisync(&ms, sync_info, NULL, 0, (void *)&copy, queue, job,
V3DV_QUEUE_CPU, V3DV_QUEUE_CPU, V3D_CPU)) {
free(bo_handles);
free(offsets);
free(syncs);
return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
}
} else {
assert(info->pool->query_type == VK_QUERY_TYPE_PERFORMANCE_QUERY_KHR);
submit.bo_handle_count = 1;
submit.bo_handles = (uintptr_t)(void *)&bo->handle;
struct drm_v3d_copy_performance_query copy = {0};
v3d_submit_ext_set(&copy.base, NULL, DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY, 0);
/* If the queryPool was created with VK_QUERY_TYPE_PERFORMANCE_QUERY_KHR,
* results for each query are written as an array of the type indicated
* by VkPerformanceCounterKHR::storage for the counter being queried.
* For v3dv, VkPerformanceCounterKHR::storage is
* VK_PERFORMANCE_COUNTER_STORAGE_UINT64_KHR.
*/
copy.do_64bit = true;
copy.do_partial = info->flags & VK_QUERY_RESULT_PARTIAL_BIT;
copy.availability_bit = info->flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT;
copy.offset = info->offset + info->dst->mem_offset;
copy.stride = info->stride;
copy.count = info->count;
copy.nperfmons = info->pool->perfmon.nperfmons;
copy.ncounters = info->pool->perfmon.ncounters;
kperfmon_ids = (uintptr_t *) malloc(sizeof(uintptr_t) * info->count);
struct vk_sync_wait waits[info->count];
unsigned wait_count = 0;
for (uint32_t i = 0; i < info->count; i++) {
assert(info->first < info->pool->query_count);
assert(info->first + info->count <= info->pool->query_count);
struct v3dv_query *query = &info->pool->queries[info->first + i];
syncs[i] = vk_sync_as_drm_syncobj(query->perf.last_job_sync)->syncobj;
kperfmon_ids[i] = (uintptr_t)(void *)query->perf.kperfmon_ids;
if (info->flags & VK_QUERY_RESULT_WAIT_BIT) {
waits[wait_count] = (struct vk_sync_wait){
.sync = query->perf.last_job_sync
};
wait_count++;
}
}
copy.syncs = (uintptr_t)(void *)syncs;
copy.kperfmon_ids = (uintptr_t)(void *)kperfmon_ids;
if (!set_multisync(&ms, sync_info, waits, wait_count, (void *)&copy, queue, job,
V3DV_QUEUE_CPU, V3DV_QUEUE_CPU, V3D_CPU)) {
free(kperfmon_ids);
free(bo_handles);
free(offsets);
free(syncs);
return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
}
}
submit.flags |= DRM_V3D_SUBMIT_EXTENSION;
submit.extensions = (uintptr_t)(void *)&ms;
int ret = v3d_ioctl(device->pdevice->render_fd,
DRM_IOCTL_V3D_SUBMIT_CPU, &submit);
free(kperfmon_ids);
free(bo_handles);
free(offsets);
free(syncs);
v3d_multisync_free(&ms);
queue->last_job_syncs.first[V3DV_QUEUE_CPU] = false;
if (ret)
return vk_queue_set_lost(&queue->vk, "V3D_SUBMIT_CPU failed: %m");
return VK_SUCCESS;
}
static VkResult
handle_timestamp_query_cpu_job(struct v3dv_queue *queue,
struct v3dv_job *job,
struct v3dv_submit_sync_info *sync_info)
{
MESA_TRACE_FUNC();
struct v3dv_device *device = queue->device;
assert(job->type == V3DV_JOB_TYPE_CPU_TIMESTAMP_QUERY);
struct v3dv_timestamp_query_cpu_job_info *info = &job->cpu.query_timestamp;
uint32_t *offsets =
(uint32_t *) malloc(sizeof(uint32_t) * info->count);
uint32_t *syncs =
(uint32_t *) malloc(sizeof(uint32_t) * info->count);
for (uint32_t i = 0; i < info->count; i++) {
assert(info->query + i < info->pool->query_count);
struct v3dv_query *query = &info->pool->queries[info->query + i];
query->maybe_available = true;
offsets[i] = query->timestamp.offset;
syncs[i] = vk_sync_as_drm_syncobj(query->timestamp.sync)->syncobj;
}
struct v3d_multisync ms = {0};
/* The CPU job should be serialized so it only executes after all previously
* submitted work has completed
*/
job->serialize = V3DV_BARRIER_ALL;
if (!set_multisync(&ms, sync_info, NULL, 0, NULL, queue, job,
V3DV_QUEUE_CPU, V3DV_QUEUE_CPU, V3D_CPU)) {
free(offsets);
free(syncs);
return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
}
int ret = v3d_submit_timestamp_query_ioctl(
device->pdevice->render_fd, info->pool->timestamp.bo->handle,
offsets, syncs, info->count, &ms.ext.base);
free(offsets);
free(syncs);
v3d_multisync_free(&ms);
queue->last_job_syncs.first[V3DV_QUEUE_CPU] = false;
if (ret)
return vk_queue_set_lost(&queue->vk, "V3D_SUBMIT_CPU failed: %m");
return VK_SUCCESS;
}
static VkResult
handle_csd_indirect_cpu_job(struct v3dv_queue *queue,
struct v3dv_job *job,
struct v3dv_submit_sync_info *sync_info)
{
MESA_TRACE_FUNC();
struct v3dv_device *device = queue->device;
assert(job->type == V3DV_JOB_TYPE_CPU_CSD_INDIRECT);
struct v3dv_csd_indirect_cpu_job_info *info = &job->cpu.csd_indirect;
assert(info->csd_job);
assert(info->buffer && info->buffer->mem && info->buffer->mem->bo);
struct v3dv_bo *bo = info->buffer->mem->bo;
struct v3dv_job *csd_job = info->csd_job;
struct drm_v3d_submit_cpu submit = {0};
submit.bo_handle_count = 1;
submit.bo_handles = (uintptr_t)(void *)&bo->handle;
uint32_t *bo_handles = (uint32_t *) malloc(sizeof(uint32_t) * csd_job->bo_count);
uint32_t bo_idx = 0;
set_foreach (csd_job->bos, entry) {
struct v3dv_bo *csd_bo = (struct v3dv_bo *)entry->key;
/* dedup against cpu_job bo */
if (csd_bo->handle == bo->handle)
continue;
bo_handles[bo_idx++] = csd_bo->handle;
}
csd_job->csd.submit.bo_handle_count = bo_idx;
csd_job->csd.submit.bo_handles = (uintptr_t)(void *)bo_handles;
struct drm_v3d_indirect_csd indirect = {0};
v3d_submit_ext_set(&indirect.base, NULL, DRM_V3D_EXT_ID_CPU_INDIRECT_CSD, 0);
indirect.submit = csd_job->csd.submit;
indirect.offset = info->buffer->mem_offset + info->offset;
indirect.wg_size = info->wg_size;
for (int i = 0; i < 3; i++) {
if (info->wg_uniform_offsets[i]) {
assert(info->wg_uniform_offsets[i] >= (uint32_t *) csd_job->indirect.base);
indirect.wg_uniform_offsets[i] = info->wg_uniform_offsets[i] - (uint32_t *) csd_job->indirect.base;
} else {
indirect.wg_uniform_offsets[i] = 0xffffffff; /* No rewrite */
}
}
indirect.indirect = csd_job->indirect.bo->handle;
struct v3d_multisync ms = {0};
/* We need to configure the semaphores of this job with the indirect
* CSD job, as the CPU job must obey to the CSD job synchronization
* demands, such as barriers.
*/
if (!set_multisync(&ms, sync_info, NULL, 0, (void *)&indirect, queue, csd_job,
V3DV_QUEUE_CPU, V3DV_QUEUE_CSD, V3D_CPU)) {
return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
}
submit.flags |= DRM_V3D_SUBMIT_EXTENSION;
submit.extensions = (uintptr_t)(void *)&ms.ext;
int ret = v3d_ioctl(device->pdevice->render_fd,
DRM_IOCTL_V3D_SUBMIT_CPU, &submit);
free(bo_handles);
v3d_multisync_free(&ms);
queue->last_job_syncs.first[V3DV_QUEUE_CPU] = false;
queue->last_job_syncs.first[V3DV_QUEUE_CSD] = false;
if (ret)
return vk_queue_set_lost(&queue->vk, "V3D_SUBMIT_CPU failed: %m");
return VK_SUCCESS;
}
static inline void
job_add_device_address_bos(struct v3dv_job *job, struct v3dv_queue *queue)
{
if (!job->uses_buffer_device_address)
return;
struct v3dv_device *device = queue->device;
mtx_lock(&device->queue_mutex);
util_dynarray_foreach(&device->device_address_bo_list,
struct v3dv_bo *, bo) {
v3dv_job_add_bo(job, *bo);
}
mtx_unlock(&device->queue_mutex);
}
static VkResult
handle_cl_job(struct v3dv_queue *queue,
struct v3dv_job *job,
uint32_t counter_pass_idx,
struct v3dv_submit_sync_info *sync_info)
{
MESA_TRACE_FUNC();
struct v3dv_device *device = queue->device;
struct drm_v3d_submit_cl submit = { 0 };
/* Sanity check: we should only flag a bcl sync on a job that needs to be
* serialized.
*/
assert(job->serialize || !job->needs_bcl_sync);
/* We expect to have just one RCL per job which should fit in just one BO.
* Our BCL, could chain multiple BOS together though.
*/
assert(list_length(&job->rcl.bo_list) == 1);
assert(list_length(&job->bcl.bo_list) >= 1);
struct v3dv_bo *bcl_fist_bo =
list_first_entry(&job->bcl.bo_list, struct v3dv_bo, list_link);
submit.bcl_start = bcl_fist_bo->offset;
submit.bcl_end = job->suspending ? job->suspended_bcl_end :
job->bcl.bo->offset + v3dv_cl_offset(&job->bcl);
submit.rcl_start = job->rcl.bo->offset;
submit.rcl_end = job->rcl.bo->offset + v3dv_cl_offset(&job->rcl);
submit.qma = job->tile_alloc->offset;
submit.qms = job->tile_alloc->size;
submit.qts = job->tile_state->offset;
submit.flags = 0;
if (job->tmu_dirty_rcl)
submit.flags |= DRM_V3D_SUBMIT_CL_FLUSH_CACHE;
/* If the job uses VK_KHR_buffer_device_address we need to ensure all
* buffers flagged with VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT
* are included.
*/
job_add_device_address_bos(job, queue);
submit.bo_handle_count = job->bo_count;
uint32_t *bo_handles =
(uint32_t *) malloc(sizeof(uint32_t) * submit.bo_handle_count);
uint32_t bo_idx = 0;
set_foreach(job->bos, entry) {
struct v3dv_bo *bo = (struct v3dv_bo *)entry->key;
bo_handles[bo_idx++] = bo->handle;
}
assert(bo_idx == submit.bo_handle_count);
submit.bo_handles = (uintptr_t)(void *)bo_handles;
submit.perfmon_id = job->perf ?
job->perf->kperfmon_ids[counter_pass_idx] : 0;
mtx_lock(&device->queue_mutex);
const bool needs_perf_sync = device->last_perfmon_id != submit.perfmon_id;
device->last_perfmon_id = submit.perfmon_id;
mtx_unlock(&device->queue_mutex);
/* We need a binning sync if we are the first CL job waiting on a semaphore
* with a wait stage that involves the geometry pipeline, or if the job
* comes after a pipeline barrier that involves geometry stages
* (needs_bcl_sync) or when performance queries are in use.
*
* We need a render sync if the job doesn't need a binning sync but has
* still been flagged for serialization. It should be noted that RCL jobs
* don't start until the previous RCL job has finished so we don't really
* need to add a fence for those, however, we might need to wait on a CSD or
* TFU job, which are not automatically serialized with CL jobs.
*/
bool needs_bcl_sync = job->needs_bcl_sync || needs_perf_sync;
if (queue->last_job_syncs.first[V3DV_QUEUE_CL]) {
for (int i = 0; !needs_bcl_sync && i < sync_info->wait_count; i++) {
needs_bcl_sync = sync_info->waits[i].stage_mask &
(VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT |
VK_PIPELINE_STAGE_2_ALL_GRAPHICS_BIT |
VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT |
VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT |
VK_PIPELINE_STAGE_2_INDEX_INPUT_BIT |
VK_PIPELINE_STAGE_2_VERTEX_INPUT_BIT |
VK_PIPELINE_STAGE_2_VERTEX_ATTRIBUTE_INPUT_BIT |
VK_PIPELINE_STAGE_2_VERTEX_SHADER_BIT |
VK_PIPELINE_STAGE_2_TESSELLATION_CONTROL_SHADER_BIT |
VK_PIPELINE_STAGE_2_TESSELLATION_EVALUATION_SHADER_BIT |
VK_PIPELINE_STAGE_2_GEOMETRY_SHADER_BIT |
VK_PIPELINE_STAGE_2_PRE_RASTERIZATION_SHADERS_BIT);
}
}
bool needs_rcl_sync = job->serialize && !needs_bcl_sync;
/* Replace single semaphore settings whenever our kernel-driver supports
* multiple semaphores extension.
*/
struct v3d_multisync ms = { 0 };
enum v3d_queue wait_stage = needs_rcl_sync ? V3D_RENDER : V3D_BIN;
if (!set_multisync(&ms, sync_info, NULL, 0, NULL, queue, job,
V3DV_QUEUE_CL, V3DV_QUEUE_CL, wait_stage)) {
free(bo_handles);
return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
}
submit.flags |= DRM_V3D_SUBMIT_EXTENSION;
submit.extensions = (uintptr_t)(void *)&ms.ext;
/* We are using multisync so disable legacy single-sync interface */
submit.in_sync_rcl = 0;
submit.in_sync_bcl = 0;
submit.out_sync = 0;
v3dv_clif_dump(device, job, &submit);
int ret = v3d_ioctl(device->pdevice->render_fd,
DRM_IOCTL_V3D_SUBMIT_CL, &submit);
static bool warned = false;
if (ret && !warned) {
mesa_loge("Draw call returned %s. Expect corruption.\n",
strerror(errno));
warned = true;
}
free(bo_handles);
v3d_multisync_free(&ms);
queue->last_job_syncs.first[V3DV_QUEUE_CL] = false;
if (ret)
return vk_queue_set_lost(&queue->vk, "V3D_SUBMIT_CL failed: %m");
return VK_SUCCESS;
}
static VkResult
handle_tfu_job(struct v3dv_queue *queue,
struct v3dv_job *job,
struct v3dv_submit_sync_info *sync_info)
{
MESA_TRACE_FUNC();
assert(!V3D_DBG(DISABLE_TFU));
struct v3dv_device *device = queue->device;
/* Replace single semaphore settings whenever our kernel-driver supports
* multiple semaphore extension.
*/
struct v3d_multisync ms = { 0 };
if (!set_multisync(&ms, sync_info, NULL, 0, NULL, queue, job,
V3DV_QUEUE_TFU, V3DV_QUEUE_TFU, V3D_TFU)) {
return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
}
job->tfu.flags |= DRM_V3D_SUBMIT_EXTENSION;
job->tfu.extensions = (uintptr_t)(void *)&ms.ext;
/* We are using multisync so disable legacy single-sync interface */
job->tfu.in_sync = 0;
job->tfu.out_sync = 0;
int ret = v3d_ioctl(device->pdevice->render_fd,
DRM_IOCTL_V3D_SUBMIT_TFU, &job->tfu);
v3d_multisync_free(&ms);
queue->last_job_syncs.first[V3DV_QUEUE_TFU] = false;
if (ret != 0)
return vk_queue_set_lost(&queue->vk, "V3D_SUBMIT_TFU failed: %m");
return VK_SUCCESS;
}
static VkResult
handle_csd_job(struct v3dv_queue *queue,
struct v3dv_job *job,
uint32_t counter_pass_idx,
struct v3dv_submit_sync_info *sync_info)
{
MESA_TRACE_FUNC();
struct v3dv_device *device = queue->device;
struct drm_v3d_submit_csd *submit = &job->csd.submit;
/* If the job uses VK_KHR_buffer_device_address we need to ensure all
* buffers flagged with VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT
* are included.
*/
job_add_device_address_bos(job, queue);
submit->bo_handle_count = job->bo_count;
uint32_t *bo_handles =
(uint32_t *) malloc(sizeof(uint32_t) * MAX2(4, submit->bo_handle_count * 2));
uint32_t bo_idx = 0;
set_foreach(job->bos, entry) {
struct v3dv_bo *bo = (struct v3dv_bo *)entry->key;
bo_handles[bo_idx++] = bo->handle;
}
assert(bo_idx == submit->bo_handle_count);
submit->bo_handles = (uintptr_t)(void *)bo_handles;
/* Replace single semaphore settings whenever our kernel-driver supports
* multiple semaphore extension.
*/
struct v3d_multisync ms = { 0 };
if (!set_multisync(&ms, sync_info, NULL, 0, NULL, queue, job,
V3DV_QUEUE_CSD, V3DV_QUEUE_CSD, V3D_CSD)) {
return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
}
submit->flags |= DRM_V3D_SUBMIT_EXTENSION;
submit->extensions = (uintptr_t)(void *)&ms.ext;
/* We are using multisync so disable legacy single-sync interface */
submit->in_sync = 0;
submit->out_sync = 0;
submit->perfmon_id = job->perf ?
job->perf->kperfmon_ids[counter_pass_idx] : 0;
mtx_lock(&device->queue_mutex);
device->last_perfmon_id = submit->perfmon_id;
mtx_unlock(&device->queue_mutex);
int ret = v3d_ioctl(device->pdevice->render_fd,
DRM_IOCTL_V3D_SUBMIT_CSD, submit);
static bool warned = false;
if (ret && !warned) {
mesa_loge("Compute dispatch returned %s. Expect corruption.\n",
strerror(errno));
warned = true;
}
free(bo_handles);
v3d_multisync_free(&ms);
queue->last_job_syncs.first[V3DV_QUEUE_CSD] = false;
if (ret)
return vk_queue_set_lost(&queue->vk, "V3D_SUBMIT_CSD failed: %m");
return VK_SUCCESS;
}
static void
queue_apply_barrier_state(struct v3dv_job *job,
struct v3dv_barrier_state *barrier)
{
if (!v3dv_job_apply_barrier_state(job, barrier))
return;
if (job->type != V3DV_JOB_TYPE_GPU_CL)
return;
if (job->serialize &&
(barrier->bcl_buffer_access || barrier->bcl_image_access)) {
job->needs_bcl_sync = true;
barrier->bcl_buffer_access = barrier->bcl_image_access = 0;
}
}
static VkResult
queue_handle_job(struct v3dv_queue *queue,
struct v3dv_job *job,
uint32_t counter_pass_idx,
struct v3dv_barrier_state *barrier,
struct v3dv_submit_sync_info *sync_info)
{
if (barrier)
queue_apply_barrier_state(job, barrier);
if (unlikely(V3D_DBG(SYNC))) {
job->serialize = V3DV_BARRIER_ALL;
job->needs_bcl_sync = job->type == V3DV_JOB_TYPE_GPU_CL;
}
job->queue = queue;
struct u_trace *trace = job->cmd_buffer ? &job->cmd_buffer->trace : NULL;
VkResult result;
switch (job->type) {
case V3DV_JOB_TYPE_GPU_CL:
job->cmd_buffer->trace_queue_mask |= 1 << V3DV_QUEUE_CL;
trace_begin_job_cl(trace, job);
result = handle_cl_job(queue, job, counter_pass_idx, sync_info);
trace_end_job_cl(trace, job, job->id, job->serialize,
job->draw_count, job->tmu_dirty_rcl);
return result;
case V3DV_JOB_TYPE_GPU_TFU:
job->cmd_buffer->trace_queue_mask |= 1 << V3DV_QUEUE_TFU;
trace_begin_job_tfu(trace, job);
result = handle_tfu_job(queue, job, sync_info);
trace_end_job_tfu(trace, job, job->id, job->serialize);
return result;
case V3DV_JOB_TYPE_GPU_CSD:
job->cmd_buffer->trace_queue_mask |= 1 << V3DV_QUEUE_CSD;
trace_begin_job_csd(trace, job);
result = handle_csd_job(queue, job, counter_pass_idx, sync_info);
trace_end_job_csd(trace, job, job->id, job->serialize,
job->csd.submit.cfg[1], /* wg_x */
job->csd.submit.cfg[2], /* wg_y */
job->csd.submit.cfg[3]); /* wg_z */
return result;
case V3DV_JOB_TYPE_CPU_RESET_QUERIES:
job->cmd_buffer->trace_queue_mask |= 1 << V3DV_QUEUE_CPU;
trace_begin_job_cpu_reset_queries(trace, job);
result = handle_reset_query_cpu_job(queue, job, sync_info);
trace_end_job_cpu_reset_queries(trace, job, job->id, job->serialize);
return result;
case V3DV_JOB_TYPE_CPU_END_QUERY:
/* V3DV_JOB_TYPE_END_QUERY doesn't involve any submission to the kernel,
* so we don't trace it.
*/
return handle_end_query_cpu_job(queue, job, counter_pass_idx);
case V3DV_JOB_TYPE_CPU_COPY_QUERY_RESULTS:
job->cmd_buffer->trace_queue_mask |= 1 << V3DV_QUEUE_CPU;
trace_begin_job_cpu_copy_query_results(trace, job);
result = handle_copy_query_results_cpu_job(queue, job, sync_info);
trace_end_job_cpu_copy_query_results(trace, job, job->id, job->serialize);
return result;
case V3DV_JOB_TYPE_CPU_CSD_INDIRECT:
job->cmd_buffer->trace_queue_mask |= 1 << V3DV_QUEUE_CPU;
trace_begin_job_cpu_csd_indirect(trace, job);
result = handle_csd_indirect_cpu_job(queue, job, sync_info);
trace_end_job_cpu_csd_indirect(trace, job, job->id, job->serialize);
return result;
case V3DV_JOB_TYPE_CPU_TIMESTAMP_QUERY:
job->cmd_buffer->trace_queue_mask |= 1 << V3DV_QUEUE_CPU;
trace_begin_job_cpu_timestamp_query(trace, job);
result = handle_timestamp_query_cpu_job(queue, job, sync_info);
trace_end_job_cpu_timestamp_query(trace, job, job->id, job->serialize);
return result;
default:
UNREACHABLE("Unhandled job type");
}
}
/* Merges an array of syncobj handles into a single target syncobj */
static VkResult
merge_syncobjs(struct v3dv_device *device,
const uint32_t *src_syncobjs,
uint32_t src_count,
uint32_t dst_syncobj)
{
int drm_fd = device->pdevice->render_fd;
int accum_fd = -1;
if (src_count == 0) {
/* If there are no jobs and no wait dependencies, the signal semaphores/fences
* must trigger immediately to satisfy the Vulkan spec.
*/
drmSyncobjSignal(drm_fd, &dst_syncobj, 1);
return VK_SUCCESS;
}
for (uint32_t i = 0; i < src_count; i++) {
int queue_fd = -1;
if (drmSyncobjExportSyncFile(drm_fd, src_syncobjs[i], &queue_fd) != 0)
goto ioctl_error;
if (accum_fd == -1) {
accum_fd = queue_fd;
} else {
int new_accum_fd = sync_merge("v3dv_merged_fence", accum_fd, queue_fd);
close(queue_fd);
if (new_accum_fd < 0)
goto ioctl_error;
close(accum_fd);
accum_fd = new_accum_fd;
}
}
/* Import the final accumulated fence fd back into the destination syncobj */
if (drmSyncobjImportSyncFile(drm_fd, dst_syncobj, accum_fd) != 0)
goto ioctl_error;
close(accum_fd);
return VK_SUCCESS;
ioctl_error:
if (accum_fd != -1)
close(accum_fd);
return VK_ERROR_DEVICE_LOST;
}
VkResult
v3dv_queue_driver_submit(struct vk_queue *vk_queue,
struct vk_queue_submit *submit)
{
MESA_TRACE_FUNC();
struct v3dv_queue *queue = container_of(vk_queue, struct v3dv_queue, vk);
VkResult result;
struct v3dv_submit_sync_info sync_info = {
.wait_count = submit->wait_count,
.waits = submit->waits,
.signal_count = submit->signal_count,
.signals = submit->signals,
};
for (int i = 0; i < V3DV_QUEUE_COUNT; i++)
queue->last_job_syncs.first[i] = true;
/* Reset every cmd_buffer's trace up front. Trace events for a job may
* be recorded against a different cmd_buffer than the one currently
* being walked below (see the suspend/resume handling), so all traces
* must be ready to receive events before any job is submitted.
*/
for (uint32_t i = 0; i < submit->command_buffer_count; i++) {
struct v3dv_cmd_buffer *cmd_buffer =
container_of(submit->command_buffers[i], struct v3dv_cmd_buffer, vk);
u_trace_fini(&cmd_buffer->trace);
u_trace_init(&cmd_buffer->trace, &queue->device->utrace.utrace_ctx);
cmd_buffer->trace_queue_mask = 0;
}
struct v3dv_barrier_state pending_barrier = { 0 };
struct v3dv_job *first_suspend_job = NULL;
struct v3dv_job *current_suspend_job = NULL;
bool has_jobs = false;
for (uint32_t i = 0; i < submit->command_buffer_count; i++) {
struct v3dv_cmd_buffer *cmd_buffer =
container_of(submit->command_buffers[i], struct v3dv_cmd_buffer, vk);
cmd_buffer->trace_marker_job.queue = queue;
trace_begin_cmdbuf(&cmd_buffer->trace, &cmd_buffer->trace_marker_job);
list_for_each_entry_safe(struct v3dv_job, job,
&cmd_buffer->jobs, list_link) {
if (job->suspending) {
job = v3d_X((&job->device->devinfo),
cmd_buffer_prepare_suspend_job_for_submit)(job);
if (!job)
return VK_ERROR_OUT_OF_DEVICE_MEMORY;
}
if (job->suspending && !job->resuming) {
assert(!first_suspend_job);
assert(!current_suspend_job);
first_suspend_job = job;
}
if (job->resuming) {
assert(first_suspend_job);
assert(current_suspend_job);
v3d_X((&job->device->devinfo), job_patch_resume_address)(first_suspend_job,
current_suspend_job,
job);
current_suspend_job = NULL;
}
if (job->suspending) {
current_suspend_job = job;
} else {
assert(!current_suspend_job);
struct v3dv_job *submit_job = first_suspend_job ?
first_suspend_job : job;
result =
queue_handle_job(queue, submit_job, submit->perf_pass_index,
&pending_barrier, &sync_info);
if (result != VK_SUCCESS)
return result;
has_jobs = true;
first_suspend_job = NULL;
}
}
trace_end_cmdbuf(&cmd_buffer->trace, &cmd_buffer->trace_marker_job,
cmd_buffer->usage_flags);
/* If the command buffer ends with a barrier, save the pending barrier
* state so we can apply it on the next command buffer.
*/
v3dv_merge_barrier_state(&pending_barrier, &cmd_buffer->state.barrier);
}
assert(!first_suspend_job);
assert(!current_suspend_job);
/* Now that every chain is fully resolved, regardless of which cmd_buffer(s)
* it spanned, it is safe to flush all traces.
*/
for (uint32_t i = 0; i < submit->command_buffer_count; i++) {
struct v3dv_cmd_buffer *cmd_buffer =
container_of(submit->command_buffers[i], struct v3dv_cmd_buffer, vk);
mtx_lock(&queue->device->utrace.process_mutex);
u_trace_flush(&cmd_buffer->trace, queue->device,
queue->device->vk.current_frame, false);
mtx_unlock(&queue->device->utrace.process_mutex);
}
mtx_lock(&queue->device->utrace.process_mutex);
u_trace_context_process(&queue->device->utrace.utrace_ctx, false /* eof */);
mtx_unlock(&queue->device->utrace.process_mutex);
/* Handle signaling now */
if (submit->signal_count > 0) {
/* We need to signal after all work (including wait dependencies)
* has completed. Vulkan also requires that submissions to the
* same queue signal in order. To ensure this, we accumulate the
* last_sync of all kernel queues into the signals of the command
* buffer regardless of whether we submitted any jobs to them in this
* command buffer (this satisfies the Vulkan signaling order
* requirement), so these are signaled as soon as all the queues
* finish. If the command buffer didn't submit any jobs, then we
* also need to gate signaling on the wait dependencies.
*/
uint32_t src_syncobjs[V3DV_QUEUE_COUNT + submit->wait_count];
uint32_t src_count = 0;
for (int i = 0; i < V3DV_QUEUE_COUNT; i++)
src_syncobjs[src_count++] = queue->last_job_syncs.syncs[i];
/* If we didn't submit any jobs, we need to merge wait dependencies */
if (!has_jobs && submit->wait_count > 0) {
for (uint32_t i = 0; i < submit->wait_count; i++)
src_syncobjs[src_count++] = vk_sync_as_drm_syncobj(submit->waits[i].sync)->syncobj;
}
/* Merge the accumulated syncobjs into each Vulkan signal semaphore */
for (uint32_t i = 0; i < submit->signal_count; i++) {
uint32_t dst_syncobj = vk_sync_as_drm_syncobj(submit->signals[i].sync)->syncobj;
result = merge_syncobjs(queue->device, src_syncobjs, src_count, dst_syncobj);
if (result != VK_SUCCESS)
return result;
}
}
return VK_SUCCESS;
}
VKAPI_ATTR VkResult VKAPI_CALL
v3dv_QueueBindSparse(VkQueue _queue,
uint32_t bindInfoCount,
const VkBindSparseInfo *pBindInfo,
VkFence fence)
{
V3DV_FROM_HANDLE(v3dv_queue, queue, _queue);
return vk_error(queue, VK_ERROR_FEATURE_NOT_PRESENT);
}