blob: c85547733d527890a33e6a1d41bc8894c160cea3 [file] [log] [blame]
/* Copyright (c) 2015-2022 The Khronos Group Inc.
* Copyright (c) 2015-2022 Valve Corporation
* Copyright (c) 2015-2022 LunarG, Inc.
* Copyright (C) 2015-2022 Google Inc.
* Modifications Copyright (C) 2020 Advanced Micro Devices, Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Author: Courtney Goeltzenleuchter <courtneygo@google.com>
* Author: Tobin Ehlis <tobine@google.com>
* Author: Chris Forbes <chrisf@ijw.co.nz>
* Author: Mark Lobodzinski <mark@lunarg.com>
* Author: Dave Houlton <daveh@lunarg.com>
* Author: John Zulauf <jzulauf@lunarg.com>
* Author: Tobias Hector <tobias.hector@amd.com>
*/
#include "queue_state.h"
#include "cmd_buffer_state.h"
#include "state_tracker.h"
using SemOp = SEMAPHORE_STATE::SemOp;
uint64_t QUEUE_STATE::Submit(CB_SUBMISSION &&submission) {
for (auto &cb_node : submission.cbs) {
auto cb_guard = cb_node->WriteLock();
for (auto *secondary_cmd_buffer : cb_node->linkedCommandBuffers) {
auto secondary_guard = secondary_cmd_buffer->WriteLock();
secondary_cmd_buffer->IncrementResources();
}
cb_node->IncrementResources();
// increment use count for all bound objects including secondary cbs
cb_node->BeginUse();
cb_node->Submit(submission.perf_submit_pass);
}
// Lock required for queue / semaphore operations, but not for command buffer
// processing above.
auto guard = WriteLock();
const uint64_t next_seq = seq_ + submissions_.size() + 1;
bool retire_early = false;
for (auto &wait : submission.wait_semaphores) {
wait.semaphore->EnqueueWait(this, next_seq, wait.payload);
wait.semaphore->BeginUse();
}
for (auto &signal : submission.signal_semaphores) {
if (signal.semaphore->EnqueueSignal(this, next_seq, signal.payload)) {
retire_early = true;
}
signal.semaphore->BeginUse();
}
if (submission.fence) {
if (submission.fence->EnqueueSignal(this, next_seq)) {
retire_early = true;
}
submission.fence->BeginUse();
}
submissions_.emplace_back(std::move(submission));
return retire_early ? next_seq : 0;
}
static void MergeResults(SEMAPHORE_STATE::RetireResult &results, const SEMAPHORE_STATE::RetireResult &sem_result) {
for (auto &entry : sem_result) {
auto &last_seq = results[entry.first];
last_seq = std::max(last_seq, entry.second);
}
}
layer_data::optional<CB_SUBMISSION> QUEUE_STATE::NextSubmission(uint64_t until_seq) {
// Pop the next submission off of the queue so that Retire() doesn't need to worry
// about locking.
auto guard = WriteLock();
layer_data::optional<CB_SUBMISSION> result;
if (seq_ < until_seq && !submissions_.empty()) {
result.emplace(std::move(submissions_.front()));
submissions_.pop_front();
seq_++;
}
return result;
}
void QUEUE_STATE::Retire(uint64_t until_seq) {
SEMAPHORE_STATE::RetireResult other_queue_seqs;
layer_data::optional<CB_SUBMISSION> submission;
// Roll this queue forward, one submission at a time.
while ((submission = NextSubmission(until_seq))) {
for (auto &wait : submission->wait_semaphores) {
auto result = wait.semaphore->Retire(this, wait.payload);
MergeResults(other_queue_seqs, result);
wait.semaphore->EndUse();
}
for (auto &signal : submission->signal_semaphores) {
auto result = signal.semaphore->Retire(this, signal.payload);
// in the case of timeline semaphores, signaling at payload == N
// may unblock waiting queues for payload <= N so we need to
// process them
MergeResults(other_queue_seqs, result);
signal.semaphore->EndUse();
}
// Handle updates to how far the current queue has progressed
// without going recursive when we call Retire on other_queue_seqs
// below.
auto self_update = other_queue_seqs.find(this);
if (self_update != other_queue_seqs.end()) {
until_seq = std::max(until_seq, self_update->second);
other_queue_seqs.erase(self_update);
}
for (auto &cb_node : submission->cbs) {
auto cb_guard = cb_node->WriteLock();
for (auto *secondary_cmd_buffer : cb_node->linkedCommandBuffers) {
auto secondary_guard = secondary_cmd_buffer->WriteLock();
secondary_cmd_buffer->Retire(submission->perf_submit_pass);
}
cb_node->Retire(submission->perf_submit_pass);
cb_node->EndUse();
}
if (submission->fence) {
submission->fence->Retire(false);
submission->fence->EndUse();
}
}
// Roll other queues forward to the highest seq we saw a wait for
for (const auto &qs : other_queue_seqs) {
qs.first->Retire(qs.second);
}
}
bool FENCE_STATE::EnqueueSignal(QUEUE_STATE *queue_state, uint64_t next_seq) {
auto guard = WriteLock();
if (scope_ != kSyncScopeInternal) {
return true;
}
// Mark fence in use
state_ = FENCE_INFLIGHT;
queue_ = queue_state;
seq_ = next_seq;
return false;
}
void FENCE_STATE::Retire(bool notify_queue) {
QUEUE_STATE *q = nullptr;
uint64_t seq = 0;
{
// Hold the lock only while updating members, but not
// while calling QUEUE_STATE::Retire()
auto guard = WriteLock();
if (scope_ == kSyncScopeInternal) {
q = queue_;
seq = seq_;
}
queue_ = nullptr;
seq_ = 0;
state_ = FENCE_RETIRED;
}
if (q && notify_queue) {
q->Retire(seq);
}
}
void FENCE_STATE::Reset() {
auto guard = WriteLock();
if (scope_ == kSyncScopeInternal) {
state_ = FENCE_UNSIGNALED;
} else if (scope_ == kSyncScopeExternalTemporary) {
scope_ = kSyncScopeInternal;
}
}
void FENCE_STATE::Import(VkExternalFenceHandleTypeFlagBits handle_type, VkFenceImportFlags flags) {
auto guard = WriteLock();
if (scope_ != kSyncScopeExternalPermanent) {
if ((handle_type == VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT || flags & VK_FENCE_IMPORT_TEMPORARY_BIT) &&
scope_ == kSyncScopeInternal) {
scope_ = kSyncScopeExternalTemporary;
} else {
scope_ = kSyncScopeExternalPermanent;
}
}
}
void FENCE_STATE::Export(VkExternalFenceHandleTypeFlagBits handle_type) {
auto guard = WriteLock();
if (handle_type != VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT) {
// Export with reference transference becomes external
scope_ = kSyncScopeExternalPermanent;
} else if (scope_ == kSyncScopeInternal) {
// Export with copy transference has a side effect of resetting the fence
state_ = FENCE_UNSIGNALED;
}
}
bool SEMAPHORE_STATE::EnqueueSignal(QUEUE_STATE *queue, uint64_t queue_seq, uint64_t &payload) {
auto guard = WriteLock();
if (scope_ != kSyncScopeInternal) {
return true; // retire early
}
if (type == VK_SEMAPHORE_TYPE_BINARY) {
payload = next_payload_++;
}
operations_.emplace(SemOp{kSignal, queue, queue_seq, payload});
return false;
}
void SEMAPHORE_STATE::EnqueueWait(QUEUE_STATE *queue, uint64_t queue_seq, uint64_t &payload) {
auto guard = WriteLock();
switch (scope_) {
case kSyncScopeExternalTemporary:
scope_ = kSyncScopeInternal;
break;
default:
break;
}
if (type == VK_SEMAPHORE_TYPE_BINARY) {
payload = next_payload_++;
}
operations_.emplace(SemOp{kWait, queue, queue_seq, payload});
}
void SEMAPHORE_STATE::EnqueueAcquire() {
auto guard = WriteLock();
assert(type == VK_SEMAPHORE_TYPE_BINARY);
operations_.emplace(SemOp{kBinaryAcquire, nullptr, 0, next_payload_++});
}
void SEMAPHORE_STATE::EnqueuePresent(QUEUE_STATE *queue) {
auto guard = WriteLock();
assert(type == VK_SEMAPHORE_TYPE_BINARY);
operations_.emplace(SemOp{kBinaryPresent, queue, 0, next_payload_++});
}
layer_data::optional<SemOp> SEMAPHORE_STATE::LastOp(std::function<bool(const SemOp &)> filter) const {
auto guard = ReadLock();
layer_data::optional<SemOp> result;
for (auto pos = operations_.rbegin(); pos != operations_.rend(); ++pos) {
if (!filter || filter(*pos)) {
result.emplace(*pos);
break;
}
}
return result;
}
bool SEMAPHORE_STATE::CanBeSignaled() const {
if (type == VK_SEMAPHORE_TYPE_TIMELINE) {
return true;
}
// both LastOp() and Completed() lock, so no locking needed in this method.
auto op = LastOp();
if (op) {
return op->CanBeSignaled();
}
auto comp = Completed();
return comp.CanBeSignaled();
}
bool SEMAPHORE_STATE::CanBeWaited() const {
if (type == VK_SEMAPHORE_TYPE_TIMELINE) {
return true;
}
// both LastOp() and Completed() lock, so no locking needed in this method.
auto op = LastOp();
if (op) {
return op->op_type == kSignal || op->op_type == kBinaryAcquire;
}
auto comp = Completed();
return comp.op_type == kSignal || comp.op_type == kBinaryAcquire;
}
SEMAPHORE_STATE::RetireResult SEMAPHORE_STATE::Retire(QUEUE_STATE *queue, uint64_t payload) {
auto guard = WriteLock();
RetireResult result;
while (!operations_.empty() && operations_.begin()->payload <= payload) {
completed_ = *operations_.begin();
operations_.erase(operations_.begin());
// Note: even though presentation is directed to a queue, there is no direct ordering between QP and subsequent work,
// so QP (and its semaphore waits) /never/ participate in any completion proof. Likewise, Acquire is not associated
// with a queue.
if (completed_.op_type != kBinaryAcquire && completed_.op_type != kBinaryPresent) {
auto &last_seq = result[completed_.queue];
last_seq = std::max(last_seq, completed_.seq);
}
}
return result;
}
void SEMAPHORE_STATE::RetireTimeline(uint64_t payload) {
if (type == VK_SEMAPHORE_TYPE_TIMELINE) {
auto results = Retire(nullptr, payload);
for (auto &entry : results) {
entry.first->Retire(entry.second);
}
}
}
void SEMAPHORE_STATE::Import(VkExternalSemaphoreHandleTypeFlagBits handle_type, VkSemaphoreImportFlags flags) {
auto guard = WriteLock();
if (scope_ != kSyncScopeExternalPermanent) {
if ((handle_type == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT || flags & VK_SEMAPHORE_IMPORT_TEMPORARY_BIT) &&
scope_ == kSyncScopeInternal) {
scope_ = kSyncScopeExternalTemporary;
} else {
scope_ = kSyncScopeExternalPermanent;
}
}
}
void SEMAPHORE_STATE::Export(VkExternalSemaphoreHandleTypeFlagBits handle_type) {
auto guard = WriteLock();
if (handle_type != VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT) {
// Cannot track semaphore state once it is exported, except for Sync FD handle types which have copy transference
scope_ = kSyncScopeExternalPermanent;
}
}