blob: 02987f7623bda62f5ffac58a5782ea49a9c661e7 [file]
// Copyright 2016 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "msd_intel_device.h"
#include <bitset>
#include <cstdio>
#include <string>
#include <fbl/string_printf.h>
#include "device_id.h"
#include "forcewake.h"
#include "magma_intel_gen_defs.h"
#include "magma_util/dlog.h"
#include "magma_util/macros.h"
#include "msd_intel_semaphore.h"
#include "platform_trace.h"
#include "registers.h"
inline uint64_t get_current_time_ns() {
return std::chrono::time_point_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now())
.time_since_epoch()
.count();
}
class MsdIntelDevice::BatchRequest : public DeviceRequest {
public:
BatchRequest(std::unique_ptr<MappedBatch> batch) : batch_(std::move(batch)) {}
protected:
magma::Status Process(MsdIntelDevice* device) override {
return device->ProcessBatch(std::move(batch_));
}
private:
std::unique_ptr<MappedBatch> batch_;
};
class MsdIntelDevice::DestroyContextRequest : public DeviceRequest {
public:
DestroyContextRequest(std::shared_ptr<MsdIntelContext> client_context)
: client_context_(std::move(client_context)) {}
protected:
magma::Status Process(MsdIntelDevice* device) override {
return device->ProcessDestroyContext(std::move(client_context_));
}
private:
std::shared_ptr<MsdIntelContext> client_context_;
};
class MsdIntelDevice::InterruptRequest : public DeviceRequest {
public:
InterruptRequest(uint64_t interrupt_time_ns, uint32_t master_interrupt_control,
uint32_t render_interrupt_status, uint32_t video_interrupt_status)
: interrupt_time_ns_(interrupt_time_ns),
master_interrupt_control_(master_interrupt_control),
render_interrupt_status_(render_interrupt_status),
video_interrupt_status_(video_interrupt_status) {}
protected:
magma::Status Process(MsdIntelDevice* device) override {
return device->ProcessInterrupts(interrupt_time_ns_, master_interrupt_control_,
render_interrupt_status_, video_interrupt_status_);
}
uint64_t interrupt_time_ns_;
uint32_t master_interrupt_control_;
uint32_t render_interrupt_status_;
uint32_t video_interrupt_status_;
};
class MsdIntelDevice::DumpRequest : public DeviceRequest {
public:
DumpRequest() {}
protected:
magma::Status Process(MsdIntelDevice* device) override {
return device->ProcessDumpStatusToLog();
}
};
class MsdIntelDevice::TimestampRequest : public DeviceRequest {
public:
TimestampRequest(std::shared_ptr<magma::PlatformBuffer> buffer) : buffer_(std::move(buffer)) {}
protected:
magma::Status Process(MsdIntelDevice* device) override {
return device->ProcessTimestampRequest(std::move(buffer_));
}
std::shared_ptr<magma::PlatformBuffer> buffer_;
};
//////////////////////////////////////////////////////////////////////////////////////////////////
std::unique_ptr<MsdIntelDevice> MsdIntelDevice::Create(void* device_handle,
bool start_device_thread) {
std::unique_ptr<MsdIntelDevice> device(new MsdIntelDevice());
if (!device->Init(device_handle, true))
return DRETP(nullptr, "Failed to initialize MsdIntelDevice");
if (start_device_thread) {
if (!device->StartDeviceThread())
return DRETP(nullptr, "Failed to start device thread");
}
return device;
}
MsdIntelDevice::MsdIntelDevice() { magic_ = kMagic; }
MsdIntelDevice::~MsdIntelDevice() { Destroy(); }
void MsdIntelDevice::Destroy() {
DLOG("Destroy");
CHECK_THREAD_NOT_CURRENT(device_thread_id_);
interrupt_manager_.reset();
device_thread_quit_flag_ = true;
if (device_request_semaphore_)
device_request_semaphore_->Signal();
if (device_thread_.joinable()) {
DLOG("joining device thread");
device_thread_.join();
DLOG("joined");
}
// TODO(fxbug.dev/82777): reset all engines
if (render_engine_cs_) {
// Ensure gpu is idle
render_engine_cs_->Reset();
}
}
std::unique_ptr<MsdIntelConnection> MsdIntelDevice::Open(msd_client_id_t client_id) {
return MsdIntelConnection::Create(this, client_id);
}
bool MsdIntelDevice::Init(void* device_handle, bool exec_init_batch) {
if (!BaseInit(device_handle))
return DRETF(false, "BaseInit failed");
InitEngine(render_engine_cs());
InitEngine(video_command_streamer());
if (exec_init_batch) {
if (!RenderInitBatch())
return DRETF(false, "RenderInitBatch failed");
}
return true;
}
bool MsdIntelDevice::BaseInit(void* device_handle) {
DASSERT(!platform_device_);
DLOG("Init device_handle %p", device_handle);
platform_device_ = MsdIntelPciDevice::CreateShim(device_handle);
if (!platform_device_)
return DRETF(false, "failed to create pci device");
uint16_t pci_dev_id;
if (!platform_device_->ReadPciConfig16(2, &pci_dev_id))
return DRETF(false, "ReadPciConfig16 failed");
uint16_t revision;
if (!platform_device_->ReadPciConfig16(8, &revision))
return DRETF(false, "ReadPciConfig16 failed");
revision_ = revision & 0xFF;
device_id_ = pci_dev_id;
DLOG("device_id 0x%x revision 0x%x", device_id_, revision);
std::unique_ptr<magma::PlatformMmio> mmio(
platform_device_->CpuMapPciMmio(0, magma::PlatformMmio::CACHE_POLICY_UNCACHED_DEVICE));
if (!mmio)
return DRETF(false, "failed to map pci bar 0");
register_io_ = std::make_unique<magma::RegisterIo>(std::move(mmio));
if (DeviceId::is_gen9(device_id_)) {
ForceWake::reset(register_io_.get(), registers::ForceWake::GEN9_RENDER);
ForceWake::request(register_io_.get(), registers::ForceWake::GEN9_RENDER);
} else {
MAGMA_LOG(WARNING, "Unrecognized graphics PCI device id 0x%x", device_id_);
return false;
}
bus_mapper_ = magma::PlatformBusMapper::Create(platform_device_->GetBusTransactionInitiator());
if (!bus_mapper_)
return DRETF(false, "failed to create bus mapper");
// Clear faults
registers::AllEngineFault::clear(register_io_.get());
QuerySliceInfo(&subslice_total_, &eu_total_);
interrupt_manager_ = InterruptManager::CreateShim(this);
if (!interrupt_manager_)
return DRETF(false, "failed to create interrupt manager");
PerProcessGtt::InitPrivatePat(register_io_.get());
gtt_ = std::shared_ptr<Gtt>(Gtt::CreateShim(this));
// Arbitrary
constexpr uint32_t kFirstSequenceNumber = 0x1000;
sequencer_ = std::unique_ptr<Sequencer>(new Sequencer(kFirstSequenceNumber));
{
auto mapping =
AddressSpace::MapBufferGpu(gtt_, MsdIntelBuffer::Create(magma::page_size(), "RCS HWSP"));
if (!mapping)
return DRETF(false, "MapBufferGpu failed for RCS HWSP");
render_engine_cs_ = std::make_unique<RenderEngineCommandStreamer>(this, std::move(mapping));
}
{
auto mapping =
AddressSpace::MapBufferGpu(gtt_, MsdIntelBuffer::Create(magma::page_size(), "VCS HWSP"));
if (!mapping)
return DRETF(false, "MapBufferGpu failed for VCS HWSP");
video_command_streamer_ = std::make_unique<VideoCommandStreamer>(this, std::move(mapping));
}
global_context_ = std::shared_ptr<MsdIntelContext>(new MsdIntelContext(gtt_));
// Creates the context backing store.
// Global context used to execute the render init batch.
if (!render_engine_cs_->InitContext(global_context_.get()))
return DRETF(false, "render_engine_cs failed to init global context");
if (!global_context_->Map(gtt_, render_engine_cs_->id()))
return DRETF(false, "global context init failed");
device_request_semaphore_ = magma::PlatformSemaphore::Create();
return true;
}
void MsdIntelDevice::InitEngine(EngineCommandStreamer* engine) {
CHECK_THREAD_IS_CURRENT(device_thread_id_);
engine->InitHardware();
switch (engine->id()) {
case RENDER_COMMAND_STREAMER:
// WaEnableGapsTsvCreditFix
registers::ArbiterControl::workaround(register_io());
// Enable render command streamer interrupts.
registers::GtInterruptMask0::write(register_io(), registers::InterruptRegisterBase::USER,
registers::InterruptRegisterBase::UNMASK);
registers::GtInterruptEnable0::write(register_io(), registers::InterruptRegisterBase::USER,
true);
registers::GtInterruptMask0::write(register_io(),
registers::InterruptRegisterBase::CONTEXT_SWITCH,
registers::InterruptRegisterBase::UNMASK);
registers::GtInterruptEnable0::write(register_io(),
registers::InterruptRegisterBase::CONTEXT_SWITCH, true);
break;
case VIDEO_COMMAND_STREAMER:
// Enable video command streamer interrupts.
registers::GtInterruptMask1::write(register_io(), registers::InterruptRegisterBase::USER,
registers::InterruptRegisterBase::UNMASK);
registers::GtInterruptEnable1::write(register_io(), registers::InterruptRegisterBase::USER,
true);
registers::GtInterruptMask1::write(register_io(),
registers::InterruptRegisterBase::CONTEXT_SWITCH,
registers::InterruptRegisterBase::UNMASK);
registers::GtInterruptEnable1::write(register_io(),
registers::InterruptRegisterBase::CONTEXT_SWITCH, true);
break;
default:
DASSERT(false);
}
}
bool MsdIntelDevice::RenderInitBatch() {
auto init_batch = render_engine_cs_->CreateRenderInitBatch(device_id_);
if (!init_batch)
return DRETF(false, "failed to create render init batch");
if (!render_engine_cs_->RenderInit(global_context_, std::move(init_batch), gtt_))
return DRETF(false, "render_engine_cs failed RenderInit");
return true;
}
bool MsdIntelDevice::EngineReset(EngineCommandStreamer* engine) {
MAGMA_LOG(WARNING, "resetting engine %s", engine->Name());
engine->ResetCurrentContext();
InitEngine(engine);
registers::AllEngineFault::clear(register_io_.get());
if (engine->id() == RENDER_COMMAND_STREAMER) {
if (!RenderInitBatch())
return false;
}
return true;
}
bool MsdIntelDevice::StartDeviceThread() {
DASSERT(!device_thread_.joinable());
device_thread_ = std::thread([this] { this->DeviceThreadLoop(); });
// Don't start interrupt processing until the device thread is running.
return interrupt_manager_->RegisterCallback(
InterruptCallback, this,
registers::MasterInterruptControl::kRenderInterruptsPendingBitMask |
registers::MasterInterruptControl::kVideoInterruptsPendingBitMask);
}
void MsdIntelDevice::InterruptCallback(void* data, uint32_t master_interrupt_control,
uint64_t interrupt_timestamp) {
DASSERT(data);
auto device = reinterpret_cast<MsdIntelDevice*>(data);
device->last_interrupt_callback_timestamp_ = magma::get_monotonic_ns();
device->last_interrupt_timestamp_ = interrupt_timestamp;
magma::RegisterIo* register_io = device->register_io_for_interrupt();
uint64_t now = get_current_time_ns();
uint32_t render_interrupt_status = 0;
uint32_t video_interrupt_status = 0;
if (master_interrupt_control &
registers::MasterInterruptControl::kRenderInterruptsPendingBitMask) {
render_interrupt_status = registers::GtInterruptIdentity0::read(register_io);
DLOG("gt IIR0 0x%08x", render_interrupt_status);
if (render_interrupt_status & registers::InterruptRegisterBase::kUserInterruptBit) {
registers::GtInterruptIdentity0::clear(register_io, registers::InterruptRegisterBase::USER);
}
if (render_interrupt_status & registers::InterruptRegisterBase::kContextSwitchBit) {
registers::GtInterruptIdentity0::clear(register_io,
registers::InterruptRegisterBase::CONTEXT_SWITCH);
}
}
if (master_interrupt_control &
registers::MasterInterruptControl::kVideoInterruptsPendingBitMask) {
video_interrupt_status = registers::GtInterruptIdentity1::read(register_io);
DLOG("gt IIR1 0x%08x", video_interrupt_status);
if (video_interrupt_status & registers::InterruptRegisterBase::kUserInterruptBit) {
registers::GtInterruptIdentity1::clear(register_io, registers::InterruptRegisterBase::USER);
}
if (video_interrupt_status & registers::InterruptRegisterBase::kContextSwitchBit) {
registers::GtInterruptIdentity1::clear(register_io,
registers::InterruptRegisterBase::CONTEXT_SWITCH);
}
}
if (render_interrupt_status || video_interrupt_status) {
device->EnqueueDeviceRequest(std::make_unique<InterruptRequest>(
now, master_interrupt_control, render_interrupt_status, video_interrupt_status));
}
}
void MsdIntelDevice::DumpStatusToLog() { EnqueueDeviceRequest(std::make_unique<DumpRequest>()); }
void MsdIntelDevice::SubmitBatch(std::unique_ptr<MappedBatch> batch) {
DLOG("SubmitBatch");
CHECK_THREAD_NOT_CURRENT(device_thread_id_);
EnqueueDeviceRequest(std::make_unique<BatchRequest>(std::move(batch)));
}
void MsdIntelDevice::DestroyContext(std::shared_ptr<MsdIntelContext> client_context) {
DLOG("DestroyContext");
CHECK_THREAD_NOT_CURRENT(device_thread_id_);
EnqueueDeviceRequest(std::make_unique<DestroyContextRequest>(std::move(client_context)));
}
//////////////////////////////////////////////////////////////////////////////////////////////////
void MsdIntelDevice::EnqueueDeviceRequest(std::unique_ptr<DeviceRequest> request,
bool enqueue_front) {
TRACE_DURATION("magma", "EnqueueDeviceRequest");
std::unique_lock<std::mutex> lock(device_request_mutex_);
if (enqueue_front) {
device_request_list_.emplace_front(std::move(request));
} else {
device_request_list_.emplace_back(std::move(request));
}
device_request_semaphore_->Signal();
}
#ifdef HANGCHECK_TIMEOUT_MS
constexpr uint32_t kHangcheckTimeoutMs = HANGCHECK_TIMEOUT_MS;
#else
constexpr uint32_t kHangcheckTimeoutMs = 1000;
#endif
constexpr uint32_t kFreqPollPeriodMs = 16;
std::chrono::milliseconds MsdIntelDevice::GetDeviceRequestTimeoutMs(
const std::vector<EngineCommandStreamer*>& engines) {
auto timeout = std::chrono::steady_clock::duration::max();
for (auto& engine : engines) {
timeout = std::min(timeout, engine->progress()->GetHangcheckTimeout(
kHangcheckTimeoutMs, std::chrono::steady_clock::now()));
}
std::chrono::milliseconds timeout_ms =
std::max(std::chrono::milliseconds(0), std::chrono::ceil<std::chrono::milliseconds>(timeout));
if (timeout != std::chrono::steady_clock::duration::max()) {
timeout_ms = std::min(timeout_ms, std::chrono::milliseconds(kFreqPollPeriodMs));
}
return timeout_ms;
}
void MsdIntelDevice::DeviceRequestTimedOut(const std::vector<EngineCommandStreamer*>& engines) {
// Sometimes the interrupt thread has been observed to be massively delayed in
// responding to a pending interrupt. In that case the InterruptRequest can be posted
// after the timeout has expired, so always check if there is work to do before jumping
// to conclusions.
{
std::unique_lock<std::mutex> lock(device_request_mutex_);
if (!device_request_list_.empty())
return;
}
auto now = std::chrono::steady_clock::now();
for (auto& engine : engines) {
if (engine->progress()->GetHangcheckTimeout(kHangcheckTimeoutMs, now) <=
std::chrono::steady_clock::duration::zero()) {
HangCheckTimeout(kHangcheckTimeoutMs, engine->id());
}
}
}
void MsdIntelDevice::TraceFreq(std::chrono::steady_clock::time_point& last_freq_poll_time) {
auto now = std::chrono::steady_clock::now();
if (std::chrono::ceil<std::chrono::milliseconds>(now - last_freq_poll_time).count() >=
kFreqPollPeriodMs) {
last_freq_poll_time = now;
if (TRACE_ENABLED()) {
uint32_t ATTRIBUTE_UNUSED actual_mhz =
registers::RenderPerformanceStatus::read_current_frequency_gen9(register_io_.get());
uint32_t ATTRIBUTE_UNUSED requested_mhz =
registers::RenderPerformanceNormalFrequencyRequest::read(register_io_.get());
TRACE_COUNTER("magma", "gpu freq", 0, "request_mhz", requested_mhz, "actual_mhz", actual_mhz);
}
}
}
int MsdIntelDevice::DeviceThreadLoop() {
magma::PlatformThreadHelper::SetCurrentThreadName("DeviceThread");
std::unique_lock<std::mutex> lock(device_request_mutex_);
// Manipulate device_thread_id_ while locked, here and below
device_thread_id_ = std::make_unique<magma::PlatformThreadId>();
lock.unlock();
CHECK_THREAD_IS_CURRENT(device_thread_id_);
DLOG("DeviceThreadLoop starting thread 0x%lx", device_thread_id_->id());
std::vector<EngineCommandStreamer*> engines{render_engine_cs(), video_command_streamer()};
std::chrono::steady_clock::time_point last_freq_poll_time;
while (true) {
std::chrono::milliseconds timeout_ms = GetDeviceRequestTimeoutMs(engines);
// When the semaphore wait returns the semaphore will be reset.
// The reset may race with subsequent enqueue/signals on the semaphore,
// which is fine because we process everything available in the queue
// before returning here to wait.
DASSERT(timeout_ms.count() >= 0);
magma::Status status = device_request_semaphore_->Wait(timeout_ms.count());
switch (status.get()) {
case MAGMA_STATUS_OK:
break;
case MAGMA_STATUS_TIMED_OUT: {
DeviceRequestTimedOut(engines);
break;
}
default:
MAGMA_LOG(WARNING, "device_request_semaphore_ Wait failed: %d", status.get());
DASSERT(false);
// TODO(fxbug.dev/13287): should we trigger a restart of the driver?
std::this_thread::sleep_for(std::chrono::seconds(5));
}
while (true) {
lock.lock();
if (!device_request_list_.size())
break;
auto request = std::move(device_request_list_.front());
device_request_list_.pop_front();
lock.unlock();
request->ProcessAndReply(this);
}
lock.unlock();
if (device_thread_quit_flag_)
break;
TraceFreq(last_freq_poll_time);
}
DLOG("DeviceThreadLoop exit");
lock.lock();
device_thread_id_.reset();
return 0;
}
void MsdIntelDevice::ProcessCompletedCommandBuffers(EngineCommandStreamerId id) {
CHECK_THREAD_IS_CURRENT(device_thread_id_);
TRACE_DURATION("magma", "ProcessCompletedCommandBuffers");
switch (id) {
case RENDER_COMMAND_STREAMER:
render_engine_cs_->ProcessCompletedCommandBuffers(
render_engine_cs()->hardware_status_page()->read_sequence_number());
break;
case VIDEO_COMMAND_STREAMER:
video_command_streamer_->ProcessCompletedCommandBuffers(
video_command_streamer()->hardware_status_page()->read_sequence_number());
break;
default:
DASSERT(false);
}
}
magma::Status MsdIntelDevice::ProcessInterrupts(uint64_t interrupt_time_ns,
uint32_t master_interrupt_control,
uint32_t render_interrupt_status,
uint32_t video_interrupt_status) {
TRACE_DURATION("magma", "ProcessInterrupts");
if (master_interrupt_control &
registers::MasterInterruptControl::kRenderInterruptsPendingBitMask) {
if (render_interrupt_status & registers::InterruptRegisterBase::kUserInterruptBit) {
ProcessCompletedCommandBuffers(RENDER_COMMAND_STREAMER);
}
if (render_interrupt_status & registers::InterruptRegisterBase::kContextSwitchBit) {
render_engine_cs_->ContextSwitched();
}
}
if (master_interrupt_control &
registers::MasterInterruptControl::kVideoInterruptsPendingBitMask) {
if (video_interrupt_status & registers::InterruptRegisterBase::kUserInterruptBit) {
ProcessCompletedCommandBuffers(VIDEO_COMMAND_STREAMER);
}
if (video_interrupt_status & registers::InterruptRegisterBase::kContextSwitchBit) {
video_command_streamer_->ContextSwitched();
}
}
uint32_t fault = registers::AllEngineFault::read(register_io_.get());
if (registers::AllEngineFault::valid(fault)) {
std::vector<std::string> dump;
DumpToString(dump);
MAGMA_LOG(WARNING, "GPU fault detected\n");
for (auto& str : dump) {
MAGMA_LOG(WARNING, "%s", str.c_str());
}
switch (registers::AllEngineFault::engine(fault)) {
case registers::AllEngineFault::RCS:
EngineReset(render_engine_cs());
break;
case registers::AllEngineFault::VCS1:
EngineReset(video_command_streamer());
break;
default:
DASSERT(false);
}
}
return MAGMA_STATUS_OK;
}
magma::Status MsdIntelDevice::ProcessDumpStatusToLog() {
std::vector<std::string> dump;
DumpToString(dump);
for (auto& str : dump) {
MAGMA_LOG(INFO, "%s", str.c_str());
}
return MAGMA_STATUS_OK;
}
void MsdIntelDevice::HangCheckTimeout(uint64_t timeout_ms, EngineCommandStreamerId id) {
std::vector<std::string> dump;
DumpToString(dump);
uint32_t master_interrupt_control = registers::MasterInterruptControl::read(register_io_.get());
bool pending_interrupt;
EngineCommandStreamer* engine;
switch (id) {
case RENDER_COMMAND_STREAMER:
pending_interrupt = (master_interrupt_control &
registers::MasterInterruptControl::kRenderInterruptsPendingBitMask);
engine = render_engine_cs();
break;
case VIDEO_COMMAND_STREAMER:
pending_interrupt = (master_interrupt_control &
registers::MasterInterruptControl::kVideoInterruptsPendingBitMask);
engine = video_command_streamer();
break;
default:
DASSERT(false);
return;
}
if (pending_interrupt) {
MAGMA_LOG(WARNING,
"%s: Hang check timeout (%lu ms) while pending interrupt; slow interrupt handler?\n"
"last submitted sequence number 0x%x master_interrupt_control 0x%08x "
"last_interrupt_callback_timestamp %lu last_interrupt_timestamp %lu",
engine->Name(), timeout_ms, engine->progress()->last_submitted_sequence_number(),
master_interrupt_control, last_interrupt_callback_timestamp_.load(),
last_interrupt_timestamp_.load());
for (auto& str : dump) {
MAGMA_LOG(WARNING, "%s", str.c_str());
}
return;
}
MAGMA_LOG(WARNING,
"%s: Suspected GPU hang (%lu ms):\nlast submitted sequence number "
"0x%x master_interrupt_control 0x%08x last_interrupt_callback_timestamp %lu "
"last_interrupt_timestamp %lu",
engine->Name(), timeout_ms, engine->progress()->last_submitted_sequence_number(),
master_interrupt_control, last_interrupt_callback_timestamp_.load(),
last_interrupt_timestamp_.load());
for (auto& str : dump) {
MAGMA_LOG(WARNING, "%s", str.c_str());
}
suspected_gpu_hang_count_ += 1;
EngineReset(engine);
}
bool MsdIntelDevice::InitContextForEngine(MsdIntelContext* context,
EngineCommandStreamer* command_streamer) {
if (!command_streamer->InitContext(context))
return DRETF(false, "failed to initialize context");
if (!context->Map(gtt(), command_streamer->id()))
return DRETF(false, "failed to map context");
// TODO(fxbug.dev/80906): any workarounds or cache config for VCS?
if (command_streamer->id() == RENDER_COMMAND_STREAMER) {
if (!command_streamer->InitContextWorkarounds(context))
return DRETF(false, "failed to init workarounds");
if (!command_streamer->InitContextCacheConfig(context))
return DRETF(false, "failed to init cache config");
}
return true;
}
magma::Status MsdIntelDevice::ProcessBatch(std::unique_ptr<MappedBatch> batch) {
CHECK_THREAD_IS_CURRENT(device_thread_id_);
TRACE_DURATION("magma", "Device::ProcessBatch");
DLOG("preparing command buffer for execution");
auto context = batch->GetContext().lock();
DASSERT(context);
if (context->killed())
return DRET_MSG(MAGMA_STATUS_CONTEXT_KILLED, "Context killed");
auto context_command_streamer = context->GetTargetCommandStreamer();
DASSERT(context_command_streamer);
EngineCommandStreamer* command_streamer = nullptr;
switch (*context_command_streamer) {
case RENDER_COMMAND_STREAMER:
command_streamer = render_engine_cs_.get();
break;
case VIDEO_COMMAND_STREAMER:
command_streamer = video_command_streamer_.get();
break;
}
DASSERT(command_streamer);
if (!context->IsInitializedForEngine(command_streamer->id())) {
if (!InitContextForEngine(context.get(), command_streamer))
return DRET_MSG(MAGMA_STATUS_INTERNAL_ERROR, "Failed to initialize context");
}
uint64_t ATTRIBUTE_UNUSED buffer_id = batch->GetBatchBufferId();
{
TRACE_DURATION("magma", "Device::SubmitBatch");
TRACE_FLOW_STEP("magma", "command_buffer", buffer_id);
command_streamer->SubmitBatch(std::move(batch));
}
RequestMaxFreq();
return MAGMA_STATUS_OK;
}
magma::Status MsdIntelDevice::ProcessDestroyContext(
std::shared_ptr<MsdIntelContext> client_context) {
DLOG("ProcessDestroyContext");
TRACE_DURATION("magma", "ProcessDestroyContext");
CHECK_THREAD_IS_CURRENT(device_thread_id_);
// Just let it go out of scope
return MAGMA_STATUS_OK;
}
bool MsdIntelDevice::WaitIdleForTest(uint32_t timeout_ms) {
CHECK_THREAD_IS_CURRENT(device_thread_id_);
std::vector<EngineCommandStreamer*> engines{render_engine_cs(), video_command_streamer()};
uint32_t sequence_number = Sequencer::kInvalidSequenceNumber;
auto start = std::chrono::high_resolution_clock::now();
for (auto engine : engines) {
while (!engine->IsIdle()) {
ProcessCompletedCommandBuffers(engine->id());
auto end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double, std::milli> elapsed = end - start;
if (engine->progress()->last_completed_sequence_number() != sequence_number) {
sequence_number = engine->progress()->last_completed_sequence_number();
start = end;
} else {
if (elapsed.count() > timeout_ms)
return DRETF(false, "WaitIdle timeout (%u ms)", timeout_ms);
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
}
}
return true;
}
void MsdIntelDevice::RequestMaxFreq() {
CHECK_THREAD_IS_CURRENT(device_thread_id_);
uint32_t mhz = registers::RenderPerformanceStateCapability::read_rp0_frequency(register_io());
registers::RenderPerformanceNormalFrequencyRequest::write_frequency_request_gen9(register_io(),
mhz);
}
uint32_t MsdIntelDevice::GetCurrentFrequency() {
CHECK_THREAD_IS_CURRENT(device_thread_id_);
if (DeviceId::is_gen9(device_id_))
return registers::RenderPerformanceStatus::read_current_frequency_gen9(register_io());
DLOG("GetCurrentGraphicsFrequency not implemented");
return 0;
}
void MsdIntelDevice::QuerySliceInfo(uint32_t* subslice_total_out, uint32_t* eu_total_out) {
uint32_t slice_enable_mask;
uint32_t subslice_enable_mask;
registers::Fuse2ControlDwordMirror::read(register_io_.get(), &slice_enable_mask,
&subslice_enable_mask);
DLOG("slice_enable_mask 0x%x subslice_enable_mask 0x%x", slice_enable_mask, subslice_enable_mask);
std::bitset<registers::MirrorEuDisable::kMaxSliceCount> slice_bitset(slice_enable_mask);
std::bitset<registers::MirrorEuDisable::kMaxSubsliceCount> subslice_bitset(subslice_enable_mask);
*subslice_total_out = magma::to_uint32(slice_bitset.count() * subslice_bitset.count());
*eu_total_out = 0;
for (uint8_t slice = 0; slice < registers::MirrorEuDisable::kMaxSliceCount; slice++) {
if ((slice_enable_mask & (1 << slice)) == 0)
continue; // skip disabled slice
std::vector<uint32_t> eu_disable_mask;
registers::MirrorEuDisable::read(register_io_.get(), slice, eu_disable_mask);
for (uint32_t subslice = 0; subslice < eu_disable_mask.size(); subslice++) {
if ((subslice_enable_mask & (1 << subslice)) == 0)
continue; // skip disabled subslice
DLOG("subslice %u eu_disable_mask 0x%x", subslice, eu_disable_mask[subslice]);
size_t eu_disable_count =
std::bitset<registers::MirrorEuDisable::kEuPerSubslice>(eu_disable_mask[subslice])
.count();
*eu_total_out += registers::MirrorEuDisable::kEuPerSubslice - eu_disable_count;
}
}
}
magma::Status MsdIntelDevice::QueryTimestamp(std::unique_ptr<magma::PlatformBuffer> buffer) {
auto request = std::make_unique<TimestampRequest>(std::move(buffer));
auto reply = request->GetReply();
EnqueueDeviceRequest(std::move(request));
constexpr uint32_t kWaitTimeoutMs = 1000;
magma::Status status = reply->Wait(kWaitTimeoutMs);
if (!status.ok())
return DRET_MSG(status.get(), "reply wait failed");
return MAGMA_STATUS_OK;
}
static uint64_t get_ns_monotonic(bool raw) {
struct timespec time;
int ret = clock_gettime(raw ? CLOCK_MONOTONIC_RAW : CLOCK_MONOTONIC, &time);
if (ret < 0)
return 0;
return static_cast<uint64_t>(time.tv_sec) * 1000000000ULL + time.tv_nsec;
}
magma::Status MsdIntelDevice::ProcessTimestampRequest(
std::shared_ptr<magma::PlatformBuffer> buffer) {
magma_intel_gen_timestamp_query* query;
{
void* ptr;
if (!buffer->MapCpu(&ptr))
return DRET_MSG(MAGMA_STATUS_INTERNAL_ERROR, "failed to map query buffer");
query = reinterpret_cast<magma_intel_gen_timestamp_query*>(ptr);
}
// The monotonic raw timestamps represent the start/end of the sample interval.
query->monotonic_raw_timestamp[0] = get_ns_monotonic(true);
query->monotonic_timestamp = get_ns_monotonic(false);
query->device_timestamp =
registers::Timestamp::read(register_io_.get(), render_engine_cs()->mmio_base());
query->monotonic_raw_timestamp[1] = get_ns_monotonic(true);
buffer->UnmapCpu();
return MAGMA_STATUS_OK;
}
//////////////////////////////////////////////////////////////////////////////////////////////////
msd_connection_t* msd_device_open(msd_device_t* dev, msd_client_id_t client_id) {
auto connection = MsdIntelDevice::cast(dev)->Open(client_id);
if (!connection)
return DRETP(nullptr, "MsdIntelDevice::Open failed");
return new MsdIntelAbiConnection(std::move(connection));
}
void msd_device_destroy(msd_device_t* dev) { delete MsdIntelDevice::cast(dev); }
magma_status_t msd_device_query(msd_device_t* device, uint64_t id, uint64_t* value_out) {
switch (id) {
case MAGMA_QUERY_VENDOR_ID:
*value_out = MAGMA_VENDOR_ID_INTEL;
return MAGMA_STATUS_OK;
case MAGMA_QUERY_DEVICE_ID:
*value_out = MsdIntelDevice::cast(device)->device_id();
return MAGMA_STATUS_OK;
case MAGMA_QUERY_IS_TOTAL_TIME_SUPPORTED:
*value_out = 0;
return MAGMA_STATUS_OK;
case kMagmaIntelGenQuerySubsliceAndEuTotal:
*value_out = MsdIntelDevice::cast(device)->subslice_total();
*value_out = (*value_out << 32) | MsdIntelDevice::cast(device)->eu_total();
return MAGMA_STATUS_OK;
case kMagmaIntelGenQueryGttSize:
*value_out = 1ul << 48;
return MAGMA_STATUS_OK;
case kMagmaIntelGenQueryExtraPageCount:
*value_out = PerProcessGtt::ExtraPageCount();
return MAGMA_STATUS_OK;
}
return DRET_MSG(MAGMA_STATUS_INVALID_ARGS, "unhandled id %" PRIu64, id);
}
magma_status_t msd_device_query_returns_buffer(msd_device_t* device, uint64_t id,
uint32_t* buffer_out) {
switch (id) {
case kMagmaIntelGenQueryTimestamp: {
auto buffer = magma::PlatformBuffer::Create(magma::page_size(), "timestamps");
if (!buffer)
return DRET_MSG(MAGMA_STATUS_INTERNAL_ERROR, "failed to create timestamp buffer");
if (!buffer->duplicate_handle(buffer_out))
return DRET_MSG(MAGMA_STATUS_INTERNAL_ERROR, "failed to dupe timestamp buffer");
return MsdIntelDevice::cast(device)->QueryTimestamp(std::move(buffer)).get();
}
}
return DRET_MSG(MAGMA_STATUS_INVALID_ARGS, "unhandled id %" PRIu64, id);
}
void msd_device_dump_status(msd_device_t* device, uint32_t dump_type) {
MsdIntelDevice::cast(device)->DumpStatusToLog();
}
magma_status_t msd_device_get_icd_list(struct msd_device_t* abi_device, uint64_t count,
msd_icd_info_t* icd_info_out, uint64_t* actual_count_out) {
const char* kSuffixes[] = {"_test", ""};
constexpr uint32_t kMediaIcdCount = 1;
constexpr uint32_t kTotalIcdCount = std::size(kSuffixes) + kMediaIcdCount;
if (icd_info_out && count < kTotalIcdCount) {
return MAGMA_STATUS_INVALID_ARGS;
}
*actual_count_out = kTotalIcdCount;
if (icd_info_out) {
for (uint32_t i = 0; i < std::size(kSuffixes); i++) {
strncpy(icd_info_out[i].component_url,
fbl::StringPrintf("fuchsia-pkg://fuchsia.com/libvulkan_intel_gen%s#meta/vulkan.cm",
kSuffixes[i])
.c_str(),
sizeof(icd_info_out[i].component_url) - 1);
icd_info_out[i].support_flags = ICD_SUPPORT_FLAG_VULKAN;
}
{
size_t media_index = std::size(kSuffixes);
strncpy(icd_info_out[media_index].component_url,
"fuchsia-pkg://fuchsia.com/codec_runner_intel_gen#meta/codec_runner_intel_gen.cm",
sizeof(icd_info_out[media_index].component_url) - 1);
icd_info_out[media_index].support_flags = ICD_SUPPORT_FLAG_MEDIA_CODEC_FACTORY;
}
}
return MAGMA_STATUS_OK;
}