| // Copyright 2018 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "aml-gpu.h" |
| |
| #include <fidl/fuchsia.driver.compat/cpp/wire.h> |
| #include <fidl/fuchsia.hardware.gpu.amlogic/cpp/wire.h> |
| #include <fidl/fuchsia.hardware.gpu.mali/cpp/wire.h> |
| #include <lib/ddk/platform-defs.h> |
| #include <lib/driver/component/cpp/driver_export.h> |
| #include <lib/driver/component/cpp/node_add_args.h> |
| #include <lib/fdf/cpp/dispatcher.h> |
| #include <lib/trace/event.h> |
| #include <stdint.h> |
| #include <stdio.h> |
| #include <stdlib.h> |
| #include <string.h> |
| #include <unistd.h> |
| #include <zircon/errors.h> |
| #include <zircon/process.h> |
| #include <zircon/syscalls.h> |
| #include <zircon/syscalls/smc.h> |
| |
| #include <soc/aml-common/aml-registers.h> |
| |
| #include "s905d2-gpu.h" |
| #include "s912-gpu.h" |
| #include "src/devices/tee/drivers/optee/tee-smc.h" |
| #include "t931-gpu.h" |
| |
| namespace aml_gpu { |
| |
| AmlGpu::AmlGpu(fdf::DriverStartArgs start_args, |
| fdf::UnownedSynchronizedDispatcher driver_dispatcher) |
| : fdf::DriverBase("aml-gpu", std::move(start_args), std::move(driver_dispatcher)) {} |
| |
| AmlGpu::~AmlGpu() {} |
| |
| void AmlGpu::Stop() { |
| if (loop_dispatcher_.get()) { |
| loop_dispatcher_.ShutdownAsync(); |
| // At this point the Mali device has been released and won't call into this driver, so the loop |
| // should shutdown quickly. |
| loop_shutdown_completion_.Wait(); |
| } |
| } |
| |
| void AmlGpu::SetClkFreqSource(int32_t clk_source) { |
| if (current_clk_source_ == clk_source) { |
| return; |
| } |
| |
| FDF_LOG(INFO, "Setting clock source to %d: %d\n", clk_source, |
| gpu_block_->gpu_clk_freq[clk_source]); |
| uint32_t current_clk_cntl = hiu_buffer_->Read32(4 * gpu_block_->hhi_clock_cntl_offset); |
| uint32_t enabled_mux = current_clk_cntl & (1 << kFinalMuxBitShift); |
| uint32_t new_mux = enabled_mux == 0; |
| uint32_t mux_shift = new_mux ? 16 : 0; |
| |
| // clear existing values |
| current_clk_cntl &= ~(kClockMuxMask << mux_shift); |
| // set the divisor, enable & source for the unused mux |
| current_clk_cntl |= CalculateClockMux(true, gpu_block_->gpu_clk_freq[clk_source], 1) << mux_shift; |
| |
| // Write the new values to the unused mux |
| hiu_buffer_->Write32(current_clk_cntl, 4 * gpu_block_->hhi_clock_cntl_offset); |
| zx_nanosleep(zx_deadline_after(ZX_USEC(10))); |
| |
| // Toggle current mux selection |
| current_clk_cntl ^= (1 << kFinalMuxBitShift); |
| |
| // Select the unused input mux |
| hiu_buffer_->Write32(current_clk_cntl, 4 * gpu_block_->hhi_clock_cntl_offset); |
| |
| current_clk_source_ = clk_source; |
| UpdateClockProperties(); |
| } |
| |
| void AmlGpu::SetInitialClkFreqSource(int32_t clk_source) { |
| uint32_t current_clk_cntl = hiu_buffer_->Read32(4 * gpu_block_->hhi_clock_cntl_offset); |
| uint32_t enabled_mux = (current_clk_cntl & (1 << kFinalMuxBitShift)) != 0; |
| uint32_t mux_shift = enabled_mux ? 16 : 0; |
| |
| if (current_clk_cntl & (1 << (mux_shift + kClkEnabledBitShift))) { |
| SetClkFreqSource(clk_source); |
| } else { |
| FDF_LOG(INFO, "Setting initial clock source to %d: %d\n", clk_source, |
| gpu_block_->gpu_clk_freq[clk_source]); |
| // Switching the final dynamic mux from a disabled source to an enabled |
| // source doesn't work. If the current clock source is disabled, then |
| // enable it instead of switching. |
| current_clk_cntl &= ~(kClockMuxMask << mux_shift); |
| current_clk_cntl |= CalculateClockMux(true, gpu_block_->gpu_clk_freq[clk_source], 1) |
| << mux_shift; |
| |
| // Write the new values to the existing mux. |
| hiu_buffer_->Write32(current_clk_cntl, 4 * gpu_block_->hhi_clock_cntl_offset); |
| zx_nanosleep(zx_deadline_after(ZX_USEC(10))); |
| current_clk_source_ = clk_source; |
| UpdateClockProperties(); |
| } |
| } |
| |
| void AmlGpu::UpdateClockProperties() { |
| current_clk_source_property_.Set(current_clk_source_); |
| uint32_t clk_mux_source = gpu_block_->gpu_clk_freq[current_clk_source_]; |
| current_clk_mux_source_property_.Set(clk_mux_source); |
| ZX_DEBUG_ASSERT(clk_mux_source < kClockInputs); |
| uint32_t current_clk_freq_hz = gpu_block_->input_freq_map[clk_mux_source]; |
| current_clk_freq_hz_property_.Set(current_clk_freq_hz); |
| TRACE_INSTANT("magma", "AmlGpu::UpdateClockProperties", TRACE_SCOPE_PROCESS, "current_clk_source", |
| current_clk_source_, "clk_mux_source", clk_mux_source, "current_clk_freq_hz", |
| current_clk_freq_hz); |
| } |
| |
| zx_status_t AmlGpu::Gp0Init() { |
| auto clock_client = incoming()->Connect<fuchsia_hardware_clock::Service::Clock>("clock-gp0-pll"); |
| if (clock_client.is_error() || !clock_client.value().is_valid()) { |
| FDF_LOG(ERROR, "could not get clock fragment"); |
| return ZX_ERR_NO_RESOURCES; |
| } |
| |
| gp0_clock_ = fidl::WireSyncClient(std::move(clock_client.value())); |
| |
| // Errors setting the clock should be logged but ignored; initialization can continue either way. |
| auto set_result = gp0_clock_->SetRate(846000000); |
| if (set_result.status() != ZX_OK) { |
| FDF_LOG(ERROR, "Setting clock frequency failed, %s", set_result.FormatDescription().c_str()); |
| return ZX_OK; |
| } |
| |
| if (set_result->is_error()) { |
| FDF_LOG(ERROR, "Setting clock frequency failed, %s", |
| zx_status_get_string(set_result->error_value())); |
| return ZX_OK; |
| } |
| |
| auto enable_result = gp0_clock_->Enable(); |
| if (enable_result.status() != ZX_OK) { |
| FDF_LOG(ERROR, "Enabling clock failed, %s", enable_result.FormatDescription().c_str()); |
| return ZX_OK; |
| } |
| |
| if (enable_result->is_error()) { |
| FDF_LOG(ERROR, "Enabling clock failed, %s", zx_status_get_string(enable_result->error_value())); |
| return ZX_OK; |
| } |
| |
| gp0_init_succeeded_ = true; |
| root_.RecordBool("gp0_init_succeeded", true); |
| return ZX_OK; |
| } |
| |
| void AmlGpu::InitClock() { |
| { |
| auto result = reset_register_->WriteRegister32(gpu_block_->reset0_mask_offset, |
| aml_registers::MALI_RESET0_MASK, 0); |
| if ((result.status() != ZX_OK) || result->is_error()) { |
| FDF_LOG(ERROR, "Reset0 Mask Clear failed\n"); |
| } |
| } |
| |
| { |
| auto result = reset_register_->WriteRegister32(gpu_block_->reset0_level_offset, |
| aml_registers::MALI_RESET0_MASK, 0); |
| if ((result.status() != ZX_OK) || result->is_error()) { |
| FDF_LOG(ERROR, "Reset0 Level Clear failed\n"); |
| } |
| } |
| |
| { |
| auto result = reset_register_->WriteRegister32(gpu_block_->reset2_mask_offset, |
| aml_registers::MALI_RESET2_MASK, 0); |
| if ((result.status() != ZX_OK) || result->is_error()) { |
| FDF_LOG(ERROR, "Reset2 Mask Clear failed\n"); |
| } |
| } |
| |
| { |
| auto result = reset_register_->WriteRegister32(gpu_block_->reset2_level_offset, |
| aml_registers::MALI_RESET2_MASK, 0); |
| if ((result.status() != ZX_OK) || result->is_error()) { |
| FDF_LOG(ERROR, "Reset2 Level Clear failed\n"); |
| } |
| } |
| |
| uint32_t initial_clock_index = gpu_block_->initial_clock_index; |
| if (gpu_block_->enable_gp0 && !gp0_init_succeeded_) { |
| initial_clock_index = gpu_block_->non_gp0_index; |
| } |
| |
| SetInitialClkFreqSource(static_cast<int32_t>(initial_clock_index)); |
| |
| { |
| auto result = reset_register_->WriteRegister32(gpu_block_->reset0_level_offset, |
| aml_registers::MALI_RESET0_MASK, |
| aml_registers::MALI_RESET0_MASK); |
| if ((result.status() != ZX_OK) || result->is_error()) { |
| FDF_LOG(ERROR, "Reset2 Level Set failed\n"); |
| } |
| } |
| |
| { |
| auto result = reset_register_->WriteRegister32(gpu_block_->reset2_level_offset, |
| aml_registers::MALI_RESET2_MASK, |
| aml_registers::MALI_RESET2_MASK); |
| if ((result.status() != ZX_OK) || result->is_error()) { |
| FDF_LOG(ERROR, "Reset2 Level Set failed\n"); |
| } |
| } |
| |
| gpu_buffer_->Write32(0x2968A819, 4 * kPwrKey); |
| gpu_buffer_->Write32(0xfff | (0x20 << 16), 4 * kPwrOverride1); |
| } |
| |
| void AmlGpu::GetProperties(fdf::Arena& arena, GetPropertiesCompleter::Sync& completer) { |
| completer.buffer(arena).Reply(properties_); |
| } |
| |
| // Match the definitions in the Amlogic OPTEE implementation. |
| #define DMC_DEV_ID_GPU 1 |
| |
| #define DMC_DEV_TYPE_NON_SECURE 0 |
| #define DMC_DEV_TYPE_SECURE 1 |
| #define DMC_DEV_TYPE_INACCESSIBLE 2 |
| |
| zx_status_t AmlGpu::SetProtected(uint32_t protection_mode) { |
| if (!secure_monitor_) |
| return ZX_ERR_NOT_SUPPORTED; |
| |
| // Call into the TEE to mark a particular hardware unit as able to access |
| // protected memory or not. |
| zx_smc_parameters_t params = {}; |
| zx_smc_result_t result = {}; |
| constexpr uint32_t kFuncIdConfigDeviceSecure = 14; |
| params.func_id = tee_smc::CreateFunctionId(tee_smc::kFastCall, tee_smc::kSmc32CallConv, |
| tee_smc::kTrustedOsService, kFuncIdConfigDeviceSecure); |
| params.arg1 = DMC_DEV_ID_GPU; |
| params.arg2 = protection_mode; |
| zx_status_t status = zx_smc_call(secure_monitor_.get(), ¶ms, &result); |
| if (status != ZX_OK) { |
| FDF_LOG(ERROR, "Failed to set unit %ld protected status %ld code: %d", params.arg1, params.arg2, |
| status); |
| return status; |
| } |
| if (result.arg0 != 0) { |
| FDF_LOG(ERROR, "Failed to set unit %ld protected status %ld: %lx", params.arg1, params.arg2, |
| result.arg0); |
| return ZX_ERR_INTERNAL; |
| } |
| current_protected_mode_property_.Set(protection_mode); |
| return ZX_OK; |
| } |
| |
| void AmlGpu::EnterProtectedMode(fdf::Arena& arena, EnterProtectedModeCompleter::Sync& completer) { |
| if (!secure_monitor_) { |
| completer.buffer(arena).ReplyError(ZX_ERR_NOT_SUPPORTED); |
| return; |
| } |
| |
| zx_status_t status = SetProtected(DMC_DEV_TYPE_SECURE); |
| if (status == ZX_OK) { |
| completer.buffer(arena).ReplySuccess(); |
| } else { |
| completer.buffer(arena).ReplyError(status); |
| } |
| } |
| |
| void AmlGpu::StartExitProtectedMode(fdf::Arena& arena, |
| StartExitProtectedModeCompleter::Sync& completer) { |
| if (!secure_monitor_) { |
| completer.buffer(arena).ReplyError(ZX_ERR_NOT_SUPPORTED); |
| return; |
| } |
| // Switch device to inaccessible mode. This will prevent writes to all memory |
| // and start resetting the GPU. |
| zx_status_t status = SetProtected(DMC_DEV_TYPE_INACCESSIBLE); |
| if (status == ZX_OK) { |
| completer.buffer(arena).ReplySuccess(); |
| } else { |
| completer.buffer(arena).ReplyError(status); |
| } |
| } |
| |
| void AmlGpu::FinishExitProtectedMode(fdf::Arena& arena, |
| FinishExitProtectedModeCompleter::Sync& completer) { |
| if (!secure_monitor_) { |
| completer.buffer(arena).ReplyError(ZX_ERR_NOT_SUPPORTED); |
| return; |
| } |
| // Switch to non-secure mode. This will check that the device has been reset |
| // and will re-enable access to non-protected memory. |
| zx_status_t status = SetProtected(DMC_DEV_TYPE_NON_SECURE); |
| if (status == ZX_OK) { |
| completer.buffer(arena).ReplySuccess(); |
| } else { |
| completer.buffer(arena).ReplyError(status); |
| } |
| } |
| |
| zx_status_t AmlGpu::ProcessMetadata( |
| std::vector<uint8_t> raw_metadata, |
| fidl::WireTableBuilder<fuchsia_hardware_gpu_mali::wire::MaliProperties>& builder) { |
| fit::result decoded = fidl::InplaceUnpersist<fuchsia_hardware_gpu_amlogic::wire::Metadata>( |
| cpp20::span(raw_metadata)); |
| if (!decoded.is_ok()) { |
| FDF_LOG(ERROR, "Unable to parse metadata %s", |
| decoded.error_value().FormatDescription().c_str()); |
| return ZX_ERR_INTERNAL; |
| } |
| const auto& metadata = *decoded.value(); |
| builder.supports_protected_mode(metadata.has_supports_protected_mode() && |
| metadata.supports_protected_mode()); |
| return ZX_OK; |
| } |
| |
| zx::result<> AmlGpu::Start() { |
| component_inspector_ = std::make_unique<inspect::ComponentInspector>( |
| dispatcher(), inspect::PublishOptions{ |
| .inspector = inspector_, |
| .client_end = incoming()->Connect<fuchsia_inspect::InspectSink>().value()}); |
| |
| auto loop_dispatcher = fdf::UnsynchronizedDispatcher::Create( |
| fdf::UnsynchronizedDispatcher::Options{}, "aml-gpu-thread", |
| [this](fdf_dispatcher_t* dispatcher) { loop_shutdown_completion_.Signal(); }, |
| "fuchsia.graphics.drivers.aml-gpu"); |
| |
| if (!loop_dispatcher.is_ok()) { |
| FDF_LOG(ERROR, "Creating dispatcher failed, status=%s\n", loop_dispatcher.status_string()); |
| return loop_dispatcher.take_error(); |
| } |
| loop_dispatcher_ = *std::move(loop_dispatcher); |
| root_ = inspector_.GetRoot().CreateChild("aml-gpu"); |
| current_clk_source_property_ = root_.CreateUint("current_clk_source", current_clk_source_); |
| current_clk_mux_source_property_ = root_.CreateUint("current_clk_mux_source", 0); |
| current_clk_freq_hz_property_ = root_.CreateUint("current_clk_freq_hz", 0); |
| // GPU is in unknown mode on Bind. |
| current_protected_mode_property_ = root_.CreateInt("current_protected_mode", -1); |
| auto builder = fuchsia_hardware_gpu_mali::wire::MaliProperties::Builder(arena_); |
| { |
| zx::result result = incoming()->Connect<fuchsia_driver_compat::Service::Device>(); |
| if (result.is_error()) { |
| FDF_LOG(ERROR, "Failed to open compat service: %s", result.status_string()); |
| return result.take_error(); |
| } |
| auto compat = fidl::WireSyncClient(std::move(result.value())); |
| if (!compat.is_valid()) { |
| FDF_LOG(ERROR, "%s: failed to get compat", __func__); |
| return zx::error(ZX_ERR_NO_RESOURCES); |
| } |
| |
| auto metadata = compat->GetMetadata(); |
| if (!metadata.ok()) { |
| FDF_LOG(ERROR, "%s: failed to GetMetadata %s", __func__, metadata.FormatDescription().data()); |
| return zx::error(metadata.status()); |
| } |
| |
| if (!metadata->is_ok()) { |
| FDF_LOG(ERROR, "Metadata error: %d", metadata->error_value()); |
| return zx::error(metadata->error_value()); |
| } |
| // Metadata may or may not exist; if not, default values are used. |
| for (auto& entry : metadata->value()->metadata) { |
| if (entry.type == fuchsia_hardware_gpu_amlogic::wire::kMaliMetadata) { |
| uint64_t size; |
| zx_status_t status = entry.data.get_prop_content_size(&size); |
| if (status != ZX_OK) { |
| FDF_LOG(ERROR, "Failed to get metadata size, %s", zx_status_get_string(status)); |
| return zx::error(status); |
| } |
| std::vector<uint8_t> raw_metadata(size); |
| status = entry.data.read(raw_metadata.data(), 0, size); |
| if (status != ZX_OK) { |
| FDF_LOG(ERROR, "Failed to read metadata, %s", zx_status_get_string(status)); |
| return zx::error(status); |
| } |
| status = ProcessMetadata(std::move(raw_metadata), builder); |
| if (status != ZX_OK) { |
| FDF_LOG(ERROR, "Error processing metadata %d", status); |
| return zx::error(status); |
| } |
| break; |
| } |
| } |
| } |
| |
| auto platform_device = |
| incoming()->Connect<fuchsia_hardware_platform_device::Service::Device>("pdev"); |
| pdev_ = ddk::PDevFidl(std::move(platform_device.value())); |
| if (!pdev_.is_valid()) { |
| FDF_LOG(ERROR, "could not get platform device protocol"); |
| return zx::error(ZX_ERR_NOT_SUPPORTED); |
| } |
| |
| zx_status_t status = pdev_.MapMmio(MMIO_GPU, &gpu_buffer_); |
| if (status != ZX_OK) { |
| FDF_LOG(ERROR, "MapMmio failed"); |
| return zx::error(status); |
| } |
| |
| status = pdev_.MapMmio(MMIO_HIU, &hiu_buffer_); |
| if (status != ZX_OK) { |
| FDF_LOG(ERROR, "MapMmio failed"); |
| return zx::error(status); |
| } |
| |
| pdev_device_info_t info; |
| status = pdev_.GetDeviceInfo(&info); |
| if (status != ZX_OK) { |
| FDF_LOG(ERROR, "GetDeviceInfo failed"); |
| return zx::error(status); |
| } |
| |
| if (info.vid == PDEV_VID_GENERIC && info.pid == PDEV_PID_GENERIC && |
| info.did == PDEV_DID_DEVICETREE_NODE) { |
| // TODO(https://fxbug.dev/318736574) : Remove and rely only on GetDeviceInfo. |
| pdev_board_info_t board_info; |
| if ((status = pdev_.GetBoardInfo(&board_info)) != ZX_OK) { |
| FDF_LOG(ERROR, "GetBoardInfo failed"); |
| return zx::error(status); |
| } |
| if (board_info.vid == PDEV_VID_KHADAS) { |
| switch (board_info.pid) { |
| case PDEV_PID_VIM3: |
| info.pid = PDEV_PID_AMLOGIC_A311D; |
| break; |
| default: |
| FDF_LOG(ERROR, "Unsupported PID 0x%x for VID 0x%x", board_info.pid, board_info.vid); |
| return zx::error(ZX_ERR_INVALID_ARGS); |
| } |
| } else { |
| FDF_LOG(ERROR, "Unsupported VID 0x%x", board_info.vid); |
| return zx::error(ZX_ERR_INVALID_ARGS); |
| } |
| } |
| |
| switch (info.pid) { |
| case PDEV_PID_AMLOGIC_S912: |
| gpu_block_ = &s912_gpu_blocks; |
| break; |
| case PDEV_PID_AMLOGIC_S905D2: |
| case PDEV_PID_AMLOGIC_S905D3: |
| gpu_block_ = &s905d2_gpu_blocks; |
| break; |
| // A311D and T931 have the same GPU registers. |
| case PDEV_PID_AMLOGIC_T931: |
| case PDEV_PID_AMLOGIC_A311D: |
| gpu_block_ = &t931_gpu_blocks; |
| break; |
| default: |
| FDF_LOG(ERROR, "unsupported SOC PID %u\n", info.pid); |
| return zx::error(ZX_ERR_INVALID_ARGS); |
| } |
| |
| auto reset_register_client = |
| incoming()->Connect<fuchsia_hardware_registers::Service::Device>("register-reset"); |
| if (reset_register_client.is_error() || !reset_register_client.value().is_valid()) { |
| FDF_LOG(ERROR, "could not get register-reset fragment"); |
| return zx::error(ZX_ERR_NO_RESOURCES); |
| } |
| |
| reset_register_ = fidl::WireSyncClient(std::move(reset_register_client.value())); |
| |
| if (info.pid == PDEV_PID_AMLOGIC_S905D3 && builder.supports_protected_mode()) { |
| zx_status_t status; |
| // S905D3 needs to use an SMC into the TEE to do protected mode switching. |
| static constexpr uint32_t kTrustedOsSmcIndex = 0; |
| status = pdev_.GetSmc(kTrustedOsSmcIndex, &secure_monitor_); |
| if (status != ZX_OK) { |
| FDF_LOG(ERROR, "Unable to retrieve secure monitor SMC: %d", status); |
| return zx::error(status); |
| } |
| builder.use_protected_mode_callbacks(true); |
| } |
| |
| if (gpu_block_->enable_gp0) { |
| zx_status_t status = Gp0Init(); |
| if (status != ZX_OK) { |
| FDF_LOG(ERROR, "aml_gp0_init failed: %d. Falling back to lower clock.\n", status); |
| return zx::error(status); |
| } |
| } |
| |
| properties_ = builder.Build(); |
| |
| InitClock(); |
| |
| auto protocol = [this](fdf::ServerEnd<fuchsia_hardware_gpu_mali::ArmMali> server_end) mutable { |
| fdf::BindServer(loop_dispatcher_.get(), std::move(server_end), this); |
| }; |
| |
| fuchsia_hardware_gpu_mali::Service::InstanceHandler handler({.arm_mali = std::move(protocol)}); |
| { |
| auto status = outgoing()->AddService<fuchsia_hardware_gpu_mali::Service>(std::move(handler)); |
| if (status.is_error()) { |
| FDF_LOG(ERROR, "%s(): Failed to add service to outgoing directory: %s\n", __func__, |
| status.status_string()); |
| return status.take_error(); |
| } |
| } |
| |
| fidl::Arena arena; |
| node_ = fidl::WireSyncClient<fuchsia_driver_framework::Node>(std::move(node())); |
| |
| auto offers = std::vector{fdf::MakeOffer2<fuchsia_hardware_gpu_mali::Service>("default")}; |
| auto args = |
| fuchsia_driver_framework::NodeAddArgs{{.name = "aml-gpu", .offers2 = std::move(offers)}}; |
| |
| zx::result controller_endpoints = |
| fidl::CreateEndpoints<fuchsia_driver_framework::NodeController>(); |
| ZX_ASSERT(controller_endpoints.is_ok()); |
| |
| auto result = node_->AddChild(fidl::ToWire(arena, std::move(args)), |
| std::move(controller_endpoints->server), {}); |
| if (!result.ok()) { |
| FDF_LOG(ERROR, "Failed to add child: %s", result.status_string()); |
| return zx::error(result.status()); |
| } |
| |
| return zx::ok(); |
| } |
| |
| } // namespace aml_gpu |
| |
| // clang-format off |
| FUCHSIA_DRIVER_EXPORT(aml_gpu::AmlGpu); |