src/devices/bin/driver_loader/loader.cc

// Copyright 2024 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/devices/bin/driver_loader/loader.h"

#include <fidl/fuchsia.io/cpp/wire.h>
#include <lib/fdio/directory.h>
#include <lib/fdio/fdio.h>
#include <lib/fdio/io.h>
#include <lib/fit/defer.h>
#include <lib/ld/abi.h>
#include <lib/ld/remote-abi-heap.h>
#include <lib/ld/remote-abi-stub.h>
#include <lib/ld/remote-abi.h>

#include <filesystem>

#include <fbl/unique_fd.h>

#include "src/devices/bin/driver_loader/diagnostics.h"
#include "src/devices/lib/log/log.h"

namespace fio = fuchsia_io;

namespace {

zx::result<zx::vmo> GetStubLdVmo() {
  std::filesystem::path stub_path{"/pkg/lib"};
  stub_path /= std::string{ld::RemoteAbiStub<>::kFilename};
  const fio::wire::Flags kFlags =
      fio::wire::Flags::kProtocolFile | fio::wire::kPermReadable | fio::wire::kPermExecutable;
  fbl::unique_fd fd;
  zx_status_t status =
      fdio_open3_fd(stub_path.c_str(), static_cast<uint64_t>(kFlags), fd.reset_and_get_address());
  if (status != ZX_OK) {
    return zx::error(status);
  }
  zx::vmo vmo;
  status = fdio_get_vmo_exec(fd.get(), vmo.reset_and_get_address());
  if (status != ZX_OK) {
    return zx::error(status);
  }
  return zx::ok(std::move(vmo));
}

}  // namespace

namespace driver_loader {

// static
std::unique_ptr<Loader> Loader::Create(async_dispatcher_t* dispatcher,
                                       LoadDriverHandler load_driver_handler_for_testing) {
  zx::result<zx::vmo> stub_ld_vmo = GetStubLdVmo();
  if (stub_ld_vmo.is_error()) {
    LOGF(ERROR, "Failed to get stub ld vmo for loader: %s", stub_ld_vmo.status_string());
    return nullptr;
  }
  auto diag = MakeDiagnostics();
  return std::unique_ptr<Loader>(new Loader(dispatcher, std::move(load_driver_handler_for_testing),
                                            diag, std::move(*stub_ld_vmo)));
}

void Loader::Connect(fidl::ServerEnd<fuchsia_driver_loader::DriverHostLauncher> server_end) {
  bindings_.AddBinding(dispatcher_, std::move(server_end), this, fidl::kIgnoreBindingClosure);
}

zx::result<> Loader::Start(zx::process process, zx::vmar root_vmar, zx::vmo exec_vmo,
                           zx::vmo vdso_vmo, fidl::ClientEnd<fuchsia_io::Directory> lib_dir,
                           fidl::ServerEnd<fuchsia_driver_loader::DriverHost> server_end) {
  auto diag = MakeDiagnostics();

  Linker linker;
  linker.set_abi_stub(remote_abi_stub_);
  if (!linker.abi_stub()) {
    return zx::error(ZX_ERR_INTERNAL);
  }

  Linker::Module::DecodedPtr decoded_executable =
      Linker::Module::Decoded::Create(diag, std::move(exec_vmo), kPageSize);
  if (!decoded_executable) {
    return zx::error(ZX_ERR_INVALID_ARGS);
  }
  // TODO(https://fxbug.dev/365122208): we should cache the decoded ptr per vdso version.
  Linker::Module::DecodedPtr decoded_vdso =
      Linker::Module::Decoded::Create(diag, std::move(vdso_vmo), kPageSize);
  if (!decoded_vdso) {
    return zx::error(ZX_ERR_INVALID_ARGS);
  }

  auto init_result = linker.Init(diag,
                                 {Linker::Executable(std::move(decoded_executable)),
                                  Linker::Implicit(std::move(decoded_vdso))},
                                 GetDepFunction(diag, std::move(lib_dir)));
  if (!init_result) {
    // We usually do not expect this to fail. It's possible that there are errors
    // reported to the diagnostics (such as for missing deps), but |init_result| will still be
    // true. For non fatal errors, we will try to proceed as far as possible in order to catch as
    // many issues as possible.
    LOGF(ERROR, "Linker init failed, probably due to a very invalid module");
    return zx::error(ZX_ERR_INTERNAL);
  }

  // We expect this to be equal to the size of the initial_modules list passed to |linker.Init|.
  ZX_ASSERT_MSG(init_result->size() == 2u,
                "Linker.Init returned an unexpected number of elements, want %u got %lu", 2u,
                init_result->size());

  // Allocate new child vmars.
  if (!linker.Allocate(diag, root_vmar.borrow())) {
    // As |Init| has succeeded, |Allocate| should only fail due to system errors such as resource
    // constraints, rather than issues with the module itself. This may be due to kernel OOM, or
    // that there are too many existing mappings.
    LOGF(ERROR, "Linker failed to allocate in vmar, address space may be full");
    return zx::error(ZX_ERR_INTERNAL);
  }

  const Linker::Module& loaded_vdso = *init_result->back();
  struct ProcessState::ProcessStartArgs process_start_args = {
      .entry = linker.main_entry(),
      .vdso_base = loaded_vdso.module().vaddr_start(),
      .stack_size = linker.main_stack_size(),
  };

  // Apply relocations to segment VMOs.
  if (!linker.Relocate(diag)) {
    return zx::error(ZX_ERR_INTERNAL);
  }

  // Since |DiagnosticsFlags::multiple_errors| is set to true, we would not have returned early
  // for non-fatal errors or warnings. Check now before attempting to continue loading.
  auto check_errors = [&diag](std::string_view what, std::string_view error_msg) -> zx::result<> {
    if ((diag.errors() != 0) || (diag.warnings() != 0)) {
      LOGF(ERROR, "Linker diagnostics reported %u errors, %u warnings during %s, %s aborted. %s.",
           diag.errors(), diag.warnings(), what, what, error_msg);
      return zx::error(ZX_ERR_INTERNAL);
    }
    return zx::ok();
  };

  if (auto result = check_errors("relocation", "This is likely due to a bad driver host package");
      result.is_error()) {
    return result.take_error();
  }

  // Load the segments into the allocated vmars.
  if (!linker.Load(diag)) {
    return zx::error(ZX_ERR_INTERNAL);
  }
  if (auto result = check_errors("loading", "This is an unexpected error"); result.is_error()) {
    return result.take_error();
  }

  // Commit the VMARs and mappings.
  linker.Commit();

  constexpr std::string_view kThreadName = "driver-host-initial-thread";
  zx::thread thread;
  zx_status_t status =
      zx::thread::create(process, kThreadName.data(), kThreadName.size(), 0, &thread);
  if (status != ZX_OK) {
    return zx::error(status);
  }
  zx::channel bootstrap_sender, bootstrap_receiver;
  status = zx::channel::create(0, &bootstrap_sender, &bootstrap_receiver);
  if (status != ZX_OK) {
    return zx::error(status);
  }

  auto preloaded_modules = linker.PreloadedImplicit(*init_result);
  // Start the process.
  auto process_state = ProcessState::Create(
      remote_abi_stub_, std::move(process), std::move(thread), std::move(root_vmar),
      std::move(bootstrap_sender), load_driver_handler_for_testing_.share(),
      std::move(process_start_args), std::move(preloaded_modules));
  if (process_state.is_error()) {
    LOGF(ERROR, "Failed to create process state: %s", process_state.status_string());
    return process_state.take_error();
  }

  status = process_state->BindServer(dispatcher_, std::move(server_end));
  if (status != ZX_OK) {
    LOGF(ERROR, "Failed to bind server endpoint to process state: %s",
         zx_status_get_string(status));
    return zx::error(status);
  }

  status = process_state->Start(std::move(bootstrap_receiver));
  if (status != ZX_OK) {
    LOGF(ERROR, "Failed to start process: %s", zx_status_get_string(status));
    return zx::error(status);
  }
  started_processes_.push_back(std::move(*process_state));
  return zx::ok();
}

void Loader::Launch(LaunchRequestView request, LaunchCompleter::Sync& completer) {
  auto result = Start(std::move(request->process()), std::move(request->root_vmar()),
                      std::move(request->driver_host_binary()), std::move(request->vdso()),
                      std::move(request->driver_host_libs()), std::move(request->driver_host()));
  if (result.is_error()) {
    completer.ReplyError(result.status_value());
    return;
  }
  completer.ReplySuccess();
}

// static
zx::result<zx::vmo> Loader::GetDepVmo(fidl::UnownedClientEnd<fuchsia_io::Directory> lib_dir,
                                      const char* libname) {
  auto [client_end, server_end] = fidl::Endpoints<fio::File>::Create();
  zx_status_t status =
      fdio_open3_at(lib_dir.channel()->get(), libname,
                    static_cast<uint64_t>(fio::wire::kPermReadable | fio::wire::kPermExecutable),
                    server_end.TakeChannel().release());
  if (status != ZX_OK) {
    return zx::error(status);
  }
  fidl::SyncClient file(std::move(client_end));
  auto result = file->GetBackingMemory(fio::VmoFlags::kRead | fio::VmoFlags::kExecute |
                                       fio::VmoFlags::kPrivateClone);
  if (result.is_error()) {
    return zx::error(ZX_ERR_NOT_FOUND);
  }
  return zx::ok(std::move(result->vmo()));
}

// static
zx::result<std::unique_ptr<Loader::ProcessState>> Loader::ProcessState::Create(
    ld::RemoteAbiStub<>::Ptr remote_abi_stub, zx::process process, zx::thread thread,
    zx::vmar root_vmar, zx::channel bootstrap_sender,
    LoadDriverHandler load_driver_handler_for_testing, const ProcessStartArgs& args,
    Linker::InitModuleList preloaded_modules) {
  auto process_state = std::unique_ptr<ProcessState>(new ProcessState());

  process_state->remote_abi_stub_ = std::move(remote_abi_stub);
  process_state->process_ = std::move(process);
  process_state->thread_ = std::move(thread);
  process_state->root_vmar_ = std::move(root_vmar);
  process_state->process_start_args_ = std::move(args);
  process_state->preloaded_modules_ = std::move(preloaded_modules);
  process_state->bootstrap_sender_ = std::move(bootstrap_sender);
  process_state->load_driver_handler_for_testing_ = std::move(load_driver_handler_for_testing);

  zx_status_t status = process_state->AllocateStack();
  if (status != ZX_OK) {
    return zx::error(status);
  }
  return zx::ok(std::move(process_state));
}

// TODO(https://fxbug.dev/349913885): this should be replaced by a call to a helper library
// once available.
zx_status_t Loader::ProcessState::AllocateStack() {
  zx::vmo stack_vmo;

  size_t bootstrap_stack_size = process_start_args_.stack_size.value_or(kDefaultStackSize);

  const size_t guard_size = Loader::kPageSize;
  const size_t stack_vmo_size = (bootstrap_stack_size + Loader::kPageSize - 1) & -Loader::kPageSize;
  const size_t stack_vmar_size = stack_vmo_size + guard_size;

  zx_status_t status = zx::vmo::create(stack_vmo_size, 0, &stack_vmo);
  if (status != ZX_OK) {
    LOGF(ERROR, "Failed to create vmo for stack: %s", zx_status_get_string(status));
    return status;
  }

  zx::vmar stack_vmar;
  uintptr_t stack_vmar_base;
  status = root_vmar_.allocate(ZX_VM_CAN_MAP_SPECIFIC | ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE, 0,
                               stack_vmar_size, &stack_vmar, &stack_vmar_base);
  if (status != ZX_OK) {
    LOGF(ERROR, "Failed to allocate child vmar for stack: %s", zx_status_get_string(status));
    return status;
  }

  zx_vaddr_t stack_base;
  status = stack_vmar.map(ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_SPECIFIC | ZX_VM_ALLOW_FAULTS,
                          guard_size, stack_vmo, 0, stack_vmo_size, &stack_base);
  if (status != ZX_OK) {
    LOGF(ERROR, "Failed to map stack vmar: %s", zx_status_get_string(status));
    return status;
  }

  stack_ = std::move(stack_vmo);
  initial_stack_pointer_ = elfldltl::AbiTraits<>::InitialStackPointer(stack_base, stack_vmo_size);
  return ZX_OK;
}

zx_status_t Loader::ProcessState::BindServer(
    async_dispatcher_t* dispatcher, fidl::ServerEnd<fuchsia_driver_loader::DriverHost> server_end) {
  if (binding_.has_value()) {
    return ZX_ERR_ALREADY_BOUND;
  }
  binding_.emplace(dispatcher, std::move(server_end), this, fidl::kIgnoreBindingClosure);
  return ZX_OK;
}

zx_status_t Loader::ProcessState::Start(zx::channel bootstrap_receiver) {
  return process_.start(thread_, process_start_args_.entry, initial_stack_pointer_,
                        std::move(bootstrap_receiver), process_start_args_.vdso_base);
}

void Loader::ProcessState::LoadDriver(LoadDriverRequestView request,
                                      LoadDriverCompleter::Sync& completer) {
  fidl::VectorView<fuchsia_driver_loader::wire::RootModule> additional_root_modules;
  if (request->has_additional_root_modules()) {
    additional_root_modules = request->additional_root_modules();
  }
  auto result = LoadDriverModule(std::string(request->driver_soname().get()),
                                 std::move(request->driver_binary()),
                                 std::move(request->driver_libs()), additional_root_modules);
  if (result.is_error()) {
    completer.ReplyError(result.status_value());
    return;
  }
  fidl::Arena arena;
  completer.ReplySuccess(fuchsia_driver_loader::wire::DriverHostLoadDriverResponse::Builder(arena)
                             .dynamic_linking_abi(*result)
                             .Build());
}

zx::result<Loader::DynamicLinkingPassiveAbi> Loader::ProcessState::LoadDriverModule(
    std::string driver_name, zx::vmo driver_module, fidl::ClientEnd<fuchsia_io::Directory> lib_dir,
    fidl::VectorView<fuchsia_driver_loader::wire::RootModule> additional_root_modules) {
  auto diag = MakeDiagnostics();

  if (driver_name == "compat.so") {
    driver_name = "libdriver.so";
  }

  Linker::Soname root_module_name{driver_name};

  Linker linker;
  linker.set_abi_stub(remote_abi_stub_);

  Linker::Module::DecodedPtr decoded_module =
      Linker::Module::Decoded::Create(diag, std::move(driver_module), kPageSize);
  if (!decoded_module) {
    LOGF(ERROR, "Failed to decode driver module for %s", root_module_name.c_str());
    return zx::error(ZX_ERR_INVALID_ARGS);
  }

  // Make a copy of the list of driver host initial modules that will be in the
  // driver's dynamic linking namespace - this is the driver host (libdriver.so) and the vDSO.
  Linker::InitModuleList initial_modules = preloaded_modules_;
  initial_modules.emplace_back(Linker::RootModule(decoded_module, root_module_name));

  // We need to keep these strings alive for |Linker.Init|.
  std::vector<std::string> additional_root_modules_names;
  for (auto& module : additional_root_modules) {
    additional_root_modules_names.emplace_back(module.name().get());
    Linker::Soname name{additional_root_modules_names.back()};
    Linker::Module::DecodedPtr decoded_module =
        Linker::Module::Decoded::Create(diag, std::move(module.binary()), kPageSize);
    if (!decoded_module) {
      LOGF(ERROR, "Failed to decode additional root module module %s", name.c_str());
      return zx::error(ZX_ERR_INVALID_ARGS);
    }
    initial_modules.emplace_back(Linker::RootModule(decoded_module, name));
  }

  auto init_result =
      linker.Init(diag, std::move(initial_modules), GetDepFunction(diag, std::move(lib_dir)));
  if (!init_result) {
    // We usually do not expect this to fail. It's possible that there are errors
    // reported to the diagnostics (such as for missing deps), but |init_result| will still be
    // true. For non fatal errors, we will try to proceed as far as possible in order to catch as
    // many issues as possible.
    LOGF(ERROR, "Linker init failed, probably due to a very invalid module");
    return zx::error(ZX_ERR_INVALID_ARGS);
  }

  // We expect the |init_result| vector to be the same size as the number of |initial_modules|.
  // This includes the driver module, the two preloaded modules (driver host and vdso),
  // and any additional root modules.
  size_t expected_init_result_size = 3u + additional_root_modules.size();
  ZX_ASSERT_MSG(init_result->size() == expected_init_result_size,
                "Linker.Init returned an unexpected number of elements, want %lu got %lu",
                expected_init_result_size, init_result->size());

  if (!linker.Allocate(diag, root_vmar_.borrow())) {
    // As |Init| has succeeded, |Allocate| should only fail due to system errors such as resource
    // constraints, rather than issues with the module itself. This may be due to kernel OOM, or
    // that there are too many existing mappings.
    LOGF(ERROR, "Linker failed to allocate in vmar, address space may be full");
    return zx::error(ZX_ERR_INTERNAL);
  }

  if (!linker.Relocate(diag)) {
    return zx::error(ZX_ERR_INTERNAL);
  }

  // Since |DiagnosticsFlags::multiple_errors| is set to true, we would not have returned early
  // for non-fatal errors or warnings. Check now before attempting to continue loading.
  auto check_errors = [&diag](std::string_view what, std::string_view error_msg) -> zx::result<> {
    if ((diag.errors() != 0) || (diag.warnings() != 0)) {
      LOGF(ERROR, "Linker diagnostics reported %u errors, %u warnings during %s, %s aborted. %s.",
           diag.errors(), diag.warnings(), what, what, error_msg);
      return zx::error(ZX_ERR_INTERNAL);
    }
    return zx::ok();
  };

  if (auto result = check_errors("relocation", "This is likely due to a bad driver host package");
      result.is_error()) {
    return result.take_error();
  }

  if (!linker.Load(diag)) {
    return zx::error(ZX_ERR_INTERNAL);
  }
  if (auto result = check_errors("loading", "This is an unexpected error"); result.is_error()) {
    return result.take_error();
  }

  linker.Commit();

  DynamicLinkingPassiveAbi passive_abi = linker.abi_vaddr();

  ZX_ASSERT_MSG(passive_abi != 0u, "Got null for dynamic linking passive abi address");

  if (load_driver_handler_for_testing_) {
    load_driver_handler_for_testing_(bootstrap_sender_.borrow(), passive_abi);
  }

  return zx::ok(passive_abi);
}

}  // namespace driver_loader