src/gallium/auxiliary/gallivm/lp_bld_fuchsia_memory_mapper.cpp

/*
 * Copyright © 2023 Google, LLC
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "lp_bld_fuchsia_memory_mapper.hpp"

#include <fidl/fuchsia.vulkan.loader/cpp/wire.h>
#include <lib/component/incoming/cpp/protocol.h>

#include <zircon/process.h>
#include <zircon/status.h>
#include <zircon/syscalls.h>
#include "lib/zx/vmo.h"

#include "llvm/Support/Process.h"

using llvm::sys::MemoryBlock;
using llvm::sys::Process;

// TODO(https://fxbug.dev/321122796): Replace fprintf() with Mesa-idiomatic logging.

namespace {

static_assert(LLVM_ENABLE_THREADS);

int getZirconProtectionFlags(unsigned Flags)
{
   switch (Flags) {
   case llvm::sys::Memory::MF_READ:
      return ZX_VM_FLAG_PERM_READ;
   case llvm::sys::Memory::MF_WRITE:
      return ZX_VM_FLAG_PERM_WRITE;
   case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE:
      return ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE;
   case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_EXEC:
      return ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_EXECUTE;
   case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE | llvm::sys::Memory::MF_EXEC:
      return ZX_VM_FLAG_PERM_READ | ZX_VM_FLAG_PERM_WRITE | ZX_VM_FLAG_PERM_EXECUTE;
   case llvm::sys::Memory::MF_EXEC:
      return ZX_VM_FLAG_PERM_EXECUTE;
   default:
      // TODO(https://fxbug.dev/321122796): use Mesa-idiomatic logging.
      llvm_unreachable("Illegal memory protection flag specified!");
   }
   // Provide a default return value as required by some compilers.
   return 0;
}

}

LavapipeMemoryMapper::LavapipeMemoryMapper()
{
    const zx::result client_end_or = component::Connect<fuchsia_vulkan_loader::Loader>();
    if (client_end_or.is_error()) {
        fprintf(stderr, "LavapipeMemoryMapper failed to connect to Vulkan Loader: %s\n",
                client_end_or.status_string());
        return;
    }

    auto result = fidl::WireCall(*client_end_or)->GetVmexResource();
    if (!result.ok()) {
        fprintf(stderr,
                "LavapipeMemoryMapper failed to get vmex from Vulkan Loader: %s\n",
                result.status_string());
        return;
    }

    vmex_ = std::move(result.value()->resource);
    if (!vmex_) {
        fprintf(stderr, "LavapipeMemoryMapper VMEX request succeeded but it is NULL!\n");
    }
}

llvm::sys::MemoryBlock
LavapipeMemoryMapper::allocateMappedMemory(llvm::SectionMemoryManager::AllocationPurpose Purpose,
                                           size_t NumBytes,
                                           const llvm::sys::MemoryBlock* const NearBlock,
                                           unsigned Flags, std::error_code& EC)
{
    allocation_count_++;

    if (allocation_count_ % 100 == 0) {
        fprintf(stderr, "Lavapipe-LLVM allocations: %zu  releases: %zu  outstanding: %zi\n",
                allocation_count_, release_count_, allocation_count_ - release_count_);
    }

    EC = std::error_code();
    if (NumBytes == 0) {
        fprintf(stderr, "LavapipeMemoryManager::allocateMappedMemory: NumBytes == 0\n");

        return MemoryBlock();
    }

    int Protect = getZirconProtectionFlags(Flags);
    if (Purpose == llvm::SectionMemoryManager::AllocationPurpose::Code) {
        Protect |= ZX_VM_FLAG_PERM_EXECUTE;
    }

    // Use near hint (if available) and the page size to set a page-aligned starting address.
    uintptr_t Start =
        NearBlock ? reinterpret_cast<uintptr_t>(NearBlock->base()) + NearBlock->allocatedSize()
                    : 0;
    static const size_t PageSize = Process::getPageSizeEstimate();
    const size_t NumPages = (NumBytes + PageSize - 1) / PageSize;
    const size_t MappingSize = PageSize * NumPages;

    if (Start && Start % PageSize)
        Start += PageSize - Start % PageSize;

    zx_handle_t VMO;
    zx_status_t status = _zx_vmo_create(MappingSize, 0, &VMO);
    if (status != ZX_OK) {
        EC = std::error_code(status, std::generic_category());

        fprintf(stderr,
                "LavapipeMemoryManager::allocateMappedMemory: error in zx_vmo_create: %s\n",
                zx_status_get_string(status));

        return MemoryBlock();
    }

    if (Protect & ZX_VM_FLAG_PERM_EXECUTE) {
        status = zx_vmo_replace_as_executable(VMO, vmex_.get(), &VMO);
        if (status != ZX_OK) {
            fprintf(stderr,
                    "LavapipeMemoryManager::allocateMappedMemory: "
                    "zx_vmo_replace_as_executable() failed: %s\n", zx_status_get_string(status));

            EC = std::error_code(status, std::generic_category());
            return MemoryBlock();
        }
    }

    // If anything below here fails, delete the VMO before returning to avoid leakage.
    zx::vmo VMO_raii(VMO);

    const char* vmo_name = "[lavapipe-llvm]";
    status = zx_object_set_property(VMO, ZX_PROP_NAME, vmo_name, strlen(vmo_name));
    if (status != ZX_OK) {
        // Handle error.  This isn't good, but it's not a show-stopper.
        fprintf(stderr, "LavapipeMemoryManager::allocateMappedMemory: failed to set VMO name");
    }

    uintptr_t Addr;
    status =
        zx_vmar_map(_zx_vmar_root_self(), /*options=*/Protect, /*vmar_offset=*/0, /*vmo=*/VMO,
                    /*vmo_offset=*/0, /*len=*/MappingSize, /*mapped_address=*/&Addr);
    if (status != ZX_OK) {
        if (NearBlock) { // Try again without a near hint
            fprintf(stderr,
                    "LavapipeMemoryManager::allocateMappedMemory: retrying without near "
                    "hint after error: %s\n",
                    zx_status_get_string(status));
            return allocateMappedMemory(Purpose, NumBytes, nullptr, Flags, EC);
        }

        EC = std::error_code(status, std::generic_category());

        fprintf(stderr,
                "LavapipeMemoryManager::allocateMappedMemory: mapping failed with error: %s "
                "root-vmar: %d _root-vmar: %d \n",
                zx_status_get_string(status), zx_vmar_root_self(), _zx_vmar_root_self());

        return MemoryBlock();
    }

    MemoryBlock Result(reinterpret_cast<void*>(Addr), MappingSize, Flags);

    // Rely on protectMappedMemory to invalidate instruction cache.
    if (Flags & llvm::sys::Memory::MF_EXEC) {
        EC = protectMappedMemory(Result, Flags);
        if (EC != std::error_code()) {
            fprintf(stderr,
                    "LavapipeMemoryManager::allocateMappedMemory: failed to protect memory: %d\n",
                    EC.value());

            return MemoryBlock();
        }
    }

    return Result;
}

std::error_code
LavapipeMemoryMapper::protectMappedMemory(const llvm::sys::MemoryBlock& Block, unsigned Flags)
{
    if (!Flags) {
        fprintf(stderr, "LavapipeMemoryManager::protectMappedMemory failed because no flags set\n");

        return std::error_code(EINVAL, std::generic_category());
    }

    static const llvm::Align PageSize = llvm::Align(Process::getPageSizeEstimate());
    if (Block.base() == nullptr || Block.allocatedSize() == 0) {
        fprintf(stderr,
                "LavapipeMemoryManager::protectMappedMemory failed because "
                "address or allocated size is zero\n");

        return std::error_code();
    }

    int Protect = getZirconProtectionFlags(Flags);
    uintptr_t Start = alignAddr((const uint8_t*)Block.base() - PageSize.value() + 1, PageSize);
    uintptr_t End = alignAddr((const uint8_t*)Block.base() + Block.allocatedSize(), PageSize);

    bool InvalidateCache = (Flags & llvm::sys::Memory::MF_EXEC);

#if defined(__arm__) || defined(__aarch64__)
    // Certain ARM implementations treat icache clear instruction as a memory read,
    // and CPU segfaults on trying to clear cache on !PROT_READ page.  Therefore we need
    // to temporarily add PROT_READ for the sake of flushing the instruction caches.
    if (InvalidateCache && !(Protect & PROT_READ)) {
        Memory::InvalidateInstructionCache(Block.base(), Block.allocatedSize());
        InvalidateCache = false;
    }
#endif

    zx_status_t status = zx_vmar_protect(_zx_vmar_root_self(), /*options=*/Protect,
                                        /*addr=*/(uintptr_t)Start, /*len=*/End - Start);
    if (status != ZX_OK) {
        fprintf(stderr,
            "LavapipeMemoryManager::protectMappedMemory: zx_vmar_protect() failed with error: "
            "%s   options: %u   addr: %p  len: %p\n",
            zx_status_get_string(status), Protect, (void*)Start, (void*)(End - Start));

        return std::error_code(status, std::generic_category());
    }

    if (Flags & llvm::sys::Memory::MF_EXEC)
        llvm::sys::Memory::InvalidateInstructionCache(Block.base(), Block.allocatedSize());

    return std::error_code();
}

std::error_code LavapipeMemoryMapper::releaseMappedMemory(llvm::sys::MemoryBlock& M)
{
    release_count_++;

    return llvm::sys::Memory::releaseMappedMemory(M);
}