// Copyright 2017 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "address_manager.h"

#include <chrono>

#include "address_space.h"
#include "msd_arm_connection.h"
#include "platform_barriers.h"

// Normal memory, outer non-cacheable, inner cacheable with
// implementation-defined allocation. The definition of this is similar to
// normal LPAE memory attributes, but is undocumented.
constexpr uint8_t kMmuNormalMemoryAttr = 0x48;
// Memory with this attribute is also outer cacheable with
// implementation-defined allocation.
constexpr uint8_t kMmuOuterCacheableMemoryAttr = 0x88;

// The memory attribute register has 8 8-bit slots.
static constexpr uint64_t SlotAttribute(int slot, uint8_t attributes)
{
    return static_cast<uint64_t>(attributes) << (slot * 8);
}

constexpr uint64_t kMemoryAttributes =
    SlotAttribute(AddressSpace::kNormalMemoryAttributeSlot, kMmuNormalMemoryAttr) |
    SlotAttribute(AddressSpace::kOuterCacheableAttributeSlot, kMmuOuterCacheableMemoryAttr);

AddressManager::AddressManager(Owner* owner, uint32_t address_slot_count) : owner_(owner)
{
    address_slots_.resize(address_slot_count);
    for (uint32_t i = 0; i < address_slot_count; i++) {
        registers_.push_back(std::make_unique<HardwareSlot>(i));
    }
}

bool AddressManager::AssignAddressSpace(MsdArmAtom* atom)
{
    DASSERT(!atom->address_slot_mapping());
    auto connection = atom->connection().lock();
    if (!connection)
        return false;
    if (connection->address_space_lost())
        return false;

    std::shared_ptr<AddressSlotMapping> mapping = AllocateMappingForAddressSpace(connection);
    atom->set_address_slot_mapping(mapping);

    return mapping ? true : false;
}

void AddressManager::AtomFinished(MsdArmAtom* atom)
{
    if (!atom->address_slot_mapping())
        return;
    atom->set_address_slot_mapping(nullptr);
    address_slot_free_.notify_one();
}

std::shared_ptr<AddressSlotMapping> AddressManager::GetMappingForSlot(uint32_t slot_number)
{
    std::lock_guard<std::mutex> lock(address_slot_lock_);
    auto& slot = address_slots_[slot_number];
    return slot.mapping.lock();
}

std::shared_ptr<AddressSlotMapping>
AddressManager::GetMappingForAddressSpaceUnlocked(const AddressSpace* address_space)
{
    DASSERT(address_space);
    for (size_t i = 0; i < address_slots_.size(); ++i) {
        AddressSlot& slot = address_slots_[i];
        if (slot.address_space != address_space)
            continue;
        auto mapping = slot.mapping.lock();
        if (!mapping) {
            mapping = std::make_shared<AddressSlotMapping>(
                i, std::static_pointer_cast<MsdArmConnection>(address_space->owner()));
            slot.mapping = mapping;
        }
        return mapping;
    }

    return nullptr;
}

void AddressManager::FlushAddressMappingRange(AddressSpace* address_space, uint64_t start,
                                              uint64_t length, bool synchronous)
{
    HardwareSlot* slot;
    std::shared_ptr<AddressSlotMapping> mapping;
    {
        std::lock_guard<std::mutex> lock(address_slot_lock_);
        mapping = AddressManager::GetMappingForAddressSpaceUnlocked(address_space);
        if (!mapping)
            return;
        slot = registers_[mapping->slot_number()].get();
        // Grab the hardware lock inside the address slot lock so we can be
        // sure the address slot still maps to the same address space.
        // std::unique_lock can't be used because it interacts poorly with
        // thread-safety analysis
        slot->lock.lock();
    }
    slot->FlushMmuRange(owner_->register_io(), start, length, synchronous);

    // The mapping will be released before the hardware lock, so that we
    // can be sure that the mapping will be expired after ReleaseSpaceMappings
    // acquires the hardware lock.
    mapping.reset();
    address_slot_free_.notify_one();
    slot->lock.unlock();
}

void AddressManager::UnlockAddressSpace(AddressSpace* address_space)
{
    HardwareSlot* slot;
    std::shared_ptr<AddressSlotMapping> mapping;
    {
        std::lock_guard<std::mutex> lock(address_slot_lock_);
        mapping = AddressManager::GetMappingForAddressSpaceUnlocked(address_space);
        if (!mapping)
            return;
        slot = registers_[mapping->slot_number()].get();
        // Grab the hardware lock inside the address slot lock so we can be
        // sure the address slot still maps to the same address space.
        // std::unique_lock can't be used because it interacts poorly with
        // thread-safety analysis
        slot->lock.lock();
    }
    slot->UnlockMmu(owner_->register_io());

    // The mapping will be released before the hardware lock, so that we
    // can be sure that the mapping will be expired after ReleaseSpaceMappings
    // acquires the hardware lock.
    mapping.reset();
    address_slot_free_.notify_one();
    slot->lock.unlock();
}

// Disable thread safety analysis because it doesn't understand unique_lock.
std::shared_ptr<AddressSlotMapping> AddressManager::AllocateMappingForAddressSpace(
    std::shared_ptr<MsdArmConnection> connection) __TA_NO_THREAD_SAFETY_ANALYSIS
{
    std::unique_lock<std::mutex> lock(address_slot_lock_);
    while (true) {
        std::shared_ptr<AddressSlotMapping> mapping =
            GetMappingForAddressSpaceUnlocked(connection->const_address_space());
        if (mapping)
            return mapping;

        // Allocate new mapping (trying to avoid evicting).
        for (size_t i = 0; i < address_slots_.size(); ++i) {
            if (!address_slots_[i].address_space)
                return AssignToSlot(connection, i);
        }

        // TODO(MA-386): Evict the LRU slot.
        for (size_t i = 0; i < address_slots_.size(); ++i) {
            if (address_slots_[i].mapping.expired())
                return AssignToSlot(connection, i);
        }

        // There are normally 8 hardware address slots but only 6 jobs can be running in hardware at
        // a time (and also the profiler can use an address slot). So the only way we can be
        // completely out of address slots is that a connection thread is flushing the MMU. Because
        // of that there's no deadlock if we block the device thread, because the connection can
        // finish flushing and release its mapping without the device thread. Starvation is still
        // possible, though.
        if (address_slot_free_.wait_for(lock,
                                        std::chrono::seconds(acquire_slot_timeout_seconds_)) ==
            std::cv_status::timeout) {
            magma::log(magma::LOG_WARNING, "Timeout waiting for address slot");
            return DRETP(nullptr, "Timeout waiting for address slot");
        }
    }
}

std::shared_ptr<AddressSlotMapping>
AddressManager::AssignToSlot(std::shared_ptr<MsdArmConnection> connection, uint32_t slot_number)
{
    DLOG("Assigning connection %p to slot %d\n", connection.get(), slot_number);
    AddressSlot& slot = address_slots_[slot_number];
    HardwareSlot& hardware_slot = *registers_[slot_number];
    std::lock_guard<std::mutex> lock(hardware_slot.lock);

    auto old_address_space = slot.address_space;
    magma::RegisterIo* io = owner_->register_io();
    if (old_address_space)
        hardware_slot.InvalidateSlot(io);
    auto mapping = std::make_shared<AddressSlotMapping>(slot_number, connection);
    slot.mapping = mapping;
    slot.address_space = connection->const_address_space();

    registers::AsRegisters& as_reg = hardware_slot.registers;
    hardware_slot.WaitForMmuIdle(io);

    uint64_t translation_table_entry = connection->const_address_space()->translation_table_entry();
    as_reg.TranslationTable().FromValue(translation_table_entry).WriteTo(io);

    as_reg.MemoryAttributes().FromValue(kMemoryAttributes).WriteTo(io);

    as_reg.Command().FromValue(registers::AsCommand::kCmdUpdate).WriteTo(io);
    return mapping;
}

void AddressManager::ReleaseSpaceMappings(const AddressSpace* address_space)
{
    std::lock_guard<std::mutex> lock(address_slot_lock_);
    for (size_t i = 0; i < address_slots_.size(); ++i) {
        AddressSlot& slot = address_slots_[i];
        if (slot.address_space != address_space)
            continue;
        // Grab lock to ensure the registers aren't being modified during
        // invalidate.
        HardwareSlot& hardware_slot = *registers_[i];
        std::lock_guard<std::mutex> lock(hardware_slot.lock);
        DASSERT(slot.mapping.expired());
        hardware_slot.InvalidateSlot(owner_->register_io());
        slot.address_space = nullptr;
    }
}

void AddressManager::HardwareSlot::InvalidateSlot(magma::RegisterIo* io)
{
    WaitForMmuIdle(io);
    constexpr uint64_t kFullAddressSpaceSize = 1ul << AddressSpace::kVirtualAddressSize;
    FlushMmuRange(io, 0, kFullAddressSpaceSize, true);

    registers.TranslationTable().FromValue(0).WriteTo(io);
    registers.MemoryAttributes().FromValue(kMemoryAttributes).WriteTo(io);

    registers.Command().FromValue(registers::AsCommand::kCmdUpdate).WriteTo(io);

    // Ensure CPU reads and writes to buffers in the address space don't happen
    // until after the hardware got the command to finish using the buffer.
    magma::barriers::Barrier();
}

void AddressManager::HardwareSlot::WaitForMmuIdle(magma::RegisterIo* io)
{
    auto status_reg = registers.Status();
    if (!status_reg.ReadFrom(io).reg_value())
        return;

    auto timeout = std::chrono::steady_clock::now() + std::chrono::seconds(5);
    while (status_reg.ReadFrom(io).reg_value() && std::chrono::steady_clock::now() < timeout)
        ;

    uint32_t status = status_reg.ReadFrom(io).reg_value();
    if (status)
        magma::log(magma::LOG_WARNING, "Wait for MMU %d to idle timed out with status 0x%x\n",
                   registers.address_space(), status);
}

void AddressManager::HardwareSlot::FlushMmuRange(magma::RegisterIo* io, uint64_t start,
                                                 uint64_t length, bool synchronous)
{
    DASSERT(magma::is_page_aligned(start));
    uint64_t region = start;
    uint64_t num_pages = length >> PAGE_SHIFT;
    uint8_t log2_num_pages = 0;
    if (num_pages > 0) {
        log2_num_pages = 63 - __builtin_clzl(num_pages);
        if ((1ul << log2_num_pages) < num_pages)
            log2_num_pages++;
    }

    // Ensure page table writes are completed before the hardware tries to
    // access the buffer.
    magma::barriers::WriteBarrier();
    constexpr uint32_t kRegionLengthOffset = 11;

    // The low 12 bits are used to specify how many pages are to be locked in
    // this operation.
    static_assert(kRegionLengthOffset + 64 < PAGE_SIZE, "maximum region length is too large");

    uint8_t region_width = log2_num_pages + kRegionLengthOffset;

    region |= region_width;
    registers.LockAddress().FromValue(region).WriteTo(io);
    registers.Command().FromValue(registers::AsCommand::kCmdLock).WriteTo(io);
    WaitForMmuIdle(io);
    if (synchronous) {
        // Both invalidate the TLB entries and throw away data in the L2 cache
        // corresponding to them, or otherwise the cache may be written back to
        // memory after the memory's started being used for something else.
        registers.Command().FromValue(registers::AsCommand::kCmdFlushMem).WriteTo(io);
    } else {
        registers.Command().FromValue(registers::AsCommand::kCmdFlushPageTable).WriteTo(io);
    }
    WaitForMmuIdle(io);
    // If a page range was unmapped, ensure the hardware is no longer accessing
    // it before any CPU reads or writes to the memory.
    magma::barriers::Barrier();
}

void AddressManager::HardwareSlot::UnlockMmu(magma::RegisterIo* io)
{
    WaitForMmuIdle(io);
    registers.Command().FromValue(registers::AsCommand::kCmdUnlock).WriteTo(io);
}