drivers/gpu/msd-intel-gen/src/ppgtt.h - garnet - Git at Google

 // Copyright 2017 The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 #ifndef PPGTT_H
 #define PPGTT_H

 #include "address_space.h"
 #include "magma_util/address_space_allocator.h"
 #include "magma_util/register_io.h"
 #include "platform_buffer.h"

 using gen_pde_t = uint64_t;
 using gen_pdpe_t = uint64_t;
 using gen_pml4e_t = uint64_t;

 static inline gen_pde_t gen_pde_encode(uint64_t bus_addr)
 {
     return bus_addr | PAGE_RW | PAGE_PRESENT;
 }

 static inline gen_pdpe_t gen_pdpe_encode(uint64_t bus_addr)
 {
     return bus_addr | PAGE_RW | PAGE_PRESENT;
 }

 static inline gen_pml4e_t gen_pml4_encode(uint64_t bus_addr)
 {
     return bus_addr | PAGE_RW | PAGE_PRESENT;
 }

 class PerProcessGtt : public AddressSpace {
 public:
     class Owner : public AddressSpace::Owner {
     };

     static std::unique_ptr<PerProcessGtt> Create(Owner* owner);

     uint64_t Size() const override { return kSize; }

     static void InitPrivatePat(magma::RegisterIo* reg_io);

     uint64_t get_pml4_bus_addr() { return pml4_table_->bus_addr(); }

     // Legacy 48-bit ppgtt = 512 PDPs; each PDP has 512 PDs; each PD handles 1GB
     // (512 * 512 * 4096)
     static constexpr uint64_t kPml4Entries = 512;

     static constexpr uint64_t kPageDirectoryPtrShift = 9;
     static constexpr uint64_t kPageDirectoryPtrEntries = 1 << kPageDirectoryPtrShift;
     static constexpr uint64_t kPageDirectoryPtrMask = kPageDirectoryPtrEntries - 1;

     static constexpr uint64_t kPageDirectoryShift = 9;
     static constexpr uint64_t kPageDirectoryEntries = 1 << kPageDirectoryShift;
     static constexpr uint64_t kPageDirectoryMask = kPageDirectoryEntries - 1;

     static constexpr uint64_t kPageTableShift = 9;
     static constexpr uint64_t kPageTableEntries = 1 << kPageTableShift;
     static constexpr uint64_t kPageTableMask = kPageTableEntries - 1;

     // These structures are overlayed onto mapped buffers
     struct PageTableGpu {
         gen_pte_t entry[kPageTableEntries];
     };

     struct PageDirectoryTableGpu {
         gen_pde_t entry[kPageDirectoryEntries];
     };

     struct PageDirectoryPtrTableGpu {
         gen_pdpe_t entry[kPageDirectoryPtrEntries];
     };

     struct Pml4TableGpu {
         gen_pml4e_t entry[kPml4Entries];
     };

     class Page {
     public:
         bool Init(Owner* owner)
         {
             buffer_ = magma::PlatformBuffer::Create(PAGE_SIZE, "ppgtt table");
             if (!buffer_)
                 return DRETF(false, "couldn't create buffer");

             if (!buffer_->MapCpu(&mapping_))
                 return DRETF(false, "failed to map cpu");

             bus_mapping_ = owner->GetBusMapper()->MapPageRangeBus(buffer_.get(), 0, 1);
             if (!bus_mapping_)
                 return DRETF(false, "failed to map page range bus");

             owner_ = owner;
             return true;
         }

         void* mapping() { return mapping_; }

         uint64_t bus_addr() { return bus_mapping_->Get()[0]; }

         Owner* owner() { return owner_; }

     private:
         std::unique_ptr<magma::PlatformBuffer> buffer_;
         void* mapping_ = nullptr;
         std::unique_ptr<magma::PlatformBusMapper::BusMapping> bus_mapping_;
         Owner* owner_ = nullptr;
     };

     class PageTable : public Page {
     public:
         gen_pte_t* page_table_entry(uint32_t page_index)
         {
             DASSERT(page_index < kPageTableEntries);
             return &reinterpret_cast<PageTableGpu*>(mapping())->entry[page_index];
         }

         static std::unique_ptr<PageTable> Create(Owner* owner, std::shared_ptr<Page> scratch_page);

         std::shared_ptr<Page> scratch_page() { return scratch_page_; }

     private:
         PageTable(std::shared_ptr<Page> scratch_page) : scratch_page_(std::move(scratch_page)) {}

         std::shared_ptr<Page> scratch_page_;
     };

     class PageDirectory : public Page {
     public:
         PageDirectoryTableGpu* page_directory_table_gpu()
         {
             return reinterpret_cast<PageDirectoryTableGpu*>(mapping());
         }

         PageTable* page_table(uint32_t index, bool alloc)
         {
             DASSERT(index < kPageDirectoryEntries);
             if (!page_tables_[index]) {
                 if (!alloc)
                     return scratch_table_.get();
                 page_tables_[index] = PageTable::Create(owner(), scratch_table_->scratch_page());
                 if (!page_tables_[index])
                     return DRETP(nullptr, "couldn't create page table");
                 page_directory_table_gpu()->entry[index] =
                     gen_pde_encode(page_tables_[index]->bus_addr());
             }
             return page_tables_[index].get();
         }

         gen_pte_t* page_table_entry(uint32_t page_directory_index, uint32_t page_table_index)
         {
             auto table = page_table(page_directory_index, true);
             if (!table)
                 return nullptr;
             return table->page_table_entry(page_table_index);
         }

         static std::unique_ptr<PageDirectory> Create(Owner* owner,
                                                      std::shared_ptr<PageTable> scratch_table);

         std::shared_ptr<PageTable> scratch_table() { return scratch_table_; }

     private:
         PageDirectory(std::shared_ptr<PageTable> scratch_table)
             : page_tables_(kPageDirectoryEntries), scratch_table_(std::move(scratch_table))
         {
         }

         std::vector<std::unique_ptr<PageTable>> page_tables_;
         std::shared_ptr<PageTable> scratch_table_;
     };

     class PageDirectoryPtrTable : public Page {
     public:
         PageDirectoryPtrTableGpu* page_directory_ptr_table_gpu()
         {
             return reinterpret_cast<PageDirectoryPtrTableGpu*>(mapping());
         }

         PageDirectory* page_directory(uint32_t index, bool alloc)
         {
             DASSERT(index < kPageDirectoryPtrEntries);
             if (!page_directories_[index]) {
                 if (!alloc)
                     return scratch_dir_.get();
                 page_directories_[index] =
                     PageDirectory::Create(owner(), scratch_dir_->scratch_table());
                 if (!page_directories_[index])
                     return DRETP(nullptr, "couldn't create page directory");
                 page_directory_ptr_table_gpu()->entry[index] =
                     gen_pdpe_encode(page_directories_[index]->bus_addr());
             }
             return page_directories_[index].get();
         }

         gen_pte_t* page_table_entry(uint32_t page_directory_ptr_index,
                                     uint32_t page_directory_index, uint32_t page_table_index)
         {
             auto dir = page_directory(page_directory_ptr_index, true);
             if (!dir)
                 return nullptr;
             return dir->page_table_entry(page_directory_index, page_table_index);
         }

         static std::unique_ptr<PageDirectoryPtrTable>
         Create(Owner* owner, std::shared_ptr<PageDirectory> scratch_dir);

         std::shared_ptr<PageDirectory> scratch_dir() { return scratch_dir_; }

     private:
         PageDirectoryPtrTable(std::shared_ptr<PageDirectory> scratch_dir)
             : page_directories_(kPageDirectoryPtrEntries), scratch_dir_(std::move(scratch_dir))
         {
         }

         std::vector<std::unique_ptr<PageDirectory>> page_directories_;
         std::shared_ptr<PageDirectory> scratch_dir_;
     };

     class Pml4Table : public Page {
     public:
         Pml4TableGpu* pml4_table_gpu() { return reinterpret_cast<Pml4TableGpu*>(mapping()); }

         PageDirectoryPtrTable* page_directory_ptr(uint32_t index, bool alloc)
         {
             DASSERT(index < kPml4Entries);
             if (!directory_ptrs_[index]) {
                 if (!alloc)
                     return scratch_directory_ptr_.get();
                 directory_ptrs_[index] =
                     PageDirectoryPtrTable::Create(owner(), scratch_directory_ptr_->scratch_dir());
                 if (!directory_ptrs_[index])
                     return DRETP(nullptr, "couldn't create page directory ptr table");
                 pml4_table_gpu()->entry[index] =
                     gen_pml4_encode(directory_ptrs_[index]->bus_addr());
             }
             return directory_ptrs_[index].get();
         }

         PageDirectory* page_directory(uint32_t pml4_index, uint32_t page_directory_ptr_index)
         {
             auto dir_ptr = page_directory_ptr(pml4_index, true);
             if (!dir_ptr)
                 return nullptr;
             return dir_ptr->page_directory(page_directory_ptr_index, true);
         }

         uint64_t scratch_page_bus_addr() { return scratch_page_bus_addr_; }

         static std::unique_ptr<Pml4Table> Create(Owner* owner);

     private:
         Pml4Table(uint64_t scratch_page_bus_addr,
                   std::unique_ptr<PageDirectoryPtrTable> scratch_directory_ptr)
             : directory_ptrs_(kPml4Entries + 1), scratch_page_bus_addr_(scratch_page_bus_addr),
               scratch_directory_ptr_(std::move(scratch_directory_ptr))
         {
         }

         std::vector<std::unique_ptr<PageDirectoryPtrTable>> directory_ptrs_;
         uint64_t scratch_page_bus_addr_;
         std::unique_ptr<PageDirectoryPtrTable> scratch_directory_ptr_;
     };

     Pml4Table* pml4_table() { return pml4_table_.get(); }

 private:
     PerProcessGtt(Owner* owner, std::unique_ptr<Pml4Table> pml4_table);

     static constexpr uint64_t kSize = kPml4Entries * kPageDirectoryPtrEntries *
                                       kPageDirectoryEntries * kPageTableEntries * PAGE_SIZE;

     static constexpr uint32_t kOverfetchPageCount = 1;
     static constexpr uint32_t kGuardPageCount = 8;

     static_assert(kSize == 1ull << 48, "ppgtt size calculation");

     // AddressSpace overrides
     bool AllocLocked(size_t size, uint8_t align_pow2, uint64_t* addr_out) override;
     bool FreeLocked(uint64_t addr) override;

     bool InsertLocked(uint64_t addr, magma::PlatformBusMapper::BusMapping* buffer) override;
     bool ClearLocked(uint64_t start, uint64_t length) override;

     std::unique_ptr<Pml4Table> pml4_table_;
     std::unique_ptr<magma::AddressSpaceAllocator> allocator_;

     // For testing
     friend class TestPerProcessGtt;
     gen_pte_t get_pte(gpu_addr_t gpu_addr);
 };

 #endif // PPGTT_H
	// Copyright 2017 The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	#ifndef PPGTT_H
	#define PPGTT_H

	#include "address_space.h"
	#include "magma_util/address_space_allocator.h"
	#include "magma_util/register_io.h"
	#include "platform_buffer.h"

	using gen_pde_t = uint64_t;
	using gen_pdpe_t = uint64_t;
	using gen_pml4e_t = uint64_t;

	static inline gen_pde_t gen_pde_encode(uint64_t bus_addr)
	{
	return bus_addr \| PAGE_RW \| PAGE_PRESENT;
	}

	static inline gen_pdpe_t gen_pdpe_encode(uint64_t bus_addr)
	{
	return bus_addr \| PAGE_RW \| PAGE_PRESENT;
	}

	static inline gen_pml4e_t gen_pml4_encode(uint64_t bus_addr)
	{
	return bus_addr \| PAGE_RW \| PAGE_PRESENT;
	}

	class PerProcessGtt : public AddressSpace {
	public:
	class Owner : public AddressSpace::Owner {
	};

	static std::unique_ptr<PerProcessGtt> Create(Owner* owner);

	uint64_t Size() const override { return kSize; }

	static void InitPrivatePat(magma::RegisterIo* reg_io);

	uint64_t get_pml4_bus_addr() { return pml4_table_->bus_addr(); }

	// Legacy 48-bit ppgtt = 512 PDPs; each PDP has 512 PDs; each PD handles 1GB
	// (512 * 512 * 4096)
	static constexpr uint64_t kPml4Entries = 512;

	static constexpr uint64_t kPageDirectoryPtrShift = 9;
	static constexpr uint64_t kPageDirectoryPtrEntries = 1 << kPageDirectoryPtrShift;
	static constexpr uint64_t kPageDirectoryPtrMask = kPageDirectoryPtrEntries - 1;

	static constexpr uint64_t kPageDirectoryShift = 9;
	static constexpr uint64_t kPageDirectoryEntries = 1 << kPageDirectoryShift;
	static constexpr uint64_t kPageDirectoryMask = kPageDirectoryEntries - 1;

	static constexpr uint64_t kPageTableShift = 9;
	static constexpr uint64_t kPageTableEntries = 1 << kPageTableShift;
	static constexpr uint64_t kPageTableMask = kPageTableEntries - 1;

	// These structures are overlayed onto mapped buffers
	struct PageTableGpu {
	gen_pte_t entry[kPageTableEntries];
	};

	struct PageDirectoryTableGpu {
	gen_pde_t entry[kPageDirectoryEntries];
	};

	struct PageDirectoryPtrTableGpu {
	gen_pdpe_t entry[kPageDirectoryPtrEntries];
	};

	struct Pml4TableGpu {
	gen_pml4e_t entry[kPml4Entries];
	};

	class Page {
	public:
	bool Init(Owner* owner)
	{
	buffer_ = magma::PlatformBuffer::Create(PAGE_SIZE, "ppgtt table");
	if (!buffer_)
	return DRETF(false, "couldn't create buffer");

	if (!buffer_->MapCpu(&mapping_))
	return DRETF(false, "failed to map cpu");

	bus_mapping_ = owner->GetBusMapper()->MapPageRangeBus(buffer_.get(), 0, 1);
	if (!bus_mapping_)
	return DRETF(false, "failed to map page range bus");

	owner_ = owner;
	return true;
	}

	void* mapping() { return mapping_; }

	uint64_t bus_addr() { return bus_mapping_->Get()[0]; }

	Owner* owner() { return owner_; }

	private:
	std::unique_ptr<magma::PlatformBuffer> buffer_;
	void* mapping_ = nullptr;
	std::unique_ptr<magma::PlatformBusMapper::BusMapping> bus_mapping_;
	Owner* owner_ = nullptr;
	};

	class PageTable : public Page {
	public:
	gen_pte_t* page_table_entry(uint32_t page_index)
	{
	DASSERT(page_index < kPageTableEntries);
	return &reinterpret_cast<PageTableGpu*>(mapping())->entry[page_index];
	}

	static std::unique_ptr<PageTable> Create(Owner* owner, std::shared_ptr<Page> scratch_page);

	std::shared_ptr<Page> scratch_page() { return scratch_page_; }

	private:
	PageTable(std::shared_ptr<Page> scratch_page) : scratch_page_(std::move(scratch_page)) {}

	std::shared_ptr<Page> scratch_page_;
	};

	class PageDirectory : public Page {
	public:
	PageDirectoryTableGpu* page_directory_table_gpu()
	{
	return reinterpret_cast<PageDirectoryTableGpu*>(mapping());
	}

	PageTable* page_table(uint32_t index, bool alloc)
	{
	DASSERT(index < kPageDirectoryEntries);
	if (!page_tables_[index]) {
	if (!alloc)
	return scratch_table_.get();
	page_tables_[index] = PageTable::Create(owner(), scratch_table_->scratch_page());
	if (!page_tables_[index])
	return DRETP(nullptr, "couldn't create page table");
	page_directory_table_gpu()->entry[index] =
	gen_pde_encode(page_tables_[index]->bus_addr());
	}
	return page_tables_[index].get();
	}

	gen_pte_t* page_table_entry(uint32_t page_directory_index, uint32_t page_table_index)
	{
	auto table = page_table(page_directory_index, true);
	if (!table)
	return nullptr;
	return table->page_table_entry(page_table_index);
	}

	static std::unique_ptr<PageDirectory> Create(Owner* owner,
	std::shared_ptr<PageTable> scratch_table);

	std::shared_ptr<PageTable> scratch_table() { return scratch_table_; }

	private:
	PageDirectory(std::shared_ptr<PageTable> scratch_table)
	: page_tables_(kPageDirectoryEntries), scratch_table_(std::move(scratch_table))
	{
	}

	std::vector<std::unique_ptr<PageTable>> page_tables_;
	std::shared_ptr<PageTable> scratch_table_;
	};

	class PageDirectoryPtrTable : public Page {
	public:
	PageDirectoryPtrTableGpu* page_directory_ptr_table_gpu()
	{
	return reinterpret_cast<PageDirectoryPtrTableGpu*>(mapping());
	}

	PageDirectory* page_directory(uint32_t index, bool alloc)
	{
	DASSERT(index < kPageDirectoryPtrEntries);
	if (!page_directories_[index]) {
	if (!alloc)
	return scratch_dir_.get();
	page_directories_[index] =
	PageDirectory::Create(owner(), scratch_dir_->scratch_table());
	if (!page_directories_[index])
	return DRETP(nullptr, "couldn't create page directory");
	page_directory_ptr_table_gpu()->entry[index] =
	gen_pdpe_encode(page_directories_[index]->bus_addr());
	}
	return page_directories_[index].get();
	}

	gen_pte_t* page_table_entry(uint32_t page_directory_ptr_index,
	uint32_t page_directory_index, uint32_t page_table_index)
	{
	auto dir = page_directory(page_directory_ptr_index, true);
	if (!dir)
	return nullptr;
	return dir->page_table_entry(page_directory_index, page_table_index);
	}

	static std::unique_ptr<PageDirectoryPtrTable>
	Create(Owner* owner, std::shared_ptr<PageDirectory> scratch_dir);

	std::shared_ptr<PageDirectory> scratch_dir() { return scratch_dir_; }

	private:
	PageDirectoryPtrTable(std::shared_ptr<PageDirectory> scratch_dir)
	: page_directories_(kPageDirectoryPtrEntries), scratch_dir_(std::move(scratch_dir))
	{
	}

	std::vector<std::unique_ptr<PageDirectory>> page_directories_;
	std::shared_ptr<PageDirectory> scratch_dir_;
	};

	class Pml4Table : public Page {
	public:
	Pml4TableGpu* pml4_table_gpu() { return reinterpret_cast<Pml4TableGpu*>(mapping()); }

	PageDirectoryPtrTable* page_directory_ptr(uint32_t index, bool alloc)
	{
	DASSERT(index < kPml4Entries);
	if (!directory_ptrs_[index]) {
	if (!alloc)
	return scratch_directory_ptr_.get();
	directory_ptrs_[index] =
	PageDirectoryPtrTable::Create(owner(), scratch_directory_ptr_->scratch_dir());
	if (!directory_ptrs_[index])
	return DRETP(nullptr, "couldn't create page directory ptr table");
	pml4_table_gpu()->entry[index] =
	gen_pml4_encode(directory_ptrs_[index]->bus_addr());
	}
	return directory_ptrs_[index].get();
	}

	PageDirectory* page_directory(uint32_t pml4_index, uint32_t page_directory_ptr_index)
	{
	auto dir_ptr = page_directory_ptr(pml4_index, true);
	if (!dir_ptr)
	return nullptr;
	return dir_ptr->page_directory(page_directory_ptr_index, true);
	}

	uint64_t scratch_page_bus_addr() { return scratch_page_bus_addr_; }

	static std::unique_ptr<Pml4Table> Create(Owner* owner);

	private:
	Pml4Table(uint64_t scratch_page_bus_addr,
	std::unique_ptr<PageDirectoryPtrTable> scratch_directory_ptr)
	: directory_ptrs_(kPml4Entries + 1), scratch_page_bus_addr_(scratch_page_bus_addr),
	scratch_directory_ptr_(std::move(scratch_directory_ptr))
	{
	}

	std::vector<std::unique_ptr<PageDirectoryPtrTable>> directory_ptrs_;
	uint64_t scratch_page_bus_addr_;
	std::unique_ptr<PageDirectoryPtrTable> scratch_directory_ptr_;
	};

	Pml4Table* pml4_table() { return pml4_table_.get(); }

	private:
	PerProcessGtt(Owner* owner, std::unique_ptr<Pml4Table> pml4_table);

	static constexpr uint64_t kSize = kPml4Entries * kPageDirectoryPtrEntries *
	kPageDirectoryEntries * kPageTableEntries * PAGE_SIZE;

	static constexpr uint32_t kOverfetchPageCount = 1;
	static constexpr uint32_t kGuardPageCount = 8;

	static_assert(kSize == 1ull << 48, "ppgtt size calculation");

	// AddressSpace overrides
	bool AllocLocked(size_t size, uint8_t align_pow2, uint64_t* addr_out) override;
	bool FreeLocked(uint64_t addr) override;

	bool InsertLocked(uint64_t addr, magma::PlatformBusMapper::BusMapping* buffer) override;
	bool ClearLocked(uint64_t start, uint64_t length) override;

	std::unique_ptr<Pml4Table> pml4_table_;
	std::unique_ptr<magma::AddressSpaceAllocator> allocator_;

	// For testing
	friend class TestPerProcessGtt;
	gen_pte_t get_pte(gpu_addr_t gpu_addr);
	};

	#endif // PPGTT_H