zircon/system/dev/sysmem/sysmem/contiguous_pooled_memory_allocator.cc - fuchsia - Git at Google

 // Copyright 2019 The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 #include "contiguous_pooled_memory_allocator.h"

 #include "macros.h"

 ContiguousPooledMemoryAllocator::ContiguousPooledMemoryAllocator(
     Owner* parent_device, const char* allocation_name, uint64_t size,
     bool is_cpu_accessible)
     : parent_device_(parent_device),
       allocation_name_(allocation_name),
       region_allocator_(RegionAllocator::RegionPool::Create(std::numeric_limits<size_t>::max())),
       size_(size),
       is_cpu_accessible_(is_cpu_accessible) {}

 zx_status_t ContiguousPooledMemoryAllocator::Init(uint32_t alignment_log2) {
   zx::vmo local_contiguous_vmo;
   zx_status_t status = zx::vmo::create_contiguous(
       parent_device_->bti(), size_, alignment_log2, &local_contiguous_vmo);
   if (status != ZX_OK) {
     DRIVER_ERROR("Could allocate contiguous memory, status %d allocation_name_: %s\n",
                  status, allocation_name_);
     return status;
   }

   return InitCommon(std::move(local_contiguous_vmo));
 }

 zx_status_t ContiguousPooledMemoryAllocator::InitPhysical(zx_paddr_t paddr) {
   zx::vmo local_contiguous_vmo;
   zx_status_t status = parent_device_->CreatePhysicalVmo(paddr, size_, &local_contiguous_vmo);
   if (status != ZX_OK) {
     DRIVER_ERROR("Failed to create physical VMO: %d allocation_name_: %s\n",
                  status, allocation_name_);
     return status;
   }

   return InitCommon(std::move(local_contiguous_vmo));
 }

 zx_status_t ContiguousPooledMemoryAllocator::InitCommon(zx::vmo local_contiguous_vmo) {
   zx_status_t status = local_contiguous_vmo.set_property(
       ZX_PROP_NAME, allocation_name_, strlen(allocation_name_));
   if (status != ZX_OK) {
     DRIVER_ERROR("Failed vmo.set_property(ZX_PROP_NAME, ...): %d\n", status);
     return status;
   }

   zx_info_vmo_t info;
   status = local_contiguous_vmo.get_info(ZX_INFO_VMO, &info, sizeof(info), nullptr, nullptr);
   if (status != ZX_OK) {
     DRIVER_ERROR("Failed local_contiguous_vmo.get_info(ZX_INFO_VMO, ...) - status: %d", status);
     return status;
   }
   // Only secure/protected RAM ever uses a physical VMO.  Not all secure/protected RAM uses a
   // physical VMO.
   ZX_DEBUG_ASSERT(ZX_INFO_VMO_TYPE(info.flags) == ZX_INFO_VMO_TYPE_PAGED || !is_cpu_accessible_);
   // Paged VMOs are cached by default.  Physical VMOs are uncached by default.
   ZX_DEBUG_ASSERT((ZX_INFO_VMO_TYPE(info.flags) == ZX_INFO_VMO_TYPE_PAGED) ==
                   (info.cache_policy == ZX_CACHE_POLICY_CACHED));
   // We'd have this assert, except it doesn't work with fake-bti, so for now we trust that when not
   // running a unit test, we have a VMO with info.flags & ZX_INFO_VMO_CONTIGUOUS.
   //ZX_DEBUG_ASSERT(info.flags & ZX_INFO_VMO_CONTIGUOUS);

   // Regardless of CPU or RAM domain, and regardless of contig VMO or physical VMO, if we use the
   // CPU to access the RAM, we want to use the CPU cache.  If we can't use the CPU to access the RAM
   // (on REE side), then we don't want to use the CPU cache.
   //
   // Why we want cached when is_cpu_accessible_:
   //
   // Without setting cached, in addition to presumably being slower, memcpy tends to fail with
   // non-aligned access faults / syscalls that are trying to copy directly to the VMO can fail
   // without it being obvious that it's an underlying non-aligned access fault triggered by memcpy.
   //
   // Why we want uncached when !is_cpu_accessible_:
   //
   // IIUC, it's possible on aarch64 for a cached mapping to protected memory + speculative execution
   // to cause random faults, while a non-cached mapping only faults if a non-cached mapping is
   // actually touched.
   uint32_t desired_cache_policy =
       is_cpu_accessible_ ? ZX_CACHE_POLICY_CACHED : ZX_CACHE_POLICY_UNCACHED;
   if (info.cache_policy != desired_cache_policy) {
     status = local_contiguous_vmo.set_cache_policy(desired_cache_policy);
     if (status != ZX_OK) {
       // TODO(ZX-4807): Ideally we'd set ZX_CACHE_POLICY_UNCACHED when !is_cpu_accessible_, since
       // IIUC on aarch64 it's possible for a cached mapping to secure/protected memory + speculative
       // execution to cause random faults, while an uncached mapping only faults if the uncached
       // mapping is actually touched.  However, currently for a VMO created with
       // zx::vmo::create_contiguous(), the .set_cache_policy() doesn't work because the VMO already
       // has pages.  Cases where !is_cpu_accessible_ include both Init() and InitPhysical(), so we
       // can't rely on local_contiguous_vmo being a physical VMO.
       if (ZX_INFO_VMO_TYPE(info.flags) == ZX_INFO_VMO_TYPE_PAGED) {
         DRIVER_ERROR(
             "Ignoring failure to set_cache_policy() on contig VMO - see ZX-4807 - status: %d\n",
             status);
         status = ZX_OK;
         goto keepGoing;
       }
       DRIVER_ERROR("Failed to set_cache_policy(): %d\n", status);
       return status;
     }
   }
 keepGoing:;

   zx_paddr_t addrs;
   zx::pmt pmt;
   // When running a unit test, the zircon/system/dev/lib/fake-bti provides a fake zx_bti_pin() that
   // should tolerate ZX_BTI_CONTIGUOUS here despite the local_contiguous_vmo not actually having
   // info.flags ZX_INFO_VMO_CONTIGUOUS.
   status = parent_device_->bti().pin(ZX_BTI_PERM_READ | ZX_BTI_PERM_WRITE | ZX_BTI_CONTIGUOUS,
                                      local_contiguous_vmo, 0, size_, &addrs, 1, &pmt);
   if (status != ZX_OK) {
     DRIVER_ERROR("Could not pin memory, status %d\n", status);
     return status;
   }

   start_ = addrs;
   contiguous_vmo_ = std::move(local_contiguous_vmo);
   ralloc_region_t region = {0, size_};
   region_allocator_.AddRegion(region);
   // It is intentional here that ~pmt doesn't imply zx_pmt_unpin().  If sysmem dies, we'll reboot.
   return ZX_OK;
 }

 zx_status_t ContiguousPooledMemoryAllocator::Allocate(uint64_t size, zx::vmo* parent_vmo) {
   RegionAllocator::Region::UPtr region;
   zx::vmo result_parent_vmo;

   // TODO: Use a fragmentation-reducing allocator (such as best fit).
   //
   // The "region" param is an out ref.
   zx_status_t status = region_allocator_.GetRegion(size, ZX_PAGE_SIZE, region);
   if (status != ZX_OK) {
     DRIVER_INFO("GetRegion failed (out of space?) - size: %zu status: %d\n", size, status);
     DumpPoolStats();
     return status;
   }

   // The result_parent_vmo created here is a VMO window to a sub-region of contiguous_vmo_.
   status = contiguous_vmo_.create_child(ZX_VMO_CHILD_SLICE, region->base, size, &result_parent_vmo);
   if (status != ZX_OK) {
     DRIVER_ERROR("Failed vmo.create_child(ZX_VMO_CHILD_SLICE, ...): %d\n", status);
     return status;
   }

   // If you see a sysmem-contig VMO you should know that it doesn't actually
   // take up any space, because the same memory is backed by contiguous_vmo_.
   const char* kSysmemContig = "sysmem-contig";
   status = result_parent_vmo.set_property(ZX_PROP_NAME, kSysmemContig, strlen(kSysmemContig));
   if (status != ZX_OK) {
     DRIVER_ERROR("Failed vmo.set_property(ZX_PROP_NAME, ...): %d\n", status);
     return status;
   }

   regions_.emplace(std::make_pair(result_parent_vmo.get(), std::move(region)));
   *parent_vmo = std::move(result_parent_vmo);
   return ZX_OK;
 }

 zx_status_t ContiguousPooledMemoryAllocator::SetupChildVmo(const zx::vmo& parent_vmo, const zx::vmo& child_vmo) {
   // nothing to do here
   return ZX_OK;
 }

 void ContiguousPooledMemoryAllocator::Delete(zx::vmo parent_vmo) {
   auto it = regions_.find(parent_vmo.get());
   ZX_ASSERT(it != regions_.end());
   regions_.erase(it);
   // ~parent_vmo
 }

 void ContiguousPooledMemoryAllocator::DumpPoolStats() {
   uint64_t unused_size = 0;
   uint64_t max_free_size = 0;
   region_allocator_.WalkAvailableRegions(
       [&unused_size, &max_free_size](const ralloc_region_t* r) -> bool {
         unused_size += r->size;
         max_free_size = std::max(max_free_size, r->size);
         return true;
       });

   DRIVER_ERROR("Contiguous pool unused total: %ld bytes, max free size %ld bytes "
                "AllocatedRegionCount(): %zu AvailableRegionCount(): %zu\n",
                unused_size, max_free_size,
                region_allocator_.AllocatedRegionCount(), region_allocator_.AvailableRegionCount());
 }
	// Copyright 2019 The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	#include "contiguous_pooled_memory_allocator.h"

	#include "macros.h"

	ContiguousPooledMemoryAllocator::ContiguousPooledMemoryAllocator(
	Owner* parent_device, const char* allocation_name, uint64_t size,
	bool is_cpu_accessible)
	: parent_device_(parent_device),
	allocation_name_(allocation_name),
	region_allocator_(RegionAllocator::RegionPool::Create(std::numeric_limits<size_t>::max())),
	size_(size),
	is_cpu_accessible_(is_cpu_accessible) {}

	zx_status_t ContiguousPooledMemoryAllocator::Init(uint32_t alignment_log2) {
	zx::vmo local_contiguous_vmo;
	zx_status_t status = zx::vmo::create_contiguous(
	parent_device_->bti(), size_, alignment_log2, &local_contiguous_vmo);
	if (status != ZX_OK) {
	DRIVER_ERROR("Could allocate contiguous memory, status %d allocation_name_: %s\n",
	status, allocation_name_);
	return status;
	}

	return InitCommon(std::move(local_contiguous_vmo));
	}

	zx_status_t ContiguousPooledMemoryAllocator::InitPhysical(zx_paddr_t paddr) {
	zx::vmo local_contiguous_vmo;
	zx_status_t status = parent_device_->CreatePhysicalVmo(paddr, size_, &local_contiguous_vmo);
	if (status != ZX_OK) {
	DRIVER_ERROR("Failed to create physical VMO: %d allocation_name_: %s\n",
	status, allocation_name_);
	return status;
	}

	return InitCommon(std::move(local_contiguous_vmo));
	}

	zx_status_t ContiguousPooledMemoryAllocator::InitCommon(zx::vmo local_contiguous_vmo) {
	zx_status_t status = local_contiguous_vmo.set_property(
	ZX_PROP_NAME, allocation_name_, strlen(allocation_name_));
	if (status != ZX_OK) {
	DRIVER_ERROR("Failed vmo.set_property(ZX_PROP_NAME, ...): %d\n", status);
	return status;
	}

	zx_info_vmo_t info;
	status = local_contiguous_vmo.get_info(ZX_INFO_VMO, &info, sizeof(info), nullptr, nullptr);
	if (status != ZX_OK) {
	DRIVER_ERROR("Failed local_contiguous_vmo.get_info(ZX_INFO_VMO, ...) - status: %d", status);
	return status;
	}
	// Only secure/protected RAM ever uses a physical VMO. Not all secure/protected RAM uses a
	// physical VMO.
	ZX_DEBUG_ASSERT(ZX_INFO_VMO_TYPE(info.flags) == ZX_INFO_VMO_TYPE_PAGED \|\| !is_cpu_accessible_);
	// Paged VMOs are cached by default. Physical VMOs are uncached by default.
	ZX_DEBUG_ASSERT((ZX_INFO_VMO_TYPE(info.flags) == ZX_INFO_VMO_TYPE_PAGED) ==
	(info.cache_policy == ZX_CACHE_POLICY_CACHED));
	// We'd have this assert, except it doesn't work with fake-bti, so for now we trust that when not
	// running a unit test, we have a VMO with info.flags & ZX_INFO_VMO_CONTIGUOUS.
	//ZX_DEBUG_ASSERT(info.flags & ZX_INFO_VMO_CONTIGUOUS);

	// Regardless of CPU or RAM domain, and regardless of contig VMO or physical VMO, if we use the
	// CPU to access the RAM, we want to use the CPU cache. If we can't use the CPU to access the RAM
	// (on REE side), then we don't want to use the CPU cache.
	//
	// Why we want cached when is_cpu_accessible_:
	//
	// Without setting cached, in addition to presumably being slower, memcpy tends to fail with
	// non-aligned access faults / syscalls that are trying to copy directly to the VMO can fail
	// without it being obvious that it's an underlying non-aligned access fault triggered by memcpy.
	//
	// Why we want uncached when !is_cpu_accessible_:
	//
	// IIUC, it's possible on aarch64 for a cached mapping to protected memory + speculative execution
	// to cause random faults, while a non-cached mapping only faults if a non-cached mapping is
	// actually touched.
	uint32_t desired_cache_policy =
	is_cpu_accessible_ ? ZX_CACHE_POLICY_CACHED : ZX_CACHE_POLICY_UNCACHED;
	if (info.cache_policy != desired_cache_policy) {
	status = local_contiguous_vmo.set_cache_policy(desired_cache_policy);
	if (status != ZX_OK) {
	// TODO(ZX-4807): Ideally we'd set ZX_CACHE_POLICY_UNCACHED when !is_cpu_accessible_, since
	// IIUC on aarch64 it's possible for a cached mapping to secure/protected memory + speculative
	// execution to cause random faults, while an uncached mapping only faults if the uncached
	// mapping is actually touched. However, currently for a VMO created with
	// zx::vmo::create_contiguous(), the .set_cache_policy() doesn't work because the VMO already
	// has pages. Cases where !is_cpu_accessible_ include both Init() and InitPhysical(), so we
	// can't rely on local_contiguous_vmo being a physical VMO.
	if (ZX_INFO_VMO_TYPE(info.flags) == ZX_INFO_VMO_TYPE_PAGED) {
	DRIVER_ERROR(
	"Ignoring failure to set_cache_policy() on contig VMO - see ZX-4807 - status: %d\n",
	status);
	status = ZX_OK;
	goto keepGoing;
	}
	DRIVER_ERROR("Failed to set_cache_policy(): %d\n", status);
	return status;
	}
	}
	keepGoing:;

	zx_paddr_t addrs;
	zx::pmt pmt;
	// When running a unit test, the zircon/system/dev/lib/fake-bti provides a fake zx_bti_pin() that
	// should tolerate ZX_BTI_CONTIGUOUS here despite the local_contiguous_vmo not actually having
	// info.flags ZX_INFO_VMO_CONTIGUOUS.
	status = parent_device_->bti().pin(ZX_BTI_PERM_READ \| ZX_BTI_PERM_WRITE \| ZX_BTI_CONTIGUOUS,
	local_contiguous_vmo, 0, size_, &addrs, 1, &pmt);
	if (status != ZX_OK) {
	DRIVER_ERROR("Could not pin memory, status %d\n", status);
	return status;
	}

	start_ = addrs;
	contiguous_vmo_ = std::move(local_contiguous_vmo);
	ralloc_region_t region = {0, size_};
	region_allocator_.AddRegion(region);
	// It is intentional here that ~pmt doesn't imply zx_pmt_unpin(). If sysmem dies, we'll reboot.
	return ZX_OK;
	}

	zx_status_t ContiguousPooledMemoryAllocator::Allocate(uint64_t size, zx::vmo* parent_vmo) {
	RegionAllocator::Region::UPtr region;
	zx::vmo result_parent_vmo;

	// TODO: Use a fragmentation-reducing allocator (such as best fit).
	//
	// The "region" param is an out ref.
	zx_status_t status = region_allocator_.GetRegion(size, ZX_PAGE_SIZE, region);
	if (status != ZX_OK) {
	DRIVER_INFO("GetRegion failed (out of space?) - size: %zu status: %d\n", size, status);
	DumpPoolStats();
	return status;
	}

	// The result_parent_vmo created here is a VMO window to a sub-region of contiguous_vmo_.
	status = contiguous_vmo_.create_child(ZX_VMO_CHILD_SLICE, region->base, size, &result_parent_vmo);
	if (status != ZX_OK) {
	DRIVER_ERROR("Failed vmo.create_child(ZX_VMO_CHILD_SLICE, ...): %d\n", status);
	return status;
	}

	// If you see a sysmem-contig VMO you should know that it doesn't actually
	// take up any space, because the same memory is backed by contiguous_vmo_.
	const char* kSysmemContig = "sysmem-contig";
	status = result_parent_vmo.set_property(ZX_PROP_NAME, kSysmemContig, strlen(kSysmemContig));
	if (status != ZX_OK) {
	DRIVER_ERROR("Failed vmo.set_property(ZX_PROP_NAME, ...): %d\n", status);
	return status;
	}

	regions_.emplace(std::make_pair(result_parent_vmo.get(), std::move(region)));
	*parent_vmo = std::move(result_parent_vmo);
	return ZX_OK;
	}

	zx_status_t ContiguousPooledMemoryAllocator::SetupChildVmo(const zx::vmo& parent_vmo, const zx::vmo& child_vmo) {
	// nothing to do here
	return ZX_OK;
	}

	void ContiguousPooledMemoryAllocator::Delete(zx::vmo parent_vmo) {
	auto it = regions_.find(parent_vmo.get());
	ZX_ASSERT(it != regions_.end());
	regions_.erase(it);
	// ~parent_vmo
	}

	void ContiguousPooledMemoryAllocator::DumpPoolStats() {
	uint64_t unused_size = 0;
	uint64_t max_free_size = 0;
	region_allocator_.WalkAvailableRegions(
	[&unused_size, &max_free_size](const ralloc_region_t* r) -> bool {
	unused_size += r->size;
	max_free_size = std::max(max_free_size, r->size);
	return true;
	});

	DRIVER_ERROR("Contiguous pool unused total: %ld bytes, max free size %ld bytes "
	"AllocatedRegionCount(): %zu AvailableRegionCount(): %zu\n",
	unused_size, max_free_size,
	region_allocator_.AllocatedRegionCount(), region_allocator_.AvailableRegionCount());
	}