| // Copyright 2019 The Fuchsia Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| #include "contiguous_pooled_memory_allocator.h" | |
| #include <fidl/fuchsia.sysmem2/cpp/wire.h> | |
| #include <lib/fidl/cpp/wire/arena.h> | |
| #include <lib/fit/defer.h> | |
| #include <lib/trace/event.h> | |
| #include <lib/zx/clock.h> | |
| #include <stdlib.h> | |
| #include <zircon/errors.h> | |
| #include <zircon/syscalls.h> | |
| #include <zircon/types.h> | |
| #include <algorithm> | |
| #include <numeric> | |
| #include <fbl/string_printf.h> | |
| #include "fbl/algorithm.h" | |
| #include "macros.h" | |
| #include "protected_ranges.h" | |
| #include "region-alloc/region-alloc.h" | |
| #include "src/sysmem/metrics/metrics.cb.h" | |
| #include "utils.h" | |
| namespace sysmem_service { | |
| namespace { | |
| zx::duration kGuardCheckInterval = zx::sec(5); | |
| constexpr uint64_t kMiB = 1024ull * 1024; | |
| fuchsia_hardware_sysmem::HeapProperties BuildHeapProperties(bool is_cpu_accessible, | |
| bool is_ever_zircon_accessible) { | |
| using fuchsia_hardware_sysmem::CoherencyDomainSupport; | |
| using fuchsia_hardware_sysmem::HeapProperties; | |
| CoherencyDomainSupport coherency_domain_support; | |
| coherency_domain_support.cpu_supported() = is_cpu_accessible; | |
| coherency_domain_support.ram_supported() = is_cpu_accessible; | |
| coherency_domain_support.inaccessible_supported() = true; | |
| HeapProperties heap_properties; | |
| heap_properties.coherency_domain_support() = std::move(coherency_domain_support); | |
| // New buffers do need to be zeroed (regardless of is_ever_cpu_accessible_ and | |
| // is_always_cpu_accessible_), and we want to do the zeroing in ContiguousPooledMemoryAllocator, | |
| // either via Zircon's zeroing of reclaimed pages, our own zeroing of just-checked pattern pages. | |
| // So we set need_clear true and return true from is_already_cleared_on_allocate(). For secure | |
| // buffers, these are always cleared via the TEE even if some of the pages may have also been | |
| // cleared by Zircon page reclaim, since any "scramble" HW setting feature would potentially make | |
| // zeroes look like non-zero to a device reading the buffer. | |
| // The exception is if zircon can never access the buffer, such as with any physical VMOs created | |
| // for the VDEC range, then there is no risk of stray zeroes from the kernel. | |
| heap_properties.need_clear() = is_ever_zircon_accessible; | |
| // is_cpu_accessible true: We don't do (all the) flushing in this class, so caller will help with | |
| // that. | |
| // | |
| // is_cpu_accessible false: The only zeroing that matters re. cache flushing is the last one which | |
| // is done via the TEE and the TEE flushes after that zeroing. We shouldn't flush from the REE | |
| // since it will/could cause HW errors. | |
| heap_properties.need_flush() = is_cpu_accessible; | |
| return heap_properties; | |
| } | |
| // true - a and b have at least one page in common, and the pages in common are returned via out. | |
| // false - a and b don't have any pages in common and out is unmodified. | |
| bool Intersect(const ralloc_region_t& a, const ralloc_region_t& b, ralloc_region_t* out) { | |
| uint64_t a_base = a.base; | |
| uint64_t a_end = a.base + a.size; | |
| uint64_t b_base = b.base; | |
| uint64_t b_end = b.base + b.size; | |
| uint64_t intersected_base = std::max(a_base, b_base); | |
| uint64_t intersected_end = std::min(a_end, b_end); | |
| if (intersected_end <= intersected_base) { | |
| return false; | |
| } | |
| if (out) { | |
| out->base = intersected_base; | |
| out->size = intersected_end - intersected_base; | |
| } | |
| return true; | |
| } | |
| std::atomic<trace_counter_id_t> next_counter_id; | |
| } // namespace | |
| ContiguousPooledMemoryAllocator::ContiguousPooledMemoryAllocator( | |
| Owner* parent_device, const char* heap_name, inspect::Node* parent_node, | |
| std::optional<fuchsia_sysmem2::Heap> heap, uint64_t size, bool is_always_cpu_accessible, | |
| bool is_ever_cpu_accessible, bool is_ever_zircon_accessible, bool is_ready, | |
| bool can_be_torn_down, async_dispatcher_t* dispatcher) | |
| : MemoryAllocator(BuildHeapProperties(is_always_cpu_accessible, is_ever_zircon_accessible)), | |
| parent_device_(parent_device), | |
| dispatcher_(dispatcher), | |
| heap_name_(heap_name), | |
| heap_(std::move(heap)), | |
| counter_id_(next_counter_id.fetch_add(1, std::memory_order_relaxed)), | |
| region_allocator_(RegionAllocator::RegionPool::Create(std::numeric_limits<size_t>::max())), | |
| size_(size), | |
| is_always_cpu_accessible_(is_always_cpu_accessible), | |
| is_ever_cpu_accessible_(is_ever_cpu_accessible), | |
| is_ready_(is_ready), | |
| can_be_torn_down_(can_be_torn_down), | |
| metrics_(parent_device->metrics()) { | |
| snprintf(child_name_, sizeof(child_name_), "%s-child", heap_name_); | |
| // Ensure NUL-terminated. | |
| child_name_[sizeof(child_name_) - 1] = 0; | |
| node_ = parent_node->CreateChild(heap_name); | |
| node_.CreateUint("size", size, &properties_); | |
| node_.CreateUint("id", id(), &properties_); | |
| high_water_mark_property_ = node_.CreateUint("high_water_mark", 0); | |
| free_at_high_water_mark_property_ = node_.CreateUint("free_at_high_water_mark", size); | |
| used_size_property_ = node_.CreateUint("used_size", 0); | |
| allocations_failed_property_ = node_.CreateUint("allocations_failed", 0); | |
| last_allocation_failed_timestamp_ns_property_ = | |
| node_.CreateUint("last_allocation_failed_timestamp_ns", 0); | |
| commits_failed_property_ = node_.CreateUint("commits_failed", 0); | |
| last_commit_failed_timestamp_ns_property_ = node_.CreateUint("last_commit_failed_timstamp_ns", 0); | |
| allocations_failed_fragmentation_property_ = | |
| node_.CreateUint("allocations_failed_fragmentation", 0); | |
| max_free_at_high_water_property_ = node_.CreateUint("max_free_at_high_water", size); | |
| is_ready_property_ = node_.CreateBool("is_ready", is_ready_); | |
| failed_guard_region_checks_property_ = | |
| node_.CreateUint("failed_guard_region_checks", failed_guard_region_checks_); | |
| last_failed_guard_region_check_timestamp_ns_property_ = | |
| node_.CreateUint("last_failed_guard_region_check_timestamp_ns", 0); | |
| large_contiguous_region_sum_property_ = node_.CreateUint("large_contiguous_region_sum", 0); | |
| // CMM/PCMM properties - these values aren't quite true yet, but will be soon. | |
| loanable_efficiency_property_ = | |
| node_.CreateDouble("loanable_efficiency", is_ever_cpu_accessible_ ? 1.0 : 0.0); | |
| loanable_ratio_property_ = | |
| node_.CreateDouble("loanable_ratio", is_ever_cpu_accessible_ ? 1.0 : 0.0); | |
| loanable_bytes_property_ = node_.CreateUint("loanable_bytes", is_ever_cpu_accessible_ ? size : 0); | |
| loanable_mebibytes_property_ = | |
| node_.CreateUint("loanable_mebibytes", is_ever_cpu_accessible_ ? size / kMiB : 0); | |
| if (dispatcher) { | |
| zx_status_t status = zx::event::create(0, &trace_observer_event_); | |
| ZX_ASSERT(status == ZX_OK); | |
| status = trace_register_observer(trace_observer_event_.get()); | |
| ZX_ASSERT(status == ZX_OK); | |
| wait_.set_object(trace_observer_event_.get()); | |
| wait_.set_trigger(ZX_EVENT_SIGNALED); | |
| status = wait_.Begin(dispatcher); | |
| ZX_ASSERT(status == ZX_OK); | |
| } | |
| } | |
| void ContiguousPooledMemoryAllocator::InitGuardRegion( | |
| size_t guard_region_size, bool unused_pages_guarded, int64_t unused_guard_pattern_period_bytes, | |
| zx::duration unused_page_check_cycle_period, bool internal_guard_regions, | |
| bool crash_on_guard_failure, async_dispatcher_t* dispatcher) { | |
| ZX_DEBUG_ASSERT(!regions_.size()); | |
| ZX_DEBUG_ASSERT(!guard_region_size_); | |
| ZX_DEBUG_ASSERT(!guard_region_data_.size()); | |
| ZX_DEBUG_ASSERT(contiguous_vmo_.get()); | |
| ZX_DEBUG_ASSERT(!unused_pages_guarded_); | |
| ZX_DEBUG_ASSERT(is_ever_cpu_accessible_); | |
| zx_status_t status; | |
| uint64_t min_guard_data_size = guard_region_size; | |
| if (unused_pages_guarded) { | |
| ZX_DEBUG_ASSERT(is_ever_cpu_accessible_); | |
| unused_pages_guarded_ = true; | |
| if (unused_guard_pattern_period_bytes > 0) { | |
| unused_guard_pattern_period_bytes_ = unused_guard_pattern_period_bytes; | |
| } | |
| unused_page_check_cycle_period_ = unused_page_check_cycle_period; | |
| ZX_DEBUG_ASSERT(mapping_); | |
| unused_checker_.PostDelayed(dispatcher, | |
| unused_page_check_cycle_period_ / kUnusedCheckPartialCount); | |
| unused_recently_checker_.Post(dispatcher); | |
| min_guard_data_size = std::max(min_guard_data_size, unused_guard_data_size_); | |
| deleted_regions_.resize(kNumDeletedRegions); | |
| } | |
| ZX_DEBUG_ASSERT(guard_region_size % zx_system_get_page_size() == 0); | |
| ZX_DEBUG_ASSERT(min_guard_data_size % zx_system_get_page_size() == 0); | |
| guard_region_data_.resize(min_guard_data_size); | |
| for (size_t i = 0; i < min_guard_data_size; i++) { | |
| guard_region_data_[i] = debug_safe_cast<uint8_t>(((i + 1) % 256)); | |
| } | |
| if (!guard_region_size) { | |
| return; | |
| } | |
| guard_region_size_ = guard_region_size; | |
| // Internal guard regions expect pages to be CPU accessible always. Internal guard regions for | |
| // part-time protected memory would be severely limited anyway due to granularity of protection | |
| // ranges and limited number of HW protection ranges. | |
| has_internal_guard_regions_ = internal_guard_regions && is_always_cpu_accessible_; | |
| guard_region_copy_.resize(guard_region_size); | |
| crash_on_guard_failure_ = crash_on_guard_failure; | |
| // Initialize external guard regions. | |
| ralloc_region_t regions[] = {{.base = 0, .size = guard_region_size}, | |
| {.base = size_ - guard_region_size, .size = guard_region_size}}; | |
| for (auto& region : regions) { | |
| status = region_allocator_.SubtractRegion(region); | |
| ZX_DEBUG_ASSERT(status == ZX_OK); | |
| status = contiguous_vmo_.write(guard_region_data_.data(), region.base, guard_region_size_); | |
| ZX_DEBUG_ASSERT(status == ZX_OK); | |
| } | |
| status = guard_checker_.PostDelayed(dispatcher, kGuardCheckInterval); | |
| ZX_ASSERT(status == ZX_OK); | |
| } | |
| void ContiguousPooledMemoryAllocator::SetupUnusedPages() { | |
| ZX_DEBUG_ASSERT(is_ever_cpu_accessible_); | |
| ZX_DEBUG_ASSERT((is_always_cpu_accessible_ && is_ready_ && !protected_ranges_.has_value()) || | |
| (protected_ranges_.has_value() && protected_ranges_->ranges().size() == 0)); | |
| ZX_DEBUG_ASSERT(!is_setup_unused_pages_called_); | |
| is_setup_unused_pages_called_ = true; | |
| std::vector<ralloc_region_t> todo; | |
| region_allocator_.WalkAvailableRegions([&todo](const ralloc_region_t* region) { | |
| todo.emplace_back(*region); | |
| return true; | |
| }); | |
| for (auto& region : todo) { | |
| OnRegionUnused(region); | |
| } | |
| } | |
| void ContiguousPooledMemoryAllocator::FillUnusedRangeWithGuard(uint64_t start_offset, | |
| uint64_t size) { | |
| ZX_DEBUG_ASSERT(mapping_); | |
| ZX_DEBUG_ASSERT(start_offset % zx_system_get_page_size() == 0); | |
| ZX_DEBUG_ASSERT(size % zx_system_get_page_size() == 0); | |
| ZX_DEBUG_ASSERT(unused_guard_pattern_period_bytes_ % zx_system_get_page_size() == 0); | |
| uint64_t end = start_offset + size; | |
| uint64_t to_copy_size; | |
| for (uint64_t offset = start_offset; offset < end; offset += to_copy_size) { | |
| to_copy_size = std::min(unused_guard_data_size_, end - offset); | |
| memcpy(&mapping_[offset], guard_region_data_.data(), to_copy_size); | |
| zx_cache_flush(&mapping_[offset], to_copy_size, ZX_CACHE_FLUSH_DATA); | |
| // zx_cache_flush() takes care of dsb sy when __aarch64__. | |
| } | |
| } | |
| ContiguousPooledMemoryAllocator::~ContiguousPooledMemoryAllocator() { | |
| ZX_DEBUG_ASSERT(is_empty()); | |
| wait_.Cancel(); | |
| if (trace_observer_event_) { | |
| trace_unregister_observer(trace_observer_event_.get()); | |
| } | |
| step_toward_optimal_protected_ranges_.Cancel(); | |
| guard_checker_.Cancel(); | |
| unused_checker_.Cancel(); | |
| unused_recently_checker_.Cancel(); | |
| if (mapping_) { | |
| zx_status_t status = zx::vmar::root_self()->unmap(reinterpret_cast<uintptr_t>(mapping_), size_); | |
| ZX_ASSERT(status == ZX_OK); | |
| } | |
| } | |
| zx_status_t ContiguousPooledMemoryAllocator::Init(uint32_t alignment_log2) { | |
| zx::vmo local_contiguous_vmo; | |
| const long system_page_alignment = __builtin_ctzl(zx_system_get_page_size()); | |
| if (alignment_log2 < system_page_alignment) { | |
| alignment_log2 = safe_cast<uint32_t>(system_page_alignment); | |
| } | |
| max_supported_alignment_log2_ = alignment_log2; | |
| auto vmo_size = CheckRoundUp(size_, zx_system_get_page_size()); | |
| if (!vmo_size.IsValid()) { | |
| LOG(ERROR, "size_ overflow heap_name_: %s", heap_name_); | |
| return ZX_ERR_INVALID_ARGS; | |
| } | |
| zx_status_t status = zx::vmo::create_contiguous(parent_device_->bti(), vmo_size.ValueOrDie(), | |
| alignment_log2, &local_contiguous_vmo); | |
| if (status != ZX_OK) { | |
| LOG(ERROR, "Could not allocate contiguous memory, status %d heap_name_: %s", status, | |
| heap_name_); | |
| return status; | |
| } | |
| return InitCommon(std::move(local_contiguous_vmo)); | |
| } | |
| zx_status_t ContiguousPooledMemoryAllocator::InitPhysical(zx_paddr_t paddr) { | |
| // __builtin_ctzll is UB when paddr is 0. Zircon will never allocate a physically contiguous VMO | |
| // using physical page 0, but guard against 0 here anyway to avoid taking a dep on that here. | |
| max_supported_alignment_log2_ = paddr != 0 ? __builtin_ctzll(paddr) : sizeof(paddr) * 8; | |
| auto vmo_size = CheckRoundUp(size_, zx_system_get_page_size()); | |
| if (!vmo_size.IsValid()) { | |
| LOG(ERROR, "size_ overflow heap_name_: %s", heap_name_); | |
| return ZX_ERR_INVALID_ARGS; | |
| } | |
| zx::result<zx::vmo> physical_vmo_result = | |
| parent_device_->CreatePhysicalVmo(paddr, vmo_size.ValueOrDie()); | |
| if (!physical_vmo_result.is_ok()) { | |
| LOG(ERROR, "Failed to create physical VMO: %s heap_name_: %s", | |
| physical_vmo_result.status_string(), heap_name_); | |
| return physical_vmo_result.status_value(); | |
| } | |
| zx::vmo& local_contiguous_vmo = physical_vmo_result.value(); | |
| return InitCommon(std::move(local_contiguous_vmo)); | |
| } | |
| zx_status_t ContiguousPooledMemoryAllocator::InitCommon(zx::vmo local_contiguous_vmo) { | |
| zx_status_t status = | |
| local_contiguous_vmo.set_property(ZX_PROP_NAME, heap_name_, strlen(heap_name_)); | |
| if (status != ZX_OK) { | |
| LOG(ERROR, "Failed vmo.set_property(ZX_PROP_NAME, ...): %d", status); | |
| return status; | |
| } | |
| zx_info_vmo_t info; | |
| status = local_contiguous_vmo.get_info(ZX_INFO_VMO, &info, sizeof(info), nullptr, nullptr); | |
| if (status != ZX_OK) { | |
| LOG(ERROR, "Failed local_contiguous_vmo.get_info(ZX_INFO_VMO, ...) - status: %d", status); | |
| return status; | |
| } | |
| // Only secure/protected RAM ever uses a physical VMO. Not all secure/protected RAM uses a | |
| // physical VMO. | |
| ZX_DEBUG_ASSERT(ZX_INFO_VMO_TYPE(info.flags) == ZX_INFO_VMO_TYPE_PAGED || | |
| !is_ever_cpu_accessible_); | |
| // Paged VMOs are cached by default. Physical VMOs are uncached by default. | |
| ZX_DEBUG_ASSERT((ZX_INFO_VMO_TYPE(info.flags) == ZX_INFO_VMO_TYPE_PAGED) == | |
| (info.cache_policy == ZX_CACHE_POLICY_CACHED)); | |
| // We'd have this assert, except it doesn't work with fake-bti, so for now we trust that when not | |
| // running a unit test, we have a VMO with info.flags & ZX_INFO_VMO_CONTIGUOUS. | |
| // ZX_DEBUG_ASSERT(info.flags & ZX_INFO_VMO_CONTIGUOUS); | |
| // Regardless of CPU or RAM domain, and regardless of contig VMO or physical VMO, if we use the | |
| // CPU to access the RAM, we want to use the CPU cache, if we can do so safely. | |
| // | |
| // Why we want cached when is_always_cpu_accessible_: | |
| // | |
| // Without setting cached, in addition to presumably being slower, memcpy tends to fail with | |
| // non-aligned access faults / syscalls that are trying to copy directly to the VMO can fail | |
| // without it being obvious that it's an underlying non-aligned access fault triggered by memcpy. | |
| // | |
| // Why we want uncached when !is_always_cpu_accessible_: | |
| // | |
| // If the memory is sometimes protected, we can't use the CPU cache safely, since speculative | |
| // prefetching may occur and interact badly (but not necessarily in immedidately-obvious ways) | |
| // with the HW-protected ranges (on aarch64, this causes SErrors of type DECERR). A non-cached | |
| // mapping doesn't do any speculative prefetching so doesn't trigger errors as long as we don't | |
| // access a page while it's HW-protected. | |
| // | |
| // An alternative would be to only map pages of the VMO that are known to not be under a | |
| // HW-protected range while mapped, but since a non-cached mapping seems fast enough, this is | |
| // simpler. | |
| uint32_t desired_cache_policy = | |
| is_always_cpu_accessible_ ? ZX_CACHE_POLICY_CACHED : ZX_CACHE_POLICY_UNCACHED; | |
| if (info.cache_policy != desired_cache_policy) { | |
| status = local_contiguous_vmo.set_cache_policy(desired_cache_policy); | |
| if (status != ZX_OK) { | |
| if (ZX_INFO_VMO_TYPE(info.flags) == ZX_INFO_VMO_TYPE_PAGED) { | |
| LOG(ERROR, "Failed to set_cache_policy() (contig paged VMO) - status: %d", status); | |
| } else { | |
| LOG(ERROR, "Failed to set_cache_policy() (not paged VMO) - status: %d", status); | |
| } | |
| return status; | |
| } | |
| } | |
| zx_paddr_t addrs; | |
| // When running a unit test, the src/devices/testing/fake-bti provides a fake zx_bti_pin() that | |
| // should tolerate ZX_BTI_CONTIGUOUS here despite the local_contiguous_vmo not actually having | |
| // info.flags ZX_INFO_VMO_CONTIGUOUS. | |
| status = parent_device_->bti().pin(ZX_BTI_PERM_READ | ZX_BTI_PERM_WRITE | ZX_BTI_CONTIGUOUS, | |
| local_contiguous_vmo, 0, size_, &addrs, 1, &pool_pmt_); | |
| if (status != ZX_OK) { | |
| LOG(ERROR, "Could not pin memory, status %d", status); | |
| return status; | |
| } | |
| phys_start_ = addrs; | |
| // We intentionally do not have max_supported_alignment_log2_ depend on luck here. Please do not | |
| // do max_supported_alignment_log2_ = std::max(max_supported_alignment_log2_, | |
| // __builtin_ctzll(phys_start_)) here, so that requests for particular alignment can either fail | |
| // or succeed based on what's guaranteed (assuming we get this far etc), not what happened to | |
| // occur this boot. | |
| // Since the VMO is contiguous or physical, we don't need to keep the VMO pinned for it to remain | |
| // physically contiguous. A physical VMO can't have any pages decommitted, while a contiguous | |
| // VMO can. In order to decommit pages from a contiguous VMO, we can't have the decommitting | |
| // pages pinned (from user mode, ignoring any pinning internal to Zircon). | |
| status = pool_pmt_.unpin(); | |
| // All possible failures are bugs in how we called zx_pmt_unpin(). | |
| ZX_DEBUG_ASSERT(status == ZX_OK); | |
| // We map contiguous_vmo_ as cached only if is_always_cpu_accessible_, to avoid SError(s) that can | |
| // result from speculative prefetch from a physical page under a HW-protected range. A non-cached | |
| // mapping prevents speculative prefetch. | |
| // | |
| // TODO(https://fxbug.dev/42179016): Currently Zircon's physmap has !is_always_cpu_accessible_ | |
| // pages mapped cached, which we believe is likely the cause of some SError(s) related to | |
| // protected_memory_size. One way to fix would be to change the physmap mapping to non-cached | |
| // when a contiguous VMO | |
| // | |
| // So far, when !is_always_cpu_accessible_, a non-cached mapping seems fast enough; we only read | |
| // or write using the mapping for a low % of pages. | |
| status = zx::vmar::root_self()->map( | |
| /*options=*/(ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_MAP_RANGE), | |
| /*vmar_offset=*/0, local_contiguous_vmo, /*vmo_offset=*/0, /*len=*/size_, | |
| reinterpret_cast<zx_vaddr_t*>(&mapping_)); | |
| if (status != ZX_OK) { | |
| LOG(ERROR, "mapping contiguous_vmo_ failed - fatal - status: %d", status); | |
| return status; | |
| } | |
| // Assume cannot decommit until proven otherwise. | |
| can_decommit_ = false; | |
| if (ZX_INFO_VMO_TYPE(info.flags) == ZX_INFO_VMO_TYPE_PAGED) { | |
| // Paged VMOs might support decommit depending on whether it was enabled by the kernel cmdline | |
| // flag or not. Probe support for this by attempting a decommit. | |
| status = | |
| local_contiguous_vmo.op_range(ZX_VMO_OP_DECOMMIT, 0, zx_system_get_page_size(), nullptr, 0); | |
| if (status == ZX_OK) { | |
| // Decommit succeeded, commit the page back before continuing. | |
| status = | |
| local_contiguous_vmo.op_range(ZX_VMO_OP_COMMIT, 0, zx_system_get_page_size(), nullptr, 0); | |
| if (status != ZX_OK) { | |
| LOG(ERROR, | |
| "ZX_VMO_OP_COMMIT failed on contiguous vmo where ZX_VMO_OP_DECOMMIT succeeded - status: %d", | |
| status); | |
| return status; | |
| } | |
| can_decommit_ = true; | |
| } | |
| } | |
| contiguous_vmo_ = std::move(local_contiguous_vmo); | |
| ralloc_region_t region = {0, size_}; | |
| region_allocator_.AddRegion(region); | |
| return ZX_OK; | |
| } | |
| zx_status_t ContiguousPooledMemoryAllocator::Allocate( | |
| uint64_t raw_vmo_size, std::optional<uint64_t> min_physical_alignment, | |
| const fuchsia_sysmem2::SingleBufferSettings& settings, std::optional<std::string> name, | |
| uint64_t buffer_collection_id, uint32_t buffer_index, zx::vmo* parent_vmo) { | |
| TRACE_DURATION("gfx", "ContiguousPooledMemoryAllocator::Allocate", "size_bytes", | |
| *settings.buffer_settings()->size_bytes()); | |
| ZX_DEBUG_ASSERT_MSG(raw_vmo_size % zx_system_get_page_size() == 0, "raw_vmo_size: 0x%" PRIx64, | |
| raw_vmo_size); | |
| ZX_DEBUG_ASSERT_MSG(*settings.buffer_settings()->raw_vmo_size() == raw_vmo_size, | |
| " settings raw_vmo_size: %" PRIu64 " raw_vmo_size: %" PRIu64, | |
| *settings.buffer_settings()->raw_vmo_size(), raw_vmo_size); | |
| if (!is_ready_) { | |
| LOG(ERROR, "heap_name_: %s is not ready_, failing", heap_name_); | |
| return ZX_ERR_BAD_STATE; | |
| } | |
| RegionAllocator::Region::UPtr region; | |
| zx::vmo result_parent_vmo; | |
| const uint64_t guard_region_size = has_internal_guard_regions_ ? guard_region_size_ : 0; | |
| uint64_t allocation_size = raw_vmo_size + guard_region_size_ * 2; | |
| uint64_t alignment = zx_system_get_page_size(); | |
| if (min_physical_alignment.has_value()) { | |
| if (has_internal_guard_regions_) { | |
| LOG(ERROR, "has_internal_guard_regions_ and min_physical_alignment not supported together"); | |
| return ZX_ERR_NOT_SUPPORTED; | |
| } | |
| alignment = std::max(alignment, *min_physical_alignment); | |
| if (alignment > (1 << max_supported_alignment_log2_)) { | |
| LOG(ERROR, "alignment > (1 << max_supported_alignment_log2_)"); | |
| return ZX_ERR_NOT_SUPPORTED; | |
| } | |
| } | |
| // TODO(https://fxbug.dev/42119460): Use a fragmentation-reducing allocator (such as best fit). | |
| // | |
| // The "region" param is an out ref. | |
| zx_status_t status = region_allocator_.GetRegion(allocation_size, alignment, region); | |
| if (status != ZX_OK) { | |
| LOG(WARNING, "GetRegion failed (out of space?) - size: %" PRIu64 " status: %d", raw_vmo_size, | |
| status); | |
| DumpPoolStats(); | |
| // Log buffer lifetime related info for existing buffers. This is via the owner since this | |
| // allocator doesn't have all the relevant info; we only want to log this info if we're failing | |
| // because we're out of limited space (this failure path). | |
| parent_device_->OnAllocationFailure(); | |
| allocations_failed_property_.Add(1); | |
| last_allocation_failed_timestamp_ns_property_.Set(zx::clock::get_monotonic().get()); | |
| uint64_t unused_size = 0; | |
| region_allocator_.WalkAvailableRegions([&unused_size](const ralloc_region_t* r) -> bool { | |
| unused_size += r->size; | |
| return true; | |
| }); | |
| if (unused_size >= allocation_size) { | |
| // There's enough unused memory total, so the allocation must have failed due to | |
| // fragmentation. | |
| allocations_failed_fragmentation_property_.Add(1); | |
| } | |
| return status; | |
| } | |
| // We rely on this commit not destroying existing unused region guard pages (since we never | |
| // decommitted those), and not touching any protected pages (since those aren't decommitted). | |
| // This commit will commit the gaps between guard pages, if any of those pages are decommitted | |
| // currently. These gaps are what we may have previously decommitted if the pages weren't | |
| // protected. In contrast to decommitting, when we commit we don't need to separately commit only | |
| // the gaps, since a commit range that also overlaps the unused range guard pages doesn't change | |
| // the contents of the already-committed guard pages, and doesn't touch any already-committed | |
| // protected pages. This CommitRegion() can also overlap (spatially not temporally) with a | |
| // (possibly-larger) CommitRegion() requested by protected_ranges_ via UseRange() (if we're using | |
| // protected_ranges_). | |
| status = CommitRegion(*region.get()); | |
| if (status != ZX_OK) { | |
| LOG(WARNING, "CommitRegion() failed (OOM?) - allocation_size: %" PRIu64 " status %d", | |
| allocation_size, status); | |
| commits_failed_property_.Add(1); | |
| last_commit_failed_timestamp_ns_property_.Set(zx::clock::get_monotonic().get()); | |
| return status; | |
| } | |
| // If !is_always_cpu_accessible_, no point in doing any zeroing other than the zeroing later via | |
| // the TEE once the region is fully protected. This is because zeroing via memset() before the | |
| // range is protected isn't really necessarily making the protected range appear to be zero to | |
| // protected mode devices that read the just-protected range, due to any potential HW "scramble". | |
| CheckUnusedRange(region->base, region->size, /*and_also_zero=*/is_always_cpu_accessible_); | |
| ZX_DEBUG_ASSERT(protected_ranges_.has_value() == | |
| (!is_always_cpu_accessible_ && is_ever_cpu_accessible_)); | |
| if (!is_always_cpu_accessible_) { | |
| const auto new_range = protected_ranges::Range::BeginLength(region->base, region->size); | |
| if (protected_ranges_.has_value()) { | |
| ZX_DEBUG_ASSERT(protected_ranges_.has_value()); | |
| protected_ranges_->AddRange(new_range); | |
| EnsureSteppingTowardOptimalProtectedRanges(); | |
| } else { | |
| ZX_DEBUG_ASSERT(!is_ever_cpu_accessible_); | |
| // The covering range is VDEC (or similar), which is not an explicitly-created range, but | |
| // rather an implicit range. In some cases this range may still be checked against by layers | |
| // above the TEE, but it's not a range that was created via Range | |
| // | |
| // If we're running with new FW, IsDynamic() is true. If we're not, then we can't call | |
| // ZeroProtectedSubRange() because the FW doesn't have it, in which case we can't zero the | |
| // protected buffer. | |
| if (protected_ranges_control_->IsDynamic()) { | |
| protected_ranges_control_->ZeroProtectedSubRange(false, new_range); | |
| } | |
| } | |
| } | |
| TracePoolSize(false); | |
| // We don't attempt to have guard regions on either side of a !is_cpu_accessible_ buffer (aka | |
| // "internal" guard regions), since a guard page could already be under a protected range and | |
| // since deprotecting a page is expected to change its contents (not necessarily to zero, but | |
| // change the contents to ensure that no protected data can be read / un-scrambled). | |
| ZX_DEBUG_ASSERT(is_always_cpu_accessible_ || !guard_region_size); | |
| if (guard_region_size) { | |
| status = contiguous_vmo_.write(guard_region_data_.data(), region->base, guard_region_size_); | |
| if (status != ZX_OK) { | |
| LOG(ERROR, "Failed to write pre-guard region."); | |
| return status; | |
| } | |
| status = | |
| contiguous_vmo_.write(guard_region_data_.data(), | |
| region->base + guard_region_size + raw_vmo_size, guard_region_size_); | |
| if (status != ZX_OK) { | |
| LOG(ERROR, "Failed to write post-guard region."); | |
| return status; | |
| } | |
| } | |
| // The result_parent_vmo created here is a VMO window to a sub-region of contiguous_vmo_. | |
| status = contiguous_vmo_.create_child(ZX_VMO_CHILD_SLICE, region->base + guard_region_size, | |
| raw_vmo_size, &result_parent_vmo); | |
| if (status != ZX_OK) { | |
| LOG(ERROR, "Failed vmo.create_child(ZX_VMO_CHILD_SLICE, ...): %d", status); | |
| return status; | |
| } | |
| // If you see a Sysmem*-child VMO you should know that it doesn't actually | |
| // take up any space, because the same memory is backed by contiguous_vmo_. | |
| status = result_parent_vmo.set_property(ZX_PROP_NAME, child_name_, strlen(child_name_)); | |
| if (status != ZX_OK) { | |
| LOG(ERROR, "Failed vmo.set_property(ZX_PROP_NAME, ...): %d", status); | |
| return status; | |
| } | |
| if (!name.has_value()) { | |
| name = "Unknown"; | |
| } | |
| RegionData data; | |
| data.name = std::move(*name); | |
| zx_info_handle_basic_t handle_info; | |
| status = result_parent_vmo.get_info(ZX_INFO_HANDLE_BASIC, &handle_info, sizeof(handle_info), | |
| nullptr, nullptr); | |
| ZX_ASSERT(status == ZX_OK); | |
| data.node = node_.CreateChild(fbl::StringPrintf("vmo-%ld", handle_info.koid).c_str()); | |
| data.size_property = data.node.CreateUint("raw_vmo_size", raw_vmo_size); | |
| data.koid = handle_info.koid; | |
| data.koid_property = data.node.CreateUint("koid", handle_info.koid); | |
| allocated_bytes_ += region->size; | |
| data.ptr = std::move(region); | |
| regions_.emplace(std::make_pair(result_parent_vmo.get(), std::move(data))); | |
| UpdateLoanableMetrics(); | |
| LOG(DEBUG, "Allocate() - loaned ratio: %g loaning efficiency: %g", GetLoanableRatio(), | |
| GetLoanableEfficiency()); | |
| *parent_vmo = std::move(result_parent_vmo); | |
| return ZX_OK; | |
| } | |
| void ContiguousPooledMemoryAllocator::Delete(zx::vmo parent_vmo) { | |
| TRACE_DURATION("gfx", "ContiguousPooledMemoryAllocator::Delete"); | |
| auto it = regions_.find(parent_vmo.get()); | |
| ZX_ASSERT(it != regions_.end()); | |
| auto& region_data = it->second; | |
| CheckGuardRegionData(region_data); | |
| StashDeletedRegion(region_data); | |
| ZX_DEBUG_ASSERT(protected_ranges_.has_value() == | |
| (!is_always_cpu_accessible_ && is_ever_cpu_accessible_)); | |
| if (protected_ranges_.has_value()) { | |
| const auto& region = *region_data.ptr; | |
| const auto old_range = protected_ranges::Range::BeginLength(region.base, region.size); | |
| protected_ranges_->DeleteRange(old_range); | |
| EnsureSteppingTowardOptimalProtectedRanges(); | |
| } else { | |
| OnRegionUnused(*it->second.ptr.get()); | |
| } | |
| allocated_bytes_ -= region_data.ptr->size; | |
| regions_.erase(it); | |
| // region_data now invalid | |
| parent_vmo.reset(); | |
| TracePoolSize(false); | |
| UpdateLoanableMetrics(); | |
| LOG(DEBUG, "Delete() - loaned ratio: %g loaning efficiency: %g", GetLoanableRatio(), | |
| GetLoanableEfficiency()); | |
| if (is_empty()) { | |
| parent_device_->CheckForUnbind(); | |
| } | |
| } | |
| void ContiguousPooledMemoryAllocator::set_heap(fuchsia_sysmem2::Heap heap) { | |
| ZX_DEBUG_ASSERT(!heap_.has_value()); | |
| ZX_DEBUG_ASSERT(!is_ready_); | |
| heap_ = std::move(heap); | |
| } | |
| void ContiguousPooledMemoryAllocator::set_ready() { | |
| ZX_ASSERT(!is_ready_); | |
| if (!is_always_cpu_accessible_) { | |
| protected_ranges_control_.emplace(this); | |
| if (is_ever_cpu_accessible_) { | |
| protected_ranges_.emplace(&*protected_ranges_control_, | |
| parent_device_->protected_ranges_disable_dynamic()); | |
| SetupUnusedPages(); | |
| } | |
| } | |
| is_ready_ = true; | |
| is_ready_property_.Set(is_ready_); | |
| } | |
| void ContiguousPooledMemoryAllocator::clear_ready() { | |
| ZX_ASSERT(is_ready_); | |
| if (!is_always_cpu_accessible_) { | |
| protected_ranges_control_.reset(); | |
| if (is_ever_cpu_accessible_) { | |
| protected_ranges_->DiscardAllRanges(); | |
| protected_ranges_.reset(); | |
| } | |
| } | |
| is_ready_ = false; | |
| is_ready_property_.Set(is_ready_); | |
| } | |
| void ContiguousPooledMemoryAllocator::CheckGuardRegion(const char* region_name, size_t region_size, | |
| bool pre, uint64_t start_offset) { | |
| const uint64_t guard_region_size = guard_region_size_; | |
| zx_status_t status = contiguous_vmo_.op_range(ZX_VMO_OP_CACHE_CLEAN_INVALIDATE, start_offset, | |
| guard_region_size, nullptr, 0); | |
| ZX_ASSERT(status == ZX_OK); | |
| status = contiguous_vmo_.read(guard_region_copy_.data(), start_offset, guard_region_copy_.size()); | |
| ZX_ASSERT(status == ZX_OK); | |
| if (memcmp(guard_region_copy_.data(), guard_region_data_.data(), guard_region_size_) != 0) { | |
| size_t error_start = UINT64_MAX; | |
| size_t error_end = 0; | |
| for (size_t i = 0; i < guard_region_copy_.size(); i++) { | |
| if (guard_region_copy_[i] != guard_region_data_[i]) { | |
| error_start = std::min(i, error_start); | |
| error_end = std::max(i, error_end); | |
| } | |
| } | |
| ZX_DEBUG_ASSERT(error_start < guard_region_copy_.size()); | |
| std::string bad_str; | |
| std::string good_str; | |
| constexpr uint32_t kRegionSizeToOutput = 16; | |
| for (uint32_t i = safe_cast<uint32_t>(error_start); | |
| i < error_start + kRegionSizeToOutput && i < guard_region_size_; i++) { | |
| bad_str += fbl::StringPrintf(" 0x%x", guard_region_copy_[i]).c_str(); | |
| good_str += fbl::StringPrintf(" 0x%x", guard_region_data_[i]).c_str(); | |
| } | |
| LOG(ERROR, | |
| "Region %s of size %ld has corruption in %s guard pages between [0x%lx, 0x%lx] - good " | |
| "\"%s\" bad \"%s\"", | |
| region_name, region_size, pre ? "pre" : "post", error_start, error_end, good_str.c_str(), | |
| bad_str.c_str()); | |
| // For now, if unused page checking is enabled, also print the guard region diffs using | |
| // ReportPatternCheckFailedRange(). While this is mainly intended for printing diffs after | |
| // pattern check failure on unused pages (in contrast to per-allocation or per-reserved-VMO | |
| // guard pages), we _might_ find the DeletedRegion info useful, and the diffs may have more | |
| // info. | |
| // | |
| // TODO(dustingreen): In a later CL, integrate anything that's needed from the code above into | |
| // ReportPatternCheckFailedRange(), and make ReportPatternCheckFailedRange() work even if unused | |
| // page checking is disabled. | |
| auto page_aligned_base = CheckRoundDown(error_start, zx_system_get_page_size()); | |
| auto page_aligned_end = | |
| CheckRoundUp(safemath::CheckAdd(error_end, 1), zx_system_get_page_size()); | |
| ZX_ASSERT(page_aligned_base.IsValid()); | |
| ZX_ASSERT(page_aligned_end.IsValid()); | |
| auto page_aligned_size = safemath::CheckSub(page_aligned_end, page_aligned_base); | |
| ZX_ASSERT(page_aligned_size.IsValid()); | |
| ralloc_region_t diff_range{ | |
| .base = page_aligned_base.ValueOrDie(), | |
| .size = page_aligned_size.ValueOrDie(), | |
| }; | |
| ReportPatternCheckFailedRange(diff_range, "guard"); | |
| IncrementGuardRegionFailureInspectData(); | |
| } | |
| } | |
| void ContiguousPooledMemoryAllocator::IncrementGuardRegionFailureInspectData() { | |
| ZX_ASSERT_MSG(!crash_on_guard_failure_, "Crashing due to guard region failure"); | |
| failed_guard_region_checks_++; | |
| failed_guard_region_checks_property_.Set(failed_guard_region_checks_); | |
| last_failed_guard_region_check_timestamp_ns_property_.Set(zx::clock::get_monotonic().get()); | |
| auto& snapshot_annotation_register = parent_device_->snapshot_annotation_register(); | |
| if (snapshot_annotation_register.has_value()) { | |
| snapshot_annotation_register->IncrementNumDmaCorruptions(); | |
| } | |
| } | |
| void ContiguousPooledMemoryAllocator::CheckGuardRegionData(const RegionData& region) { | |
| const uint64_t guard_region_size = guard_region_size_; | |
| if (guard_region_size == 0 || !has_internal_guard_regions_) | |
| return; | |
| uint64_t size = region.ptr->size; | |
| uint64_t vmo_size = size - guard_region_size * 2; | |
| ZX_DEBUG_ASSERT(guard_region_size_ == guard_region_copy_.size()); | |
| for (uint32_t i = 0; i < 2; i++) { | |
| uint64_t start_offset = region.ptr->base; | |
| if (i == 1) { | |
| // Size includes guard regions. | |
| start_offset += size - guard_region_size; | |
| } | |
| CheckGuardRegion(region.name.c_str(), vmo_size, (i == 0), start_offset); | |
| } | |
| } | |
| void ContiguousPooledMemoryAllocator::CheckExternalGuardRegions() { | |
| size_t guard_region_size = guard_region_size_; | |
| if (!guard_region_size) | |
| return; | |
| ralloc_region_t regions[] = {{.base = 0, .size = guard_region_size}, | |
| {.base = size_ - guard_region_size, .size = guard_region_size}}; | |
| for (size_t i = 0; i < std::size(regions); i++) { | |
| auto& region = regions[i]; | |
| ZX_DEBUG_ASSERT(i < 2); | |
| ZX_DEBUG_ASSERT(region.size == guard_region_size_); | |
| CheckGuardRegion("External", 0, (i == 0), region.base); | |
| } | |
| } | |
| bool ContiguousPooledMemoryAllocator::is_ready() { return is_ready_; } | |
| void ContiguousPooledMemoryAllocator::TraceObserverCallback(async_dispatcher_t* dispatcher, | |
| async::WaitBase* wait, | |
| zx_status_t status, | |
| const zx_packet_signal_t* signal) { | |
| if (status != ZX_OK) | |
| return; | |
| trace_observer_event_.signal(ZX_EVENT_SIGNALED, 0); | |
| // We don't care if tracing was enabled or disabled - if the category is now disabled, the trace | |
| // will just be ignored anyway. | |
| // | |
| // If the category is now enabled, this ensures the trace interval has a contiguous pool size | |
| // datapoint at/near the start. | |
| TracePoolSize(true); | |
| trace_notify_observer_updated(trace_observer_event_.get()); | |
| wait_.Begin(dispatcher); | |
| } | |
| void ContiguousPooledMemoryAllocator::CheckGuardPageCallback(async_dispatcher_t* dispatcher, | |
| async::TaskBase* task, | |
| zx_status_t status) { | |
| if (status != ZX_OK) | |
| return; | |
| // Ignore status - if the post fails, that means the driver is being shut down. | |
| guard_checker_.PostDelayed(dispatcher, kGuardCheckInterval); | |
| CheckExternalGuardRegions(); | |
| if (!has_internal_guard_regions_) | |
| return; | |
| for (auto& region_data : regions_) { | |
| auto& region = region_data.second; | |
| CheckGuardRegionData(region); | |
| } | |
| } | |
| void ContiguousPooledMemoryAllocator::CheckUnusedPagesCallback(async_dispatcher_t* dispatcher, | |
| async::TaskBase* task, | |
| zx_status_t status) { | |
| if (status != ZX_OK) { | |
| return; | |
| } | |
| uint64_t page_size = zx_system_get_page_size(); | |
| auto start = CheckRoundDown( | |
| safemath::CheckDiv(safemath::CheckMul(unused_check_phase_, size_), kUnusedCheckPartialCount), | |
| page_size); | |
| auto end = CheckRoundDown(safemath::CheckDiv(safemath::CheckMul(unused_check_phase_ + 1, size_), | |
| kUnusedCheckPartialCount), | |
| page_size); | |
| // These are always valid since unused_check_phase_ + 1 <= kUnusedCheckPartialCount (64) and | |
| // size_ (the size of a mapped contiguous VMO) is far below UINT64_MAX / kUnusedCheckPartialCount. | |
| ZX_ASSERT(start.IsValid()); | |
| ZX_ASSERT(end.IsValid()); | |
| CheckAnyUnusedPages(start.ValueOrDie(), end.ValueOrDie()); | |
| unused_check_phase_ = (unused_check_phase_ + 1) % kUnusedCheckPartialCount; | |
| // Ignore status - if the post fails, that means the driver is being shut down. | |
| unused_checker_.PostDelayed(dispatcher, | |
| unused_page_check_cycle_period_ / kUnusedCheckPartialCount); | |
| } | |
| void ContiguousPooledMemoryAllocator::CheckUnusedRecentlyPagesCallback( | |
| async_dispatcher_t* dispatcher, async::TaskBase* task, zx_status_t status) { | |
| if (status != ZX_OK) { | |
| return; | |
| } | |
| zx::time now_ish = zx::clock::get_monotonic(); | |
| int32_t deleted_region_index = deleted_regions_next_ - 1; | |
| for (int32_t i = 0; i < deleted_regions_count_; ++i) { | |
| if (deleted_region_index == -1) { | |
| deleted_region_index = kNumDeletedRegions - 1; | |
| } | |
| const DeletedRegion& deleted_region = deleted_regions_[deleted_region_index]; | |
| if (now_ish - deleted_region.when_freed > kUnusedRecentlyAgeThreshold) { | |
| break; | |
| } | |
| CheckAnyUnusedPages(deleted_region.region.base, | |
| deleted_region.region.base + deleted_region.region.size); | |
| --deleted_region_index; | |
| } | |
| unused_recently_checker_.PostDelayed(dispatcher, kUnusedRecentlyPageCheckPeriod); | |
| } | |
| void ContiguousPooledMemoryAllocator::EnsureSteppingTowardOptimalProtectedRanges() { | |
| step_toward_optimal_protected_ranges_min_time_ = | |
| zx::clock::get_monotonic() + kStepTowardOptimalProtectedRangesPeriod; | |
| zx_status_t post_status = step_toward_optimal_protected_ranges_.PostForTime( | |
| dispatcher_, step_toward_optimal_protected_ranges_min_time_); | |
| ZX_ASSERT(post_status == ZX_OK || post_status == ZX_ERR_ALREADY_EXISTS); | |
| } | |
| void ContiguousPooledMemoryAllocator::StepTowardOptimalProtectedRanges( | |
| async_dispatcher_t* dispatcher, async::TaskBase* task, zx_status_t status) { | |
| if (status != ZX_OK) { | |
| return; | |
| } | |
| zx::time now = zx::clock::get_monotonic(); | |
| if (now >= step_toward_optimal_protected_ranges_min_time_) { | |
| bool done = protected_ranges_->StepTowardOptimalRanges(); | |
| UpdateLoanableMetrics(); | |
| if (done) { | |
| LOG(INFO, | |
| "StepTowardOptimalProtectedRanges() - done: %d loaned ratio: %g loaning efficiency: %g", | |
| done, GetLoanableRatio(), GetLoanableEfficiency()); | |
| return; | |
| } else { | |
| LOG(DEBUG, | |
| "StepTowardOptimalProtectedRanges() - done: %d loaned ratio: %g loaning efficiency: %g", | |
| done, GetLoanableRatio(), GetLoanableEfficiency()); | |
| } | |
| step_toward_optimal_protected_ranges_min_time_ = now + kStepTowardOptimalProtectedRangesPeriod; | |
| } | |
| ZX_DEBUG_ASSERT(!step_toward_optimal_protected_ranges_.is_pending()); | |
| step_toward_optimal_protected_ranges_.PostForTime(dispatcher, | |
| step_toward_optimal_protected_ranges_min_time_); | |
| } | |
| protected_ranges::ProtectedRangesCoreControl& | |
| ContiguousPooledMemoryAllocator::protected_ranges_core_control(const fuchsia_sysmem2::Heap& heap) { | |
| ZX_DEBUG_ASSERT(heap_.has_value()); | |
| if (!protected_ranges_core_control_) { | |
| protected_ranges_core_control_ = &parent_device_->protected_ranges_core_control(*heap_); | |
| } | |
| return *protected_ranges_core_control_; | |
| } | |
| void ContiguousPooledMemoryAllocator::DumpRanges() const { | |
| if (protected_ranges_->ranges().empty()) { | |
| return; | |
| } | |
| LOG(INFO, "ContiguousPooledMemoryAllocator::DumpRanges() ###############"); | |
| for (const auto& iter : protected_ranges_->ranges()) { | |
| LOG(INFO, "DumpRanges() - begin: 0x%" PRIx64 " length: 0x%" PRIx64 " end: 0x%" PRIx64, | |
| iter.begin(), iter.length(), iter.end()); | |
| } | |
| } | |
| void ContiguousPooledMemoryAllocator::CheckAnyUnusedPages(uint64_t start_offset, | |
| uint64_t end_offset) { | |
| // This is a list of non-zero-size portions of unused regions within [start_offset, end_offset). | |
| std::vector<ralloc_region_t> todo; | |
| auto process_unused_region = [&todo, start_offset, end_offset](const ralloc_region_t& range) { | |
| ralloc_region_t r = range; | |
| if (r.base + r.size <= start_offset) { | |
| return true; | |
| } | |
| if (r.base >= end_offset) { | |
| return true; | |
| } | |
| ZX_DEBUG_ASSERT((r.base < end_offset) && (r.base + r.size > start_offset)); | |
| // make r be the intersection of r and [start, end) | |
| if (r.base + r.size > end_offset) { | |
| r.size = end_offset - r.base; | |
| } | |
| if (r.base < start_offset) { | |
| uint64_t delta = start_offset - r.base; | |
| r.base += delta; | |
| r.size -= delta; | |
| } | |
| todo.push_back(r); | |
| return true; | |
| }; | |
| if (!protected_ranges_.has_value()) { | |
| // Without protected_ranges_, the unused ranges (in this context, that are check-able) are just | |
| // the raw unused ranges from region_allocator_. | |
| region_allocator_.WalkAvailableRegions([&process_unused_region](const ralloc_region_t* region) { | |
| // struct copy | |
| ralloc_region_t r = *region; | |
| return process_unused_region(r); | |
| }); | |
| } else { | |
| // With protected_ranges_, the unused ranges that are check-able are only the gaps in between | |
| // the protected ranges, as we can't check pages that are protected even if they're not in use | |
| // by an allocated VMO. | |
| // | |
| // Any range that is not protected by protected_ranges_ is also not used according to | |
| // region_allocator_. Some ranges which are protected by protected_ranges_ are not used | |
| // according to region_allocator_, but we can't check those unused pages here. | |
| protected_ranges_->ForUnprotectedRanges( | |
| [&process_unused_region](const protected_ranges::Range& range) { | |
| ralloc_region_t r{.base = range.begin(), .size = range.length()}; | |
| return process_unused_region(r); | |
| }); | |
| } | |
| for (auto& r : todo) { | |
| CheckUnusedRange(r.base, r.size, /*and_also_zero=*/false); | |
| } | |
| } | |
| void ContiguousPooledMemoryAllocator::StashDeletedRegion(const RegionData& region_data) { | |
| if (deleted_regions_.size() != kNumDeletedRegions) { | |
| return; | |
| } | |
| // Remember basic info regarding up to kNumDeletedRegions regions, for potential reporting of | |
| // pattern check failures later. | |
| deleted_regions_[deleted_regions_next_] = | |
| DeletedRegion{.region = {.base = region_data.ptr->base, .size = region_data.ptr->size}, | |
| .when_freed = zx::clock::get_monotonic(), | |
| .name = region_data.name}; | |
| ++deleted_regions_next_; | |
| ZX_DEBUG_ASSERT(deleted_regions_next_ <= kNumDeletedRegions); | |
| if (deleted_regions_next_ == kNumDeletedRegions) { | |
| deleted_regions_next_ = 0; | |
| } | |
| if (deleted_regions_count_ < kNumDeletedRegions) { | |
| ++deleted_regions_count_; | |
| } | |
| } | |
| // The data structure for old regions is optimized for limiting the overall size and limting the | |
| // cost of upkeep of the old region info. It's not optimized for lookup; this lookup can be a bit | |
| // slow, but _should_ never need to happen... | |
| ContiguousPooledMemoryAllocator::DeletedRegion* | |
| ContiguousPooledMemoryAllocator::FindMostRecentDeletedRegion(uint64_t offset) { | |
| uint64_t page_size = zx_system_get_page_size(); | |
| DeletedRegion* deleted_region = nullptr; | |
| int32_t index = deleted_regions_next_ - 1; | |
| int32_t count = 0; | |
| ralloc_region_t offset_page{.base = safe_cast<uint64_t>(offset), | |
| .size = safe_cast<uint64_t>(page_size)}; | |
| while (count < deleted_regions_count_) { | |
| if (index < 0) { | |
| index = kNumDeletedRegions - 1; | |
| } | |
| if (Intersect(offset_page, deleted_regions_[index].region, nullptr)) { | |
| deleted_region = &deleted_regions_[index]; | |
| break; | |
| } | |
| --index; | |
| } | |
| return deleted_region; | |
| } | |
| void ContiguousPooledMemoryAllocator::ReportPatternCheckFailedRange( | |
| const ralloc_region_t& failed_range, const char* which_type) { | |
| if (!unused_pages_guarded_) { | |
| // for now | |
| // | |
| // TODO(dustingreen): Remove this restriction. | |
| LOG(ERROR, "!unused_pages_guarded_ so ReportPatternCheckFailedRange() returning early"); | |
| return; | |
| } | |
| ZX_ASSERT(deleted_regions_.size() == kNumDeletedRegions); | |
| ZX_ASSERT(failed_range.base % zx_system_get_page_size() == 0); | |
| ZX_ASSERT(failed_range.size % zx_system_get_page_size() == 0); | |
| LOG(ERROR, | |
| "########################### SYSMEM DETECTS BAD DMA WRITE (%s) - paddr range start: " | |
| "0x%" PRIx64 " size: 0x%" PRIx64 " (internal offset: 0x%" PRIx64 ")", | |
| which_type, phys_start_ + failed_range.base, failed_range.size, failed_range.base); | |
| std::optional<DeletedRegion*> prev_deleted_region; | |
| bool skipped_since_last = false; | |
| LOG(ERROR, | |
| "DeletedRegion info for failed range expanded by 1 page on either side (... - omitted " | |
| "entries are same DeletedRegion info):"); | |
| int32_t page_size = zx_system_get_page_size(); | |
| auto handle_skip_since_last = [&skipped_since_last] { | |
| if (!skipped_since_last) { | |
| return; | |
| } | |
| LOG(ERROR, "..."); | |
| skipped_since_last = false; | |
| }; | |
| for (int64_t offset = safe_cast<int64_t>(failed_range.base) - page_size; | |
| offset < safe_cast<int64_t>(failed_range.base + failed_range.size) + page_size; | |
| offset += page_size) { | |
| ZX_ASSERT(offset >= -page_size); | |
| if (offset == -page_size) { | |
| LOG(ERROR, "offset -page_size (out of bounds)"); | |
| continue; | |
| } | |
| ZX_ASSERT(offset <= safe_cast<int64_t>(size_)); | |
| if (offset == safe_cast<int64_t>(size_)) { | |
| LOG(ERROR, "offset == size_ (out of bounds)"); | |
| continue; | |
| } | |
| DeletedRegion* deleted_region = FindMostRecentDeletedRegion(offset); | |
| // This can sometimes be comparing nullptr and nullptr, or nullptr and not nullptr, and that's | |
| // fine/expected. | |
| if (prev_deleted_region.has_value() && deleted_region == prev_deleted_region.value()) { | |
| skipped_since_last = true; | |
| continue; | |
| } | |
| prev_deleted_region.emplace(deleted_region); | |
| handle_skip_since_last(); | |
| if (deleted_region) { | |
| zx::time now = zx::clock::get_monotonic(); | |
| zx::duration deleted_ago = now - deleted_region->when_freed; | |
| uint64_t deleted_micros_ago = deleted_ago.to_usecs(); | |
| LOG(ERROR, | |
| "paddr: 0x%" PRIx64 " previous region: 0x%p - paddr base: 0x%" PRIx64 | |
| " reserved-relative offset: 0x%" PRIx64 " size: 0x%" PRIx64 " freed micros ago: %" PRIu64 | |
| " name: %s", | |
| phys_start_ + offset, deleted_region, phys_start_ + deleted_region->region.base, | |
| deleted_region->region.base, deleted_region->region.size, deleted_micros_ago, | |
| deleted_region->name.c_str()); | |
| } else { | |
| LOG(ERROR, "paddr: 0x%" PRIx64 " no previous region found within history window", | |
| phys_start_ + offset); | |
| } | |
| } | |
| // Indicate that the same deleted region was repeated more times at the end, as appropriate. | |
| handle_skip_since_last(); | |
| LOG(ERROR, "END DeletedRangeInfo"); | |
| LOG(ERROR, "Data not matching pattern (... - no diffs, ...... - skipping middle even if diffs):"); | |
| constexpr uint32_t kBytesPerLine = 32; | |
| ZX_ASSERT(zx_system_get_page_size() % kBytesPerLine == 0); | |
| // 2 per byte for hex digits + '!' or '=', not counting terminating \000 | |
| constexpr uint32_t kCharsPerByte = 3; | |
| constexpr uint32_t kMaxStrLen = kCharsPerByte * kBytesPerLine; | |
| char str[kMaxStrLen + 1]; | |
| constexpr uint32_t kMaxDiffBytes = 1024; | |
| static_assert((kMaxDiffBytes / 2) % kBytesPerLine == 0); | |
| uint32_t diff_bytes = 0; | |
| static_assert(kMaxDiffBytes % 2 == 0); | |
| ZX_ASSERT(failed_range.size % kBytesPerLine == 0); | |
| for (uint64_t offset = failed_range.base; offset < failed_range.base + failed_range.size; | |
| offset += kBytesPerLine) { | |
| if (failed_range.size > kMaxDiffBytes && diff_bytes >= kMaxDiffBytes / 2) { | |
| // skip past the middle to keep total diff bytes <= kMaxDiffBytes | |
| offset = failed_range.base + failed_range.size - kMaxDiffBytes / 2; | |
| // indicate per-line skips as appropriate | |
| handle_skip_since_last(); | |
| LOG(ERROR, "......"); | |
| // The part near the end of the failed_range won't satisfy the enclosing if's condition due to | |
| // starting kMaxDiffBytes / 2 from the end, so the enclosing loop will stop first. | |
| diff_bytes = 0; | |
| } | |
| bool match = !memcmp(&mapping_[offset], &guard_region_data_[offset % page_size], kBytesPerLine); | |
| if (!match) { | |
| handle_skip_since_last(); | |
| LOG(ERROR, "paddr: 0x%" PRIx64 " offset 0x%" PRIx64, phys_start_ + offset, offset); | |
| for (uint32_t i = 0; i < kBytesPerLine; ++i) { | |
| bool byte_match = mapping_[offset + i] == guard_region_data_[(offset + i) % page_size]; | |
| // printing 2 hex characters + 1 indicator char + \000 | |
| int result = sprintf(&str[i * kCharsPerByte], "%02x%s", mapping_[offset + i], | |
| byte_match ? "=" : "!"); | |
| ZX_DEBUG_ASSERT(result == kCharsPerByte); | |
| } | |
| diff_bytes += kBytesPerLine; | |
| LOG(ERROR, "%s", str); | |
| } else { | |
| skipped_since_last = true; | |
| } | |
| } | |
| // Indicate no diffs at end, as appropriate. | |
| handle_skip_since_last(); | |
| LOG(ERROR, "END data not matching pattern"); | |
| } | |
| void ContiguousPooledMemoryAllocator::CheckUnusedRange(uint64_t offset, uint64_t size, | |
| bool and_also_zero) { | |
| ZX_DEBUG_ASSERT(mapping_); | |
| uint32_t succeeded_count = 0; | |
| uint32_t failed_count = 0; | |
| uint32_t page_size = zx_system_get_page_size(); | |
| auto nop_loan_range = [](const ralloc_region_t& range) { | |
| // We shouldn't read the pages in this case, and we shouldn't call zx_cache_flush() on these | |
| // pages either. | |
| }; | |
| auto maybe_zero_range = [this, and_also_zero](const ralloc_region_t& range) { | |
| if (!and_also_zero) { | |
| return; | |
| } | |
| // We don't have to cache flush here because the logical_buffer_collection.cc caller does that. | |
| memset(&mapping_[range.base], 0x00, range.size); | |
| }; | |
| const ralloc_region_t unused_range{.base = offset, .size = size}; | |
| ForUnusedGuardPatternRanges( | |
| unused_range, | |
| /*pattern_func=*/ | |
| [this, page_size, and_also_zero, &succeeded_count, | |
| &failed_count](const ralloc_region_t& range) { | |
| // ensure the CPU will see DMA writes up to this point (unless pattern happens to match of | |
| // course) | |
| zx_cache_flush(&mapping_[range.base], range.size, | |
| ZX_CACHE_FLUSH_DATA | ZX_CACHE_FLUSH_INVALIDATE); | |
| std::optional<ralloc_region_t> zero_range; | |
| auto handle_zero_range = [this, &zero_range, and_also_zero] { | |
| if (!and_also_zero || !zero_range.has_value()) { | |
| return; | |
| } | |
| // We don't have to cache flush here because the logical_buffer_collection.cc caller does | |
| // that. | |
| memset(&mapping_[zero_range->base], 0x00, zero_range->size); | |
| zero_range.reset(); | |
| }; | |
| auto ensure_handle_zero_range = fit::defer([&handle_zero_range] { handle_zero_range(); }); | |
| std::optional<ralloc_region_t> failed_range; | |
| auto handle_failed_range = [this, &failed_range, and_also_zero] { | |
| if (!failed_range.has_value()) { | |
| return; | |
| } | |
| ReportPatternCheckFailedRange(failed_range.value(), "unused"); | |
| IncrementGuardRegionFailureInspectData(); | |
| // So we don't keep finding the same corruption over and over. | |
| if (!and_also_zero) { | |
| FillUnusedRangeWithGuard(failed_range->base, failed_range->size); | |
| } | |
| failed_range.reset(); | |
| }; | |
| auto ensure_handle_failed_range = | |
| fit::defer([&handle_failed_range] { handle_failed_range(); }); | |
| uint64_t end = range.base + range.size; | |
| uint64_t todo_size; | |
| for (uint64_t iter = range.base; iter != end; iter += todo_size) { | |
| todo_size = std::min(unused_guard_data_size_, end - iter); | |
| ZX_DEBUG_ASSERT(todo_size % page_size == 0); | |
| if (unlikely(memcmp(guard_region_data_.data(), &mapping_[iter], todo_size))) { | |
| if (!failed_range.has_value()) { | |
| failed_range.emplace(ralloc_region_t{.base = iter, .size = 0}); | |
| } | |
| failed_range->size += todo_size; | |
| ++failed_count; | |
| } else { | |
| // if any failed range is active, it's ending here, so report it | |
| handle_failed_range(); | |
| ++succeeded_count; | |
| } | |
| // We memset() incrementally for better cache locality (vs. forwarding to | |
| // maybe_zero_range to zero the whole incoming range). However, if we have a failed | |
| // pattern check range in progress, we don't immediately zero because in that case we | |
| // need to print diffs first. This is somewhat more complicated than just checking a big | |
| // range then zeroing a big range, but this should also be quite a bit faster by staying | |
| // in cache until we're done reading and writing the data. | |
| if (and_also_zero) { | |
| if (!zero_range.has_value()) { | |
| zero_range.emplace(ralloc_region_t{.base = iter, .size = 0}); | |
| } | |
| zero_range->size += todo_size; | |
| if (!failed_range.has_value()) { | |
| // Zero immediately if we don't need to keep the data around for failed_range reasons. | |
| handle_zero_range(); | |
| } | |
| } | |
| } | |
| // ~ensure_handle_failed_range | |
| // ~ensure_handle_zero_range | |
| }, | |
| /*loan_func=*/nop_loan_range, | |
| /*zero_func=*/maybe_zero_range); | |
| metrics_.LogUnusedPageCheckCounts(succeeded_count, failed_count); | |
| } | |
| uint64_t ContiguousPooledMemoryAllocator::CalculateLargeContiguousRegionSize() { | |
| constexpr uint32_t kRegionTrackerCount = 10; | |
| std::array<uint64_t, kRegionTrackerCount> largest_regions{}; | |
| // All elements are identical, so largest_regions is already a heap. | |
| region_allocator_.WalkAvailableRegions([&](const ralloc_region_t* r) -> bool { | |
| if (r->size > largest_regions[0]) { | |
| // Pop the smallest element. | |
| std::pop_heap(largest_regions.begin(), largest_regions.end(), std::greater<uint64_t>()); | |
| // Push the region size onto the heap. | |
| largest_regions[kRegionTrackerCount - 1] = r->size; | |
| std::push_heap(largest_regions.begin(), largest_regions.end(), std::greater<uint64_t>()); | |
| } | |
| return true; | |
| }); | |
| uint64_t top_region_sum = std::accumulate(largest_regions.begin(), largest_regions.end(), 0); | |
| return top_region_sum; | |
| } | |
| void ContiguousPooledMemoryAllocator::DumpPoolStats() { | |
| uint64_t unused_size = 0; | |
| uint64_t max_free_size = 0; | |
| region_allocator_.WalkAvailableRegions( | |
| [&unused_size, &max_free_size](const ralloc_region_t* r) -> bool { | |
| unused_size += r->size; | |
| max_free_size = std::max(max_free_size, r->size); | |
| return true; | |
| }); | |
| uint64_t top_region_sum = CalculateLargeContiguousRegionSize(); | |
| LOG(INFO, | |
| "%s unused total: %ld bytes, max free size %ld bytes " | |
| "AllocatedRegionCount(): %zu AvailableRegionCount(): %zu, largest 10 regions %zu", | |
| heap_name_, unused_size, max_free_size, region_allocator_.AllocatedRegionCount(), | |
| region_allocator_.AvailableRegionCount(), top_region_sum); | |
| for (auto& [vmo, region] : regions_) { | |
| LOG(INFO, "Region koid %ld name %s size %zu", region.koid, region.name.c_str(), | |
| region.ptr->size); | |
| } | |
| } | |
| void ContiguousPooledMemoryAllocator::DumpPoolHighWaterMark() { | |
| LOG(INFO, "%s high_water_mark_used_size_: %ld bytes, max_free_size_at_high_water_mark_ %ld bytes", | |
| heap_name_, high_water_mark_used_size_, max_free_size_at_high_water_mark_); | |
| } | |
| void ContiguousPooledMemoryAllocator::TracePoolSize(bool initial_trace) { | |
| uint64_t used_size = 0; | |
| region_allocator_.WalkAllocatedRegions([&used_size](const ralloc_region_t* r) -> bool { | |
| used_size += r->size; | |
| return true; | |
| }); | |
| used_size_property_.Set(used_size); | |
| large_contiguous_region_sum_property_.Set(CalculateLargeContiguousRegionSize()); | |
| // heap_name_ points at a string literal, as required by TRACE_COUNTER | |
| TRACE_COUNTER("gfx", heap_name_, counter_id_, "size", used_size); | |
| bool trace_high_water_mark = initial_trace; | |
| if (used_size > high_water_mark_used_size_) { | |
| high_water_mark_used_size_ = used_size; | |
| trace_high_water_mark = true; | |
| high_water_mark_property_.Set(high_water_mark_used_size_); | |
| free_at_high_water_mark_property_.Set(size_ - high_water_mark_used_size_); | |
| uint64_t max_free_size = 0; | |
| region_allocator_.WalkAvailableRegions([&max_free_size](const ralloc_region_t* r) -> bool { | |
| max_free_size = std::max(max_free_size, r->size); | |
| return true; | |
| }); | |
| max_free_size_at_high_water_mark_ = max_free_size; | |
| max_free_at_high_water_property_.Set(max_free_size_at_high_water_mark_); | |
| // This can be a bit noisy at first, but then settles down quickly. | |
| DumpPoolHighWaterMark(); | |
| } | |
| if (trace_high_water_mark) { | |
| TRACE_INSTANT("gfx", "Increased high water mark", TRACE_SCOPE_THREAD, "heap_name", heap_name_, | |
| "size", high_water_mark_used_size_); | |
| } | |
| } | |
| void ContiguousPooledMemoryAllocator::UpdateLoanableMetrics() { | |
| double efficiency = GetLoanableEfficiency(); | |
| if (efficiency < min_efficiency_) { | |
| min_efficiency_ = efficiency; | |
| } | |
| loanable_efficiency_property_.Set(efficiency); | |
| loanable_ratio_property_.Set(GetLoanableRatio()); | |
| uint64_t loanable_bytes = GetLoanableBytes(); | |
| loanable_bytes_property_.Set(loanable_bytes); | |
| loanable_mebibytes_property_.Set(loanable_bytes / kMiB); | |
| } | |
| uint64_t ContiguousPooledMemoryAllocator::GetVmoRegionOffsetForTest(const zx::vmo& vmo) { | |
| return regions_[vmo.get()].ptr->base + guard_region_size_; | |
| } | |
| bool ContiguousPooledMemoryAllocator::is_already_cleared_on_allocate() { | |
| // This is accurate for CPU-accessible, non-VDEC part-time protected, and VDEC full-time | |
| // protected. | |
| // | |
| // We zero these ways: | |
| // * Zircon zeroing reclaimed pages | |
| // * zeroing just-checked pattern pages | |
| // * using the TEE to zero as appropriate | |
| return true; | |
| } | |
| template <typename F1, typename F2, typename F3> | |
| void ContiguousPooledMemoryAllocator::ForUnusedGuardPatternRanges(const ralloc_region_t& region, | |
| F1 pattern_func, F2 loan_func, | |
| F3 zero_func) { | |
| if (!protected_ranges_.has_value()) { | |
| ForUnusedGuardPatternRangesInternal(region, pattern_func, loan_func, zero_func); | |
| } else { | |
| const protected_ranges::Range unused_range = | |
| protected_ranges::Range::BeginLength(region.base, region.size); | |
| protected_ranges_->ForUnprotectedRangesOverlappingRange( | |
| unused_range, | |
| /*unprotected_range=*/[this, &pattern_func, &loan_func, | |
| &zero_func](const protected_ranges::Range& unprotected_range) { | |
| // The extent of unprotected_range is already clamped to only include pages that are also | |
| // in unused_range. | |
| ralloc_region_t region{.base = unprotected_range.begin(), | |
| .size = unprotected_range.length()}; | |
| ForUnusedGuardPatternRangesInternal(region, pattern_func, loan_func, zero_func); | |
| }); | |
| } | |
| } | |
| template <typename F1, typename F2, typename F3> | |
| void ContiguousPooledMemoryAllocator::ForUnusedGuardPatternRangesInternal( | |
| const ralloc_region_t& region, F1 pattern_func, F2 loan_func, F3 zero_func) { | |
| if (!can_decommit_ && !unused_pages_guarded_) { | |
| zero_func(region); | |
| return; | |
| } | |
| if (!can_decommit_) { | |
| pattern_func(region); | |
| return; | |
| } | |
| if (!unused_pages_guarded_) { | |
| loan_func(region); | |
| return; | |
| } | |
| // We already know that the passed-in region doesn't overlap with any used region. It may be | |
| // adjacent to another unused range. | |
| uint64_t region_base = region.base; | |
| uint64_t region_end = region.base + region.size; | |
| ZX_DEBUG_ASSERT(region_end > region_base); | |
| // The "meta pattern" is just a page aligned to unused_guard_pattern_period_ that's kept for | |
| // DMA-write-after-free detection purposes, followed by the rest of unused_guard_pattern_period_ | |
| // that's loaned. The meta pattern repeats through the whole offset space from 0 to size_, but | |
| // only applies to portions of the space which are not currently used. | |
| auto meta_pattern_start = CheckRoundDown(region_base, unused_guard_pattern_period_bytes_); | |
| auto meta_pattern_end = CheckRoundUp(region_end, unused_guard_pattern_period_bytes_); | |
| if (!meta_pattern_start.IsValid() || !meta_pattern_end.IsValid()) { | |
| return; | |
| } | |
| for (uint64_t meta_pattern_base = meta_pattern_start.ValueOrDie(); | |
| meta_pattern_base < meta_pattern_end.ValueOrDie(); | |
| meta_pattern_base += unused_guard_pattern_period_bytes_) { | |
| ralloc_region_t raw_keep{ | |
| .base = meta_pattern_base, | |
| .size = unused_to_pattern_bytes_, | |
| }; | |
| ralloc_region_t keep; | |
| if (Intersect(raw_keep, region, &keep)) { | |
| pattern_func(keep); | |
| } | |
| ralloc_region_t raw_loan{ | |
| .base = raw_keep.base + raw_keep.size, | |
| .size = unused_guard_pattern_period_bytes_ - unused_to_pattern_bytes_, | |
| }; | |
| ralloc_region_t loan; | |
| if (Intersect(raw_loan, region, &loan)) { | |
| loan_func(loan); | |
| } | |
| } | |
| } | |
| void ContiguousPooledMemoryAllocator::OnRegionUnused(const ralloc_region_t& region) { | |
| ForUnusedGuardPatternRanges( | |
| region, | |
| /*pattern_func=*/ | |
| [this](const ralloc_region_t& pattern_range) { | |
| ZX_DEBUG_ASSERT(unused_pages_guarded_); | |
| FillUnusedRangeWithGuard(pattern_range.base, pattern_range.size); | |
| }, | |
| /*loan_func=*/ | |
| [this](const ralloc_region_t& loan_range) { | |
| ZX_DEBUG_ASSERT(can_decommit_); | |
| zx_status_t zero_status = ZX_OK; | |
| zx_status_t decommit_status = contiguous_vmo_.op_range(ZX_VMO_OP_DECOMMIT, loan_range.base, | |
| loan_range.size, nullptr, 0); | |
| if (decommit_status != ZX_OK) { | |
| // sysmem only calls the current method on one thread | |
| static zx::time next_log_time = zx::time::infinite_past(); | |
| zx::time now = zx::clock::get_monotonic(); | |
| if (now >= next_log_time) { | |
| ZX_DEBUG_ASSERT(heap_.has_value()); | |
| LOG(INFO, | |
| "(log rate limited) ZX_VMO_OP_DECOMMIT failed on contiguous VMO - decommit_status: " | |
| "%d base: 0x%" PRIx64 " size: 0x%" PRIx64 " heap_type: %s heap_id: 0x%" PRIx64, | |
| decommit_status, loan_range.base, loan_range.size, heap_->heap_type()->c_str(), | |
| heap_->id().value()); | |
| next_log_time = now + zx::sec(30); | |
| } | |
| // If we can't decommit (unexpected), we try to zero before giving up. Overall, we rely | |
| // on all loan_range(s) to be zeroed by a decommit/commit to be able to skip zeroing of | |
| // previously loaned ranges, so we need to zero here to make it look as if the decommit | |
| // worked from a zeroing point of view. If we also can't zero, we assert. The decommit | |
| // is not expected to fail unless decommit of contiguous VMO pages is disabled via kernel | |
| // command line flag. If the VMO is cached, i.e. is always cpu accessible, then we can | |
| // ask the kernel to zero it, otherwise we must do it ourselves. | |
| if (is_always_cpu_accessible_) { | |
| zero_status = contiguous_vmo_.op_range(ZX_VMO_OP_ZERO, loan_range.base, loan_range.size, | |
| nullptr, 0); | |
| // We don't expect DECOMMIT or ZERO to ever fail. | |
| ZX_ASSERT_MSG(zero_status == ZX_OK, | |
| "ZX_VMO_OP_DECOMMIT and ZX_VMO_OP_ZERO both failed - zero_status: %d\n", | |
| zero_status); | |
| } else { | |
| ZX_DEBUG_ASSERT(mapping_); | |
| // Although our zeroing loop only requires the range be word aligned we know that we | |
| // should have only been asked to loan page aligned regions to begin with. | |
| ZX_DEBUG_ASSERT(loan_range.base % zx_system_get_page_size() == 0); | |
| ZX_DEBUG_ASSERT(loan_range.size % zx_system_get_page_size() == 0); | |
| // Need to explicitly zero using a volatile word pointer since memset might use a | |
| // non-word sized access, or the 'dc zva' instruction on aarch64, both of which are | |
| // invalid on an uncached address. | |
| volatile uint64_t* addr = reinterpret_cast<uint64_t*>(&mapping_[loan_range.base]); | |
| for (size_t i = 0; i < loan_range.size / sizeof(uint64_t); i++) { | |
| addr[i] = 0; | |
| } | |
| } | |
| } | |
| }, | |
| /*zero_func=*/ | |
| [this](const ralloc_region_t& zero_range) { | |
| ZX_DEBUG_ASSERT(!can_decommit_); | |
| ZX_DEBUG_ASSERT(!unused_pages_guarded_); | |
| // We don't actually need to zero here since this is during delete. We zero during allocate | |
| // instead. | |
| }); | |
| } | |
| zx_status_t ContiguousPooledMemoryAllocator::CommitRegion(const ralloc_region_t& region) { | |
| TRACE_DURATION("gfx", "ContiguousPooledMemoryAllocator::CommitRegion"); | |
| if (!can_decommit_) { | |
| return ZX_OK; | |
| } | |
| zx_status_t status = | |
| contiguous_vmo_.op_range(ZX_VMO_OP_COMMIT, region.base, region.size, nullptr, 0); | |
| return status; | |
| } | |
| // loanable pages / un-used pages | |
| double ContiguousPooledMemoryAllocator::GetLoanableEfficiency() { | |
| if (protected_ranges_.has_value()) { | |
| return protected_ranges_->GetEfficiency(); | |
| } else { | |
| if (is_ever_cpu_accessible_) { | |
| return 1.0; | |
| } else { | |
| return 0.0; | |
| } | |
| } | |
| } | |
| // loanable pages / total pages | |
| double ContiguousPooledMemoryAllocator::GetLoanableRatio() { | |
| if (protected_ranges_.has_value()) { | |
| return protected_ranges_->GetLoanableRatio(); | |
| } else { | |
| if (is_ever_cpu_accessible_) { | |
| uint64_t loanable_bytes = size_ - allocated_bytes_; | |
| return safe_cast<double>(loanable_bytes) / safe_cast<double>(size_); | |
| } else { | |
| return 0.0; | |
| } | |
| } | |
| } | |
| uint64_t ContiguousPooledMemoryAllocator::GetLoanableBytes() { | |
| if (protected_ranges_.has_value()) { | |
| return protected_ranges_->GetLoanableBytes(); | |
| } else { | |
| if (is_ever_cpu_accessible_) { | |
| return size_ - allocated_bytes_; | |
| } else { | |
| return 0; | |
| } | |
| } | |
| } | |
| bool ContiguousPooledMemoryAllocator::RangesControl::IsDynamic() { | |
| return parent_->protected_ranges_core_control(parent_->heap()).IsDynamic(); | |
| } | |
| uint64_t ContiguousPooledMemoryAllocator::RangesControl::MaxRangeCount() { | |
| return parent_->protected_ranges_core_control(parent_->heap()).MaxRangeCount(); | |
| } | |
| uint64_t ContiguousPooledMemoryAllocator::RangesControl::GetRangeGranularity() { | |
| return parent_->protected_ranges_core_control(parent_->heap()).GetRangeGranularity(); | |
| } | |
| bool ContiguousPooledMemoryAllocator::RangesControl::HasModProtectedRange() { | |
| return parent_->protected_ranges_core_control(parent_->heap()).HasModProtectedRange(); | |
| } | |
| void ContiguousPooledMemoryAllocator::RangesControl::AddProtectedRange( | |
| const protected_ranges::Range& zero_based_range) { | |
| // We pin/unpin in AddProtectedRange() / DelProtectedRange() instead of UseRange()/UnUseRange(), | |
| // because UnUseRange() isn't guaranteed to line up with any previous UseRange(), while the former | |
| // is required to specify specific tracked ranges. | |
| // | |
| // The point of pinning is entirely about preventing Zircon from potentially trying to use | |
| // HW-protected pages between when sysmem hypothetically crashes and when that sysmem crash | |
| // triggers a hard reboot. | |
| // | |
| // TODO(https://fxbug.dev/42178137): When possible, configure sysmem to trigger reboot on driver | |
| // remove. | |
| zx::pmt pmt; | |
| zx_paddr_t paddr; | |
| zx_status_t pin_result = parent_->parent_device_->bti().pin( | |
| ZX_BTI_PERM_READ | ZX_BTI_PERM_READ | ZX_BTI_CONTIGUOUS, parent_->contiguous_vmo_, | |
| zero_based_range.begin(), zero_based_range.length(), &paddr, 1, &pmt); | |
| // If sysmem can't pin an already-committed range, do a hard reboot so things work again. We do | |
| // not ZX_ASSERT() if UseRange()'s commit fails; that can fail cleanly, but once the pages are | |
| // committed we expect pin to work here since pages don't need to be allocated by this pin. This | |
| // is because Zircon doesn't implicitly decommit pages from contiguous VMOs (and is unlikely to | |
| // in future given how currently-present pages of contiguous VMOs tend to get pinned again fairly | |
| // soon anyway, else why did they need to be contiguous). But if this changes, we'll see this | |
| // ZX_ASSERT() fire and we'll need to accomodate the possibility of pin failing. | |
| ZX_ASSERT(pin_result == ZX_OK); | |
| zero_based_range.SetMutablePmt(std::move(pmt)); | |
| const auto range = protected_ranges::Range::BeginLength( | |
| parent_->phys_start_ + zero_based_range.begin(), zero_based_range.length()); | |
| parent_->protected_ranges_core_control(parent_->heap()).AddProtectedRange(range); | |
| } | |
| void ContiguousPooledMemoryAllocator::RangesControl::DelProtectedRange( | |
| const protected_ranges::Range& zero_based_range) { | |
| const auto range = protected_ranges::Range::BeginLength( | |
| parent_->phys_start_ + zero_based_range.begin(), zero_based_range.length()); | |
| parent_->protected_ranges_core_control(parent_->heap()).DelProtectedRange(range); | |
| // The pin_count will prevent actual un-pinning for any page that's still covered by a different | |
| // pin. | |
| zx::pmt pmt = zero_based_range.TakeMutablePmt(); | |
| zx_status_t unpin_status = pmt.unpin(); | |
| ZX_ASSERT(unpin_status == ZX_OK); | |
| } | |
| void ContiguousPooledMemoryAllocator::RangesControl::ModProtectedRange( | |
| const protected_ranges::Range& old_zero_based_range, | |
| const protected_ranges::Range& new_zero_based_range) { | |
| // pin new | |
| zx::pmt pmt; | |
| zx_paddr_t paddr; | |
| zx_status_t pin_result = parent_->parent_device_->bti().pin( | |
| ZX_BTI_PERM_READ | ZX_BTI_PERM_READ | ZX_BTI_CONTIGUOUS, parent_->contiguous_vmo_, | |
| new_zero_based_range.begin(), new_zero_based_range.length(), &paddr, 1, &pmt); | |
| // If sysmem can't pin an already-committed range, do a hard reboot so things work again. We do | |
| // not ZX_ASSERT() if UseRange()'s commit fails; that can fail cleanly, but once the pages are | |
| // committed we expect pin to work here since pages don't need to be allocated by this pin. This | |
| // is because Zircon doesn't implicitly decommit pages from contiguous VMOs (and is unlikely to | |
| // in future given how currently-present pages of contiguous VMOs tend to get pinned again fairly | |
| // soon anyway, else why did they need to be contiguous). But if this changes, we'll see this | |
| // ZX_ASSERT() fire and we'll need to accomodate the possibility of pin failing. | |
| ZX_ASSERT(pin_result == ZX_OK); | |
| new_zero_based_range.SetMutablePmt(std::move(pmt)); | |
| const auto old_range = protected_ranges::Range::BeginLength( | |
| parent_->phys_start_ + old_zero_based_range.begin(), old_zero_based_range.length()); | |
| const auto new_range = protected_ranges::Range::BeginLength( | |
| parent_->phys_start_ + new_zero_based_range.begin(), new_zero_based_range.length()); | |
| parent_->protected_ranges_core_control(parent_->heap()).ModProtectedRange(old_range, new_range); | |
| // unpin old | |
| // | |
| // The pin_count will prevent actual un-pinning for any page that's still covered by a different | |
| // pin. | |
| pmt = old_zero_based_range.TakeMutablePmt(); | |
| zx_status_t unpin_status = pmt.unpin(); | |
| ZX_ASSERT(unpin_status == ZX_OK); | |
| } | |
| void ContiguousPooledMemoryAllocator::RangesControl::ZeroProtectedSubRange( | |
| bool is_covering_range_explicit, const protected_ranges::Range& zero_based_range) { | |
| const auto range = protected_ranges::Range::BeginLength( | |
| parent_->phys_start_ + zero_based_range.begin(), zero_based_range.length()); | |
| parent_->protected_ranges_core_control(parent_->heap()) | |
| .ZeroProtectedSubRange(is_covering_range_explicit, range); | |
| } | |
| uint64_t ContiguousPooledMemoryAllocator::RangesControl::GetBase() { return 0; } | |
| uint64_t ContiguousPooledMemoryAllocator::RangesControl::GetSize() { return parent_->size_; } | |
| bool ContiguousPooledMemoryAllocator::RangesControl::UseRange( | |
| const protected_ranges::Range& range) { | |
| ralloc_region_t region{.base = range.begin(), .size = range.length()}; | |
| bool result = (ZX_OK == parent_->CommitRegion(region)); | |
| return result; | |
| } | |
| void ContiguousPooledMemoryAllocator::RangesControl::UnUseRange( | |
| const protected_ranges::Range& range) { | |
| ralloc_region_t region{.base = range.begin(), .size = range.length()}; | |
| parent_->OnRegionUnused(region); | |
| } | |
| } // namespace sysmem_service |