| // Copyright 2021 The Fuchsia Authors | |
| // | |
| // Use of this source code is governed by a MIT-style | |
| // license that can be found in the LICENSE file or at | |
| // https://opensource.org/licenses/MIT | |
| #include <lib/boot-options/boot-options.h> | |
| #include <lib/counters.h> | |
| #include <lib/fit/defer.h> | |
| #include <lib/zircon-internal/macros.h> | |
| #include <cassert> | |
| #include <cstdint> | |
| #include <kernel/lockdep.h> | |
| #include <ktl/algorithm.h> | |
| #include <vm/loan_sweeper.h> | |
| #include <vm/pmm.h> | |
| #include <vm/scanner.h> | |
| #include <vm/vm_cow_pages.h> | |
| #include "pmm_node.h" | |
| #include <ktl/enforce.h> | |
| KCOUNTER(sweep_count, "vm.reclamation.sweep_count") | |
| KCOUNTER(sweep_looped, "vm.reclamation.sweep_looped") | |
| KCOUNTER(sweep_pages_examined, "vm.reclamation.sweep_pages_examined") | |
| KCOUNTER(sweep_pages_swept_to_loaned, "vm.reclamation.sweep_pages_swept_to_loaned") | |
| KCOUNTER(sweep_page_chase_retried, "vm.reclamation.sweep_page_chase_retried") | |
| KCOUNTER(sweep_page_chase_gave_up, "vm.reclamation.sweep_page_chase_gave_up") | |
| LoanSweeper::LoanSweeper() | |
| : page_queues_(pmm_page_queues()), ppb_config_(pmm_physical_page_borrowing_config()) {} | |
| LoanSweeper::LoanSweeper(PageQueues* queues, PhysicalPageBorrowingConfig* config) | |
| : page_queues_(queues), ppb_config_(config) {} | |
| void LoanSweeper::Init() { | |
| num_arenas_ = pmm_num_arenas(); | |
| fbl::AllocChecker ac; | |
| arenas_ = ktl::make_unique<pmm_arena_info[]>(&ac, num_arenas_); | |
| // This allocation happens super early, and only super early, so require it to succeed. If it | |
| // doesn't succeed, most likely something has gone quite wrong quite early. | |
| ASSERT(ac.check()); | |
| zx_status_t status = | |
| pmm_get_arena_info(num_arenas_, /*i=*/0, &arenas_[0], num_arenas_ * sizeof(arenas_[0])); | |
| // The only failures are caller bugs, but also check in release in case that changes. | |
| ASSERT(status == ZX_OK); | |
| min_paddr_ = ktl::numeric_limits<paddr_t>::max(); | |
| max_paddr_ = ktl::numeric_limits<paddr_t>::min(); | |
| for (uint32_t i = 0; i < num_arenas_; ++i) { | |
| auto& arena = arenas_[i]; | |
| min_paddr_ = ktl::min(min_paddr_, arena.base); | |
| max_paddr_ = ktl::max(max_paddr_, arena.base + arena.size - 1); | |
| #if ZX_DEBUG_ASSERT_IMPLEMENTED | |
| for (uint32_t j = 0; j < num_arenas_; ++j) { | |
| auto& arena_2 = arenas_[j]; | |
| if (&arena_2 != &arena) { | |
| // Failing this assert would mean two arenas overlap on the same physical range, which would | |
| // break assumptions elsewhere in LoanSweeper. | |
| DEBUG_ASSERT(arena_2.base + arena_2.size - 1 < arena.base || | |
| arena_2.base > arena.base + arena.size - 1); | |
| } | |
| } | |
| #endif | |
| } | |
| next_start_paddr_ = min_paddr_; | |
| } | |
| uint64_t LoanSweeper::ForceSynchronousSweep() { return SynchronousSweepInternal(); } | |
| // For now, we don't expect the number of loaned pages to typically exceed the number of non-loaned | |
| // non-pinned pages (replaceable pages, roughly speaking) so it's reasonable enough for now to just | |
| // sweep the pmm's page array looking for non-loaned non-pinned used pages when we have free loaned | |
| // pages available. In the event that there are so many pinned pages that we run out of replaceable | |
| // pages before we run out of loaned pages, we'll end up scanning the whole pmm page array and find | |
| // nothing. In that event, we'll count the occurrence for now. | |
| // | |
| // Other than too many pages pinned to be able to make use of all loaned pages, we expect the | |
| // density of replace-able pages to be high enough that sweeping in physical order is amortized | |
| // reasonably efficient. | |
| // | |
| // We sweep from a starting offset that's persistent from the end of last sweep, since typically | |
| // any sweeps due to low free pages will end early when we exhaust all loaned pages, and there's a | |
| // better chance of finding replace-able non-loaned pages when we start from where we left off. | |
| uint64_t LoanSweeper::SynchronousSweepInternal() { | |
| // Sweep (up to) all the pages to find any VMO pages we can move to loaned physical pages, while | |
| // we have any free loaned physical pages available. | |
| // | |
| // We iterate in physical page order because the info we need is in the pmm physical page array, | |
| // not in VmCowPages. For now, there's no particular reason to expect a VmPageListNode to | |
| // typically contain physically-contiguous pages, so we'd be jumping around in the pmm physical | |
| // page array if we iterated in VmCowPages order. Non-sequential access is only done for pages we | |
| // can probably replace with a loaned physical page. | |
| sweep_count.Add(1); | |
| pmm_arena_info_t* cached_arena = nullptr; | |
| for (uint32_t i = 0; i < num_arenas_; ++i) { | |
| auto& arena = arenas_[i]; | |
| if (arena.base <= next_start_paddr_ && next_start_paddr_ < arena.base + arena.size) { | |
| cached_arena = &arena; | |
| break; | |
| } | |
| } | |
| auto get_next_iter = [this, &cached_arena](paddr_t iter) { | |
| DEBUG_ASSERT(IS_PAGE_ALIGNED(iter)); | |
| iter += PAGE_SIZE; | |
| if (cached_arena && iter < cached_arena->base + cached_arena->size) { | |
| return iter; | |
| } | |
| pmm_arena_info_t* next_arena = nullptr; | |
| pmm_arena_info_t* min_arena = &arenas_[0]; | |
| for (uint32_t i = 0; i < num_arenas_; ++i) { | |
| auto& arena = arenas_[i]; | |
| if (arena.base >= iter && (!next_arena || arena.base < next_arena->base)) { | |
| next_arena = &arena; | |
| } | |
| if (arena.base < min_arena->base) { | |
| min_arena = &arena; | |
| } | |
| } | |
| if (!next_arena) { | |
| next_arena = min_arena; | |
| } | |
| cached_arena = next_arena; | |
| return next_arena->base; | |
| }; | |
| bool ppb_enabled = ppb_config_->is_any_borrowing_enabled(); | |
| uint64_t replaced_non_loaned_page_count = 0; | |
| paddr_t iter; | |
| auto set_next_start_addr = fit::defer([this, &iter] { next_start_paddr_ = iter; }); | |
| Guard<Mutex> guard(&lock_); | |
| bool first = true; | |
| for (iter = next_start_paddr_; iter != next_start_paddr_ || first; iter = get_next_iter(iter)) { | |
| first = false; | |
| if (ppb_enabled) { | |
| if (!pmm_count_loaned_free_pages()) { | |
| return replaced_non_loaned_page_count; | |
| } | |
| } else { | |
| if (!pmm_count_loaned_used_pages()) { | |
| return replaced_non_loaned_page_count; | |
| } | |
| } | |
| vm_page_t* page = paddr_to_vm_page(iter); | |
| DEBUG_ASSERT(page); | |
| DEBUG_ASSERT(page->paddr() == iter); | |
| sweep_pages_examined.Add(1); | |
| // We're willing to try a limited number of times to chase down a non-loaned page as it moves | |
| // between VmCowPages, but limit the iteration count since it's not critical that we replace | |
| // every single non-loaned page we iterate over, as there should typically be plenty of | |
| // non-loaned replaceable pages to use up all the loaned pages. | |
| bool replaced = false; | |
| const uint32_t kMaxPageChaseIterations = 3; | |
| uint32_t page_try_ordinal; | |
| for (page_try_ordinal = 0; page_try_ordinal < kMaxPageChaseIterations; ++page_try_ordinal) { | |
| if (page_try_ordinal != 0) { | |
| sweep_page_chase_retried.Add(1); | |
| } | |
| // These are approximate checks, as we're not holding the PageQueues lock or the pmm lock | |
| // continuously until we replace the pagel. | |
| if (page->state() != vm_page_state::OBJECT) { | |
| // next page | |
| break; | |
| } | |
| if (ppb_enabled == pmm_is_loaned(page)) { | |
| // next page | |
| break; | |
| } | |
| // That's enough pre-checking to filter out most pages that won't work. Now try to find the | |
| // owning VmCowPages and replace this page with a loaned page (or non-loaned page). | |
| // | |
| // Despite the efforts of GetCowWithReplaceablePage, we may still find below that a returned | |
| // VmCowPages doesn't have the page any more, which is the reason for the directly enclosing | |
| // loop. | |
| auto maybe_vmo_backlink = | |
| pmm_page_queues()->GetCowWithReplaceablePage(page, /*owning_cow=*/nullptr); | |
| if (!maybe_vmo_backlink) { | |
| // There may not be a backlink if page already became FREE or if the page state wasn't | |
| // immediately consistent with the page being replaceable (without any waiting). | |
| // | |
| // next page | |
| break; | |
| } | |
| auto& vmo_backlink = maybe_vmo_backlink.value(); | |
| // Else GetCowWithReplaceablePage wouldn't have set the optional backlink. | |
| DEBUG_ASSERT(vmo_backlink.cow_container); | |
| auto& cow_container = vmo_backlink.cow_container; | |
| // TODO(fxbug.dev/99890): Implement way to replace loaned with non-loaned that supports | |
| // delayed allocations. | |
| ASSERT(ppb_enabled); | |
| // vmo_backlink.offset is offset in cow | |
| zx_status_t replace_result = cow_container->ReplacePageWithLoaned(page, vmo_backlink.offset); | |
| if (replace_result == ZX_ERR_NOT_FOUND) { | |
| // No longer owned by cow or no longer replaceable. Go around again to figure out which and | |
| // continue chasing it down. We limit the iteration count however, since it's not critical | |
| // that we catch up with the page here, and we don't want to get stuck on a page that's | |
| // moving super often, or pinning/unpinning super often. Counters track times where we | |
| // tried more than once, and times when we tried max times and still didn't replace the | |
| // page. | |
| // | |
| // this page again | |
| continue; | |
| } | |
| if (replace_result == ZX_ERR_NO_MEMORY) { | |
| // Out of pages of the appropriate type, so don't try the next page. | |
| return replaced_non_loaned_page_count; | |
| } | |
| if (replace_result != ZX_OK) { | |
| // Not replaceable after all. | |
| // | |
| // next page | |
| break; | |
| } | |
| // The page has been replaced with a different page that doesn't have loan_cancelled set. | |
| replaced = true; | |
| // next page | |
| break; | |
| } // page chase loop | |
| if (page_try_ordinal == kMaxPageChaseIterations) { | |
| sweep_page_chase_gave_up.Add(1); | |
| } | |
| if (replaced && ppb_enabled) { | |
| ++replaced_non_loaned_page_count; | |
| sweep_pages_swept_to_loaned.Add(1); | |
| } | |
| } | |
| if (iter == next_start_paddr_) { | |
| sweep_looped.Add(1); | |
| } | |
| return replaced_non_loaned_page_count; | |
| } |