| // Copyright 2021 The Fuchsia Authors |
| // |
| // Use of this source code is governed by a MIT-style |
| // license that can be found in the LICENSE file or at |
| // https://opensource.org/licenses/MIT |
| |
| #ifndef ZIRCON_KERNEL_VM_INCLUDE_VM_EVICTOR_H_ |
| #define ZIRCON_KERNEL_VM_INCLUDE_VM_EVICTOR_H_ |
| |
| #include <lib/zircon-internal/thread_annotations.h> |
| #include <sys/types.h> |
| #include <zircon/time.h> |
| |
| #include <kernel/event.h> |
| #include <kernel/spinlock.h> |
| |
| class PmmNode; |
| class PageQueues; |
| |
| namespace vm_unittest { |
| class TestPmmNode; |
| } |
| |
| // Implements page evictor logic to free pages belonging to a PmmNode under memory pressure. |
| // Provides APIs for |
| // 1) one-shot eviction, which involves arming an eviction target and triggering eviction |
| // and |
| // 2) continuous eviction, which creates a dedicated thread to perform periodic evictions to |
| // maintain a free memory level. |
| // Eviction in this context is both direct eviction of pager backed memory and discardable VMO |
| // memory, as well as by performing compression. |
| // This class is thread-safe. |
| class Evictor { |
| public: |
| enum class EvictionLevel : uint8_t { |
| OnlyOldest = 0, |
| IncludeNewest = 1, |
| }; |
| |
| enum class Output : bool { |
| Print = true, |
| NoPrint = false, |
| }; |
| |
| enum class TriggerReason : bool { |
| OOM = true, |
| Other = false, |
| }; |
| |
| // Eviction target state is grouped together behind a lock to allow different threads to safely |
| // trigger and perform the eviction. |
| struct EvictionTarget { |
| bool pending = false; |
| // The desired value to get |pmm_node_|'s free page count to |
| uint64_t free_pages_target = 0; |
| // A minimum amount of pages we want to evict, regardless of how much free memory is available. |
| uint64_t min_pages_to_free = 0; |
| EvictionLevel level = EvictionLevel::OnlyOldest; |
| bool print_counts = false; |
| bool oom_trigger = false; |
| }; |
| |
| // We count non-loaned and loaned evicted pages separately since the eviction goal is set in terms |
| // of non-loaned pages (for now) so in order to verify expected behavior in tests we keep separate |
| // counts. |
| struct EvictedPageCounts { |
| // The pager_backed and pager_backed_loaned counts are exclusive; any given evicted page is |
| // counted as pager_backed or pager_backed_loaned depending on whether it's non-loaned or loaned |
| // respectively. |
| // |
| // evicted from pager-backed VMO non-loaned page count |
| uint64_t pager_backed = 0; |
| // evicted from pager-backed VMO loaned page count |
| uint64_t pager_backed_loaned = 0; |
| // evicted from/via discardable VMO page count |
| uint64_t discardable = 0; |
| // evicted from an anonymous VMO via compression |
| uint64_t compressed = 0; |
| }; |
| |
| explicit Evictor(PmmNode *node); |
| Evictor() = delete; |
| ~Evictor(); |
| |
| // Called from the scanner with kernel cmdline values. |
| void SetDiscardableEvictionsPercent(uint32_t discardable_percent); |
| void SetContinuousEvictionInterval(zx_time_t eviction_interval); |
| // Called from the scanner to enable eviction if required. Creates an eviction thread to process |
| // asynchronous eviction requests (both one-shot and continuous). |
| // By default this only enables user pager based eviction and |use_compression| can be used to |
| // also perform compression. |
| void EnableEviction(bool use_compression); |
| // Called from the scanner to disable all eviction if needed, will shut down any in existing |
| // eviction thread. It is a responsibility of the scanner to not have multiple concurrent calls |
| // to this and EnableEviction. |
| void DisableEviction(); |
| |
| // Set |one_shot_eviction_target_| to the specified |target|. The previous values are overridden. |
| void SetOneShotEvictionTarget(EvictionTarget target); |
| |
| // Combine the specified |target| with the pre-existing |one_shot_eviction_target_|. |
| void CombineOneShotEvictionTarget(EvictionTarget target); |
| |
| // Perform a one-shot eviction based on the current values of |one_shot_eviction_target_|. The |
| // expectation is that the user will have set the target before calling this function with either |
| // SetOneShotEvictionTarget() or CombineOneShotEvictionTarget(). This may acquire arbitrary vmo |
| // and aspace locks. |
| EvictedPageCounts EvictOneShotFromPreloadedTarget(); |
| |
| // Performs a synchronous request to evict |min_mem_to_free| (in bytes). The return value is the |
| // number of pages evicted. The |eviction_level| is a rough control that maps to how old a page |
| // needs to be for being considered for eviction. This may acquire arbitrary vmo and aspace locks. |
| uint64_t EvictOneShotSynchronous(uint64_t min_mem_to_free, |
| EvictionLevel eviction_level = EvictionLevel::OnlyOldest, |
| Output output = Output::NoPrint, |
| TriggerReason reason = TriggerReason::Other); |
| |
| // Reclaim memory until free memory equals the |free_mem_target| (in bytes) and at least |
| // |min_mem_to_free| (in bytes) has been reclaimed. Reclamation will happen asynchronously on the |
| // eviction thread and this function returns immediately. Once the target is reached, or there is |
| // no more memory that can be reclaimed, this process will stop and the free memory target will be |
| // cleared. The |eviction_level| is a rough control on how hard to try and evict. Multiple calls |
| // to EvictOneShotAsynchronous will cause all the targets to get merged by adding together |
| // |min_mem_to_free|, taking the max of |free_mem_target| and the highest or most aggressive of |
| // any |eviction_level|. |
| void EvictOneShotAsynchronous(uint64_t min_mem_to_free, uint64_t free_mem_target, |
| EvictionLevel eviction_level = EvictionLevel::OnlyOldest, |
| Output output = Output::NoPrint); |
| |
| // Enable continuous eviction on the eviction thread. Pages are evicted until the free memory |
| // level is restored to |free_mem_target| (in bytes) and at least |min_mem_to_free| (in bytes) has |
| // been evicted. The eviction thread will re-evaluate these two conditions at a fixed cadence of |
| // |default_eviction_interval_| (controlled by the kernel cmdline option |
| // `kernel.page-scanner.eviction-interval-seconds`), and continue to evict pages if required, |
| // until eviction is explicitly disabled with DisableContinuousEviction(). The |eviction_level| is |
| // a rough control that maps to how old a page needs to be for being considered for eviction. The |
| // |output| controls whether the eviction thread prints its progress each time it frees pages. |
| void EnableContinuousEviction(uint64_t min_mem_to_free, uint64_t free_mem_target, |
| EvictionLevel eviction_level = EvictionLevel::OnlyOldest, |
| Output output = Output::NoPrint); |
| |
| // Disable continuous eviction on the eviction thread. Use EnableContinuousEviction() to re-enable |
| // eviction when required. |
| void DisableContinuousEviction(); |
| |
| // Whether any eviction (one-shot and continuous) can occur. |
| bool IsEvictionEnabled() const; |
| |
| // Whether eviction should attempt to use compression. |
| bool IsCompressionEnabled() const; |
| |
| struct EvictorStats { |
| uint64_t pager_backed_oom = 0; |
| uint64_t pager_backed_other = 0; |
| uint64_t compression_oom = 0; |
| uint64_t compression_other = 0; |
| uint64_t discarded_oom = 0; |
| uint64_t discarded_other = 0; |
| }; |
| // Return global eviction stats from all instantiations of the Evictor. |
| static EvictorStats GetGlobalStats(); |
| |
| private: |
| // Private constructor for test code to specify |queues| not owned by |node|. |
| Evictor(PmmNode *node, PageQueues *queues); |
| |
| // Helpers for testing. |
| EvictionTarget DebugGetOneShotEvictionTarget() const; |
| void DebugSetMinDiscardableAge(zx_time_t age); |
| |
| friend class vm_unittest::TestPmmNode; |
| |
| // Evict until |min_pages_to_evict| have been evicted and there are at least |free_pages_target| |
| // free pages on the system. Note that the eviction operation here is one-shot, i.e. as soon as |
| // the targets are met, eviction will stop and the function will return. Returns the number of |
| // discardable and pager-backed pages evicted. This may acquire arbitrary vmo and aspace locks. |
| EvictedPageCounts EvictUntilTargetsMet(uint64_t min_pages_to_evict, uint64_t free_pages_target, |
| EvictionLevel level) TA_EXCL(lock_); |
| |
| // Evict the requested number of |target_pages| from discardable vmos. The return value is the |
| // number of pages evicted. This may acquire arbitrary vmo and aspace locks. |
| uint64_t EvictDiscardable(uint64_t target_pages) const TA_EXCL(lock_); |
| |
| // Evict the requested number of |target_pages| from vmos by querying the page queues. The |
| // returned struct has the number of pages evicted (discardable will be 0). The |eviction_level| |
| // is a rough control that maps to how old a page needs to be for being considered for eviction. |
| // This may acquire arbitrary vmo and aspace locks. |
| EvictedPageCounts EvictPageQueues(uint64_t target_pages, EvictionLevel eviction_level) const |
| TA_EXCL(lock_); |
| |
| // The main loop for the eviction thread. |
| int EvictionThreadLoop() TA_EXCL(lock_); |
| |
| // Control parameters for continuous eviction. |
| EvictionTarget continuous_eviction_target_ TA_GUARDED(lock_) = {}; |
| zx_time_t next_eviction_interval_ TA_GUARDED(lock_) = ZX_TIME_INFINITE; |
| |
| // Targets for one-shot eviction, kept separate from the continuous eviction control parameters |
| // above. |
| EvictionTarget one_shot_eviction_target_ TA_GUARDED(lock_) = {}; |
| |
| // Event that enforces only one eviction attempt to be active at any time. This prevents us from |
| // overshooting the free memory targets required by various simultaneous eviction requests. |
| AutounsignalEvent no_ongoing_eviction_{true}; |
| |
| // Use MonitoredSpinLock to provide lockup detector diagnostics for the critical sections |
| // protected by this lock. |
| mutable DECLARE_SPINLOCK_WITH_TYPE(Evictor, MonitoredSpinLock) lock_; |
| |
| // The eviction thread used to process asynchronous requests (both one-shot and continuous). |
| // Created only if eviction is enabled i.e. |eviction_enabled_| is set to true. |
| Thread *eviction_thread_ = nullptr; |
| ktl::atomic<bool> eviction_thread_exiting_ = false; |
| |
| // Used by the eviction thread to wait for eviction requests. |
| AutounsignalEvent eviction_signal_; |
| |
| // The PmmNode whose free level the Evictor monitors, and frees pages to. |
| PmmNode *const pmm_node_; |
| |
| // The set of PageQueues that the Evictor evicts pages from. |
| // |
| // This is technically not needed and is mostly for the benefit of unit tests. The Evictor can |
| // just call pmm_node_->GetPageQueues() to get the right set of page queues to work on. However, |
| // the VMO side code is currently PmmNode agnostic, and until there exists a way for VMOs to |
| // allocate from (and free to) a particular PmmNode, we'll need to track the PageQueues separately |
| // in order to write meaningful tests. |
| // |
| // This is set to pmm_node_->GetPageQueues() by the public constructor that passes in the PmmNode |
| // associated with this evictor. The private constructor which also passes in PageQueues (not |
| // necessarily owned by the PmmNode) is only used in test code. |
| PageQueues *const page_queues_; |
| |
| // These parameters are initialized later from kernel cmdline options. |
| // Whether eviction is enabled. |
| bool eviction_enabled_ TA_GUARDED(lock_) = false; |
| // Whether eviction should attempt compression. |
| bool use_compression_ TA_GUARDED(lock_) = false; |
| // A rough percentage of page evictions that should be satisfied from discardable vmos (as opposed |
| // to pager-backed vmos). Will require tuning when discardable vmos start being used. Currently |
| // sets the number of discardable pages to evict to 0, putting all the burden of eviction on |
| // pager-backed pages. |
| uint32_t discardable_evictions_percent_ TA_GUARDED(lock_) = 0; |
| // The minimum interval a discardable VMO has to be unlocked for to be considered for eviction. |
| zx_time_t min_discardable_age_ TA_GUARDED(lock_) = ZX_SEC(10); |
| // Default continuous eviction interval. Set to 10s to match the scanner aging interval, since we |
| // won't find any new pages to evict before the next aging round. |
| zx_time_t default_eviction_interval_ TA_GUARDED(lock_) = ZX_SEC(10); |
| }; |
| |
| #endif // ZIRCON_KERNEL_VM_INCLUDE_VM_EVICTOR_H_ |