blob: f16ea6df15307f1eeeae2543dcd91c17262ad011 [file] [log] [blame]
// Copyright 2016 The Fuchsia Authors
// Copyright (c) 2014 Travis Geiselbrecht
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT
#include "vm/pmm.h"
#include <assert.h>
#include <err.h>
#include <inttypes.h>
#include <lib/cmdline.h>
#include <lib/console.h>
#include <platform.h>
#include <pow2.h>
#include <stdlib.h>
#include <string.h>
#include <trace.h>
#include <zircon/time.h>
#include <zircon/types.h>
#include <new>
#include <fbl/auto_lock.h>
#include <fbl/intrusive_double_list.h>
#include <kernel/mp.h>
#include <kernel/mutex.h>
#include <kernel/timer.h>
#include <lk/init.h>
#include <vm/bootalloc.h>
#include <vm/physmap.h>
#include <vm/pmm_checker.h>
#include <vm/scanner.h>
#include <vm/vm.h>
#include "pmm_arena.h"
#include "pmm_node.h"
#include "vm_priv.h"
#define LOCAL_TRACE VM_GLOBAL_TRACE(0)
// The (currently) one and only pmm node
static PmmNode pmm_node;
static void pmm_fill_free_pages(uint level) { pmm_node.FillFreePages(); }
LK_INIT_HOOK(pmm_fill, &pmm_fill_free_pages, LK_INIT_LEVEL_VM)
vm_page_t* paddr_to_vm_page(paddr_t addr) { return pmm_node.PaddrToPage(addr); }
zx_status_t pmm_add_arena(const pmm_arena_info_t* info) { return pmm_node.AddArena(info); }
size_t pmm_num_arenas() { return pmm_node.NumArenas(); }
zx_status_t pmm_get_arena_info(size_t count, uint64_t i, pmm_arena_info_t* buffer,
size_t buffer_size) {
return pmm_node.GetArenaInfo(count, i, buffer, buffer_size);
}
zx_status_t pmm_alloc_page(uint alloc_flags, paddr_t* pa) {
return pmm_node.AllocPage(alloc_flags, nullptr, pa);
}
zx_status_t pmm_alloc_page(uint alloc_flags, vm_page_t** page) {
return pmm_node.AllocPage(alloc_flags, page, nullptr);
}
zx_status_t pmm_alloc_page(uint alloc_flags, vm_page_t** page, paddr_t* pa) {
return pmm_node.AllocPage(alloc_flags, page, pa);
}
zx_status_t pmm_alloc_pages(size_t count, uint alloc_flags, list_node* list) {
return pmm_node.AllocPages(count, alloc_flags, list);
}
zx_status_t pmm_alloc_range(paddr_t address, size_t count, list_node* list) {
return pmm_node.AllocRange(address, count, list);
}
zx_status_t pmm_alloc_contiguous(size_t count, uint alloc_flags, uint8_t alignment_log2,
paddr_t* pa, list_node* list) {
// if we're called with a single page, just fall through to the regular allocation routine
if (unlikely(count == 1 && alignment_log2 <= PAGE_SIZE_SHIFT)) {
vm_page_t* page;
zx_status_t status = pmm_node.AllocPage(alloc_flags, &page, pa);
if (status != ZX_OK) {
return status;
}
list_add_tail(list, &page->queue_node);
return ZX_OK;
}
return pmm_node.AllocContiguous(count, alloc_flags, alignment_log2, pa, list);
}
void pmm_alloc_pages(uint alloc_flags, page_request_t* req) {
pmm_node.AllocPages(alloc_flags, req);
}
bool pmm_clear_request(page_request_t* req) { return pmm_node.ClearRequest(req); }
void pmm_swap_request(page_request_t* old, page_request_t* new_req) {
pmm_node.SwapRequest(old, new_req);
}
void pmm_free(list_node* list) { pmm_node.FreeList(list); }
void pmm_free_page(vm_page* page) { pmm_node.FreePage(page); }
uint64_t pmm_count_free_pages() { return pmm_node.CountFreePages(); }
uint64_t pmm_count_total_bytes() { return pmm_node.CountTotalBytes(); }
PageQueues* pmm_page_queues() { return pmm_node.GetPageQueues(); }
zx_status_t pmm_init_reclamation(const uint64_t* watermarks, uint8_t watermark_count,
uint64_t debounce, mem_avail_state_updated_callback_t callback) {
return pmm_node.InitReclamation(watermarks, watermark_count, debounce, callback);
}
void pmm_checker_init_from_cmdline() {
bool enabled = gCmdline.GetBool("kernel.pmm-checker.enable", false);
if (enabled) {
pmm_node.EnableChecker();
}
}
void pmm_checker_check_all_free_pages() { pmm_node.CheckAllFreePages(); }
static void pmm_checker_enable() {
// Enable filling of pages going forward.
pmm_node.EnableChecker();
// From this point on, pages will be filled when they are freed. However, the free list may still
// have a lot of unfilled pages so make a pass over them and fill them all. Node, |FillFreePages|
// will also |Arm| the checker.
pmm_node.FillFreePages();
}
static void pmm_checker_disable() { pmm_node.DisableChecker(); }
static bool pmm_checker_is_enabled() { return pmm_node.Checker()->IsArmed(); }
static void pmm_checker_print_status() {
printf("pmm checker %s\n", pmm_checker_is_enabled() ? "enabled" : "disabled");
}
static void pmm_dump_timer(Timer* t, zx_time_t now, void*) {
zx_time_t deadline = zx_time_add_duration(now, ZX_SEC(1));
t->SetOneshot(deadline, &pmm_dump_timer, nullptr);
pmm_node.DumpFree();
}
static void init_request_thread(unsigned int level) { pmm_node.InitRequestThread(); }
LK_INIT_HOOK(pmm, init_request_thread, LK_INIT_LEVEL_THREADING)
static int cmd_pmm(int argc, const cmd_args* argv, uint32_t flags) {
bool is_panic = flags & CMD_FLAG_PANIC;
if (argc < 2) {
printf("not enough arguments\n");
usage:
printf("usage:\n");
printf("%s dump : dump pmm info \n", argv[0].str);
if (!is_panic) {
printf("%s free : periodically dump free mem count\n",
argv[0].str);
printf(
"%s oom [<rate>] : leak memory until oom is triggered,\n"
" optionally specify the rate at which to leak "
"(in MB per second)\n",
argv[0].str);
printf("%s mem_avail_state info : dump memory availability state info\n",
argv[0].str);
printf(
"%s mem_avail_state <state> [<nsecs>] : allocate memory to go to memstate <state>, hold "
"the state for <nsecs> (default 10s) \n"
" only works if going to <state> from current "
"state requires allocating memory,\n"
" can't free up pre-allocated memory\n",
argv[0].str);
printf("%s drop_user_pt : drop all user hardware page tables\n",
argv[0].str);
printf("%s checker status : prints the status of the pmm checker\n",
argv[0].str);
printf("%s checker enable : enables the pmm checker\n", argv[0].str);
printf("%s checker disable : disables the pmm checker\n", argv[0].str);
printf("%s checker check : forces a check of all free pages in the pmm\n",
argv[0].str);
}
return ZX_ERR_INTERNAL;
}
if (!strcmp(argv[1].str, "dump")) {
pmm_node.Dump(is_panic);
} else if (is_panic) {
// No other operations will work during a panic.
printf("Only the \"arenas\" command is available during a panic.\n");
goto usage;
} else if (!strcmp(argv[1].str, "free")) {
static bool show_mem = false;
static Timer timer;
if (!show_mem) {
printf("pmm free: issue the same command to stop.\n");
zx_time_t deadline = zx_time_add_duration(current_time(), ZX_SEC(1));
const TimerSlack slack{ZX_MSEC(20), TIMER_SLACK_CENTER};
const Deadline slackDeadline(deadline, slack);
timer.Set(slackDeadline, &pmm_dump_timer, nullptr);
show_mem = true;
} else {
timer.Cancel();
show_mem = false;
}
} else if (!strcmp(argv[1].str, "oom")) {
uint64_t rate = 0;
if (argc > 2) {
rate = strtoul(argv[2].str, nullptr, 0) * 1024 * 1024 / PAGE_SIZE;
}
uint64_t pages_till_oom;
// In case we are racing with someone freeing pages we will leak in a loop until we are sure
// we have hit the oom state.
while ((pages_till_oom = pmm_node.DebugNumPagesTillMemState(0)) > 0) {
list_node list = LIST_INITIAL_VALUE(list);
if (rate > 0) {
uint64_t pages_leaked = 0;
while (pages_leaked < pages_till_oom) {
uint64_t alloc_pages = fbl::min(rate, pages_till_oom - pages_leaked);
if (pmm_node.AllocPages(alloc_pages, 0, &list) == ZX_OK) {
pages_leaked += alloc_pages;
printf("Leaked %lu pages\n", pages_leaked);
}
Thread::Current::SleepRelative(ZX_SEC(1));
}
} else {
if (pmm_node.AllocPages(pages_till_oom, 0, &list) == ZX_OK) {
printf("Leaked %lu pages\n", pages_till_oom);
}
}
// Ignore any errors under the assumption we had a racy allocation and try again next time
// around the loop.
}
} else if (!strcmp(argv[1].str, "mem_avail_state")) {
if (argc < 3) {
goto usage;
}
if (!strcmp(argv[2].str, "info")) {
pmm_node.DumpMemAvailState();
} else {
uint8_t state = static_cast<uint8_t>(argv[2].u);
if (state > pmm_node.DebugMaxMemAvailState()) {
printf("Invalid memstate %u. Specify a value between 0 and %u.\n", state,
pmm_node.DebugMaxMemAvailState());
goto usage;
}
uint64_t pages_to_alloc, pages_to_free = 0;
list_node list = LIST_INITIAL_VALUE(list);
// In case we are racing with someone freeing pages we will leak in a loop until we are sure
// we have hit the required memory availability state.
while ((pages_to_alloc = pmm_node.DebugNumPagesTillMemState(state)) > 0) {
if (pmm_node.AllocPages(pages_to_alloc, 0, &list) == ZX_OK) {
printf("Leaked %lu pages\n", pages_to_alloc);
pages_to_free += pages_to_alloc;
}
}
if (pages_to_free > 0) {
uint64_t nsecs = 10;
if (argc > 3) {
nsecs = argv[3].u;
}
printf("Sleeping for %lu seconds...\n", nsecs);
Thread::Current::SleepRelative(ZX_SEC(nsecs));
pmm_node.FreeList(&list);
printf("Freed %lu pages\n", pages_to_free);
}
}
} else if (!strcmp(argv[1].str, "drop_user_pt")) {
VmAspace::DropAllUserPageTables();
} else if (!strcmp(argv[1].str, "checker")) {
if (argc != 3) {
goto usage;
}
if (!strcmp(argv[2].str, "status")) {
pmm_checker_print_status();
} else if (!strcmp(argv[2].str, "enable")) {
pmm_checker_enable();
pmm_checker_print_status();
} else if (!strcmp(argv[2].str, "disable")) {
pmm_checker_disable();
pmm_checker_print_status();
} else if (!strcmp(argv[2].str, "check")) {
if (!pmm_checker_is_enabled()) {
printf("error: pmm checker is not enabled\n");
return ZX_ERR_INTERNAL;
}
printf("checking all free pages...\n");
pmm_checker_check_all_free_pages();
printf("done\n");
} else {
goto usage;
}
} else {
printf("unknown command\n");
goto usage;
}
return ZX_OK;
}
STATIC_COMMAND_START
STATIC_COMMAND_MASKED("pmm", "physical memory manager", &cmd_pmm, CMD_AVAIL_ALWAYS)
STATIC_COMMAND_END(pmm)