blob: 5e8f791e0f06b53da9bd6edbe81cfd755a9a9ece [file] [log] [blame]
// Copyright 2016 The Fuchsia Authors
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
#include <inttypes.h>
#include <lib/boot-options/boot-options.h>
#include <lib/console.h>
#include <lib/counters.h>
#include <lib/crashlog.h>
#include <lib/elfldltl/machine.h>
#include <lib/instrumentation/vmo.h>
#include <lib/userabi/rodso.h>
#include <lib/userabi/userboot.h>
#include <lib/userabi/userboot_internal.h>
#include <lib/userabi/vdso.h>
#include <lib/zircon-internal/default_stack_size.h>
#include <platform.h>
#include <stdio.h>
#include <trace.h>
#include <zircon/errors.h>
#include <zircon/types.h>
#include <cassert>
#include <cstdlib>
#include <cstring>
#include <lk/init.h>
#include <object/channel_dispatcher.h>
#include <object/handle.h>
#include <object/job_dispatcher.h>
#include <object/message_packet.h>
#include <object/process_dispatcher.h>
#include <object/thread_dispatcher.h>
#include <object/vm_address_region_dispatcher.h>
#include <object/vm_object_dispatcher.h>
#include <phys/handoff.h>
#include <platform/crashlog.h>
#include <vm/vm_object_paged.h>
#include <lib/crypto/entropy/quality_test.h>
namespace {
class VmoBuffer {
explicit VmoBuffer(fbl::RefPtr<VmObjectPaged> vmo) : size_(vmo->size()), vmo_(ktl::move(vmo)) {}
explicit VmoBuffer(size_t size = PAGE_SIZE, uint32_t options = VmObjectPaged::kResizable) {
zx_status_t status = VmObjectPaged::Create(PMM_ALLOC_FLAG_ANY, options, size,
AttributionObject::GetKernelAttribution(), &vmo_);
ZX_ASSERT(status == ZX_OK);
size_ = vmo_->size();
int Write(const ktl::string_view str) {
// Enlarge the VMO as needed if it was created as resizable.
if (str.size() > size_ - offset_ && vmo_->is_resizable()) {
DEBUG_ASSERT(vmo_->size() < offset_ + str.size());
zx_status_t status = vmo_->Resize(offset_ + str.size());
if (status == ZX_OK) {
// Update unlocked cache of the size.
size_ = vmo_->size();
} else if (offset_ == size_) {
// None left to write without the resize.
// Otherwise proceed to write what can be written.
return static_cast<int>(status);
const size_t todo = ktl::min(str.size(), size_ - offset_);
if (zx_status_t res = vmo_->Write(, offset_, todo); res != ZX_OK) {
DEBUG_ASSERT(static_cast<int>(res) < 0);
return res;
offset_ += todo;
DEBUG_ASSERT(todo <= ktl::numeric_limits<int>::max());
return static_cast<int>(todo);
const fbl::RefPtr<VmObjectPaged>& vmo() const { return vmo_; }
size_t content_size() const { return offset_; }
size_t offset_{0};
size_t size_;
fbl::RefPtr<VmObjectPaged> vmo_;
constexpr const char kStackVmoName[] = "userboot-initial-stack";
constexpr const char kCrashlogVmoName[] = "crashlog";
constexpr const char kBootOptionsVmoname[] = "boot-options.txt";
constexpr const char kZbiVmoName[] = "zbi";
constexpr size_t stack_size = ZIRCON_DEFAULT_STACK_SIZE;
#include "userboot-code.h"
// This is defined in assembly via RODSO_IMAGE (see rodso-asm.h);
// userboot-code.h gives details about the image's size and layout.
extern "C" const char userboot_image[];
KCOUNTER(timeline_userboot, "boot.timeline.userboot")
KCOUNTER(init_time, "init.userboot.time.msec")
class UserbootImage : private RoDso {
UserbootImage(const VDso* vdso, KernelHandle<VmObjectDispatcher>* vmo_kernel_handle)
: RoDso("userboot", userboot_image, USERBOOT_CODE_END, USERBOOT_CODE_START,
vdso_(vdso) {}
// The whole userboot image consists of the userboot rodso image
// immediately followed by the vDSO image. This returns the size
// of that combined image.
size_t size() const { return RoDso::size() + vdso_->size(); }
zx_status_t Map(fbl::RefPtr<VmAddressRegionDispatcher> root_vmar, uintptr_t* vdso_base,
uintptr_t* entry) {
// Create a VMAR (placed anywhere) to hold the combined image.
KernelHandle<VmAddressRegionDispatcher> vmar_handle;
zx_rights_t vmar_rights;
zx_status_t status = root_vmar->Allocate(
0, size(),
&vmar_handle, &vmar_rights);
if (status != ZX_OK)
return status;
// Map userboot proper.
status = RoDso::Map(vmar_handle.dispatcher(), 0);
if (status == ZX_OK) {
*entry = vmar_handle.dispatcher()->vmar()->base() + USERBOOT_ENTRY;
// Map the vDSO right after it.
*vdso_base = vmar_handle.dispatcher()->vmar()->base() + RoDso::size();
// Releasing |vmar_handle| is safe because it has a no-op
// on_zero_handles(), otherwise the mapping routines would have
// to take ownership of the handle and manage its lifecycle.
status = vdso_->Map(vmar_handle.release(), RoDso::size());
return status;
const VDso* vdso_;
// Keep a global reference to the kcounters vmo so that the kcounters
// memory always remains valid, even if userspace closes the last handle.
fbl::RefPtr<VmObject> kcounters_vmo_ref;
// Get a handle to a VM object, with full rights except perhaps for writing.
zx_status_t get_vmo_handle(fbl::RefPtr<VmObject> vmo, bool readonly, uint64_t content_size,
fbl::RefPtr<VmObjectDispatcher>* disp_ptr, Handle** ptr) {
if (!vmo)
fbl::RefPtr<ContentSizeManager> content_size_manager;
zx_status_t result = ContentSizeManager::Create(content_size, &content_size_manager);
if (result != ZX_OK) {
return result;
zx_rights_t rights;
KernelHandle<VmObjectDispatcher> vmo_kernel_handle;
result = VmObjectDispatcher::Create(ktl::move(vmo), ktl::move(content_size_manager),
&vmo_kernel_handle, &rights);
if (result == ZX_OK) {
if (disp_ptr)
*disp_ptr = vmo_kernel_handle.dispatcher();
if (readonly)
rights &= ~ZX_RIGHT_WRITE;
if (ptr)
*ptr = Handle::Make(ktl::move(vmo_kernel_handle), rights).release();
return result;
HandleOwner get_job_handle() {
return Handle::Dup(GetRootJobHandle(), JobDispatcher::default_rights());
// Converts platform crashlog into a VMO
zx_status_t crashlog_to_vmo(fbl::RefPtr<VmObject>* out, size_t* out_size) {
PlatformCrashlog::Interface& crashlog = PlatformCrashlog::Get();
size_t size = crashlog.Recover(nullptr);
fbl::RefPtr<VmObjectPaged> crashlog_vmo;
zx_status_t status = VmObjectPaged::Create(
PMM_ALLOC_FLAG_ANY, 0u, size, AttributionObject::GetKernelAttribution(), &crashlog_vmo);
if (status != ZX_OK) {
return status;
if (size) {
VmoBuffer vmo_buffer{crashlog_vmo};
FILE vmo_file = FILE{&vmo_buffer};
crashlog_vmo->set_name(kCrashlogVmoName, sizeof(kCrashlogVmoName) - 1);
// Stash the recovered crashlog so that it may be propagated to the next
// kernel instance in case we later mexec.
*out = ktl::move(crashlog_vmo);
*out_size = size;
// Now that we have recovered the old crashlog, enable crashlog uptime
// updates. This will cause systems with a RAM based crashlog to periodically
// create a payload-less crashlog indicating a SW reboot reason of "unknown"
// along with an uptime indicator. If the system spontaneously reboots (due
// to something like a WDT, or brownout) we will be able to recover this log
// and know that we spontaneously rebooted, and have some idea of how long we
// were running before we did.
return ZX_OK;
void bootstrap_vmos(Handle** handles) {
ktl::span<ktl::byte> zbi = ZbiInPhysmap(true);
void* rbase =;
size_t rsize = ROUNDUP_PAGE_SIZE(zbi.size_bytes());
dprintf(INFO, "userboot: ramdisk %#15zx @ %p\n", rsize, rbase);
// The instrumentation VMOs need to be created prior to the rootfs as the information for these
// vmos is in the phys handoff region, which becomes inaccessible once the rootfs is created.
zx_status_t status = InstrumentationData::GetVmos(&handles[userboot::kFirstInstrumentationData]);
ASSERT(status == ZX_OK);
// The ZBI.
fbl::RefPtr<VmObjectPaged> rootfs_vmo;
status = VmObjectPaged::CreateFromWiredPages(
rbase, rsize, true, AttributionObject::GetKernelAttribution(), &rootfs_vmo);
ASSERT(status == ZX_OK);
rootfs_vmo->set_name(kZbiVmoName, sizeof(kZbiVmoName) - 1);
status = get_vmo_handle(rootfs_vmo, false, rsize, nullptr, &handles[userboot::kZbi]);
ASSERT(status == ZX_OK);
// The rootfs vmo was created with exclusive=true, which means that the VMO is sole owner of those
// pages. gPhysHandoff represents a pointer to the previous physmap location of the data, but the
// VMO is free to move and use different pages to represent the data, as such attempting to use
// this old reference is essentially a use-after-free.
gPhysHandoff = nullptr;
// Crashlog.
fbl::RefPtr<VmObject> crashlog_vmo;
size_t crashlog_size = 0;
status = crashlog_to_vmo(&crashlog_vmo, &crashlog_size);
ASSERT(status == ZX_OK);
status =
get_vmo_handle(crashlog_vmo, true, crashlog_size, nullptr, &handles[userboot::kCrashlog]);
ASSERT(status == ZX_OK);
// Boot options.
VmoBuffer boot_options;
FILE boot_options_file{&boot_options};
gBootOptions->Show(/*defaults=*/false, &boot_options_file);
boot_options.vmo()->set_name(kBootOptionsVmoname, sizeof(kBootOptionsVmoname) - 1);
status = get_vmo_handle(boot_options.vmo(), false, boot_options.content_size(), nullptr,
ZX_ASSERT(status == ZX_OK);
status =
get_vmo_handle(crypto::entropy::entropy_vmo, true, crypto::entropy::entropy_vmo_content_size,
nullptr, &handles[kEntropyTestData]);
ASSERT(status == ZX_OK);
// kcounters names table.
fbl::RefPtr<VmObjectPaged> kcountdesc_vmo;
status = VmObjectPaged::CreateFromWiredPages(CounterDesc().VmoData(), CounterDesc().VmoDataSize(),
true, AttributionObject::GetKernelAttribution(),
ASSERT(status == ZX_OK);
sizeof(counters::DescriptorVmo::kVmoName) - 1);
status = get_vmo_handle(ktl::move(kcountdesc_vmo), true, CounterDesc().VmoContentSize(), nullptr,
ASSERT(status == ZX_OK);
// kcounters live data.
fbl::RefPtr<VmObjectPaged> kcounters_vmo;
status = VmObjectPaged::CreateFromWiredPages(
CounterArena().VmoData(), CounterArena().VmoDataSize(), false,
AttributionObject::GetKernelAttribution(), &kcounters_vmo);
ASSERT(status == ZX_OK);
kcounters_vmo_ref = kcounters_vmo;
kcounters_vmo->set_name(counters::kArenaVmoName, sizeof(counters::kArenaVmoName) - 1);
status = get_vmo_handle(ktl::move(kcounters_vmo), true, CounterArena().VmoContentSize(), nullptr,
ASSERT(status == ZX_OK);
void userboot_init(uint) {
// Prepare the bootstrap message packet. This allocates space for its
// handles, which we'll fill in as we create things.
MessagePacketPtr msg;
zx_status_t status = MessagePacket::Create(nullptr, 0, userboot::kHandleCount, &msg);
ASSERT(status == ZX_OK);
Handle** const handles = msg->mutable_handles();
DEBUG_ASSERT(msg->num_handles() == userboot::kHandleCount);
// Create the process.
KernelHandle<ProcessDispatcher> process_handle;
KernelHandle<VmAddressRegionDispatcher> vmar_handle;
zx_rights_t process_rights, vmar_rights;
status = ProcessDispatcher::Create(GetRootJobDispatcher(), "userboot", 0, &process_handle,
&process_rights, &vmar_handle, &vmar_rights);
ASSERT(status == ZX_OK);
// It needs its own process and root VMAR handles.
auto process = process_handle.dispatcher();
auto vmar = vmar_handle.dispatcher();
HandleOwner proc_handle_owner = Handle::Make(ktl::move(process_handle), process_rights);
HandleOwner vmar_handle_owner = Handle::Make(ktl::move(vmar_handle), vmar_rights);
handles[userboot::kProcSelf] = proc_handle_owner.release();
handles[userboot::kVmarRootSelf] = vmar_handle_owner.release();
// It gets the root resource and job handles.
handles[userboot::kRootResource] = get_resource_handle(ZX_RSRC_KIND_ROOT).release();
handles[userboot::kMmioResource] = get_resource_handle(ZX_RSRC_KIND_MMIO).release();
handles[userboot::kIrqResource] = get_resource_handle(ZX_RSRC_KIND_IRQ).release();
#if ARCH_X86
handles[userboot::kIoportResource] = get_resource_handle(ZX_RSRC_KIND_IOPORT).release();
#elif ARCH_ARM64
handles[userboot::kSmcResource] = get_resource_handle(ZX_RSRC_KIND_SMC).release();
handles[userboot::kSystemResource] = get_resource_handle(ZX_RSRC_KIND_SYSTEM).release();
handles[userboot::kRootJob] = get_job_handle().release();
// It also gets many VMOs for VDSOs and other things.
constexpr int kVariants = static_cast<int>(userboot::VdsoVariant::COUNT);
KernelHandle<VmObjectDispatcher> vdso_kernel_handles[kVariants];
const VDso* vdso = VDso::Create(vdso_kernel_handles);
for (int i = 0; i < kVariants; ++i) {
handles[userboot::kFirstVdso + i] =
Handle::Make(ktl::move(vdso_kernel_handles[i]), vdso->vmo_rights()).release();
ASSERT(handles[userboot::kFirstVdso + i]);
DEBUG_ASSERT(handles[userboot::kFirstVdso + 1]->dispatcher() == vdso->vmo());
if (gBootOptions->always_use_next_vdso) {
std::swap(handles[userboot::kFirstVdso], handles[userboot::kFirstVdso + 1]);
// Make the channel that will hold the message.
KernelHandle<ChannelDispatcher> user_handle, kernel_handle;
zx_rights_t channel_rights;
status = ChannelDispatcher::Create(&user_handle, &kernel_handle, &channel_rights);
ASSERT(status == ZX_OK);
// Transfer it in.
status = kernel_handle.dispatcher()->Write(ZX_KOID_INVALID, ktl::move(msg));
ASSERT(status == ZX_OK);
// Inject the user-side channel handle into the process.
HandleOwner user_handle_owner = Handle::Make(ktl::move(user_handle), channel_rights);
zx_handle_t hv = process->handle_table().MapHandleToValue(user_handle_owner);
// Map in the userboot image along with the vDSO.
KernelHandle<VmObjectDispatcher> userboot_vmo_kernel_handle;
UserbootImage userboot(vdso, &userboot_vmo_kernel_handle);
uintptr_t vdso_base = 0;
uintptr_t entry = 0;
status = userboot.Map(vmar, &vdso_base, &entry);
ASSERT(status == ZX_OK);
// Map the stack anywhere.
uintptr_t stack_base;
fbl::RefPtr<VmObjectPaged> stack_vmo;
status = VmObjectPaged::Create(PMM_ALLOC_FLAG_ANY, 0u, stack_size,
process->attribution_object(), &stack_vmo);
ASSERT(status == ZX_OK);
stack_vmo->set_name(kStackVmoName, sizeof(kStackVmoName) - 1);
zx::result<VmAddressRegion::MapResult> stack_mapping_result =
vmar->Map(0, ktl::move(stack_vmo), 0, stack_size, ZX_VM_PERM_READ | ZX_VM_PERM_WRITE);
stack_base = stack_mapping_result->base;
uintptr_t sp = elfldltl::AbiTraits<>::InitialStackPointer(stack_base, stack_size);
// Create the user thread.
fbl::RefPtr<ThreadDispatcher> thread;
KernelHandle<ThreadDispatcher> thread_handle;
zx_rights_t thread_rights;
status =
ThreadDispatcher::Create(ktl::move(process), 0, "userboot", &thread_handle, &thread_rights);
ASSERT(status == ZX_OK);
status = thread_handle.dispatcher()->Initialize();
ASSERT(status == ZX_OK);
thread = thread_handle.dispatcher();
// Create a root job observer, restarting the system if the root job becomes childless.
dprintf(SPEW, "userboot: %-23s @ %#" PRIxPTR "\n", "entry point", entry);
// Start the process's initial thread.
auto arg1 = static_cast<uintptr_t>(hv);
status = thread->Start(ThreadDispatcher::EntryState{entry, sp, arg1, vdso_base},
/* ensure_initial_thread= */ true);
ASSERT(status == ZX_OK);
init_time.Add(current_time() / 1000000LL);
} // namespace
LK_INIT_HOOK(userboot, userboot_init, LK_INIT_LEVEL_USER + 1)