[kernel][object][vm] Add no-op pager vmos
This change adds the ability to create a no-op pager vmo. The no-op
implementation behaves like a normal vmo, except there are some extra
objects which will manage the pager state in the future.
Test: runtests
Change-Id: I39e0213e6c578206620640ea25eed96898d0f023
diff --git a/kernel/object/include/object/pager_dispatcher.h b/kernel/object/include/object/pager_dispatcher.h
index 2140e2d..3b9c11a 100644
--- a/kernel/object/include/object/pager_dispatcher.h
+++ b/kernel/object/include/object/pager_dispatcher.h
@@ -9,16 +9,51 @@
#include <fbl/canary.h>
#include <fbl/ref_ptr.h>
#include <object/dispatcher.h>
+#include <object/port_dispatcher.h>
#include <zircon/types.h>
+#include <vm/page_source.h>
+
+// Wrapper which maintains the object layer state of a PageSource.
+class PageSourceWrapper : public PageSourceCallback,
+ public fbl::DoublyLinkedListable<fbl::unique_ptr<PageSourceWrapper>> {
+public:
+ PageSourceWrapper(PagerDispatcher* dispatcher, fbl::RefPtr<PortDispatcher> port, uint64_t key);
+ virtual ~PageSourceWrapper();
+
+ void OnClose() override;
+
+private:
+ PagerDispatcher* const pager_;
+ const fbl::RefPtr<PortDispatcher> port_;
+ const uint64_t key_;
+
+ fbl::Mutex mtx_;
+ bool closed_ TA_GUARDED(mtx_) = false;
+
+ // The PageSource this is wrapping.
+ fbl::RefPtr<PageSource> src_ TA_GUARDED(mtx_);
+
+ friend PagerDispatcher;
+};
class PagerDispatcher final : public SoloDispatcher<PagerDispatcher, ZX_DEFAULT_PAGER_RIGHTS> {
public:
static zx_status_t Create(fbl::RefPtr<Dispatcher>* dispatcher, zx_rights_t* rights);
+ ~PagerDispatcher() final;
+
+ zx_status_t CreateSource(fbl::RefPtr<PortDispatcher> port,
+ uint64_t key, fbl::RefPtr<PageSource>* src);
+ void ReleaseSource(PageSourceWrapper* src);
zx_obj_type_t get_type() const final { return ZX_OBJ_TYPE_PAGER; }
+ void on_zero_handles() final;
+
private:
explicit PagerDispatcher();
fbl::Canary<fbl::magic("PGRD")> canary_;
+
+ fbl::Mutex mtx_;
+ fbl::DoublyLinkedList<fbl::unique_ptr<PageSourceWrapper>> srcs_;
};
diff --git a/kernel/object/pager_dispatcher.cpp b/kernel/object/pager_dispatcher.cpp
index c3b888f..660e8cd 100644
--- a/kernel/object/pager_dispatcher.cpp
+++ b/kernel/object/pager_dispatcher.cpp
@@ -5,6 +5,10 @@
// https://opensource.org/licenses/MIT
#include <object/pager_dispatcher.h>
+#include <trace.h>
+#include <vm/page_source.h>
+
+#define LOCAL_TRACE 0
zx_status_t PagerDispatcher::Create(fbl::RefPtr<Dispatcher>* dispatcher, zx_rights_t* rights) {
fbl::AllocChecker ac;
@@ -19,3 +23,74 @@
}
PagerDispatcher::PagerDispatcher() : SoloDispatcher() {}
+
+PagerDispatcher::~PagerDispatcher() {}
+
+zx_status_t PagerDispatcher::CreateSource(fbl::RefPtr<PortDispatcher> port,
+ uint64_t key, fbl::RefPtr<PageSource>* src_out) {
+ fbl::AllocChecker ac;
+ auto wrapper = fbl::make_unique_checked<PageSourceWrapper>(&ac, this, ktl::move(port), key);
+ if (!ac.check()) {
+ return ZX_ERR_NO_MEMORY;
+ }
+
+ auto src = fbl::AdoptRef(new (&ac) PageSource(wrapper.get(), get_koid()));
+ if (!ac.check()) {
+ return ZX_ERR_NO_MEMORY;
+ }
+
+ fbl::AutoLock lock(&wrapper->mtx_);
+ wrapper->src_ = src;
+
+ fbl::AutoLock lock2(&mtx_);
+ srcs_.push_front(ktl::move(wrapper));
+
+ *src_out = ktl::move(src);
+ return ZX_OK;
+}
+
+void PagerDispatcher::ReleaseSource(PageSourceWrapper* src) {
+ fbl::AutoLock lock(&mtx_);
+ srcs_.erase(*src);
+}
+
+void PagerDispatcher::on_zero_handles() {
+ fbl::DoublyLinkedList<fbl::unique_ptr<PageSourceWrapper>> srcs;
+
+ mtx_.Acquire();
+ while (!srcs_.is_empty()) {
+ auto& src = srcs_.front();
+ fbl::RefPtr<PageSource> inner;
+ {
+ fbl::AutoLock lock(&src.mtx_);
+ inner = src.src_;
+ }
+
+ // Call close outside of the lock, since it will call back into ::OnClose.
+ mtx_.Release();
+ if (inner) {
+ inner->Close();
+ }
+ mtx_.Acquire();
+ }
+ mtx_.Release();
+}
+
+PageSourceWrapper::PageSourceWrapper(PagerDispatcher* dispatcher,
+ fbl::RefPtr<PortDispatcher> port, uint64_t key)
+ : pager_(dispatcher), port_(ktl::move(port)), key_(key) {
+ LTRACEF("%p key %lx\n", this, key_);
+}
+
+PageSourceWrapper::~PageSourceWrapper() {
+ LTRACEF("%p\n", this);
+ DEBUG_ASSERT(closed_);
+}
+
+void PageSourceWrapper::OnClose() {
+ {
+ fbl::AutoLock lock(&mtx_);
+ closed_ = true;
+ }
+ pager_->ReleaseSource(this);
+}
diff --git a/kernel/syscalls/pager.cpp b/kernel/syscalls/pager.cpp
index c840369..a61c62d 100644
--- a/kernel/syscalls/pager.cpp
+++ b/kernel/syscalls/pager.cpp
@@ -9,6 +9,8 @@
#include <fbl/ref_ptr.h>
#include <ktl/move.h>
#include <object/pager_dispatcher.h>
+#include <object/vm_object_dispatcher.h>
+#include <vm/vm_object_paged.h>
// zx_status_t zx_pager_create
zx_status_t sys_pager_create(uint32_t options, user_out_handle* out) {
@@ -25,3 +27,45 @@
return out->make(ktl::move(dispatcher), rights);
}
+
+// zx_status_t zx_pager_create_vmo
+zx_status_t sys_pager_create_vmo(zx_handle_t pager, zx_handle_t port, uint64_t key,
+ uint64_t size, uint32_t options, user_out_handle* out) {
+ if (options) {
+ return ZX_ERR_INVALID_ARGS;
+ }
+
+ auto up = ProcessDispatcher::GetCurrent();
+ fbl::RefPtr<PagerDispatcher> pager_dispatcher;
+ zx_status_t status = up->GetDispatcher(pager, &pager_dispatcher);
+ if (status != ZX_OK) {
+ return status;
+ }
+
+ fbl::RefPtr<PortDispatcher> port_dispatcher;
+ status = up->GetDispatcherWithRights(port, ZX_RIGHT_WRITE, &port_dispatcher);
+ if (status != ZX_OK) {
+ return status;
+ }
+
+ fbl::RefPtr<PageSource> src;
+ status = pager_dispatcher->CreateSource(ktl::move(port_dispatcher), key, &src);
+ if (status != ZX_OK) {
+ return status;
+ }
+
+ fbl::RefPtr<VmObject> vmo;
+ status = VmObjectPaged::CreateExternal(ktl::move(src), size, &vmo);
+ if (status != ZX_OK) {
+ return status;
+ }
+
+ fbl::RefPtr<Dispatcher> dispatcher;
+ zx_rights_t rights;
+ status = VmObjectDispatcher::Create(vmo, &dispatcher, &rights);
+ if (status != ZX_OK) {
+ return status;
+ }
+
+ return out->make(ktl::move(dispatcher), rights);
+}
diff --git a/kernel/vm/include/vm/page_source.h b/kernel/vm/include/vm/page_source.h
new file mode 100644
index 0000000..dac6995
--- /dev/null
+++ b/kernel/vm/include/vm/page_source.h
@@ -0,0 +1,44 @@
+// Copyright 2018 The Fuchsia Authors
+//
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT
+
+#pragma once
+
+#include <fbl/mutex.h>
+#include <fbl/ref_counted.h>
+#include <fbl/ref_ptr.h>
+#include <kernel/mutex.h>
+#include <vm/vm.h>
+#include <zircon/types.h>
+
+// Callback to whatever is backing the PageSource.
+class PageSourceCallback {
+public:
+ // OnClose should be called once no more requests will be made to the page source. The
+ // callback can keep a reference to the page source, so it must be called outside of
+ // the PageSource destructor.
+ virtual void OnClose() = 0;
+};
+
+// Object which bridges a vm_object to some external data source.
+class PageSource : public fbl::RefCounted<PageSource> {
+public:
+ PageSource(PageSourceCallback* callback, uint64_t page_source_id);
+ ~PageSource();
+
+ // Closes the source. All pending transactions will be aborted and all future
+ // calls will fail.
+ void Close();
+
+ // Gets an id used for ownership verification.
+ uint64_t get_page_source_id() const { return page_source_id_; }
+
+private:
+ PageSourceCallback* const callback_;
+ const uint64_t page_source_id_;
+
+ fbl::Mutex mtx_;
+ bool closed_ TA_GUARDED(mtx_) = false;
+};
diff --git a/kernel/vm/include/vm/vm_object_paged.h b/kernel/vm/include/vm/vm_object_paged.h
index a06fa04..327bfe8 100644
--- a/kernel/vm/include/vm/vm_object_paged.h
+++ b/kernel/vm/include/vm/vm_object_paged.h
@@ -17,6 +17,7 @@
#include <lib/user_copy/user_ptr.h>
#include <list.h>
#include <stdint.h>
+#include <vm/page_source.h>
#include <vm/pmm.h>
#include <vm/vm.h>
#include <vm/vm_aspace.h>
@@ -44,6 +45,9 @@
static zx_status_t CreateFromROData(const void* data, size_t size, fbl::RefPtr<VmObject>* vmo);
+ static zx_status_t CreateExternal(fbl::RefPtr<PageSource> src,
+ uint64_t size, fbl::RefPtr<VmObject>* vmo);
+
zx_status_t Resize(uint64_t size) override;
zx_status_t ResizeLocked(uint64_t size) override TA_REQ(lock_);
uint32_t create_options() const override { return options_; }
@@ -101,7 +105,8 @@
private:
// private constructor (use Create())
VmObjectPaged(
- uint32_t options, uint32_t pmm_alloc_flags, uint64_t size, fbl::RefPtr<VmObject> parent);
+ uint32_t options, uint32_t pmm_alloc_flags, uint64_t size,
+ fbl::RefPtr<VmObject> parent, fbl::RefPtr<PageSource> page_source);
// private destructor, only called from refptr
~VmObjectPaged() override;
@@ -144,6 +149,9 @@
uint32_t pmm_alloc_flags_ TA_GUARDED(lock_) = PMM_ALLOC_FLAG_ANY;
uint32_t cache_policy_ TA_GUARDED(lock_) = ARCH_MMU_FLAG_CACHED;
+ // The page source, if any.
+ const fbl::RefPtr<PageSource> page_source_;
+
// a tree of pages
VmPageList page_list_ TA_GUARDED(lock_);
};
diff --git a/kernel/vm/page_source.cpp b/kernel/vm/page_source.cpp
new file mode 100644
index 0000000..982953a
--- /dev/null
+++ b/kernel/vm/page_source.cpp
@@ -0,0 +1,31 @@
+// Copyright 2018 The Fuchsia Authors
+//
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT
+
+#include <fbl/auto_lock.h>
+#include <trace.h>
+#include <vm/page_source.h>
+
+#define LOCAL_TRACE 0
+
+PageSource::PageSource(PageSourceCallback* callback, uint64_t page_source_id)
+ : callback_(callback), page_source_id_(page_source_id) {
+ LTRACEF("%p callback %p\n", this, callback_);
+}
+
+PageSource::~PageSource() {
+ LTRACEF("%p\n", this);
+ DEBUG_ASSERT(closed_);
+}
+
+void PageSource::Close() {
+ fbl::AutoLock info_lock(&mtx_);
+ LTRACEF("%p\n", this);
+
+ if (!closed_) {
+ closed_ = true;
+ callback_->OnClose();
+ }
+}
diff --git a/kernel/vm/rules.mk b/kernel/vm/rules.mk
index c84d074..8820f52 100644
--- a/kernel/vm/rules.mk
+++ b/kernel/vm/rules.mk
@@ -20,6 +20,7 @@
$(LOCAL_DIR)/bootreserve.cpp \
$(LOCAL_DIR)/kstack.cpp \
$(LOCAL_DIR)/page.cpp \
+ $(LOCAL_DIR)/page_source.cpp \
$(LOCAL_DIR)/pinned_vm_object.cpp \
$(LOCAL_DIR)/pmm.cpp \
$(LOCAL_DIR)/pmm_arena.cpp \
diff --git a/kernel/vm/vm_object_paged.cpp b/kernel/vm/vm_object_paged.cpp
index 22e3226..489c003 100644
--- a/kernel/vm/vm_object_paged.cpp
+++ b/kernel/vm/vm_object_paged.cpp
@@ -65,14 +65,17 @@
} // namespace
VmObjectPaged::VmObjectPaged(
- uint32_t options, uint32_t pmm_alloc_flags, uint64_t size, fbl::RefPtr<VmObject> parent)
+ uint32_t options, uint32_t pmm_alloc_flags, uint64_t size,
+ fbl::RefPtr<VmObject> parent, fbl::RefPtr<PageSource> page_source)
: VmObject(ktl::move(parent)),
options_(options),
size_(size),
- pmm_alloc_flags_(pmm_alloc_flags) {
+ pmm_alloc_flags_(pmm_alloc_flags),
+ page_source_(ktl::move(page_source)) {
LTRACEF("%p\n", this);
DEBUG_ASSERT(IS_PAGE_ALIGNED(size_));
+ DEBUG_ASSERT(page_source_ == nullptr || parent_ == nullptr);
}
VmObjectPaged::~VmObjectPaged() {
@@ -91,6 +94,10 @@
// free all of the pages attached to us
page_list_.FreeAllPages();
+
+ if (page_source_) {
+ page_source_->Close();
+ }
}
zx_status_t VmObjectPaged::Create(uint32_t pmm_alloc_flags,
@@ -109,7 +116,7 @@
fbl::AllocChecker ac;
auto vmo = fbl::AdoptRef<VmObject>(
- new (&ac) VmObjectPaged(options, pmm_alloc_flags, size, nullptr));
+ new (&ac) VmObjectPaged(options, pmm_alloc_flags, size, nullptr, nullptr));
if (!ac.check()) {
return ZX_ERR_NO_MEMORY;
}
@@ -130,7 +137,7 @@
fbl::AllocChecker ac;
auto vmo = fbl::AdoptRef<VmObject>(
- new (&ac) VmObjectPaged(kContiguous, pmm_alloc_flags, size, nullptr));
+ new (&ac) VmObjectPaged(kContiguous, pmm_alloc_flags, size, nullptr, nullptr));
if (!ac.check()) {
return ZX_ERR_NO_MEMORY;
}
@@ -236,6 +243,26 @@
return ZX_OK;
}
+zx_status_t VmObjectPaged::CreateExternal(fbl::RefPtr<PageSource> src,
+ uint64_t size, fbl::RefPtr<VmObject>* obj) {
+ // make sure size is page aligned
+ zx_status_t status = RoundSize(size, &size);
+ if (status != ZX_OK) {
+ return status;
+ }
+
+ fbl::AllocChecker ac;
+ auto vmo = fbl::AdoptRef<VmObject>(new (&ac) VmObjectPaged(
+ kResizable, PMM_ALLOC_FLAG_ANY, size, nullptr, ktl::move(src)));
+ if (!ac.check()) {
+ return ZX_ERR_NO_MEMORY;
+ }
+
+ *obj = ktl::move(vmo);
+
+ return ZX_OK;
+}
+
zx_status_t VmObjectPaged::CloneCOW(bool resizable, uint64_t offset, uint64_t size,
bool copy_name, fbl::RefPtr<VmObject>* clone_vmo) {
LTRACEF("vmo %p offset %#" PRIx64 " size %#" PRIx64 "\n", this, offset, size);
@@ -253,7 +280,7 @@
// allocate the clone up front outside of our lock
fbl::AllocChecker ac;
auto vmo = fbl::AdoptRef<VmObjectPaged>(
- new (&ac) VmObjectPaged(options, pmm_alloc_flags_, size, fbl::WrapRefPtr(this)));
+ new (&ac) VmObjectPaged(options, pmm_alloc_flags_, size, fbl::WrapRefPtr(this), nullptr));
if (!ac.check()) {
return ZX_ERR_NO_MEMORY;
}
diff --git a/system/public/zircon/syscalls.abigen b/system/public/zircon/syscalls.abigen
index 214788b..3a51d71 100644
--- a/system/public/zircon/syscalls.abigen
+++ b/system/public/zircon/syscalls.abigen
@@ -991,6 +991,10 @@
(options: uint32_t)
returns (zx_status_t, out_pager: zx_handle_t pager);
+syscall pager_create_vmo
+ (pager: zx_handle_t, port: zx_handle_t, key: uint64_t, size: uint64_t, options: uint32_t)
+ returns (zx_status_t, out_pager_vmo: zx_handle_t out_pager_vmo);
+
# Test syscalls (keep at the end)
syscall syscall_test_0() returns (zx_status_t);
