zircon/kernel/vm/include/vm/page_source.h

// Copyright 2018 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT

#ifndef ZIRCON_KERNEL_VM_INCLUDE_VM_PAGE_SOURCE_H_
#define ZIRCON_KERNEL_VM_INCLUDE_VM_PAGE_SOURCE_H_

#include <zircon/types.h>

#include <fbl/intrusive_double_list.h>
#include <fbl/intrusive_wavl_tree.h>
#include <fbl/ref_counted.h>
#include <fbl/ref_ptr.h>
#include <kernel/event.h>
#include <kernel/lockdep.h>
#include <kernel/mutex.h>
#include <ktl/unique_ptr.h>
#include <vm/page.h>
#include <vm/page_request.h>
#include <vm/vm.h>

class PageRequest;
class PageSource;

struct VmoDebugInfo {
  uintptr_t vmo_ptr;
  uint64_t vmo_id;
};

// Interface for providing pages to a VMO through page requests.
class PageProvider {
 public:
  virtual ~PageProvider() = default;

 private:
  // Synchronously gets a page from the backing source.
  virtual bool GetPageSync(uint64_t offset, VmoDebugInfo vmo_debug_info, vm_page_t** const page_out,
                           paddr_t* const pa_out) = 0;
  // Informs the backing source of a page request. The provider has ownership
  // of |request| until the async request is cancelled.
  virtual void GetPageAsync(page_request_t* request) = 0;
  // Informs the backing source that a page request has been fulfilled. This
  // must be called for all requests that are raised.
  virtual void ClearAsyncRequest(page_request_t* request) = 0;
  // Swaps the backing memory for a request. Assumes that |old|
  // and |new_request| have the same type, offset, and length.
  virtual void SwapRequest(page_request_t* old, page_request_t* new_req) = 0;

  // OnDetach is called once no more calls to GetPageSync/GetPageAsync will be made. It
  // will be called before OnClose and will only be called once.
  virtual void OnDetach() = 0;
  // After OnClose is called, no more calls will be made except for ::WaitOnEvent.
  virtual void OnClose() = 0;

  // Called from the backing source dispatcher when it is going away, in order to perform any
  // cleanup as required. The difference between this call and OnDetach/OnClose is that typically
  // OnDetach/OnClose are called from the VMO side, whereas OnDispatcherClose is called from the
  // backing source side (e.g. a pager).
  virtual void OnDispatcherClose() = 0;

  // Waits on an |event| associated with a page request.
  virtual zx_status_t WaitOnEvent(Event* event) = 0;

  // Dumps relevant state for debugging purposes.
  virtual void Dump() = 0;

  friend PageSource;
  friend PageRequest;
};

// A page source is responsible for fulfilling page requests from a VMO with backing pages.
// The PageSource class mostly contains generic functionality around managing
// the lifecycle of VMO page requests. The PageSource contains an reference to a PageProvider
// implementation, which is responsible for actually providing the pages. (E.g. for VMOs backed by a
// userspace pager, the PageProvider is a PagerProxy instance which talks to the userspace pager
// service.)
//
// The synchronous fulfillment of requests is fairly straightforward, with direct calls
// from the vm object to the PageSource to the PageProvider.
//
// For asynchronous requests, the lifecycle is as follows:
//   1) A vm object requests a page with PageSource::GetPage.
//   2) PageSource starts tracking the request's PageRequest and then
//      forwards the request to PageProvider::GetPageAsync.
//   3) The caller waits for the request with PageRequest::Wait.
//   4) At some point, whatever is backing the PageProvider provides pages
//      to the vm object (e.g. with VmObjectPaged::SupplyPages).
//   5) The vm object calls PageSource::OnPagesSupplied, which signals
//      any PageRequests that have been fulfilled.
//   6) The caller wakes up and queries the vm object again, by which
//      point the requested page will be present.

// Object which provides pages to a vm_object.
class PageSource : public fbl::RefCounted<PageSource>,
                   public fbl::DoublyLinkedListable<fbl::RefPtr<PageSource>> {
 public:
  PageSource() = delete;
  explicit PageSource(ktl::unique_ptr<PageProvider> page_provider);

  // Sends a request to the backing source to provide the requested page.
  //
  // Returns ZX_OK if the request was synchronously fulfilled.
  // Returns ZX_ERR_SHOULD_WAIT if the request will be asynchronously
  // fulfilled. The caller should wait on |req|.
  // Returns ZX_ERR_NEXT if the PageRequest is in batch mode and the caller
  // can continue to add more pages to the request.
  // Returns ZX_ERR_NOT_FOUND if the request cannot be fulfilled.
  zx_status_t GetPage(uint64_t offset, PageRequest* req, VmoDebugInfo vmo_debug_info,
                      vm_page_t** const page_out, paddr_t* const pa_out);

  // Called to complete a batched PageRequest if the last call to GetPage
  // returned ZX_ERR_NEXT.
  //
  // Returns ZX_ERR_SHOULD_WAIT if the PageRequest will be fulfilled after
  // being waited upon.
  // Returns ZX_ERR_NOT_FOUND if the request will never be resolved.
  zx_status_t FinalizeRequest(PageRequest* request);

  // Updates the request tracking metadata to account for pages [offset, offset + len) having
  // been supplied to the owning vmo.
  void OnPagesSupplied(uint64_t offset, uint64_t len);

  // Fails outstanding page requests in the range [offset, offset + len). Events associated with the
  // failed page requests are signaled with the |error_status|, and any waiting threads are
  // unblocked.
  void OnPagesFailed(uint64_t offset, uint64_t len, zx_status_t error_status);

  // Returns true if |error_status| is a valid provider failure error code, which can be used with
  // |OnPagesFailed|.
  //
  // Not every error code is supported, since these errors can get returned via a zx_vmo_read() or a
  // zx_vmo_op_range(), if those calls resulted in a page fault. So the |error_status| should be a
  // supported return error code for those syscalls.
  static bool IsValidFailureCode(zx_status_t error_status);

  // Detaches the source from the VMO. All future calls into the page source will fail. All
  // pending read transactions are aborted. Pending flush transactions will still
  // be serviced.
  void Detach();

  // Closes the source. Will call Detach() if the source is not already detached. All pending
  // transactions will be aborted and all future calls will fail.
  void Close();

  // Called when the PageProvider's backing dispatcher (e.g. a pager dispatcher) is being torn down.
  // See PagerDispatcher::on_zero_handles().
  void OnPageProviderDispatcherClose();

  void Dump() const;

 protected:
  // destructor should only be invoked from RefPtr
  virtual ~PageSource();
  friend fbl::RefPtr<PageSource>;

 private:
  fbl::Canary<fbl::magic("VMPS")> canary_;

  mutable DECLARE_MUTEX(PageSource) page_source_mtx_;
  bool detached_ TA_GUARDED(page_source_mtx_) = false;
  bool closed_ TA_GUARDED(page_source_mtx_) = false;

  // Tree of outstanding requests which have been sent to the PageProvider. The list
  // is keyed by the end offset of the requests (not the start offsets).
  fbl::WAVLTree<uint64_t, PageRequest*> outstanding_requests_ TA_GUARDED(page_source_mtx_);

#ifdef DEBUG_ASSERT_IMPLEMENTED
  // Tracks the request currently being processed (only used for verifying batching assertions).
  PageRequest* current_request_ TA_GUARDED(page_source_mtx_) = nullptr;
#endif  // DEBUG_ASSERT_IMPLEMENTED

  // PageProvider instance that will provide pages asynchronously (e.g. a userspace pager, see
  // PagerProxy for details).
  ktl::unique_ptr<PageProvider> page_provider_;

  // Sends a read request to the backing source, or adds the request to the overlap_ list if the
  // needed region has already been requested from the source.
  void SendRequestToProviderLocked(PageRequest* request) TA_REQ(page_source_mtx_);

  // Wakes up the given PageRequest and all overlapping requests, with an optional |status|.
  void CompleteRequestLocked(PageRequest* request, zx_status_t status = ZX_OK)
      TA_REQ(page_source_mtx_);

  // Removes |request| from any internal tracking. Called by a PageRequest if
  // it needs to abort itself.
  void CancelRequest(PageRequest* request) TA_EXCL(page_source_mtx_);

  friend PageRequest;
};

// Object which is used to make delayed page requests to a PageSource
class PageRequest : public fbl::WAVLTreeContainable<PageRequest*>,
                    public fbl::DoublyLinkedListable<PageRequest*> {
 public:
  // If |allow_batching| is true, then a single request can be used to service
  // multiple consecutive pages.
  explicit PageRequest(bool allow_batching = false) : allow_batching_(allow_batching) {}
  ~PageRequest();

  // Returns ZX_OK on success, or a permitted error code if the backing page provider explicitly
  // failed this page request. Returns ZX_ERR_INTERNAL_INTR_KILLED if the thread was killed.
  zx_status_t Wait();

  DISALLOW_COPY_ASSIGN_AND_MOVE(PageRequest);

 private:
  // PageRequests passed to GetPage may or may not be initialized. offset_ must be checked
  // and the object must be initialized if necessary.
  void Init(fbl::RefPtr<PageSource> src, uint64_t offset, VmoDebugInfo vmo_debug_info);

  const bool allow_batching_;

  // The page source this request is currently associated with.
  fbl::RefPtr<PageSource> src_;
  // Event signaled when the request is fulfilled.
  AutounsignalEvent event_;
  // PageRequests are active if offset_ is not UINT64_MAX. In an inactive request, the
  // only other valid field is src_.
  uint64_t offset_ = UINT64_MAX;
  // The total length of the request.
  uint64_t len_ = 0;
  // The vmobject this page request is for.
  VmoDebugInfo vmo_debug_info_ = {};

  // Keeps track of the size of the request that still needs to be fulfilled. This
  // can become incorrect if some pages get supplied, decommitted, and then
  // re-supplied. If that happens, then it will cause the page request to complete
  // prematurely. However, page source clients should be operating in a loop to handle
  // evictions, so this will simply result in some redundant read requests to the
  // page source. Given the rarity in which this situation should arise, it's not
  // worth the complexity of tracking it.
  uint64_t pending_size_ = 0;

  // List node for overlapping requests.
  fbl::DoublyLinkedList<PageRequest*> overlap_;

  // Request struct for the PageProvider.
  page_request_t read_request_;

  uint64_t GetEnd() const {
    // Assert on overflow, since it means vmobject made an out-of-bounds request.
    uint64_t unused;
    DEBUG_ASSERT(!add_overflow(offset_, len_, &unused));

    return offset_ + len_;
  }

  uint64_t GetKey() const { return GetEnd(); }

  friend PageSource;
  friend fbl::DefaultKeyedObjectTraits<uint64_t, PageRequest>;
};

#endif  // ZIRCON_KERNEL_VM_INCLUDE_VM_PAGE_SOURCE_H_