src/storage/blobfs/pager/user-pager.h

// Copyright 2019 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef SRC_STORAGE_BLOBFS_PAGER_USER_PAGER_H_
#define SRC_STORAGE_BLOBFS_PAGER_USER_PAGER_H_

#ifndef __Fuchsia__
#error Fuchsia-only Header
#endif

#include <lib/async-loop/cpp/loop.h>
#include <lib/async-loop/default.h>
#include <lib/async/dispatcher.h>
#include <lib/fzl/vmo-mapper.h>
#include <lib/zx/pager.h>

#include <memory>

#include "src/storage/blobfs/compression/external-decompressor.h"
#include "src/storage/blobfs/metrics.h"
#include "src/storage/blobfs/pager/transfer-buffer.h"
#include "src/storage/blobfs/pager/user-pager-info.h"
#include "src/storage/blobfs/transaction-manager.h"
#include "src/storage/lib/watchdog/include/lib/watchdog/watchdog.h"

namespace blobfs {
namespace pager {

// The size of the transfer buffer for reading from storage.
//
// The decision to use a single global transfer buffer is arbitrary; a pool of them could also be
// available in the future for more fine-grained access. Moreover, the blobfs pager uses a single
// thread at the moment, so a global buffer should be sufficient.
//
// 256 MB; but the size is arbitrary, since pages will become decommitted as they are moved to
// destination VMOS.
constexpr uint64_t kTransferBufferSize = 256 * (1 << 20);

// The size of the scratch buffer used for decompression.
//
// The decision to use a single global transfer buffer is arbitrary; a pool of them could also be
// available in the future for more fine-grained access. Moreover, the blobfs pager uses a single
// thread at the moment, so a global buffer should be sufficient.
//
// 256 MB; but the size is arbitrary, since pages will become decommitted as they are moved to
// destination VMOS.
constexpr uint64_t kDecompressionBufferSize = 256 * (1 << 20);

// Make sure blocks are page-aligned.
static_assert(kBlobfsBlockSize % PAGE_SIZE == 0);
// Make sure the pager transfer buffer is block-aligned.
static_assert(kTransferBufferSize % kBlobfsBlockSize == 0);
// Make sure the decompression scratch buffer is block-aligned.
static_assert(kDecompressionBufferSize % kBlobfsBlockSize == 0);
// Make sure the pager transfer buffer and decompression buffer are sized per the worst case
// compression ratio of 1.
static_assert(kTransferBufferSize >= kDecompressionBufferSize);

// Wrapper enum for error codes supported by the zx_pager_op_range(ZX_PAGER_OP_FAIL) syscall, used
// to communicate userpager errors to the kernel, so that the error can be propagated to the
// originator of the page request (if required), and the waiting thread can be unblocked. We use
// this wrapper enum instead of a raw zx_status_t as not all error codes are supported.
enum class PagerErrorStatus : zx_status_t {
  kErrIO = ZX_ERR_IO,
  kErrDataIntegrity = ZX_ERR_IO_DATA_INTEGRITY,
  kErrBadState = ZX_ERR_BAD_STATE,
  // This value is not supported by zx_pager_op_range(). Instead, it is used to determine if the
  // zx_pager_op_range() call is required in the first place - PagerErrorStatus::kOK indicates no
  // error, so we don't make the call.
  kOK = ZX_OK,
};

constexpr PagerErrorStatus ToPagerErrorStatus(zx_status_t status) {
  switch (status) {
    case ZX_OK:
      return PagerErrorStatus::kOK;
    // ZX_ERR_IO_DATA_INTEGRITY is the only error code in the I/O class of errors that we
    // distinguish. For everything else return ZX_ERR_IO.
    case ZX_ERR_IO_DATA_INTEGRITY:
      return PagerErrorStatus::kErrDataIntegrity;
    case ZX_ERR_IO:
    case ZX_ERR_IO_DATA_LOSS:
    case ZX_ERR_IO_INVALID:
    case ZX_ERR_IO_MISSED_DEADLINE:
    case ZX_ERR_IO_NOT_PRESENT:
    case ZX_ERR_IO_OVERRUN:
    case ZX_ERR_IO_REFUSED:
      return PagerErrorStatus::kErrIO;
    // Return ZX_ERR_BAD_STATE by default.
    default:
      return PagerErrorStatus::kErrBadState;
  }
}

// Encapsulates a user pager, its associated thread and transfer buffer.
class UserPager {
 public:
  DISALLOW_COPY_ASSIGN_AND_MOVE(UserPager);

  // Creates an instance of UserPager.
  // A new thread is created and started to process page fault requests.
  // |uncompressed_buffer| is used to retrieve and buffer uncompressed data from the underlying
  // storage. |compressed_buffer| is used to retrieve and buffer compressed data from the underlying
  // storage. |decompression_buffer_size| is the size of the scratch buffer to use for
  // decompression.
  [[nodiscard]] static zx::status<std::unique_ptr<UserPager>> Create(
      std::unique_ptr<TransferBuffer> compressed_buffer, std::unique_ptr<TransferBuffer> buffer,
      size_t decompression_buffer_size, BlobfsMetrics* metrics, bool sandbox_decompression);

  // Returns the pager handle.
  const zx::pager& Pager() const { return pager_; }

  // Returns the pager dispatcher.
  async_dispatcher_t* Dispatcher() const { return pager_loop_.dispatcher(); }

  // Invoked by the |PageWatcher| on a read request. Reads in the requested byte range
  // [|offset|, |offset| + |length|) for the inode associated with |info->identifier| into the
  // |transfer_buffer_|, and then moves those pages to the destination |vmo|. If |verifier_info| is
  // not null, uses it to verify the pages prior to transferring them to the destination vmo.
  //
  // If an error is encountered, the error code is returned as a |PagerErrorStatus|. This error code
  // is communicated to the kernel with the zx_pager_op_range(ZX_PAGER_OP_FAIL) syscall.
  // |PagerErrorStatus| wraps the supported error codes for ZX_PAGER_OP_FAIL.
  // 1. ZX_ERR_IO - Failure while reading the required data from the block device.
  // 2. ZX_ERR_IO_DATA_INTEGRITY - Failure while verifying the contents read in.
  // 3. ZX_ERR_BAD_STATE - Any other type of failure which prevents us from successfully completing
  // the page request, e.g. failure while decompressing, or while transferring pages from the
  // transfer buffer etc.
  [[nodiscard]] PagerErrorStatus TransferPagesToVmo(uint64_t offset, uint64_t length,
                                                    const zx::vmo& vmo, const UserPagerInfo& info);

 protected:
  // Protected for unit test access.
  zx::pager pager_;

 private:
  UserPager(size_t decompression_buffer_size, BlobfsMetrics* metrics);

  // Helper function to apply a scheduling deadline profile to the pager |thread|. Called from
  // |UserPager::Create| after starting the pager thread.
  static void SetDeadlineProfile(thrd_t thread);

  struct ReadRange {
    uint64_t offset;
    uint64_t length;
  };
  // Returns a range which covers [offset, offset+length), adjusted for alignment.
  //
  // The returned range will have the following guarantees:
  //  - The range will contain [offset, offset+length).
  //  - The returned offset will be block-aligned.
  //  - The end of the returned range is *either* block-aligned or is the end of the file.
  //  - The range will be adjusted for verification (see |BlobVerifier::Align|).
  //
  // The range needs to be extended before actually populating the transfer buffer with pages, as
  // absent pages will cause page faults during verification on the userpager thread, causing it to
  // block against itself indefinitely.
  //
  // For example:
  //                  |...input_range...|
  // |..data_block..|..data_block..|..data_block..|
  //                |........output_range.........|
  ReadRange GetBlockAlignedReadRange(const UserPagerInfo& info, uint64_t offset,
                                     uint64_t length) const;
  // Returns a range at least as big as GetBlockAlignedReadRange(), extended by an implementation
  // defined read-ahead algorithm.
  //
  // The same alignment guarantees for GetBlockAlignedReadRange() apply.
  ReadRange GetBlockAlignedExtendedRange(const UserPagerInfo& info, uint64_t offset,
                                         uint64_t length) const;

  PagerErrorStatus TransferChunkedPagesToVmo(uint64_t offset, uint64_t length, const zx::vmo& vmo,
                                             const UserPagerInfo& info);
  PagerErrorStatus TransferUncompressedPagesToVmo(uint64_t offset, uint64_t length,
                                                  const zx::vmo& vmo, const UserPagerInfo& info);

  // Scratch buffer for pager transfers of uncompressed data.
  // NOTE: Per the constraints imposed by |zx_pager_supply_pages|, the VMO owned by this buffer
  // needs to be unmapped before calling |zx_pager_supply_pages|. Map
  // |uncompressed_transfer_buffer_.vmo()| only when an explicit address is required, e.g. for
  // verification, and unmap it immediately after.
  std::unique_ptr<TransferBuffer> uncompressed_transfer_buffer_;

  // Scratch buffer for pager transfers of compressed data.
  // Unlike the above transfer buffer, this never needs to be unmapped since we will be calling
  // |zx_pager_supply_pages| on the |decompression_buffer_|.
  std::unique_ptr<TransferBuffer> compressed_transfer_buffer_;

  // A persistent mapping for |compressed_transfer_buffer_|.
  fzl::VmoMapper compressed_mapper_;

  // This is the buffer that can be written to by the other end of the
  // |decompressor_client_| connection. The contents are not to be trusted and
  // may be changed at any time, so they need to be copied out prior to
  // verification.
  zx::vmo sandbox_buffer_;

  // Scratch buffer for decompression.
  // NOTE: Per the constraints imposed by |zx_pager_supply_pages|, this needs to be unmapped before
  // calling |zx_pager_supply_pages|.
  zx::vmo decompression_buffer_;

  // Size of |decompression_buffer_|, stashed at vmo creation time to avoid a syscall each time the
  // size needs to be queried.
  const size_t decompression_buffer_size_;

  // Watchdog which triggers if any page faults exceed a threshold deadline.  This *must* come
  // before the loop below so that the loop, which might have references to the watchdog, is
  // destryoed first.
  std::unique_ptr<fs_watchdog::WatchdogInterface> watchdog_;

  // Async loop for pager requests.
  async::Loop pager_loop_ = async::Loop(&kAsyncLoopConfigNoAttachToCurrentThread);

  // Records all metrics for this instance of blobfs.
  BlobfsMetrics* metrics_ = nullptr;

  // Maintains a connection to the external decompressor.
  std::unique_ptr<ExternalDecompressorClient> decompressor_client_ = nullptr;
};

}  // namespace pager
}  // namespace blobfs

#endif  // SRC_STORAGE_BLOBFS_PAGER_USER_PAGER_H_