zircon/kernel/object/io_buffer_dispatcher.cc

// Copyright 2022 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT

#include <assert.h>
#include <lib/counters.h>
#include <lib/iob/blob-id-allocator.h>
#include <lib/user_copy/user_ptr.h>
#include <lib/zx/result.h>
#include <string.h>
#include <zircon/errors.h>
#include <zircon/rights.h>
#include <zircon/syscalls/iob.h>
#include <zircon/syscalls/object.h>
#include <zircon/types.h>

#include <cstdint>

#include <fbl/alloc_checker.h>
#include <fbl/array.h>
#include <fbl/ref_ptr.h>
#include <kernel/koid.h>
#include <kernel/lockdep.h>
#include <kernel/mutex.h>
#include <ktl/move.h>
#include <object/dispatcher.h>
#include <object/handle.h>
#include <object/io_buffer_dispatcher.h>
#include <object/vm_object_dispatcher.h>
#include <vm/pinned_vm_object.h>
#include <vm/pmm.h>
#include <vm/vm_address_region.h>
#include <vm/vm_object.h>

#include <ktl/enforce.h>

#define LOCAL_TRACE 0

KCOUNTER(dispatcher_iob_create_count, "dispatcher.iob.create")
KCOUNTER(dispatcher_iob_destroy_count, "dispatcher.iob.destroy")

// static
zx::result<fbl::Array<IoBufferDispatcher::IobRegionVariant>> IoBufferDispatcher::CreateRegions(
    const IoBufferDispatcher::RegionArray& region_configs, VmObjectChildObserver* ep0,
    VmObjectChildObserver* ep1) {
  fbl::AllocChecker ac;
  fbl::Array<IobRegionVariant> region_inner =
      fbl::MakeArray<IobRegionVariant>(&ac, region_configs.size());
  if (!ac.check()) {
    return zx::error(ZX_ERR_NO_MEMORY);
  }

  for (unsigned i = 0; i < region_configs.size(); i++) {
    zx_iob_region_t region_config = region_configs[i];
    if (region_config.type != ZX_IOB_REGION_TYPE_PRIVATE) {
      return zx::error(ZX_ERR_INVALID_ARGS);
    }
    if (region_config.access == 0) {
      return zx::error(ZX_ERR_INVALID_ARGS);
    }

    // A note on resource management:
    //
    // Everything we allocate in this loop ultimately gets owned by the SharedIobState and will be
    // cleaned up when both PeeredDispatchers are destroyed and drop their reference to the
    // SharedIobState.
    //
    // However, there is a complication. The VmoChildObservers have a reference to the dispatchers
    // which creates a cycle: IoBufferDispatcher -> SharedIobState -> IobRegion -> VmObject ->
    // VmoChildObserver -> IoBufferDispatcher.
    //
    // Since the VmoChildObservers keep a raw pointers to the dispatchers, we need to be sure to
    // reset the the corresponding pointers when we destroy an IoBufferDispatcher. Otherwise when an
    // IoBufferDispatcher maps a region, it could try to update a destroyed peer.
    //
    // See: IoBufferDispatcher~IoBufferDispatcher.

    // We effectively duplicate the logic from sys_vmo_create here, but instead of creating a
    // kernel handle and dispatcher, we keep ownership of it and assign it to a region.

    zx::result<VmObjectDispatcher::CreateStats> parse_result =
        VmObjectDispatcher::parse_create_syscall_flags(region_config.private_region.options,
                                                       region_config.size);
    if (parse_result.is_error()) {
      return zx::error(parse_result.error_value());
    }
    VmObjectDispatcher::CreateStats stats = parse_result.value();

    fbl::RefPtr<VmObjectPaged> vmo;
    if (zx_status_t status =
            VmObjectPaged::Create(PMM_ALLOC_FLAG_ANY, stats.flags, stats.size, &vmo);
        status != ZX_OK) {
      return zx::error(status);
    }

    // parse_create_syscall_flags will round up the size to the nearest page, or set the size to the
    // maximum possible VMO size if ZX_VMO_UNBOUNDED is used. We need to know the actual size of the
    // vmo to later return if asked.
    region_config.size = stats.size;
    zx_koid_t koid = KernelObjectId::Generate();
    vmo->set_user_id(koid);

    // In order to track mappings and unmappings separately for each endpoint, we give each
    // endpoint a child reference instead of the created vmo.
    fbl::RefPtr<VmObject> ep0_reference;
    fbl::RefPtr<VmObject> ep1_reference;

    Resizability resizability =
        vmo->is_resizable() ? Resizability::Resizable : Resizability::NonResizable;
    if (zx_status_t status =
            vmo->CreateChildReference(resizability, 0, 0, true, nullptr, &ep0_reference);
        status != ZX_OK) {
      return zx::error(status);
    }
    if (zx_status_t status =
            vmo->CreateChildReference(resizability, 0, 0, true, nullptr, &ep1_reference);
        status != ZX_OK) {
      return zx::error(status);
    }

    ep0_reference->set_user_id(koid);
    ep1_reference->set_user_id(koid);

    // Now each endpoint can observe the mappings created by the other
    ep0_reference->SetChildObserver(ep1);
    ep1_reference->SetChildObserver(ep0);

    if (zx::result result =
            CreateIobRegionVariant(ktl::move(ep0_reference), ktl::move(ep1_reference),
                                   ktl::move(vmo), region_config, koid);
        result.is_error()) {
      return result.take_error();
    } else {
      region_inner[i] = ktl::move(result.value());
    }
  }
  return zx::ok(ktl::move(region_inner));
}

// static
zx_status_t IoBufferDispatcher::Create(uint64_t options,
                                       const IoBufferDispatcher::RegionArray& region_configs,
                                       KernelHandle<IoBufferDispatcher>* handle0,
                                       KernelHandle<IoBufferDispatcher>* handle1,
                                       zx_rights_t* rights) {
  if (region_configs.size() == 0) {
    return ZX_ERR_INVALID_ARGS;
  }

  fbl::AllocChecker ac;
  auto holder0 = fbl::AdoptRef(new (&ac) PeerHolder<IoBufferDispatcher>());
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }
  auto holder1 = holder0;

  fbl::RefPtr<SharedIobState> shared_regions = fbl::AdoptRef(new (&ac) SharedIobState{
      .regions = nullptr,
  });

  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }

  KernelHandle new_handle0(fbl::AdoptRef(
      new (&ac) IoBufferDispatcher(ktl::move(holder0), IobEndpointId::Ep0, shared_regions)));
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }

  KernelHandle new_handle1(fbl::AdoptRef(
      new (&ac) IoBufferDispatcher(ktl::move(holder1), IobEndpointId::Ep1, shared_regions)));
  if (!ac.check()) {
    return ZX_ERR_NO_MEMORY;
  }

  zx::result<fbl::Array<IobRegionVariant>> regions =
      CreateRegions(region_configs, new_handle0.dispatcher().get(), new_handle1.dispatcher().get());
  if (regions.is_error()) {
    return regions.error_value();
  }
  {
    Guard<CriticalMutex> guard{&shared_regions->state_lock};
    shared_regions->regions = *ktl::move(regions);
  }

  new_handle0.dispatcher()->InitPeer(new_handle1.dispatcher());
  new_handle1.dispatcher()->InitPeer(new_handle0.dispatcher());

  *rights = default_rights();
  *handle0 = ktl::move(new_handle0);
  *handle1 = ktl::move(new_handle1);

  return ZX_OK;
}

IoBufferDispatcher::IoBufferDispatcher(fbl::RefPtr<PeerHolder<IoBufferDispatcher>> holder,
                                       IobEndpointId endpoint_id,
                                       fbl::RefPtr<SharedIobState> shared_state)
    : PeeredDispatcher(ktl::move(holder)),
      shared_state_(ktl::move(shared_state)),
      endpoint_id_(endpoint_id) {
  kcounter_add(dispatcher_iob_create_count, 1);
}

zx_rights_t IoBufferDispatcher::GetMapRights(zx_rights_t iob_rights, size_t region_index) const {
  zx_rights_t region_rights =
      shared_state_->GetRegion<>(region_index)->GetMapRights(GetEndpointId());
  region_rights &= (iob_rights | ZX_RIGHT_MAP);
  return region_rights;
}

zx_rights_t IoBufferDispatcher::GetMediatedRights(size_t region_index) const {
  return shared_state_->GetRegion<>(region_index)->GetMediatedRights(GetEndpointId());
}

const fbl::RefPtr<VmObject>& IoBufferDispatcher::GetVmo(size_t region_index) const {
  return shared_state_->GetRegion<>(region_index)->GetVmo(GetEndpointId());
}

zx_iob_region_t IoBufferDispatcher::GetRegion(size_t region_index) const {
  return shared_state_->GetRegion<>(region_index)->region();
}

size_t IoBufferDispatcher::RegionCount() const {
  canary_.Assert();
  Guard<CriticalMutex> guard{&shared_state_->state_lock};
  return shared_state_->regions.size();
}

void IoBufferDispatcher::on_zero_handles_locked() { canary_.Assert(); }

void IoBufferDispatcher::OnPeerZeroHandlesLocked() {
  canary_.Assert();
  peer_zero_handles_ = true;
  if (peer_mapped_regions_ == 0) {
    UpdateStateLocked(0, ZX_IOB_PEER_CLOSED);
  }
}

void IoBufferDispatcher::OnZeroChild() {
  Guard<CriticalMutex> guard{get_lock()};

  // OnZeroChildren gets called every time the number of children is equal to zero. Due to races, by
  // the time this method is called we could already have children again. This is fine, since all
  // we need to do is ensure that every increment of peer_mapped_regions_ is paired with a
  // decrement.
  DEBUG_ASSERT(peer_mapped_regions_ > 0);
  peer_mapped_regions_--;
  if (peer_mapped_regions_ == 0 && peer_zero_handles_) {
    UpdateStateLocked(0, ZX_IOB_PEER_CLOSED);
  }
}

IoBufferDispatcher::~IoBufferDispatcher() {
  // The other endpoint's vmos are set up to notify this endpoint when they map regions via a raw
  // pointer. Since we're about to be destroyed, we need to unregister.
  //
  // VMObject unregisters in when the last handle is closed, but we do it a bit differently here.
  //
  // If we unregister in OnPeerZeroHandlesLocked, then we update the ChildObserver, we create the
  // dependency get_lock() -> child_observer_lock_. Then when we OnZeroChild is called, we first get
  // the child_observer_lock_, then try to update the state, locking get_lock establishing
  // child_observer_lock_ -> get_lock() which could lead to some lock ordering issues.
  //
  // However, cleaning up in the destructor means that we could be notified while we are being
  // destroyed. This should be okay because:
  // - Each vmo's child observer is protected by its child_observer_lock_.
  // - While we are in this destructor, a notification may attempt to get a vmo's.
  //   child_observer_lock_.
  // - If we have already destroyed updated the observer, it will see a nullptr and not access our
  //   state.
  // - If we have not already destroyed the observer, it will notify us while continuing to hold the
  //   child observer lock.
  // - If we attempt to reset the observer that is notifying us, we will block until it is completed
  //   and releases the lock.
  // - Thus we cannot continue to the Destruction Sequence until there are no observers in progress
  //   accessing our state, and no observer can start accessing our state.
  Guard<CriticalMutex> guard{&shared_state_->state_lock};
  IobEndpointId other_id =
      endpoint_id_ == IobEndpointId::Ep0 ? IobEndpointId::Ep1 : IobEndpointId::Ep0;
  for (size_t i = 0; i < shared_state_->regions.size(); ++i) {
    if (const fbl::RefPtr<VmObject>& vmo = shared_state_->GetRegionLocked<>(i)->GetVmo(other_id);
        vmo != nullptr) {
      vmo->SetChildObserver(nullptr);
    }
  }

  kcounter_add(dispatcher_iob_destroy_count, 1);
}

zx_info_iob_t IoBufferDispatcher::GetInfo() const {
  canary_.Assert();
  Guard<CriticalMutex> guard{&shared_state_->state_lock};
  return {.options = 0, .region_count = static_cast<uint32_t>(shared_state_->regions.size())};
}

zx_iob_region_info_t IoBufferDispatcher::GetRegionInfo(size_t index) const {
  canary_.Assert();
  return shared_state_->GetRegion<>(index)->GetRegionInfo(endpoint_id_ == IobEndpointId::Ep1);
}

zx_status_t IoBufferDispatcher::get_name(char (&out_name)[ZX_MAX_NAME_LEN]) const {
  shared_state_->GetRegion<>(0)->GetVmo(IobEndpointId::Ep0)->get_name(out_name, ZX_MAX_NAME_LEN);
  return ZX_OK;
}

zx_status_t IoBufferDispatcher::set_name(const char* name, size_t len) {
  canary_.Assert();
  Guard<CriticalMutex> guard{&shared_state_->state_lock};
  for (size_t i = 0; i < shared_state_->regions.size(); ++i) {
    const IobRegion* region = shared_state_->GetRegionLocked<>(i);
    switch (region->region().type) {
      case ZX_IOB_REGION_TYPE_PRIVATE: {
        zx_status_t region_result = region->GetVmo(IobEndpointId::Ep0)->set_name(name, len);
        if (region_result != ZX_OK) {
          return region_result;
        }
        region_result = region->GetVmo(IobEndpointId::Ep1)->set_name(name, len);
        if (region_result != ZX_OK) {
          return region_result;
        }
        break;
      }
      default:
        continue;
    }
  }
  return ZX_OK;
}

zx::result<fbl::RefPtr<VmObject>> IoBufferDispatcher::CreateMappableVmoForRegion(
    size_t region_index) {
  fbl::RefPtr<VmObject> child_reference;
  const fbl::RefPtr<VmObject>& vmo = GetVmo(region_index);
  Resizability resizability =
      vmo->is_resizable() ? Resizability::Resizable : Resizability::NonResizable;
  {
    bool first_child = false;

    zx_status_t status =
        vmo->CreateChildReference(resizability, 0, 0, true, &first_child, &child_reference);
    if (status != ZX_OK) {
      return zx::error(status);
    }
    if (first_child) {
      // Need to record 0->1 transition. As we are holding a refptr to the child we know that
      // OnZeroChildren cannot get called to decrement before we can increment. In this case the
      // lock is not attempting to synchronize anything beyond the raw access of
      // peer_mapped_regions_.
      Guard<CriticalMutex> guard{get_lock()};
      const fbl::RefPtr<IoBufferDispatcher>& p = peer();
      AssertHeld(*p->get_lock());
      p->peer_mapped_regions_++;
    }
  }
  child_reference->set_user_id(vmo->user_id());
  return zx::ok(child_reference);
}

zx::result<uint32_t> IoBufferDispatcher::AllocateId(size_t region_index,
                                                    user_in_ptr<const ktl::byte> blob_ptr,
                                                    size_t blob_size) {
  if (region_index >= RegionCount()) {
    return zx::error{ZX_ERR_OUT_OF_RANGE};
  }
  if (auto* region = shared_state_->GetRegion<IobRegionIdAllocator>(region_index)) {
    return region->AllocateId(GetEndpointId(), blob_ptr, blob_size);
  }
  return zx::error{ZX_ERR_WRONG_TYPE};
}

// static
zx::result<IoBufferDispatcher::IobRegionVariant> IoBufferDispatcher::CreateIobRegionVariant(
    fbl::RefPtr<VmObject> ep0_vmo,     //
    fbl::RefPtr<VmObject> ep1_vmo,     //
    fbl::RefPtr<VmObject> kernel_vmo,  //
    const zx_iob_region_t& region,     //
    zx_koid_t koid) {
  constexpr zx_iob_access_t kEp0MedR = ZX_IOB_ACCESS_EP0_CAN_MEDIATED_READ;
  constexpr zx_iob_access_t kEp0MedW = ZX_IOB_ACCESS_EP0_CAN_MEDIATED_WRITE;
  constexpr zx_iob_access_t kEp0MapW = ZX_IOB_ACCESS_EP0_CAN_MAP_WRITE;
  constexpr zx_iob_access_t kEp1MedR = ZX_IOB_ACCESS_EP1_CAN_MEDIATED_READ;
  constexpr zx_iob_access_t kEp1MedW = ZX_IOB_ACCESS_EP1_CAN_MEDIATED_WRITE;
  constexpr zx_iob_access_t kEp1MapW = ZX_IOB_ACCESS_EP1_CAN_MAP_WRITE;

  zx_iob_access_t mediated0 = region.access & (kEp0MedR | kEp0MedW);
  zx_iob_access_t mediated1 = region.access & (kEp1MedR | kEp1MedW);
  bool mediated = mediated0 != 0 || mediated1 != 0;
  bool map_writable = (region.access & kEp0MapW) != 0 || (region.access & kEp1MapW) != 0;

  // The region memory must be directly accessible by userspace or the kernel.
  if (!map_writable && !mediated) {
    return zx::error{ZX_ERR_INVALID_ARGS};
  }

  fbl::RefPtr<VmMapping> mapping;
  zx_vaddr_t base = 0;
  if (mediated) {
    const char* mapping_name;
    switch (region.discipline.type) {
      case ZX_IOB_DISCIPLINE_TYPE_ID_ALLOCATOR:
        // If an ID allocator endpoint requests mediated access, that access must at
        // least admit mediated writes. Equivalently, no ID allocator endpoint can
        // be read-only in terms of mediated access.
        if (mediated0 == kEp0MedR || mediated1 == kEp1MedR) {
          return zx::error{ZX_ERR_INVALID_ARGS};
        }
        mapping_name = "IOBuffer region (ID allocator)";
        break;
      case ZX_IOB_DISCIPLINE_TYPE_NONE:
        // NONE type discipline does not support mediated access.
        [[fallthrough]];
      default:
        // Unknown discipline.
        return zx::error{ZX_ERR_INVALID_ARGS};
    }

    // Create a mapping of this VMO in the kernel aspace. It is convenient to
    // create the mapping object first before any discipline-specific pinning
    // and mapping is actually performed, allowing that to be done in
    // subclasses, so we pass `VMAR_FLAG_DEBUG_DYNAMIC_KERNEL_MAPPING`.
    fbl::RefPtr<VmAddressRegion> kernel_vmar =
        VmAspace::kernel_aspace()->RootVmar()->as_vm_address_region();
    zx::result<VmAddressRegion::MapResult> result = kernel_vmar->CreateVmMapping(
        0, region.size, 0, VMAR_FLAG_DEBUG_DYNAMIC_KERNEL_MAPPING, ktl::move(kernel_vmo), 0,
        ARCH_MMU_FLAG_PERM_READ | ARCH_MMU_FLAG_PERM_WRITE, mapping_name);
    if (result.is_error()) {
      return result.take_error();
    }
    mapping = ktl::move(result.value().mapping);
    base = result.value().base;
  }

  IobRegionVariant variant;
  switch (region.discipline.type) {
    case ZX_IOB_DISCIPLINE_TYPE_NONE:
      variant = IobRegionNone{
          ktl::move(ep0_vmo), ktl::move(ep1_vmo), ktl::move(mapping), base, region, koid};
      break;
    case ZX_IOB_DISCIPLINE_TYPE_ID_ALLOCATOR: {
      IobRegionIdAllocator allocator{
          ktl::move(ep0_vmo), ktl::move(ep1_vmo), ktl::move(mapping), base, region, koid};
      if (zx::result result = allocator.Init(); result.is_error()) {
        return result.take_error();
      }
      variant = ktl::move(allocator);
      break;
    }
    default:
      // Unknown discipline.
      return zx::error{ZX_ERR_INVALID_ARGS};
  }
  return zx::ok(ktl::move(variant));
}

zx::result<> IoBufferDispatcher::IobRegionIdAllocator::Init() {
  AssertDiscipline(ZX_IOB_DISCIPLINE_TYPE_ID_ALLOCATOR);

  if (!mapping()) {
    return zx::ok();
  }

  zx_status_t status =
      PinnedVmObject::Create(mapping()->vmo(), 0, region().size, /*write=*/true, &pin_);
  if (status != ZX_OK) {
    return zx::error{status};
  }
  status = mapping()->MapRange(0, region().size, /*commit=*/true);
  if (status != ZX_OK) {
    return zx::error{status};
  }

  iob::BlobIdAllocator allocator(mediated_bytes());
  allocator.Init(/*zero_fill=*/false);  // A fresh VMO, so already zero-filled.

  return zx::ok();
}

zx::result<uint32_t> IoBufferDispatcher::IobRegionIdAllocator::AllocateId(
    IobEndpointId id, user_in_ptr<const ktl::byte> blob_ptr, size_t blob_size) {
  AssertDiscipline(ZX_IOB_DISCIPLINE_TYPE_ID_ALLOCATOR);

  zx_rights_t mediated_rights = GetMediatedRights(id);
  if ((mediated_rights & ZX_RIGHT_WRITE) == 0) {
    return zx::error{ZX_ERR_ACCESS_DENIED};
  }

  iob::BlobIdAllocator allocator(mediated_bytes());
  zx_status_t copy_status = ZX_OK;
  auto result = allocator.Allocate(
      [&copy_status](auto blob_ptr, ktl::span<ktl::byte> dest) {
        copy_status = blob_ptr.copy_array_from_user(dest.data(), dest.size());
      },
      blob_ptr, blob_size);
  if (copy_status != ZX_OK) {
    return zx::error{copy_status};
  }
  if (result.is_error()) {
    switch (result.error_value()) {
      case iob::BlobIdAllocator::AllocateError::kOutOfMemory:
        return zx::error{ZX_ERR_NO_MEMORY};
      default:
        return zx::error{ZX_ERR_IO_DATA_INTEGRITY};
    }
  }
  return zx::ok(result.value());
}