src/lib/elfldltl/include/lib/elfldltl/segment-with-vmo.h - fuchsia - Git at Google

 // Copyright 2023 The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 #ifndef SRC_LIB_ELFLDLTL_INCLUDE_LIB_ELFLDLTL_SEGMENT_WITH_VMO_H_
 #define SRC_LIB_ELFLDLTL_INCLUDE_LIB_ELFLDLTL_SEGMENT_WITH_VMO_H_

 #include <lib/fit/result.h>
 #include <lib/zx/vmo.h>
 #include <zircon/assert.h>

 #include <type_traits>

 #include "diagnostics.h"
 #include "load.h"
 #include "mapped-vmo-file.h"
 #include "vmar-loader.h"
 #include "zircon.h"

 namespace elfldltl {

 // elfldltl::SegmentWithVmo::Copy and elfldltl::SegmentWithVmo::NoCopy can be
 // used as the SegmentWrapper template template parameter in elfldltl::LoadInfo
 // instantiations.  Partial specializations of elfldltl::VmarLoader::SegmentVmo
 // provided below will then check segment.vmo() when mapping a segment.  If
 // it's a valid handle, then the segment is mapped from that VMO at offset 0;
 // otherwise it's mapped from the main file VMO at segment.offset() as usual.
 //
 // The Copy version has the usual behavior of treating the VMO as read-only and
 // making a copy-on-write child VMO for a writable segment.  The NoCopy version
 // will instead map segment.vmo() itself writable, so it can only be used once
 // as that VMO's contents will be modified by the process using it.
 //
 // The NoCopy case allows for simple out-of-process dynamic linking where
 // nothing is cached for reuse.  The Copy case allows for the "zygote" model
 // where VMOs can be reused to get copy-on-write sharing of segments where
 // relocation has already been done.
 //
 // elfldltl::SegmentWithVmo::AlignSegments adjusts any LoadInfo segments that
 // have a partial page of data followed by zero-fill bytes so that all segments
 // can be mapped page-aligned without additional partial-page zeroing work (as
 // elfldltl::AlignedRemoteVmarLoader requires).  To do this, it installs a
 // per-segment VMO that copies (via copy-on-write snapshot) the data and then
 // zero-fills the partial page directly in the VMO contents.  Using this with
 // an elfldltl::SegmentWithVmo::NoCopy instantiation of elfldltl::LoadInfo
 // really just does the zeroing work ahead of time; elfldltl::RemoteVmarLoader
 // would do the same thing when mapping.  An elfldltl::SegmentWithVmo::Copy
 // instantiation can be used either for a full zygote model where the
 // relocations are applied in the stored VMOs, or simply to share the zeroing
 // work across multiple separate loads with their own relocations applied to
 // their own copies (this may be beneficial to overall page-sharing if the
 // final data page didn't need to be touched for any relocations).
 //
 // elfldltl::SegmentWithVmo::MakeMutable can be applied to a particular segment
 // to install its own copied VMO.  This is what out-of-process dynamic linking
 // must do before applying relocations that touch that segment's contents.
 //
 // The elfldltl::SegmentWithVmo::GetMutableMemory class is meant for use with
 // the elfldltl::LoadInfoMutableMemory adapter object.

 class SegmentWithVmo {
  public:
   // This becomes an additional base type along with the original Segment type.
   struct VmoHolder {
     VmoHolder() = default;

     VmoHolder(VmoHolder&&) = default;
     VmoHolder& operator=(VmoHolder&&) = default;

     explicit VmoHolder(zx::vmo vmo) : vmo_{std::move(vmo)} {}

     zx::vmo& vmo() { return vmo_; }
     const zx::vmo& vmo() const { return vmo_; }

    private:
     zx::vmo vmo_;
   };
   static_assert(std::is_move_constructible_v<VmoHolder>);
   static_assert(std::is_move_assignable_v<VmoHolder>);

   // The Copy and NoCopy templates are shorthands using this.
   template <class CopyT>
   struct Wrapper {
     // Segment types that have a filesz() are replaced with this subclass.
     // ZeroFillSegment never needs a VMO handle.
     template <class Segment>
     class WithVmo : public Segment, public VmoHolder {
      public:
       using Segment::Segment;

       // It's movable, though not copyable.  The base Segment type is copyable.
       WithVmo(WithVmo&&) = default;

       // It's explicitly constructible from the base Segment type.
       explicit WithVmo(const Segment& other) : Segment{other} {}

       WithVmo& operator=(WithVmo&&) = default;

       // Assigning from the base Segment type drops any previous VMO.
       WithVmo& operator=(const Segment& other) {
         Segment::operator=(other);
         vmo().reset();
       }

       // Don't merge with any other segment if there's a VMO installed.  Its
       // contents won't cover the other segment.  Both segments will get the
       // reciprocal check, so if this segment has no VMO (yet) but the other
       // does, its CanMergeWith is the one to return false.
       template <class Other>
       bool CanMergeWith(const Other& other) const {
         static_assert(std::is_move_constructible_v<WithVmo>);
         static_assert(std::is_move_assignable_v<WithVmo>);
         return !vmo();
       }

       // Similar to `CanMergeWith`, this segment can't be replaced by another
       // segment if there's a VMO installed.
       bool CanReplace() const {
         static_assert(std::is_move_constructible_v<WithVmo>);
         static_assert(std::is_move_assignable_v<WithVmo>);
         return !vmo();
       }

       // This accepts the corresponding segment type either from the base
       // (unwrapped) LoadInfo::*Segment type or from a SegmentWithVmo wrapper
       // like this one.  Both SegmentWithVmo::Copy and SegmentWithVmo::NoCopy
       // versions of the corresponding Segment subclass are accepted.  The VMOs
       // are always cloned--the only distinction of NoCopy is what a VmarLoader
       // does with these new segment VMOs (NoCopy writes them in place).
       template <class Diagnostics, class OtherSegment>
       static fit::result<bool, WithVmo> Copy(Diagnostics& diag, const OtherSegment& other) {
         // If the other segment has its own VMO, then the copied segment
         // needs a copy-on-write clone of that whole VMO.
         zx::unowned_vmo other_vmo;

         // For copy from the unadorned Segment, there's never a VMO to copy.
         if constexpr (!std::is_same_v<OtherSegment, Segment>) {
           // Copy from the corresponding Segment subclass of this wrapper is
           // the other supported option, to clone the existing VMO, if any.
           static_assert(std::is_same_v<OtherSegment, SameSegment<CopyT>> ||
                         std::is_same_v<OtherSegment, SameSegment<OtherCopyT>>);
           other_vmo = other.vmo().borrow();
         }

         // Copy the base Segment type's fields first.  It cannot fail.
         WithVmo copy{*Segment::Copy(diag, static_cast<const Segment&>(other))};

         if (*other_vmo) {
           assert(copy.filesz() == other.filesz());
           zx_status_t status =  //
               CopyVmo(other_vmo->borrow(), 0, copy.filesz(), copy.vmo());
           if (status != ZX_OK) [[unlikely]] {
             return fit::error{SystemError(diag, copy, kCopyVmoFail, status)};
           }
         }

         return fit::ok(std::move(copy));
       }

      private:
       using OtherCopyT = std::integral_constant<bool, !CopyT::value>;

       template <class SomeCopyT>
       using SameSegment = typename Wrapper<SomeCopyT>::template WithVmo<Segment>;
     };

     // Use the wrapper if need be, or the original type if not.
     template <class Segment>
     using Type = std::conditional_t<kSegmentHasFilesz<Segment>, WithVmo<Segment>, Segment>;

     // This implements the VmarLoader::SegmentVmo class for LoadInfo types used
     // with SegmentWithVmo::Copy and SegmentWithVmo::NoCopy.  It's the base
     // class for the partial specialization of VmarLoader::SegmentVmo below.
     class SegmentVmo {
      public:
       SegmentVmo() = delete;

       template <class Segment>
       SegmentVmo(const Segment& segment, zx::unowned_vmo vmo)
           : vmo_{vmo->borrow()}, offset_{segment.offset()} {
         if constexpr (kSegmentHasFilesz<Segment>) {
           // This is not a ZeroFillSegment, so it might have a VMO stored.
           if (segment.vmo()) {
             // This VMO at offset 0 replaces the file VMO at segment.offset().
             vmo_ = segment.vmo().borrow();
             offset_ = 0;

             // In SegmentWithVmo::NoCopy, copy_on_write() is true only when
             // using the file VMO, not the stored VMO.
             copy_on_write_ = CopyFalse{};
           }
         }
       }

       zx::unowned_vmo vmo() const { return vmo_->borrow(); }

       constexpr auto copy_on_write() const { return copy_on_write_; }

       uint64_t offset() const { return offset_; }

      private:
       // In Copy, copy_on_write() is statically true all the time.  In NoCopy,
       // it's true by default but can be set to false in the constructor.
       using CopyTrue = std::conditional_t<CopyT{}, CopyT, std::true_type>;
       using CopyFalse = std::conditional_t<CopyT{}, CopyT, std::false_type>;

       zx::unowned_vmo vmo_;
       std::conditional_t<CopyT{}, CopyT, bool> copy_on_write_{CopyTrue{}};
       uint64_t offset_;
     };
   };

   template <class Segment>
   using Copy = typename Wrapper<std::true_type>::template Type<Segment>;
   using CopySegmentVmo = Wrapper<std::true_type>::SegmentVmo;

   template <class Segment>
   using NoCopy = Wrapper<std::false_type>::template Type<Segment>;
   using NoCopySegmentVmo = Wrapper<std::true_type>::SegmentVmo;

   // This takes a LoadInfo::*Segment type that has file contents (i.e. not
   // ZeroFillSegment), and ensures that segment.vmo() is a valid segment.
   // Unless there's a VMO handle there already, it creates a new copy-on-write
   // child VMO from the original file VMO.
   template <class Diagnostics, class Segment>
   static bool MakeMutable(Diagnostics& diag, Segment& segment, zx::unowned_vmo vmo) {
     if (!segment.vmo()) {
       zx_status_t status =
           CopyVmo(vmo->borrow(), segment.offset(), segment.filesz(), segment.vmo());
       if (status != ZX_OK) [[unlikely]] {
         return SystemError(diag, segment, kCopyVmoFail, status);
       }
     }
     return true;
   }

   // elfldltl::SegmentWithVmo::AlignSegments modifies a LoadInfo that uses the
   // SegmentWithVmo wrappers.  It ensures that all segment.filesz() values are
   // page-aligned, so that elfldltl::AlignedRemoteVmarLoader can be used later.
   // To start with, each segment must have no vmo() handle installed.  If a
   // segment has a misaligned filesz(), then a new copy-on-write child of the
   // main file VMO is installed as its vmo().  In the modified VMO, the partial
   // page has been cleared to all zero bytes.  The segment.filesz() has been
   // rounded up to whole pages so it can be mapped with no additional zeroing.
   //
   // With the optional readonly flag set, make each new per-segment VMO
   // immutable by replacing the segment .vmo() handle without ZX_RIGHT_WRITE.
   template <class Diagnostics, class LoadInfo>
   [[nodiscard]] static bool AlignSegments(Diagnostics& diag, LoadInfo& info, zx::unowned_vmo vmo,
                                           size_t page_size, bool readonly = false) {
     using DataWithZeroFillSegment = typename LoadInfo::DataWithZeroFillSegment;
     auto align_segment = [vmo, page_size, readonly, &diag](auto& segment) {
       using Segment = std::decay_t<decltype(segment)>;

       // When there's a partial page to zero, this will create a new VMO.  With
       // the readonly flag, we want to ensure this VMO won't be modified in the
       // future, so we want to drop ZX_VM_RIGHT_WRITE from our handle to it.
       // What's not obvious is that ZX_DEFAULT_VMO_RIGHTS &~ ZX_VM_RIGHT_WRITE
       // cannot be used here.  The handle from the VMO created in MakeMutable
       // will certainly have ZX_VM_RIGHT_WRITE, but it won't necessarily have
       // all the rights in ZX_DEFAULT_VMO_RIGHTS.  The zx_handle_replace system
       // call does not have a way to mask off rights, only choose a whole new
       // set of rights that must be a subset of the existing rights--so you
       // have to be sure what's truly a subset of the existing rights.
       //
       // We shouldn't presume exactly what rights the filesystem or loader
       // service or whatnot gave us on the file VMO handle.  The per-segment
       // VMOs are from zx_vmo_create_child (in CopyVmo, below), which returns
       // the new VMO via a handle with rights that depend on the original VMO
       // handle's rights.  So even though segment.vmo() is a handle we made
       // ourselves, we can't presume exactly what rights it has.
       //
       // So to drop ZX_RIGHT_WRITE for the readonly case below, we'd need to do
       // one of two things: do a ZX_INFO_HANDLE_BASIC query to find the rights
       // before masking off ZX_RIGHT_WRITE in zx_handle_replace; or, just use a
       // fixed set of rights that's small enough to be surely a subset of the
       // VMO rights we have (as derived from the file VMO handle's rights by
       // the behavior of zx_vmo_create_child).  Since we know how these
       // particular VMOs will need to be used later, we do the latter.
       constexpr zx_rights_t kReadonlyRights =
           ZX_RIGHTS_BASIC | ZX_RIGHTS_PROPERTY | ZX_RIGHT_MAP | ZX_RIGHT_READ;

       if constexpr (std::is_same_v<Segment, DataWithZeroFillSegment>) {
         const size_t zero_size = segment.MakeAligned(page_size);
         if (zero_size > 0) {
           ZX_DEBUG_ASSERT(!segment.vmo());
           if (!MakeMutable(diag, segment, vmo->borrow())) [[unlikely]] {
             return false;
           }
           const uint64_t zero_offset = segment.filesz() - zero_size;
           zx_status_t status = ZeroVmo(segment.vmo().borrow(), zero_offset, zero_size);
           if (status != ZX_OK) [[unlikely]] {
             return SystemError(diag, segment, kZeroVmoFail, status);
           }
           if (readonly) {
             status = segment.vmo().replace(kReadonlyRights, &segment.vmo());
             if (status != ZX_OK) [[unlikely]] {
               return SystemError(diag, segment, kProtectVmoFail, status);
             }
           }
         }
       }
       return true;
     };
     return info.VisitSegments(align_segment);
   }

   // elfldltl::SegmentWithVmo::GetMutableMemory must be instantiated with a
   // LoadInfo type using SegmentWithVmo wrapper types, and created with a
   // zx::unowned_vmo for the original file contents.  It can then be used to
   // construct an elfldltl::LoadInfoMutableMemory adapter object (see details
   // in <lib/elfldltl/loadinfo-mutable-memory.h>), which provides the mutation
   // calls in the Memory API.  When mutation requests are made on that Memory
   // object, the GetMutableMemory callback will be used to map in the mutable
   // VMO for the segment contents.  The mutable VMO is created via MakeMutable
   // if needed, and then stored in the LoadInfo::Segment object for later use.
   //
   // The callable object itself is copyable and default-constructible
   // (requiring later copy-assignment to be usable).  Note that it holds (and
   // copies) the unowned handles passed in the constructor, so users are
   // responsible for ensuring those handles remain valid for the lifetime of
   // the callable object.  The optional second argument to the constructor is a
   // zx::unowned_vmar used for making mappings, default zx::vmar::root_self().
   template <class LoadInfo>
   class GetMutableMemory {
    public:
     using Result = fit::result<bool, MappedVmoFile>;
     using Segment = typename LoadInfo::Segment;

     GetMutableMemory() = default;

     GetMutableMemory(const GetMutableMemory&) = default;

     explicit GetMutableMemory(zx::unowned_vmo vmo, zx::unowned_vmar vmar = zx::vmar::root_self())
         : vmo_{vmo->borrow()}, vmar_{vmar->borrow()} {}

     GetMutableMemory& operator=(const GetMutableMemory&) = default;

     template <class Diagnostics>
     Result operator()(Diagnostics& diag, Segment& segment) const {
       auto get_memory = [this, &diag](auto& segment) -> Result {
         using SegmentType = std::decay_t<decltype(segment)>;
         if constexpr (kSegmentHasFilesz<SegmentType>) {
           // Make sure there's a mutable VMO available.
           if (!MakeMutable(diag, segment, vmo_->borrow())) [[unlikely]] {
             return fit::error(false);
           }
           if (!segment.vmo()) [[unlikely]] {
             // MakeMutable failed but Diagnostics said to keep going.
             // No sense also reporting the failure to map the invalid handle.
             return fit::error{true};
           }

           // Now map it in for mutation.
           MappedVmoFile memory;
           if (auto result = memory.InitMutable(segment.vmo().borrow(), segment.filesz(),
                                                segment.vaddr(), vmar_->borrow());
               result.is_error()) {
             return fit::error{SystemError(diag, segment, kMapFail, result.error_value())};
           }

           return fit::ok(std::move(memory));
         } else {
           // This should be impossible via LoadInfoMutableMemory.
           return fit::error{diag.FormatError(kMutableZeroFill)};
         }
       };
       return std::visit(get_memory, segment);
     }

    private:
     zx::unowned_vmo vmo_;
     zx::unowned_vmar vmar_;
   };

  private:
   static constexpr std::string_view kCopyVmoFail =
       "cannot create copy-on-write VMO for segment contents";
   static constexpr std::string_view kZeroVmoFail =
       "cannot zero partial page in VMO for data segment";
   static constexpr std::string_view kProtectVmoFail =
       "cannot drop ZX_RIGHT_WRITE on VMO for data segment";
   static constexpr std::string_view kColonSpace = ": ";
   static constexpr std::string_view kMapFail = "cannot map segment to apply relocations";
   static constexpr std::string_view kMutableZeroFill = "cannot make zero-fill segment mutable";

   template <class Diagnostics, class Segment>
   static constexpr bool SystemError(Diagnostics& diag, const Segment& segment,
                                     std::string_view fail, zx_status_t status) {
     return diag.SystemError(fail, FileOffset{segment.offset()}, kColonSpace, ZirconError{status});
   }

   static zx_status_t CopyVmo(zx::unowned_vmo vmo, uint64_t offset, uint64_t size,
                              zx::vmo& segment_vmo) {
     return vmo->create_child(ZX_VMO_CHILD_SNAPSHOT_AT_LEAST_ON_WRITE, offset, size, &segment_vmo);
   }

   static zx_status_t ZeroVmo(zx::unowned_vmo segment_vmo, uint64_t offset, uint64_t size) {
     return segment_vmo->op_range(ZX_VMO_OP_ZERO, offset, size, nullptr, 0);
   }
 };

 // These are the SegmentVmo types used for LoadInfo<..., SegmentWithVmo::...>.
 // They use a segment.vmo() if it's present.
 //
 // Note that a specialization using a template template parameter matches only
 // that exact parameter, not even a template alias to the same thing.  So two
 // separate partial specializations are required here to match all LoadInfo
 // instantiations using either SegmentWithVmo::... template.

 template <class Elf, template <class> class Container, PhdrLoadPolicy Policy>
 class VmarLoader::SegmentVmo<LoadInfo<Elf, Container, Policy, SegmentWithVmo::Copy>>
     : public SegmentWithVmo::CopySegmentVmo {
   using SegmentWithVmo::CopySegmentVmo::CopySegmentVmo;
 };

 template <class Elf, template <class> class Container, PhdrLoadPolicy Policy>
 class VmarLoader::SegmentVmo<LoadInfo<Elf, Container, Policy, SegmentWithVmo::NoCopy>>
     : public SegmentWithVmo::NoCopySegmentVmo {
   using SegmentWithVmo::NoCopySegmentVmo::NoCopySegmentVmo;
 };

 }  // namespace elfldltl

 #endif  // SRC_LIB_ELFLDLTL_INCLUDE_LIB_ELFLDLTL_SEGMENT_WITH_VMO_H_
	// Copyright 2023 The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	#ifndef SRC_LIB_ELFLDLTL_INCLUDE_LIB_ELFLDLTL_SEGMENT_WITH_VMO_H_
	#define SRC_LIB_ELFLDLTL_INCLUDE_LIB_ELFLDLTL_SEGMENT_WITH_VMO_H_

	#include <lib/fit/result.h>
	#include <lib/zx/vmo.h>
	#include <zircon/assert.h>

	#include <type_traits>

	#include "diagnostics.h"
	#include "load.h"
	#include "mapped-vmo-file.h"
	#include "vmar-loader.h"
	#include "zircon.h"

	namespace elfldltl {

	// elfldltl::SegmentWithVmo::Copy and elfldltl::SegmentWithVmo::NoCopy can be
	// used as the SegmentWrapper template template parameter in elfldltl::LoadInfo
	// instantiations. Partial specializations of elfldltl::VmarLoader::SegmentVmo
	// provided below will then check segment.vmo() when mapping a segment. If
	// it's a valid handle, then the segment is mapped from that VMO at offset 0;
	// otherwise it's mapped from the main file VMO at segment.offset() as usual.
	//
	// The Copy version has the usual behavior of treating the VMO as read-only and
	// making a copy-on-write child VMO for a writable segment. The NoCopy version
	// will instead map segment.vmo() itself writable, so it can only be used once
	// as that VMO's contents will be modified by the process using it.
	//
	// The NoCopy case allows for simple out-of-process dynamic linking where
	// nothing is cached for reuse. The Copy case allows for the "zygote" model
	// where VMOs can be reused to get copy-on-write sharing of segments where
	// relocation has already been done.
	//
	// elfldltl::SegmentWithVmo::AlignSegments adjusts any LoadInfo segments that
	// have a partial page of data followed by zero-fill bytes so that all segments
	// can be mapped page-aligned without additional partial-page zeroing work (as
	// elfldltl::AlignedRemoteVmarLoader requires). To do this, it installs a
	// per-segment VMO that copies (via copy-on-write snapshot) the data and then
	// zero-fills the partial page directly in the VMO contents. Using this with
	// an elfldltl::SegmentWithVmo::NoCopy instantiation of elfldltl::LoadInfo
	// really just does the zeroing work ahead of time; elfldltl::RemoteVmarLoader
	// would do the same thing when mapping. An elfldltl::SegmentWithVmo::Copy
	// instantiation can be used either for a full zygote model where the
	// relocations are applied in the stored VMOs, or simply to share the zeroing
	// work across multiple separate loads with their own relocations applied to
	// their own copies (this may be beneficial to overall page-sharing if the
	// final data page didn't need to be touched for any relocations).
	//
	// elfldltl::SegmentWithVmo::MakeMutable can be applied to a particular segment
	// to install its own copied VMO. This is what out-of-process dynamic linking
	// must do before applying relocations that touch that segment's contents.
	//
	// The elfldltl::SegmentWithVmo::GetMutableMemory class is meant for use with
	// the elfldltl::LoadInfoMutableMemory adapter object.

	class SegmentWithVmo {
	public:
	// This becomes an additional base type along with the original Segment type.
	struct VmoHolder {
	VmoHolder() = default;

	VmoHolder(VmoHolder&&) = default;
	VmoHolder& operator=(VmoHolder&&) = default;

	explicit VmoHolder(zx::vmo vmo) : vmo_{std::move(vmo)} {}

	zx::vmo& vmo() { return vmo_; }
	const zx::vmo& vmo() const { return vmo_; }

	private:
	zx::vmo vmo_;
	};
	static_assert(std::is_move_constructible_v<VmoHolder>);
	static_assert(std::is_move_assignable_v<VmoHolder>);

	// The Copy and NoCopy templates are shorthands using this.
	template <class CopyT>
	struct Wrapper {
	// Segment types that have a filesz() are replaced with this subclass.
	// ZeroFillSegment never needs a VMO handle.
	template <class Segment>
	class WithVmo : public Segment, public VmoHolder {
	public:
	using Segment::Segment;

	// It's movable, though not copyable. The base Segment type is copyable.
	WithVmo(WithVmo&&) = default;

	// It's explicitly constructible from the base Segment type.
	explicit WithVmo(const Segment& other) : Segment{other} {}

	WithVmo& operator=(WithVmo&&) = default;

	// Assigning from the base Segment type drops any previous VMO.
	WithVmo& operator=(const Segment& other) {
	Segment::operator=(other);
	vmo().reset();
	}

	// Don't merge with any other segment if there's a VMO installed. Its
	// contents won't cover the other segment. Both segments will get the
	// reciprocal check, so if this segment has no VMO (yet) but the other
	// does, its CanMergeWith is the one to return false.
	template <class Other>
	bool CanMergeWith(const Other& other) const {
	static_assert(std::is_move_constructible_v<WithVmo>);
	static_assert(std::is_move_assignable_v<WithVmo>);
	return !vmo();
	}

	// Similar to `CanMergeWith`, this segment can't be replaced by another
	// segment if there's a VMO installed.
	bool CanReplace() const {
	static_assert(std::is_move_constructible_v<WithVmo>);
	static_assert(std::is_move_assignable_v<WithVmo>);
	return !vmo();
	}

	// This accepts the corresponding segment type either from the base
	// (unwrapped) LoadInfo::*Segment type or from a SegmentWithVmo wrapper
	// like this one. Both SegmentWithVmo::Copy and SegmentWithVmo::NoCopy
	// versions of the corresponding Segment subclass are accepted. The VMOs
	// are always cloned--the only distinction of NoCopy is what a VmarLoader
	// does with these new segment VMOs (NoCopy writes them in place).
	template <class Diagnostics, class OtherSegment>
	static fit::result<bool, WithVmo> Copy(Diagnostics& diag, const OtherSegment& other) {
	// If the other segment has its own VMO, then the copied segment
	// needs a copy-on-write clone of that whole VMO.
	zx::unowned_vmo other_vmo;

	// For copy from the unadorned Segment, there's never a VMO to copy.
	if constexpr (!std::is_same_v<OtherSegment, Segment>) {
	// Copy from the corresponding Segment subclass of this wrapper is
	// the other supported option, to clone the existing VMO, if any.
	static_assert(std::is_same_v<OtherSegment, SameSegment<CopyT>> \|\|
	std::is_same_v<OtherSegment, SameSegment<OtherCopyT>>);
	other_vmo = other.vmo().borrow();
	}

	// Copy the base Segment type's fields first. It cannot fail.
	WithVmo copy{*Segment::Copy(diag, static_cast<const Segment&>(other))};

	if (*other_vmo) {
	assert(copy.filesz() == other.filesz());
	zx_status_t status = //
	CopyVmo(other_vmo->borrow(), 0, copy.filesz(), copy.vmo());
	if (status != ZX_OK) [[unlikely]] {
	return fit::error{SystemError(diag, copy, kCopyVmoFail, status)};
	}
	}

	return fit::ok(std::move(copy));
	}

	private:
	using OtherCopyT = std::integral_constant<bool, !CopyT::value>;

	template <class SomeCopyT>
	using SameSegment = typename Wrapper<SomeCopyT>::template WithVmo<Segment>;
	};

	// Use the wrapper if need be, or the original type if not.
	template <class Segment>
	using Type = std::conditional_t<kSegmentHasFilesz<Segment>, WithVmo<Segment>, Segment>;

	// This implements the VmarLoader::SegmentVmo class for LoadInfo types used
	// with SegmentWithVmo::Copy and SegmentWithVmo::NoCopy. It's the base
	// class for the partial specialization of VmarLoader::SegmentVmo below.
	class SegmentVmo {
	public:
	SegmentVmo() = delete;

	template <class Segment>
	SegmentVmo(const Segment& segment, zx::unowned_vmo vmo)
	: vmo_{vmo->borrow()}, offset_{segment.offset()} {
	if constexpr (kSegmentHasFilesz<Segment>) {
	// This is not a ZeroFillSegment, so it might have a VMO stored.
	if (segment.vmo()) {
	// This VMO at offset 0 replaces the file VMO at segment.offset().
	vmo_ = segment.vmo().borrow();
	offset_ = 0;

	// In SegmentWithVmo::NoCopy, copy_on_write() is true only when
	// using the file VMO, not the stored VMO.
	copy_on_write_ = CopyFalse{};
	}
	}
	}

	zx::unowned_vmo vmo() const { return vmo_->borrow(); }

	constexpr auto copy_on_write() const { return copy_on_write_; }

	uint64_t offset() const { return offset_; }

	private:
	// In Copy, copy_on_write() is statically true all the time. In NoCopy,
	// it's true by default but can be set to false in the constructor.
	using CopyTrue = std::conditional_t<CopyT{}, CopyT, std::true_type>;
	using CopyFalse = std::conditional_t<CopyT{}, CopyT, std::false_type>;

	zx::unowned_vmo vmo_;
	std::conditional_t<CopyT{}, CopyT, bool> copy_on_write_{CopyTrue{}};
	uint64_t offset_;
	};
	};

	template <class Segment>
	using Copy = typename Wrapper<std::true_type>::template Type<Segment>;
	using CopySegmentVmo = Wrapper<std::true_type>::SegmentVmo;

	template <class Segment>
	using NoCopy = Wrapper<std::false_type>::template Type<Segment>;
	using NoCopySegmentVmo = Wrapper<std::true_type>::SegmentVmo;

	// This takes a LoadInfo::*Segment type that has file contents (i.e. not
	// ZeroFillSegment), and ensures that segment.vmo() is a valid segment.
	// Unless there's a VMO handle there already, it creates a new copy-on-write
	// child VMO from the original file VMO.
	template <class Diagnostics, class Segment>
	static bool MakeMutable(Diagnostics& diag, Segment& segment, zx::unowned_vmo vmo) {
	if (!segment.vmo()) {
	zx_status_t status =
	CopyVmo(vmo->borrow(), segment.offset(), segment.filesz(), segment.vmo());
	if (status != ZX_OK) [[unlikely]] {
	return SystemError(diag, segment, kCopyVmoFail, status);
	}
	}
	return true;
	}

	// elfldltl::SegmentWithVmo::AlignSegments modifies a LoadInfo that uses the
	// SegmentWithVmo wrappers. It ensures that all segment.filesz() values are
	// page-aligned, so that elfldltl::AlignedRemoteVmarLoader can be used later.
	// To start with, each segment must have no vmo() handle installed. If a
	// segment has a misaligned filesz(), then a new copy-on-write child of the
	// main file VMO is installed as its vmo(). In the modified VMO, the partial
	// page has been cleared to all zero bytes. The segment.filesz() has been
	// rounded up to whole pages so it can be mapped with no additional zeroing.
	//
	// With the optional readonly flag set, make each new per-segment VMO
	// immutable by replacing the segment .vmo() handle without ZX_RIGHT_WRITE.
	template <class Diagnostics, class LoadInfo>
	[[nodiscard]] static bool AlignSegments(Diagnostics& diag, LoadInfo& info, zx::unowned_vmo vmo,
	size_t page_size, bool readonly = false) {
	using DataWithZeroFillSegment = typename LoadInfo::DataWithZeroFillSegment;
	auto align_segment = [vmo, page_size, readonly, &diag](auto& segment) {
	using Segment = std::decay_t<decltype(segment)>;

	// When there's a partial page to zero, this will create a new VMO. With
	// the readonly flag, we want to ensure this VMO won't be modified in the
	// future, so we want to drop ZX_VM_RIGHT_WRITE from our handle to it.
	// What's not obvious is that ZX_DEFAULT_VMO_RIGHTS &~ ZX_VM_RIGHT_WRITE
	// cannot be used here. The handle from the VMO created in MakeMutable
	// will certainly have ZX_VM_RIGHT_WRITE, but it won't necessarily have
	// all the rights in ZX_DEFAULT_VMO_RIGHTS. The zx_handle_replace system
	// call does not have a way to mask off rights, only choose a whole new
	// set of rights that must be a subset of the existing rights--so you
	// have to be sure what's truly a subset of the existing rights.
	//
	// We shouldn't presume exactly what rights the filesystem or loader
	// service or whatnot gave us on the file VMO handle. The per-segment
	// VMOs are from zx_vmo_create_child (in CopyVmo, below), which returns
	// the new VMO via a handle with rights that depend on the original VMO
	// handle's rights. So even though segment.vmo() is a handle we made
	// ourselves, we can't presume exactly what rights it has.
	//
	// So to drop ZX_RIGHT_WRITE for the readonly case below, we'd need to do
	// one of two things: do a ZX_INFO_HANDLE_BASIC query to find the rights
	// before masking off ZX_RIGHT_WRITE in zx_handle_replace; or, just use a
	// fixed set of rights that's small enough to be surely a subset of the
	// VMO rights we have (as derived from the file VMO handle's rights by
	// the behavior of zx_vmo_create_child). Since we know how these
	// particular VMOs will need to be used later, we do the latter.
	constexpr zx_rights_t kReadonlyRights =
	ZX_RIGHTS_BASIC \| ZX_RIGHTS_PROPERTY \| ZX_RIGHT_MAP \| ZX_RIGHT_READ;

	if constexpr (std::is_same_v<Segment, DataWithZeroFillSegment>) {
	const size_t zero_size = segment.MakeAligned(page_size);
	if (zero_size > 0) {
	ZX_DEBUG_ASSERT(!segment.vmo());
	if (!MakeMutable(diag, segment, vmo->borrow())) [[unlikely]] {
	return false;
	}
	const uint64_t zero_offset = segment.filesz() - zero_size;
	zx_status_t status = ZeroVmo(segment.vmo().borrow(), zero_offset, zero_size);
	if (status != ZX_OK) [[unlikely]] {
	return SystemError(diag, segment, kZeroVmoFail, status);
	}
	if (readonly) {
	status = segment.vmo().replace(kReadonlyRights, &segment.vmo());
	if (status != ZX_OK) [[unlikely]] {
	return SystemError(diag, segment, kProtectVmoFail, status);
	}
	}
	}
	}
	return true;
	};
	return info.VisitSegments(align_segment);
	}

	// elfldltl::SegmentWithVmo::GetMutableMemory must be instantiated with a
	// LoadInfo type using SegmentWithVmo wrapper types, and created with a
	// zx::unowned_vmo for the original file contents. It can then be used to
	// construct an elfldltl::LoadInfoMutableMemory adapter object (see details
	// in <lib/elfldltl/loadinfo-mutable-memory.h>), which provides the mutation
	// calls in the Memory API. When mutation requests are made on that Memory
	// object, the GetMutableMemory callback will be used to map in the mutable
	// VMO for the segment contents. The mutable VMO is created via MakeMutable
	// if needed, and then stored in the LoadInfo::Segment object for later use.
	//
	// The callable object itself is copyable and default-constructible
	// (requiring later copy-assignment to be usable). Note that it holds (and
	// copies) the unowned handles passed in the constructor, so users are
	// responsible for ensuring those handles remain valid for the lifetime of
	// the callable object. The optional second argument to the constructor is a
	// zx::unowned_vmar used for making mappings, default zx::vmar::root_self().
	template <class LoadInfo>
	class GetMutableMemory {
	public:
	using Result = fit::result<bool, MappedVmoFile>;
	using Segment = typename LoadInfo::Segment;

	GetMutableMemory() = default;

	GetMutableMemory(const GetMutableMemory&) = default;

	explicit GetMutableMemory(zx::unowned_vmo vmo, zx::unowned_vmar vmar = zx::vmar::root_self())
	: vmo_{vmo->borrow()}, vmar_{vmar->borrow()} {}

	GetMutableMemory& operator=(const GetMutableMemory&) = default;

	template <class Diagnostics>
	Result operator()(Diagnostics& diag, Segment& segment) const {
	auto get_memory = [this, &diag](auto& segment) -> Result {
	using SegmentType = std::decay_t<decltype(segment)>;
	if constexpr (kSegmentHasFilesz<SegmentType>) {
	// Make sure there's a mutable VMO available.
	if (!MakeMutable(diag, segment, vmo_->borrow())) [[unlikely]] {
	return fit::error(false);
	}
	if (!segment.vmo()) [[unlikely]] {
	// MakeMutable failed but Diagnostics said to keep going.
	// No sense also reporting the failure to map the invalid handle.
	return fit::error{true};
	}

	// Now map it in for mutation.
	MappedVmoFile memory;
	if (auto result = memory.InitMutable(segment.vmo().borrow(), segment.filesz(),
	segment.vaddr(), vmar_->borrow());
	result.is_error()) {
	return fit::error{SystemError(diag, segment, kMapFail, result.error_value())};
	}

	return fit::ok(std::move(memory));
	} else {
	// This should be impossible via LoadInfoMutableMemory.
	return fit::error{diag.FormatError(kMutableZeroFill)};
	}
	};
	return std::visit(get_memory, segment);
	}

	private:
	zx::unowned_vmo vmo_;
	zx::unowned_vmar vmar_;
	};

	private:
	static constexpr std::string_view kCopyVmoFail =
	"cannot create copy-on-write VMO for segment contents";
	static constexpr std::string_view kZeroVmoFail =
	"cannot zero partial page in VMO for data segment";
	static constexpr std::string_view kProtectVmoFail =
	"cannot drop ZX_RIGHT_WRITE on VMO for data segment";
	static constexpr std::string_view kColonSpace = ": ";
	static constexpr std::string_view kMapFail = "cannot map segment to apply relocations";
	static constexpr std::string_view kMutableZeroFill = "cannot make zero-fill segment mutable";

	template <class Diagnostics, class Segment>
	static constexpr bool SystemError(Diagnostics& diag, const Segment& segment,
	std::string_view fail, zx_status_t status) {
	return diag.SystemError(fail, FileOffset{segment.offset()}, kColonSpace, ZirconError{status});
	}

	static zx_status_t CopyVmo(zx::unowned_vmo vmo, uint64_t offset, uint64_t size,
	zx::vmo& segment_vmo) {
	return vmo->create_child(ZX_VMO_CHILD_SNAPSHOT_AT_LEAST_ON_WRITE, offset, size, &segment_vmo);
	}

	static zx_status_t ZeroVmo(zx::unowned_vmo segment_vmo, uint64_t offset, uint64_t size) {
	return segment_vmo->op_range(ZX_VMO_OP_ZERO, offset, size, nullptr, 0);
	}
	};

	// These are the SegmentVmo types used for LoadInfo<..., SegmentWithVmo::...>.
	// They use a segment.vmo() if it's present.
	//
	// Note that a specialization using a template template parameter matches only
	// that exact parameter, not even a template alias to the same thing. So two
	// separate partial specializations are required here to match all LoadInfo
	// instantiations using either SegmentWithVmo::... template.

	template <class Elf, template <class> class Container, PhdrLoadPolicy Policy>
	class VmarLoader::SegmentVmo<LoadInfo<Elf, Container, Policy, SegmentWithVmo::Copy>>
	: public SegmentWithVmo::CopySegmentVmo {
	using SegmentWithVmo::CopySegmentVmo::CopySegmentVmo;
	};

	template <class Elf, template <class> class Container, PhdrLoadPolicy Policy>
	class VmarLoader::SegmentVmo<LoadInfo<Elf, Container, Policy, SegmentWithVmo::NoCopy>>
	: public SegmentWithVmo::NoCopySegmentVmo {
	using SegmentWithVmo::NoCopySegmentVmo::NoCopySegmentVmo;
	};

	} // namespace elfldltl

	#endif // SRC_LIB_ELFLDLTL_INCLUDE_LIB_ELFLDLTL_SEGMENT_WITH_VMO_H_