src/tests/microbenchmarks/vmo.cc - fuchsia - Git at Google

 // Copyright 2017 The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 #include <lib/zx/vmar.h>
 #include <lib/zx/vmo.h>

 #include <vector>

 #include <fbl/string_printf.h>
 #include <perftest/perftest.h>

 #include "assert.h"

 // This file contains various test cases that measure the cost of copying to/from a VMO, effectively
 // measuring the cost of a memcpy() for different cases. These include:
 //  * Vmo/Read, Vmo/Write: cost of copying from/to a VMO with zx_vmo_read()/zx_vmo_write(). The
 //    operated-on VMO is already mapped, with page table entries populated.
 //    * Subcase: Vmo/Write/ZeroPage: cost of zx_vmo_write() when the memory being read from is the
 //      shared zero page (as implemented by the kernel).
 //  * VmoMap/Read, VmoMap/Write: cost of mapping a VMO and then copying it. The operated-on VMO
 //    already has its pages committed.
 //    * Subcase: "/Kernel" variants use zx_vmo_read()/zx_vmo_write() to copy to/from the VMO; other
 //      variants use memcpy() in userland.
 //    * Subcase: VmoMapRange: uses ZX_VM_MAP_RANGE so that the map operation pre-populates the page
 //      table entries for the mappings.
 //  * Vmo/Memcpy: cost of creating a VMO, mapping it, then copying it using memcpy().
 //    * Subcase: "/WithPrecommit" variants use ZX_VMAR_OP_COMMIT to map and commit the VMO prior to
 //      issuing a memcpy.
 //    * Subcase: "/WithoutPrecommit" variants perform a memcpy into the VMO without committing it.
 //  * Vmo/TransferData: cost of zx_vmo_transfer_data().

 namespace {

 // Measure the time taken to write or read a chunk of data to/from a VMO using the zx_vmo_write() or
 // zx_vmo_read() syscalls respectively. If |do_write| and |zero_page| are true, this measures the
 // time to do a zx_vmo_write() that copies from a buffer that maps to the kernel's shared zero page
 // into the VMO. One reason for testing this case is that this uses a different code path in the
 // kernel than if non-zero pages were used. For multi-page buffers, it will also read fewer pages of
 // physical memory.
 bool VmoReadOrWriteTest(perftest::RepeatState* state, uint32_t copy_size, bool do_write,
                         bool zero_page) {
   if (zero_page) {
     // This is the only meaningful combination. See comments where the test is registered below.
     ZX_ASSERT(do_write);
   }

   zx::vmo vmo;
   ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo));

   // Use a vmo as the buffer to read from / write to, so we can exactly control whether we're using
   // distinct physical pages or the singleton zero page.
   zx::vmo buffer_vmo;
   ASSERT_OK(zx::vmo::create(copy_size, 0, &buffer_vmo));
   zx_vaddr_t buffer_addr;
   ASSERT_OK(zx::vmar::root_self()->map(ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, buffer_vmo, 0,
                                        copy_size, &buffer_addr));

   // If |zero_page| is not specified, memset to non-zero to make sure buffer_vmo's pages are
   // populated and not eligible for zero page deduping, otherwise let the kernel fault in the zero
   // page as required.
   //
   // This can alter the runtime of the vmo write below. If |zero_page| is true, for vmo
   // write, the buffer is being read from, so we will just use the singleton zero page.
   //
   // Also when performing page lookups in the vmo to retrieve backing pages, the logic in the kernel
   // for handling distinct physical pages differs from the zero page.
   if (!zero_page) {
     memset(reinterpret_cast<void*>(buffer_addr), 0xa, copy_size);
   }

   // Write the VMO so that the pages are pre-committed. This matters
   // more for the read case.
   ASSERT_OK(vmo.write(reinterpret_cast<void*>(buffer_addr), 0, copy_size));

   if (do_write) {
     while (state->KeepRunning()) {
       ASSERT_OK(vmo.write(reinterpret_cast<void*>(buffer_addr), 0, copy_size));
     }
   } else {
     while (state->KeepRunning()) {
       ASSERT_OK(vmo.read(reinterpret_cast<void*>(buffer_addr), 0, copy_size));
     }
   }

   ASSERT_OK(zx::vmar::root_self()->unmap(buffer_addr, copy_size));
   return true;
 }

 // Measure the time taken to write or read a chunk of data to/from a mapped VMO. The writing/reading
 // is either done from userland using memcpy() (when user_memcpy=true) or by the kernel using
 // zx_vmo_read()/zx_vmo_write() (when user_memcpy=false).
 bool VmoReadOrWriteMapTestImpl(perftest::RepeatState* state, uint32_t copy_size, bool do_write,
                                int flags, bool user_memcpy) {
   zx::vmo vmo;
   ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo));
   std::vector<char> buffer(copy_size);
   zx_vaddr_t mapped_addr;

   zx::vmo vmo_buf;
   if (!user_memcpy) {
     // Create a temporary VMO that we can use to get the kernel to read/write our mapped memory.
     ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo_buf));
   }

   // Write the VMO so that the pages are pre-committed. This matters
   // more for the read case.
   ASSERT_OK(vmo.write(buffer.data(), 0, copy_size));

   if (do_write) {
     while (state->KeepRunning()) {
       ASSERT_OK(zx::vmar::root_self()->map(ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | flags, 0, vmo, 0,
                                            copy_size, &mapped_addr));
       if (user_memcpy) {
         std::memcpy(reinterpret_cast<void*>(mapped_addr), buffer.data(), copy_size);
       } else {
         // To write to the mapped in portion we *read* from the temporary VMO.
         ASSERT_OK(vmo_buf.read(reinterpret_cast<void*>(mapped_addr), 0, copy_size));
       }
       ASSERT_OK(zx::vmar::root_self()->unmap(mapped_addr, copy_size));
     }
   } else {  // read
     while (state->KeepRunning()) {
       ASSERT_OK(zx::vmar::root_self()->map(ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | flags, 0, vmo, 0,
                                            copy_size, &mapped_addr));
       if (user_memcpy) {
         std::memcpy(buffer.data(), reinterpret_cast<void*>(mapped_addr), copy_size);
       } else {
         // To read from the mapped in portion we *write* it to the temporary VMO.
         ASSERT_OK(vmo_buf.write(reinterpret_cast<void*>(mapped_addr), 0, copy_size));
       }
       ASSERT_OK(zx::vmar::root_self()->unmap(mapped_addr, copy_size));
     }
   }
   return true;
 }

 bool VmoReadOrWriteMapTest(perftest::RepeatState* state, uint32_t copy_size, bool do_write,
                            bool user_memcpy) {
   return VmoReadOrWriteMapTestImpl(state, copy_size, do_write, 0, user_memcpy);
 }

 bool VmoReadOrWriteMapRangeTest(perftest::RepeatState* state, uint32_t copy_size, bool do_write,
                                 bool user_memcpy) {
   return VmoReadOrWriteMapTestImpl(state, copy_size, do_write, ZX_VM_MAP_RANGE, user_memcpy);
 }

 // Measure the time taken to create a VMO, map it into the root VMAR, optionally commit and
 // map the pages, memcpy data into the VMO, then unmap and destroy the VMO. This is used as
 // an indirect way to measure the overhead induced by page faulting during a memcpy.
 bool VmoMemcpyPrecommitTest(perftest::RepeatState* state, uint32_t size, bool precommit) {
   state->DeclareStep("create_and_map_vmo");
   if (precommit) {
     state->DeclareStep("precommit");
   }
   state->DeclareStep("memcpy");
   state->DeclareStep("unmap_and_destroy_vmo");

   // Set up a source buffer and initialize it.
   std::unique_ptr<char[]> src(new char[size]);
   memset(src.get(), 0xff, size);

   while (state->KeepRunning()) {
     // Create and map the destination VMO.
     zx::vmo dst_vmo;
     ASSERT_OK(zx::vmo::create(size, 0, &dst_vmo));
     zx_vaddr_t dst;
     ASSERT_OK(
         zx::vmar::root_self()->map(ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, dst_vmo, 0, size, &dst));
     state->NextStep();

     // Commit the destination VMO if we're running the precommit case.
     if (precommit) {
       ASSERT_OK(zx::vmar::root_self()->op_range(ZX_VMAR_OP_COMMIT, dst, size, 0, 0));
       state->NextStep();
     }

     // Memcpy from source into dst.
     memcpy(reinterpret_cast<void*>(dst), src.get(), size);
     state->NextStep();

     // Unmap the destination VMO.
     ASSERT_OK(zx::vmar::root_self()->unmap(dst, size));
   }
   return true;
 }

 // Measure the time taken to clone a vmo and destroy it. If map_size is non zero, then this function
 // tests the case where the original vmo is mapped in chunks of map_size; otherwise it tests the
 // case where the original vmo is not mapped.
 bool VmoCloneTest(perftest::RepeatState* state, uint32_t copy_size, uint32_t map_size) {
   if (map_size > 0) {
     state->DeclareStep("map");
   }
   state->DeclareStep("clone");
   state->DeclareStep("close");
   if (map_size > 0) {
     state->DeclareStep("unmap");
   }

   zx::vmo vmo;
   ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo));
   ASSERT_OK(vmo.op_range(ZX_VMO_OP_COMMIT, 0, copy_size, nullptr, 0));

   zx::vmar vmar;
   zx_vaddr_t addr = 0;
   // Allocate a single vmar so we have a single reserved block if mapping in using multiple chunks.
   ASSERT_OK(zx::vmar::root_self()->allocate(
       ZX_VM_CAN_MAP_SPECIFIC | ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE, 0, copy_size, &vmar,
       &addr));

   while (state->KeepRunning()) {
     if (map_size > 0) {
       zx_vaddr_t chunk_addr;
       for (uint32_t off = 0; off < copy_size; off += map_size) {
         ASSERT_OK(vmar.map(ZX_VM_MAP_RANGE | ZX_VM_PERM_READ | ZX_VM_SPECIFIC, off, vmo, off,
                            map_size, &chunk_addr));
       }
       state->NextStep();
     }

     zx::vmo clone;
     ASSERT_OK(vmo.create_child(ZX_VMO_CHILD_SNAPSHOT, 0, copy_size, &clone));
     state->NextStep();

     clone.reset();

     if (map_size > 0) {
       state->NextStep();
       ASSERT_OK(vmar.unmap(addr, copy_size));
     }
   }

   return true;
 }

 // Measure the time taken to create a clone, map, unmap and then destroy it.
 bool VmoMapCloneTest(perftest::RepeatState* state, uint32_t copy_size) {
   state->DeclareStep("clone");
   state->DeclareStep("map");
   state->DeclareStep("unmap");
   state->DeclareStep("close");

   zx::vmo vmo;
   ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo));
   // Fully commit the parent vmo's pages, so that the clone mapping has backing pages to map in.
   ASSERT_OK(vmo.op_range(ZX_VMO_OP_COMMIT, 0, copy_size, nullptr, 0));

   while (state->KeepRunning()) {
     zx::vmo clone;
     ASSERT_OK(vmo.create_child(ZX_VMO_CHILD_SNAPSHOT, 0, copy_size, &clone));
     state->NextStep();

     zx_vaddr_t addr = 0;
     // ZX_VM_MAP_RANGE will fully populate the mapping.
     ASSERT_OK(zx::vmar::root_self()->map(ZX_VM_MAP_RANGE | ZX_VM_PERM_READ, 0, clone, 0, copy_size,
                                          &addr));
     state->NextStep();

     ASSERT_OK(zx::vmar::root_self()->unmap(addr, copy_size));
     state->NextStep();

     clone.reset();
   }

   return true;
 }

 // Measure the time it takes to clone a vmo. Specifically, this measures:
 //   - Clone a vmo.
 //   - Read or write either the original vmo (do_target_clone=false) or the
 //     clone (do_target_clone=true).
 //     - For bidirectional clones, we don't expect varying do_target_clone to
 //       significantly affect this performance.
 //     - do_full_op controls whether we read or write the whole vmo or just
 //       a subset of the pages, as the performance characteristics of a
 //       partially populated clone and a fully populated clone can differ.
 //   - Destroy the clone.
 bool VmoCloneReadOrWriteTest(perftest::RepeatState* state, uint32_t copy_size, bool do_write,
                              bool do_target_clone, bool do_full_op) {
   const size_t kPageSize = zx_system_get_page_size();
   state->DeclareStep("clone");
   state->DeclareStep(do_write ? "write" : "read");
   state->DeclareStep("close");

   zx::vmo vmo;
   ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo));
   ASSERT_OK(vmo.op_range(ZX_VMO_OP_COMMIT, 0, copy_size, nullptr, 0));

   std::vector<char> buffer(copy_size);

   while (state->KeepRunning()) {
     zx::vmo clone;
     ASSERT_OK(vmo.create_child(ZX_VMO_CHILD_SNAPSHOT, 0, copy_size, &clone));
     state->NextStep();

     const zx::vmo& target = do_target_clone ? clone : vmo;
     if (do_full_op) {
       if (do_write) {
         ASSERT_OK(target.write(buffer.data(), 0, copy_size));
       } else {
         ASSERT_OK(target.read(buffer.data(), 0, copy_size));
       }
     } else {
       // There's no special meaning behind the particular value of this
       // constant. It just needs to result in a couple of writes into
       // the vmo without populating it too densely.
       const uint64_t kWriteInterval = 8 * kPageSize;
       for (uint64_t offset = 0; offset < copy_size; offset += kWriteInterval) {
         if (do_write) {
           ASSERT_OK(target.write(buffer.data(), offset, kPageSize));
         } else {
           ASSERT_OK(target.read(buffer.data(), offset, kPageSize));
         }
       }
     }

     state->NextStep();
     // The clone goes out of scope and is implicitly closed.
   }

   return true;
 }

 // Measure the times taken to create, write and then read some data from a VMO on a single thread.
 // This is used to measure the performance of a brand new VMO's entire lifecycle up to data read
 // completion time. This test is useful because this is essentially what users of `fuchsia.mem.Data`
 // or `fuchsia.mem.Buffer` must do on top of their default zx.channel write/read operations. It's
 // worth measuring these operations together (when they are also tested separately) because we
 // expect them to have different performance behavior together.
 //
 // The zx_vmo_write() call will cause pages to be allocated in the VMO, and closing the VMO handle
 // will free those pages.
 bool VmoCreateWriteReadCloseTest(perftest::RepeatState* state, uint32_t copy_size) {
   state->DeclareStep("create");
   state->DeclareStep("write");
   state->DeclareStep("read");
   state->DeclareStep("close");

   // Use a vmo as the buffer to read from / write to.
   zx::vmo buffer_vmo;
   ASSERT_OK(zx::vmo::create(copy_size, 0, &buffer_vmo));
   zx_vaddr_t buffer_addr;
   ASSERT_OK(zx::vmar::root_self()->map(ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, buffer_vmo, 0,
                                        copy_size, &buffer_addr));
   memset(reinterpret_cast<void*>(buffer_addr), 0xa, copy_size);

   while (state->KeepRunning()) {
     zx::vmo vmo;
     ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo));
     state->NextStep();
     ASSERT_OK(vmo.write(reinterpret_cast<void*>(buffer_addr), 0, copy_size));
     state->NextStep();
     ASSERT_OK(vmo.read(reinterpret_cast<void*>(buffer_addr), 0, copy_size));
     state->NextStep();
   }

   ASSERT_OK(zx::vmar::root_self()->unmap(buffer_addr, copy_size));
   return true;
 }

 // Measure the time taken to transfer data between VMOs using zx_vmo_transfer_data().
 bool VmoTransferDataTest(perftest::RepeatState* state, uint64_t transfer_size) {
   const uint64_t vmo_size = transfer_size * 2;
   const uint64_t src_offset = transfer_size;
   const uint64_t dst_offset = 0;
   zx::vmo src_vmo;
   ASSERT_OK(zx::vmo::create(vmo_size, 0, &src_vmo));
   zx_vaddr_t buffer_addr;
   ASSERT_OK(zx::vmar::root_self()->map(ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, src_vmo, 0, vmo_size,
                                        &buffer_addr));
   zx::vmo dst_vmo;
   ASSERT_OK(zx::vmo::create(vmo_size, 0, &dst_vmo));

   state->DeclareStep("fill_source");
   state->DeclareStep("transfer");
   while (state->KeepRunning()) {
     memset((void*)buffer_addr, 'f', vmo_size);
     state->NextStep();
     ASSERT_OK(dst_vmo.transfer_data(0, dst_offset, transfer_size, &src_vmo, src_offset));
   }
   ASSERT_OK(zx::vmar::root_self()->unmap(buffer_addr, vmo_size));
   return true;
 }

 template <typename Func, typename... Args>
 void RegisterVmoTest(const char* name, Func fn, Args... args) {
   for (unsigned size_in_kbytes : {4, 32, 128, 512, 2048}) {
     auto full_name = fbl::StringPrintf("%s/%ukbytes", name, size_in_kbytes);
     perftest::RegisterTest(full_name.c_str(), fn, size_in_kbytes * 1024, args...);
   }
 }

 void RegisterTests() {
   for (bool do_write : {false, true}) {
     for (bool zero : {false, true}) {
       // The zero case for vmo read is not meaningful since it will only operate on the zero page in
       // the first iteration; the remaining iterations will use forked pages which is equivalent to
       // the non-zero case. Skip this combo.
       if (zero && !do_write) {
         continue;
       }
       const char* rw = do_write ? "Write" : "Read";
       const char* z = zero ? "/ZeroPage" : "";
       auto rw_name = fbl::StringPrintf("Vmo/%s%s", rw, z);
       RegisterVmoTest(rw_name.c_str(), VmoReadOrWriteTest, do_write, zero);
     }
   }

   for (bool do_write : {false, true}) {
     for (bool user_memcpy : {false, true}) {
       const char* rw = do_write ? "Write" : "Read";
       const char* user_kernel = user_memcpy ? "" : "/Kernel";

       auto rw_name = fbl::StringPrintf("VmoMap/%s%s", rw, user_kernel);
       RegisterVmoTest(rw_name.c_str(), VmoReadOrWriteMapTest, do_write, user_memcpy);

       rw_name = fbl::StringPrintf("VmoMapRange/%s%s", rw, user_kernel);
       RegisterVmoTest(rw_name.c_str(), VmoReadOrWriteMapRangeTest, do_write, user_memcpy);
     }
   }

   for (bool precommit : {false, true}) {
     const char* pc = precommit ? "WithPrecommit" : "WithoutPrecommit";
     auto precommit_name = fbl::StringPrintf("Vmo/Memcpy/%s", pc);
     RegisterVmoTest(precommit_name.c_str(), VmoMemcpyPrecommitTest, precommit);
   }

   for (bool map : {false, true}) {
     auto clone_name = fbl::StringPrintf("Vmo/Clone%s", map ? "/MapParent" : "");
     RegisterVmoTest(clone_name.c_str(), [map](perftest::RepeatState* state, uint32_t size) {
       return VmoCloneTest(state, size, map ? size : 0);
     });
   }

   for (unsigned map_chunk_kb : {4, 64, 2048, 32768}) {
     constexpr uint32_t vmo_size_kb = 32768;
     auto name = fbl::StringPrintf("Vmo/Clone/MapParent%usegments/%ukbytes",
                                   vmo_size_kb / map_chunk_kb, vmo_size_kb);
     perftest::RegisterTest(name.c_str(), VmoCloneTest, vmo_size_kb * 1024, map_chunk_kb * 1024);
   }

   auto name = fbl::StringPrintf("Vmo/MapClone");
   RegisterVmoTest(name.c_str(), VmoMapCloneTest);

   for (bool do_write : {false, true}) {
     for (bool do_target_clone : {false, true}) {
       for (bool do_full_op : {false, true}) {
         const char* rw = do_write ? "Write" : "Read";
         const char* target = do_target_clone ? "Clone" : "Orig";
         const char* density = do_full_op ? "All" : "Some";
         auto clone_rw_name = fbl::StringPrintf("Vmo/Clone/%s%s%s", rw, target, density);
         RegisterVmoTest(clone_rw_name.c_str(), VmoCloneReadOrWriteTest, do_write, do_target_clone,
                         do_full_op);
       }
     }
   }

   name = fbl::StringPrintf("Vmo/CreateWriteReadClose");
   RegisterVmoTest(name.c_str(), VmoCreateWriteReadCloseTest);

   name = fbl::StringPrintf("Vmo/TransferData");
   RegisterVmoTest(name.c_str(), VmoTransferDataTest);
 }
 PERFTEST_CTOR(RegisterTests)

 }  // namespace
	// Copyright 2017 The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	#include <lib/zx/vmar.h>
	#include <lib/zx/vmo.h>

	#include <vector>

	#include <fbl/string_printf.h>
	#include <perftest/perftest.h>

	#include "assert.h"

	// This file contains various test cases that measure the cost of copying to/from a VMO, effectively
	// measuring the cost of a memcpy() for different cases. These include:
	// * Vmo/Read, Vmo/Write: cost of copying from/to a VMO with zx_vmo_read()/zx_vmo_write(). The
	// operated-on VMO is already mapped, with page table entries populated.
	// * Subcase: Vmo/Write/ZeroPage: cost of zx_vmo_write() when the memory being read from is the
	// shared zero page (as implemented by the kernel).
	// * VmoMap/Read, VmoMap/Write: cost of mapping a VMO and then copying it. The operated-on VMO
	// already has its pages committed.
	// * Subcase: "/Kernel" variants use zx_vmo_read()/zx_vmo_write() to copy to/from the VMO; other
	// variants use memcpy() in userland.
	// * Subcase: VmoMapRange: uses ZX_VM_MAP_RANGE so that the map operation pre-populates the page
	// table entries for the mappings.
	// * Vmo/Memcpy: cost of creating a VMO, mapping it, then copying it using memcpy().
	// * Subcase: "/WithPrecommit" variants use ZX_VMAR_OP_COMMIT to map and commit the VMO prior to
	// issuing a memcpy.
	// * Subcase: "/WithoutPrecommit" variants perform a memcpy into the VMO without committing it.
	// * Vmo/TransferData: cost of zx_vmo_transfer_data().

	namespace {

	// Measure the time taken to write or read a chunk of data to/from a VMO using the zx_vmo_write() or
	// zx_vmo_read() syscalls respectively. If \|do_write\| and \|zero_page\| are true, this measures the
	// time to do a zx_vmo_write() that copies from a buffer that maps to the kernel's shared zero page
	// into the VMO. One reason for testing this case is that this uses a different code path in the
	// kernel than if non-zero pages were used. For multi-page buffers, it will also read fewer pages of
	// physical memory.
	bool VmoReadOrWriteTest(perftest::RepeatState* state, uint32_t copy_size, bool do_write,
	bool zero_page) {
	if (zero_page) {
	// This is the only meaningful combination. See comments where the test is registered below.
	ZX_ASSERT(do_write);
	}

	zx::vmo vmo;
	ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo));

	// Use a vmo as the buffer to read from / write to, so we can exactly control whether we're using
	// distinct physical pages or the singleton zero page.
	zx::vmo buffer_vmo;
	ASSERT_OK(zx::vmo::create(copy_size, 0, &buffer_vmo));
	zx_vaddr_t buffer_addr;
	ASSERT_OK(zx::vmar::root_self()->map(ZX_VM_PERM_READ \| ZX_VM_PERM_WRITE, 0, buffer_vmo, 0,
	copy_size, &buffer_addr));

	// If \|zero_page\| is not specified, memset to non-zero to make sure buffer_vmo's pages are
	// populated and not eligible for zero page deduping, otherwise let the kernel fault in the zero
	// page as required.
	//
	// This can alter the runtime of the vmo write below. If \|zero_page\| is true, for vmo
	// write, the buffer is being read from, so we will just use the singleton zero page.
	//
	// Also when performing page lookups in the vmo to retrieve backing pages, the logic in the kernel
	// for handling distinct physical pages differs from the zero page.
	if (!zero_page) {
	memset(reinterpret_cast<void*>(buffer_addr), 0xa, copy_size);
	}

	// Write the VMO so that the pages are pre-committed. This matters
	// more for the read case.
	ASSERT_OK(vmo.write(reinterpret_cast<void*>(buffer_addr), 0, copy_size));

	if (do_write) {
	while (state->KeepRunning()) {
	ASSERT_OK(vmo.write(reinterpret_cast<void*>(buffer_addr), 0, copy_size));
	}
	} else {
	while (state->KeepRunning()) {
	ASSERT_OK(vmo.read(reinterpret_cast<void*>(buffer_addr), 0, copy_size));
	}
	}

	ASSERT_OK(zx::vmar::root_self()->unmap(buffer_addr, copy_size));
	return true;
	}

	// Measure the time taken to write or read a chunk of data to/from a mapped VMO. The writing/reading
	// is either done from userland using memcpy() (when user_memcpy=true) or by the kernel using
	// zx_vmo_read()/zx_vmo_write() (when user_memcpy=false).
	bool VmoReadOrWriteMapTestImpl(perftest::RepeatState* state, uint32_t copy_size, bool do_write,
	int flags, bool user_memcpy) {
	zx::vmo vmo;
	ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo));
	std::vector<char> buffer(copy_size);
	zx_vaddr_t mapped_addr;

	zx::vmo vmo_buf;
	if (!user_memcpy) {
	// Create a temporary VMO that we can use to get the kernel to read/write our mapped memory.
	ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo_buf));
	}

	// Write the VMO so that the pages are pre-committed. This matters
	// more for the read case.
	ASSERT_OK(vmo.write(buffer.data(), 0, copy_size));

	if (do_write) {
	while (state->KeepRunning()) {
	ASSERT_OK(zx::vmar::root_self()->map(ZX_VM_PERM_READ \| ZX_VM_PERM_WRITE \| flags, 0, vmo, 0,
	copy_size, &mapped_addr));
	if (user_memcpy) {
	std::memcpy(reinterpret_cast<void*>(mapped_addr), buffer.data(), copy_size);
	} else {
	// To write to the mapped in portion we read from the temporary VMO.
	ASSERT_OK(vmo_buf.read(reinterpret_cast<void*>(mapped_addr), 0, copy_size));
	}
	ASSERT_OK(zx::vmar::root_self()->unmap(mapped_addr, copy_size));
	}
	} else { // read
	while (state->KeepRunning()) {
	ASSERT_OK(zx::vmar::root_self()->map(ZX_VM_PERM_READ \| ZX_VM_PERM_WRITE \| flags, 0, vmo, 0,
	copy_size, &mapped_addr));
	if (user_memcpy) {
	std::memcpy(buffer.data(), reinterpret_cast<void*>(mapped_addr), copy_size);
	} else {
	// To read from the mapped in portion we write it to the temporary VMO.
	ASSERT_OK(vmo_buf.write(reinterpret_cast<void*>(mapped_addr), 0, copy_size));
	}
	ASSERT_OK(zx::vmar::root_self()->unmap(mapped_addr, copy_size));
	}
	}
	return true;
	}

	bool VmoReadOrWriteMapTest(perftest::RepeatState* state, uint32_t copy_size, bool do_write,
	bool user_memcpy) {
	return VmoReadOrWriteMapTestImpl(state, copy_size, do_write, 0, user_memcpy);
	}

	bool VmoReadOrWriteMapRangeTest(perftest::RepeatState* state, uint32_t copy_size, bool do_write,
	bool user_memcpy) {
	return VmoReadOrWriteMapTestImpl(state, copy_size, do_write, ZX_VM_MAP_RANGE, user_memcpy);
	}

	// Measure the time taken to create a VMO, map it into the root VMAR, optionally commit and
	// map the pages, memcpy data into the VMO, then unmap and destroy the VMO. This is used as
	// an indirect way to measure the overhead induced by page faulting during a memcpy.
	bool VmoMemcpyPrecommitTest(perftest::RepeatState* state, uint32_t size, bool precommit) {
	state->DeclareStep("create_and_map_vmo");
	if (precommit) {
	state->DeclareStep("precommit");
	}
	state->DeclareStep("memcpy");
	state->DeclareStep("unmap_and_destroy_vmo");

	// Set up a source buffer and initialize it.
	std::unique_ptr<char[]> src(new char[size]);
	memset(src.get(), 0xff, size);

	while (state->KeepRunning()) {
	// Create and map the destination VMO.
	zx::vmo dst_vmo;
	ASSERT_OK(zx::vmo::create(size, 0, &dst_vmo));
	zx_vaddr_t dst;
	ASSERT_OK(
	zx::vmar::root_self()->map(ZX_VM_PERM_READ \| ZX_VM_PERM_WRITE, 0, dst_vmo, 0, size, &dst));
	state->NextStep();

	// Commit the destination VMO if we're running the precommit case.
	if (precommit) {
	ASSERT_OK(zx::vmar::root_self()->op_range(ZX_VMAR_OP_COMMIT, dst, size, 0, 0));
	state->NextStep();
	}

	// Memcpy from source into dst.
	memcpy(reinterpret_cast<void*>(dst), src.get(), size);
	state->NextStep();

	// Unmap the destination VMO.
	ASSERT_OK(zx::vmar::root_self()->unmap(dst, size));
	}
	return true;
	}

	// Measure the time taken to clone a vmo and destroy it. If map_size is non zero, then this function
	// tests the case where the original vmo is mapped in chunks of map_size; otherwise it tests the
	// case where the original vmo is not mapped.
	bool VmoCloneTest(perftest::RepeatState* state, uint32_t copy_size, uint32_t map_size) {
	if (map_size > 0) {
	state->DeclareStep("map");
	}
	state->DeclareStep("clone");
	state->DeclareStep("close");
	if (map_size > 0) {
	state->DeclareStep("unmap");
	}

	zx::vmo vmo;
	ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo));
	ASSERT_OK(vmo.op_range(ZX_VMO_OP_COMMIT, 0, copy_size, nullptr, 0));

	zx::vmar vmar;
	zx_vaddr_t addr = 0;
	// Allocate a single vmar so we have a single reserved block if mapping in using multiple chunks.
	ASSERT_OK(zx::vmar::root_self()->allocate(
	ZX_VM_CAN_MAP_SPECIFIC \| ZX_VM_CAN_MAP_READ \| ZX_VM_CAN_MAP_WRITE, 0, copy_size, &vmar,
	&addr));

	while (state->KeepRunning()) {
	if (map_size > 0) {
	zx_vaddr_t chunk_addr;
	for (uint32_t off = 0; off < copy_size; off += map_size) {
	ASSERT_OK(vmar.map(ZX_VM_MAP_RANGE \| ZX_VM_PERM_READ \| ZX_VM_SPECIFIC, off, vmo, off,
	map_size, &chunk_addr));
	}
	state->NextStep();
	}

	zx::vmo clone;
	ASSERT_OK(vmo.create_child(ZX_VMO_CHILD_SNAPSHOT, 0, copy_size, &clone));
	state->NextStep();

	clone.reset();

	if (map_size > 0) {
	state->NextStep();
	ASSERT_OK(vmar.unmap(addr, copy_size));
	}
	}

	return true;
	}

	// Measure the time taken to create a clone, map, unmap and then destroy it.
	bool VmoMapCloneTest(perftest::RepeatState* state, uint32_t copy_size) {
	state->DeclareStep("clone");
	state->DeclareStep("map");
	state->DeclareStep("unmap");
	state->DeclareStep("close");

	zx::vmo vmo;
	ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo));
	// Fully commit the parent vmo's pages, so that the clone mapping has backing pages to map in.
	ASSERT_OK(vmo.op_range(ZX_VMO_OP_COMMIT, 0, copy_size, nullptr, 0));

	while (state->KeepRunning()) {
	zx::vmo clone;
	ASSERT_OK(vmo.create_child(ZX_VMO_CHILD_SNAPSHOT, 0, copy_size, &clone));
	state->NextStep();

	zx_vaddr_t addr = 0;
	// ZX_VM_MAP_RANGE will fully populate the mapping.
	ASSERT_OK(zx::vmar::root_self()->map(ZX_VM_MAP_RANGE \| ZX_VM_PERM_READ, 0, clone, 0, copy_size,
	&addr));
	state->NextStep();

	ASSERT_OK(zx::vmar::root_self()->unmap(addr, copy_size));
	state->NextStep();

	clone.reset();
	}

	return true;
	}

	// Measure the time it takes to clone a vmo. Specifically, this measures:
	// - Clone a vmo.
	// - Read or write either the original vmo (do_target_clone=false) or the
	// clone (do_target_clone=true).
	// - For bidirectional clones, we don't expect varying do_target_clone to
	// significantly affect this performance.
	// - do_full_op controls whether we read or write the whole vmo or just
	// a subset of the pages, as the performance characteristics of a
	// partially populated clone and a fully populated clone can differ.
	// - Destroy the clone.
	bool VmoCloneReadOrWriteTest(perftest::RepeatState* state, uint32_t copy_size, bool do_write,
	bool do_target_clone, bool do_full_op) {
	const size_t kPageSize = zx_system_get_page_size();
	state->DeclareStep("clone");
	state->DeclareStep(do_write ? "write" : "read");
	state->DeclareStep("close");

	zx::vmo vmo;
	ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo));
	ASSERT_OK(vmo.op_range(ZX_VMO_OP_COMMIT, 0, copy_size, nullptr, 0));

	std::vector<char> buffer(copy_size);

	while (state->KeepRunning()) {
	zx::vmo clone;
	ASSERT_OK(vmo.create_child(ZX_VMO_CHILD_SNAPSHOT, 0, copy_size, &clone));
	state->NextStep();

	const zx::vmo& target = do_target_clone ? clone : vmo;
	if (do_full_op) {
	if (do_write) {
	ASSERT_OK(target.write(buffer.data(), 0, copy_size));
	} else {
	ASSERT_OK(target.read(buffer.data(), 0, copy_size));
	}
	} else {
	// There's no special meaning behind the particular value of this
	// constant. It just needs to result in a couple of writes into
	// the vmo without populating it too densely.
	const uint64_t kWriteInterval = 8 * kPageSize;
	for (uint64_t offset = 0; offset < copy_size; offset += kWriteInterval) {
	if (do_write) {
	ASSERT_OK(target.write(buffer.data(), offset, kPageSize));
	} else {
	ASSERT_OK(target.read(buffer.data(), offset, kPageSize));
	}
	}
	}

	state->NextStep();
	// The clone goes out of scope and is implicitly closed.
	}

	return true;
	}

	// Measure the times taken to create, write and then read some data from a VMO on a single thread.
	// This is used to measure the performance of a brand new VMO's entire lifecycle up to data read
	// completion time. This test is useful because this is essentially what users of `fuchsia.mem.Data`
	// or `fuchsia.mem.Buffer` must do on top of their default zx.channel write/read operations. It's
	// worth measuring these operations together (when they are also tested separately) because we
	// expect them to have different performance behavior together.
	//
	// The zx_vmo_write() call will cause pages to be allocated in the VMO, and closing the VMO handle
	// will free those pages.
	bool VmoCreateWriteReadCloseTest(perftest::RepeatState* state, uint32_t copy_size) {
	state->DeclareStep("create");
	state->DeclareStep("write");
	state->DeclareStep("read");
	state->DeclareStep("close");

	// Use a vmo as the buffer to read from / write to.
	zx::vmo buffer_vmo;
	ASSERT_OK(zx::vmo::create(copy_size, 0, &buffer_vmo));
	zx_vaddr_t buffer_addr;
	ASSERT_OK(zx::vmar::root_self()->map(ZX_VM_PERM_READ \| ZX_VM_PERM_WRITE, 0, buffer_vmo, 0,
	copy_size, &buffer_addr));
	memset(reinterpret_cast<void*>(buffer_addr), 0xa, copy_size);

	while (state->KeepRunning()) {
	zx::vmo vmo;
	ASSERT_OK(zx::vmo::create(copy_size, 0, &vmo));
	state->NextStep();
	ASSERT_OK(vmo.write(reinterpret_cast<void*>(buffer_addr), 0, copy_size));
	state->NextStep();
	ASSERT_OK(vmo.read(reinterpret_cast<void*>(buffer_addr), 0, copy_size));
	state->NextStep();
	}

	ASSERT_OK(zx::vmar::root_self()->unmap(buffer_addr, copy_size));
	return true;
	}

	// Measure the time taken to transfer data between VMOs using zx_vmo_transfer_data().
	bool VmoTransferDataTest(perftest::RepeatState* state, uint64_t transfer_size) {
	const uint64_t vmo_size = transfer_size * 2;
	const uint64_t src_offset = transfer_size;
	const uint64_t dst_offset = 0;
	zx::vmo src_vmo;
	ASSERT_OK(zx::vmo::create(vmo_size, 0, &src_vmo));
	zx_vaddr_t buffer_addr;
	ASSERT_OK(zx::vmar::root_self()->map(ZX_VM_PERM_READ \| ZX_VM_PERM_WRITE, 0, src_vmo, 0, vmo_size,
	&buffer_addr));
	zx::vmo dst_vmo;
	ASSERT_OK(zx::vmo::create(vmo_size, 0, &dst_vmo));

	state->DeclareStep("fill_source");
	state->DeclareStep("transfer");
	while (state->KeepRunning()) {
	memset((void*)buffer_addr, 'f', vmo_size);
	state->NextStep();
	ASSERT_OK(dst_vmo.transfer_data(0, dst_offset, transfer_size, &src_vmo, src_offset));
	}
	ASSERT_OK(zx::vmar::root_self()->unmap(buffer_addr, vmo_size));
	return true;
	}

	template <typename Func, typename... Args>
	void RegisterVmoTest(const char* name, Func fn, Args... args) {
	for (unsigned size_in_kbytes : {4, 32, 128, 512, 2048}) {
	auto full_name = fbl::StringPrintf("%s/%ukbytes", name, size_in_kbytes);
	perftest::RegisterTest(full_name.c_str(), fn, size_in_kbytes * 1024, args...);
	}
	}

	void RegisterTests() {
	for (bool do_write : {false, true}) {
	for (bool zero : {false, true}) {
	// The zero case for vmo read is not meaningful since it will only operate on the zero page in
	// the first iteration; the remaining iterations will use forked pages which is equivalent to
	// the non-zero case. Skip this combo.
	if (zero && !do_write) {
	continue;
	}
	const char* rw = do_write ? "Write" : "Read";
	const char* z = zero ? "/ZeroPage" : "";
	auto rw_name = fbl::StringPrintf("Vmo/%s%s", rw, z);
	RegisterVmoTest(rw_name.c_str(), VmoReadOrWriteTest, do_write, zero);
	}
	}

	for (bool do_write : {false, true}) {
	for (bool user_memcpy : {false, true}) {
	const char* rw = do_write ? "Write" : "Read";
	const char* user_kernel = user_memcpy ? "" : "/Kernel";

	auto rw_name = fbl::StringPrintf("VmoMap/%s%s", rw, user_kernel);
	RegisterVmoTest(rw_name.c_str(), VmoReadOrWriteMapTest, do_write, user_memcpy);

	rw_name = fbl::StringPrintf("VmoMapRange/%s%s", rw, user_kernel);
	RegisterVmoTest(rw_name.c_str(), VmoReadOrWriteMapRangeTest, do_write, user_memcpy);
	}
	}

	for (bool precommit : {false, true}) {
	const char* pc = precommit ? "WithPrecommit" : "WithoutPrecommit";
	auto precommit_name = fbl::StringPrintf("Vmo/Memcpy/%s", pc);
	RegisterVmoTest(precommit_name.c_str(), VmoMemcpyPrecommitTest, precommit);
	}

	for (bool map : {false, true}) {
	auto clone_name = fbl::StringPrintf("Vmo/Clone%s", map ? "/MapParent" : "");
	RegisterVmoTest(clone_name.c_str(), [map](perftest::RepeatState* state, uint32_t size) {
	return VmoCloneTest(state, size, map ? size : 0);
	});
	}

	for (unsigned map_chunk_kb : {4, 64, 2048, 32768}) {
	constexpr uint32_t vmo_size_kb = 32768;
	auto name = fbl::StringPrintf("Vmo/Clone/MapParent%usegments/%ukbytes",
	vmo_size_kb / map_chunk_kb, vmo_size_kb);
	perftest::RegisterTest(name.c_str(), VmoCloneTest, vmo_size_kb * 1024, map_chunk_kb * 1024);
	}

	auto name = fbl::StringPrintf("Vmo/MapClone");
	RegisterVmoTest(name.c_str(), VmoMapCloneTest);

	for (bool do_write : {false, true}) {
	for (bool do_target_clone : {false, true}) {
	for (bool do_full_op : {false, true}) {
	const char* rw = do_write ? "Write" : "Read";
	const char* target = do_target_clone ? "Clone" : "Orig";
	const char* density = do_full_op ? "All" : "Some";
	auto clone_rw_name = fbl::StringPrintf("Vmo/Clone/%s%s%s", rw, target, density);
	RegisterVmoTest(clone_rw_name.c_str(), VmoCloneReadOrWriteTest, do_write, do_target_clone,
	do_full_op);
	}
	}
	}

	name = fbl::StringPrintf("Vmo/CreateWriteReadClose");
	RegisterVmoTest(name.c_str(), VmoCreateWriteReadCloseTest);

	name = fbl::StringPrintf("Vmo/TransferData");
	RegisterVmoTest(name.c_str(), VmoTransferDataTest);
	}
	PERFTEST_CTOR(RegisterTests)

	} // namespace