absl/crc/internal/crc_memcpy_x86_arm_combined.cc - third_party/abseil-cpp - Git at Google

 // Copyright 2022 The Abseil Authors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //     https://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 // Simultaneous memcopy and CRC-32C for x86-64 and ARM 64. Uses integer
 // registers because XMM registers do not support the CRC instruction (yet).
 // While copying, compute the running CRC of the data being copied.
 //
 // It is assumed that any CPU running this code has SSE4.2 instructions
 // available (for CRC32C).  This file will do nothing if that is not true.
 //
 // The CRC instruction has a 3-byte latency, and we are stressing the ALU ports
 // here (unlike a traditional memcopy, which has almost no ALU use), so we will
 // need to copy in such a way that the CRC unit is used efficiently. We have two
 // regimes in this code:
 //  1. For operations of size < kCrcSmallSize, do the CRC then the memcpy
 //  2. For operations of size > kCrcSmallSize:
 //      a) compute an initial CRC + copy on a small amount of data to align the
 //         destination pointer on a 16-byte boundary.
 //      b) Split the data into 3 main regions and a tail (smaller than 48 bytes)
 //      c) Do the copy and CRC of the 3 main regions, interleaving (start with
 //         full cache line copies for each region, then move to single 16 byte
 //         pieces per region).
 //      d) Combine the CRCs with CRC32C::Concat.
 //      e) Copy the tail and extend the CRC with the CRC of the tail.
 // This method is not ideal for op sizes between ~1k and ~8k because CRC::Concat
 // takes a significant amount of time.  A medium-sized approach could be added
 // using 3 CRCs over fixed-size blocks where the zero-extensions required for
 // CRC32C::Concat can be precomputed.

 #ifdef __SSE4_2__
 #include <immintrin.h>
 #endif

 #ifdef _MSC_VER
 #include <intrin.h>
 #endif

 #include <array>
 #include <cstddef>
 #include <cstdint>
 #include <cstring>
 #include <memory>

 #include "absl/base/attributes.h"
 #include "absl/base/config.h"
 #include "absl/base/optimization.h"
 #include "absl/base/prefetch.h"
 #include "absl/crc/crc32c.h"
 #include "absl/crc/internal/cpu_detect.h"
 #include "absl/crc/internal/crc32_x86_arm_combined_simd.h"
 #include "absl/crc/internal/crc_memcpy.h"
 #include "absl/strings/string_view.h"

 #if defined(ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE) || \
     defined(ABSL_INTERNAL_HAVE_ARM_ACCELERATED_CRC_MEMCPY_ENGINE)

 namespace absl {
 ABSL_NAMESPACE_BEGIN
 namespace crc_internal {

 namespace {

 inline crc32c_t ShortCrcCopy(char* dst, const char* src, std::size_t length,
                              crc32c_t crc) {
   // Small copy: just go 1 byte at a time: being nice to the branch predictor
   // is more important here than anything else
   uint32_t crc_uint32 = static_cast<uint32_t>(crc);
   for (std::size_t i = 0; i < length; i++) {
     uint8_t data = *reinterpret_cast<const uint8_t*>(src);
     crc_uint32 = CRC32_u8(crc_uint32, data);
     *reinterpret_cast<uint8_t*>(dst) = data;
     ++src;
     ++dst;
   }
   return crc32c_t{crc_uint32};
 }

 constexpr size_t kIntLoadsPerVec = sizeof(V128) / sizeof(uint64_t);

 // Common function for copying the tails of multiple large regions.
 // Disable ubsan for benign unaligned access. See b/254108538.
 template <size_t vec_regions, size_t int_regions>
 ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED inline void LargeTailCopy(
     crc32c_t* crcs, char** dst, const char** src, size_t region_size,
     size_t copy_rounds) {
   std::array<V128, vec_regions> data;
   std::array<uint64_t, kIntLoadsPerVec * int_regions> int_data;

   while (copy_rounds > 0) {
     for (size_t i = 0; i < vec_regions; i++) {
       size_t region = i;

       auto* vsrc = reinterpret_cast<const V128*>(*src + region_size * region);
       auto* vdst = reinterpret_cast<V128*>(*dst + region_size * region);

       // Load the blocks, unaligned
       data[i] = V128_LoadU(vsrc);

       // Store the blocks, aligned
       V128_Store(vdst, data[i]);

       // Compute the running CRC
       crcs[region] = crc32c_t{static_cast<uint32_t>(
           CRC32_u64(static_cast<uint32_t>(crcs[region]),
                     static_cast<uint64_t>(V128_Extract64<0>(data[i]))))};
       crcs[region] = crc32c_t{static_cast<uint32_t>(
           CRC32_u64(static_cast<uint32_t>(crcs[region]),
                     static_cast<uint64_t>(V128_Extract64<1>(data[i]))))};
     }

     for (size_t i = 0; i < int_regions; i++) {
       size_t region = vec_regions + i;

       auto* usrc =
           reinterpret_cast<const uint64_t*>(*src + region_size * region);
       auto* udst = reinterpret_cast<uint64_t*>(*dst + region_size * region);

       for (size_t j = 0; j < kIntLoadsPerVec; j++) {
         size_t data_index = i * kIntLoadsPerVec + j;

         int_data[data_index] = *(usrc + j);
         crcs[region] = crc32c_t{CRC32_u64(static_cast<uint32_t>(crcs[region]),
                                           int_data[data_index])};

         *(udst + j) = int_data[data_index];
       }
     }

     // Increment pointers
     *src += sizeof(V128);
     *dst += sizeof(V128);
     --copy_rounds;
   }
 }

 }  // namespace

 template <size_t vec_regions, size_t int_regions>
 class AcceleratedCrcMemcpyEngine : public CrcMemcpyEngine {
  public:
   AcceleratedCrcMemcpyEngine() = default;
   AcceleratedCrcMemcpyEngine(const AcceleratedCrcMemcpyEngine&) = delete;
   AcceleratedCrcMemcpyEngine operator=(const AcceleratedCrcMemcpyEngine&) =
       delete;

   crc32c_t Compute(void* __restrict dst, const void* __restrict src,
                    std::size_t length, crc32c_t initial_crc) const override;
 };

 // Disable ubsan for benign unaligned access. See b/254108538.
 template <size_t vec_regions, size_t int_regions>
 ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED crc32c_t
 AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute(
     void* __restrict dst, const void* __restrict src, std::size_t length,
     crc32c_t initial_crc) const {
   constexpr std::size_t kRegions = vec_regions + int_regions;
   static_assert(kRegions > 0, "Must specify at least one region.");
   constexpr uint32_t kCrcDataXor = uint32_t{0xffffffff};
   constexpr std::size_t kBlockSize = sizeof(V128);
   constexpr std::size_t kCopyRoundSize = kRegions * kBlockSize;

   // Number of blocks per cacheline.
   constexpr std::size_t kBlocksPerCacheLine = ABSL_CACHELINE_SIZE / kBlockSize;

   char* dst_bytes = static_cast<char*>(dst);
   const char* src_bytes = static_cast<const char*>(src);

   // Make sure that one prefetch per big block is enough to cover the whole
   // dataset, and we don't prefetch too much.
   static_assert(ABSL_CACHELINE_SIZE % kBlockSize == 0,
                 "Cache lines are not divided evenly into blocks, may have "
                 "unintended behavior!");

   // Experimentally-determined boundary between a small and large copy.
   // Below this number, spin-up and concatenation of CRCs takes enough time that
   // it kills the throughput gains of using 3 regions and wide vectors.
   constexpr size_t kCrcSmallSize = 256;

   // Experimentally-determined prefetch distance.  Main loop copies will
   // prefeth data 2 cache lines ahead.
   constexpr std::size_t kPrefetchAhead = 2 * ABSL_CACHELINE_SIZE;

   // Small-size CRC-memcpy : just do CRC + memcpy
   if (length < kCrcSmallSize) {
     crc32c_t crc =
         ExtendCrc32c(initial_crc, absl::string_view(src_bytes, length));
     memcpy(dst, src, length);
     return crc;
   }

   // Start work on the CRC: undo the XOR from the previous calculation or set up
   // the initial value of the CRC.
   initial_crc = crc32c_t{static_cast<uint32_t>(initial_crc) ^ kCrcDataXor};

   // Do an initial alignment copy, so we can use aligned store instructions to
   // the destination pointer.  We align the destination pointer because the
   // penalty for an unaligned load is small compared to the penalty of an
   // unaligned store on modern CPUs.
   std::size_t bytes_from_last_aligned =
       reinterpret_cast<uintptr_t>(dst) & (kBlockSize - 1);
   if (bytes_from_last_aligned != 0) {
     std::size_t bytes_for_alignment = kBlockSize - bytes_from_last_aligned;

     // Do the short-sized copy and CRC.
     initial_crc =
         ShortCrcCopy(dst_bytes, src_bytes, bytes_for_alignment, initial_crc);
     src_bytes += bytes_for_alignment;
     dst_bytes += bytes_for_alignment;
     length -= bytes_for_alignment;
   }

   // We are going to do the copy and CRC in kRegions regions to make sure that
   // we can saturate the CRC unit.  The CRCs will be combined at the end of the
   // run.  Copying will use the SSE registers, and we will extract words from
   // the SSE registers to add to the CRC.  Initially, we run the loop one full
   // cache line per region at a time, in order to insert prefetches.

   // Initialize CRCs for kRegions regions.
   crc32c_t crcs[kRegions];
   crcs[0] = initial_crc;
   for (size_t i = 1; i < kRegions; i++) {
     crcs[i] = crc32c_t{kCrcDataXor};
   }

   // Find the number of rounds to copy and the region size.  Also compute the
   // tail size here.
   size_t copy_rounds = length / kCopyRoundSize;

   // Find the size of each region and the size of the tail.
   const std::size_t region_size = copy_rounds * kBlockSize;
   const std::size_t tail_size = length - (kRegions * region_size);

   // Holding registers for data in each region.
   std::array<V128, vec_regions> vec_data;
   std::array<uint64_t, int_regions * kIntLoadsPerVec> int_data;

   // Main loop.
   while (copy_rounds > kBlocksPerCacheLine) {
     // Prefetch kPrefetchAhead bytes ahead of each pointer.
     for (size_t i = 0; i < kRegions; i++) {
       absl::PrefetchToLocalCache(src_bytes + kPrefetchAhead + region_size * i);
 #ifdef ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE
       // TODO(b/297082454): investigate dropping prefetch on x86.
       absl::PrefetchToLocalCache(dst_bytes + kPrefetchAhead + region_size * i);
 #endif
     }

     // Load and store data, computing CRC on the way.
     for (size_t i = 0; i < kBlocksPerCacheLine; i++) {
       // Copy and CRC the data for the CRC regions.
       for (size_t j = 0; j < vec_regions; j++) {
         // Cycle which regions get vector load/store and integer load/store, to
         // engage prefetching logic around vector load/stores and save issue
         // slots by using the integer registers.
         size_t region = (j + i) % kRegions;

         auto* vsrc =
             reinterpret_cast<const V128*>(src_bytes + region_size * region);
         auto* vdst = reinterpret_cast<V128*>(dst_bytes + region_size * region);

         // Load and CRC data.
         vec_data[j] = V128_LoadU(vsrc + i);
         crcs[region] = crc32c_t{static_cast<uint32_t>(
             CRC32_u64(static_cast<uint32_t>(crcs[region]),
                       static_cast<uint64_t>(V128_Extract64<0>(vec_data[j]))))};
         crcs[region] = crc32c_t{static_cast<uint32_t>(
             CRC32_u64(static_cast<uint32_t>(crcs[region]),
                       static_cast<uint64_t>(V128_Extract64<1>(vec_data[j]))))};

         // Store the data.
         V128_Store(vdst + i, vec_data[j]);
       }

       // Preload the partial CRCs for the CLMUL subregions.
       for (size_t j = 0; j < int_regions; j++) {
         // Cycle which regions get vector load/store and integer load/store, to
         // engage prefetching logic around vector load/stores and save issue
         // slots by using the integer registers.
         size_t region = (j + vec_regions + i) % kRegions;

         auto* usrc =
             reinterpret_cast<const uint64_t*>(src_bytes + region_size * region);
         auto* udst =
             reinterpret_cast<uint64_t*>(dst_bytes + region_size * region);

         for (size_t k = 0; k < kIntLoadsPerVec; k++) {
           size_t data_index = j * kIntLoadsPerVec + k;

           // Load and CRC the data.
           int_data[data_index] = *(usrc + i * kIntLoadsPerVec + k);
           crcs[region] = crc32c_t{CRC32_u64(static_cast<uint32_t>(crcs[region]),
                                             int_data[data_index])};

           // Store the data.
           *(udst + i * kIntLoadsPerVec + k) = int_data[data_index];
         }
       }
     }

     // Increment pointers
     src_bytes += kBlockSize * kBlocksPerCacheLine;
     dst_bytes += kBlockSize * kBlocksPerCacheLine;
     copy_rounds -= kBlocksPerCacheLine;
   }

   // Copy and CRC the tails of each region.
   LargeTailCopy<vec_regions, int_regions>(crcs, &dst_bytes, &src_bytes,
                                           region_size, copy_rounds);

   // Move the source and destination pointers to the end of the region
   src_bytes += region_size * (kRegions - 1);
   dst_bytes += region_size * (kRegions - 1);

   // Copy and CRC the tail through the XMM registers.
   std::size_t tail_blocks = tail_size / kBlockSize;
   LargeTailCopy<0, 1>(&crcs[kRegions - 1], &dst_bytes, &src_bytes, 0,
                       tail_blocks);

   // Final tail copy for under 16 bytes.
   crcs[kRegions - 1] =
       ShortCrcCopy(dst_bytes, src_bytes, tail_size - tail_blocks * kBlockSize,
                    crcs[kRegions - 1]);

   if (kRegions == 1) {
     // If there is only one region, finalize and return its CRC.
     return crc32c_t{static_cast<uint32_t>(crcs[0]) ^ kCrcDataXor};
   }

   // Finalize the first CRCs: XOR the internal CRCs by the XOR mask to undo the
   // XOR done before doing block copy + CRCs.
   for (size_t i = 0; i + 1 < kRegions; i++) {
     crcs[i] = crc32c_t{static_cast<uint32_t>(crcs[i]) ^ kCrcDataXor};
   }

   // Build a CRC of the first kRegions - 1 regions.
   crc32c_t full_crc = crcs[0];
   for (size_t i = 1; i + 1 < kRegions; i++) {
     full_crc = ConcatCrc32c(full_crc, crcs[i], region_size);
   }

   // Finalize and concatenate the final CRC, then return.
   crcs[kRegions - 1] =
       crc32c_t{static_cast<uint32_t>(crcs[kRegions - 1]) ^ kCrcDataXor};
   return ConcatCrc32c(full_crc, crcs[kRegions - 1], region_size + tail_size);
 }

 CrcMemcpy::ArchSpecificEngines CrcMemcpy::GetArchSpecificEngines() {
 #ifdef UNDEFINED_BEHAVIOR_SANITIZER
   // UBSAN does not play nicely with unaligned loads (which we use a lot).
   // Get the underlying architecture.
   CpuType cpu_type = GetCpuType();
   switch (cpu_type) {
     case CpuType::kAmdRome:
     case CpuType::kAmdNaples:
     case CpuType::kAmdMilan:
     case CpuType::kAmdGenoa:
     case CpuType::kAmdRyzenV3000:
     case CpuType::kIntelCascadelakeXeon:
     case CpuType::kIntelSkylakeXeon:
     case CpuType::kIntelSkylake:
     case CpuType::kIntelBroadwell:
     case CpuType::kIntelHaswell:
     case CpuType::kIntelIvybridge:
       return {
           /*.temporal=*/new FallbackCrcMemcpyEngine(),
           /*.non_temporal=*/new CrcNonTemporalMemcpyAVXEngine(),
       };
     // INTEL_SANDYBRIDGE performs better with SSE than AVX.
     case CpuType::kIntelSandybridge:
       return {
           /*.temporal=*/new FallbackCrcMemcpyEngine(),
           /*.non_temporal=*/new CrcNonTemporalMemcpyEngine(),
       };
     default:
       return {/*.temporal=*/new FallbackCrcMemcpyEngine(),
               /*.non_temporal=*/new FallbackCrcMemcpyEngine()};
   }
 #else
   // Get the underlying architecture.
   CpuType cpu_type = GetCpuType();
   switch (cpu_type) {
     // On Zen 2, PEXTRQ uses 2 micro-ops, including one on the vector store port
     // which data movement from the vector registers to the integer registers
     // (where CRC32C happens) to crowd the same units as vector stores.  As a
     // result, using that path exclusively causes bottlenecking on this port.
     // We can avoid this bottleneck by using the integer side of the CPU for
     // most operations rather than the vector side.  We keep a vector region to
     // engage some of the prefetching logic in the cache hierarchy which seems
     // to give vector instructions special treatment.  These prefetch units see
     // strided access to each region, and do the right thing.
     case CpuType::kAmdRome:
     case CpuType::kAmdNaples:
     case CpuType::kAmdMilan:
     case CpuType::kAmdGenoa:
     case CpuType::kAmdRyzenV3000:
       return {
           /*.temporal=*/new AcceleratedCrcMemcpyEngine<1, 2>(),
           /*.non_temporal=*/new CrcNonTemporalMemcpyAVXEngine(),
       };
     // PCLMULQDQ is slow and we don't have wide enough issue width to take
     // advantage of it.  For an unknown architecture, don't risk using CLMULs.
     case CpuType::kIntelCascadelakeXeon:
     case CpuType::kIntelSkylakeXeon:
     case CpuType::kIntelSkylake:
     case CpuType::kIntelBroadwell:
     case CpuType::kIntelHaswell:
     case CpuType::kIntelIvybridge:
       return {
           /*.temporal=*/new AcceleratedCrcMemcpyEngine<3, 0>(),
           /*.non_temporal=*/new CrcNonTemporalMemcpyAVXEngine(),
       };
     // INTEL_SANDYBRIDGE performs better with SSE than AVX.
     case CpuType::kIntelSandybridge:
       return {
           /*.temporal=*/new AcceleratedCrcMemcpyEngine<3, 0>(),
           /*.non_temporal=*/new CrcNonTemporalMemcpyEngine(),
       };
     default:
       return {/*.temporal=*/new FallbackCrcMemcpyEngine(),
               /*.non_temporal=*/new FallbackCrcMemcpyEngine()};
   }
 #endif  // UNDEFINED_BEHAVIOR_SANITIZER
 }

 // For testing, allow the user to specify which engine they want.
 std::unique_ptr<CrcMemcpyEngine> CrcMemcpy::GetTestEngine(int vector,
                                                           int integer) {
   if (vector == 3 && integer == 0) {
     return std::make_unique<AcceleratedCrcMemcpyEngine<3, 0>>();
   } else if (vector == 1 && integer == 2) {
     return std::make_unique<AcceleratedCrcMemcpyEngine<1, 2>>();
   } else if (vector == 1 && integer == 0) {
     return std::make_unique<AcceleratedCrcMemcpyEngine<1, 0>>();
   }
   return nullptr;
 }

 }  // namespace crc_internal
 ABSL_NAMESPACE_END
 }  // namespace absl

 #endif  // ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE ||
         // ABSL_INTERNAL_HAVE_ARM_ACCELERATED_CRC_MEMCPY_ENGINE
	// Copyright 2022 The Abseil Authors
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// https://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	// Simultaneous memcopy and CRC-32C for x86-64 and ARM 64. Uses integer
	// registers because XMM registers do not support the CRC instruction (yet).
	// While copying, compute the running CRC of the data being copied.
	//
	// It is assumed that any CPU running this code has SSE4.2 instructions
	// available (for CRC32C). This file will do nothing if that is not true.
	//
	// The CRC instruction has a 3-byte latency, and we are stressing the ALU ports
	// here (unlike a traditional memcopy, which has almost no ALU use), so we will
	// need to copy in such a way that the CRC unit is used efficiently. We have two
	// regimes in this code:
	// 1. For operations of size < kCrcSmallSize, do the CRC then the memcpy
	// 2. For operations of size > kCrcSmallSize:
	// a) compute an initial CRC + copy on a small amount of data to align the
	// destination pointer on a 16-byte boundary.
	// b) Split the data into 3 main regions and a tail (smaller than 48 bytes)
	// c) Do the copy and CRC of the 3 main regions, interleaving (start with
	// full cache line copies for each region, then move to single 16 byte
	// pieces per region).
	// d) Combine the CRCs with CRC32C::Concat.
	// e) Copy the tail and extend the CRC with the CRC of the tail.
	// This method is not ideal for op sizes between ~1k and ~8k because CRC::Concat
	// takes a significant amount of time. A medium-sized approach could be added
	// using 3 CRCs over fixed-size blocks where the zero-extensions required for
	// CRC32C::Concat can be precomputed.

	#ifdef __SSE4_2__
	#include <immintrin.h>
	#endif

	#ifdef _MSC_VER
	#include <intrin.h>
	#endif

	#include <array>
	#include <cstddef>
	#include <cstdint>
	#include <cstring>
	#include <memory>

	#include "absl/base/attributes.h"
	#include "absl/base/config.h"
	#include "absl/base/optimization.h"
	#include "absl/base/prefetch.h"
	#include "absl/crc/crc32c.h"
	#include "absl/crc/internal/cpu_detect.h"
	#include "absl/crc/internal/crc32_x86_arm_combined_simd.h"
	#include "absl/crc/internal/crc_memcpy.h"
	#include "absl/strings/string_view.h"

	#if defined(ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE) \|\| \
	defined(ABSL_INTERNAL_HAVE_ARM_ACCELERATED_CRC_MEMCPY_ENGINE)

	namespace absl {
	ABSL_NAMESPACE_BEGIN
	namespace crc_internal {

	namespace {

	inline crc32c_t ShortCrcCopy(char* dst, const char* src, std::size_t length,
	crc32c_t crc) {
	// Small copy: just go 1 byte at a time: being nice to the branch predictor
	// is more important here than anything else
	uint32_t crc_uint32 = static_cast<uint32_t>(crc);
	for (std::size_t i = 0; i < length; i++) {
	uint8_t data = reinterpret_cast<const uint8_t>(src);
	crc_uint32 = CRC32_u8(crc_uint32, data);
	reinterpret_cast<uint8_t>(dst) = data;
	++src;
	++dst;
	}
	return crc32c_t{crc_uint32};
	}

	constexpr size_t kIntLoadsPerVec = sizeof(V128) / sizeof(uint64_t);

	// Common function for copying the tails of multiple large regions.
	// Disable ubsan for benign unaligned access. See b/254108538.
	template <size_t vec_regions, size_t int_regions>
	ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED inline void LargeTailCopy(
	crc32c_t* crcs, char dst, const char src, size_t region_size,
	size_t copy_rounds) {
	std::array<V128, vec_regions> data;
	std::array<uint64_t, kIntLoadsPerVec * int_regions> int_data;

	while (copy_rounds > 0) {
	for (size_t i = 0; i < vec_regions; i++) {
	size_t region = i;

	auto* vsrc = reinterpret_cast<const V128>(src + region_size * region);
	auto* vdst = reinterpret_cast<V128>(dst + region_size * region);

	// Load the blocks, unaligned
	data[i] = V128_LoadU(vsrc);

	// Store the blocks, aligned
	V128_Store(vdst, data[i]);

	// Compute the running CRC
	crcs[region] = crc32c_t{static_cast<uint32_t>(
	CRC32_u64(static_cast<uint32_t>(crcs[region]),
	static_cast<uint64_t>(V128_Extract64<0>(data[i]))))};
	crcs[region] = crc32c_t{static_cast<uint32_t>(
	CRC32_u64(static_cast<uint32_t>(crcs[region]),
	static_cast<uint64_t>(V128_Extract64<1>(data[i]))))};
	}

	for (size_t i = 0; i < int_regions; i++) {
	size_t region = vec_regions + i;

	auto* usrc =
	reinterpret_cast<const uint64_t>(src + region_size * region);
	auto* udst = reinterpret_cast<uint64_t>(dst + region_size * region);

	for (size_t j = 0; j < kIntLoadsPerVec; j++) {
	size_t data_index = i * kIntLoadsPerVec + j;

	int_data[data_index] = *(usrc + j);
	crcs[region] = crc32c_t{CRC32_u64(static_cast<uint32_t>(crcs[region]),
	int_data[data_index])};

	*(udst + j) = int_data[data_index];
	}
	}

	// Increment pointers
	*src += sizeof(V128);
	*dst += sizeof(V128);
	--copy_rounds;
	}
	}

	} // namespace

	template <size_t vec_regions, size_t int_regions>
	class AcceleratedCrcMemcpyEngine : public CrcMemcpyEngine {
	public:
	AcceleratedCrcMemcpyEngine() = default;
	AcceleratedCrcMemcpyEngine(const AcceleratedCrcMemcpyEngine&) = delete;
	AcceleratedCrcMemcpyEngine operator=(const AcceleratedCrcMemcpyEngine&) =
	delete;

	crc32c_t Compute(void* __restrict dst, const void* __restrict src,
	std::size_t length, crc32c_t initial_crc) const override;
	};

	// Disable ubsan for benign unaligned access. See b/254108538.
	template <size_t vec_regions, size_t int_regions>
	ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED crc32c_t
	AcceleratedCrcMemcpyEngine<vec_regions, int_regions>::Compute(
	void* __restrict dst, const void* __restrict src, std::size_t length,
	crc32c_t initial_crc) const {
	constexpr std::size_t kRegions = vec_regions + int_regions;
	static_assert(kRegions > 0, "Must specify at least one region.");
	constexpr uint32_t kCrcDataXor = uint32_t{0xffffffff};
	constexpr std::size_t kBlockSize = sizeof(V128);
	constexpr std::size_t kCopyRoundSize = kRegions * kBlockSize;

	// Number of blocks per cacheline.
	constexpr std::size_t kBlocksPerCacheLine = ABSL_CACHELINE_SIZE / kBlockSize;

	char* dst_bytes = static_cast<char*>(dst);
	const char* src_bytes = static_cast<const char*>(src);

	// Make sure that one prefetch per big block is enough to cover the whole
	// dataset, and we don't prefetch too much.
	static_assert(ABSL_CACHELINE_SIZE % kBlockSize == 0,
	"Cache lines are not divided evenly into blocks, may have "
	"unintended behavior!");

	// Experimentally-determined boundary between a small and large copy.
	// Below this number, spin-up and concatenation of CRCs takes enough time that
	// it kills the throughput gains of using 3 regions and wide vectors.
	constexpr size_t kCrcSmallSize = 256;

	// Experimentally-determined prefetch distance. Main loop copies will
	// prefeth data 2 cache lines ahead.
	constexpr std::size_t kPrefetchAhead = 2 * ABSL_CACHELINE_SIZE;

	// Small-size CRC-memcpy : just do CRC + memcpy
	if (length < kCrcSmallSize) {
	crc32c_t crc =
	ExtendCrc32c(initial_crc, absl::string_view(src_bytes, length));
	memcpy(dst, src, length);
	return crc;
	}

	// Start work on the CRC: undo the XOR from the previous calculation or set up
	// the initial value of the CRC.
	initial_crc = crc32c_t{static_cast<uint32_t>(initial_crc) ^ kCrcDataXor};

	// Do an initial alignment copy, so we can use aligned store instructions to
	// the destination pointer. We align the destination pointer because the
	// penalty for an unaligned load is small compared to the penalty of an
	// unaligned store on modern CPUs.
	std::size_t bytes_from_last_aligned =
	reinterpret_cast<uintptr_t>(dst) & (kBlockSize - 1);
	if (bytes_from_last_aligned != 0) {
	std::size_t bytes_for_alignment = kBlockSize - bytes_from_last_aligned;

	// Do the short-sized copy and CRC.
	initial_crc =
	ShortCrcCopy(dst_bytes, src_bytes, bytes_for_alignment, initial_crc);
	src_bytes += bytes_for_alignment;
	dst_bytes += bytes_for_alignment;
	length -= bytes_for_alignment;
	}

	// We are going to do the copy and CRC in kRegions regions to make sure that
	// we can saturate the CRC unit. The CRCs will be combined at the end of the
	// run. Copying will use the SSE registers, and we will extract words from
	// the SSE registers to add to the CRC. Initially, we run the loop one full
	// cache line per region at a time, in order to insert prefetches.

	// Initialize CRCs for kRegions regions.
	crc32c_t crcs[kRegions];
	crcs[0] = initial_crc;
	for (size_t i = 1; i < kRegions; i++) {
	crcs[i] = crc32c_t{kCrcDataXor};
	}

	// Find the number of rounds to copy and the region size. Also compute the
	// tail size here.
	size_t copy_rounds = length / kCopyRoundSize;

	// Find the size of each region and the size of the tail.
	const std::size_t region_size = copy_rounds * kBlockSize;
	const std::size_t tail_size = length - (kRegions * region_size);

	// Holding registers for data in each region.
	std::array<V128, vec_regions> vec_data;
	std::array<uint64_t, int_regions * kIntLoadsPerVec> int_data;

	// Main loop.
	while (copy_rounds > kBlocksPerCacheLine) {
	// Prefetch kPrefetchAhead bytes ahead of each pointer.
	for (size_t i = 0; i < kRegions; i++) {
	absl::PrefetchToLocalCache(src_bytes + kPrefetchAhead + region_size * i);
	#ifdef ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE
	// TODO(b/297082454): investigate dropping prefetch on x86.
	absl::PrefetchToLocalCache(dst_bytes + kPrefetchAhead + region_size * i);
	#endif
	}

	// Load and store data, computing CRC on the way.
	for (size_t i = 0; i < kBlocksPerCacheLine; i++) {
	// Copy and CRC the data for the CRC regions.
	for (size_t j = 0; j < vec_regions; j++) {
	// Cycle which regions get vector load/store and integer load/store, to
	// engage prefetching logic around vector load/stores and save issue
	// slots by using the integer registers.
	size_t region = (j + i) % kRegions;

	auto* vsrc =
	reinterpret_cast<const V128>(src_bytes + region_size region);
	auto* vdst = reinterpret_cast<V128>(dst_bytes + region_size region);

	// Load and CRC data.
	vec_data[j] = V128_LoadU(vsrc + i);
	crcs[region] = crc32c_t{static_cast<uint32_t>(
	CRC32_u64(static_cast<uint32_t>(crcs[region]),
	static_cast<uint64_t>(V128_Extract64<0>(vec_data[j]))))};
	crcs[region] = crc32c_t{static_cast<uint32_t>(
	CRC32_u64(static_cast<uint32_t>(crcs[region]),
	static_cast<uint64_t>(V128_Extract64<1>(vec_data[j]))))};

	// Store the data.
	V128_Store(vdst + i, vec_data[j]);
	}

	// Preload the partial CRCs for the CLMUL subregions.
	for (size_t j = 0; j < int_regions; j++) {
	// Cycle which regions get vector load/store and integer load/store, to
	// engage prefetching logic around vector load/stores and save issue
	// slots by using the integer registers.
	size_t region = (j + vec_regions + i) % kRegions;

	auto* usrc =
	reinterpret_cast<const uint64_t>(src_bytes + region_size region);
	auto* udst =
	reinterpret_cast<uint64_t>(dst_bytes + region_size region);

	for (size_t k = 0; k < kIntLoadsPerVec; k++) {
	size_t data_index = j * kIntLoadsPerVec + k;

	// Load and CRC the data.
	int_data[data_index] = (usrc + i kIntLoadsPerVec + k);
	crcs[region] = crc32c_t{CRC32_u64(static_cast<uint32_t>(crcs[region]),
	int_data[data_index])};

	// Store the data.
	(udst + i kIntLoadsPerVec + k) = int_data[data_index];
	}
	}
	}

	// Increment pointers
	src_bytes += kBlockSize * kBlocksPerCacheLine;
	dst_bytes += kBlockSize * kBlocksPerCacheLine;
	copy_rounds -= kBlocksPerCacheLine;
	}

	// Copy and CRC the tails of each region.
	LargeTailCopy<vec_regions, int_regions>(crcs, &dst_bytes, &src_bytes,
	region_size, copy_rounds);

	// Move the source and destination pointers to the end of the region
	src_bytes += region_size * (kRegions - 1);
	dst_bytes += region_size * (kRegions - 1);

	// Copy and CRC the tail through the XMM registers.
	std::size_t tail_blocks = tail_size / kBlockSize;
	LargeTailCopy<0, 1>(&crcs[kRegions - 1], &dst_bytes, &src_bytes, 0,
	tail_blocks);

	// Final tail copy for under 16 bytes.
	crcs[kRegions - 1] =
	ShortCrcCopy(dst_bytes, src_bytes, tail_size - tail_blocks * kBlockSize,
	crcs[kRegions - 1]);

	if (kRegions == 1) {
	// If there is only one region, finalize and return its CRC.
	return crc32c_t{static_cast<uint32_t>(crcs[0]) ^ kCrcDataXor};
	}

	// Finalize the first CRCs: XOR the internal CRCs by the XOR mask to undo the
	// XOR done before doing block copy + CRCs.
	for (size_t i = 0; i + 1 < kRegions; i++) {
	crcs[i] = crc32c_t{static_cast<uint32_t>(crcs[i]) ^ kCrcDataXor};
	}

	// Build a CRC of the first kRegions - 1 regions.
	crc32c_t full_crc = crcs[0];
	for (size_t i = 1; i + 1 < kRegions; i++) {
	full_crc = ConcatCrc32c(full_crc, crcs[i], region_size);
	}

	// Finalize and concatenate the final CRC, then return.
	crcs[kRegions - 1] =
	crc32c_t{static_cast<uint32_t>(crcs[kRegions - 1]) ^ kCrcDataXor};
	return ConcatCrc32c(full_crc, crcs[kRegions - 1], region_size + tail_size);
	}

	CrcMemcpy::ArchSpecificEngines CrcMemcpy::GetArchSpecificEngines() {
	#ifdef UNDEFINED_BEHAVIOR_SANITIZER
	// UBSAN does not play nicely with unaligned loads (which we use a lot).
	// Get the underlying architecture.
	CpuType cpu_type = GetCpuType();
	switch (cpu_type) {
	case CpuType::kAmdRome:
	case CpuType::kAmdNaples:
	case CpuType::kAmdMilan:
	case CpuType::kAmdGenoa:
	case CpuType::kAmdRyzenV3000:
	case CpuType::kIntelCascadelakeXeon:
	case CpuType::kIntelSkylakeXeon:
	case CpuType::kIntelSkylake:
	case CpuType::kIntelBroadwell:
	case CpuType::kIntelHaswell:
	case CpuType::kIntelIvybridge:
	return {
	/.temporal=/new FallbackCrcMemcpyEngine(),
	/.non_temporal=/new CrcNonTemporalMemcpyAVXEngine(),
	};
	// INTEL_SANDYBRIDGE performs better with SSE than AVX.
	case CpuType::kIntelSandybridge:
	return {
	/.temporal=/new FallbackCrcMemcpyEngine(),
	/.non_temporal=/new CrcNonTemporalMemcpyEngine(),
	};
	default:
	return {/.temporal=/new FallbackCrcMemcpyEngine(),
	/.non_temporal=/new FallbackCrcMemcpyEngine()};
	}
	#else
	// Get the underlying architecture.
	CpuType cpu_type = GetCpuType();
	switch (cpu_type) {
	// On Zen 2, PEXTRQ uses 2 micro-ops, including one on the vector store port
	// which data movement from the vector registers to the integer registers
	// (where CRC32C happens) to crowd the same units as vector stores. As a
	// result, using that path exclusively causes bottlenecking on this port.
	// We can avoid this bottleneck by using the integer side of the CPU for
	// most operations rather than the vector side. We keep a vector region to
	// engage some of the prefetching logic in the cache hierarchy which seems
	// to give vector instructions special treatment. These prefetch units see
	// strided access to each region, and do the right thing.
	case CpuType::kAmdRome:
	case CpuType::kAmdNaples:
	case CpuType::kAmdMilan:
	case CpuType::kAmdGenoa:
	case CpuType::kAmdRyzenV3000:
	return {
	/.temporal=/new AcceleratedCrcMemcpyEngine<1, 2>(),
	/.non_temporal=/new CrcNonTemporalMemcpyAVXEngine(),
	};
	// PCLMULQDQ is slow and we don't have wide enough issue width to take
	// advantage of it. For an unknown architecture, don't risk using CLMULs.
	case CpuType::kIntelCascadelakeXeon:
	case CpuType::kIntelSkylakeXeon:
	case CpuType::kIntelSkylake:
	case CpuType::kIntelBroadwell:
	case CpuType::kIntelHaswell:
	case CpuType::kIntelIvybridge:
	return {
	/.temporal=/new AcceleratedCrcMemcpyEngine<3, 0>(),
	/.non_temporal=/new CrcNonTemporalMemcpyAVXEngine(),
	};
	// INTEL_SANDYBRIDGE performs better with SSE than AVX.
	case CpuType::kIntelSandybridge:
	return {
	/.temporal=/new AcceleratedCrcMemcpyEngine<3, 0>(),
	/.non_temporal=/new CrcNonTemporalMemcpyEngine(),
	};
	default:
	return {/.temporal=/new FallbackCrcMemcpyEngine(),
	/.non_temporal=/new FallbackCrcMemcpyEngine()};
	}
	#endif // UNDEFINED_BEHAVIOR_SANITIZER
	}

	// For testing, allow the user to specify which engine they want.
	std::unique_ptr<CrcMemcpyEngine> CrcMemcpy::GetTestEngine(int vector,
	int integer) {
	if (vector == 3 && integer == 0) {
	return std::make_unique<AcceleratedCrcMemcpyEngine<3, 0>>();
	} else if (vector == 1 && integer == 2) {
	return std::make_unique<AcceleratedCrcMemcpyEngine<1, 2>>();
	} else if (vector == 1 && integer == 0) {
	return std::make_unique<AcceleratedCrcMemcpyEngine<1, 0>>();
	}
	return nullptr;
	}

	} // namespace crc_internal
	ABSL_NAMESPACE_END
	} // namespace absl

	#endif // ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE \|\|
	// ABSL_INTERNAL_HAVE_ARM_ACCELERATED_CRC_MEMCPY_ENGINE