src/storage/blobfs/delivery_blob.cc - fuchsia - Git at Google

 // Copyright 2023 The Fuchsia Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 #include "src/storage/blobfs/delivery_blob.h"

 #include <lib/cksum.h>
 #include <zircon/assert.h>

 #include "src/lib/chunked-compression/chunked-decompressor.h"
 #include "src/lib/chunked-compression/multithreaded-chunked-compressor.h"
 #include "src/lib/digest/merkle-tree.h"
 #include "src/storage/blobfs/compression/configs/chunked_compression_params.h"
 #include "src/storage/blobfs/delivery_blob_private.h"

 #ifndef __APPLE__
 #include <endian.h>
 #else
 #include <machine/endian.h>
 #endif
 #include <filesystem>
 #include <thread>
 #include <type_traits>

 #include <safemath/safe_conversions.h>

 namespace {

 constexpr bool IsValidDeliveryType(blobfs::DeliveryBlobType delivery_type) {
   switch (delivery_type) {
     case blobfs::DeliveryBlobType::kType1:
       return true;
     default:
       return false;
   }
 }

 zx::result<fbl::Array<uint8_t>> CompressData(cpp20::span<const uint8_t> data,
                                              const chunked_compression::CompressionParams& params) {
   const size_t num_chunks = fbl::round_up(data.size_bytes(), params.chunk_size) / params.chunk_size;
   // If `data` spans multiple chunks, we compress each chunk in parallel.
   if (num_chunks > 1) {
     const size_t num_threads = std::min<size_t>(num_chunks, std::thread::hardware_concurrency());
     chunked_compression::MultithreadedChunkedCompressor compressor(num_threads);
     return compressor.Compress(params, data);
   }

   chunked_compression::ChunkedCompressor compressor(params);
   const size_t output_limit = params.ComputeOutputSizeLimit(data.size_bytes());
   fbl::Array<uint8_t> compressed_result = fbl::MakeArray<uint8_t>(output_limit);
   size_t compressed_size;
   const chunked_compression::Status status = compressor.Compress(
       data.data(), data.size_bytes(), compressed_result.data(), output_limit, &compressed_size);
   if (status != chunked_compression::kStatusOk) {
     return zx::error(chunked_compression::ToZxStatus(status));
   }
   return zx::ok(fbl::Array<uint8_t>(compressed_result.release(), compressed_size));
 }

 }  // namespace

 namespace blobfs {

 // Format layout assumptions.
 // **WARNING**: Use caution when updating these assertions as it may break format compatibility.
 static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__);
 static_assert(sizeof(DeliveryBlobHeader) == 12);
 static_assert(std::is_trivial_v<DeliveryBlobHeader>);
 static_assert(sizeof(MetadataType1) == 16);
 static_assert(std::is_trivial_v<MetadataType1>);

 /// Delivery blob magic number (0xfc1ab10b or "Fuchsia Blob") in big-endian.
 constexpr uint8_t kDeliveryBlobMagic[4] = {0xfc, 0x1a, 0xb1, 0x0b};

 DeliveryBlobHeader DeliveryBlobHeader::Create(DeliveryBlobType type, size_t metadata_length) {
   return {.magic = {kDeliveryBlobMagic[0], kDeliveryBlobMagic[1], kDeliveryBlobMagic[2],
                     kDeliveryBlobMagic[3]},
           .type = type,
           .header_length =
               safemath::checked_cast<uint32_t>(sizeof(DeliveryBlobHeader) + metadata_length)};
 }

 bool DeliveryBlobHeader::IsValid() const {
   return std::equal(std::begin(magic), std::end(magic), std::begin(kDeliveryBlobMagic)) &&
          IsValidDeliveryType(type);
 }

 zx::result<DeliveryBlobHeader> DeliveryBlobHeader::FromBuffer(cpp20::span<const uint8_t> buffer) {
   if (buffer.size_bytes() < sizeof(DeliveryBlobHeader)) {
     return zx::error(ZX_ERR_BUFFER_TOO_SMALL);
   }
   // We cannot guarantee `buffer` is aligned, so we must use `memcpy`.
   DeliveryBlobHeader header = {};
   std::memcpy(&header, buffer.data(), sizeof(DeliveryBlobHeader));
   if (!header.IsValid()) {
     return zx::error(ZX_ERR_IO_DATA_INTEGRITY);
   }
   return zx::ok(header);
 }

 const DeliveryBlobHeader MetadataType1::kHeader =
     DeliveryBlobHeader::Create(DeliveryBlobType::kType1, sizeof(MetadataType1));

 bool MetadataType1::IsValid(const DeliveryBlobHeader& header) const {
   constexpr size_t kExpectedHeaderLength = sizeof(DeliveryBlobHeader) + sizeof(MetadataType1);
   return
       // Validate header.
       header.IsValid()
       // Validate header length.
       && (header.header_length == kExpectedHeaderLength)
       // Validate CRC.
       && (checksum == Checksum(header))
       // Validate flags.
       && ((flags & ~kValidFlagsMask) == 0);
 }

 uint32_t MetadataType1::Checksum(const DeliveryBlobHeader& header) const {
   const uint32_t header_crc =
       ::crc32(0, reinterpret_cast<const uint8_t*>(&header), sizeof(blobfs::DeliveryBlobHeader));
   // Make a copy of `metadata` so we can zero the checksum field before the calculation.
   MetadataType1 metadata_copy = *this;
   metadata_copy.checksum = 0;
   const uint32_t metadata_crc =
       ::crc32(0, reinterpret_cast<const uint8_t*>(&metadata_copy), sizeof(MetadataType1));
   return crc32_combine(header_crc, metadata_crc, sizeof(MetadataType1));
 }

 MetadataType1 MetadataType1::Create(const DeliveryBlobHeader& header, size_t payload_length,
                                     bool is_compressed) {
   MetadataType1 metadata = {
       .payload_length = safemath::checked_cast<uint64_t>(payload_length),
       .checksum = {},
       .flags = (is_compressed ? kIsCompressed : 0),
   };
   metadata.checksum = metadata.Checksum(header);
   ZX_ASSERT(metadata.IsValid(header));
   return metadata;
 }

 zx::result<MetadataType1> MetadataType1::FromBuffer(cpp20::span<const uint8_t> buffer,
                                                     const blobfs::DeliveryBlobHeader& header) {
   if (buffer.size_bytes() < sizeof(MetadataType1)) {
     return zx::error(ZX_ERR_BUFFER_TOO_SMALL);
   }
   // We cannot guarantee `buffer` is aligned, so we must use `memcpy`.
   MetadataType1 metadata = {};
   std::memcpy(&metadata, buffer.data(), sizeof(MetadataType1));
   // Ensure the parsed metadata is valid (i.e. checksums or other sanity checks pass).
   if (!metadata.IsValid(header)) {
     return zx::error(ZX_ERR_IO_DATA_INTEGRITY);
   }
   return zx::ok(metadata);
 }

 std::string GetDeliveryBlobPath(const std::string_view& path) {
   std::filesystem::path fs_path(path);
   const std::string new_filename = std::string(kDeliveryBlobPrefix) + fs_path.filename().string();
   return fs_path.replace_filename(new_filename);
 }

 zx::result<fbl::Array<uint8_t>> GenerateDeliveryBlobType1(cpp20::span<const uint8_t> data,
                                                           std::optional<bool> compress) {
   constexpr size_t kPayloadOffset = sizeof(DeliveryBlobHeader) + sizeof(MetadataType1);

   fbl::Array<uint8_t> delivery_blob;

   if (compress.value_or(true)) {
     // WARNING: If we use different compression parameters here, the `compressed_file_size` in the
     // blob info JSON file will be incorrect.
     const chunked_compression::CompressionParams params =
         blobfs::GetDefaultChunkedCompressionParams(data.size_bytes());
     zx::result compressed_data = CompressData(data, params);
     if (compressed_data.is_error()) {
       return compressed_data.take_error();
     }
     // Only use the compressed result if it saves space or we require the payload to be compressed.
     if (compressed_data->size() < data.size_bytes() || compress.value_or(false)) {
       delivery_blob = fbl::MakeArray<uint8_t>(kPayloadOffset + compressed_data->size());
       std::memcpy(delivery_blob.data() + kPayloadOffset, compressed_data->data(),
                   compressed_data->size());
     }
   }
   const bool use_compressed_result = !delivery_blob.empty();
   if (!use_compressed_result) {
     delivery_blob = fbl::MakeArray<uint8_t>(kPayloadOffset + data.size_bytes());
   }
   // Write delivery blob header and metadata.
   std::memcpy(delivery_blob.data(), &MetadataType1::kHeader, sizeof MetadataType1::kHeader);
   const MetadataType1 metadata =
       MetadataType1::Create(MetadataType1::kHeader, delivery_blob.size() - kPayloadOffset,
                             /*is_compressed=*/use_compressed_result);
   std::memcpy(delivery_blob.data() + sizeof MetadataType1::kHeader, &metadata, sizeof metadata);
   // Copy uncompressed data into payload if we aren't using compression or we aborted compression.
   if (!use_compressed_result && !data.empty()) {
     std::memcpy(delivery_blob.data() + kPayloadOffset, data.data(), data.size_bytes());
   }
   return zx::ok(std::move(delivery_blob));
 }

 zx::result<digest::Digest> CalculateDeliveryBlobDigest(cpp20::span<const uint8_t> data) {
   zx::result header = DeliveryBlobHeader::FromBuffer(data);
   if (header.is_error()) {
     return header.take_error();
   }
   if (header->type != DeliveryBlobType::kType1) {
     return zx::error(ZX_ERR_NOT_SUPPORTED);
   }

   // Currently, Type 1 blobs have a fixed header size, but this may change if we move the seek table
   // from the payload section into the metadata.
   constexpr size_t kExpectedHeaderLength = sizeof(DeliveryBlobHeader) + sizeof(MetadataType1);
   if (data.size() < kExpectedHeaderLength) {
     return zx::error(ZX_ERR_BUFFER_TOO_SMALL);
   }
   if (header->header_length != kExpectedHeaderLength) {
     return zx::error(ZX_ERR_IO_DATA_INTEGRITY);
   }

   zx::result metadata =
       MetadataType1::FromBuffer(data.subspan(sizeof(DeliveryBlobHeader)), *header);
   if (metadata.is_error()) {
     return metadata.take_error();
   }

   cpp20::span<const uint8_t> payload = data.subspan(header->header_length);
   if (payload.size() != metadata->payload_length) {
     return zx::error(ZX_ERR_IO_DATA_INTEGRITY);
   }

   // Decompress `payload` if required, and update the span to point to the uncompressed result.
   fbl::Array<uint8_t> decompressed_result;
   if (metadata->IsCompressed() && !payload.empty()) {
     size_t unused_bytes_written;
     if (chunked_compression::Status status =
             chunked_compression::ChunkedDecompressor::DecompressBytes(
                 payload.data(), payload.size(), &decompressed_result, &unused_bytes_written);
         status != chunked_compression::kStatusOk) {
       return zx::error(chunked_compression::ToZxStatus(status));
     }
     payload = decompressed_result;
   }

   // Calculate Merkle root of `payload` which points to the uncompressed blob data (may be empty).
   std::unique_ptr<uint8_t[]> unused_tree;
   size_t unused_tree_len;
   digest::Digest root;

   if (zx_status_t status = digest::MerkleTreeCreator::Create(payload.data(), payload.size(),
                                                              &unused_tree, &unused_tree_len, &root);
       status != ZX_OK) {
     return zx::error(status);
   }

   return zx::ok(root);
 }

 }  // namespace blobfs
	// Copyright 2023 The Fuchsia Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	#include "src/storage/blobfs/delivery_blob.h"

	#include <lib/cksum.h>
	#include <zircon/assert.h>

	#include "src/lib/chunked-compression/chunked-decompressor.h"
	#include "src/lib/chunked-compression/multithreaded-chunked-compressor.h"
	#include "src/lib/digest/merkle-tree.h"
	#include "src/storage/blobfs/compression/configs/chunked_compression_params.h"
	#include "src/storage/blobfs/delivery_blob_private.h"

	#ifndef __APPLE__
	#include <endian.h>
	#else
	#include <machine/endian.h>
	#endif
	#include <filesystem>
	#include <thread>
	#include <type_traits>

	#include <safemath/safe_conversions.h>

	namespace {

	constexpr bool IsValidDeliveryType(blobfs::DeliveryBlobType delivery_type) {
	switch (delivery_type) {
	case blobfs::DeliveryBlobType::kType1:
	return true;
	default:
	return false;
	}
	}

	zx::result<fbl::Array<uint8_t>> CompressData(cpp20::span<const uint8_t> data,
	const chunked_compression::CompressionParams& params) {
	const size_t num_chunks = fbl::round_up(data.size_bytes(), params.chunk_size) / params.chunk_size;
	// If `data` spans multiple chunks, we compress each chunk in parallel.
	if (num_chunks > 1) {
	const size_t num_threads = std::min<size_t>(num_chunks, std::thread::hardware_concurrency());
	chunked_compression::MultithreadedChunkedCompressor compressor(num_threads);
	return compressor.Compress(params, data);
	}

	chunked_compression::ChunkedCompressor compressor(params);
	const size_t output_limit = params.ComputeOutputSizeLimit(data.size_bytes());
	fbl::Array<uint8_t> compressed_result = fbl::MakeArray<uint8_t>(output_limit);
	size_t compressed_size;
	const chunked_compression::Status status = compressor.Compress(
	data.data(), data.size_bytes(), compressed_result.data(), output_limit, &compressed_size);
	if (status != chunked_compression::kStatusOk) {
	return zx::error(chunked_compression::ToZxStatus(status));
	}
	return zx::ok(fbl::Array<uint8_t>(compressed_result.release(), compressed_size));
	}

	} // namespace

	namespace blobfs {

	// Format layout assumptions.
	// WARNING: Use caution when updating these assertions as it may break format compatibility.
	static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__);
	static_assert(sizeof(DeliveryBlobHeader) == 12);
	static_assert(std::is_trivial_v<DeliveryBlobHeader>);
	static_assert(sizeof(MetadataType1) == 16);
	static_assert(std::is_trivial_v<MetadataType1>);

	/// Delivery blob magic number (0xfc1ab10b or "Fuchsia Blob") in big-endian.
	constexpr uint8_t kDeliveryBlobMagic[4] = {0xfc, 0x1a, 0xb1, 0x0b};

	DeliveryBlobHeader DeliveryBlobHeader::Create(DeliveryBlobType type, size_t metadata_length) {
	return {.magic = {kDeliveryBlobMagic[0], kDeliveryBlobMagic[1], kDeliveryBlobMagic[2],
	kDeliveryBlobMagic[3]},
	.type = type,
	.header_length =
	safemath::checked_cast<uint32_t>(sizeof(DeliveryBlobHeader) + metadata_length)};
	}

	bool DeliveryBlobHeader::IsValid() const {
	return std::equal(std::begin(magic), std::end(magic), std::begin(kDeliveryBlobMagic)) &&
	IsValidDeliveryType(type);
	}

	zx::result<DeliveryBlobHeader> DeliveryBlobHeader::FromBuffer(cpp20::span<const uint8_t> buffer) {
	if (buffer.size_bytes() < sizeof(DeliveryBlobHeader)) {
	return zx::error(ZX_ERR_BUFFER_TOO_SMALL);
	}
	// We cannot guarantee `buffer` is aligned, so we must use `memcpy`.
	DeliveryBlobHeader header = {};
	std::memcpy(&header, buffer.data(), sizeof(DeliveryBlobHeader));
	if (!header.IsValid()) {
	return zx::error(ZX_ERR_IO_DATA_INTEGRITY);
	}
	return zx::ok(header);
	}

	const DeliveryBlobHeader MetadataType1::kHeader =
	DeliveryBlobHeader::Create(DeliveryBlobType::kType1, sizeof(MetadataType1));

	bool MetadataType1::IsValid(const DeliveryBlobHeader& header) const {
	constexpr size_t kExpectedHeaderLength = sizeof(DeliveryBlobHeader) + sizeof(MetadataType1);
	return
	// Validate header.
	header.IsValid()
	// Validate header length.
	&& (header.header_length == kExpectedHeaderLength)
	// Validate CRC.
	&& (checksum == Checksum(header))
	// Validate flags.
	&& ((flags & ~kValidFlagsMask) == 0);
	}

	uint32_t MetadataType1::Checksum(const DeliveryBlobHeader& header) const {
	const uint32_t header_crc =
	::crc32(0, reinterpret_cast<const uint8_t*>(&header), sizeof(blobfs::DeliveryBlobHeader));
	// Make a copy of `metadata` so we can zero the checksum field before the calculation.
	MetadataType1 metadata_copy = *this;
	metadata_copy.checksum = 0;
	const uint32_t metadata_crc =
	::crc32(0, reinterpret_cast<const uint8_t*>(&metadata_copy), sizeof(MetadataType1));
	return crc32_combine(header_crc, metadata_crc, sizeof(MetadataType1));
	}

	MetadataType1 MetadataType1::Create(const DeliveryBlobHeader& header, size_t payload_length,
	bool is_compressed) {
	MetadataType1 metadata = {
	.payload_length = safemath::checked_cast<uint64_t>(payload_length),
	.checksum = {},
	.flags = (is_compressed ? kIsCompressed : 0),
	};
	metadata.checksum = metadata.Checksum(header);
	ZX_ASSERT(metadata.IsValid(header));
	return metadata;
	}

	zx::result<MetadataType1> MetadataType1::FromBuffer(cpp20::span<const uint8_t> buffer,
	const blobfs::DeliveryBlobHeader& header) {
	if (buffer.size_bytes() < sizeof(MetadataType1)) {
	return zx::error(ZX_ERR_BUFFER_TOO_SMALL);
	}
	// We cannot guarantee `buffer` is aligned, so we must use `memcpy`.
	MetadataType1 metadata = {};
	std::memcpy(&metadata, buffer.data(), sizeof(MetadataType1));
	// Ensure the parsed metadata is valid (i.e. checksums or other sanity checks pass).
	if (!metadata.IsValid(header)) {
	return zx::error(ZX_ERR_IO_DATA_INTEGRITY);
	}
	return zx::ok(metadata);
	}

	std::string GetDeliveryBlobPath(const std::string_view& path) {
	std::filesystem::path fs_path(path);
	const std::string new_filename = std::string(kDeliveryBlobPrefix) + fs_path.filename().string();
	return fs_path.replace_filename(new_filename);
	}

	zx::result<fbl::Array<uint8_t>> GenerateDeliveryBlobType1(cpp20::span<const uint8_t> data,
	std::optional<bool> compress) {
	constexpr size_t kPayloadOffset = sizeof(DeliveryBlobHeader) + sizeof(MetadataType1);

	fbl::Array<uint8_t> delivery_blob;

	if (compress.value_or(true)) {
	// WARNING: If we use different compression parameters here, the `compressed_file_size` in the
	// blob info JSON file will be incorrect.
	const chunked_compression::CompressionParams params =
	blobfs::GetDefaultChunkedCompressionParams(data.size_bytes());
	zx::result compressed_data = CompressData(data, params);
	if (compressed_data.is_error()) {
	return compressed_data.take_error();
	}
	// Only use the compressed result if it saves space or we require the payload to be compressed.
	if (compressed_data->size() < data.size_bytes() \|\| compress.value_or(false)) {
	delivery_blob = fbl::MakeArray<uint8_t>(kPayloadOffset + compressed_data->size());
	std::memcpy(delivery_blob.data() + kPayloadOffset, compressed_data->data(),
	compressed_data->size());
	}
	}
	const bool use_compressed_result = !delivery_blob.empty();
	if (!use_compressed_result) {
	delivery_blob = fbl::MakeArray<uint8_t>(kPayloadOffset + data.size_bytes());
	}
	// Write delivery blob header and metadata.
	std::memcpy(delivery_blob.data(), &MetadataType1::kHeader, sizeof MetadataType1::kHeader);
	const MetadataType1 metadata =
	MetadataType1::Create(MetadataType1::kHeader, delivery_blob.size() - kPayloadOffset,
	/is_compressed=/use_compressed_result);
	std::memcpy(delivery_blob.data() + sizeof MetadataType1::kHeader, &metadata, sizeof metadata);
	// Copy uncompressed data into payload if we aren't using compression or we aborted compression.
	if (!use_compressed_result && !data.empty()) {
	std::memcpy(delivery_blob.data() + kPayloadOffset, data.data(), data.size_bytes());
	}
	return zx::ok(std::move(delivery_blob));
	}

	zx::result<digest::Digest> CalculateDeliveryBlobDigest(cpp20::span<const uint8_t> data) {
	zx::result header = DeliveryBlobHeader::FromBuffer(data);
	if (header.is_error()) {
	return header.take_error();
	}
	if (header->type != DeliveryBlobType::kType1) {
	return zx::error(ZX_ERR_NOT_SUPPORTED);
	}

	// Currently, Type 1 blobs have a fixed header size, but this may change if we move the seek table
	// from the payload section into the metadata.
	constexpr size_t kExpectedHeaderLength = sizeof(DeliveryBlobHeader) + sizeof(MetadataType1);
	if (data.size() < kExpectedHeaderLength) {
	return zx::error(ZX_ERR_BUFFER_TOO_SMALL);
	}
	if (header->header_length != kExpectedHeaderLength) {
	return zx::error(ZX_ERR_IO_DATA_INTEGRITY);
	}

	zx::result metadata =
	MetadataType1::FromBuffer(data.subspan(sizeof(DeliveryBlobHeader)), *header);
	if (metadata.is_error()) {
	return metadata.take_error();
	}

	cpp20::span<const uint8_t> payload = data.subspan(header->header_length);
	if (payload.size() != metadata->payload_length) {
	return zx::error(ZX_ERR_IO_DATA_INTEGRITY);
	}

	// Decompress `payload` if required, and update the span to point to the uncompressed result.
	fbl::Array<uint8_t> decompressed_result;
	if (metadata->IsCompressed() && !payload.empty()) {
	size_t unused_bytes_written;
	if (chunked_compression::Status status =
	chunked_compression::ChunkedDecompressor::DecompressBytes(
	payload.data(), payload.size(), &decompressed_result, &unused_bytes_written);
	status != chunked_compression::kStatusOk) {
	return zx::error(chunked_compression::ToZxStatus(status));
	}
	payload = decompressed_result;
	}

	// Calculate Merkle root of `payload` which points to the uncompressed blob data (may be empty).
	std::unique_ptr<uint8_t[]> unused_tree;
	size_t unused_tree_len;
	digest::Digest root;

	if (zx_status_t status = digest::MerkleTreeCreator::Create(payload.data(), payload.size(),
	&unused_tree, &unused_tree_len, &root);
	status != ZX_OK) {
	return zx::error(status);
	}

	return zx::ok(root);
	}

	} // namespace blobfs