[volume_image]: Lz4DecompressReader.
Allows streaming a compressed sparse image without decompressing
entirely beforehand.
Fixed decompression where, the decompressor would resize its internal
buffer to match whatever size the hint requested. This is wrong, since
it would break expected bounds. That aside, the internal decompressor
has its own decompressor buffers, aligned to the compression blocks.
Removed harmlesss incorrect behavior. Decompressing a compressed sparse
fvm flag, would remove the no fill required flag from the header. This
is not correct, since when a compressed image is generated, all zeroes
are emitted, and left for the compressor to reduce the space. This was
harmless because it just resulted in more zeroes being written when
writing a decompressed sparse image.
Update default constructor for Lz4Compressor, to set the default
arguments for the LZ4_preferences_t required to compress.
Test: storage-volume-image-test
Change-Id: Ibeb0bec400f722a47d036e581e4586fd48bfc04d
Reviewed-on: https://fuchsia-review.googlesource.com/c/fuchsia/+/513512
Reviewed-by: James Sullivan <jfsulliv@google.com>
Commit-Queue: Gianfranco Valentino <gevalentino@google.com>
diff --git a/src/storage/volume_image/fvm/fvm_sparse_image.cc b/src/storage/volume_image/fvm/fvm_sparse_image.cc
index 04506e87..fc81df3 100644
--- a/src/storage/volume_image/fvm/fvm_sparse_image.cc
+++ b/src/storage/volume_image/fvm/fvm_sparse_image.cc
@@ -598,7 +598,6 @@
}
// Remove the compression flag.
header_or.value().flags ^= fvm::kSparseFlagLz4;
- header_or.value().flags ^= fvm::kSparseFlagZeroFillNotRequired;
memcpy(metadata_buffer.data(), &header_or.value(), sizeof(fvm::SparseImage));
auto metadata_write_result = writer.Write(0, metadata_buffer);
diff --git a/src/storage/volume_image/fvm/fvm_sparse_image_test.cc b/src/storage/volume_image/fvm/fvm_sparse_image_test.cc
index 0918fbe..4e6d410 100644
--- a/src/storage/volume_image/fvm/fvm_sparse_image_test.cc
+++ b/src/storage/volume_image/fvm/fvm_sparse_image_test.cc
@@ -1662,6 +1662,11 @@
write_result = FvmSparseWriteImage(decompressed_descriptor, &expected_container.writer());
ASSERT_TRUE(write_result.is_ok()) << write_result.error();
+ // When decompressing this flag should remain, since the zeroes where already emitted as part of
+ // the compressed image, and the were decompressed. In general, not keeping this flag, would
+ // apply fill to all extents, even those who do not need it.
+ expected_container.serialized_image().header.flags |= fvm::kSparseFlagZeroFillNotRequired;
+
auto decompress_or = FvmSparseDecompressImage(
0, BufferReader(0, &compressed_container.serialized_image(), sizeof(SerializedSparseImage)),
decompressed_container.writer());
diff --git a/src/storage/volume_image/utils/BUILD.gn b/src/storage/volume_image/utils/BUILD.gn
index 8e363c2..7130600 100644
--- a/src/storage/volume_image/utils/BUILD.gn
+++ b/src/storage/volume_image/utils/BUILD.gn
@@ -47,6 +47,36 @@
]
}
+source_set("lz4-decompress-reader") {
+ sources = [ "lz4_decompress_reader.cc" ]
+ public = [ "lz4_decompress_reader.h" ]
+ public_deps = [
+ ":io-interfaces",
+ ":lz4-compression",
+ "//sdk/lib/fit",
+ "//zircon/public/lib/fbl",
+ ]
+}
+
+source_set("lz4-decompress-reader-test") {
+ testonly = true
+ sources = [ "lz4_decompress_reader_test.cc" ]
+ deps = [
+ ":fd-io",
+ ":fd-test-helper",
+ ":lz4-decompress-reader",
+ "//src/storage/volume_image/fvm:fvm-sparse-image",
+ "//third_party/googletest:gmock",
+ "//third_party/googletest:gtest",
+ ]
+
+ configs += [ "//src/storage/volume_image/adapter:test-image-path" ]
+
+ if (is_host) {
+ deps += [ "//src/storage/volume_image/adapter:host-test-images" ]
+ }
+}
+
source_set("guid") {
sources = [ "guid.cc" ]
public = [ "guid.h" ]
@@ -174,5 +204,6 @@
":fd-io-test",
":guid-test",
":lz4-compression-test",
+ ":lz4-decompress-reader-test",
]
}
diff --git a/src/storage/volume_image/utils/lz4_compressor.cc b/src/storage/volume_image/utils/lz4_compressor.cc
index 15086f9..78052f7 100644
--- a/src/storage/volume_image/utils/lz4_compressor.cc
+++ b/src/storage/volume_image/utils/lz4_compressor.cc
@@ -48,6 +48,10 @@
} // namespace
+Lz4Compressor::Lz4Compressor()
+ : Lz4Compressor(
+ ConvertOptionsToPreferences(CompressionOptions{.schema = CompressionSchema::kLz4})) {}
+
Lz4Compressor::~Lz4Compressor() {
if (context_ != nullptr) {
LZ4F_freeCompressionContext(context_);
diff --git a/src/storage/volume_image/utils/lz4_compressor.h b/src/storage/volume_image/utils/lz4_compressor.h
index 6b64507..fab24ba 100644
--- a/src/storage/volume_image/utils/lz4_compressor.h
+++ b/src/storage/volume_image/utils/lz4_compressor.h
@@ -29,7 +29,7 @@
// On failure, returns a string describing the error.
static fit::result<Lz4Compressor, std::string> Create(const CompressionOptions& options);
- Lz4Compressor() = default;
+ Lz4Compressor();
explicit Lz4Compressor(const Preferences& preferences) : preferences_(preferences) {}
Lz4Compressor(const Lz4Compressor&) = delete;
Lz4Compressor(Lz4Compressor&&) noexcept = default;
diff --git a/src/storage/volume_image/utils/lz4_decompress_reader.cc b/src/storage/volume_image/utils/lz4_decompress_reader.cc
new file mode 100644
index 0000000..bd09483
--- /dev/null
+++ b/src/storage/volume_image/utils/lz4_decompress_reader.cc
@@ -0,0 +1,152 @@
+// Copyright 2021 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/storage/volume_image/utils/lz4_decompress_reader.h"
+
+#include <lib/fit/result.h>
+#include <string.h>
+
+#include <cstdint>
+#include <iostream>
+#include <memory>
+
+#include "src/storage/volume_image/utils/lz4_decompressor.h"
+
+namespace storage::volume_image {
+
+fit::result<void, std::string> Lz4DecompressReader::Initialize(uint64_t max_buffer_size) const {
+ context_.decompressed_data.resize(max_buffer_size, 0);
+ context_.decompressed_offset = offset_;
+ context_.decompressed_length = 0;
+
+ context_.compressed_data.resize(max_buffer_size, 0);
+ context_.compressed_offset = offset_;
+
+ context_.decompressor = std::make_unique<Lz4Decompressor>();
+ context_.hint = std::nullopt;
+ context_.decompressor->ProvideSizeHint(max_buffer_size);
+
+ return context_.decompressor->Prepare(
+ [this](auto decompressed_data) { return this->DecompressionHandler(decompressed_data); });
+}
+
+fit::result<void, std::string> Lz4DecompressReader::DecompressionHandler(
+ fbl::Span<const uint8_t> decompressed_data) const {
+ memcpy(context_.decompressed_data.data(), decompressed_data.data(), decompressed_data.size());
+ context_.decompressed_offset += context_.decompressed_length;
+ context_.decompressed_length = decompressed_data.size();
+
+ return fit::ok();
+}
+
+fit::result<void, std::string> Lz4DecompressReader::Seek(uint64_t offset) const {
+ // Offset in uncompressed area.
+ if (offset < offset_) {
+ return fit::ok();
+ }
+
+ if (offset < context_.decompressed_offset) {
+ if (auto result = Initialize(context_.decompressed_data.size()); result.is_error()) {
+ return result.take_error_result();
+ }
+ }
+
+ // Offset is in range.
+ auto offset_in_range = [&]() {
+ return context_.decompressed_length > 0 && offset >= context_.decompressed_offset &&
+ offset < context_.decompressed_offset + context_.decompressed_length;
+ };
+
+ auto end_of_compressed_data = [&]() {
+ return context_.compressed_offset == compressed_reader_->length();
+ };
+
+ auto end_of_frame = [&]() { return context_.hint.has_value() && context_.hint.value() == 0; };
+
+ // Decompress until offset is in range.
+ while (!offset_in_range() && !end_of_frame() && !end_of_compressed_data()) {
+ if (auto result = NextDecompressedChunk(); result.is_error()) {
+ return result;
+ }
+ }
+
+ if (!offset_in_range() && (end_of_frame() || end_of_compressed_data())) {
+ return fit::error("Reached end of compressed data before reaching offset.");
+ };
+ return fit::ok();
+}
+
+fit::result<void, std::string> Lz4DecompressReader::NextDecompressedChunk() const {
+ auto read_view = fbl::Span<uint8_t>(context_.compressed_data);
+ uint64_t remaining_compressed_bytes = compressed_reader_->length() - context_.compressed_offset;
+
+ if (read_view.size() > remaining_compressed_bytes) {
+ read_view = read_view.subspan(0, remaining_compressed_bytes);
+ }
+
+ if (context_.hint.has_value() && read_view.size() > context_.hint.value()) {
+ read_view = read_view.subspan(0, context_.hint.value());
+ }
+
+ if (auto result = compressed_reader_->Read(context_.compressed_offset, read_view);
+ result.is_error()) {
+ return result;
+ }
+
+ auto decompress_result = context_.decompressor->Decompress(read_view);
+ if (decompress_result.is_error()) {
+ return decompress_result.take_error_result();
+ }
+
+ auto [hint, consumed_bytes] = decompress_result.value();
+ context_.hint = hint;
+ context_.compressed_offset += consumed_bytes;
+ return fit::ok();
+}
+
+fit::result<void, std::string> Lz4DecompressReader::Read(uint64_t offset,
+ fbl::Span<uint8_t> buffer) const {
+ // Base recursion case.
+ if (buffer.empty()) {
+ return fit::ok();
+ }
+
+ // Attempting to read out of the uncompressed range.
+ if (offset < offset_) {
+ uint64_t uncompressed_bytes = offset_ - offset;
+ uint64_t uncompressed_bytes_to_copy =
+ std::min(static_cast<uint64_t>(buffer.size()), uncompressed_bytes);
+ if (auto result =
+ compressed_reader_->Read(offset, buffer.subspan(0, uncompressed_bytes_to_copy));
+ result.is_error()) {
+ return result;
+ }
+
+ offset += uncompressed_bytes_to_copy;
+ buffer = buffer.subspan(uncompressed_bytes_to_copy);
+ if (buffer.empty()) {
+ return fit::ok();
+ }
+ }
+
+ while (!buffer.empty()) {
+ if (auto result = Seek(offset); result.is_error()) {
+ return result;
+ }
+
+ // Now the data is in the buffer, or at least some of it.
+ uint64_t decompressed_buffer_offset = offset - context_.decompressed_offset;
+ uint64_t decompressed_buffer_bytes = context_.decompressed_length - decompressed_buffer_offset;
+ uint64_t decompressed_bytes_to_copy =
+ std::min(static_cast<uint64_t>(buffer.size()), decompressed_buffer_bytes);
+ memcpy(buffer.data(), context_.decompressed_data.data() + decompressed_buffer_offset,
+ decompressed_bytes_to_copy);
+
+ offset += decompressed_bytes_to_copy;
+ buffer = buffer.subspan(decompressed_bytes_to_copy);
+ }
+ return fit::ok();
+}
+
+} // namespace storage::volume_image
diff --git a/src/storage/volume_image/utils/lz4_decompress_reader.h b/src/storage/volume_image/utils/lz4_decompress_reader.h
new file mode 100644
index 0000000..8380f09
--- /dev/null
+++ b/src/storage/volume_image/utils/lz4_decompress_reader.h
@@ -0,0 +1,81 @@
+// Copyright 2021 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef SRC_STORAGE_VOLUME_IMAGE_UTILS_LZ4_DECOMPRESS_READER_H_
+#define SRC_STORAGE_VOLUME_IMAGE_UTILS_LZ4_DECOMPRESS_READER_H_
+
+#include <cstdint>
+#include <limits>
+#include <memory>
+#include <string>
+
+#include <fbl/span.h>
+
+#include "src/storage/volume_image/utils/lz4_decompressor.h"
+#include "src/storage/volume_image/utils/reader.h"
+
+namespace storage::volume_image {
+
+// Provides a decompressed view of the underlying compressed data.
+class Lz4DecompressReader final : public Reader {
+ public:
+ // Default size for |StreamContext| buffers.
+ static constexpr uint64_t kMaxBufferSize = 20 * (1 << 10);
+
+ // Lz4DecompressReader will decompress data starting at |offset|. That is the compressed data is
+ // embedded in |compressed_reader| and the first compressed byte is at |offset|.
+ Lz4DecompressReader(uint64_t offset, uint64_t decompressed_length,
+ std::shared_ptr<Reader> compressed_reader)
+ : offset_(offset),
+ length_(decompressed_length),
+ compressed_reader_(std::move(compressed_reader)) {}
+
+ // Initializes the underlying |StreamContext|.
+ fit::result<void, std::string> Initialize(uint64_t max_buffer_size = kMaxBufferSize) const;
+
+ // Returns the number of bytes readable from this reader.
+ uint64_t length() const final { return length_; }
+
+ // On success data at [|offset|, |offset| + |buffer.size()|] are read into
+ // |buffer|.
+ //
+ // On error the returned result to contains a string describing the error.
+ fit::result<void, std::string> Read(uint64_t offset, fbl::Span<uint8_t> buffer) const final;
+
+ private:
+ fit::result<void, std::string> DecompressionHandler(
+ fbl::Span<const uint8_t> decompressed_data) const;
+
+ fit::result<void, std::string> Seek(uint64_t offset) const;
+
+ fit::result<void, std::string> NextDecompressedChunk() const;
+
+ // Describes the current state of the decompression stream.
+ struct StreamContext {
+ std::vector<uint8_t> compressed_data;
+ uint64_t compressed_offset = 0;
+
+ std::vector<uint8_t> decompressed_data;
+ uint64_t decompressed_offset = 0;
+ uint64_t decompressed_length = 0;
+
+ std::optional<uint64_t> hint = std::nullopt;
+
+ std::unique_ptr<Lz4Decompressor> decompressor = nullptr;
+ };
+
+ // Reinitializes the streaming context.
+ fit::result<void, std::string> ResetStreamContext();
+
+ uint64_t offset_ = 0;
+ uint64_t length_ = 0;
+ std::shared_ptr<Reader> compressed_reader_ = nullptr;
+
+ // Mutable since this will never change the contents of a given range in the exposed view.
+ mutable StreamContext context_;
+};
+
+} // namespace storage::volume_image
+
+#endif // SRC_STORAGE_VOLUME_IMAGE_UTILS_LZ4_DECOMPRESS_READER_H_
diff --git a/src/storage/volume_image/utils/lz4_decompress_reader_test.cc b/src/storage/volume_image/utils/lz4_decompress_reader_test.cc
new file mode 100644
index 0000000..f23022c
--- /dev/null
+++ b/src/storage/volume_image/utils/lz4_decompress_reader_test.cc
@@ -0,0 +1,291 @@
+// Copyright 2021 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/storage/volume_image/utils/lz4_decompress_reader.h"
+
+#include <sys/types.h>
+
+#include <cstdint>
+#include <string_view>
+
+#include <fbl/span.h>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "src/storage/fvm/fvm_sparse.h"
+#include "src/storage/volume_image/fvm/fvm_sparse_image.h"
+#include "src/storage/volume_image/utils/fd_reader.h"
+#include "src/storage/volume_image/utils/fd_test_helper.h"
+#include "src/storage/volume_image/utils/fd_writer.h"
+#include "src/storage/volume_image/utils/lz4_compressor.h"
+#include "src/storage/volume_image/utils/lz4_decompressor.h"
+
+namespace storage::volume_image {
+namespace {
+// Path to a compressed sparse image.
+constexpr std::string_view kFvmSparseImagePath =
+ STORAGE_VOLUME_IMAGE_ADAPTER_TEST_IMAGE_PATH "test_fvm.sparse.blk";
+
+constexpr std::string_view kLoremIpsum =
+ R"(Lorem ipsum dolor sit amet, consectetur adipiscing elit,
+ sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+ Elit pellentesque habitant morbi tristique senectus et netus et. Blandit
+ aliquam etiam erat velit scelerisque in. Placerat orci nulla pellentesque
+ dignissim enim sit amet. Suspendisse ultrices gravida dictum fusce ut placerat
+ orci. Pretium aenean pharetra magna ac placerat vestibulum lectus mauris ultrices.
+ Nibh venenatis cras sed felis eget velit aliquet sagittis. Risus quis varius quam
+ quisque id diam vel. Sed enim ut sem viverra. Fusce id velit ut tortor pretium.
+ Amet dictum sit amet justo donec enim diam vulputate ut. Faucibus scelerisque eleifend
+ donec pretium vulputate sapien nec. Curabitur gravida arcu ac tortor dignissim
+ convallis aenean. Morbi non arcu risus quis varius quam quisque. Vitae suscipit
+ tellus mauris a diam maecenas. Mattis enim ut tellus elementum sagittis vitae et leo
+ duis. Lacinia quis vel eros donec ac odio.
+
+ Feugiat in ante metus dictum at. Amet nisl suscipit adipiscing bibendum est.
+ Bibendum ut tristique et egestas quis ipsum suspendisse ultrices. Sed euismod nisi
+ porta lorem mollis aliquam ut porttitor leo. Libero id faucibus nisl tincidunt eget.
+ Gravida in fermentum et sollicitudin ac orci. Accumsan sit amet nulla facilisi morbi
+ tempus. Sed euismod nisi porta lorem mollis aliquam ut. Sed velit dignissim sodales
+ ut eu sem integer. Purus in massa tempor nec feugiat nisl pretium. Eros in cursus
+ turpis massa.
+
+ A diam maecenas sed enim ut. Leo in vitae turpis massa sed. Lobortis scelerisque
+ fermentum dui faucibus in ornare. Nullam eget felis eget nunc lobortis mattis. A cras
+ semper auctor neque vitae tempus. Dignissim suspendisse in est ante in nibh mauris
+ cursus. Dictumst quisque sagittis purus sit amet volutpat consequat mauris nunc. Vel
+ quam elementum pulvinar etiam non quam lacus suspendisse faucibus. Libero just
+ laoreet sit amet cursus sit amet. Imperdiet dui accumsan sit amet nulla. Platea
+ dictumst quisque sagittis purus. Lobortis mattis aliquam faucibus purus in massa. Nec
+ sagittis aliquam malesuada bibendum. Eu sem integer vitae justo. Sit amet dictum sit
+ amet justo donec enim. Aliquet sagittis id consectetur purus ut faucibus pulvinar
+ elementum integer. Diam vulputate ut pharetra sit amet aliquam. At consectetur lorem
+ donec massa sapien faucibus et.)";
+
+fit::result<std::vector<uint8_t>, std::string> CompressedData(
+ fbl::Span<const uint8_t> source_data) {
+ std::vector<uint8_t> compressed_data;
+ Lz4Compressor compressor;
+ if (auto result =
+ compressor.Prepare([&compressed_data](fbl::Span<const uint8_t> compressed_chunk) {
+ compressed_data.insert(compressed_data.end(), compressed_chunk.begin(),
+ compressed_chunk.end());
+ return fit::ok();
+ });
+ result.is_error()) {
+ return result.take_error_result();
+ }
+
+ if (auto result = compressor.Compress(source_data); result.is_error()) {
+ return result.take_error_result();
+ }
+
+ if (auto result = compressor.Finalize(); result.is_error()) {
+ return result.take_error_result();
+ }
+
+ return fit::ok(compressed_data);
+}
+
+// Compressed Reader.
+class FakeReader : public Reader {
+ public:
+ FakeReader(std::vector<uint8_t> data) : data_(data) {}
+
+ uint64_t length() const final { return data_.size(); }
+
+ fit::result<void, std::string> Read(uint64_t offset, fbl::Span<uint8_t> buffer) const final {
+ if (buffer.empty()) {
+ return fit::ok();
+ }
+ if (offset + buffer.size() > data_.size()) {
+ return fit::error("FakeReader::Read out of bounds.");
+ }
+ memcpy(buffer.data(), data_.data() + offset, buffer.size());
+ return fit::ok();
+ }
+
+ private:
+ std::vector<uint8_t> data_;
+};
+
+constexpr uint64_t kUncompressedDataPrefix = 128;
+constexpr uint64_t kDecompressedLength = kUncompressedDataPrefix + kLoremIpsum.size();
+constexpr uint64_t kMaxBufferLength = kUncompressedDataPrefix + 1;
+// constexpr uint64_t kMaxReadBufferLength = kUncompressedDataPrefix / 3;
+
+fit::result<std::vector<uint8_t>, std::string> GetData() {
+ auto data_or = CompressedData(fbl::Span<const uint8_t>(
+ reinterpret_cast<const uint8_t*>(kLoremIpsum.data()), kLoremIpsum.size()));
+ if (data_or.is_error()) {
+ return data_or.take_error_result();
+ }
+ auto data = data_or.take_value();
+
+ data.insert(data.begin(), kLoremIpsum.begin(), kLoremIpsum.begin() + kUncompressedDataPrefix);
+ return fit::ok(data);
+}
+
+void CheckRangeMatch(uint64_t offset, const Reader& reader,
+ fbl::Span<const uint8_t> expected_data) {
+ uint64_t bytes_to_read = reader.length() - offset;
+ if (bytes_to_read > expected_data.size()) {
+ bytes_to_read = expected_data.size();
+ }
+
+ std::vector<uint8_t> data;
+ data.resize(bytes_to_read, 0);
+
+ auto result = reader.Read(offset, data);
+ ASSERT_TRUE(result.is_ok()) << result.error();
+
+ EXPECT_TRUE(memcmp(data.data(), expected_data.data(), bytes_to_read) == 0);
+
+ if (data.empty()) {
+ return;
+ }
+}
+
+TEST(Lz4DecompressReaderTest, ReadingUncompressedAreaIsOk) {
+ auto data_or = GetData();
+ ASSERT_TRUE(data_or.is_ok()) << data_or.error();
+ auto data = data_or.take_value();
+
+ std::shared_ptr<FakeReader> compressed_reader = std::make_shared<FakeReader>(data);
+ Lz4DecompressReader decompressed_reader(kUncompressedDataPrefix, kDecompressedLength,
+ compressed_reader);
+ auto init_result = decompressed_reader.Initialize(kMaxBufferLength);
+ ASSERT_TRUE(init_result.is_ok()) << init_result.error();
+
+ auto view = fbl::Span<uint8_t>(data);
+
+ // Read part of uncompressed data only.
+ ASSERT_NO_FATAL_FAILURE(
+ CheckRangeMatch(0, decompressed_reader, view.subspan(0, kUncompressedDataPrefix / 4)));
+
+ // The entire uncompressed data.
+ ASSERT_NO_FATAL_FAILURE(
+ CheckRangeMatch(0, decompressed_reader, view.subspan(0, kUncompressedDataPrefix)));
+}
+
+TEST(Lz4DecompressReaderTest, ReadingCompressedAreaIsOk) {
+ auto data_or = GetData();
+ ASSERT_TRUE(data_or.is_ok()) << data_or.error();
+ auto data = data_or.take_value();
+
+ std::shared_ptr<FakeReader> compressed_reader = std::make_shared<FakeReader>(data);
+ Lz4DecompressReader decompressed_reader(kUncompressedDataPrefix, kDecompressedLength,
+ compressed_reader);
+ auto init_result = decompressed_reader.Initialize(kMaxBufferLength);
+ ASSERT_TRUE(init_result.is_ok()) << init_result.error();
+
+ auto lorem_ipsum = fbl::Span<const uint8_t>(reinterpret_cast<const uint8_t*>(kLoremIpsum.data()),
+ kLoremIpsum.size());
+
+ // Random chunk.
+ ASSERT_NO_FATAL_FAILURE(CheckRangeMatch(kUncompressedDataPrefix + 500, decompressed_reader,
+ lorem_ipsum.subspan(500)));
+
+ // Read part of uncompressed data only.
+ ASSERT_NO_FATAL_FAILURE(
+ CheckRangeMatch(kUncompressedDataPrefix, decompressed_reader, lorem_ipsum.subspan(0, 1)));
+
+ // The entire uncompressed data.
+ ASSERT_NO_FATAL_FAILURE(
+ CheckRangeMatch(kUncompressedDataPrefix, decompressed_reader, lorem_ipsum));
+}
+
+TEST(Lz4DecompressReaderTest, ReadingBothAreasIsOk) {
+ auto data_or = GetData();
+ ASSERT_TRUE(data_or.is_ok()) << data_or.error();
+ auto data = data_or.take_value();
+
+ std::shared_ptr<FakeReader> compressed_reader = std::make_shared<FakeReader>(data);
+ Lz4DecompressReader decompressed_reader(kUncompressedDataPrefix, kDecompressedLength,
+ compressed_reader);
+ auto init_result = decompressed_reader.Initialize(kMaxBufferLength);
+ ASSERT_TRUE(init_result.is_ok()) << init_result.error();
+
+ std::array<uint8_t, 2> expected_data = {data[kUncompressedDataPrefix - 1], kLoremIpsum[0]};
+
+ // The entire uncompressed data.
+ ASSERT_NO_FATAL_FAILURE(
+ CheckRangeMatch(kUncompressedDataPrefix - 1, decompressed_reader, expected_data));
+}
+
+TEST(Lz4DecompressReaderTest, DecompressingSparseFvmIsOk) {
+ auto decompressed_image_or = TempFile::Create();
+ ASSERT_TRUE(decompressed_image_or.is_ok()) << decompressed_image_or.error();
+ auto decompressed_image = decompressed_image_or.take_value();
+
+ auto compressed_reader_or = FdReader::Create(kFvmSparseImagePath);
+ ASSERT_TRUE(compressed_reader_or.is_ok()) << compressed_reader_or.error();
+ auto compressed_reader = compressed_reader_or.take_value();
+
+ auto decompressed_writer_or = FdWriter::Create(decompressed_image.path());
+ ASSERT_TRUE(decompressed_writer_or.is_ok()) << decompressed_writer_or.error();
+ auto decompressed_writer = decompressed_writer_or.take_value();
+
+ auto decompress_result = FvmSparseDecompressImage(0, compressed_reader, decompressed_writer);
+ ASSERT_TRUE(decompress_result.is_ok()) << decompress_result.error();
+ ASSERT_TRUE(decompress_result.value());
+
+ // Read the header.
+ fvm::SparseImage header;
+ fbl::Span<uint8_t> header_buffer(reinterpret_cast<uint8_t*>(&header), sizeof(header));
+ auto header_read_result = compressed_reader.Read(0, header_buffer);
+ ASSERT_TRUE(header_read_result.is_ok()) << header_read_result.error();
+
+ uint64_t compressed_data_offset = header.header_length;
+
+ auto expected_decompressed_reader_or = FdReader::Create(decompressed_image.path());
+ ASSERT_TRUE(expected_decompressed_reader_or.is_ok()) << expected_decompressed_reader_or.error();
+ auto expected_decompressed_reader = expected_decompressed_reader_or.take_value();
+
+ std::shared_ptr<Reader> shared_compressed_reader =
+ std::make_shared<FdReader>(std::move(compressed_reader));
+ // For a fvm sparse image, we can either decompress and calculate the length in a single pass,
+ // or calculate the expected uncompressed size based on the accumulated extent length.
+ Lz4DecompressReader decompressed_reader(
+ compressed_data_offset, expected_decompressed_reader.length(), shared_compressed_reader);
+ ASSERT_TRUE(decompressed_reader.Initialize().is_ok());
+
+ // Now compare offsets.
+ constexpr uint64_t kDecompressedBufferSize = 64u << 10;
+ std::vector<uint8_t> actual_decompressed_buffer;
+ actual_decompressed_buffer.resize(kDecompressedBufferSize, 0);
+
+ std::vector<uint8_t> expected_decompressed_buffer;
+ expected_decompressed_buffer.resize(kDecompressedBufferSize, 0);
+
+ // We skip the header itself, since some flags might be different, from the compressed and the non
+ // compressed. Though this sectionis not compressed.
+ uint64_t read_bytes = sizeof(fvm::SparseImage);
+ while (read_bytes < decompressed_reader.length()) {
+ uint64_t bytes_to_read = kDecompressedBufferSize;
+ if (bytes_to_read > decompressed_reader.length() - read_bytes) {
+ bytes_to_read = decompressed_reader.length() - read_bytes;
+ }
+ auto actual_decompressed_view =
+ fbl::Span<uint8_t>(actual_decompressed_buffer).subspan(0, bytes_to_read);
+ auto read_result = decompressed_reader.Read(read_bytes, actual_decompressed_view);
+ ASSERT_TRUE(read_result.is_ok()) << read_result.error();
+
+ auto expected_decompressed_view =
+ fbl::Span<uint8_t>(expected_decompressed_buffer).subspan(0, bytes_to_read);
+ auto expected_read_result =
+ expected_decompressed_reader.Read(read_bytes, expected_decompressed_view);
+ ASSERT_TRUE(expected_read_result.is_ok()) << expected_read_result.error();
+
+ EXPECT_TRUE(memcmp(actual_decompressed_view.data(), expected_decompressed_view.data(),
+ actual_decompressed_view.size()) == 0)
+ << " offset " << read_bytes << " size " << bytes_to_read;
+ read_bytes += bytes_to_read;
+ }
+
+ // Check that read_bytes contain all data from the decompressed image.
+ EXPECT_EQ(read_bytes, expected_decompressed_reader.length());
+}
+
+} // namespace
+} // namespace storage::volume_image
diff --git a/src/storage/volume_image/utils/lz4_decompressor.cc b/src/storage/volume_image/utils/lz4_decompressor.cc
index 414a21a..e2b617f9 100644
--- a/src/storage/volume_image/utils/lz4_decompressor.cc
+++ b/src/storage/volume_image/utils/lz4_decompressor.cc
@@ -88,11 +88,6 @@
}
state_ = State::kDecompressed;
- // Extend the buffer to hint size.
- if (decompress_result.byte_count() > decompression_buffer_.size()) {
- decompression_buffer_.resize(decompress_result.byte_count(), 0);
- }
-
// lz4_decompress returns 0 when the end of the decompression frame has been reached.
return fit::ok(
DecompressResult{.hint = decompress_result.byte_count(), .read_bytes = read_bytes});
