[volume_image]: FvmSparseReadImage support for lz4
Allows directly opening a sparse fvm image compressed with lz4, by using
FvmSpaseReadImage. Increased the default buffer size, to speed things up
a bit to 2 MB per buffer.
Test: storage-volume-image-tests
Change-Id: Icc3795930663a5f492594a053c2fc522ed1bad5d
Reviewed-on: https://fuchsia-review.googlesource.com/c/fuchsia/+/514656
Commit-Queue: Gianfranco Valentino <gevalentino@google.com>
Reviewed-by: James Sullivan <jfsulliv@google.com>
diff --git a/src/storage/volume_image/adapter/adapter_integration_test.cc b/src/storage/volume_image/adapter/adapter_integration_test.cc
index 3e367e8..ba3ddb1 100644
--- a/src/storage/volume_image/adapter/adapter_integration_test.cc
+++ b/src/storage/volume_image/adapter/adapter_integration_test.cc
@@ -427,5 +427,24 @@
ASSERT_NO_FATAL_FAILURE(CheckPartitionsInRamdisk(fvm_descriptor));
}
+TEST(AdapterTest, CompressedSparseImageWithoutExplicitDecompressionToFvmImagePassesFsck) {
+ auto compressed_sparse_reader_or = FdReader::Create(kFvmSparseImagePath);
+ ASSERT_TRUE(compressed_sparse_reader_or.is_ok()) << compressed_sparse_reader_or.error();
+ FdReader compressed_sparse_reader = compressed_sparse_reader_or.take_value();
+
+ // Read the decompressed image.
+ auto fvm_descriptor_or =
+ FvmSparseReadImage(0, std::make_unique<FdReader>(std::move(compressed_sparse_reader)));
+ ASSERT_TRUE(fvm_descriptor_or.is_ok()) << fvm_descriptor_or.error();
+ auto fvm_descriptor = fvm_descriptor_or.take_value();
+
+ auto write_result = WriteFvmImage(fvm_descriptor);
+ ASSERT_TRUE(write_result.is_ok()) << write_result.error();
+ auto [fvm_vmo, fvm_writer] = write_result.take_value();
+
+ auto ramdisk_handle = LaunchFvm(fvm_vmo);
+ ASSERT_NO_FATAL_FAILURE(CheckPartitionsInRamdisk(fvm_descriptor));
+}
+
} // namespace
} // namespace storage::volume_image
diff --git a/src/storage/volume_image/fvm/BUILD.gn b/src/storage/volume_image/fvm/BUILD.gn
index 76b0773..95dfedb 100644
--- a/src/storage/volume_image/fvm/BUILD.gn
+++ b/src/storage/volume_image/fvm/BUILD.gn
@@ -51,6 +51,7 @@
deps = [
"//src/storage/volume_image/utils:block-utils",
"//src/storage/volume_image/utils:lz4-compression",
+ "//src/storage/volume_image/utils:lz4-decompress-reader",
]
}
diff --git a/src/storage/volume_image/fvm/fvm_sparse_image.cc b/src/storage/volume_image/fvm/fvm_sparse_image.cc
index fc81df3..723c013 100644
--- a/src/storage/volume_image/fvm/fvm_sparse_image.cc
+++ b/src/storage/volume_image/fvm/fvm_sparse_image.cc
@@ -25,6 +25,7 @@
#include "src/storage/volume_image/options.h"
#include "src/storage/volume_image/utils/block_utils.h"
#include "src/storage/volume_image/utils/compressor.h"
+#include "src/storage/volume_image/utils/lz4_decompress_reader.h"
#include "src/storage/volume_image/utils/lz4_decompressor.h"
#include "src/storage/volume_image/volume_descriptor.h"
@@ -691,12 +692,6 @@
}
auto header = header_or.take_value();
- if (fvm_sparse_internal::GetCompressionOptions(header).schema != CompressionSchema::kNone) {
- return fit::error(
- "FvmSparseReadImage only supports uncompressed images. Use FvmSparseDecompressImage "
- "first.");
- }
-
// Get the partition entries.
auto partition_entries_or =
fvm_sparse_internal::GetPartitions(sizeof(fvm::SparseImage), *image_reader, header);
@@ -704,9 +699,16 @@
return partition_entries_or.take_error_result();
}
+ // This is the maximum offset allowed for the sparse image.
+ uint64_t total_image_size = header.header_length;
+ for (const auto& partition_entry : partition_entries_or.value()) {
+ for (const auto& extent : partition_entry.extents) {
+ total_image_size += extent.extent_length;
+ }
+ }
+
// Get the matching options.
FvmOptions options;
-
options.slice_size = header.slice_size;
if (header.maximum_disk_size != 0) {
@@ -717,7 +719,7 @@
builder.SetOptions(options);
// Generate the address map for each partition entry.
- uint64_t imaged_extent_offset = header.header_length;
+ uint64_t image_extent_offset = header.header_length;
for (auto& partition_entry : partition_entries_or.value()) {
VolumeDescriptor volume_descriptor;
AddressDescriptor address_descriptor;
@@ -753,10 +755,22 @@
address_descriptor.mappings.push_back(mapping);
accumulated_extent_offset += extent.extent_length;
}
- std::unique_ptr<SharedReader> partition_reader = std::make_unique<SharedReader>(
- imaged_extent_offset, accumulated_extent_offset, image_reader);
- imaged_extent_offset += accumulated_extent_offset;
+ // If the image is compressed wrap it with a Lz4DecompressReader.
+ std::shared_ptr<Reader> base_reader = image_reader;
+ if (fvm_sparse_internal::GetCompressionOptions(header).schema == CompressionSchema::kLz4) {
+ auto decompress_reader = std::make_shared<Lz4DecompressReader>(
+ header.header_length, total_image_size, image_reader);
+ if (auto result = decompress_reader->Initialize(); result.is_error()) {
+ return result.take_error_result();
+ }
+ base_reader = decompress_reader;
+ }
+
+ std::unique_ptr<SharedReader> partition_reader =
+ std::make_unique<SharedReader>(image_extent_offset, accumulated_extent_offset, base_reader);
+
+ image_extent_offset += accumulated_extent_offset;
builder.AddPartition(
Partition(volume_descriptor, address_descriptor, std::move(partition_reader)));
diff --git a/src/storage/volume_image/fvm/fvm_sparse_image_test.cc b/src/storage/volume_image/fvm/fvm_sparse_image_test.cc
index 4e6d410..3cae533 100644
--- a/src/storage/volume_image/fvm/fvm_sparse_image_test.cc
+++ b/src/storage/volume_image/fvm/fvm_sparse_image_test.cc
@@ -1697,45 +1697,18 @@
ASSERT_TRUE(FvmSparseReadImage(0, nullptr).is_error());
}
-TEST(FvmSparseReadImageTest, CompressedImageIsError) {
- SerializedImageContainer compressed_container;
- auto descriptor = MakeFvmDescriptorWithOptions(MakeOptions(8192, CompressionSchema::kLz4));
+void CheckGeneratedDescriptor(const FvmDescriptor& actual_descriptor,
+ const FvmDescriptor& original_descriptor) {
+ EXPECT_EQ(actual_descriptor.options().slice_size, original_descriptor.options().slice_size);
+ EXPECT_EQ(actual_descriptor.options().max_volume_size,
+ original_descriptor.options().max_volume_size);
+ EXPECT_EQ(actual_descriptor.options().target_volume_size, std::nullopt);
+ EXPECT_EQ(actual_descriptor.options().compression.schema, CompressionSchema::kNone);
- Lz4Compressor compressor = Lz4Compressor::Create(descriptor.options().compression).take_value();
-
- // Write the compressed data that we will decompress later.
- auto write_result = FvmSparseWriteImage(descriptor, &compressed_container.writer(), &compressor);
- ASSERT_TRUE(write_result.is_ok()) << write_result.error();
-
- EXPECT_TRUE(FvmSparseReadImage(
- 0, std::make_unique<BufferReader>(0, &compressed_container.serialized_image(),
- sizeof(SerializedSparseImage)))
- .is_error());
-}
-
-TEST(FvmSparseReadImageTest, ReturnsFvmDescriptorAndIsCorrect) {
- SerializedImageContainer compressed_container;
- auto descriptor = MakeFvmDescriptorWithOptions(MakeOptions(8192, CompressionSchema::kNone));
-
- // Write the compressed data that we will decompress later.
- auto write_result = FvmSparseWriteImage(descriptor, &compressed_container.writer());
- ASSERT_TRUE(write_result.is_ok()) << write_result.error();
-
- auto read_descriptor_or = FvmSparseReadImage(
- 0, std::make_unique<BufferReader>(0, &compressed_container.serialized_image(),
- sizeof(SerializedSparseImage)));
- EXPECT_TRUE(read_descriptor_or.is_ok());
- auto read_descriptor = read_descriptor_or.take_value();
-
- EXPECT_EQ(read_descriptor.options().slice_size, descriptor.options().slice_size);
- EXPECT_EQ(read_descriptor.options().max_volume_size, descriptor.options().max_volume_size);
- EXPECT_EQ(read_descriptor.options().target_volume_size, std::nullopt);
- EXPECT_EQ(read_descriptor.options().compression.schema, CompressionSchema::kNone);
-
- ASSERT_EQ(read_descriptor.partitions().size(), descriptor.partitions().size());
- auto read_partition_it = read_descriptor.partitions().begin();
- auto expected_partition_it = descriptor.partitions().begin();
- for (size_t i = 0; i < read_descriptor.partitions().size(); ++i) {
+ ASSERT_EQ(actual_descriptor.partitions().size(), original_descriptor.partitions().size());
+ auto read_partition_it = actual_descriptor.partitions().begin();
+ auto expected_partition_it = original_descriptor.partitions().begin();
+ for (size_t i = 0; i < actual_descriptor.partitions().size(); ++i) {
const auto& actual_partition = *(read_partition_it);
const auto& expected_partition = *(expected_partition_it);
@@ -1772,9 +1745,13 @@
EXPECT_EQ(actual_mapping.size, expected_size);
// Non compressed images will require zero filling.
- ASSERT_EQ(actual_mapping.options.size(), 1u);
- EXPECT_NE(actual_mapping.options.find(EnumAsString(AddressMapOption::kFill)),
- actual_mapping.options.end());
+ if (original_descriptor.options().compression.schema == CompressionSchema::kNone) {
+ ASSERT_EQ(actual_mapping.options.size(), 1u);
+ EXPECT_NE(actual_mapping.options.find(EnumAsString(AddressMapOption::kFill)),
+ actual_mapping.options.end());
+ } else {
+ ASSERT_EQ(actual_mapping.options.size(), 0u);
+ }
actual_data.resize(actual_mapping.count, 0);
expected_data.resize(expected_mapping.count, 0);
@@ -1794,5 +1771,41 @@
}
}
+TEST(FvmSparseReadImageTest, CompressedImageIsOk) {
+ SerializedImageContainer compressed_container;
+ auto descriptor = MakeFvmDescriptorWithOptions(MakeOptions(8192, CompressionSchema::kLz4));
+
+ Lz4Compressor compressor = Lz4Compressor::Create(descriptor.options().compression).take_value();
+
+ // Write the compressed data that we will decompress later.
+ auto write_result = FvmSparseWriteImage(descriptor, &compressed_container.writer(), &compressor);
+ ASSERT_TRUE(write_result.is_ok()) << write_result.error();
+
+ auto read_descriptor_or = FvmSparseReadImage(
+ 0, std::make_unique<BufferReader>(0, &compressed_container.serialized_image(),
+ sizeof(SerializedSparseImage)));
+ EXPECT_TRUE(read_descriptor_or.is_ok()) << read_descriptor_or.error();
+ auto read_descriptor = read_descriptor_or.take_value();
+
+ ASSERT_NO_FATAL_FAILURE(CheckGeneratedDescriptor(read_descriptor, descriptor));
+}
+
+TEST(FvmSparseReadImageTest, ReturnsFvmDescriptorAndIsCorrect) {
+ SerializedImageContainer compressed_container;
+ auto descriptor = MakeFvmDescriptorWithOptions(MakeOptions(8192, CompressionSchema::kNone));
+
+ // Write the compressed data that we will decompress later.
+ auto write_result = FvmSparseWriteImage(descriptor, &compressed_container.writer());
+ ASSERT_TRUE(write_result.is_ok()) << write_result.error();
+
+ auto read_descriptor_or = FvmSparseReadImage(
+ 0, std::make_unique<BufferReader>(0, &compressed_container.serialized_image(),
+ sizeof(SerializedSparseImage)));
+ EXPECT_TRUE(read_descriptor_or.is_ok());
+ auto read_descriptor = read_descriptor_or.take_value();
+
+ ASSERT_NO_FATAL_FAILURE(CheckGeneratedDescriptor(read_descriptor, descriptor));
+}
+
} // namespace
} // namespace storage::volume_image
diff --git a/src/storage/volume_image/utils/fd_reader.cc b/src/storage/volume_image/utils/fd_reader.cc
index de0607e..3eed878d 100644
--- a/src/storage/volume_image/utils/fd_reader.cc
+++ b/src/storage/volume_image/utils/fd_reader.cc
@@ -10,6 +10,7 @@
#include <cstdio>
#include <cstdlib>
+#include <iostream>
#include <fbl/span.h>
#include <fbl/unique_fd.h>
diff --git a/src/storage/volume_image/utils/lz4_decompress_reader.h b/src/storage/volume_image/utils/lz4_decompress_reader.h
index 8380f09..346a7e1 100644
--- a/src/storage/volume_image/utils/lz4_decompress_reader.h
+++ b/src/storage/volume_image/utils/lz4_decompress_reader.h
@@ -21,7 +21,7 @@
class Lz4DecompressReader final : public Reader {
public:
// Default size for |StreamContext| buffers.
- static constexpr uint64_t kMaxBufferSize = 20 * (1 << 10);
+ static constexpr uint64_t kMaxBufferSize = 2 * (1 << 20);
// Lz4DecompressReader will decompress data starting at |offset|. That is the compressed data is
// embedded in |compressed_reader| and the first compressed byte is at |offset|.
