[volume_image]: FvmSparseReadImage support for lz4

Allows directly opening a sparse fvm image compressed with lz4, by using
FvmSpaseReadImage. Increased the default buffer size, to speed things up
a bit to 2 MB per buffer.

Test: storage-volume-image-tests
Change-Id: Icc3795930663a5f492594a053c2fc522ed1bad5d
Reviewed-on: https://fuchsia-review.googlesource.com/c/fuchsia/+/514656
Commit-Queue: Gianfranco Valentino <gevalentino@google.com>
Reviewed-by: James Sullivan <jfsulliv@google.com>
diff --git a/src/storage/volume_image/adapter/adapter_integration_test.cc b/src/storage/volume_image/adapter/adapter_integration_test.cc
index 3e367e8..ba3ddb1 100644
--- a/src/storage/volume_image/adapter/adapter_integration_test.cc
+++ b/src/storage/volume_image/adapter/adapter_integration_test.cc
@@ -427,5 +427,24 @@
   ASSERT_NO_FATAL_FAILURE(CheckPartitionsInRamdisk(fvm_descriptor));
 }
 
+TEST(AdapterTest, CompressedSparseImageWithoutExplicitDecompressionToFvmImagePassesFsck) {
+  auto compressed_sparse_reader_or = FdReader::Create(kFvmSparseImagePath);
+  ASSERT_TRUE(compressed_sparse_reader_or.is_ok()) << compressed_sparse_reader_or.error();
+  FdReader compressed_sparse_reader = compressed_sparse_reader_or.take_value();
+
+  // Read the decompressed image.
+  auto fvm_descriptor_or =
+      FvmSparseReadImage(0, std::make_unique<FdReader>(std::move(compressed_sparse_reader)));
+  ASSERT_TRUE(fvm_descriptor_or.is_ok()) << fvm_descriptor_or.error();
+  auto fvm_descriptor = fvm_descriptor_or.take_value();
+
+  auto write_result = WriteFvmImage(fvm_descriptor);
+  ASSERT_TRUE(write_result.is_ok()) << write_result.error();
+  auto [fvm_vmo, fvm_writer] = write_result.take_value();
+
+  auto ramdisk_handle = LaunchFvm(fvm_vmo);
+  ASSERT_NO_FATAL_FAILURE(CheckPartitionsInRamdisk(fvm_descriptor));
+}
+
 }  // namespace
 }  // namespace storage::volume_image
diff --git a/src/storage/volume_image/fvm/BUILD.gn b/src/storage/volume_image/fvm/BUILD.gn
index 76b0773..95dfedb 100644
--- a/src/storage/volume_image/fvm/BUILD.gn
+++ b/src/storage/volume_image/fvm/BUILD.gn
@@ -51,6 +51,7 @@
   deps = [
     "//src/storage/volume_image/utils:block-utils",
     "//src/storage/volume_image/utils:lz4-compression",
+    "//src/storage/volume_image/utils:lz4-decompress-reader",
   ]
 }
 
diff --git a/src/storage/volume_image/fvm/fvm_sparse_image.cc b/src/storage/volume_image/fvm/fvm_sparse_image.cc
index fc81df3..723c013 100644
--- a/src/storage/volume_image/fvm/fvm_sparse_image.cc
+++ b/src/storage/volume_image/fvm/fvm_sparse_image.cc
@@ -25,6 +25,7 @@
 #include "src/storage/volume_image/options.h"
 #include "src/storage/volume_image/utils/block_utils.h"
 #include "src/storage/volume_image/utils/compressor.h"
+#include "src/storage/volume_image/utils/lz4_decompress_reader.h"
 #include "src/storage/volume_image/utils/lz4_decompressor.h"
 #include "src/storage/volume_image/volume_descriptor.h"
 
@@ -691,12 +692,6 @@
   }
   auto header = header_or.take_value();
 
-  if (fvm_sparse_internal::GetCompressionOptions(header).schema != CompressionSchema::kNone) {
-    return fit::error(
-        "FvmSparseReadImage only supports uncompressed images. Use FvmSparseDecompressImage "
-        "first.");
-  }
-
   // Get the partition entries.
   auto partition_entries_or =
       fvm_sparse_internal::GetPartitions(sizeof(fvm::SparseImage), *image_reader, header);
@@ -704,9 +699,16 @@
     return partition_entries_or.take_error_result();
   }
 
+  // This is the maximum offset allowed for the sparse image.
+  uint64_t total_image_size = header.header_length;
+  for (const auto& partition_entry : partition_entries_or.value()) {
+    for (const auto& extent : partition_entry.extents) {
+      total_image_size += extent.extent_length;
+    }
+  }
+
   // Get the matching options.
   FvmOptions options;
-
   options.slice_size = header.slice_size;
 
   if (header.maximum_disk_size != 0) {
@@ -717,7 +719,7 @@
   builder.SetOptions(options);
 
   // Generate the address map for each partition entry.
-  uint64_t imaged_extent_offset = header.header_length;
+  uint64_t image_extent_offset = header.header_length;
   for (auto& partition_entry : partition_entries_or.value()) {
     VolumeDescriptor volume_descriptor;
     AddressDescriptor address_descriptor;
@@ -753,10 +755,22 @@
       address_descriptor.mappings.push_back(mapping);
       accumulated_extent_offset += extent.extent_length;
     }
-    std::unique_ptr<SharedReader> partition_reader = std::make_unique<SharedReader>(
-        imaged_extent_offset, accumulated_extent_offset, image_reader);
 
-    imaged_extent_offset += accumulated_extent_offset;
+    // If the image is compressed wrap it with a Lz4DecompressReader.
+    std::shared_ptr<Reader> base_reader = image_reader;
+    if (fvm_sparse_internal::GetCompressionOptions(header).schema == CompressionSchema::kLz4) {
+      auto decompress_reader = std::make_shared<Lz4DecompressReader>(
+          header.header_length, total_image_size, image_reader);
+      if (auto result = decompress_reader->Initialize(); result.is_error()) {
+        return result.take_error_result();
+      }
+      base_reader = decompress_reader;
+    }
+
+    std::unique_ptr<SharedReader> partition_reader =
+        std::make_unique<SharedReader>(image_extent_offset, accumulated_extent_offset, base_reader);
+
+    image_extent_offset += accumulated_extent_offset;
 
     builder.AddPartition(
         Partition(volume_descriptor, address_descriptor, std::move(partition_reader)));
diff --git a/src/storage/volume_image/fvm/fvm_sparse_image_test.cc b/src/storage/volume_image/fvm/fvm_sparse_image_test.cc
index 4e6d410..3cae533 100644
--- a/src/storage/volume_image/fvm/fvm_sparse_image_test.cc
+++ b/src/storage/volume_image/fvm/fvm_sparse_image_test.cc
@@ -1697,45 +1697,18 @@
   ASSERT_TRUE(FvmSparseReadImage(0, nullptr).is_error());
 }
 
-TEST(FvmSparseReadImageTest, CompressedImageIsError) {
-  SerializedImageContainer compressed_container;
-  auto descriptor = MakeFvmDescriptorWithOptions(MakeOptions(8192, CompressionSchema::kLz4));
+void CheckGeneratedDescriptor(const FvmDescriptor& actual_descriptor,
+                              const FvmDescriptor& original_descriptor) {
+  EXPECT_EQ(actual_descriptor.options().slice_size, original_descriptor.options().slice_size);
+  EXPECT_EQ(actual_descriptor.options().max_volume_size,
+            original_descriptor.options().max_volume_size);
+  EXPECT_EQ(actual_descriptor.options().target_volume_size, std::nullopt);
+  EXPECT_EQ(actual_descriptor.options().compression.schema, CompressionSchema::kNone);
 
-  Lz4Compressor compressor = Lz4Compressor::Create(descriptor.options().compression).take_value();
-
-  // Write the compressed data that we will decompress later.
-  auto write_result = FvmSparseWriteImage(descriptor, &compressed_container.writer(), &compressor);
-  ASSERT_TRUE(write_result.is_ok()) << write_result.error();
-
-  EXPECT_TRUE(FvmSparseReadImage(
-                  0, std::make_unique<BufferReader>(0, &compressed_container.serialized_image(),
-                                                    sizeof(SerializedSparseImage)))
-                  .is_error());
-}
-
-TEST(FvmSparseReadImageTest, ReturnsFvmDescriptorAndIsCorrect) {
-  SerializedImageContainer compressed_container;
-  auto descriptor = MakeFvmDescriptorWithOptions(MakeOptions(8192, CompressionSchema::kNone));
-
-  // Write the compressed data that we will decompress later.
-  auto write_result = FvmSparseWriteImage(descriptor, &compressed_container.writer());
-  ASSERT_TRUE(write_result.is_ok()) << write_result.error();
-
-  auto read_descriptor_or = FvmSparseReadImage(
-      0, std::make_unique<BufferReader>(0, &compressed_container.serialized_image(),
-                                        sizeof(SerializedSparseImage)));
-  EXPECT_TRUE(read_descriptor_or.is_ok());
-  auto read_descriptor = read_descriptor_or.take_value();
-
-  EXPECT_EQ(read_descriptor.options().slice_size, descriptor.options().slice_size);
-  EXPECT_EQ(read_descriptor.options().max_volume_size, descriptor.options().max_volume_size);
-  EXPECT_EQ(read_descriptor.options().target_volume_size, std::nullopt);
-  EXPECT_EQ(read_descriptor.options().compression.schema, CompressionSchema::kNone);
-
-  ASSERT_EQ(read_descriptor.partitions().size(), descriptor.partitions().size());
-  auto read_partition_it = read_descriptor.partitions().begin();
-  auto expected_partition_it = descriptor.partitions().begin();
-  for (size_t i = 0; i < read_descriptor.partitions().size(); ++i) {
+  ASSERT_EQ(actual_descriptor.partitions().size(), original_descriptor.partitions().size());
+  auto read_partition_it = actual_descriptor.partitions().begin();
+  auto expected_partition_it = original_descriptor.partitions().begin();
+  for (size_t i = 0; i < actual_descriptor.partitions().size(); ++i) {
     const auto& actual_partition = *(read_partition_it);
     const auto& expected_partition = *(expected_partition_it);
 
@@ -1772,9 +1745,13 @@
       EXPECT_EQ(actual_mapping.size, expected_size);
 
       // Non compressed images will require zero filling.
-      ASSERT_EQ(actual_mapping.options.size(), 1u);
-      EXPECT_NE(actual_mapping.options.find(EnumAsString(AddressMapOption::kFill)),
-                actual_mapping.options.end());
+      if (original_descriptor.options().compression.schema == CompressionSchema::kNone) {
+        ASSERT_EQ(actual_mapping.options.size(), 1u);
+        EXPECT_NE(actual_mapping.options.find(EnumAsString(AddressMapOption::kFill)),
+                  actual_mapping.options.end());
+      } else {
+        ASSERT_EQ(actual_mapping.options.size(), 0u);
+      }
 
       actual_data.resize(actual_mapping.count, 0);
       expected_data.resize(expected_mapping.count, 0);
@@ -1794,5 +1771,41 @@
   }
 }
 
+TEST(FvmSparseReadImageTest, CompressedImageIsOk) {
+  SerializedImageContainer compressed_container;
+  auto descriptor = MakeFvmDescriptorWithOptions(MakeOptions(8192, CompressionSchema::kLz4));
+
+  Lz4Compressor compressor = Lz4Compressor::Create(descriptor.options().compression).take_value();
+
+  // Write the compressed data that we will decompress later.
+  auto write_result = FvmSparseWriteImage(descriptor, &compressed_container.writer(), &compressor);
+  ASSERT_TRUE(write_result.is_ok()) << write_result.error();
+
+  auto read_descriptor_or = FvmSparseReadImage(
+      0, std::make_unique<BufferReader>(0, &compressed_container.serialized_image(),
+                                        sizeof(SerializedSparseImage)));
+  EXPECT_TRUE(read_descriptor_or.is_ok()) << read_descriptor_or.error();
+  auto read_descriptor = read_descriptor_or.take_value();
+
+  ASSERT_NO_FATAL_FAILURE(CheckGeneratedDescriptor(read_descriptor, descriptor));
+}
+
+TEST(FvmSparseReadImageTest, ReturnsFvmDescriptorAndIsCorrect) {
+  SerializedImageContainer compressed_container;
+  auto descriptor = MakeFvmDescriptorWithOptions(MakeOptions(8192, CompressionSchema::kNone));
+
+  // Write the compressed data that we will decompress later.
+  auto write_result = FvmSparseWriteImage(descriptor, &compressed_container.writer());
+  ASSERT_TRUE(write_result.is_ok()) << write_result.error();
+
+  auto read_descriptor_or = FvmSparseReadImage(
+      0, std::make_unique<BufferReader>(0, &compressed_container.serialized_image(),
+                                        sizeof(SerializedSparseImage)));
+  EXPECT_TRUE(read_descriptor_or.is_ok());
+  auto read_descriptor = read_descriptor_or.take_value();
+
+  ASSERT_NO_FATAL_FAILURE(CheckGeneratedDescriptor(read_descriptor, descriptor));
+}
+
 }  // namespace
 }  // namespace storage::volume_image
diff --git a/src/storage/volume_image/utils/fd_reader.cc b/src/storage/volume_image/utils/fd_reader.cc
index de0607e..3eed878 100644
--- a/src/storage/volume_image/utils/fd_reader.cc
+++ b/src/storage/volume_image/utils/fd_reader.cc
@@ -10,6 +10,7 @@
 
 #include <cstdio>
 #include <cstdlib>
+#include <iostream>
 
 #include <fbl/span.h>
 #include <fbl/unique_fd.h>
diff --git a/src/storage/volume_image/utils/lz4_decompress_reader.h b/src/storage/volume_image/utils/lz4_decompress_reader.h
index 8380f09..346a7e1 100644
--- a/src/storage/volume_image/utils/lz4_decompress_reader.h
+++ b/src/storage/volume_image/utils/lz4_decompress_reader.h
@@ -21,7 +21,7 @@
 class Lz4DecompressReader final : public Reader {
  public:
   // Default size for |StreamContext| buffers.
-  static constexpr uint64_t kMaxBufferSize = 20 * (1 << 10);
+  static constexpr uint64_t kMaxBufferSize = 2 * (1 << 20);
 
   // Lz4DecompressReader will decompress data starting at |offset|. That is the compressed data is
   // embedded in |compressed_reader| and the first compressed byte is at |offset|.