[fvm] [host] Use SparseReader to read sparse files
Test: fvm-test
Change-Id: I4eeaa1e1085f5391ec2ead6a29f8c31f0a5e5f32
diff --git a/system/host/fvm/container/sparse.cpp b/system/host/fvm/container/sparse.cpp
index 0b46813..e55fd3c 100644
--- a/system/host/fvm/container/sparse.cpp
+++ b/system/host/fvm/container/sparse.cpp
@@ -102,34 +102,31 @@
if (s.st_size > 0) {
disk_size_ = s.st_size;
- if (read(fd_.get(), &image_, sizeof(fvm::sparse_image_t)) != sizeof(fvm::sparse_image_t)) {
- fprintf(stderr, "SparseContainer: Failed to read the sparse header\n");
+ fbl::unique_fd dup_fd(dup(fd_.get()));
+ if (fvm::SparseReader::CreateSilent(std::move(dup_fd), &reader_) != ZX_OK) {
+ fprintf(stderr, "SparseContainer: Failed to read metadata from sparse file\n");
return;
}
- if (image_.flags & fvm::kSparseFlagLz4) {
- return;
- }
+ memcpy(&image_, reader_->Image(), sizeof(fvm::sparse_image_t));
extent_size_ = disk_size_ - image_.header_length;
+ uintptr_t partition_ptr = reinterpret_cast<uintptr_t>(reader_->Partitions());
+
for (unsigned i = 0; i < image_.partition_count; i++) {
partition_info_t partition;
+ memcpy(&partition.descriptor, reinterpret_cast<void*>(partition_ptr),
+ sizeof(fvm::partition_descriptor_t));
partitions_.push_back(std::move(partition));
- if (read(fd_.get(), &partitions_[i].descriptor, sizeof(fvm::partition_descriptor_t)) !=
- sizeof(fvm::partition_descriptor_t)) {
- fprintf(stderr, "SparseContainer: Failed to read partition %u\n", i);
- return;
- }
+ partition_ptr += sizeof(fvm::partition_descriptor_t);
- for (unsigned j = 0; j < partitions_[i].descriptor.extent_count; j++) {
+ for (size_t j = 0; j < partitions_[i].descriptor.extent_count; j++) {
fvm::extent_descriptor_t extent;
+ memcpy(&extent, reinterpret_cast<void*>(partition_ptr),
+ sizeof(fvm::extent_descriptor_t));
partitions_[i].extents.push_back(extent);
- if (read(fd_.get(), &partitions_[i].extents[j], sizeof(fvm::extent_descriptor_t)) !=
- sizeof(fvm::extent_descriptor_t)) {
- fprintf(stderr, "SparseContainer: Failed to read extent\n");
- return;
- }
+ partition_ptr += sizeof(fvm::extent_descriptor_t);
}
}
@@ -160,6 +157,15 @@
fprintf(stderr, "SparseContainer: Found invalid container\n");
return ZX_ERR_INTERNAL;
}
+
+ if (image_.flags & fvm::kSparseFlagLz4) {
+ // Decompression must occur before verification, since all contents must be available for
+ // fsck.
+ fprintf(stderr, "SparseContainer: Found compressed container; contents cannot be"
+ " verified\n");
+ return ZX_ERR_INVALID_ARGS;
+ }
+
if (image_.magic != fvm::kSparseFormatMagic) {
fprintf(stderr, "SparseContainer: Bad magic\n");
return ZX_ERR_IO;
@@ -330,6 +336,23 @@
return ZX_OK;
}
+zx_status_t SparseContainer::Decompress(const char* path) {
+ if ((flags_ & fvm::kSparseFlagLz4) == 0) {
+ fprintf(stderr, "Cannot decompress un-compressed sparse file\n");
+ return ZX_ERR_NOT_SUPPORTED;
+ }
+
+ fbl::unique_fd fd;
+
+ fd.reset(open(path, O_WRONLY | O_CREAT | O_EXCL, 0644));
+ if (!fd) {
+ fprintf(stderr, "could not open %s: %s\n", path, strerror(errno));
+ return ZX_ERR_IO;
+ }
+
+ return reader_->WriteDecompressed(std::move(fd));
+}
+
zx_status_t SparseContainer::AllocatePartition(fbl::unique_ptr<Format> format) {
partition_info_t partition;
format->GetPartitionInfo(&partition.descriptor);
diff --git a/system/host/fvm/include/fvm/container.h b/system/host/fvm/include/fvm/container.h
index 6165ad8..199b1c5 100644
--- a/system/host/fvm/include/fvm/container.h
+++ b/system/host/fvm/include/fvm/container.h
@@ -10,6 +10,7 @@
#include <fbl/string_buffer.h>
#include <fbl/vector.h>
#include <fbl/unique_fd.h>
+#include <fvm/fvm-lz4.h>
#include <fvm/fvm-sparse.h>
#include "format.h"
@@ -211,6 +212,10 @@
size_t SliceSize() const final;
zx_status_t AddPartition(const char* path, const char* type_name) final;
+ // Decompresses the contents of the sparse file (if they are compressed), and writes the output
+ // to |path|.
+ zx_status_t Decompress(const char* path);
+
private:
bool valid_;
bool dirty_;
@@ -219,6 +224,7 @@
fvm::sparse_image_t image_;
fbl::Vector<partition_info_t> partitions_;
CompressionContext compression_;
+ fbl::unique_ptr<fvm::SparseReader> reader_;
zx_status_t AllocatePartition(fbl::unique_ptr<Format> format);
zx_status_t AllocateExtent(uint32_t part_index, uint64_t slice_start, uint64_t slice_count,
diff --git a/system/host/fvm/main.cpp b/system/host/fvm/main.cpp
index e26ce82..29d5e8a 100644
--- a/system/host/fvm/main.cpp
+++ b/system/host/fvm/main.cpp
@@ -23,7 +23,7 @@
" required)\n");
fprintf(stderr, " sparse : Creates a sparse file. One or more input paths are required.\n");
fprintf(stderr, " verify : Report basic information about sparse/fvm files and run fsck on"
- " contained partitions\n");
+ " contained partitions.\n");
fprintf(stderr, " decompress : Decompresses a compressed sparse file. --sparse input path is"
" required.\n");
fprintf(stderr, "Flags (neither or both of offset/length must be specified):\n");
@@ -265,12 +265,13 @@
return -1;
}
- if (fvm::decompress_sparse(input_path, path) != ZX_OK) {
+ SparseContainer compressedContainer(input_path, slice_size, flags);
+ if (compressedContainer.Decompress(path) != ZX_OK) {
return -1;
}
- fbl::unique_ptr<SparseContainer> sparseData(new SparseContainer(path, slice_size, flags));
- if (sparseData->Verify() != ZX_OK) {
+ SparseContainer sparseContainer(path, slice_size, flags);
+ if (sparseContainer.Verify() != ZX_OK) {
return -1;
}
} else {
diff --git a/system/ulib/fvm/fvm-lz4.cpp b/system/ulib/fvm/fvm-lz4.cpp
index f47dbae..2cf176b 100644
--- a/system/ulib/fvm/fvm-lz4.cpp
+++ b/system/ulib/fvm/fvm-lz4.cpp
@@ -8,8 +8,16 @@
namespace fvm {
zx_status_t SparseReader::Create(fbl::unique_fd fd, fbl::unique_ptr<SparseReader>* out) {
+ return SparseReader::CreateHelper(std::move(fd), true /* verbose */, out);
+}
+zx_status_t SparseReader::CreateSilent(fbl::unique_fd fd, fbl::unique_ptr<SparseReader>* out) {
+ return SparseReader::CreateHelper(std::move(fd), false /* verbose */, out);
+}
+
+zx_status_t SparseReader::CreateHelper(fbl::unique_fd fd, bool verbose,
+ fbl::unique_ptr<SparseReader>* out) {
fbl::AllocChecker ac;
- fbl::unique_ptr<SparseReader> reader(new (&ac) SparseReader(std::move(fd)));
+ fbl::unique_ptr<SparseReader> reader(new (&ac) SparseReader(std::move(fd), verbose));
if (!ac.check()) {
return ZX_ERR_NO_MEMORY;
}
@@ -23,7 +31,8 @@
return ZX_OK;
}
-SparseReader::SparseReader(fbl::unique_fd fd) : compressed_(false), fd_(std::move(fd)) {}
+SparseReader::SparseReader(fbl::unique_fd fd, bool verbose) : compressed_(false), verbose_(verbose),
+ fd_(std::move(fd)) {}
zx_status_t SparseReader::ReadMetadata() {
// Read sparse image header.
@@ -63,7 +72,10 @@
// If image is compressed, additional setup is required
if (image.flags & fvm::kSparseFlagLz4) {
- printf("Found compressed file\n");
+ if (verbose_) {
+ printf("Found compressed file\n");
+ }
+
compressed_ = true;
// Initialize decompression context
LZ4F_errorCode_t errc = LZ4F_createDecompressionContext(&dctx_, LZ4F_VERSION);
@@ -293,39 +305,17 @@
}
void SparseReader::PrintStats() const {
- printf("Reading FVM from compressed file: %s\n", compressed_ ? "true" : "false");
- printf("Remaining bytes read into compression buffer: %lu\n", in_buf_.size);
- printf("Remaining bytes written to decompression buffer: %lu\n", out_buf_.size);
+ if (verbose_) {
+ printf("Reading FVM from compressed file: %s\n", compressed_ ? "true" : "false");
+ printf("Remaining bytes read into compression buffer: %lu\n", in_buf_.size);
+ printf("Remaining bytes written to decompression buffer: %lu\n", out_buf_.size);
#ifdef __Fuchsia__
- printf("Time reading bytes from sparse FVM file: %lu (%lu s)\n", read_time_,
- read_time_ / zx_ticks_per_second());
- printf("Time reading bytes AND decompressing them: %lu (%lu s)\n", total_time_,
- total_time_ / zx_ticks_per_second());
+ printf("Time reading bytes from sparse FVM file: %lu (%lu s)\n", read_time_,
+ read_time_ / zx_ticks_per_second());
+ printf("Time reading bytes AND decompressing them: %lu (%lu s)\n", total_time_,
+ total_time_ / zx_ticks_per_second());
#endif
-}
-
-zx_status_t decompress_sparse(const char* infile, const char* outfile) {
- fbl::unique_fd infd, outfd, testfd;
-
- infd.reset(open(infile, O_RDONLY));
- if (!infd) {
- fprintf(stderr, "could not open %s: %s\n", infile, strerror(errno));
- return ZX_ERR_IO;
}
-
- outfd.reset(open(outfile, O_WRONLY | O_CREAT | O_EXCL, 0644));
- if (!outfd) {
- fprintf(stderr, "could not open %s: %s\n", outfile, strerror(errno));
- return ZX_ERR_IO;
- }
-
- zx_status_t status;
- fbl::unique_ptr<SparseReader> reader;
- if ((status = SparseReader::Create(std::move(infd), &reader))) {
- return status;
- }
-
- return reader->WriteDecompressed(std::move(outfd));
}
} // namespace fvm
diff --git a/system/ulib/fvm/include/fvm/fvm-lz4.h b/system/ulib/fvm/include/fvm/fvm-lz4.h
index 9d4a607..be8476f 100644
--- a/system/ulib/fvm/include/fvm/fvm-lz4.h
+++ b/system/ulib/fvm/include/fvm/fvm-lz4.h
@@ -2,6 +2,8 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
+#pragma once
+
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
@@ -30,6 +32,7 @@
class SparseReader {
public:
static zx_status_t Create(fbl::unique_fd fd, fbl::unique_ptr<SparseReader>* out);
+ static zx_status_t CreateSilent(fbl::unique_fd fd, fbl::unique_ptr<SparseReader>* out);
~SparseReader();
fvm::sparse_image_t* Image();
@@ -42,7 +45,7 @@
private:
typedef struct buffer {
// Write |length| bytes from |indata| into buffer.
- bool is_empty() {
+ bool is_empty() const {
return offset == 0 && size == 0;
}
@@ -85,7 +88,10 @@
size_t max_size;
} buffer_t;
- SparseReader(fbl::unique_fd fd);
+ static zx_status_t CreateHelper(fbl::unique_fd fd, bool verbose,
+ fbl::unique_ptr<SparseReader>* out);
+
+ SparseReader(fbl::unique_fd fd, bool verbose);
// Read in header data, prepare buffers and decompression context if necessary
zx_status_t ReadMetadata();
// Initialize buffer with a given |size|
@@ -98,6 +104,9 @@
// True if sparse file is compressed
bool compressed_;
+ // If true, all logs are printed.
+ bool verbose_;
+
fbl::unique_fd fd_;
fbl::unique_ptr<uint8_t[]> metadata_;
LZ4F_decompressionContext_t dctx_;
@@ -119,6 +128,4 @@
#endif
};
-// Read from compressed |infile|, decompress, and write to |outfile|.
-zx_status_t decompress_sparse(const char* infile, const char* outfile);
} // namespace fvm
diff --git a/system/ulib/minfs/minfs.cpp b/system/ulib/minfs/minfs.cpp
index 1c81489..7506eb3 100644
--- a/system/ulib/minfs/minfs.cpp
+++ b/system/ulib/minfs/minfs.cpp
@@ -1173,6 +1173,11 @@
zx_status_t SparseFsck(fbl::unique_fd fd, off_t start, off_t end,
const fbl::Vector<size_t>& extent_lengths) {
+ if (start >= end) {
+ fprintf(stderr, "error: Insufficient space allocated\n");
+ return ZX_ERR_INVALID_ARGS;
+ }
+
if (extent_lengths.size() != kExtentCount) {
FS_TRACE_ERROR("error: invalid number of extents\n");
return ZX_ERR_INVALID_ARGS;
diff --git a/system/utest/fvm-host/main.cpp b/system/utest/fvm-host/main.cpp
index e0e64ba..40fbba3 100644
--- a/system/utest/fvm-host/main.cpp
+++ b/system/utest/fvm-host/main.cpp
@@ -181,23 +181,26 @@
}
bool ReportContainer(const char* path, off_t offset) {
+ BEGIN_HELPER;
fbl::unique_ptr<Container> container;
off_t length;
ASSERT_TRUE(StatFile(path, &length));
ASSERT_EQ(Container::Create(path, offset, length - offset, 0, &container), ZX_OK,
"Failed to initialize container");
ASSERT_EQ(container->Verify(), ZX_OK, "File check failed\n");
- return true;
+ END_HELPER;
}
bool ReportSparse(uint32_t flags) {
+ BEGIN_HELPER;
if ((flags & fvm::kSparseFlagLz4) != 0) {
unittest_printf("Decompressing sparse file\n");
- if (fvm::decompress_sparse(sparse_lz4_path, sparse_path) != ZX_OK) {
- return false;
- }
+ SparseContainer compressedContainer(sparse_lz4_path, DEFAULT_SLICE_SIZE, flags);
+ ASSERT_EQ(compressedContainer.Decompress(sparse_path), ZX_OK);
}
- return ReportContainer(sparse_path, 0);
+
+ ASSERT_TRUE(ReportContainer(sparse_path, 0));
+ END_HELPER;
}
bool CreateFvm(bool create_before, off_t offset, size_t slice_size) {
