| // Copyright 2017 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include <inttypes.h> |
| #include <string.h> |
| #include <sys/stat.h> |
| #include <zircon/errors.h> |
| |
| #include <algorithm> |
| #include <filesystem> |
| #include <fstream> |
| #include <memory> |
| #include <string> |
| #include <thread> |
| #include <utility> |
| #include <vector> |
| |
| #include <fbl/algorithm.h> |
| #include <fbl/unique_fd.h> |
| |
| #include "blobfs.h" |
| #include "src/storage/blobfs/blob_layout.h" |
| #include "src/storage/blobfs/format.h" |
| #include "src/storage/blobfs/fsck_host.h" |
| #include "src/storage/blobfs/host.h" |
| #include "src/storage/blobfs/iterator/node_populator.h" |
| |
| namespace { |
| |
| void WriteBlobInfoToJson(std::ofstream& file, const blobfs::BlobInfo& blob) { |
| std::filesystem::path path = |
| std::filesystem::relative(std::filesystem::canonical(blob.GetSrcFilePath())); |
| const auto& blob_layout = blob.GetBlobLayout(); |
| uint64_t total_size = uint64_t{blob_layout.TotalBlockCount()} * blobfs::kBlobfsBlockSize; |
| file << " {\n"; |
| file << " \"source_path\": " << path << ",\n"; |
| file << " \"merkle\": \"" << blob.GetDigest().ToString() << "\",\n"; |
| file << " \"bytes\": " << blob_layout.FileSize() << ",\n"; |
| file << " \"size\": " << total_size << ",\n"; |
| file << " \"file_size\": " << blob_layout.FileSize() << ",\n"; |
| file << " \"compressed_file_size\": " << blob_layout.DataSizeUpperBound() << ",\n"; |
| file << " \"merkle_tree_size\": " << blob_layout.MerkleTreeSize() << ",\n"; |
| file << " \"used_space_in_blobfs\": " << total_size << "\n"; |
| file << " }"; |
| } |
| |
| zx::status<> RecordBlobs(const std::filesystem::path& path, std::vector<blobfs::BlobInfo>& blobs) { |
| std::ofstream file(path); |
| if (!file.is_open()) { |
| fprintf(stderr, "Failed to open: %s\n", path.c_str()); |
| return zx::error(ZX_ERR_INVALID_ARGS); |
| } |
| file << "[\n"; |
| bool is_first_blob = true; |
| for (const auto& blob : blobs) { |
| if (is_first_blob) { |
| is_first_blob = false; |
| } else { |
| file << ",\n"; |
| } |
| WriteBlobInfoToJson(file, blob); |
| } |
| file << "]\n"; |
| file.close(); |
| if (file.fail()) { |
| fprintf(stderr, "Writing to %s failed\n", path.c_str()); |
| return zx::error(ZX_ERR_IO); |
| } |
| return zx::ok(); |
| } |
| |
| zx::status<> CreateBlobfsWithBlobs(fbl::unique_fd fd, const std::vector<blobfs::BlobInfo>& blobs) { |
| std::unique_ptr<blobfs::Blobfs> blobfs; |
| if (zx_status_t status = blobfs_create(&blobfs, std::move(fd)); status != ZX_OK) { |
| return zx::error(status); |
| } |
| for (const auto& blob : blobs) { |
| if (zx::status status = blobfs->AddBlob(blob); status.is_error()) { |
| fprintf(stderr, "Failed to add blob '%s': %d\n", blob.GetSrcFilePath().c_str(), |
| status.status_value()); |
| return status; |
| } |
| } |
| return zx::ok(); |
| } |
| |
| } // namespace |
| |
| zx_status_t BlobfsCreator::Usage() { |
| zx_status_t status = FsCreator::Usage(); |
| |
| fprintf(stderr, "\nblobfs specific options:\n"); |
| fprintf(stderr, |
| "\t--deprecated_padded_format\tFormat blobfs using the deprecated format that uses more " |
| "space.\n" |
| "Valid for the commands: mkfs and create.\n"); |
| // Additional information about manifest format. |
| fprintf(stderr, "\nEach manifest line must adhere to one of the following formats:\n"); |
| fprintf(stderr, "\t'dst/path=src/path'\n"); |
| fprintf(stderr, "\t'dst/path'\n"); |
| fprintf(stderr, "with one dst/src pair or single dst per line.\n"); |
| |
| fprintf(stderr, "\nblobfs specific commands:\n"); |
| fprintf(stderr, "\texport [IMAGE] [PATH]\n"); |
| fprintf(stderr, |
| "\nExports each blob in IMAGE to the directory in PATH. If PATH does not exist, will " |
| "attempt to " |
| "create it.\n"); |
| fprintf(stderr, |
| "\nEach blob exported to PATH is named after their merkle root, and the contents match " |
| "what IMAGE has.\n"); |
| return status; |
| } |
| |
| bool BlobfsCreator::IsCommandValid(Command command) { |
| switch (command) { |
| case Command::kMkfs: |
| case Command::kFsck: |
| case Command::kUsedDataSize: |
| case Command::kUsedInodes: |
| case Command::kUsedSize: |
| case Command::kAdd: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| bool BlobfsCreator::IsOptionValid(Option option) { |
| // TODO(planders): Add offset and length support to blobfs. |
| switch (option) { |
| case Option::kDepfile: |
| case Option::kReadonly: |
| case Option::kCompress: |
| case Option::kJsonOutput: |
| case Option::kHelp: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| bool BlobfsCreator::IsArgumentValid(Argument argument) { |
| switch (argument) { |
| case Argument::kManifest: |
| case Argument::kBlob: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| zx_status_t BlobfsCreator::ProcessManifestLine(FILE* manifest, const char* dir_path) { |
| char src[PATH_MAX]; |
| src[0] = '\0'; |
| char dst[PATH_MAX]; |
| dst[0] = '\0'; |
| |
| zx_status_t status; |
| if ((status = ParseManifestLine(manifest, dir_path, src, dst)) != ZX_OK) { |
| return status; |
| } |
| |
| if (!strlen(src)) { |
| fprintf(stderr, "Manifest line must specify source file\n"); |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| blob_list_.push_back(src); |
| return ZX_OK; |
| } |
| |
| zx_status_t BlobfsCreator::ProcessCustom(int argc, char** argv, uint8_t* processed) { |
| if (strcmp(argv[0], "--blob") == 0) { |
| constexpr uint8_t required_args = 2; |
| if (argc < required_args) { |
| fprintf(stderr, "Not enough arguments for %s\n", argv[0]); |
| return ZX_ERR_INVALID_ARGS; |
| } |
| blob_list_.push_back(argv[1]); |
| *processed = required_args; |
| return ZX_OK; |
| } |
| |
| if (strcmp(argv[0], "--deprecated_padded_format") == 0) { |
| if (GetCommand() != Command::kMkfs) { |
| fprintf(stderr, "%s is only valid for mkfs and create\n", argv[0]); |
| return ZX_ERR_INVALID_ARGS; |
| } |
| blob_layout_format_ = blobfs::BlobLayoutFormat::kDeprecatedPaddedMerkleTreeAtStart; |
| *processed = 1; |
| return ZX_OK; |
| } |
| |
| fprintf(stderr, "Argument not found: %s\n", argv[0]); |
| return ZX_ERR_INVALID_ARGS; |
| } |
| |
| zx_status_t BlobfsCreator::CalculateRequiredSize(off_t* out) { |
| std::vector<std::thread> threads; |
| unsigned blob_index = 0; |
| unsigned n_threads = std::thread::hardware_concurrency(); |
| if (!n_threads) { |
| n_threads = 4; |
| } |
| zx_status_t status = ZX_OK; |
| std::mutex mtx; |
| bool should_compress = ShouldCompress(); |
| for (unsigned j = n_threads; j > 0; j--) { |
| threads.emplace_back([&] { |
| std::vector<blobfs::BlobInfo> local_blob_info_list; |
| unsigned i = 0; |
| while (true) { |
| mtx.lock(); |
| if (status != ZX_OK) { |
| mtx.unlock(); |
| return; |
| } |
| i = blob_index++; |
| mtx.unlock(); |
| if (i >= blob_list_.size()) { |
| break; |
| } |
| const std::filesystem::path& path = blob_list_[i]; |
| zx_status_t res; |
| if ((res = AppendDepfile(path.c_str())) != ZX_OK) { |
| mtx.lock(); |
| status = res; |
| mtx.unlock(); |
| return; |
| } |
| |
| fbl::unique_fd data_fd(open(path.c_str(), O_RDONLY, 0644)); |
| zx::status<blobfs::BlobInfo> blob_info = |
| should_compress |
| ? blobfs::BlobInfo::CreateCompressed(data_fd.get(), blob_layout_format_, path) |
| : blobfs::BlobInfo::CreateUncompressed(data_fd.get(), blob_layout_format_, path); |
| if (blob_info.is_error()) { |
| mtx.lock(); |
| status = blob_info.status_value(); |
| mtx.unlock(); |
| return; |
| } |
| |
| local_blob_info_list.push_back(std::move(blob_info).value()); |
| } |
| mtx.lock(); |
| blob_info_list_.insert(blob_info_list_.end(), |
| std::make_move_iterator(local_blob_info_list.begin()), |
| std::make_move_iterator(local_blob_info_list.end())); |
| mtx.unlock(); |
| }); |
| } |
| |
| for (auto& thread : threads) { |
| thread.join(); |
| } |
| |
| if (status != ZX_OK) { |
| return status; |
| } |
| |
| // Remove all duplicate blobs by first sorting the merkle trees by |
| // digest, and then by reshuffling the vector to exclude duplicates. |
| std::sort(blob_info_list_.begin(), blob_info_list_.end(), DigestCompare()); |
| auto compare = [](const blobfs::BlobInfo& lhs, const blobfs::BlobInfo& rhs) { |
| return lhs.GetDigest() == rhs.GetDigest(); |
| }; |
| auto it = std::unique(blob_info_list_.begin(), blob_info_list_.end(), compare); |
| blob_info_list_.erase(it, blob_info_list_.end()); |
| |
| uint64_t required_node_count = 0; |
| for (const auto& blob_info : blob_info_list_) { |
| uint32_t block_count = blob_info.GetBlobLayout().TotalBlockCount(); |
| data_blocks_ += block_count; |
| uint64_t extent_count = |
| fbl::round_up(block_count, blobfs::kBlockCountMax) / blobfs::kBlockCountMax; |
| required_node_count += blobfs::NodePopulator::NodeCountForExtents(extent_count); |
| } |
| |
| required_inodes_ = std::max(blobfs::kBlobfsDefaultInodeCount, required_node_count); |
| |
| blobfs::Superblock info; |
| // Initialize enough of |info| to be able to compute the number of bytes the image will occupy. |
| info.inode_count = required_inodes_; |
| info.data_block_count = data_blocks_; |
| info.journal_block_count = blobfs::kMinimumJournalBlocks; |
| *out = blobfs::TotalBlocks(info) * blobfs::kBlobfsBlockSize; |
| return ZX_OK; |
| } |
| |
| zx_status_t BlobfsCreator::Mkfs() { |
| uint64_t block_count; |
| if (blobfs::GetBlockCount(fd_.get(), &block_count)) { |
| fprintf(stderr, "blobfs: cannot find end of underlying device\n"); |
| return ZX_ERR_IO; |
| } |
| |
| int r = blobfs::Mkfs(fd_.get(), block_count, |
| {.blob_layout_format = blob_layout_format_, .num_inodes = required_inodes_}); |
| |
| if (r >= 0 && !blob_list_.empty()) { |
| zx_status_t status; |
| if ((status = Add()) != ZX_OK) { |
| return status; |
| } |
| } |
| return r; |
| } |
| |
| zx_status_t BlobfsCreator::Fsck() { |
| zx_status_t status; |
| std::unique_ptr<blobfs::Blobfs> vn; |
| if ((status = blobfs::blobfs_create(&vn, std::move(fd_))) < 0) { |
| return status; |
| } |
| |
| return blobfs::Fsck(vn.get()); |
| } |
| |
| zx_status_t BlobfsCreator::UsedDataSize() { |
| zx_status_t status; |
| uint64_t size; |
| if ((status = blobfs::UsedDataSize(fd_, &size)) != ZX_OK) { |
| return status; |
| } |
| |
| printf("%" PRIu64 "\n", size); |
| return ZX_OK; |
| } |
| |
| zx_status_t BlobfsCreator::UsedInodes() { |
| zx_status_t status; |
| uint64_t used_inodes; |
| if ((status = blobfs::UsedInodes(fd_, &used_inodes)) != ZX_OK) { |
| return status; |
| } |
| |
| printf("%" PRIu64 "\n", used_inodes); |
| return ZX_OK; |
| } |
| |
| zx_status_t BlobfsCreator::UsedSize() { |
| zx_status_t status; |
| uint64_t size; |
| if ((status = blobfs::UsedSize(fd_, &size)) != ZX_OK) { |
| return status; |
| } |
| |
| printf("%" PRIu64 "\n", size); |
| return ZX_OK; |
| } |
| |
| zx_status_t BlobfsCreator::Add() { |
| if (blob_list_.empty()) { |
| fprintf(stderr, "Adding a blob requires an additional file argument\n"); |
| return Usage(); |
| } |
| |
| if (zx::status status = CreateBlobfsWithBlobs(std::move(fd_), blob_info_list_); |
| status.is_error()) { |
| return status.status_value(); |
| } |
| |
| if (json_output_path().has_value()) { |
| if (zx::status status = RecordBlobs(*json_output_path(), blob_info_list_); status.is_error()) { |
| return status.status_value(); |
| } |
| } |
| |
| return ZX_OK; |
| } |
| |
| int ExportBlobs(std::string& source_path, std::string& output_path) { |
| fbl::unique_fd blobfs_image(open(source_path.c_str(), O_RDONLY)); |
| if (!blobfs_image.is_valid()) { |
| fprintf(stderr, "Failed to open blobfs image at %s. More specifically: %s.\n", |
| source_path.c_str(), strerror(errno)); |
| return -1; |
| } |
| |
| std::unique_ptr<blobfs::Blobfs> fs = nullptr; |
| if (blobfs::blobfs_create(&fs, std::move(blobfs_image)) != ZX_OK) { |
| return -1; |
| } |
| |
| // Try to create path if it doesn't exist. |
| std::filesystem::create_directories(output_path); |
| fbl::unique_fd output_fd(open(output_path.c_str(), O_DIRECTORY)); |
| if (!output_fd.is_valid()) { |
| fprintf(stderr, "Failed to obtain a handle to output path at %s. More specifically: %s.\n", |
| output_path.c_str(), strerror(errno)); |
| return -1; |
| } |
| |
| auto export_result = blobfs::ExportBlobs(output_fd.get(), *fs); |
| if (export_result.is_error()) { |
| fprintf(stderr, "Encountered error while exporting blobs. More specifically: %s.\n", |
| export_result.error().c_str()); |
| return -1; |
| } |
| fprintf(stderr, "Successfully exported all blobs.\n"); |
| return 0; |
| } |
| |
| int main(int argc, char** argv) { |
| BlobfsCreator blobfs; |
| |
| if (argc > 3) { |
| if (strcmp(argv[1], "export") == 0) { |
| std::string image_path = argv[2]; |
| std::string output_path = argv[3]; |
| return ExportBlobs(image_path, output_path); |
| } |
| } |
| |
| if (blobfs.ProcessAndRun(argc, argv) != ZX_OK) { |
| return -1; |
| } |
| |
| return 0; |
| } |