blob: bd97a78ae8f778bb201e593fe597f2f431c358be [file] [log] [blame]
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fcntl.h>
#include <inttypes.h>
#include <linux/limits.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <unistd.h>
#include <string>
#include <utility>
#include <android-base/file.h>
#include <android-base/logging.h>
#include <android-base/stringprintf.h>
#include <android-base/unique_fd.h>
#include <fstab/fstab.h>
#include <gtest/gtest.h>
#include <libdm/loop_control.h>
#include <libfiemap/fiemap_writer.h>
#include <libfiemap/split_fiemap_writer.h>
#include <libgsi/libgsi.h>
#include <storage_literals/storage_literals.h>
#include "utility.h"
namespace android {
namespace fiemap {
using namespace std;
using namespace std::string_literals;
using namespace android::fiemap;
using namespace android::storage_literals;
using unique_fd = android::base::unique_fd;
using LoopDevice = android::dm::LoopDevice;
std::string gTestDir;
uint64_t testfile_size = 536870912; // default of 512MiB
size_t gBlockSize = 0;
class FiemapWriterTest : public ::testing::Test {
protected:
void SetUp() override {
const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
testfile = gTestDir + "/"s + tinfo->name();
}
void TearDown() override { unlink(testfile.c_str()); }
// name of the file we use for testing
std::string testfile;
};
class SplitFiemapTest : public ::testing::Test {
protected:
void SetUp() override {
const ::testing::TestInfo* tinfo = ::testing::UnitTest::GetInstance()->current_test_info();
testfile = gTestDir + "/"s + tinfo->name();
}
void TearDown() override {
std::string message;
if (!SplitFiemap::RemoveSplitFiles(testfile, &message)) {
cerr << "Could not remove all split files: " << message;
}
}
// name of the file we use for testing
std::string testfile;
};
TEST_F(FiemapWriterTest, CreateImpossiblyLargeFile) {
// Try creating a file of size ~100TB but aligned to
// 512 byte to make sure block alignment tests don't
// fail.
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1099511627997184);
EXPECT_EQ(fptr, nullptr);
EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
EXPECT_EQ(errno, ENOENT);
}
TEST_F(FiemapWriterTest, CreateUnalignedFile) {
// Try creating a file of size 4097 bytes which is guaranteed
// to be unaligned to all known block sizes.
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize + 1);
ASSERT_NE(fptr, nullptr);
ASSERT_EQ(fptr->size(), gBlockSize * 2);
}
TEST_F(FiemapWriterTest, CheckFilePath) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
ASSERT_NE(fptr, nullptr);
EXPECT_EQ(fptr->size(), gBlockSize);
EXPECT_EQ(fptr->file_path(), testfile);
EXPECT_EQ(access(testfile.c_str(), F_OK), 0);
}
TEST_F(FiemapWriterTest, CheckFileSize) {
// Create a large-ish file and test that the expected size matches.
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 1024 * 1024 * 16);
ASSERT_NE(fptr, nullptr);
struct stat s;
ASSERT_EQ(stat(testfile.c_str(), &s), 0);
EXPECT_EQ(static_cast<uint64_t>(s.st_size), fptr->size());
}
TEST_F(FiemapWriterTest, CheckProgress) {
std::vector<uint64_t> expected;
size_t invocations = 0;
auto callback = [&](uint64_t done, uint64_t total) -> bool {
if (invocations >= expected.size()) {
return false;
}
EXPECT_EQ(done, expected[invocations]);
EXPECT_EQ(total, gBlockSize);
invocations++;
return true;
};
expected.push_back(gBlockSize);
auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback));
EXPECT_NE(ptr, nullptr);
EXPECT_EQ(invocations, expected.size());
}
TEST_F(FiemapWriterTest, CheckPinning) {
auto ptr = FiemapWriter::Open(testfile, 4096);
ASSERT_NE(ptr, nullptr);
EXPECT_TRUE(FiemapWriter::HasPinnedExtents(testfile));
}
TEST_F(FiemapWriterTest, CheckBlockDevicePath) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
EXPECT_EQ(fptr->size(), gBlockSize);
EXPECT_EQ(fptr->bdev_path().find("/dev/block/"), size_t(0));
if (!android::gsi::IsGsiRunning()) {
EXPECT_EQ(fptr->bdev_path().find("/dev/block/dm-"), string::npos);
}
}
TEST_F(FiemapWriterTest, CheckFileCreated) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 32768);
ASSERT_NE(fptr, nullptr);
unique_fd fd(open(testfile.c_str(), O_RDONLY));
EXPECT_GT(fd, -1);
}
TEST_F(FiemapWriterTest, CheckFileSizeActual) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size);
ASSERT_NE(fptr, nullptr);
struct stat sb;
ASSERT_EQ(stat(testfile.c_str(), &sb), 0);
EXPECT_GE(sb.st_size, testfile_size);
}
TEST_F(FiemapWriterTest, CheckFileExtents) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, testfile_size);
ASSERT_NE(fptr, nullptr);
EXPECT_GT(fptr->extents().size(), 0);
}
TEST_F(FiemapWriterTest, ExistingFile) {
// Create the file.
{ ASSERT_NE(FiemapWriter::Open(testfile, gBlockSize), nullptr); }
// Test that we can still open it.
{
auto ptr = FiemapWriter::Open(testfile, 0, false);
ASSERT_NE(ptr, nullptr);
EXPECT_GT(ptr->extents().size(), 0);
}
}
TEST_F(FiemapWriterTest, FileDeletedOnError) {
auto callback = [](uint64_t, uint64_t) -> bool { return false; };
auto ptr = FiemapWriter::Open(testfile, gBlockSize, true, std::move(callback));
EXPECT_EQ(ptr, nullptr);
EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
EXPECT_EQ(errno, ENOENT);
}
TEST_F(FiemapWriterTest, MaxBlockSize) {
uint64_t max_piece_size = 0;
ASSERT_TRUE(DetermineMaximumFileSize(testfile, &max_piece_size));
ASSERT_GT(max_piece_size, 0);
}
TEST_F(FiemapWriterTest, FibmapBlockAddressing) {
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, gBlockSize);
ASSERT_NE(fptr, nullptr);
switch (fptr->fs_type()) {
case F2FS_SUPER_MAGIC:
case EXT4_SUPER_MAGIC:
// Skip the test for FIEMAP supported filesystems. This is really
// because f2fs/ext4 have caches that seem to defeat reading back
// directly from the block device, and writing directly is too
// dangerous.
std::cout << "Skipping test, filesystem does not use FIBMAP\n";
return;
}
bool uses_dm;
std::string bdev_path;
ASSERT_TRUE(FiemapWriter::GetBlockDeviceForFile(testfile, &bdev_path, &uses_dm));
if (uses_dm) {
// We could use a device-mapper wrapper here to bypass encryption, but
// really this test is for FIBMAP correctness on VFAT (where encryption
// is never used), so we don't bother.
std::cout << "Skipping test, block device is metadata encrypted\n";
return;
}
std::string data(fptr->size(), '\0');
for (size_t i = 0; i < data.size(); i++) {
data[i] = 'A' + static_cast<char>(data.size() % 26);
}
{
unique_fd fd(open(testfile.c_str(), O_WRONLY | O_CLOEXEC));
ASSERT_GE(fd, 0);
ASSERT_TRUE(android::base::WriteFully(fd, data.data(), data.size()));
ASSERT_EQ(fsync(fd), 0);
}
ASSERT_FALSE(fptr->extents().empty());
const auto& first_extent = fptr->extents()[0];
unique_fd bdev(open(fptr->bdev_path().c_str(), O_RDONLY | O_CLOEXEC));
ASSERT_GE(bdev, 0);
off_t where = first_extent.fe_physical;
ASSERT_EQ(lseek(bdev, where, SEEK_SET), where);
// Note: this will fail on encrypted folders.
std::string actual(data.size(), '\0');
ASSERT_GE(first_extent.fe_length, data.size());
ASSERT_TRUE(android::base::ReadFully(bdev, actual.data(), actual.size()));
EXPECT_EQ(memcmp(actual.data(), data.data(), data.size()), 0);
}
TEST_F(FiemapWriterTest, CheckEmptyFile) {
// Can't get any fiemap_extent out of a zero-sized file.
FiemapUniquePtr fptr = FiemapWriter::Open(testfile, 0);
EXPECT_EQ(fptr, nullptr);
EXPECT_EQ(access(testfile.c_str(), F_OK), -1);
}
TEST_F(SplitFiemapTest, Create) {
auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
ASSERT_NE(ptr, nullptr);
auto extents = ptr->extents();
// Destroy the fiemap, closing file handles. This should not delete them.
ptr = nullptr;
std::vector<std::string> files;
ASSERT_TRUE(SplitFiemap::GetSplitFileList(testfile, &files));
for (const auto& path : files) {
EXPECT_EQ(access(path.c_str(), F_OK), 0);
}
ASSERT_GE(extents.size(), files.size());
}
TEST_F(SplitFiemapTest, Open) {
{
auto ptr = SplitFiemap::Create(testfile, 1024 * 768, 1024 * 32);
ASSERT_NE(ptr, nullptr);
}
auto ptr = SplitFiemap::Open(testfile);
ASSERT_NE(ptr, nullptr);
auto extents = ptr->extents();
ASSERT_GE(extents.size(), 24);
}
TEST_F(SplitFiemapTest, DeleteOnFail) {
auto ptr = SplitFiemap::Create(testfile, 1024 * 1024 * 100, 1);
ASSERT_EQ(ptr, nullptr);
std::string first_file = testfile + ".0001";
ASSERT_NE(access(first_file.c_str(), F_OK), 0);
ASSERT_EQ(errno, ENOENT);
ASSERT_NE(access(testfile.c_str(), F_OK), 0);
ASSERT_EQ(errno, ENOENT);
}
TEST_F(SplitFiemapTest, CorruptSplit) {
unique_fd fd(open(testfile.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0700));
ASSERT_GE(fd, 0);
// Make a giant random string.
std::vector<char> data;
for (size_t i = 0x1; i < 0x7f; i++) {
for (size_t j = 0; j < 100; j++) {
data.emplace_back(i);
}
}
ASSERT_GT(data.size(), PATH_MAX);
data.emplace_back('\n');
ASSERT_TRUE(android::base::WriteFully(fd, data.data(), data.size()));
fd = {};
ASSERT_TRUE(SplitFiemap::RemoveSplitFiles(testfile));
}
static string ReadSplitFiles(const std::string& base_path, size_t num_files) {
std::string result;
for (int i = 0; i < num_files; i++) {
std::string path = base_path + android::base::StringPrintf(".%04d", i);
std::string data;
if (!android::base::ReadFileToString(path, &data)) {
return {};
}
result += data;
}
return result;
}
TEST_F(SplitFiemapTest, WriteWholeFile) {
static constexpr size_t kChunkSize = 32768;
static constexpr size_t kSize = kChunkSize * 3;
auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
ASSERT_NE(ptr, nullptr);
auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
for (size_t i = 0; i < kSize / sizeof(int); i++) {
buffer[i] = i;
}
ASSERT_TRUE(ptr->Write(buffer.get(), kSize));
std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
auto actual = ReadSplitFiles(testfile, 3);
ASSERT_EQ(expected.size(), actual.size());
EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
}
TEST_F(SplitFiemapTest, WriteFileInChunks1) {
static constexpr size_t kChunkSize = 32768;
static constexpr size_t kSize = kChunkSize * 3;
auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
ASSERT_NE(ptr, nullptr);
auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
for (size_t i = 0; i < kSize / sizeof(int); i++) {
buffer[i] = i;
}
// Write in chunks of 1000 (so some writes straddle the boundary of two
// files).
size_t bytes_written = 0;
while (bytes_written < kSize) {
size_t to_write = std::min(kSize - bytes_written, (size_t)1000);
char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;
ASSERT_TRUE(ptr->Write(data, to_write));
bytes_written += to_write;
}
std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
auto actual = ReadSplitFiles(testfile, 3);
ASSERT_EQ(expected.size(), actual.size());
EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
}
TEST_F(SplitFiemapTest, WriteFileInChunks2) {
static constexpr size_t kChunkSize = 32768;
static constexpr size_t kSize = kChunkSize * 3;
auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
ASSERT_NE(ptr, nullptr);
auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
for (size_t i = 0; i < kSize / sizeof(int); i++) {
buffer[i] = i;
}
// Write in chunks of 32KiB so every write is exactly at the end of the
// current file.
size_t bytes_written = 0;
while (bytes_written < kSize) {
size_t to_write = std::min(kSize - bytes_written, kChunkSize);
char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;
ASSERT_TRUE(ptr->Write(data, to_write));
bytes_written += to_write;
}
std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);
auto actual = ReadSplitFiles(testfile, 3);
ASSERT_EQ(expected.size(), actual.size());
EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);
}
TEST_F(SplitFiemapTest, WritePastEnd) {
static constexpr size_t kChunkSize = 32768;
static constexpr size_t kSize = kChunkSize * 3;
auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);
ASSERT_NE(ptr, nullptr);
auto buffer = std::make_unique<int[]>(kSize / sizeof(int));
for (size_t i = 0; i < kSize / sizeof(int); i++) {
buffer[i] = i;
}
ASSERT_TRUE(ptr->Write(buffer.get(), kSize));
ASSERT_FALSE(ptr->Write(buffer.get(), kSize));
}
// Get max file size and free space.
std::pair<uint64_t, uint64_t> GetBigFileLimit(const std::string& mount_point) {
struct statvfs fs;
if (statvfs(mount_point.c_str(), &fs) < 0) {
PLOG(ERROR) << "statfs failed";
return {0, 0};
}
auto fs_limit = static_cast<uint64_t>(fs.f_blocks) * (fs.f_bsize - 1);
auto fs_free = static_cast<uint64_t>(fs.f_bfree) * fs.f_bsize;
LOG(INFO) << "Big file limit: " << fs_limit << ", free space: " << fs_free;
return {fs_limit, fs_free};
}
class FsTest : public ::testing::Test {
protected:
// 2GB Filesystem and 4k block size by default
static constexpr uint64_t block_size = 4096;
static constexpr uint64_t fs_size = 64 * 1024 * 1024;
void SetUp() {
android::fs_mgr::Fstab fstab;
ASSERT_TRUE(android::fs_mgr::ReadFstabFromFile("/proc/mounts", &fstab));
ASSERT_EQ(access(tmpdir_.path, F_OK), 0);
fs_path_ = tmpdir_.path + "/fs_image"s;
mntpoint_ = tmpdir_.path + "/mnt_point"s;
auto entry = android::fs_mgr::GetEntryForMountPoint(&fstab, "/data");
ASSERT_NE(entry, nullptr);
if (entry->fs_type == "ext4") {
SetUpExt4();
} else if (entry->fs_type == "f2fs") {
SetUpF2fs();
} else {
FAIL() << "Unrecognized fs_type: " << entry->fs_type;
}
}
void SetUpExt4() {
uint64_t count = fs_size / block_size;
std::string dd_cmd =
::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64
" count=%" PRIu64 " > /dev/null 2>&1",
fs_path_.c_str(), block_size, count);
std::string mkfs_cmd =
::android::base::StringPrintf("/system/bin/mkfs.ext4 -q %s", fs_path_.c_str());
// create mount point
ASSERT_EQ(mkdir(mntpoint_.c_str(), S_IRWXU), 0);
// create file for the file system
int ret = system(dd_cmd.c_str());
ASSERT_EQ(ret, 0);
// Get and attach a loop device to the filesystem we created
LoopDevice loop_dev(fs_path_, 10s);
ASSERT_TRUE(loop_dev.valid());
// create file system
ret = system(mkfs_cmd.c_str());
ASSERT_EQ(ret, 0);
// mount the file system
ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint_.c_str(), "ext4", 0, nullptr), 0);
}
void SetUpF2fs() {
uint64_t count = fs_size / block_size;
std::string dd_cmd =
::android::base::StringPrintf("/system/bin/dd if=/dev/zero of=%s bs=%" PRIu64
" count=%" PRIu64 " > /dev/null 2>&1",
fs_path_.c_str(), block_size, count);
std::string mkfs_cmd =
::android::base::StringPrintf("/system/bin/make_f2fs -q %s", fs_path_.c_str());
// create mount point
ASSERT_EQ(mkdir(mntpoint_.c_str(), S_IRWXU), 0);
// create file for the file system
int ret = system(dd_cmd.c_str());
ASSERT_EQ(ret, 0);
// Get and attach a loop device to the filesystem we created
LoopDevice loop_dev(fs_path_, 10s);
ASSERT_TRUE(loop_dev.valid());
// create file system
ret = system(mkfs_cmd.c_str());
ASSERT_EQ(ret, 0);
// mount the file system
ASSERT_EQ(mount(loop_dev.device().c_str(), mntpoint_.c_str(), "f2fs", 0, nullptr), 0);
}
void TearDown() override {
umount(mntpoint_.c_str());
rmdir(mntpoint_.c_str());
unlink(fs_path_.c_str());
}
TemporaryDir tmpdir_;
std::string mntpoint_;
std::string fs_path_;
};
TEST_F(FsTest, LowSpaceError) {
auto limits = GetBigFileLimit(mntpoint_);
ASSERT_GE(limits.first, 0);
FiemapUniquePtr ptr;
auto test_file = mntpoint_ + "/big_file";
auto status = FiemapWriter::Open(test_file, limits.first, &ptr);
ASSERT_FALSE(status.is_ok());
ASSERT_EQ(status.error_code(), FiemapStatus::ErrorCode::NO_SPACE);
// Also test for EFBIG.
status = FiemapWriter::Open(test_file, 16_TiB, &ptr);
ASSERT_FALSE(status.is_ok());
ASSERT_NE(status.error_code(), FiemapStatus::ErrorCode::NO_SPACE);
}
bool DetermineBlockSize() {
struct statfs s;
if (statfs(gTestDir.c_str(), &s)) {
std::cerr << "Could not call statfs: " << strerror(errno) << "\n";
return false;
}
if (!s.f_bsize) {
std::cerr << "Invalid block size: " << s.f_bsize << "\n";
return false;
}
gBlockSize = s.f_bsize;
return true;
}
} // namespace fiemap
} // namespace android
using namespace android::fiemap;
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
if (argc > 1 && argv[1] == "-h"s) {
cerr << "Usage: [test_dir] [file_size]\n";
cerr << "\n";
cerr << "Note: test_dir must be a writable, unencrypted directory.\n";
exit(EXIT_FAILURE);
}
::android::base::InitLogging(argv, ::android::base::StderrLogger);
std::string root_dir = "/data/local/unencrypted";
if (access(root_dir.c_str(), F_OK)) {
root_dir = "/data";
}
std::string tempdir = root_dir + "/XXXXXX"s;
if (!mkdtemp(tempdir.data())) {
cerr << "unable to create tempdir on " << root_dir << "\n";
exit(EXIT_FAILURE);
}
if (!android::base::Realpath(tempdir, &gTestDir)) {
cerr << "unable to find realpath for " << tempdir;
exit(EXIT_FAILURE);
}
if (argc > 2) {
testfile_size = strtoull(argv[2], NULL, 0);
if (testfile_size == ULLONG_MAX) {
testfile_size = 512 * 1024 * 1024;
}
}
if (!DetermineBlockSize()) {
exit(EXIT_FAILURE);
}
auto result = RUN_ALL_TESTS();
std::string cmd = "rm -rf " + gTestDir;
system(cmd.c_str());
return result;
}