Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e9223204 authored by David Anderson's avatar David Anderson Committed by Gerrit Code Review
Browse files

Merge "Add write support to SplitFiemap."

parents 1c1850fa 6b4b265a

fs_mgr/libfiemap_writer/fiemap_writer.cpp

+0 −1
Original line number Diff line number Diff line
@@ -618,7 +618,6 @@ FiemapUniquePtr FiemapWriter::Open(const std::string& file_path, uint64_t file_s 


    fmap->file_path_ = abs_path;

    fmap->bdev_path_ = bdev_path;

    fmap->file_fd_ = std::move(file_fd);

    fmap->file_size_ = file_size;

    fmap->bdev_size_ = bdevsz;

    fmap->fs_type_ = fs_type;

fs_mgr/libfiemap_writer/fiemap_writer_test.cpp

+107 −0
Original line number Diff line number Diff line
@@ -38,6 +38,9 @@ 


#include "utility.h"



namespace android {

namespace fiemap_writer {



using namespace std;

using namespace std::string_literals;

using namespace android::fiemap_writer;
@@ -234,6 +237,105 @@ TEST_F(SplitFiemapTest, DeleteOnFail) { 
    ASSERT_EQ(errno, ENOENT);

}



static string ReadSplitFiles(const std::string& base_path, size_t num_files) {

    std::string result;

    for (int i = 0; i < num_files; i++) {

        std::string path = base_path + android::base::StringPrintf(".%04d", i);

        std::string data;

        if (!android::base::ReadFileToString(path, &data)) {

            return {};

        }

        result += data;

    }

    return result;

}



TEST_F(SplitFiemapTest, WriteWholeFile) {

    static constexpr size_t kChunkSize = 32768;

    static constexpr size_t kSize = kChunkSize * 3;

    auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);

    ASSERT_NE(ptr, nullptr);



    auto buffer = std::make_unique<int[]>(kSize / sizeof(int));

    for (size_t i = 0; i < kSize / sizeof(int); i++) {

        buffer[i] = i;

    }

    ASSERT_TRUE(ptr->Write(buffer.get(), kSize));



    std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);

    auto actual = ReadSplitFiles(testfile, 3);

    ASSERT_EQ(expected.size(), actual.size());

    EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);

}



TEST_F(SplitFiemapTest, WriteFileInChunks1) {

    static constexpr size_t kChunkSize = 32768;

    static constexpr size_t kSize = kChunkSize * 3;

    auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);

    ASSERT_NE(ptr, nullptr);



    auto buffer = std::make_unique<int[]>(kSize / sizeof(int));

    for (size_t i = 0; i < kSize / sizeof(int); i++) {

        buffer[i] = i;

    }



    // Write in chunks of 1000 (so some writes straddle the boundary of two

    // files).

    size_t bytes_written = 0;

    while (bytes_written < kSize) {

        size_t to_write = std::min(kSize - bytes_written, (size_t)1000);

        char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;

        ASSERT_TRUE(ptr->Write(data, to_write));

        bytes_written += to_write;

    }



    std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);

    auto actual = ReadSplitFiles(testfile, 3);

    ASSERT_EQ(expected.size(), actual.size());

    EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);

}



TEST_F(SplitFiemapTest, WriteFileInChunks2) {

    static constexpr size_t kChunkSize = 32768;

    static constexpr size_t kSize = kChunkSize * 3;

    auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);

    ASSERT_NE(ptr, nullptr);



    auto buffer = std::make_unique<int[]>(kSize / sizeof(int));

    for (size_t i = 0; i < kSize / sizeof(int); i++) {

        buffer[i] = i;

    }



    // Write in chunks of 32KiB so every write is exactly at the end of the

    // current file.

    size_t bytes_written = 0;

    while (bytes_written < kSize) {

        size_t to_write = std::min(kSize - bytes_written, kChunkSize);

        char* data = reinterpret_cast<char*>(buffer.get()) + bytes_written;

        ASSERT_TRUE(ptr->Write(data, to_write));

        bytes_written += to_write;

    }



    std::string expected(reinterpret_cast<char*>(buffer.get()), kSize);

    auto actual = ReadSplitFiles(testfile, 3);

    ASSERT_EQ(expected.size(), actual.size());

    EXPECT_EQ(memcmp(expected.data(), actual.data(), actual.size()), 0);

}



TEST_F(SplitFiemapTest, WritePastEnd) {

    static constexpr size_t kChunkSize = 32768;

    static constexpr size_t kSize = kChunkSize * 3;

    auto ptr = SplitFiemap::Create(testfile, kSize, kChunkSize);

    ASSERT_NE(ptr, nullptr);



    auto buffer = std::make_unique<int[]>(kSize / sizeof(int));

    for (size_t i = 0; i < kSize / sizeof(int); i++) {

        buffer[i] = i;

    }

    ASSERT_TRUE(ptr->Write(buffer.get(), kSize));

    ASSERT_FALSE(ptr->Write(buffer.get(), kSize));

}



class VerifyBlockWritesExt4 : public ::testing::Test {

    // 2GB Filesystem and 4k block size by default

    static constexpr uint64_t block_size = 4096;
@@ -333,6 +435,11 @@ bool DetermineBlockSize() { 
    return true;

}



}  // namespace fiemap_writer

}  // namespace android



using namespace android::fiemap_writer;



int main(int argc, char** argv) {

    ::testing::InitGoogleTest(&argc, argv);

    if (argc <= 1) {

fs_mgr/libfiemap_writer/include/libfiemap_writer/fiemap_writer.h

+0 −3
Original line number Diff line number Diff line
@@ -88,9 +88,6 @@ class FiemapWriter final { 
    // Block device on which we have created the file.

    std::string bdev_path_;



    // File descriptors for the file and block device

    ::android::base::unique_fd file_fd_;



    // Size in bytes of the file this class is writing

    uint64_t file_size_;

fs_mgr/libfiemap_writer/include/libfiemap_writer/split_fiemap_writer.h

+17 −0
Original line number Diff line number Diff line
@@ -23,6 +23,8 @@ 
#include <string>

#include <vector>



#include <android-base/unique_fd.h>



#include "fiemap_writer.h"



namespace android {
@@ -54,6 +56,16 @@ class SplitFiemap final { 
    // this returns true and does not report an error.

    static bool RemoveSplitFiles(const std::string& file_path, std::string* message = nullptr);



    // Return whether all components of a split file still have pinned extents.

    bool HasPinnedExtents() const;



    // Helper method for writing data that spans files. Note there is no seek

    // method (yet); this starts at 0 and increments the position by |bytes|.

    bool Write(const void* data, uint64_t bytes);



    // Flush all writes to all split files.

    bool Flush();



    const std::vector<struct fiemap_extent>& extents();

    uint32_t block_size() const;

    uint64_t size() const { return total_size_; }
@@ -73,6 +85,11 @@ class SplitFiemap final { 
    std::vector<FiemapUniquePtr> files_;

    std::vector<struct fiemap_extent> extents_;

    uint64_t total_size_ = 0;



    // Most recently open file and position for Write().

    size_t cursor_index_ = 0;

    uint64_t cursor_file_pos_ = 0;

    android::base::unique_fd cursor_fd_;

};



}  // namespace fiemap_writer

fs_mgr/libfiemap_writer/split_fiemap_writer.cpp

+79 −0
Original line number Diff line number Diff line
@@ -176,6 +176,15 @@ bool SplitFiemap::RemoveSplitFiles(const std::string& file_path, std::string* me 
    return ok;

}



bool SplitFiemap::HasPinnedExtents() const {

    for (const auto& file : files_) {

        if (!FiemapWriter::HasPinnedExtents(file->file_path())) {

            return false;

        }

    }

    return true;

}



const std::vector<struct fiemap_extent>& SplitFiemap::extents() {

    if (extents_.empty()) {

        for (const auto& file : files_) {
@@ -186,6 +195,76 @@ const std::vector<struct fiemap_extent>& SplitFiemap::extents() { 
    return extents_;

}



bool SplitFiemap::Write(const void* data, uint64_t bytes) {

    // Open the current file.

    FiemapWriter* file = files_[cursor_index_].get();



    const uint8_t* data_ptr = reinterpret_cast<const uint8_t*>(data);

    uint64_t bytes_remaining = bytes;

    while (bytes_remaining) {

        // How many bytes can we write into the current file?

        uint64_t file_bytes_left = file->size() - cursor_file_pos_;

        if (!file_bytes_left) {

            if (cursor_index_ == files_.size() - 1) {

                LOG(ERROR) << "write past end of file requested";

                return false;

            }



            // No space left in the current file, but we have more files to

            // use, so prep the next one.

            cursor_fd_ = {};

            cursor_file_pos_ = 0;

            file = files_[++cursor_index_].get();

            file_bytes_left = file->size();

        }



        // Open the current file if it's not open.

        if (cursor_fd_ < 0) {

            cursor_fd_.reset(open(file->file_path().c_str(), O_CLOEXEC | O_WRONLY));

            if (cursor_fd_ < 0) {

                PLOG(ERROR) << "open failed: " << file->file_path();

                return false;

            }

            CHECK(cursor_file_pos_ == 0);

        }



        if (!FiemapWriter::HasPinnedExtents(file->file_path())) {

            LOG(ERROR) << "file is no longer pinned: " << file->file_path();

            return false;

        }



        uint64_t bytes_to_write = std::min(file_bytes_left, bytes_remaining);

        if (!android::base::WriteFully(cursor_fd_, data_ptr, bytes_to_write)) {

            PLOG(ERROR) << "write failed: " << file->file_path();

            return false;

        }

        data_ptr += bytes_to_write;

        bytes_remaining -= bytes_to_write;

        cursor_file_pos_ += bytes_to_write;

    }



    // If we've reached the end of the current file, close it for sanity.

    if (cursor_file_pos_ == file->size()) {

        cursor_fd_ = {};

    }

    return true;

}



bool SplitFiemap::Flush() {

    for (const auto& file : files_) {

        unique_fd fd(open(file->file_path().c_str(), O_RDONLY | O_CLOEXEC));

        if (fd < 0) {

            PLOG(ERROR) << "open failed: " << file->file_path();

            return false;

        }

        if (fsync(fd)) {

            PLOG(ERROR) << "fsync failed: " << file->file_path();

            return false;

        }

    }

    return true;

}



SplitFiemap::~SplitFiemap() {

    if (!creating_) {

        return;