Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a889c87b authored by David Anderson's avatar David Anderson
Browse files

libsnapshot: Add an append mode to CowWriter. 

When in append mode, CowWriter will re-open the existing COW and resume
writing at the end of the old data position. All existing operations
will be reimported and buffered in memory.

The size calculation has been simplified to make this work. We now
advance ops_offset and no longer track the number of bytes written.

Additionally, a "header_size" field has been added to the header. This
was missing from the original format and is useful for introducing
forward compatibility later.

Finally, Finalize has been renamed to Flush. It's still mandatory, but
it can be called multiple times to continue appending data without
reopening.

Bug: 168554689
Test: cow_api_test gtest
Change-Id: I637e99ae08a4db5b273c06318e6db523ea8ec7c5
parent a652877b

fs_mgr/libsnapshot/Android.bp

+1 −0
Original line number Diff line number Diff line
@@ -412,6 +412,7 @@ cc_test { 
    test_min_api_level: 30,

    auto_gen_config: true,

    require_root: false,

    host_supported: true,

}



cc_binary {

fs_mgr/libsnapshot/cow_api_test.cpp

+61 −7
Original line number Diff line number Diff line
@@ -70,7 +70,7 @@ TEST_F(CowTest, ReadWrite) { 
    ASSERT_TRUE(writer.AddCopy(10, 20));

    ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer.AddZeroBlocks(51, 2));

    ASSERT_TRUE(writer.Finalize());

    ASSERT_TRUE(writer.Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@@ -105,7 +105,7 @@ TEST_F(CowTest, ReadWrite) { 
    ASSERT_EQ(op->compression, kCowCompressNone);

    ASSERT_EQ(op->data_length, 4096);

    ASSERT_EQ(op->new_block, 50);

    ASSERT_EQ(op->source, 104);

    ASSERT_EQ(op->source, 106);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);
@@ -145,7 +145,7 @@ TEST_F(CowTest, CompressGz) { 
    data.resize(options.block_size, '\0');



    ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer.Finalize());

    ASSERT_TRUE(writer.Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@@ -163,7 +163,7 @@ TEST_F(CowTest, CompressGz) { 
    ASSERT_EQ(op->compression, kCowCompressGz);

    ASSERT_EQ(op->data_length, 56);  // compressed!

    ASSERT_EQ(op->new_block, 50);

    ASSERT_EQ(op->source, 104);

    ASSERT_EQ(op->source, 106);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);
@@ -182,7 +182,7 @@ TEST_F(CowTest, CompressTwoBlocks) { 
    data.resize(options.block_size * 2, '\0');



    ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer.Finalize());

    ASSERT_TRUE(writer.Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@@ -224,7 +224,7 @@ TEST_P(CompressionTest, HorribleSink) { 
    data.resize(options.block_size, '\0');



    ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer.Finalize());

    ASSERT_TRUE(writer.Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@@ -259,7 +259,7 @@ TEST_F(CowTest, GetSize) { 
    ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer.AddZeroBlocks(51, 2));

    auto size_before = writer.GetCowSize();

    ASSERT_TRUE(writer.Finalize());

    ASSERT_TRUE(writer.Flush());

    auto size_after = writer.GetCowSize();

    ASSERT_EQ(size_before, size_after);

    struct stat buf;
@@ -271,6 +271,60 @@ TEST_F(CowTest, GetSize) { 
    ASSERT_EQ(buf.st_size, writer.GetCowSize());

}



TEST_F(CowTest, Append) {

    CowOptions options;

    auto writer = std::make_unique<CowWriter>(options);

    ASSERT_TRUE(writer->Initialize(cow_->fd));



    std::string data = "This is some data, believe it";

    data.resize(options.block_size, '\0');

    ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer->Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);



    writer = std::make_unique<CowWriter>(options);

    ASSERT_TRUE(writer->Initialize(cow_->fd, CowWriter::OpenMode::APPEND));



    std::string data2 = "More data!";

    data2.resize(options.block_size, '\0');

    ASSERT_TRUE(writer->AddRawBlocks(51, data2.data(), data2.size()));

    ASSERT_TRUE(writer->Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);



    struct stat buf;

    ASSERT_EQ(fstat(cow_->fd, &buf), 0);

    ASSERT_EQ(buf.st_size, writer->GetCowSize());



    // Read back both operations.

    CowReader reader;

    ASSERT_TRUE(reader.Parse(cow_->fd));



    StringSink sink;



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);



    ASSERT_FALSE(iter->Done());

    auto op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);



    iter->Next();

    sink.Reset();



    ASSERT_FALSE(iter->Done());

    op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data2);



    iter->Next();

    ASSERT_TRUE(iter->Done());

}



}  // namespace snapshot

}  // namespace android

fs_mgr/libsnapshot/cow_reader.cpp

+5 −0
Original line number Diff line number Diff line
@@ -78,6 +78,11 @@ bool CowReader::Parse(android::base::borrowed_fd fd) { 
                   << "Expected: " << kCowMagicNumber;

        return false;

    }

    if (header_.header_size != sizeof(CowHeader)) {

        LOG(ERROR) << "Header size unknown, read " << header_.header_size << ", expected "

                   << sizeof(CowHeader);

        return false;

    }



    if ((header_.major_version != kCowVersionMajor) ||

        (header_.minor_version != kCowVersionMinor)) {

fs_mgr/libsnapshot/cow_snapuserd_test.cpp

+1 −1
Original line number Diff line number Diff line
@@ -128,7 +128,7 @@ TEST_F(SnapuserdTest, ReadWrite) { 
    ASSERT_TRUE(writer.AddRawBlocks(blk_random2_replace_start, random_buffer_2.get(), size));



    // Flush operations

    ASSERT_TRUE(writer.Finalize());

    ASSERT_TRUE(writer.Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);

fs_mgr/libsnapshot/cow_writer.cpp

+90 −46
Original line number Diff line number Diff line
@@ -23,6 +23,7 @@ 
#include <android-base/logging.h>

#include <android-base/unique_fd.h>

#include <brotli/encode.h>

#include <libsnapshot/cow_reader.h>

#include <libsnapshot/cow_writer.h>

#include <zlib.h>
@@ -40,17 +41,48 @@ void CowWriter::SetupHeaders() { 
    header_.magic = kCowMagicNumber;

    header_.major_version = kCowVersionMajor;

    header_.minor_version = kCowVersionMinor;

    header_.header_size = sizeof(CowHeader);

    header_.block_size = options_.block_size;

}



bool CowWriter::Initialize(android::base::unique_fd&& fd) {

bool CowWriter::ParseOptions() {

    if (options_.compression == "gz") {

        compression_ = kCowCompressGz;

    } else if (options_.compression == "brotli") {

        compression_ = kCowCompressBrotli;

    } else if (options_.compression == "none") {

        compression_ = kCowCompressNone;

    } else if (!options_.compression.empty()) {

        LOG(ERROR) << "unrecognized compression: " << options_.compression;

        return false;

    }

    return true;

}



bool CowWriter::Initialize(android::base::unique_fd&& fd, OpenMode mode) {

    owned_fd_ = std::move(fd);

    return Initialize(android::base::borrowed_fd{owned_fd_});

    return Initialize(android::base::borrowed_fd{owned_fd_}, mode);

}



bool CowWriter::Initialize(android::base::borrowed_fd fd) {

bool CowWriter::Initialize(android::base::borrowed_fd fd, OpenMode mode) {

    fd_ = fd;



    if (!ParseOptions()) {

        return false;

    }



    switch (mode) {

        case OpenMode::WRITE:

            return OpenForWrite();

        case OpenMode::APPEND:

            return OpenForAppend();

        default:

            LOG(ERROR) << "Unknown open mode in CowWriter";

            return false;

    }

}



bool CowWriter::OpenForWrite() {

    // This limitation is tied to the data field size in CowOperation.

    if (header_.block_size > std::numeric_limits<uint16_t>::max()) {

        LOG(ERROR) << "Block size is too large";
@@ -62,35 +94,52 @@ bool CowWriter::Initialize(android::base::borrowed_fd fd) { 
        return false;

    }



    if (options_.compression == "gz") {

        compression_ = kCowCompressGz;

    } else if (options_.compression == "brotli") {

        compression_ = kCowCompressBrotli;

    } else if (options_.compression == "none") {

        compression_ = kCowCompressNone;

    } else if (!options_.compression.empty()) {

        LOG(ERROR) << "unrecognized compression: " << options_.compression;

        return false;

    }



    // Headers are not complete, but this ensures the file is at the right

    // position.

    if (!WriteFully(fd_, &header_, sizeof(header_))) {

    if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) {

        PLOG(ERROR) << "write failed";

        return false;

    }



    header_.ops_offset = header_.header_size;

    return true;

}



bool CowWriter::AddCopy(uint64_t new_block, uint64_t old_block) {

    header_.num_ops++;

bool CowWriter::OpenForAppend() {

    auto reader = std::make_unique<CowReader>();

    if (!reader->Parse(fd_) || !reader->GetHeader(&header_)) {

        return false;

    }

    options_.block_size = header_.block_size;



    // Reset this, since we're going to reimport all operations.

    header_.num_ops = 0;



    auto iter = reader->GetOpIter();

    while (!iter->Done()) {

        auto& op = iter->Get();

        AddOperation(op);



        iter->Next();

    }



    // Free reader so we own the descriptor position again.

    reader = nullptr;



    // Seek to the end of the data section.

    if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {

        PLOG(ERROR) << "lseek failed";

        return false;

    }

    return true;

}



bool CowWriter::AddCopy(uint64_t new_block, uint64_t old_block) {

    CowOperation op = {};

    op.type = kCowCopyOp;

    op.new_block = new_block;

    op.source = old_block;

    ops_ += std::basic_string<uint8_t>(reinterpret_cast<uint8_t*>(&op), sizeof(op));



    AddOperation(op);

    return true;

}
@@ -108,8 +157,6 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t 


    const uint8_t* iter = reinterpret_cast<const uint8_t*>(data);

    for (size_t i = 0; i < size / header_.block_size; i++) {

        header_.num_ops++;



        CowOperation op = {};

        op.type = kCowReplaceOp;

        op.new_block = new_block_start + i;
@@ -125,7 +172,7 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t 
                LOG(ERROR) << "Compressed block is too large: " << data.size() << " bytes";

                return false;

            }

            if (!WriteFully(fd_, data.data(), data.size())) {

            if (!WriteRawData(data.data(), data.size())) {

                PLOG(ERROR) << "AddRawBlocks: write failed";

                return false;

            }
@@ -137,11 +184,11 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t 
            pos += header_.block_size;

        }



        ops_ += std::basic_string<uint8_t>(reinterpret_cast<uint8_t*>(&op), sizeof(op));

        AddOperation(op);

        iter += header_.block_size;

    }



    if (!compression_ && !WriteFully(fd_, data, size)) {

    if (!compression_ && !WriteRawData(data, size)) {

        PLOG(ERROR) << "AddRawBlocks: write failed";

        return false;

    }
@@ -150,13 +197,11 @@ bool CowWriter::AddRawBlocks(uint64_t new_block_start, const void* data, size_t 


bool CowWriter::AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) {

    for (uint64_t i = 0; i < num_blocks; i++) {

        header_.num_ops++;



        CowOperation op = {};

        op.type = kCowZeroOp;

        op.new_block = new_block_start + i;

        op.source = 0;

        ops_ += std::basic_string<uint8_t>(reinterpret_cast<uint8_t*>(&op), sizeof(op));

        AddOperation(op);

    }

    return true;

}
@@ -212,17 +257,7 @@ static void SHA256(const void*, size_t, uint8_t[]) { 
#endif

}



bool CowWriter::Finalize() {

    // If both fields are set then Finalize is already called.

    if (header_.ops_offset > 0 && header_.ops_size > 0) {

        return true;

    }

    auto offs = lseek(fd_.get(), 0, SEEK_CUR);

    if (offs < 0) {

        PLOG(ERROR) << "lseek failed";

        return false;

    }

    header_.ops_offset = offs;

bool CowWriter::Flush() {

    header_.ops_size = ops_.size();



    memset(header_.ops_checksum, 0, sizeof(uint8_t) * 32);
@@ -235,8 +270,6 @@ bool CowWriter::Finalize() { 
        PLOG(ERROR) << "lseek failed";

        return false;

    }

    // Header is already written, calling WriteFully will increment

    // bytes_written_. So use android::base::WriteFully() here.

    if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) {

        PLOG(ERROR) << "write header failed";

        return false;
@@ -250,13 +283,16 @@ bool CowWriter::Finalize() { 
        return false;

    }



    // clear ops_ so that subsequent calls to GetSize() still works.

    ops_.clear();

    // Re-position for any subsequent writes.

    if (lseek(fd_.get(), header_.ops_offset, SEEK_SET) < 0) {

        PLOG(ERROR) << "lseek ops failed";

        return false;

    }

    return true;

}



size_t CowWriter::GetCowSize() {

    return bytes_written_ + ops_.size() * sizeof(ops_[0]);

    return header_.ops_offset + header_.num_ops * sizeof(CowOperation);

}



bool CowWriter::GetDataPos(uint64_t* pos) {
@@ -269,9 +305,17 @@ bool CowWriter::GetDataPos(uint64_t* pos) { 
    return true;

}



bool CowWriter::WriteFully(base::borrowed_fd fd, const void* data, size_t size) {

    bytes_written_ += size;

    return android::base::WriteFully(fd, data, size);

void CowWriter::AddOperation(const CowOperation& op) {

    header_.num_ops++;

    ops_.insert(ops_.size(), reinterpret_cast<const uint8_t*>(&op), sizeof(op));

}



bool CowWriter::WriteRawData(const void* data, size_t size) {

    if (!android::base::WriteFully(fd_, data, size)) {

        return false;

    }

    header_.ops_offset += size;

    return true;

}



}  // namespace snapshot