Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 1ea351b6 authored by David Anderson's avatar David Anderson
Browse files

libsnapshot: Deprecate the IByteSink API. 

This introduces a replacement for the IByteSink API, which was never
really used in any advantageous way, and is not expected to be useful.
The new API attempts to fill an entire buffer rather than request
individual slices of a buffer.

This patch simply introduces the API and refactors tests. Subsequent
patches will replace IByteSink callers and remove the API.

Bug: 278637212
Test: cow_api_test
Change-Id: Ib740de5e65fee8d61f603b106752338cc8e95967
parent 3c5fe110

fs_mgr/libsnapshot/include/libsnapshot/cow_format.h

+6 −1
Original line number Diff line number Diff line
@@ -15,7 +15,9 @@ 
#pragma once



#include <stdint.h>

#include <string>



#include <optional>

#include <string_view>



namespace android {

namespace snapshot {
@@ -196,5 +198,8 @@ bool IsMetadataOp(const CowOperation& op); 
// Ops that have dependencies on old blocks, and must take care in their merge order

bool IsOrderedOp(const CowOperation& op);



// Convert compression name to internal value.

std::optional<CowCompressionAlgorithm> CompressionAlgorithmFromString(std::string_view name);



}  // namespace snapshot

}  // namespace android

fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h

+18 −0
Original line number Diff line number Diff line
@@ -65,6 +65,7 @@ class ICowReader { 


    // Return the file header.

    virtual bool GetHeader(CowHeader* header) = 0;

    virtual CowHeader& GetHeader() = 0;



    // Return the file footer.

    virtual bool GetFooter(CowFooter* footer) = 0;
@@ -85,6 +86,19 @@ class ICowReader { 
    // Get decoded bytes from the data section, handling any decompression.

    // All retrieved data is passed to the sink.

    virtual bool ReadData(const CowOperation& op, IByteSink* sink) = 0;



    // Get decoded bytes from the data section, handling any decompression.

    //

    // If ignore_bytes is non-zero, it specifies the initial number of bytes

    // to skip writing to |buffer|.

    //

    // Returns the number of bytes written to |buffer|, or -1 on failure.

    // errno is NOT set.

    //

    // Partial reads are not possible unless |buffer_size| is less than the

    // operation block size.

    virtual ssize_t ReadData(const CowOperation& op, void* buffer, size_t buffer_size,

                             size_t ignore_bytes = 0) = 0;

};



// Iterate over a sequence of COW operations.
@@ -140,6 +154,10 @@ class CowReader final : public ICowReader { 
    std::unique_ptr<ICowOpIter> GetMergeOpIter(bool ignore_progress = false) override;



    bool ReadData(const CowOperation& op, IByteSink* sink) override;

    ssize_t ReadData(const CowOperation& op, void* buffer, size_t buffer_size,

                     size_t ignore_bytes = 0) override;



    CowHeader& GetHeader() override { return header_; }



    bool GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read);

fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp

+132 −127
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 
#include <gtest/gtest.h>

#include <libsnapshot/cow_reader.h>

#include <libsnapshot/cow_writer.h>

#include "cow_decompress.h"



using testing::AssertionFailure;

using testing::AssertionResult;
@@ -44,23 +45,10 @@ class CowTest : public ::testing::Test { 
    std::unique_ptr<TemporaryFile> cow_;

};



// Sink that always appends to the end of a string.

class StringSink : public IByteSink {

  public:

    void* GetBuffer(size_t requested, size_t* actual) override {

        size_t old_size = stream_.size();

        stream_.resize(old_size + requested, '\0');

        *actual = requested;

        return stream_.data() + old_size;

// Helper to check read sizes.

static inline bool ReadData(CowReader& reader, const CowOperation& op, void* buffer, size_t size) {

    return reader.ReadData(op, buffer, size) == size;

}

    bool ReturnData(void*, size_t) override { return true; }

    void Reset() { stream_.clear(); }



    std::string& stream() { return stream_; }



  private:

    std::string stream_;

};



TEST_F(CowTest, CopyContiguous) {

    CowOptions options;
@@ -144,7 +132,7 @@ TEST_F(CowTest, ReadWrite) { 
    ASSERT_EQ(op->new_block, 10);

    ASSERT_EQ(op->source, 20);



    StringSink sink;

    std::string sink(data.size(), '\0');



    iter->Next();

    ASSERT_FALSE(iter->Done());
@@ -154,8 +142,8 @@ TEST_F(CowTest, ReadWrite) { 
    ASSERT_EQ(op->compression, kCowCompressNone);

    ASSERT_EQ(op->data_length, 4096);

    ASSERT_EQ(op->new_block, 50);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data);



    iter->Next();

    ASSERT_FALSE(iter->Done());
@@ -222,7 +210,7 @@ TEST_F(CowTest, ReadWriteXor) { 
    ASSERT_EQ(op->new_block, 10);

    ASSERT_EQ(op->source, 20);



    StringSink sink;

    std::string sink(data.size(), '\0');



    iter->Next();

    ASSERT_FALSE(iter->Done());
@@ -233,8 +221,8 @@ TEST_F(CowTest, ReadWriteXor) { 
    ASSERT_EQ(op->data_length, 4096);

    ASSERT_EQ(op->new_block, 50);

    ASSERT_EQ(op->source, 98314);  // 4096 * 24 + 10

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data);



    iter->Next();

    ASSERT_FALSE(iter->Done());
@@ -285,22 +273,22 @@ TEST_F(CowTest, CompressGz) { 
    ASSERT_FALSE(iter->Done());

    auto op = &iter->Get();



    StringSink sink;

    std::string sink(data.size(), '\0');



    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_EQ(op->compression, kCowCompressGz);

    ASSERT_EQ(op->data_length, 56);  // compressed!

    ASSERT_EQ(op->new_block, 50);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data);



    iter->Next();

    ASSERT_TRUE(iter->Done());

}



class CompressionRWTest : public CowTest, public testing::WithParamInterface<const char*> {};

class CompressionTest : public CowTest, public testing::WithParamInterface<const char*> {};



TEST_P(CompressionRWTest, ThreadedBatchWrites) {

TEST_P(CompressionTest, ThreadedBatchWrites) {

    CowOptions options;

    options.compression = GetParam();

    options.num_compress_threads = 2;
@@ -342,31 +330,32 @@ TEST_P(CompressionRWTest, ThreadedBatchWrites) { 


        if (op->type == kCowXorOp) {

            total_blocks += 1;

            StringSink sink;

            std::string sink(xor_data.size(), '\0');

            ASSERT_EQ(op->new_block, 50);

            ASSERT_EQ(op->source, 98314);  // 4096 * 24 + 10

            ASSERT_TRUE(reader.ReadData(*op, &sink));

            ASSERT_EQ(sink.stream(), xor_data);

            ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

            ASSERT_EQ(sink, xor_data);

        }



        if (op->type == kCowReplaceOp) {

            total_blocks += 1;

            if (op->new_block == 100) {

                StringSink sink;

                ASSERT_TRUE(reader.ReadData(*op, &sink));

                data.resize(options.block_size);

                ASSERT_EQ(sink.stream(), data);

                std::string sink(data.size(), '\0');

                ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

                ASSERT_EQ(sink.size(), data.size());

                ASSERT_EQ(sink, data);

            }

            if (op->new_block == 6000) {

                StringSink sink;

                ASSERT_TRUE(reader.ReadData(*op, &sink));

                data2.resize(options.block_size);

                ASSERT_EQ(sink.stream(), data2);

                std::string sink(data2.size(), '\0');

                ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

                ASSERT_EQ(sink, data2);

            }

            if (op->new_block == 9000) {

                StringSink sink;

                ASSERT_TRUE(reader.ReadData(*op, &sink));

                ASSERT_EQ(sink.stream(), data3);

                std::string sink(data3.size(), '\0');

                ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

                ASSERT_EQ(sink, data3);

            }

        }
@@ -376,7 +365,7 @@ TEST_P(CompressionRWTest, ThreadedBatchWrites) { 
    ASSERT_EQ(total_blocks, expected_blocks);

}



TEST_P(CompressionRWTest, NoBatchWrites) {

TEST_P(CompressionTest, NoBatchWrites) {

    CowOptions options;

    options.compression = GetParam();

    options.num_compress_threads = 1;
@@ -416,21 +405,21 @@ TEST_P(CompressionRWTest, NoBatchWrites) { 
        if (op->type == kCowReplaceOp) {

            total_blocks += 1;

            if (op->new_block == 50) {

                StringSink sink;

                ASSERT_TRUE(reader.ReadData(*op, &sink));

                data.resize(options.block_size);

                ASSERT_EQ(sink.stream(), data);

                std::string sink(data.size(), '\0');

                ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

                ASSERT_EQ(sink, data);

            }

            if (op->new_block == 3000) {

                StringSink sink;

                ASSERT_TRUE(reader.ReadData(*op, &sink));

                data2.resize(options.block_size);

                ASSERT_EQ(sink.stream(), data2);

                std::string sink(data2.size(), '\0');

                ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

                ASSERT_EQ(sink, data2);

            }

            if (op->new_block == 5000) {

                StringSink sink;

                ASSERT_TRUE(reader.ReadData(*op, &sink));

                ASSERT_EQ(sink.stream(), data3);

                std::string sink(data3.size(), '\0');

                ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

                ASSERT_EQ(sink, data3);

            }

        }
@@ -440,7 +429,66 @@ TEST_P(CompressionRWTest, NoBatchWrites) { 
    ASSERT_EQ(total_blocks, expected_blocks);

}



INSTANTIATE_TEST_SUITE_P(CowApi, CompressionRWTest, testing::Values("none", "gz", "brotli", "lz4"));

template <typename T>

class HorribleStream : public IByteStream {

  public:

    HorribleStream(const std::basic_string<T>& input) : input_(input) {}



    ssize_t Read(void* buffer, size_t length) override {

        if (pos_ >= input_.size()) {

            return 0;

        }

        if (length) {

            *reinterpret_cast<char*>(buffer) = input_[pos_];

        }

        pos_++;

        return 1;

    }

    size_t Size() const override { return input_.size(); }



  private:

    std::basic_string<T> input_;

    size_t pos_ = 0;

};



TEST(HorribleStream, ReadFully) {

    std::string expected = "this is some data";



    HorribleStream<char> stream(expected);



    std::string buffer(expected.size(), '\0');

    ASSERT_TRUE(stream.ReadFully(buffer.data(), buffer.size()));

    ASSERT_EQ(buffer, expected);

}



TEST_P(CompressionTest, HorribleStream) {

    if (strcmp(GetParam(), "none") == 0) {

        GTEST_SKIP();

    }



    auto algorithm = CompressionAlgorithmFromString(GetParam());

    ASSERT_TRUE(algorithm.has_value());



    std::string expected = "The quick brown fox jumps over the lazy dog.";

    expected.resize(4096, '\0');



    auto result = CompressWorker::Compress(*algorithm, expected.data(), expected.size());

    ASSERT_FALSE(result.empty());



    HorribleStream<uint8_t> stream(result);

    auto decomp = IDecompressor::FromString(GetParam());

    ASSERT_NE(decomp, nullptr);

    decomp->set_stream(&stream);



    expected = expected.substr(10, 500);



    std::string buffer(expected.size(), '\0');

    ASSERT_EQ(decomp->Decompress(buffer.data(), 500, 4096, 10), 500);

    ASSERT_EQ(buffer, expected);

}



INSTANTIATE_TEST_SUITE_P(AllCompressors, CompressionTest,

                         testing::Values("none", "gz", "brotli", "lz4"));



TEST_F(CowTest, ClusterCompressGz) {

    CowOptions options;
@@ -470,14 +518,14 @@ TEST_F(CowTest, ClusterCompressGz) { 
    ASSERT_FALSE(iter->Done());

    auto op = &iter->Get();



    StringSink sink;

    std::string sink(data.size(), '\0');



    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_EQ(op->compression, kCowCompressGz);

    ASSERT_EQ(op->data_length, 56);  // compressed!

    ASSERT_EQ(op->new_block, 50);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data);



    iter->Next();

    ASSERT_FALSE(iter->Done());
@@ -489,12 +537,13 @@ TEST_F(CowTest, ClusterCompressGz) { 
    ASSERT_FALSE(iter->Done());

    op = &iter->Get();



    sink.Reset();

    sink = {};

    sink.resize(data2.size(), '\0');

    ASSERT_EQ(op->compression, kCowCompressGz);

    ASSERT_EQ(op->data_length, 41);  // compressed!

    ASSERT_EQ(op->new_block, 51);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data2);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data2);



    iter->Next();

    ASSERT_FALSE(iter->Done());
@@ -531,55 +580,15 @@ TEST_F(CowTest, CompressTwoBlocks) { 
    iter->Next();

    ASSERT_FALSE(iter->Done());



    StringSink sink;

    std::string sink(options.block_size, '\0');



    auto op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_EQ(op->compression, kCowCompressGz);

    ASSERT_EQ(op->new_block, 51);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

}



// Only return 1-byte buffers, to stress test the partial read logic in

// CowReader.

class HorribleStringSink : public StringSink {

  public:

    void* GetBuffer(size_t, size_t* actual) override { return StringSink::GetBuffer(1, actual); }

};



class CompressionTest : public CowTest, public testing::WithParamInterface<const char*> {};



TEST_P(CompressionTest, HorribleSink) {

    CowOptions options;

    options.compression = GetParam();

    options.cluster_ops = 0;

    CowWriter writer(options);



    ASSERT_TRUE(writer.Initialize(cow_->fd));



    std::string data = "This is some data, believe it";

    data.resize(options.block_size, '\0');



    ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer.Finalize());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);



    CowReader reader;

    ASSERT_TRUE(reader.Parse(cow_->fd));



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);

    ASSERT_FALSE(iter->Done());



    HorribleStringSink sink;

    auto op = &iter->Get();

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);

}



INSTANTIATE_TEST_SUITE_P(CowApi, CompressionTest, testing::Values("none", "gz", "brotli"));



TEST_F(CowTest, GetSize) {

    CowOptions options;

    options.cluster_ops = 0;
@@ -641,7 +650,7 @@ TEST_F(CowTest, AppendLabelSmall) { 
    ASSERT_TRUE(reader.GetLastLabel(&label));

    ASSERT_EQ(label, 3);



    StringSink sink;

    std::string sink(data.size(), '\0');



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);
@@ -649,11 +658,12 @@ TEST_F(CowTest, AppendLabelSmall) { 
    ASSERT_FALSE(iter->Done());

    auto op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data);



    iter->Next();

    sink.Reset();

    sink = {};

    sink.resize(data2.size(), '\0');



    ASSERT_FALSE(iter->Done());

    op = &iter->Get();
@@ -665,8 +675,8 @@ TEST_F(CowTest, AppendLabelSmall) { 
    ASSERT_FALSE(iter->Done());

    op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data2);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data2);



    iter->Next();

    ASSERT_TRUE(iter->Done());
@@ -705,8 +715,6 @@ TEST_F(CowTest, AppendLabelMissing) { 
    CowReader reader;

    ASSERT_TRUE(reader.Parse(cow_->fd));



    StringSink sink;



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);
@@ -765,8 +773,6 @@ TEST_F(CowTest, AppendExtendedCorrupted) { 
    CowReader reader;

    ASSERT_TRUE(reader.Parse(cow_->fd));



    StringSink sink;



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);
@@ -816,7 +822,7 @@ TEST_F(CowTest, AppendbyLabel) { 
    CowReader reader;

    ASSERT_TRUE(reader.Parse(cow_->fd));



    StringSink sink;

    std::string sink(options.block_size, '\0');



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);
@@ -824,20 +830,20 @@ TEST_F(CowTest, AppendbyLabel) { 
    ASSERT_FALSE(iter->Done());

    auto op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data.substr(0, options.block_size));

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data.substr(0, options.block_size));



    iter->Next();

    sink.Reset();

    sink = {};

    sink.resize(options.block_size, '\0');



    ASSERT_FALSE(iter->Done());

    op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data.substr(options.block_size, 2 * options.block_size));

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data.substr(options.block_size, 2 * options.block_size));



    iter->Next();

    sink.Reset();



    ASSERT_FALSE(iter->Done());

    op = &iter->Get();
@@ -897,7 +903,7 @@ TEST_F(CowTest, ClusterTest) { 
    CowReader reader;

    ASSERT_TRUE(reader.Parse(cow_->fd));



    StringSink sink;

    std::string sink(data.size(), '\0');



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);
@@ -905,11 +911,10 @@ TEST_F(CowTest, ClusterTest) { 
    ASSERT_FALSE(iter->Done());

    auto op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data.substr(0, options.block_size));

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data.substr(0, options.block_size));



    iter->Next();

    sink.Reset();



    ASSERT_FALSE(iter->Done());

    op = &iter->Get();
@@ -997,7 +1002,7 @@ TEST_F(CowTest, ClusterAppendTest) { 
    ASSERT_TRUE(reader.GetLastLabel(&label));

    ASSERT_EQ(label, 50);



    StringSink sink;

    std::string sink(data2.size(), '\0');



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);
@@ -1012,8 +1017,8 @@ TEST_F(CowTest, ClusterAppendTest) { 
    ASSERT_FALSE(iter->Done());

    op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data2);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data2);



    iter->Next();
@@ -1066,13 +1071,13 @@ AssertionResult CompareDataBlock(CowReader* reader, const CowOperation& op, 
    std::string cmp = data;

    cmp.resize(header.block_size, '\0');



    StringSink sink;

    if (!reader->ReadData(op, &sink)) {

    std::string sink(cmp.size(), '\0');

    if (!reader->ReadData(op, sink.data(), sink.size())) {

        return AssertionFailure() << "Failed to read data block";

    }

    if (cmp != sink.stream()) {

    if (cmp != sink) {

        return AssertionFailure() << "Data blocks did not match, expected " << cmp << ", got "

                                  << sink.stream();

                                  << sink;

    }



    return AssertionSuccess();

fs_mgr/libsnapshot/libsnapshot_cow/cow_compress.cpp

+15 −0
Original line number Diff line number Diff line
@@ -32,6 +32,21 @@ 


namespace android {

namespace snapshot {



std::optional<CowCompressionAlgorithm> CompressionAlgorithmFromString(std::string_view name) {

    if (name == "gz") {

        return {kCowCompressGz};

    } else if (name == "brotli") {

        return {kCowCompressBrotli};

    } else if (name == "lz4") {

        return {kCowCompressLz4};

    } else if (name == "none" || name.empty()) {

        return {kCowCompressNone};

    } else {

        return {};

    }

}



std::basic_string<uint8_t> CompressWorker::Compress(const void* data, size_t length) {

    return Compress(compression_, data, length);

}

fs_mgr/libsnapshot/libsnapshot_cow/cow_decompress.cpp

+254 −15

File changed.

Preview size limit exceeded, changes collapsed.