Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9c6aa9ac authored by David Anderson's avatar David Anderson Committed by Automerger Merge Worker
Browse files

Merge changes I69f41b23,Ib740de5e am: cd21e051 

Original change: https://android-review.googlesource.com/c/platform/system/core/+/2545672



Change-Id: I6f7bddb7ed5b89e4586efa921afc147dfb0cda7e
Signed-off-by: default avatarAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
parents a6d6714d cd21e051

fs_mgr/libsnapshot/include/libsnapshot/cow_format.h

+6 −1
Original line number Diff line number Diff line
@@ -15,7 +15,9 @@ 
#pragma once



#include <stdint.h>

#include <string>



#include <optional>

#include <string_view>



namespace android {

namespace snapshot {
@@ -196,5 +198,8 @@ bool IsMetadataOp(const CowOperation& op); 
// Ops that have dependencies on old blocks, and must take care in their merge order

bool IsOrderedOp(const CowOperation& op);



// Convert compression name to internal value.

std::optional<CowCompressionAlgorithm> CompressionAlgorithmFromString(std::string_view name);



}  // namespace snapshot

}  // namespace android

fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h

+18 −0
Original line number Diff line number Diff line
@@ -65,6 +65,7 @@ class ICowReader { 


    // Return the file header.

    virtual bool GetHeader(CowHeader* header) = 0;

    virtual CowHeader& GetHeader() = 0;



    // Return the file footer.

    virtual bool GetFooter(CowFooter* footer) = 0;
@@ -85,6 +86,19 @@ class ICowReader { 
    // Get decoded bytes from the data section, handling any decompression.

    // All retrieved data is passed to the sink.

    virtual bool ReadData(const CowOperation& op, IByteSink* sink) = 0;



    // Get decoded bytes from the data section, handling any decompression.

    //

    // If ignore_bytes is non-zero, it specifies the initial number of bytes

    // to skip writing to |buffer|.

    //

    // Returns the number of bytes written to |buffer|, or -1 on failure.

    // errno is NOT set.

    //

    // Partial reads are not possible unless |buffer_size| is less than the

    // operation block size.

    virtual ssize_t ReadData(const CowOperation& op, void* buffer, size_t buffer_size,

                             size_t ignore_bytes = 0) = 0;

};



// Iterate over a sequence of COW operations.
@@ -140,6 +154,10 @@ class CowReader final : public ICowReader { 
    std::unique_ptr<ICowOpIter> GetMergeOpIter(bool ignore_progress = false) override;



    bool ReadData(const CowOperation& op, IByteSink* sink) override;

    ssize_t ReadData(const CowOperation& op, void* buffer, size_t buffer_size,

                     size_t ignore_bytes = 0) override;



    CowHeader& GetHeader() override { return header_; }



    bool GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read);

fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp

+132 −127
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 
#include <gtest/gtest.h>

#include <libsnapshot/cow_reader.h>

#include <libsnapshot/cow_writer.h>

#include "cow_decompress.h"



using testing::AssertionFailure;

using testing::AssertionResult;
@@ -44,23 +45,10 @@ class CowTest : public ::testing::Test { 
    std::unique_ptr<TemporaryFile> cow_;

};



// Sink that always appends to the end of a string.

class StringSink : public IByteSink {

  public:

    void* GetBuffer(size_t requested, size_t* actual) override {

        size_t old_size = stream_.size();

        stream_.resize(old_size + requested, '\0');

        *actual = requested;

        return stream_.data() + old_size;

// Helper to check read sizes.

static inline bool ReadData(CowReader& reader, const CowOperation& op, void* buffer, size_t size) {

    return reader.ReadData(op, buffer, size) == size;

}

    bool ReturnData(void*, size_t) override { return true; }

    void Reset() { stream_.clear(); }



    std::string& stream() { return stream_; }



  private:

    std::string stream_;

};



TEST_F(CowTest, CopyContiguous) {

    CowOptions options;
@@ -144,7 +132,7 @@ TEST_F(CowTest, ReadWrite) { 
    ASSERT_EQ(op->new_block, 10);

    ASSERT_EQ(op->source, 20);



    StringSink sink;

    std::string sink(data.size(), '\0');



    iter->Next();

    ASSERT_FALSE(iter->Done());
@@ -154,8 +142,8 @@ TEST_F(CowTest, ReadWrite) { 
    ASSERT_EQ(op->compression, kCowCompressNone);

    ASSERT_EQ(op->data_length, 4096);

    ASSERT_EQ(op->new_block, 50);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data);



    iter->Next();

    ASSERT_FALSE(iter->Done());
@@ -222,7 +210,7 @@ TEST_F(CowTest, ReadWriteXor) { 
    ASSERT_EQ(op->new_block, 10);

    ASSERT_EQ(op->source, 20);



    StringSink sink;

    std::string sink(data.size(), '\0');



    iter->Next();

    ASSERT_FALSE(iter->Done());
@@ -233,8 +221,8 @@ TEST_F(CowTest, ReadWriteXor) { 
    ASSERT_EQ(op->data_length, 4096);

    ASSERT_EQ(op->new_block, 50);

    ASSERT_EQ(op->source, 98314);  // 4096 * 24 + 10

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data);



    iter->Next();

    ASSERT_FALSE(iter->Done());
@@ -285,22 +273,22 @@ TEST_F(CowTest, CompressGz) { 
    ASSERT_FALSE(iter->Done());

    auto op = &iter->Get();



    StringSink sink;

    std::string sink(data.size(), '\0');



    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_EQ(op->compression, kCowCompressGz);

    ASSERT_EQ(op->data_length, 56);  // compressed!

    ASSERT_EQ(op->new_block, 50);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data);



    iter->Next();

    ASSERT_TRUE(iter->Done());

}



class CompressionRWTest : public CowTest, public testing::WithParamInterface<const char*> {};

class CompressionTest : public CowTest, public testing::WithParamInterface<const char*> {};



TEST_P(CompressionRWTest, ThreadedBatchWrites) {

TEST_P(CompressionTest, ThreadedBatchWrites) {

    CowOptions options;

    options.compression = GetParam();

    options.num_compress_threads = 2;
@@ -342,31 +330,32 @@ TEST_P(CompressionRWTest, ThreadedBatchWrites) { 


        if (op->type == kCowXorOp) {

            total_blocks += 1;

            StringSink sink;

            std::string sink(xor_data.size(), '\0');

            ASSERT_EQ(op->new_block, 50);

            ASSERT_EQ(op->source, 98314);  // 4096 * 24 + 10

            ASSERT_TRUE(reader.ReadData(*op, &sink));

            ASSERT_EQ(sink.stream(), xor_data);

            ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

            ASSERT_EQ(sink, xor_data);

        }



        if (op->type == kCowReplaceOp) {

            total_blocks += 1;

            if (op->new_block == 100) {

                StringSink sink;

                ASSERT_TRUE(reader.ReadData(*op, &sink));

                data.resize(options.block_size);

                ASSERT_EQ(sink.stream(), data);

                std::string sink(data.size(), '\0');

                ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

                ASSERT_EQ(sink.size(), data.size());

                ASSERT_EQ(sink, data);

            }

            if (op->new_block == 6000) {

                StringSink sink;

                ASSERT_TRUE(reader.ReadData(*op, &sink));

                data2.resize(options.block_size);

                ASSERT_EQ(sink.stream(), data2);

                std::string sink(data2.size(), '\0');

                ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

                ASSERT_EQ(sink, data2);

            }

            if (op->new_block == 9000) {

                StringSink sink;

                ASSERT_TRUE(reader.ReadData(*op, &sink));

                ASSERT_EQ(sink.stream(), data3);

                std::string sink(data3.size(), '\0');

                ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

                ASSERT_EQ(sink, data3);

            }

        }
@@ -376,7 +365,7 @@ TEST_P(CompressionRWTest, ThreadedBatchWrites) { 
    ASSERT_EQ(total_blocks, expected_blocks);

}



TEST_P(CompressionRWTest, NoBatchWrites) {

TEST_P(CompressionTest, NoBatchWrites) {

    CowOptions options;

    options.compression = GetParam();

    options.num_compress_threads = 1;
@@ -416,21 +405,21 @@ TEST_P(CompressionRWTest, NoBatchWrites) { 
        if (op->type == kCowReplaceOp) {

            total_blocks += 1;

            if (op->new_block == 50) {

                StringSink sink;

                ASSERT_TRUE(reader.ReadData(*op, &sink));

                data.resize(options.block_size);

                ASSERT_EQ(sink.stream(), data);

                std::string sink(data.size(), '\0');

                ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

                ASSERT_EQ(sink, data);

            }

            if (op->new_block == 3000) {

                StringSink sink;

                ASSERT_TRUE(reader.ReadData(*op, &sink));

                data2.resize(options.block_size);

                ASSERT_EQ(sink.stream(), data2);

                std::string sink(data2.size(), '\0');

                ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

                ASSERT_EQ(sink, data2);

            }

            if (op->new_block == 5000) {

                StringSink sink;

                ASSERT_TRUE(reader.ReadData(*op, &sink));

                ASSERT_EQ(sink.stream(), data3);

                std::string sink(data3.size(), '\0');

                ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

                ASSERT_EQ(sink, data3);

            }

        }
@@ -440,7 +429,66 @@ TEST_P(CompressionRWTest, NoBatchWrites) { 
    ASSERT_EQ(total_blocks, expected_blocks);

}



INSTANTIATE_TEST_SUITE_P(CowApi, CompressionRWTest, testing::Values("none", "gz", "brotli", "lz4"));

template <typename T>

class HorribleStream : public IByteStream {

  public:

    HorribleStream(const std::basic_string<T>& input) : input_(input) {}



    ssize_t Read(void* buffer, size_t length) override {

        if (pos_ >= input_.size()) {

            return 0;

        }

        if (length) {

            *reinterpret_cast<char*>(buffer) = input_[pos_];

        }

        pos_++;

        return 1;

    }

    size_t Size() const override { return input_.size(); }



  private:

    std::basic_string<T> input_;

    size_t pos_ = 0;

};



TEST(HorribleStream, ReadFully) {

    std::string expected = "this is some data";



    HorribleStream<char> stream(expected);



    std::string buffer(expected.size(), '\0');

    ASSERT_TRUE(stream.ReadFully(buffer.data(), buffer.size()));

    ASSERT_EQ(buffer, expected);

}



TEST_P(CompressionTest, HorribleStream) {

    if (strcmp(GetParam(), "none") == 0) {

        GTEST_SKIP();

    }



    auto algorithm = CompressionAlgorithmFromString(GetParam());

    ASSERT_TRUE(algorithm.has_value());



    std::string expected = "The quick brown fox jumps over the lazy dog.";

    expected.resize(4096, '\0');



    auto result = CompressWorker::Compress(*algorithm, expected.data(), expected.size());

    ASSERT_FALSE(result.empty());



    HorribleStream<uint8_t> stream(result);

    auto decomp = IDecompressor::FromString(GetParam());

    ASSERT_NE(decomp, nullptr);

    decomp->set_stream(&stream);



    expected = expected.substr(10, 500);



    std::string buffer(expected.size(), '\0');

    ASSERT_EQ(decomp->Decompress(buffer.data(), 500, 4096, 10), 500);

    ASSERT_EQ(buffer, expected);

}



INSTANTIATE_TEST_SUITE_P(AllCompressors, CompressionTest,

                         testing::Values("none", "gz", "brotli", "lz4"));



TEST_F(CowTest, ClusterCompressGz) {

    CowOptions options;
@@ -470,14 +518,14 @@ TEST_F(CowTest, ClusterCompressGz) { 
    ASSERT_FALSE(iter->Done());

    auto op = &iter->Get();



    StringSink sink;

    std::string sink(data.size(), '\0');



    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_EQ(op->compression, kCowCompressGz);

    ASSERT_EQ(op->data_length, 56);  // compressed!

    ASSERT_EQ(op->new_block, 50);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data);



    iter->Next();

    ASSERT_FALSE(iter->Done());
@@ -489,12 +537,13 @@ TEST_F(CowTest, ClusterCompressGz) { 
    ASSERT_FALSE(iter->Done());

    op = &iter->Get();



    sink.Reset();

    sink = {};

    sink.resize(data2.size(), '\0');

    ASSERT_EQ(op->compression, kCowCompressGz);

    ASSERT_EQ(op->data_length, 41);  // compressed!

    ASSERT_EQ(op->new_block, 51);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data2);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data2);



    iter->Next();

    ASSERT_FALSE(iter->Done());
@@ -531,55 +580,15 @@ TEST_F(CowTest, CompressTwoBlocks) { 
    iter->Next();

    ASSERT_FALSE(iter->Done());



    StringSink sink;

    std::string sink(options.block_size, '\0');



    auto op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_EQ(op->compression, kCowCompressGz);

    ASSERT_EQ(op->new_block, 51);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

}



// Only return 1-byte buffers, to stress test the partial read logic in

// CowReader.

class HorribleStringSink : public StringSink {

  public:

    void* GetBuffer(size_t, size_t* actual) override { return StringSink::GetBuffer(1, actual); }

};



class CompressionTest : public CowTest, public testing::WithParamInterface<const char*> {};



TEST_P(CompressionTest, HorribleSink) {

    CowOptions options;

    options.compression = GetParam();

    options.cluster_ops = 0;

    CowWriter writer(options);



    ASSERT_TRUE(writer.Initialize(cow_->fd));



    std::string data = "This is some data, believe it";

    data.resize(options.block_size, '\0');



    ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer.Finalize());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);



    CowReader reader;

    ASSERT_TRUE(reader.Parse(cow_->fd));



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);

    ASSERT_FALSE(iter->Done());



    HorribleStringSink sink;

    auto op = &iter->Get();

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);

}



INSTANTIATE_TEST_SUITE_P(CowApi, CompressionTest, testing::Values("none", "gz", "brotli"));



TEST_F(CowTest, GetSize) {

    CowOptions options;

    options.cluster_ops = 0;
@@ -641,7 +650,7 @@ TEST_F(CowTest, AppendLabelSmall) { 
    ASSERT_TRUE(reader.GetLastLabel(&label));

    ASSERT_EQ(label, 3);



    StringSink sink;

    std::string sink(data.size(), '\0');



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);
@@ -649,11 +658,12 @@ TEST_F(CowTest, AppendLabelSmall) { 
    ASSERT_FALSE(iter->Done());

    auto op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data);



    iter->Next();

    sink.Reset();

    sink = {};

    sink.resize(data2.size(), '\0');



    ASSERT_FALSE(iter->Done());

    op = &iter->Get();
@@ -665,8 +675,8 @@ TEST_F(CowTest, AppendLabelSmall) { 
    ASSERT_FALSE(iter->Done());

    op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data2);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data2);



    iter->Next();

    ASSERT_TRUE(iter->Done());
@@ -705,8 +715,6 @@ TEST_F(CowTest, AppendLabelMissing) { 
    CowReader reader;

    ASSERT_TRUE(reader.Parse(cow_->fd));



    StringSink sink;



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);
@@ -765,8 +773,6 @@ TEST_F(CowTest, AppendExtendedCorrupted) { 
    CowReader reader;

    ASSERT_TRUE(reader.Parse(cow_->fd));



    StringSink sink;



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);
@@ -816,7 +822,7 @@ TEST_F(CowTest, AppendbyLabel) { 
    CowReader reader;

    ASSERT_TRUE(reader.Parse(cow_->fd));



    StringSink sink;

    std::string sink(options.block_size, '\0');



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);
@@ -824,20 +830,20 @@ TEST_F(CowTest, AppendbyLabel) { 
    ASSERT_FALSE(iter->Done());

    auto op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data.substr(0, options.block_size));

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data.substr(0, options.block_size));



    iter->Next();

    sink.Reset();

    sink = {};

    sink.resize(options.block_size, '\0');



    ASSERT_FALSE(iter->Done());

    op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data.substr(options.block_size, 2 * options.block_size));

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data.substr(options.block_size, 2 * options.block_size));



    iter->Next();

    sink.Reset();



    ASSERT_FALSE(iter->Done());

    op = &iter->Get();
@@ -897,7 +903,7 @@ TEST_F(CowTest, ClusterTest) { 
    CowReader reader;

    ASSERT_TRUE(reader.Parse(cow_->fd));



    StringSink sink;

    std::string sink(data.size(), '\0');



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);
@@ -905,11 +911,10 @@ TEST_F(CowTest, ClusterTest) { 
    ASSERT_FALSE(iter->Done());

    auto op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data.substr(0, options.block_size));

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data.substr(0, options.block_size));



    iter->Next();

    sink.Reset();



    ASSERT_FALSE(iter->Done());

    op = &iter->Get();
@@ -997,7 +1002,7 @@ TEST_F(CowTest, ClusterAppendTest) { 
    ASSERT_TRUE(reader.GetLastLabel(&label));

    ASSERT_EQ(label, 50);



    StringSink sink;

    std::string sink(data2.size(), '\0');



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);
@@ -1012,8 +1017,8 @@ TEST_F(CowTest, ClusterAppendTest) { 
    ASSERT_FALSE(iter->Done());

    op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data2);

    ASSERT_TRUE(ReadData(reader, *op, sink.data(), sink.size()));

    ASSERT_EQ(sink, data2);



    iter->Next();
@@ -1066,13 +1071,13 @@ AssertionResult CompareDataBlock(CowReader* reader, const CowOperation& op, 
    std::string cmp = data;

    cmp.resize(header.block_size, '\0');



    StringSink sink;

    if (!reader->ReadData(op, &sink)) {

    std::string sink(cmp.size(), '\0');

    if (!reader->ReadData(op, sink.data(), sink.size())) {

        return AssertionFailure() << "Failed to read data block";

    }

    if (cmp != sink.stream()) {

    if (cmp != sink) {

        return AssertionFailure() << "Data blocks did not match, expected " << cmp << ", got "

                                  << sink.stream();

                                  << sink;

    }



    return AssertionSuccess();

fs_mgr/libsnapshot/libsnapshot_cow/cow_compress.cpp

+15 −0
Original line number Diff line number Diff line
@@ -32,6 +32,21 @@ 


namespace android {

namespace snapshot {



std::optional<CowCompressionAlgorithm> CompressionAlgorithmFromString(std::string_view name) {

    if (name == "gz") {

        return {kCowCompressGz};

    } else if (name == "brotli") {

        return {kCowCompressBrotli};

    } else if (name == "lz4") {

        return {kCowCompressLz4};

    } else if (name == "none" || name.empty()) {

        return {kCowCompressNone};

    } else {

        return {};

    }

}



std::basic_string<uint8_t> CompressWorker::Compress(const void* data, size_t length) {

    return Compress(compression_, data, length);

}

fs_mgr/libsnapshot/libsnapshot_cow/cow_decompress.cpp

+254 −15

File changed.

Preview size limit exceeded, changes collapsed.