Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 975ea321 authored by David Anderson's avatar David Anderson Committed by Gerrit Code Review
Browse files

Merge changes from topic "vab-brotli" 

* changes:
  libsnapshot: Add an append mode to CowWriter.
  libsnapshot: Add support for brotli compression.
parents 4fd78ec4 a889c87b

fs_mgr/libsnapshot/Android.bp

+8 −3
Original line number Diff line number Diff line
@@ -152,6 +152,7 @@ cc_library_static { 
        "liblog",

    ],

    static_libs: [

        "libbrotli",

        "libz",

    ],

    ramdisk_available: true,
@@ -362,6 +363,7 @@ cc_defaults { 


    static_libs: [

        "libbase",

        "libbrotli",

        "liblog",

        "libdm",

        "libz",
@@ -403,12 +405,14 @@ cc_test { 
        "libz",

    ],

    static_libs: [

        "libbrotli",

        "libgtest",

        "libsnapshot_cow",

    ],

    test_min_api_level: 30,

    auto_gen_config: true,

    require_root: false,

    host_supported: true,

}



cc_binary {
@@ -494,11 +498,12 @@ cc_test { 
    shared_libs: [

        "libbase",

        "liblog",

        "libz",

    ],

    static_libs: [

        "libbrotli",

        "libgtest",

        "libsnapshot_cow",

        "libz",

    ],

    header_libs: [

        "libstorage_literals_headers",

fs_mgr/libsnapshot/cow_api_test.cpp

+69 −11
Original line number Diff line number Diff line
@@ -30,12 +30,12 @@ namespace snapshot { 


class CowTest : public ::testing::Test {

  protected:

    void SetUp() override {

    virtual void SetUp() override {

        cow_ = std::make_unique<TemporaryFile>();

        ASSERT_GE(cow_->fd, 0) << strerror(errno);

    }



    void TearDown() override { cow_ = nullptr; }

    virtual void TearDown() override { cow_ = nullptr; }



    std::unique_ptr<TemporaryFile> cow_;

};
@@ -70,7 +70,7 @@ TEST_F(CowTest, ReadWrite) { 
    ASSERT_TRUE(writer.AddCopy(10, 20));

    ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer.AddZeroBlocks(51, 2));

    ASSERT_TRUE(writer.Finalize());

    ASSERT_TRUE(writer.Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@@ -105,7 +105,7 @@ TEST_F(CowTest, ReadWrite) { 
    ASSERT_EQ(op->compression, kCowCompressNone);

    ASSERT_EQ(op->data_length, 4096);

    ASSERT_EQ(op->new_block, 50);

    ASSERT_EQ(op->source, 104);

    ASSERT_EQ(op->source, 106);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);
@@ -145,7 +145,7 @@ TEST_F(CowTest, CompressGz) { 
    data.resize(options.block_size, '\0');



    ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer.Finalize());

    ASSERT_TRUE(writer.Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@@ -163,7 +163,7 @@ TEST_F(CowTest, CompressGz) { 
    ASSERT_EQ(op->compression, kCowCompressGz);

    ASSERT_EQ(op->data_length, 56);  // compressed!

    ASSERT_EQ(op->new_block, 50);

    ASSERT_EQ(op->source, 104);

    ASSERT_EQ(op->source, 106);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);
@@ -182,7 +182,7 @@ TEST_F(CowTest, CompressTwoBlocks) { 
    data.resize(options.block_size * 2, '\0');



    ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer.Finalize());

    ASSERT_TRUE(writer.Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@@ -211,9 +211,11 @@ class HorribleStringSink : public StringSink { 
    void* GetBuffer(size_t, size_t* actual) override { return StringSink::GetBuffer(1, actual); }

};



TEST_F(CowTest, HorribleSink) {

class CompressionTest : public CowTest, public testing::WithParamInterface<const char*> {};



TEST_P(CompressionTest, HorribleSink) {

    CowOptions options;

    options.compression = "gz";

    options.compression = GetParam();

    CowWriter writer(options);



    ASSERT_TRUE(writer.Initialize(cow_->fd));
@@ -222,7 +224,7 @@ TEST_F(CowTest, HorribleSink) { 
    data.resize(options.block_size, '\0');



    ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer.Finalize());

    ASSERT_TRUE(writer.Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);
@@ -239,6 +241,8 @@ TEST_F(CowTest, HorribleSink) { 
    ASSERT_EQ(sink.stream(), data);

}



INSTANTIATE_TEST_SUITE_P(CowApi, CompressionTest, testing::Values("none", "gz", "brotli"));



TEST_F(CowTest, GetSize) {

    CowOptions options;

    CowWriter writer(options);
@@ -255,7 +259,7 @@ TEST_F(CowTest, GetSize) { 
    ASSERT_TRUE(writer.AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer.AddZeroBlocks(51, 2));

    auto size_before = writer.GetCowSize();

    ASSERT_TRUE(writer.Finalize());

    ASSERT_TRUE(writer.Flush());

    auto size_after = writer.GetCowSize();

    ASSERT_EQ(size_before, size_after);

    struct stat buf;
@@ -267,6 +271,60 @@ TEST_F(CowTest, GetSize) { 
    ASSERT_EQ(buf.st_size, writer.GetCowSize());

}



TEST_F(CowTest, Append) {

    CowOptions options;

    auto writer = std::make_unique<CowWriter>(options);

    ASSERT_TRUE(writer->Initialize(cow_->fd));



    std::string data = "This is some data, believe it";

    data.resize(options.block_size, '\0');

    ASSERT_TRUE(writer->AddRawBlocks(50, data.data(), data.size()));

    ASSERT_TRUE(writer->Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);



    writer = std::make_unique<CowWriter>(options);

    ASSERT_TRUE(writer->Initialize(cow_->fd, CowWriter::OpenMode::APPEND));



    std::string data2 = "More data!";

    data2.resize(options.block_size, '\0');

    ASSERT_TRUE(writer->AddRawBlocks(51, data2.data(), data2.size()));

    ASSERT_TRUE(writer->Flush());



    ASSERT_EQ(lseek(cow_->fd, 0, SEEK_SET), 0);



    struct stat buf;

    ASSERT_EQ(fstat(cow_->fd, &buf), 0);

    ASSERT_EQ(buf.st_size, writer->GetCowSize());



    // Read back both operations.

    CowReader reader;

    ASSERT_TRUE(reader.Parse(cow_->fd));



    StringSink sink;



    auto iter = reader.GetOpIter();

    ASSERT_NE(iter, nullptr);



    ASSERT_FALSE(iter->Done());

    auto op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data);



    iter->Next();

    sink.Reset();



    ASSERT_FALSE(iter->Done());

    op = &iter->Get();

    ASSERT_EQ(op->type, kCowReplaceOp);

    ASSERT_TRUE(reader.ReadData(*op, &sink));

    ASSERT_EQ(sink.stream(), data2);



    iter->Next();

    ASSERT_TRUE(iter->Done());

}



}  // namespace snapshot

}  // namespace android

fs_mgr/libsnapshot/cow_decompress.cpp

+53 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include <utility>



#include <android-base/logging.h>

#include <brotli/decode.h>

#include <zlib.h>



namespace android {
@@ -207,5 +208,57 @@ std::unique_ptr<IDecompressor> IDecompressor::Gz() { 
    return std::unique_ptr<IDecompressor>(new GzDecompressor());

}



class BrotliDecompressor final : public StreamDecompressor {

  public:

    ~BrotliDecompressor();



    bool Init() override;

    bool DecompressInput(const uint8_t* data, size_t length) override;

    bool Done() override { return BrotliDecoderIsFinished(decoder_); }



  private:

    BrotliDecoderState* decoder_ = nullptr;

};



bool BrotliDecompressor::Init() {

    decoder_ = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);

    return true;

}



BrotliDecompressor::~BrotliDecompressor() {

    if (decoder_) {

        BrotliDecoderDestroyInstance(decoder_);

    }

}



bool BrotliDecompressor::DecompressInput(const uint8_t* data, size_t length) {

    size_t available_in = length;

    const uint8_t* next_in = data;



    bool needs_more_output = false;

    while (available_in || needs_more_output) {

        if (!output_buffer_remaining_ && !GetFreshBuffer()) {

            return false;

        }



        auto output_buffer = output_buffer_;

        auto r = BrotliDecoderDecompressStream(decoder_, &available_in, &next_in,

                                               &output_buffer_remaining_, &output_buffer_, nullptr);

        if (r == BROTLI_DECODER_RESULT_ERROR) {

            LOG(ERROR) << "brotli decode failed";

            return false;

        }

        if (!sink_->ReturnData(output_buffer, output_buffer_ - output_buffer)) {

            return false;

        }

        needs_more_output = (r == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT);

    }

    return true;

}



std::unique_ptr<IDecompressor> IDecompressor::Brotli() {

    return std::unique_ptr<IDecompressor>(new BrotliDecompressor());

}



}  // namespace snapshot

}  // namespace android

fs_mgr/libsnapshot/cow_decompress.h

+1 −0
Original line number Diff line number Diff line
@@ -40,6 +40,7 @@ class IDecompressor { 
    // Factory methods for decompression methods.

    static std::unique_ptr<IDecompressor> Uncompressed();

    static std::unique_ptr<IDecompressor> Gz();

    static std::unique_ptr<IDecompressor> Brotli();



    // |output_bytes| is the expected total number of bytes to sink.

    virtual bool Decompress(size_t output_bytes) = 0;

fs_mgr/libsnapshot/cow_reader.cpp

+8 −0
Original line number Diff line number Diff line
@@ -78,6 +78,11 @@ bool CowReader::Parse(android::base::borrowed_fd fd) { 
                   << "Expected: " << kCowMagicNumber;

        return false;

    }

    if (header_.header_size != sizeof(CowHeader)) {

        LOG(ERROR) << "Header size unknown, read " << header_.header_size << ", expected "

                   << sizeof(CowHeader);

        return false;

    }



    if ((header_.major_version != kCowVersionMajor) ||

        (header_.minor_version != kCowVersionMinor)) {
@@ -233,6 +238,9 @@ bool CowReader::ReadData(const CowOperation& op, IByteSink* sink) { 
        case kCowCompressGz:

            decompressor = IDecompressor::Gz();

            break;

        case kCowCompressBrotli:

            decompressor = IDecompressor::Brotli();

            break;

        default:

            LOG(ERROR) << "Unknown compression type: " << op.compression;

            return false;