Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 82804e5d authored by David Anderson's avatar David Anderson Committed by Gerrit Code Review
Browse files

Merge changes Ibc0f9818,I29b5617e 

* changes:
  libsnapshot: Reject bad cow versions.
  libsnapshot: Split CowReader into CowParserV2.
parents 7681f71b 0ec71157

fs_mgr/libsnapshot/Android.bp

+1 −0
Original line number Diff line number Diff line
@@ -179,6 +179,7 @@ cc_library_static { 
        "libsnapshot_cow/cow_writer.cpp",

        "libsnapshot_cow/cow_format.cpp",

        "libsnapshot_cow/cow_compress.cpp",

        "libsnapshot_cow/parser_v2.cpp",

    ],

    host_supported: true,

    recovery_available: true,

fs_mgr/libsnapshot/include/libsnapshot/cow_format.h

+10 −13
Original line number Diff line number Diff line
@@ -28,6 +28,9 @@ static constexpr uint32_t kCowVersionMinor = 0; 


static constexpr uint32_t kCowVersionManifest = 2;



static constexpr uint32_t kMinCowVersion = 1;

static constexpr uint32_t kMaxCowVersion = 2;



// This header appears as the first sequence of bytes in the COW. All fields

// in the layout are little-endian encoded. The on-disk layout is:

//
@@ -52,13 +55,15 @@ static constexpr uint32_t kCowVersionManifest = 2; 
// between writing the last operation/data pair, or the footer itself. In this

// case, the safest way to proceed is to assume the last operation is faulty.



struct CowHeader {

struct CowHeaderPrefix {

    uint64_t magic;

    uint16_t major_version;

    uint16_t minor_version;

    uint16_t header_size;  // size of CowHeader.

} __attribute__((packed));



    // Size of this struct.

    uint16_t header_size;

struct CowHeader {

    CowHeaderPrefix prefix;



    // Size of footer struct

    uint16_t footer_size;
@@ -88,7 +93,7 @@ struct CowFooterOperation { 
    // the compression type of that data (see constants below).

    uint8_t compression;



    // Length of Footer Data. Currently 64 for both checksums

    // Length of Footer Data. Currently 64.

    uint16_t data_length;



    // The amount of file space used by Cow operations
@@ -98,14 +103,6 @@ struct CowFooterOperation { 
    uint64_t num_ops;

} __attribute__((packed));



struct CowFooterData {

    // SHA256 checksums of Footer op

    uint8_t footer_checksum[32];



    // SHA256 of the operation sequence.

    uint8_t ops_checksum[32];

} __attribute__((packed));



// Cow operations are currently fixed-size entries, but this may change if

// needed.

struct CowOperation {
@@ -167,7 +164,7 @@ static constexpr uint8_t kCowReadAheadDone = 2; 


struct CowFooter {

    CowFooterOperation op;

    CowFooterData data;

    uint8_t unused[64];

} __attribute__((packed));



struct ScratchMetadata {

fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h

+0 −2
Original line number Diff line number Diff line
@@ -171,8 +171,6 @@ class CowWriter : public ICowWriter { 


    uint64_t GetCowSize() override;



    uint32_t GetCowVersion() { return header_.major_version; }



  protected:

    virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;

    virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;

fs_mgr/libsnapshot/libsnapshot_cow/cow_api_test.cpp

+9 −9
Original line number Diff line number Diff line
@@ -65,9 +65,9 @@ TEST_F(CowTest, CopyContiguous) { 
    ASSERT_TRUE(reader.Parse(cow_->fd));



    const auto& header = reader.GetHeader();

    ASSERT_EQ(header.magic, kCowMagicNumber);

    ASSERT_EQ(header.major_version, kCowVersionMajor);

    ASSERT_EQ(header.minor_version, kCowVersionMinor);

    ASSERT_EQ(header.prefix.magic, kCowMagicNumber);

    ASSERT_EQ(header.prefix.major_version, kCowVersionMajor);

    ASSERT_EQ(header.prefix.minor_version, kCowVersionMinor);

    ASSERT_EQ(header.block_size, options.block_size);



    CowFooter footer;
@@ -114,9 +114,9 @@ TEST_F(CowTest, ReadWrite) { 
    ASSERT_TRUE(reader.Parse(cow_->fd));



    const auto& header = reader.GetHeader();

    ASSERT_EQ(header.magic, kCowMagicNumber);

    ASSERT_EQ(header.major_version, kCowVersionMajor);

    ASSERT_EQ(header.minor_version, kCowVersionMinor);

    ASSERT_EQ(header.prefix.magic, kCowMagicNumber);

    ASSERT_EQ(header.prefix.major_version, kCowVersionMajor);

    ASSERT_EQ(header.prefix.minor_version, kCowVersionMinor);

    ASSERT_EQ(header.block_size, options.block_size);



    CowFooter footer;
@@ -193,9 +193,9 @@ TEST_F(CowTest, ReadWriteXor) { 
    ASSERT_TRUE(reader.Parse(cow_->fd));



    const auto& header = reader.GetHeader();

    ASSERT_EQ(header.magic, kCowMagicNumber);

    ASSERT_EQ(header.major_version, kCowVersionMajor);

    ASSERT_EQ(header.minor_version, kCowVersionMinor);

    ASSERT_EQ(header.prefix.magic, kCowMagicNumber);

    ASSERT_EQ(header.prefix.major_version, kCowVersionMajor);

    ASSERT_EQ(header.prefix.minor_version, kCowVersionMinor);

    ASSERT_EQ(header.block_size, options.block_size);



    CowFooter footer;

fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp

+13 −213
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ 
#include <zlib.h>



#include "cow_decompress.h"

#include "libsnapshot/cow_format.h"

#include "parser_v2.h"



namespace android {

namespace snapshot {
@@ -43,15 +43,6 @@ CowReader::CowReader(ReaderFlags reader_flag, bool is_merge) 
      reader_flag_(reader_flag),

      is_merge_(is_merge) {}



static void SHA256(const void*, size_t, uint8_t[]) {

#if 0

    SHA256_CTX c;

    SHA256_Init(&c);

    SHA256_Update(&c, data, length);

    SHA256_Final(out, &c);

#endif

}



std::unique_ptr<CowReader> CowReader::CloneCowReader() {

    auto cow = std::make_unique<CowReader>();

    cow->owned_fd_.reset();
@@ -63,7 +54,6 @@ std::unique_ptr<CowReader> CowReader::CloneCowReader() { 
    cow->merge_op_start_ = merge_op_start_;

    cow->num_total_data_ops_ = num_total_data_ops_;

    cow->num_ordered_ops_to_merge_ = num_ordered_ops_to_merge_;

    cow->has_seq_ops_ = has_seq_ops_;

    cow->data_loc_ = data_loc_;

    cow->block_pos_index_ = block_pos_index_;

    cow->is_merge_ = is_merge_;
@@ -101,217 +91,26 @@ bool CowReader::Parse(android::base::unique_fd&& fd, std::optional<uint64_t> lab 
bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> label) {

    fd_ = fd;



    auto pos = lseek(fd_.get(), 0, SEEK_END);

    if (pos < 0) {

        PLOG(ERROR) << "lseek end failed";

    if (!ReadCowHeader(fd, &header_)) {

        return false;

    }

    fd_size_ = pos;



    if (lseek(fd_.get(), 0, SEEK_SET) < 0) {

        PLOG(ERROR) << "lseek header failed";

        return false;

    }

    if (!android::base::ReadFully(fd_, &header_, sizeof(header_))) {

        PLOG(ERROR) << "read header failed";

    CowParserV2 parser;

    if (!parser.Parse(fd, header_, label)) {

        return false;

    }



    if (header_.magic != kCowMagicNumber) {

        LOG(ERROR) << "Header Magic corrupted. Magic: " << header_.magic

                   << "Expected: " << kCowMagicNumber;

        return false;

    }

    if (header_.footer_size != sizeof(CowFooter)) {

        LOG(ERROR) << "Footer size unknown, read " << header_.footer_size << ", expected "

                   << sizeof(CowFooter);

        return false;

    }

    if (header_.op_size != sizeof(CowOperation)) {

        LOG(ERROR) << "Operation size unknown, read " << header_.op_size << ", expected "

                   << sizeof(CowOperation);

        return false;

    }

    if (header_.cluster_ops == 1) {

        LOG(ERROR) << "Clusters must contain at least two operations to function.";

        return false;

    }

    if (header_.op_size != sizeof(CowOperation)) {

        LOG(ERROR) << "Operation size unknown, read " << header_.op_size << ", expected "

                   << sizeof(CowOperation);

        return false;

    }

    if (header_.cluster_ops == 1) {

        LOG(ERROR) << "Clusters must contain at least two operations to function.";

        return false;

    }



    if ((header_.major_version > kCowVersionMajor) || (header_.minor_version != kCowVersionMinor)) {

        LOG(ERROR) << "Header version mismatch";

        LOG(ERROR) << "Major version: " << header_.major_version

                   << "Expected: " << kCowVersionMajor;

        LOG(ERROR) << "Minor version: " << header_.minor_version

                   << "Expected: " << kCowVersionMinor;

        return false;

    }

    footer_ = parser.footer();

    fd_size_ = parser.fd_size();

    last_label_ = parser.last_label();

    ops_ = std::move(parser.ops());

    data_loc_ = parser.data_loc();



    if (!ParseOps(label)) {

        return false;

    }

    // If we're resuming a write, we're not ready to merge

    if (label.has_value()) return true;

    return PrepMergeOps();

}



bool CowReader::ParseOps(std::optional<uint64_t> label) {

    uint64_t pos;

    auto data_loc = std::make_shared<std::unordered_map<uint64_t, uint64_t>>();



    // Skip the scratch space

    if (header_.major_version >= 2 && (header_.buffer_size > 0)) {

        LOG(DEBUG) << " Scratch space found of size: " << header_.buffer_size;

        size_t init_offset = header_.header_size + header_.buffer_size;

        pos = lseek(fd_.get(), init_offset, SEEK_SET);

        if (pos != init_offset) {

            PLOG(ERROR) << "lseek ops failed";

            return false;

        }

    } else {

        pos = lseek(fd_.get(), header_.header_size, SEEK_SET);

        if (pos != header_.header_size) {

            PLOG(ERROR) << "lseek ops failed";

            return false;

        }

        // Reading a v1 version of COW which doesn't have buffer_size.

        header_.buffer_size = 0;

    }

    uint64_t data_pos = 0;



    if (header_.cluster_ops) {

        data_pos = pos + header_.cluster_ops * sizeof(CowOperation);

    } else {

        data_pos = pos + sizeof(CowOperation);

    }



    auto ops_buffer = std::make_shared<std::vector<CowOperation>>();

    uint64_t current_op_num = 0;

    uint64_t cluster_ops = header_.cluster_ops ?: 1;

    bool done = false;



    // Alternating op clusters and data

    while (!done) {

        uint64_t to_add = std::min(cluster_ops, (fd_size_ - pos) / sizeof(CowOperation));

        if (to_add == 0) break;

        ops_buffer->resize(current_op_num + to_add);

        if (!android::base::ReadFully(fd_, &ops_buffer->data()[current_op_num],

                                      to_add * sizeof(CowOperation))) {

            PLOG(ERROR) << "read op failed";

            return false;

        }

        // Parse current cluster to find start of next cluster

        while (current_op_num < ops_buffer->size()) {

            auto& current_op = ops_buffer->data()[current_op_num];

            current_op_num++;

            if (current_op.type == kCowXorOp) {

                data_loc->insert({current_op.new_block, data_pos});

            }

            pos += sizeof(CowOperation) + GetNextOpOffset(current_op, header_.cluster_ops);

            data_pos += current_op.data_length + GetNextDataOffset(current_op, header_.cluster_ops);



            if (current_op.type == kCowClusterOp) {

                break;

            } else if (current_op.type == kCowLabelOp) {

                last_label_ = {current_op.source};



                // If we reach the requested label, stop reading.

                if (label && label.value() == current_op.source) {

                    done = true;

                    break;

                }

            } else if (current_op.type == kCowFooterOp) {

                footer_.emplace();

                CowFooter* footer = &footer_.value();

                memcpy(&footer_->op, &current_op, sizeof(footer->op));

                off_t offs = lseek(fd_.get(), pos, SEEK_SET);

                if (offs < 0 || pos != static_cast<uint64_t>(offs)) {

                    PLOG(ERROR) << "lseek next op failed " << offs;

                    return false;

                }

                if (!android::base::ReadFully(fd_, &footer->data, sizeof(footer->data))) {

                    LOG(ERROR) << "Could not read COW footer";

                    return false;

                }



                // Drop the footer from the op stream.

                current_op_num--;

                done = true;

                break;

            } else if (current_op.type == kCowSequenceOp) {

                has_seq_ops_ = true;

            }

        }



        // Position for next cluster read

        off_t offs = lseek(fd_.get(), pos, SEEK_SET);

        if (offs < 0 || pos != static_cast<uint64_t>(offs)) {

            PLOG(ERROR) << "lseek next op failed " << offs;

            return false;

        }

        ops_buffer->resize(current_op_num);

    }



    LOG(DEBUG) << "COW file read complete. Total ops: " << ops_buffer->size();

    // To successfully parse a COW file, we need either:

    //  (1) a label to read up to, and for that label to be found, or

    //  (2) a valid footer.

    if (label) {

        if (!last_label_) {

            LOG(ERROR) << "Did not find label " << label.value()

                       << " while reading COW (no labels found)";

            return false;

        }

        if (last_label_.value() != label.value()) {

            LOG(ERROR) << "Did not find label " << label.value()

                       << ", last label=" << last_label_.value();

            return false;

        }

    } else if (!footer_) {

        LOG(ERROR) << "No COW footer found";

        return false;

    }



    uint8_t csum[32];

    memset(csum, 0, sizeof(uint8_t) * 32);



    if (footer_) {

        if (ops_buffer->size() != footer_->op.num_ops) {

            LOG(ERROR) << "num ops does not match, expected " << footer_->op.num_ops << ", found "

                       << ops_buffer->size();

            return false;

        }

        if (ops_buffer->size() * sizeof(CowOperation) != footer_->op.ops_size) {

            LOG(ERROR) << "ops size does not match ";

            return false;

        }

        SHA256(&footer_->op, sizeof(footer_->op), footer_->data.footer_checksum);

        if (memcmp(csum, footer_->data.ops_checksum, sizeof(csum)) != 0) {

            LOG(ERROR) << "ops checksum does not match";

            return false;

        }

        SHA256(ops_buffer->data(), footer_->op.ops_size, csum);

        if (memcmp(csum, footer_->data.ops_checksum, sizeof(csum)) != 0) {

            LOG(ERROR) << "ops checksum does not match";

            return false;

        }

    }



    ops_ = ops_buffer;

    ops_->shrink_to_fit();

    data_loc_ = data_loc;



    return true;

}



//

// This sets up the data needed for MergeOpIter. MergeOpIter presents

// data in the order we intend to merge in.
@@ -446,7 +245,8 @@ bool CowReader::PrepMergeOps() { 
            continue;

        }



        if (!has_seq_ops_ && IsOrderedOp(current_op)) {

        // Sequence ops must be the first ops in the stream.

        if (seq_ops_set.empty() && IsOrderedOp(current_op)) {

            merge_op_blocks->emplace_back(current_op.new_block);

        } else if (seq_ops_set.count(current_op.new_block) == 0) {

            other_ops.push_back(current_op.new_block);
@@ -718,8 +518,8 @@ std::unique_ptr<ICowOpIter> CowReader::GetMergeOpIter(bool ignore_progress) { 


bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) {

    // Validate the offset, taking care to acknowledge possible overflow of offset+len.

    if (offset < header_.header_size || offset >= fd_size_ - sizeof(CowFooter) || len >= fd_size_ ||

        offset + len > fd_size_ - sizeof(CowFooter)) {

    if (offset < header_.prefix.header_size || offset >= fd_size_ - sizeof(CowFooter) ||

        len >= fd_size_ || offset + len > fd_size_ - sizeof(CowFooter)) {

        LOG(ERROR) << "invalid data offset: " << offset << ", " << len << " bytes";

        return false;

    }