Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b375e238 authored by Daniel Zheng's avatar Daniel Zheng Committed by Automerger Merge Worker
Browse files

Merge "Refactor off V2 Cow Ops" into main am: adad3dbe am: 7248d1b0 am:... 

Merge "Refactor off V2 Cow Ops" into main am: adad3dbe am: 7248d1b0 am: 99ac415b am: aba30990

Original change: https://android-review.googlesource.com/c/platform/system/core/+/2736101



Change-Id: I50bcebebc2ba3528f3c8a1ebbdef0deef5490881
Signed-off-by: default avatarAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
parents 3da4ed00 aba30990

fs_mgr/libsnapshot/include/libsnapshot/cow_format.h

+26 −29
Original line number Diff line number Diff line
@@ -22,6 +22,9 @@ 
namespace android {

namespace snapshot {



struct CowOperationV3;

typedef CowOperationV3 CowOperation;



static constexpr uint64_t kCowMagicNumber = 0x436f77634f572121ULL;

static constexpr uint32_t kCowVersionMajor = 2;

static constexpr uint32_t kCowVersionMinor = 0;
@@ -109,9 +112,8 @@ struct CowFooterOperation { 
    uint64_t num_ops;

} __attribute__((packed));



// Cow operations are currently fixed-size entries, but this may change if

// needed.

struct CowOperation {

// V2 version of COW. On disk format for older devices

struct CowOperationV2 {

    // The operation code (see the constants and structures below).

    uint8_t type;
@@ -146,42 +148,32 @@ struct CowOperation { 
} __attribute__((packed));



// The on disk format of cow (currently ==  CowOperation)

struct CowOperationV2 {

struct CowOperationV3 {

    // The operation code (see the constants and structures below).

    uint8_t type;



    // If this operation reads from the data section of the COW, this contains

    // the compression type of that data (see constants below).

    uint8_t compression;



    // If this operation reads from the data section of the COW, this contains

    // the length.

    uint16_t data_length;



    // The block of data in the new image that this operation modifies.

    uint64_t new_block;

    uint32_t new_block;



    // The value of |source| depends on the operation code.

    //

    // For copy operations, this is a block location in the source image.

    //

    // For replace operations, this is a byte offset within the COW's data

    // sections (eg, not landing within the header or metadata). It is an

    // absolute position within the image.

    //

    // For zero operations (replace with all zeroes), this is unused and must

    // be zero.

    //

    // For Label operations, this is the value of the applied label.

    //

    // For Cluster operations, this is the length of the following data region

    // CopyOp: a 32-bit block location in the source image.

    // ReplaceOp: an absolute byte offset within the COW's data section.

    // XorOp: an absolute byte offset in the source image.

    // ZeroOp: unused

    // LabelOp: a 64-bit opaque identifier.

    //

    // For Xor operations, this is the byte location in the source image.

    uint64_t source;

    // For ops other than Label:

    //  Bits 47-62 are reserved and must be zero.

    // A block is compressed if it’s data is < block_sz

    uint64_t source_info;

} __attribute__((packed));



static_assert(sizeof(CowOperationV2) == sizeof(CowOperation));

static_assert(sizeof(CowOperation) == sizeof(CowFooterOperation));

static_assert(sizeof(CowOperationV2) == sizeof(CowFooterOperation));



static constexpr uint8_t kCowCopyOp = 1;

static constexpr uint8_t kCowReplaceOp = 2;
@@ -208,11 +200,14 @@ static constexpr uint8_t kCowReadAheadNotStarted = 0; 
static constexpr uint8_t kCowReadAheadInProgress = 1;

static constexpr uint8_t kCowReadAheadDone = 2;



static constexpr uint64_t kCowOpSourceInfoDataMask = (1ULL << 48) - 1;

static constexpr uint64_t kCowOpSourceInfoCompressBit = (1ULL << 63);



static inline uint64_t GetCowOpSourceInfoData(const CowOperation* op) {

    return op->source;

    return op->source_info & kCowOpSourceInfoDataMask;

}

static inline bool GetCowOpSourceInfoCompression(const CowOperation* op) {

    return op->compression != kCowCompressNone;

    return !!(op->source_info & kCowOpSourceInfoCompressBit);

}



struct CowFooter {
@@ -236,10 +231,12 @@ struct BufferState { 
// 2MB Scratch space used for read-ahead

static constexpr uint64_t BUFFER_REGION_DEFAULT_SIZE = (1ULL << 21);



std::ostream& operator<<(std::ostream& os, CowOperationV2 const& arg);



std::ostream& operator<<(std::ostream& os, CowOperation const& arg);



int64_t GetNextOpOffset(const CowOperation& op, uint32_t cluster_size);

int64_t GetNextDataOffset(const CowOperation& op, uint32_t cluster_size);

int64_t GetNextOpOffset(const CowOperationV2& op, uint32_t cluster_size);

int64_t GetNextDataOffset(const CowOperationV2& op, uint32_t cluster_size);



// Ops that are internal to the Cow Format and not OTA data

bool IsMetadataOp(const CowOperation& op);

fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h

+3 −1
Original line number Diff line number Diff line
@@ -165,9 +165,10 @@ class CowReader final : public ICowReader { 
    void UpdateMergeOpsCompleted(int num_merge_ops) { header_.num_merge_ops += num_merge_ops; }



  private:

    bool ParseV2(android::base::borrowed_fd fd, std::optional<uint64_t> label);

    bool PrepMergeOps();

    uint64_t FindNumCopyops();

    uint8_t GetCompressionType(const CowOperation* op);

    uint8_t GetCompressionType();



    android::base::unique_fd owned_fd_;

    android::base::borrowed_fd fd_;
@@ -184,6 +185,7 @@ class CowReader final : public ICowReader { 
    std::shared_ptr<std::unordered_map<uint64_t, uint64_t>> data_loc_;

    ReaderFlags reader_flag_;

    bool is_merge_{};

    uint8_t compression_type_ = kCowCompressNone;

};



}  // namespace snapshot

fs_mgr/libsnapshot/libsnapshot_cow/cow_format.cpp

+77 −37
Original line number Diff line number Diff line
@@ -14,11 +14,14 @@ 
// limitations under the License.

//



#include <inttypes.h>

#include <libsnapshot/cow_format.h>

#include <sys/types.h>

#include <unistd.h>



#include <android-base/logging.h>

#include <android-base/stringprintf.h>

#include <libsnapshot/cow_format.h>

#include "writer_v2.h"



namespace android {
@@ -26,45 +29,82 @@ namespace snapshot { 


using android::base::unique_fd;



std::ostream& operator<<(std::ostream& os, CowOperation const& op) {

    os << "CowOperation(type:";

    if (op.type == kCowCopyOp)

        os << "kCowCopyOp,    ";

    else if (op.type == kCowReplaceOp)

        os << "kCowReplaceOp, ";

    else if (op.type == kCowZeroOp)

        os << "kZeroOp,       ";

    else if (op.type == kCowFooterOp)

        os << "kCowFooterOp,  ";

    else if (op.type == kCowLabelOp)

        os << "kCowLabelOp,   ";

    else if (op.type == kCowClusterOp)

        os << "kCowClusterOp  ";

    else if (op.type == kCowXorOp)

        os << "kCowXorOp      ";

    else if (op.type == kCowSequenceOp)

        os << "kCowSequenceOp ";

    else if (op.type == kCowFooterOp)

        os << "kCowFooterOp  ";

    else

        os << (int)op.type << "?,";

    os << "compression:";

    if (op.compression == kCowCompressNone)

        os << "kCowCompressNone,   ";

    else if (op.compression == kCowCompressGz)

        os << "kCowCompressGz,     ";

    else if (op.compression == kCowCompressBrotli)

        os << "kCowCompressBrotli, ";

    else

        os << (int)op.compression << "?, ";

    os << "data_length:" << op.data_length << ",\t";

    os << "new_block:" << op.new_block << ",\t";

std::ostream& EmitCowTypeString(std::ostream& os, uint8_t cow_type) {

    switch (cow_type) {

        case kCowCopyOp:

            return os << "kCowCopyOp";

        case kCowReplaceOp:

            return os << "kCowReplaceOp";

        case kCowZeroOp:

            return os << "kZeroOp";

        case kCowFooterOp:

            return os << "kCowFooterOp";

        case kCowLabelOp:

            return os << "kCowLabelOp";

        case kCowClusterOp:

            return os << "kCowClusterOp";

        case kCowXorOp:

            return os << "kCowXorOp";

        case kCowSequenceOp:

            return os << "kCowSequenceOp";

        default:

            return os << (int)cow_type << "unknown";

    }

}



std::ostream& operator<<(std::ostream& os, CowOperationV2 const& op) {

    os << "CowOperationV2(";

    EmitCowTypeString(os, op.type) << ", ";

    switch (op.compression) {

        case kCowCompressNone:

            os << "uncompressed, ";

            break;

        case kCowCompressGz:

            os << "gz, ";

            break;

        case kCowCompressBrotli:

            os << "brotli, ";

            break;

        case kCowCompressLz4:

            os << "lz4, ";

            break;

        case kCowCompressZstd:

            os << "zstd, ";

            break;

    }

    os << "data_length:" << op.data_length << ", ";

    os << "new_block:" << op.new_block << ", ";

    os << "source:" << op.source;

    os << ")";

    return os;

}



int64_t GetNextOpOffset(const CowOperation& op, uint32_t cluster_ops) {

std::ostream& operator<<(std::ostream& os, CowOperation const& op) {

    os << "CowOperation(";

    EmitCowTypeString(os, op.type);

    if (op.type == kCowReplaceOp || op.type == kCowXorOp || op.type == kCowSequenceOp) {

        if (op.source_info & kCowOpSourceInfoCompressBit) {

            os << ", compressed";

        } else {

            os << ", uncompressed";

        }

        os << ", data_length:" << op.data_length;

    }

    if (op.type != kCowClusterOp && op.type != kCowSequenceOp && op.type != kCowLabelOp) {

        os << ", new_block:" << op.new_block;

    }

    if (op.type == kCowXorOp || op.type == kCowReplaceOp || op.type == kCowCopyOp) {

        os << ", source:" << (op.source_info & kCowOpSourceInfoDataMask);

    } else if (op.type == kCowClusterOp) {

        os << ", cluster_data:" << (op.source_info & kCowOpSourceInfoDataMask);

    } else {

        os << ", label:0x" << android::base::StringPrintf("%" PRIx64, op.source_info);

    }

    os << ")";

    return os;

}



int64_t GetNextOpOffset(const CowOperationV2& op, uint32_t cluster_ops) {

    if (op.type == kCowClusterOp) {

        return op.source;

    } else if ((op.type == kCowReplaceOp || op.type == kCowXorOp) && cluster_ops == 0) {
@@ -74,11 +114,11 @@ int64_t GetNextOpOffset(const CowOperation& op, uint32_t cluster_ops) { 
    }

}



int64_t GetNextDataOffset(const CowOperation& op, uint32_t cluster_ops) {

int64_t GetNextDataOffset(const CowOperationV2& op, uint32_t cluster_ops) {

    if (op.type == kCowClusterOp) {

        return cluster_ops * sizeof(CowOperation);

        return cluster_ops * sizeof(CowOperationV2);

    } else if (cluster_ops == 0) {

        return sizeof(CowOperation);

        return sizeof(CowOperationV2);

    } else {

        return 0;

    }

fs_mgr/libsnapshot/libsnapshot_cow/cow_reader.cpp

+42 −5
Original line number Diff line number Diff line
@@ -55,6 +55,7 @@ std::unique_ptr<CowReader> CowReader::CloneCowReader() { 
    cow->data_loc_ = data_loc_;

    cow->block_pos_index_ = block_pos_index_;

    cow->is_merge_ = is_merge_;

    cow->compression_type_ = compression_type_;

    return cow;

}
@@ -101,8 +102,44 @@ bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> lab 
    footer_ = parser.footer();

    fd_size_ = parser.fd_size();

    last_label_ = parser.last_label();

    ops_ = parser.ops();

    data_loc_ = parser.data_loc();

    ops_ = std::make_shared<std::vector<CowOperation>>(parser.ops()->size());



    // Translate the operation buffer from on disk to in memory

    for (size_t i = 0; i < parser.ops()->size(); i++) {

        const auto& v2_op = parser.ops()->at(i);



        auto& new_op = ops_->at(i);

        new_op.type = v2_op.type;

        new_op.data_length = v2_op.data_length;



        if (v2_op.new_block > std::numeric_limits<uint32_t>::max()) {

            LOG(ERROR) << "Out-of-range new block in COW op: " << v2_op;

            return false;

        }

        new_op.new_block = v2_op.new_block;



        uint64_t source_info = v2_op.source;

        if (new_op.type != kCowLabelOp) {

            source_info &= kCowOpSourceInfoDataMask;

            if (source_info != v2_op.source) {

                LOG(ERROR) << "Out-of-range source value in COW op: " << v2_op;

                return false;

            }

        }

        if (v2_op.compression != kCowCompressNone) {

            if (compression_type_ == kCowCompressNone) {

                compression_type_ = v2_op.compression;

            } else if (compression_type_ != v2_op.compression) {

                LOG(ERROR) << "COW has mixed compression types which is not supported;"

                           << " previously saw " << compression_type_ << ", got "

                           << v2_op.compression << ", op: " << v2_op;

                return false;

            }

            source_info |= kCowOpSourceInfoCompressBit;

        }

        new_op.source_info = source_info;

    }



    // If we're resuming a write, we're not ready to merge

    if (label.has_value()) return true;
@@ -597,14 +634,14 @@ class CowDataStream final : public IByteStream { 
    size_t remaining_;

};



uint8_t CowReader::GetCompressionType(const CowOperation* op) {

    return op->compression;

uint8_t CowReader::GetCompressionType() {

    return compression_type_;

}



ssize_t CowReader::ReadData(const CowOperation* op, void* buffer, size_t buffer_size,

                            size_t ignore_bytes) {

    std::unique_ptr<IDecompressor> decompressor;

    switch (GetCompressionType(op)) {

    switch (GetCompressionType()) {

        case kCowCompressNone:

            break;

        case kCowCompressGz:
@@ -624,7 +661,7 @@ ssize_t CowReader::ReadData(const CowOperation* op, void* buffer, size_t buffer_ 
            }

            break;

        default:

            LOG(ERROR) << "Unknown compression type: " << GetCompressionType(op);

            LOG(ERROR) << "Unknown compression type: " << GetCompressionType();

            return -1;

    }

fs_mgr/libsnapshot/libsnapshot_cow/parser_v2.cpp

+11 −11
Original line number Diff line number Diff line
@@ -66,18 +66,18 @@ bool CowParserV2::Parse(borrowed_fd fd, const CowHeader& header, std::optional<u 
                   << sizeof(CowFooter);

        return false;

    }

    if (header_.op_size != sizeof(CowOperation)) {

    if (header_.op_size != sizeof(CowOperationV2)) {

        LOG(ERROR) << "Operation size unknown, read " << header_.op_size << ", expected "

                   << sizeof(CowOperation);

                   << sizeof(CowOperationV2);

        return false;

    }

    if (header_.cluster_ops == 1) {

        LOG(ERROR) << "Clusters must contain at least two operations to function.";

        return false;

    }

    if (header_.op_size != sizeof(CowOperation)) {

    if (header_.op_size != sizeof(CowOperationV2)) {

        LOG(ERROR) << "Operation size unknown, read " << header_.op_size << ", expected "

                   << sizeof(CowOperation);

                   << sizeof(CowOperationV2);

        return false;

    }

    if (header_.cluster_ops == 1) {
@@ -123,23 +123,23 @@ bool CowParserV2::ParseOps(borrowed_fd fd, std::optional<uint64_t> label) { 
    uint64_t data_pos = 0;



    if (header_.cluster_ops) {

        data_pos = pos + header_.cluster_ops * sizeof(CowOperation);

        data_pos = pos + header_.cluster_ops * sizeof(CowOperationV2);

    } else {

        data_pos = pos + sizeof(CowOperation);

        data_pos = pos + sizeof(CowOperationV2);

    }



    auto ops_buffer = std::make_shared<std::vector<CowOperation>>();

    auto ops_buffer = std::make_shared<std::vector<CowOperationV2>>();

    uint64_t current_op_num = 0;

    uint64_t cluster_ops = header_.cluster_ops ?: 1;

    bool done = false;



    // Alternating op clusters and data

    while (!done) {

        uint64_t to_add = std::min(cluster_ops, (fd_size_ - pos) / sizeof(CowOperation));

        uint64_t to_add = std::min(cluster_ops, (fd_size_ - pos) / sizeof(CowOperationV2));

        if (to_add == 0) break;

        ops_buffer->resize(current_op_num + to_add);

        if (!android::base::ReadFully(fd, &ops_buffer->data()[current_op_num],

                                      to_add * sizeof(CowOperation))) {

                                      to_add * sizeof(CowOperationV2))) {

            PLOG(ERROR) << "read op failed";

            return false;

        }
@@ -150,7 +150,7 @@ bool CowParserV2::ParseOps(borrowed_fd fd, std::optional<uint64_t> label) { 
            if (current_op.type == kCowXorOp) {

                data_loc->insert({current_op.new_block, data_pos});

            }

            pos += sizeof(CowOperation) + GetNextOpOffset(current_op, header_.cluster_ops);

            pos += sizeof(CowOperationV2) + GetNextOpOffset(current_op, header_.cluster_ops);

            data_pos += current_op.data_length + GetNextDataOffset(current_op, header_.cluster_ops);



            if (current_op.type == kCowClusterOp) {
@@ -222,7 +222,7 @@ bool CowParserV2::ParseOps(borrowed_fd fd, std::optional<uint64_t> label) { 
                       << ops_buffer->size();

            return false;

        }

        if (ops_buffer->size() * sizeof(CowOperation) != footer_->op.ops_size) {

        if (ops_buffer->size() * sizeof(CowOperationV2) != footer_->op.ops_size) {

            LOG(ERROR) << "ops size does not match ";

            return false;

        }