Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9e6cef7f authored by David Anderson's avatar David Anderson Committed by Gerrit Code Review
Browse files

Merge changes Ie79ab525,I3befc71f 

* changes:
  libsnapshot: Refactor COW reading/writing of footers.
  libsnapshot: Don't try to truncate block devices
parents 70d27113 ea791367

fs_mgr/libsnapshot/cow_api_test.cpp

+2 −5
Original line number Diff line number Diff line
@@ -264,10 +264,7 @@ TEST_F(CowTest, GetSize) { 
    ASSERT_EQ(size_before, size_after);

    struct stat buf;



    if (fstat(cow_->fd, &buf) < 0) {

        perror("Fails to determine size of cow image written");

        FAIL();

    }

    ASSERT_GE(fstat(cow_->fd, &buf), 0) << strerror(errno);

    ASSERT_EQ(buf.st_size, writer.GetCowSize());

}
@@ -408,7 +405,7 @@ TEST_F(CowTest, AppendExtendedCorrupted) { 
    // Get the last known good label

    CowReader label_reader;

    uint64_t label;

    ASSERT_TRUE(label_reader.Parse(cow_->fd));

    ASSERT_TRUE(label_reader.Parse(cow_->fd, {5}));

    ASSERT_TRUE(label_reader.GetLastLabel(&label));

    ASSERT_EQ(label, 5);

fs_mgr/libsnapshot/cow_reader.cpp

+66 −71
Original line number Diff line number Diff line
@@ -31,14 +31,7 @@ 
namespace android {

namespace snapshot {



CowReader::CowReader()

    : fd_(-1),

      header_(),

      footer_(),

      fd_size_(0),

      has_footer_(false),

      last_label_(0),

      has_last_label_(false) {}

CowReader::CowReader() : fd_(-1), header_(), fd_size_(0) {}



static void SHA256(const void*, size_t, uint8_t[]) {

#if 0
@@ -49,12 +42,12 @@ static void SHA256(const void*, size_t, uint8_t[]) { 
#endif

}



bool CowReader::Parse(android::base::unique_fd&& fd) {

bool CowReader::Parse(android::base::unique_fd&& fd, std::optional<uint64_t> label) {

    owned_fd_ = std::move(fd);

    return Parse(android::base::borrowed_fd{owned_fd_});

    return Parse(android::base::borrowed_fd{owned_fd_}, label);

}



bool CowReader::Parse(android::base::borrowed_fd fd) {

bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> label) {

    fd_ = fd;



    auto pos = lseek(fd_.get(), 0, SEEK_END);
@@ -99,105 +92,107 @@ bool CowReader::Parse(android::base::borrowed_fd fd) { 
        return false;

    }



    auto footer_pos = lseek(fd_.get(), -header_.footer_size, SEEK_END);

    if (footer_pos != fd_size_ - header_.footer_size) {

        LOG(ERROR) << "Failed to read full footer!";

        return false;

    }

    if (!android::base::ReadFully(fd_, &footer_, sizeof(footer_))) {

        PLOG(ERROR) << "read footer failed";

        return false;

    }

    has_footer_ = (footer_.op.type == kCowFooterOp);

    return ParseOps();

    return ParseOps(label);

}



bool CowReader::ParseOps() {

bool CowReader::ParseOps(std::optional<uint64_t> label) {

    uint64_t pos = lseek(fd_.get(), sizeof(header_), SEEK_SET);

    if (pos != sizeof(header_)) {

        PLOG(ERROR) << "lseek ops failed";

        return false;

    }

    std::optional<uint64_t> next_last_label;



    auto ops_buffer = std::make_shared<std::vector<CowOperation>>();

    if (has_footer_) ops_buffer->reserve(footer_.op.num_ops);

    uint64_t current_op_num = 0;

    // Look until we reach the last possible non-footer position.

    uint64_t last_pos = fd_size_ - (has_footer_ ? sizeof(footer_) : sizeof(CowOperation));



    // Alternating op and data

    while (pos < last_pos) {

        ops_buffer->resize(current_op_num + 1);

        if (!android::base::ReadFully(fd_, ops_buffer->data() + current_op_num,

                                      sizeof(CowOperation))) {

    while (true) {

        ops_buffer->emplace_back();

        if (!android::base::ReadFully(fd_, &ops_buffer->back(), sizeof(CowOperation))) {

            PLOG(ERROR) << "read op failed";

            return false;

        }

        auto& current_op = ops_buffer->data()[current_op_num];

        pos = lseek(fd_.get(), GetNextOpOffset(current_op), SEEK_CUR);

        if (pos == uint64_t(-1)) {



        auto& current_op = ops_buffer->back();

        off_t offs = lseek(fd_.get(), GetNextOpOffset(current_op), SEEK_CUR);

        if (offs < 0) {

            PLOG(ERROR) << "lseek next op failed";

            return false;

        }

        current_op_num++;

        if (next_last_label) {

            last_label_ = next_last_label.value();

            has_last_label_ = true;

        }

        pos = static_cast<uint64_t>(offs);



        if (current_op.type == kCowLabelOp) {

            // If we don't have a footer, the last label may be incomplete.

            // If we see any operation after it, we can infer the flush finished.

            if (has_footer_) {

                has_last_label_ = true;

                last_label_ = current_op.source;

            } else {

                next_last_label = {current_op.source};

            last_label_ = {current_op.source};



            // If we reach the requested label, stop reading.

            if (label && label.value() == current_op.source) {

                break;

            }

        } else if (current_op.type == kCowFooterOp) {

            memcpy(&footer_.op, &current_op, sizeof(footer_.op));

            // we don't consider this an operation for the checksum

            current_op_num--;

            if (android::base::ReadFully(fd_, &footer_.data, sizeof(footer_.data))) {

                has_footer_ = true;

                if (next_last_label) {

                    last_label_ = next_last_label.value();

                    has_last_label_ = true;

                }

            footer_.emplace();



            CowFooter* footer = &footer_.value();

            memcpy(&footer_->op, &current_op, sizeof(footer->op));



            if (!android::base::ReadFully(fd_, &footer->data, sizeof(footer->data))) {

                LOG(ERROR) << "Could not read COW footer";

                return false;

            }



            // Drop the footer from the op stream.

            ops_buffer->pop_back();

            break;

        }

    }



    // To successfully parse a COW file, we need either:

    //  (1) a label to read up to, and for that label to be found, or

    //  (2) a valid footer.

    if (label) {

        if (!last_label_) {

            LOG(ERROR) << "Did not find label " << label.value()

                       << " while reading COW (no labels found)";

            return false;

        }

        if (last_label_.value() != label.value()) {

            LOG(ERROR) << "Did not find label " << label.value()

                       << ", last label=" << last_label_.value();

            return false;

        }

    } else if (!footer_) {

        LOG(ERROR) << "No COW footer found";

        return false;

    }



    uint8_t csum[32];

    memset(csum, 0, sizeof(uint8_t) * 32);



    if (has_footer_) {

        if (ops_buffer->size() != footer_.op.num_ops) {

    if (footer_) {

        if (ops_buffer->size() != footer_->op.num_ops) {

            LOG(ERROR) << "num ops does not match";

            return false;

        }

        if (ops_buffer->size() * sizeof(CowOperation) != footer_.op.ops_size) {

        if (ops_buffer->size() * sizeof(CowOperation) != footer_->op.ops_size) {

            LOG(ERROR) << "ops size does not match ";

            return false;

        }

        SHA256(&footer_.op, sizeof(footer_.op), footer_.data.footer_checksum);

        if (memcmp(csum, footer_.data.ops_checksum, sizeof(csum)) != 0) {

        SHA256(&footer_->op, sizeof(footer_->op), footer_->data.footer_checksum);

        if (memcmp(csum, footer_->data.ops_checksum, sizeof(csum)) != 0) {

            LOG(ERROR) << "ops checksum does not match";

            return false;

        }

        SHA256(ops_buffer.get()->data(), footer_.op.ops_size, csum);

        if (memcmp(csum, footer_.data.ops_checksum, sizeof(csum)) != 0) {

        SHA256(ops_buffer.get()->data(), footer_->op.ops_size, csum);

        if (memcmp(csum, footer_->data.ops_checksum, sizeof(csum)) != 0) {

            LOG(ERROR) << "ops checksum does not match";

            return false;

        }

    } else {

        LOG(INFO) << "No Footer, recovered data";

        LOG(INFO) << "No COW Footer, recovered data";

    }



    if (header_.num_merge_ops > 0) {

        uint64_t merge_ops = header_.num_merge_ops;

        uint64_t metadata_ops = 0;

        current_op_num = 0;

        uint64_t current_op_num = 0;



        CHECK(ops_buffer->size() >= merge_ops);

        while (merge_ops) {
@@ -223,14 +218,14 @@ bool CowReader::GetHeader(CowHeader* header) { 
}



bool CowReader::GetFooter(CowFooter* footer) {

    if (!has_footer_) return false;

    *footer = footer_;

    if (!footer_) return false;

    *footer = footer_.value();

    return true;

}



bool CowReader::GetLastLabel(uint64_t* label) {

    if (!has_last_label_) return false;

    *label = last_label_;

    if (!last_label_) return false;

    *label = last_label_.value();

    return true;

}
@@ -308,8 +303,8 @@ std::unique_ptr<ICowOpReverseIter> CowReader::GetRevOpIter() { 


bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) {

    // Validate the offset, taking care to acknowledge possible overflow of offset+len.

    if (offset < sizeof(header_) || offset >= fd_size_ - sizeof(footer_) || len >= fd_size_ ||

        offset + len > fd_size_ - sizeof(footer_)) {

    if (offset < sizeof(header_) || offset >= fd_size_ - sizeof(CowFooter) || len >= fd_size_ ||

        offset + len > fd_size_ - sizeof(CowFooter)) {

        LOG(ERROR) << "invalid data offset: " << offset << ", " << len << " bytes";

        return false;

    }

fs_mgr/libsnapshot/cow_writer.cpp

+24 −19
Original line number Diff line number Diff line
@@ -122,6 +122,13 @@ bool CowWriter::SetFd(android::base::borrowed_fd fd) { 
        is_dev_null_ = true;

    } else {

        fd_ = fd;



        struct stat stat;

        if (fstat(fd.get(), &stat) < 0) {

            PLOG(ERROR) << "fstat failed";

            return false;

        }

        is_block_device_ = S_ISBLK(stat.st_mode);

    }

    return true;

}
@@ -184,12 +191,10 @@ bool CowWriter::OpenForWrite() { 
bool CowWriter::OpenForAppend(uint64_t label) {

    auto reader = std::make_unique<CowReader>();

    std::queue<CowOperation> toAdd;

    bool found_label = false;



    if (!reader->Parse(fd_) || !reader->GetHeader(&header_)) {

    if (!reader->Parse(fd_, {label}) || !reader->GetHeader(&header_)) {

        return false;

    }

    reader->GetFooter(&footer_);



    options_.block_size = header_.block_size;
@@ -199,30 +204,19 @@ bool CowWriter::OpenForAppend(uint64_t label) { 
    ops_.resize(0);



    auto iter = reader->GetOpIter();

    while (!iter->Done() && !found_label) {

        const CowOperation& op = iter->Get();



        if (op.type == kCowFooterOp) break;

        if (op.type == kCowLabelOp && op.source == label) found_label = true;

        AddOperation(op);



    while (!iter->Done()) {

        AddOperation(iter->Get());

        iter->Next();

    }



    if (!found_label) {

        LOG(ERROR) << "Failed to find last label";

        return false;

    }



    // Free reader so we own the descriptor position again.

    reader = nullptr;



    // Position for new writing

    if (ftruncate(fd_.get(), next_op_pos_) != 0) {

        PLOG(ERROR) << "Failed to trim file";

    // Remove excess data

    if (!Truncate(next_op_pos_)) {

        return false;

    }

    if (lseek(fd_.get(), 0, SEEK_END) < 0) {

    if (lseek(fd_.get(), next_op_pos_, SEEK_SET) < 0) {

        PLOG(ERROR) << "lseek failed";

        return false;

    }
@@ -445,5 +439,16 @@ bool CowWriter::CommitMerge(int merged_ops) { 
    return Sync();

}



bool CowWriter::Truncate(off_t length) {

    if (is_dev_null_ || is_block_device_) {

        return true;

    }

    if (ftruncate(fd_.get(), length) < 0) {

        PLOG(ERROR) << "Failed to truncate.";

        return false;

    }

    return true;

}



}  // namespace snapshot

}  // namespace android

fs_mgr/libsnapshot/include/libsnapshot/cow_reader.h

+8 −7
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ 


#include <functional>

#include <memory>

#include <optional>



#include <android-base/unique_fd.h>

#include <libsnapshot/cow_format.h>
@@ -116,8 +117,10 @@ class CowReader : public ICowReader { 
  public:

    CowReader();



    bool Parse(android::base::unique_fd&& fd);

    bool Parse(android::base::borrowed_fd fd);

    // Parse the COW, optionally, up to the given label. If no label is

    // specified, the COW must have an intact footer.

    bool Parse(android::base::unique_fd&& fd, std::optional<uint64_t> label = {});

    bool Parse(android::base::borrowed_fd fd, std::optional<uint64_t> label = {});



    bool GetHeader(CowHeader* header) override;

    bool GetFooter(CowFooter* footer) override;
@@ -138,16 +141,14 @@ class CowReader : public ICowReader { 
    void UpdateMergeProgress(uint64_t merge_ops) { header_.num_merge_ops += merge_ops; }



  private:

    bool ParseOps();

    bool ParseOps(std::optional<uint64_t> label);



    android::base::unique_fd owned_fd_;

    android::base::borrowed_fd fd_;

    CowHeader header_;

    CowFooter footer_;

    std::optional<CowFooter> footer_;

    uint64_t fd_size_;

    bool has_footer_;

    uint64_t last_label_;

    bool has_last_label_;

    std::optional<uint64_t> last_label_;

    std::shared_ptr<std::vector<CowOperation>> ops_;

};

fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h

+2 −0
Original line number Diff line number Diff line
@@ -123,6 +123,7 @@ class CowWriter : public ICowWriter { 


    bool SetFd(android::base::borrowed_fd fd);

    bool Sync();

    bool Truncate(off_t length);



  private:

    android::base::unique_fd owned_fd_;
@@ -133,6 +134,7 @@ class CowWriter : public ICowWriter { 
    uint64_t next_op_pos_ = 0;

    bool is_dev_null_ = false;

    bool merge_in_progress_ = false;

    bool is_block_device_ = false;



    // :TODO: this is not efficient, but stringstream ubsan aborts because some

    // bytes overflow a signed char.