Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d3016bc5 authored by android-build-team Robot's avatar android-build-team Robot
Browse files

Snap for 7336869 from 6ba6672c to sc-v2-release

Change-Id: I352b7d2aa45b9c48215323af54e96a62e8c2046d
parents 583c7520 6ba6672c
Loading
Loading
Loading
Loading
+3 −1
Original line number Original line Diff line number Diff line
@@ -251,6 +251,7 @@ cc_defaults {
        "snapshot_metadata_updater_test.cpp",
        "snapshot_metadata_updater_test.cpp",
        "snapshot_reader_test.cpp",
        "snapshot_reader_test.cpp",
        "snapshot_test.cpp",
        "snapshot_test.cpp",
        "snapshot_writer_test.cpp",
    ],
    ],
    shared_libs: [
    shared_libs: [
        "libbinder",
        "libbinder",
@@ -421,6 +422,7 @@ cc_defaults {
        "snapuserd.cpp",
        "snapuserd.cpp",
        "snapuserd_daemon.cpp",
        "snapuserd_daemon.cpp",
	"snapuserd_worker.cpp",
	"snapuserd_worker.cpp",
	"snapuserd_readahead.cpp",
    ],
    ],


    cflags: [
    cflags: [
+39 −19
Original line number Original line Diff line number Diff line
@@ -94,11 +94,6 @@ bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> lab
                   << "Expected: " << kCowMagicNumber;
                   << "Expected: " << kCowMagicNumber;
        return false;
        return false;
    }
    }
    if (header_.header_size != sizeof(CowHeader)) {
        LOG(ERROR) << "Header size unknown, read " << header_.header_size << ", expected "
                   << sizeof(CowHeader);
        return false;
    }
    if (header_.footer_size != sizeof(CowFooter)) {
    if (header_.footer_size != sizeof(CowFooter)) {
        LOG(ERROR) << "Footer size unknown, read " << header_.footer_size << ", expected "
        LOG(ERROR) << "Footer size unknown, read " << header_.footer_size << ", expected "
                   << sizeof(CowFooter);
                   << sizeof(CowFooter);
@@ -123,8 +118,7 @@ bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> lab
        return false;
        return false;
    }
    }


    if ((header_.major_version != kCowVersionMajor) ||
    if ((header_.major_version > kCowVersionMajor) || (header_.minor_version != kCowVersionMinor)) {
        (header_.minor_version != kCowVersionMinor)) {
        LOG(ERROR) << "Header version mismatch";
        LOG(ERROR) << "Header version mismatch";
        LOG(ERROR) << "Major version: " << header_.major_version
        LOG(ERROR) << "Major version: " << header_.major_version
                   << "Expected: " << kCowVersionMajor;
                   << "Expected: " << kCowVersionMajor;
@@ -137,11 +131,26 @@ bool CowReader::Parse(android::base::borrowed_fd fd, std::optional<uint64_t> lab
}
}


bool CowReader::ParseOps(std::optional<uint64_t> label) {
bool CowReader::ParseOps(std::optional<uint64_t> label) {
    uint64_t pos = lseek(fd_.get(), sizeof(header_), SEEK_SET);
    uint64_t pos;
    if (pos != sizeof(header_)) {

    // Skip the scratch space
    if (header_.major_version >= 2 && (header_.buffer_size > 0)) {
        LOG(DEBUG) << " Scratch space found of size: " << header_.buffer_size;
        size_t init_offset = header_.header_size + header_.buffer_size;
        pos = lseek(fd_.get(), init_offset, SEEK_SET);
        if (pos != init_offset) {
            PLOG(ERROR) << "lseek ops failed";
            PLOG(ERROR) << "lseek ops failed";
            return false;
            return false;
        }
        }
    } else {
        pos = lseek(fd_.get(), header_.header_size, SEEK_SET);
        if (pos != header_.header_size) {
            PLOG(ERROR) << "lseek ops failed";
            return false;
        }
        // Reading a v1 version of COW which doesn't have buffer_size.
        header_.buffer_size = 0;
    }


    auto ops_buffer = std::make_shared<std::vector<CowOperation>>();
    auto ops_buffer = std::make_shared<std::vector<CowOperation>>();
    uint64_t current_op_num = 0;
    uint64_t current_op_num = 0;
@@ -360,13 +369,7 @@ void CowReader::InitializeMerge() {
    //                        Replace-op-4, Zero-op-9, Replace-op-5 }
    //                        Replace-op-4, Zero-op-9, Replace-op-5 }
    //==============================================================
    //==============================================================


    for (uint64_t i = 0; i < ops_->size(); i++) {
    num_copy_ops = FindNumCopyops();
        auto& current_op = ops_->data()[i];
        if (current_op.type != kCowCopyOp) {
            break;
        }
        num_copy_ops += 1;
    }


    std::sort(ops_.get()->begin() + num_copy_ops, ops_.get()->end(),
    std::sort(ops_.get()->begin() + num_copy_ops, ops_.get()->end(),
              [](CowOperation& op1, CowOperation& op2) -> bool {
              [](CowOperation& op1, CowOperation& op2) -> bool {
@@ -377,6 +380,23 @@ void CowReader::InitializeMerge() {
        CHECK(ops_->size() >= header_.num_merge_ops);
        CHECK(ops_->size() >= header_.num_merge_ops);
        ops_->erase(ops_.get()->begin(), ops_.get()->begin() + header_.num_merge_ops);
        ops_->erase(ops_.get()->begin(), ops_.get()->begin() + header_.num_merge_ops);
    }
    }

    num_copy_ops = FindNumCopyops();
    set_copy_ops(num_copy_ops);
}

uint64_t CowReader::FindNumCopyops() {
    uint64_t num_copy_ops = 0;

    for (uint64_t i = 0; i < ops_->size(); i++) {
        auto& current_op = ops_->data()[i];
        if (current_op.type != kCowCopyOp) {
            break;
        }
        num_copy_ops += 1;
    }

    return num_copy_ops;
}
}


bool CowReader::GetHeader(CowHeader* header) {
bool CowReader::GetHeader(CowHeader* header) {
@@ -470,7 +490,7 @@ std::unique_ptr<ICowOpReverseIter> CowReader::GetRevOpIter() {


bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) {
bool CowReader::GetRawBytes(uint64_t offset, void* buffer, size_t len, size_t* read) {
    // Validate the offset, taking care to acknowledge possible overflow of offset+len.
    // Validate the offset, taking care to acknowledge possible overflow of offset+len.
    if (offset < sizeof(header_) || offset >= fd_size_ - sizeof(CowFooter) || len >= fd_size_ ||
    if (offset < header_.header_size || offset >= fd_size_ - sizeof(CowFooter) || len >= fd_size_ ||
        offset + len > fd_size_ - sizeof(CowFooter)) {
        offset + len > fd_size_ - sizeof(CowFooter)) {
        LOG(ERROR) << "invalid data offset: " << offset << ", " << len << " bytes";
        LOG(ERROR) << "invalid data offset: " << offset << ", " << len << " bytes";
        return false;
        return false;
+83 −3
Original line number Original line Diff line number Diff line
@@ -96,6 +96,7 @@ class TempDevice {
class CowSnapuserdTest final {
class CowSnapuserdTest final {
  public:
  public:
    bool Setup();
    bool Setup();
    bool SetupCopyOverlap();
    bool Merge();
    bool Merge();
    void ValidateMerge();
    void ValidateMerge();
    void ReadSnapshotDeviceAndValidate();
    void ReadSnapshotDeviceAndValidate();
@@ -114,6 +115,7 @@ class CowSnapuserdTest final {
    void StartMerge();
    void StartMerge();


    void CreateCowDevice();
    void CreateCowDevice();
    void CreateCowDeviceWithCopyOverlap();
    void CreateBaseDevice();
    void CreateBaseDevice();
    void InitCowDevice();
    void InitCowDevice();
    void SetDeviceControlName();
    void SetDeviceControlName();
@@ -191,6 +193,24 @@ bool CowSnapuserdTest::Setup() {
    return setup_ok_;
    return setup_ok_;
}
}


bool CowSnapuserdTest::SetupCopyOverlap() {
    CreateBaseDevice();
    CreateCowDeviceWithCopyOverlap();

    SetDeviceControlName();

    StartSnapuserdDaemon();
    InitCowDevice();

    CreateDmUserDevice();
    InitDaemon();

    CreateSnapshotDevice();
    setup_ok_ = true;

    return setup_ok_;
}

void CowSnapuserdTest::StartSnapuserdDaemon() {
void CowSnapuserdTest::StartSnapuserdDaemon() {
    pid_t pid = fork();
    pid_t pid = fork();
    ASSERT_GE(pid, 0);
    ASSERT_GE(pid, 0);
@@ -255,6 +275,49 @@ void CowSnapuserdTest::ReadSnapshotDeviceAndValidate() {
    ASSERT_EQ(memcmp(snapuserd_buffer.get(), (char*)orig_buffer_.get() + (size_ * 3), size_), 0);
    ASSERT_EQ(memcmp(snapuserd_buffer.get(), (char*)orig_buffer_.get() + (size_ * 3), size_), 0);
}
}


void CowSnapuserdTest::CreateCowDeviceWithCopyOverlap() {
    std::string path = android::base::GetExecutableDirectory();
    cow_system_ = std::make_unique<TemporaryFile>(path);

    CowOptions options;
    options.compression = "gz";
    CowWriter writer(options);

    ASSERT_TRUE(writer.Initialize(cow_system_->fd));

    size_t num_blocks = size_ / options.block_size;
    size_t x = num_blocks;
    size_t blk_src_copy = num_blocks - 1;

    // Create overlapping copy operations
    while (1) {
        ASSERT_TRUE(writer.AddCopy(blk_src_copy + 1, blk_src_copy));
        x -= 1;
        if (x == 0) {
            ASSERT_EQ(blk_src_copy, 0);
            break;
        }
        blk_src_copy -= 1;
    }

    // Flush operations
    ASSERT_TRUE(writer.Finalize());

    // Construct the buffer required for validation
    orig_buffer_ = std::make_unique<uint8_t[]>(total_base_size_);

    // Read the entire base device
    ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), total_base_size_, 0),
              true);

    // Merged operations
    ASSERT_EQ(android::base::ReadFullyAtOffset(base_fd_, orig_buffer_.get(), options.block_size, 0),
              true);
    ASSERT_EQ(android::base::ReadFullyAtOffset(
                      base_fd_, (char*)orig_buffer_.get() + options.block_size, size_, 0),
              true);
}

void CowSnapuserdTest::CreateCowDevice() {
void CowSnapuserdTest::CreateCowDevice() {
    unique_fd rnd_fd;
    unique_fd rnd_fd;
    loff_t offset = 0;
    loff_t offset = 0;
@@ -707,17 +770,17 @@ void CowSnapuserdMetadataTest::ValidateMetadata() {


            de = reinterpret_cast<struct disk_exception*>((char*)buffer + offset);
            de = reinterpret_cast<struct disk_exception*>((char*)buffer + offset);
            ASSERT_EQ(de->old_chunk, 21);
            ASSERT_EQ(de->old_chunk, 21);
            ASSERT_EQ(de->new_chunk, 537);
            ASSERT_EQ(de->new_chunk, 536);
            offset += sizeof(struct disk_exception);
            offset += sizeof(struct disk_exception);


            de = reinterpret_cast<struct disk_exception*>((char*)buffer + offset);
            de = reinterpret_cast<struct disk_exception*>((char*)buffer + offset);
            ASSERT_EQ(de->old_chunk, 22);
            ASSERT_EQ(de->old_chunk, 22);
            ASSERT_EQ(de->new_chunk, 538);
            ASSERT_EQ(de->new_chunk, 537);
            offset += sizeof(struct disk_exception);
            offset += sizeof(struct disk_exception);


            de = reinterpret_cast<struct disk_exception*>((char*)buffer + offset);
            de = reinterpret_cast<struct disk_exception*>((char*)buffer + offset);
            ASSERT_EQ(de->old_chunk, 23);
            ASSERT_EQ(de->old_chunk, 23);
            ASSERT_EQ(de->new_chunk, 539);
            ASSERT_EQ(de->new_chunk, 538);
            offset += sizeof(struct disk_exception);
            offset += sizeof(struct disk_exception);


            // End of metadata
            // End of metadata
@@ -757,6 +820,23 @@ TEST(Snapuserd_Test, Snapshot_IO_TEST) {
    harness.ValidateMerge();
    harness.ValidateMerge();
    harness.Shutdown();
    harness.Shutdown();
}
}

TEST(Snapuserd_Test, Snapshot_COPY_Overlap_TEST) {
    CowSnapuserdTest harness;
    ASSERT_TRUE(harness.SetupCopyOverlap());
    ASSERT_TRUE(harness.Merge());
    harness.ValidateMerge();
    harness.Shutdown();
}

TEST(Snapuserd_Test, Snapshot_COPY_Overlap_Merge_Resume_TEST) {
    CowSnapuserdTest harness;
    ASSERT_TRUE(harness.SetupCopyOverlap());
    harness.MergeInterrupt();
    harness.ValidateMerge();
    harness.Shutdown();
}

}  // namespace snapshot
}  // namespace snapshot
}  // namespace android
}  // namespace android


+26 −25
Original line number Original line Diff line number Diff line
@@ -94,6 +94,7 @@ void CowWriter::SetupHeaders() {
    header_.block_size = options_.block_size;
    header_.block_size = options_.block_size;
    header_.num_merge_ops = 0;
    header_.num_merge_ops = 0;
    header_.cluster_ops = options_.cluster_ops;
    header_.cluster_ops = options_.cluster_ops;
    header_.buffer_size = 0;
    footer_ = {};
    footer_ = {};
    footer_.op.data_length = 64;
    footer_.op.data_length = 64;
    footer_.op.type = kCowFooterOp;
    footer_.op.type = kCowFooterOp;
@@ -139,12 +140,6 @@ bool CowWriter::SetFd(android::base::borrowed_fd fd) {
    return true;
    return true;
}
}


void CowWriter::InitializeMerge(borrowed_fd fd, CowHeader* header) {
    fd_ = fd;
    memcpy(&header_, header, sizeof(CowHeader));
    merge_in_progress_ = true;
}

bool CowWriter::Initialize(unique_fd&& fd) {
bool CowWriter::Initialize(unique_fd&& fd) {
    owned_fd_ = std::move(fd);
    owned_fd_ = std::move(fd);
    return Initialize(borrowed_fd{owned_fd_});
    return Initialize(borrowed_fd{owned_fd_});
@@ -172,7 +167,7 @@ bool CowWriter::InitializeAppend(android::base::borrowed_fd fd, uint64_t label)
}
}


void CowWriter::InitPos() {
void CowWriter::InitPos() {
    next_op_pos_ = sizeof(header_);
    next_op_pos_ = sizeof(header_) + header_.buffer_size;
    cluster_size_ = header_.cluster_ops * sizeof(CowOperation);
    cluster_size_ = header_.cluster_ops * sizeof(CowOperation);
    if (header_.cluster_ops) {
    if (header_.cluster_ops) {
        next_data_pos_ = next_op_pos_ + cluster_size_;
        next_data_pos_ = next_op_pos_ + cluster_size_;
@@ -196,6 +191,10 @@ bool CowWriter::OpenForWrite() {
        return false;
        return false;
    }
    }


    if (options_.scratch_space) {
        header_.buffer_size = BUFFER_REGION_DEFAULT_SIZE;
    }

    // Headers are not complete, but this ensures the file is at the right
    // Headers are not complete, but this ensures the file is at the right
    // position.
    // position.
    if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) {
    if (!android::base::WriteFully(fd_, &header_, sizeof(header_))) {
@@ -203,7 +202,27 @@ bool CowWriter::OpenForWrite() {
        return false;
        return false;
    }
    }


    if (options_.scratch_space) {
        // Initialize the scratch space
        std::string data(header_.buffer_size, 0);
        if (!android::base::WriteFully(fd_, data.data(), header_.buffer_size)) {
            PLOG(ERROR) << "writing scratch space failed";
            return false;
        }
    }

    if (!Sync()) {
        LOG(ERROR) << "Header sync failed";
        return false;
    }

    if (lseek(fd_.get(), sizeof(header_) + header_.buffer_size, SEEK_SET) < 0) {
        PLOG(ERROR) << "lseek failed";
        return false;
    }

    InitPos();
    InitPos();

    return true;
    return true;
}
}


@@ -517,24 +536,6 @@ bool CowWriter::Sync() {
    return true;
    return true;
}
}


bool CowWriter::CommitMerge(int merged_ops) {
    CHECK(merge_in_progress_);
    header_.num_merge_ops += merged_ops;

    if (lseek(fd_.get(), 0, SEEK_SET) < 0) {
        PLOG(ERROR) << "lseek failed";
        return false;
    }

    if (!android::base::WriteFully(fd_, reinterpret_cast<const uint8_t*>(&header_),
                                   sizeof(header_))) {
        PLOG(ERROR) << "WriteFully failed";
        return false;
    }

    return Sync();
}

bool CowWriter::Truncate(off_t length) {
bool CowWriter::Truncate(off_t length) {
    if (is_dev_null_ || is_block_device_) {
    if (is_dev_null_ || is_block_device_) {
        return true;
        return true;
+30 −2
Original line number Original line Diff line number Diff line
@@ -21,10 +21,13 @@ namespace android {
namespace snapshot {
namespace snapshot {


static constexpr uint64_t kCowMagicNumber = 0x436f77634f572121ULL;
static constexpr uint64_t kCowMagicNumber = 0x436f77634f572121ULL;
static constexpr uint32_t kCowVersionMajor = 1;
static constexpr uint32_t kCowVersionMajor = 2;
static constexpr uint32_t kCowVersionMinor = 0;
static constexpr uint32_t kCowVersionMinor = 0;


static constexpr uint32_t kCowVersionManifest = 1;
static constexpr uint32_t kCowVersionManifest = 2;

static constexpr uint32_t BLOCK_SZ = 4096;
static constexpr uint32_t BLOCK_SHIFT = (__builtin_ffs(BLOCK_SZ) - 1);


// This header appears as the first sequence of bytes in the COW. All fields
// This header appears as the first sequence of bytes in the COW. All fields
// in the layout are little-endian encoded. The on-disk layout is:
// in the layout are little-endian encoded. The on-disk layout is:
@@ -32,6 +35,8 @@ static constexpr uint32_t kCowVersionManifest = 1;
//      +-----------------------+
//      +-----------------------+
//      |     Header (fixed)    |
//      |     Header (fixed)    |
//      +-----------------------+
//      +-----------------------+
//      |     Scratch space     |
//      +-----------------------+
//      | Operation  (variable) |
//      | Operation  (variable) |
//      | Data       (variable) |
//      | Data       (variable) |
//      +-----------------------+
//      +-----------------------+
@@ -70,6 +75,9 @@ struct CowHeader {


    // Tracks merge operations completed
    // Tracks merge operations completed
    uint64_t num_merge_ops;
    uint64_t num_merge_ops;

    // Scratch space used during merge
    uint32_t buffer_size;
} __attribute__((packed));
} __attribute__((packed));


// This structure is the same size of a normal Operation, but is repurposed for the footer.
// This structure is the same size of a normal Operation, but is repurposed for the footer.
@@ -146,11 +154,31 @@ static constexpr uint8_t kCowCompressNone = 0;
static constexpr uint8_t kCowCompressGz = 1;
static constexpr uint8_t kCowCompressGz = 1;
static constexpr uint8_t kCowCompressBrotli = 2;
static constexpr uint8_t kCowCompressBrotli = 2;


static constexpr uint8_t kCowReadAheadNotStarted = 0;
static constexpr uint8_t kCowReadAheadInProgress = 1;
static constexpr uint8_t kCowReadAheadDone = 2;

struct CowFooter {
struct CowFooter {
    CowFooterOperation op;
    CowFooterOperation op;
    CowFooterData data;
    CowFooterData data;
} __attribute__((packed));
} __attribute__((packed));


struct ScratchMetadata {
    // Block of data in the image that operation modifies
    // and read-ahead thread stores the modified data
    // in the scratch space
    uint64_t new_block;
    // Offset within the file to read the data
    uint64_t file_offset;
} __attribute__((packed));

struct BufferState {
    uint8_t read_ahead_state;
} __attribute__((packed));

// 2MB Scratch space used for read-ahead
static constexpr uint64_t BUFFER_REGION_DEFAULT_SIZE = (1ULL << 21);

std::ostream& operator<<(std::ostream& os, CowOperation const& arg);
std::ostream& operator<<(std::ostream& os, CowOperation const& arg);


int64_t GetNextOpOffset(const CowOperation& op, uint32_t cluster_size);
int64_t GetNextOpOffset(const CowOperation& op, uint32_t cluster_size);
Loading