Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 812aaa96 authored by David Anderson's avatar David Anderson Committed by Gerrit Code Review
Browse files

Merge "libsnapshot: Add CowWriterBase, clean up CowWriter."

parents 1347d340 874fdaed

fs_mgr/libsnapshot/Android.bp

+5 −4
Original line number Diff line number Diff line
@@ -174,12 +174,13 @@ cc_library_static { 
        "libsnapshot_cow_defaults",

    ],

    srcs: [

        "libsnapshot_cow/cow_compress.cpp",

        "libsnapshot_cow/cow_decompress.cpp",

        "libsnapshot_cow/cow_reader.cpp",

        "libsnapshot_cow/cow_writer.cpp",

        "libsnapshot_cow/cow_format.cpp",

        "libsnapshot_cow/cow_compress.cpp",

        "libsnapshot_cow/cow_reader.cpp",

        "libsnapshot_cow/parser_v2.cpp",

        "libsnapshot_cow/writer_base.cpp",

        "libsnapshot_cow/writer_v2.cpp",

    ],

    host_supported: true,

    recovery_available: true,
@@ -371,7 +372,7 @@ cc_test { 
        "libsnapshot_cow_defaults",

    ],

    srcs: [

        "libsnapshot_cow/cow_api_test.cpp",

        "libsnapshot_cow/test_v2.cpp",

    ],

    cflags: [

        "-D_FILE_OFFSET_BITS=64",

fs_mgr/libsnapshot/include/libsnapshot/cow_format.h

+4 −0
Original line number Diff line number Diff line
@@ -31,6 +31,10 @@ static constexpr uint32_t kCowVersionManifest = 2; 
static constexpr uint32_t kMinCowVersion = 1;

static constexpr uint32_t kMaxCowVersion = 2;



// Normally, this should be kMaxCowVersion. When a new version is under testing

// it may be the previous value of kMaxCowVersion.

static constexpr uint32_t kDefaultCowVersion = 2;



// This header appears as the first sequence of bytes in the COW. All fields

// in the layout are little-endian encoded. The on-disk layout is:

//

fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h

+27 −123
Original line number Diff line number Diff line 
// Copyright (C) 2019 The Android Open Source Project

// copyright (c) 2019 the android open source project

//

// Licensed under the Apache License, Version 2.0 (the "License");

// you may not use this file except in compliance with the License.

// You may obtain a copy of the License at

// licensed under the apache license, version 2.0 (the "license");

// you may not use this file except in compliance with the license.

// you may obtain a copy of the license at

//

//      http://www.apache.org/licenses/LICENSE-2.0

//      http://www.apache.org/licenses/license-2.0

//

// Unless required by applicable law or agreed to in writing, software

// distributed under the License is distributed on an "AS IS" BASIS,

// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

// See the License for the specific language governing permissions and

// limitations under the License.

// unless required by applicable law or agreed to in writing, software

// distributed under the license is distributed on an "as is" basis,

// without warranties or conditions of any kind, either express or implied.

// see the license for the specific language governing permissions and

// limitations under the license.



#pragma once
@@ -61,30 +61,28 @@ struct CowOptions { 
// will occur in the sequence they were added to the COW.

class ICowWriter {

  public:

    explicit ICowWriter(const CowOptions& options) : options_(options) {}



    virtual ~ICowWriter() {}



    // Encode an operation that copies the contents of |old_block| to the

    // location of |new_block|. 'num_blocks' is the number of contiguous

    // COPY operations from |old_block| to |new_block|.

    bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1);

    virtual bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0;



    // Encode a sequence of raw blocks. |size| must be a multiple of the block size.

    bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size);

    virtual bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0;



    // Add a sequence of xor'd blocks. |size| must be a multiple of the block size.

    bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block,

                      uint16_t offset);

    virtual bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size,

                              uint32_t old_block, uint16_t offset) = 0;



    // Encode a sequence of zeroed blocks. |size| must be a multiple of the block size.

    bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks);

    virtual bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0;



    // Add a label to the op sequence.

    bool AddLabel(uint64_t label);

    virtual bool AddLabel(uint64_t label) = 0;



    // Add sequence data for op merging. Data is a list of the destination block numbers.

    bool AddSequenceData(size_t num_ops, const uint32_t* data);

    virtual bool AddSequenceData(size_t num_ops, const uint32_t* data) = 0;



    // Flush all pending writes. This must be called before closing the writer

    // to ensure that the correct headers and footers are written.
@@ -93,21 +91,8 @@ class ICowWriter { 
    // Return number of bytes the cow image occupies on disk.

    virtual uint64_t GetCowSize() = 0;



    const CowOptions& options() { return options_; }



  protected:

    virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0;

    virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0;

    virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,

                               uint32_t old_block, uint16_t offset) = 0;

    virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0;

    virtual bool EmitLabel(uint64_t label) = 0;

    virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) = 0;



    bool ValidateNewBlock(uint64_t new_block);



  protected:

    CowOptions options_;

    virtual uint32_t GetBlockSize() const = 0;

    virtual std::optional<uint32_t> GetMaxBlocks() const = 0;

};



class CompressWorker {
@@ -146,96 +131,15 @@ class CompressWorker { 
                        std::vector<std::basic_string<uint8_t>>* compressed_data);

};



class CowWriter : public ICowWriter {

  public:

    explicit CowWriter(const CowOptions& options);

    ~CowWriter();

// Create an ICowWriter not backed by any file. This is useful for estimating

// the final size of a cow file.

std::unique_ptr<ICowWriter> CreateCowEstimator(uint32_t version, const CowOptions& options);



    // Set up the writer.

    // The file starts from the beginning.

    //

    // If fd is < 0, the CowWriter will be opened against /dev/null. This is for

    // computing COW sizes without using storage space.

    bool Initialize(android::base::unique_fd&& fd);

    bool Initialize(android::base::borrowed_fd fd);

    // Set up a writer, assuming that the given label is the last valid label.

    // This will result in dropping any labels that occur after the given on, and will fail

    // if the given label does not appear.

    bool InitializeAppend(android::base::unique_fd&&, uint64_t label);

    bool InitializeAppend(android::base::borrowed_fd fd, uint64_t label);



    bool Finalize() override;



    uint64_t GetCowSize() override;



  protected:

    virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;

    virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;

    virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,

                               uint32_t old_block, uint16_t offset) override;

    virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;

    virtual bool EmitLabel(uint64_t label) override;

    virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) override;



  private:

    bool EmitCluster();

    bool EmitClusterIfNeeded();

    bool EmitBlocks(uint64_t new_block_start, const void* data, size_t size, uint64_t old_block,

                    uint16_t offset, uint8_t type);

    void SetupHeaders();

    void SetupWriteOptions();

    bool ParseOptions();

    bool OpenForWrite();

    bool OpenForAppend(uint64_t label);

    bool GetDataPos(uint64_t* pos);

    bool WriteRawData(const void* data, size_t size);

    bool WriteOperation(const CowOperation& op, const void* data = nullptr, size_t size = 0);

    void AddOperation(const CowOperation& op);

    void InitPos();

    void InitBatchWrites();

    void InitWorkers();

    bool FlushCluster();



    bool CompressBlocks(size_t num_blocks, const void* data);

    bool SetFd(android::base::borrowed_fd fd);

    bool Sync();

    bool Truncate(off_t length);

    bool EnsureSpaceAvailable(const uint64_t bytes_needed) const;



  private:

    android::base::unique_fd owned_fd_;

    android::base::borrowed_fd fd_;

    CowHeader header_{};

    CowFooter footer_{};

    CowCompressionAlgorithm compression_ = kCowCompressNone;

    uint64_t current_op_pos_ = 0;

    uint64_t next_op_pos_ = 0;

    uint64_t next_data_pos_ = 0;

    uint64_t current_data_pos_ = 0;

    ssize_t total_data_written_ = 0;

    uint32_t cluster_size_ = 0;

    uint32_t current_cluster_size_ = 0;

    uint64_t current_data_size_ = 0;

    bool is_dev_null_ = false;

    bool merge_in_progress_ = false;

    bool is_block_device_ = false;

    uint64_t cow_image_size_ = INT64_MAX;



    int num_compress_threads_ = 1;

    std::vector<std::unique_ptr<CompressWorker>> compress_threads_;

    std::vector<std::future<bool>> threads_;

    std::vector<std::basic_string<uint8_t>> compressed_buf_;

    std::vector<std::basic_string<uint8_t>>::iterator buf_iter_;



    std::vector<std::unique_ptr<CowOperation>> opbuffer_vec_;

    std::vector<std::unique_ptr<uint8_t[]>> databuffer_vec_;

    std::unique_ptr<struct iovec[]> cowop_vec_;

    int op_vec_index_ = 0;



    std::unique_ptr<struct iovec[]> data_vec_;

    int data_vec_index_ = 0;

    bool batch_write_ = false;

};

// Create an ICowWriter of the given version and options. If a label is given,

// the writer is opened in append mode.

std::unique_ptr<ICowWriter> CreateCowWriter(uint32_t version, const CowOptions& options,

                                            android::base::unique_fd&& fd,

                                            std::optional<uint64_t> label = {});



}  // namespace snapshot

}  // namespace android

fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h

+9 −16
Original line number Diff line number Diff line
@@ -23,30 +23,23 @@ class MockSnapshotWriter : public ISnapshotWriter { 
  public:

    using FileDescriptor = ISnapshotWriter::FileDescriptor;



    explicit MockSnapshotWriter(const CowOptions& options) : ISnapshotWriter(options) {}

    MockSnapshotWriter() : ISnapshotWriter({}) {}



    MOCK_METHOD(bool, Finalize, (), (override));



    // Return number of bytes the cow image occupies on disk.

    MOCK_METHOD(uint64_t, GetCowSize, (), (override));



    MOCK_METHOD(bool, EmitCopy, (uint64_t, uint64_t, uint64_t), (override));

    MOCK_METHOD(bool, EmitRawBlocks, (uint64_t, const void*, size_t), (override));

    MOCK_METHOD(bool, EmitXorBlocks, (uint32_t, const void*, size_t, uint32_t, uint16_t),

    MOCK_METHOD(bool, AddCopy, (uint64_t, uint64_t, uint64_t), (override));

    MOCK_METHOD(bool, AddRawBlocks, (uint64_t, const void*, size_t), (override));

    MOCK_METHOD(bool, AddXorBlocks, (uint32_t, const void*, size_t, uint32_t, uint16_t),

                (override));

    MOCK_METHOD(bool, EmitZeroBlocks, (uint64_t, uint64_t), (override));

    MOCK_METHOD(bool, EmitLabel, (uint64_t), (override));

    MOCK_METHOD(bool, EmitSequenceData, (size_t, const uint32_t*), (override));



    // Open the writer in write mode (no append).

    MOCK_METHOD(bool, AddZeroBlocks, (uint64_t, uint64_t), (override));

    MOCK_METHOD(bool, AddLabel, (uint64_t), (override));

    MOCK_METHOD(bool, AddSequenceData, (size_t, const uint32_t*), (override));

    MOCK_METHOD(bool, Initialize, (), (override));

    MOCK_METHOD(bool, InitializeAppend, (uint64_t), (override));

    MOCK_METHOD(bool, VerifyMergeOps, (), (override, const, noexcept));



    // Open the writer in append mode, with the last label to resume

    // from. See CowWriter::InitializeAppend.

    MOCK_METHOD(bool, InitializeAppend, (uint64_t label), (override));



    MOCK_METHOD(std::unique_ptr<FileDescriptor>, OpenReader, (), (override));

    MOCK_METHOD(uint32_t, GetBlockSize, (), (override, const));

    MOCK_METHOD(std::optional<uint32_t>, GetMaxBlocks, (), (override, const));

};

}  // namespace android::snapshot

fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h

+25 −25
Original line number Diff line number Diff line
@@ -31,13 +31,7 @@ class ISnapshotWriter : public ICowWriter { 
  public:

    using FileDescriptor = chromeos_update_engine::FileDescriptor;



    explicit ISnapshotWriter(const CowOptions& options);



    // Set the source device. This is used for AddCopy() operations, if the

    // underlying writer needs the original bytes (for example if backed by

    // dm-snapshot or if writing directly to an unsnapshotted region). The

    // device is only opened on the first operation that requires it.

    void SetSourceDevice(const std::string& source_device);

    virtual ~ISnapshotWriter() {}



    // Open the writer in write mode (no append).

    virtual bool Initialize() = 0;
@@ -47,15 +41,8 @@ class ISnapshotWriter : public ICowWriter { 
    virtual bool InitializeAppend(uint64_t label) = 0;



    virtual std::unique_ptr<FileDescriptor> OpenReader() = 0;

    virtual bool VerifyMergeOps() const noexcept = 0;



  protected:

    android::base::borrowed_fd GetSourceFd();



    std::optional<std::string> source_device_;



  private:

    android::base::unique_fd source_fd_;

    virtual bool VerifyMergeOps() const noexcept = 0;

};



// Send writes to a COW or a raw device directly, based on a threshold.
@@ -63,6 +50,8 @@ class CompressedSnapshotWriter final : public ISnapshotWriter { 
  public:

    CompressedSnapshotWriter(const CowOptions& options);



    void SetSourceDevice(const std::string& source_device);



    // Sets the COW device; this is required.

    bool SetCowDevice(android::base::unique_fd&& cow_device);
@@ -70,23 +59,34 @@ class CompressedSnapshotWriter final : public ISnapshotWriter { 
    bool InitializeAppend(uint64_t label) override;

    bool Finalize() override;

    uint64_t GetCowSize() override;

    uint32_t GetBlockSize() const override;

    std::optional<uint32_t> GetMaxBlocks() const override;

    std::unique_ptr<FileDescriptor> OpenReader() override;

    bool VerifyMergeOps() const noexcept;



  protected:

    bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;

    bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;

    bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block,

    bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;

    bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;

    bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block,

                      uint16_t offset) override;

    bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;

    bool EmitLabel(uint64_t label) override;

    bool EmitSequenceData(size_t num_ops, const uint32_t* data) override;

    bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;

    bool AddLabel(uint64_t label) override;

    bool AddSequenceData(size_t num_ops, const uint32_t* data) override;



  private:

    std::unique_ptr<CowReader> OpenCowReader() const;

    android::base::unique_fd cow_device_;

    android::base::borrowed_fd GetSourceFd();



    std::unique_ptr<CowWriter> cow_;

    CowOptions options_;



    // Set the source device. This is used for AddCopy() operations, if the

    // underlying writer needs the original bytes (for example if backed by

    // dm-snapshot or if writing directly to an unsnapshotted region). The

    // device is only opened on the first operation that requires it.

    std::optional<std::string> source_device_;

    android::base::unique_fd source_fd_;



    android::base::unique_fd cow_device_;

    std::unique_ptr<ICowWriter> cow_;

};



}  // namespace snapshot