Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 874fdaed authored by David Anderson's avatar David Anderson
Browse files

libsnapshot: Add CowWriterBase, clean up CowWriter. 

To support multiple implementations of CowWriter, we will need to move
direct usage of CowWriter to ICowWriter. This CL does this while also
adding some small cleanups:

- Move ICowWriter implementation methods to a new CowWriterBase class.
  This keeps ICowWriter as a clean interface.
- Make the "Add" methods pure virtual, move the "Emit" methods to
  CowWriterBase as an implementation detail.
- Simplify Initialize/InitializeAppend so they can be shared.
- Rename CowWriter to CowWriterV2.
- Rename cow_writer.cpp to writer_v2.cpp.
- Rename cow_api_test.cpp to test_v2.cpp.
- Remove ICowWriter::options, replace with GetBlockSize.
- Add a CreateCowWriter helper to avoid implementation details in
  update_engine.

Bug: 280529365
Test: builds
Change-Id: If50faf03b292c6c8b23a6170e3f37329fb759ff6
parent aa453194

fs_mgr/libsnapshot/Android.bp

+5 −4
Original line number Diff line number Diff line
@@ -174,12 +174,13 @@ cc_library_static { 
        "libsnapshot_cow_defaults",

    ],

    srcs: [

        "libsnapshot_cow/cow_compress.cpp",

        "libsnapshot_cow/cow_decompress.cpp",

        "libsnapshot_cow/cow_reader.cpp",

        "libsnapshot_cow/cow_writer.cpp",

        "libsnapshot_cow/cow_format.cpp",

        "libsnapshot_cow/cow_compress.cpp",

        "libsnapshot_cow/cow_reader.cpp",

        "libsnapshot_cow/parser_v2.cpp",

        "libsnapshot_cow/writer_base.cpp",

        "libsnapshot_cow/writer_v2.cpp",

    ],

    host_supported: true,

    recovery_available: true,
@@ -371,7 +372,7 @@ cc_test { 
        "libsnapshot_cow_defaults",

    ],

    srcs: [

        "libsnapshot_cow/cow_api_test.cpp",

        "libsnapshot_cow/test_v2.cpp",

    ],

    cflags: [

        "-D_FILE_OFFSET_BITS=64",

fs_mgr/libsnapshot/include/libsnapshot/cow_format.h

+4 −0
Original line number Diff line number Diff line
@@ -31,6 +31,10 @@ static constexpr uint32_t kCowVersionManifest = 2; 
static constexpr uint32_t kMinCowVersion = 1;

static constexpr uint32_t kMaxCowVersion = 2;



// Normally, this should be kMaxCowVersion. When a new version is under testing

// it may be the previous value of kMaxCowVersion.

static constexpr uint32_t kDefaultCowVersion = 2;



// This header appears as the first sequence of bytes in the COW. All fields

// in the layout are little-endian encoded. The on-disk layout is:

//

fs_mgr/libsnapshot/include/libsnapshot/cow_writer.h

+27 −123
Original line number Diff line number Diff line 
// Copyright (C) 2019 The Android Open Source Project

// copyright (c) 2019 the android open source project

//

// Licensed under the Apache License, Version 2.0 (the "License");

// you may not use this file except in compliance with the License.

// You may obtain a copy of the License at

// licensed under the apache license, version 2.0 (the "license");

// you may not use this file except in compliance with the license.

// you may obtain a copy of the license at

//

//      http://www.apache.org/licenses/LICENSE-2.0

//      http://www.apache.org/licenses/license-2.0

//

// Unless required by applicable law or agreed to in writing, software

// distributed under the License is distributed on an "AS IS" BASIS,

// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

// See the License for the specific language governing permissions and

// limitations under the License.

// unless required by applicable law or agreed to in writing, software

// distributed under the license is distributed on an "as is" basis,

// without warranties or conditions of any kind, either express or implied.

// see the license for the specific language governing permissions and

// limitations under the license.



#pragma once
@@ -61,30 +61,28 @@ struct CowOptions { 
// will occur in the sequence they were added to the COW.

class ICowWriter {

  public:

    explicit ICowWriter(const CowOptions& options) : options_(options) {}



    virtual ~ICowWriter() {}



    // Encode an operation that copies the contents of |old_block| to the

    // location of |new_block|. 'num_blocks' is the number of contiguous

    // COPY operations from |old_block| to |new_block|.

    bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1);

    virtual bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0;



    // Encode a sequence of raw blocks. |size| must be a multiple of the block size.

    bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size);

    virtual bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0;



    // Add a sequence of xor'd blocks. |size| must be a multiple of the block size.

    bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block,

                      uint16_t offset);

    virtual bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size,

                              uint32_t old_block, uint16_t offset) = 0;



    // Encode a sequence of zeroed blocks. |size| must be a multiple of the block size.

    bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks);

    virtual bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0;



    // Add a label to the op sequence.

    bool AddLabel(uint64_t label);

    virtual bool AddLabel(uint64_t label) = 0;



    // Add sequence data for op merging. Data is a list of the destination block numbers.

    bool AddSequenceData(size_t num_ops, const uint32_t* data);

    virtual bool AddSequenceData(size_t num_ops, const uint32_t* data) = 0;



    // Flush all pending writes. This must be called before closing the writer

    // to ensure that the correct headers and footers are written.
@@ -93,21 +91,8 @@ class ICowWriter { 
    // Return number of bytes the cow image occupies on disk.

    virtual uint64_t GetCowSize() = 0;



    const CowOptions& options() { return options_; }



  protected:

    virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) = 0;

    virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) = 0;

    virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,

                               uint32_t old_block, uint16_t offset) = 0;

    virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) = 0;

    virtual bool EmitLabel(uint64_t label) = 0;

    virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) = 0;



    bool ValidateNewBlock(uint64_t new_block);



  protected:

    CowOptions options_;

    virtual uint32_t GetBlockSize() const = 0;

    virtual std::optional<uint32_t> GetMaxBlocks() const = 0;

};



class CompressWorker {
@@ -146,96 +131,15 @@ class CompressWorker { 
                        std::vector<std::basic_string<uint8_t>>* compressed_data);

};



class CowWriter : public ICowWriter {

  public:

    explicit CowWriter(const CowOptions& options);

    ~CowWriter();

// Create an ICowWriter not backed by any file. This is useful for estimating

// the final size of a cow file.

std::unique_ptr<ICowWriter> CreateCowEstimator(uint32_t version, const CowOptions& options);



    // Set up the writer.

    // The file starts from the beginning.

    //

    // If fd is < 0, the CowWriter will be opened against /dev/null. This is for

    // computing COW sizes without using storage space.

    bool Initialize(android::base::unique_fd&& fd);

    bool Initialize(android::base::borrowed_fd fd);

    // Set up a writer, assuming that the given label is the last valid label.

    // This will result in dropping any labels that occur after the given on, and will fail

    // if the given label does not appear.

    bool InitializeAppend(android::base::unique_fd&&, uint64_t label);

    bool InitializeAppend(android::base::borrowed_fd fd, uint64_t label);



    bool Finalize() override;



    uint64_t GetCowSize() override;



  protected:

    virtual bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;

    virtual bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;

    virtual bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size,

                               uint32_t old_block, uint16_t offset) override;

    virtual bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;

    virtual bool EmitLabel(uint64_t label) override;

    virtual bool EmitSequenceData(size_t num_ops, const uint32_t* data) override;



  private:

    bool EmitCluster();

    bool EmitClusterIfNeeded();

    bool EmitBlocks(uint64_t new_block_start, const void* data, size_t size, uint64_t old_block,

                    uint16_t offset, uint8_t type);

    void SetupHeaders();

    void SetupWriteOptions();

    bool ParseOptions();

    bool OpenForWrite();

    bool OpenForAppend(uint64_t label);

    bool GetDataPos(uint64_t* pos);

    bool WriteRawData(const void* data, size_t size);

    bool WriteOperation(const CowOperation& op, const void* data = nullptr, size_t size = 0);

    void AddOperation(const CowOperation& op);

    void InitPos();

    void InitBatchWrites();

    void InitWorkers();

    bool FlushCluster();



    bool CompressBlocks(size_t num_blocks, const void* data);

    bool SetFd(android::base::borrowed_fd fd);

    bool Sync();

    bool Truncate(off_t length);

    bool EnsureSpaceAvailable(const uint64_t bytes_needed) const;



  private:

    android::base::unique_fd owned_fd_;

    android::base::borrowed_fd fd_;

    CowHeader header_{};

    CowFooter footer_{};

    CowCompressionAlgorithm compression_ = kCowCompressNone;

    uint64_t current_op_pos_ = 0;

    uint64_t next_op_pos_ = 0;

    uint64_t next_data_pos_ = 0;

    uint64_t current_data_pos_ = 0;

    ssize_t total_data_written_ = 0;

    uint32_t cluster_size_ = 0;

    uint32_t current_cluster_size_ = 0;

    uint64_t current_data_size_ = 0;

    bool is_dev_null_ = false;

    bool merge_in_progress_ = false;

    bool is_block_device_ = false;

    uint64_t cow_image_size_ = INT64_MAX;



    int num_compress_threads_ = 1;

    std::vector<std::unique_ptr<CompressWorker>> compress_threads_;

    std::vector<std::future<bool>> threads_;

    std::vector<std::basic_string<uint8_t>> compressed_buf_;

    std::vector<std::basic_string<uint8_t>>::iterator buf_iter_;



    std::vector<std::unique_ptr<CowOperation>> opbuffer_vec_;

    std::vector<std::unique_ptr<uint8_t[]>> databuffer_vec_;

    std::unique_ptr<struct iovec[]> cowop_vec_;

    int op_vec_index_ = 0;



    std::unique_ptr<struct iovec[]> data_vec_;

    int data_vec_index_ = 0;

    bool batch_write_ = false;

};

// Create an ICowWriter of the given version and options. If a label is given,

// the writer is opened in append mode.

std::unique_ptr<ICowWriter> CreateCowWriter(uint32_t version, const CowOptions& options,

                                            android::base::unique_fd&& fd,

                                            std::optional<uint64_t> label = {});



}  // namespace snapshot

}  // namespace android

fs_mgr/libsnapshot/include/libsnapshot/mock_snapshot_writer.h

+9 −16
Original line number Diff line number Diff line
@@ -23,30 +23,23 @@ class MockSnapshotWriter : public ISnapshotWriter { 
  public:

    using FileDescriptor = ISnapshotWriter::FileDescriptor;



    explicit MockSnapshotWriter(const CowOptions& options) : ISnapshotWriter(options) {}

    MockSnapshotWriter() : ISnapshotWriter({}) {}



    MOCK_METHOD(bool, Finalize, (), (override));



    // Return number of bytes the cow image occupies on disk.

    MOCK_METHOD(uint64_t, GetCowSize, (), (override));



    MOCK_METHOD(bool, EmitCopy, (uint64_t, uint64_t, uint64_t), (override));

    MOCK_METHOD(bool, EmitRawBlocks, (uint64_t, const void*, size_t), (override));

    MOCK_METHOD(bool, EmitXorBlocks, (uint32_t, const void*, size_t, uint32_t, uint16_t),

    MOCK_METHOD(bool, AddCopy, (uint64_t, uint64_t, uint64_t), (override));

    MOCK_METHOD(bool, AddRawBlocks, (uint64_t, const void*, size_t), (override));

    MOCK_METHOD(bool, AddXorBlocks, (uint32_t, const void*, size_t, uint32_t, uint16_t),

                (override));

    MOCK_METHOD(bool, EmitZeroBlocks, (uint64_t, uint64_t), (override));

    MOCK_METHOD(bool, EmitLabel, (uint64_t), (override));

    MOCK_METHOD(bool, EmitSequenceData, (size_t, const uint32_t*), (override));



    // Open the writer in write mode (no append).

    MOCK_METHOD(bool, AddZeroBlocks, (uint64_t, uint64_t), (override));

    MOCK_METHOD(bool, AddLabel, (uint64_t), (override));

    MOCK_METHOD(bool, AddSequenceData, (size_t, const uint32_t*), (override));

    MOCK_METHOD(bool, Initialize, (), (override));

    MOCK_METHOD(bool, InitializeAppend, (uint64_t), (override));

    MOCK_METHOD(bool, VerifyMergeOps, (), (override, const, noexcept));



    // Open the writer in append mode, with the last label to resume

    // from. See CowWriter::InitializeAppend.

    MOCK_METHOD(bool, InitializeAppend, (uint64_t label), (override));



    MOCK_METHOD(std::unique_ptr<FileDescriptor>, OpenReader, (), (override));

    MOCK_METHOD(uint32_t, GetBlockSize, (), (override, const));

    MOCK_METHOD(std::optional<uint32_t>, GetMaxBlocks, (), (override, const));

};

}  // namespace android::snapshot

fs_mgr/libsnapshot/include/libsnapshot/snapshot_writer.h

+25 −25
Original line number Diff line number Diff line
@@ -31,13 +31,7 @@ class ISnapshotWriter : public ICowWriter { 
  public:

    using FileDescriptor = chromeos_update_engine::FileDescriptor;



    explicit ISnapshotWriter(const CowOptions& options);



    // Set the source device. This is used for AddCopy() operations, if the

    // underlying writer needs the original bytes (for example if backed by

    // dm-snapshot or if writing directly to an unsnapshotted region). The

    // device is only opened on the first operation that requires it.

    void SetSourceDevice(const std::string& source_device);

    virtual ~ISnapshotWriter() {}



    // Open the writer in write mode (no append).

    virtual bool Initialize() = 0;
@@ -47,15 +41,8 @@ class ISnapshotWriter : public ICowWriter { 
    virtual bool InitializeAppend(uint64_t label) = 0;



    virtual std::unique_ptr<FileDescriptor> OpenReader() = 0;

    virtual bool VerifyMergeOps() const noexcept = 0;



  protected:

    android::base::borrowed_fd GetSourceFd();



    std::optional<std::string> source_device_;



  private:

    android::base::unique_fd source_fd_;

    virtual bool VerifyMergeOps() const noexcept = 0;

};



// Send writes to a COW or a raw device directly, based on a threshold.
@@ -63,6 +50,8 @@ class CompressedSnapshotWriter final : public ISnapshotWriter { 
  public:

    CompressedSnapshotWriter(const CowOptions& options);



    void SetSourceDevice(const std::string& source_device);



    // Sets the COW device; this is required.

    bool SetCowDevice(android::base::unique_fd&& cow_device);
@@ -70,23 +59,34 @@ class CompressedSnapshotWriter final : public ISnapshotWriter { 
    bool InitializeAppend(uint64_t label) override;

    bool Finalize() override;

    uint64_t GetCowSize() override;

    uint32_t GetBlockSize() const override;

    std::optional<uint32_t> GetMaxBlocks() const override;

    std::unique_ptr<FileDescriptor> OpenReader() override;

    bool VerifyMergeOps() const noexcept;



  protected:

    bool EmitCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;

    bool EmitRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;

    bool EmitXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block,

    bool AddCopy(uint64_t new_block, uint64_t old_block, uint64_t num_blocks = 1) override;

    bool AddRawBlocks(uint64_t new_block_start, const void* data, size_t size) override;

    bool AddXorBlocks(uint32_t new_block_start, const void* data, size_t size, uint32_t old_block,

                      uint16_t offset) override;

    bool EmitZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;

    bool EmitLabel(uint64_t label) override;

    bool EmitSequenceData(size_t num_ops, const uint32_t* data) override;

    bool AddZeroBlocks(uint64_t new_block_start, uint64_t num_blocks) override;

    bool AddLabel(uint64_t label) override;

    bool AddSequenceData(size_t num_ops, const uint32_t* data) override;



  private:

    std::unique_ptr<CowReader> OpenCowReader() const;

    android::base::unique_fd cow_device_;

    android::base::borrowed_fd GetSourceFd();



    std::unique_ptr<CowWriter> cow_;

    CowOptions options_;



    // Set the source device. This is used for AddCopy() operations, if the

    // underlying writer needs the original bytes (for example if backed by

    // dm-snapshot or if writing directly to an unsnapshotted region). The

    // device is only opened on the first operation that requires it.

    std::optional<std::string> source_device_;

    android::base::unique_fd source_fd_;



    android::base::unique_fd cow_device_;

    std::unique_ptr<ICowWriter> cow_;

};



}  // namespace snapshot