Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 71651543 authored by David Anderson's avatar David Anderson
Browse files

liblp: Split super.img for retrofit devices. 

On retrofit devices, we need images that can be flashed in the
bootloader for the component "super"partitions. This change rewrites
SparseBuilder so that it generates one sparse image per block device,
and now handles partitions that span across multiple block devices.

A new API function has been added to write the set of sparse images to
an output folder.

Bug: 118887112
Test: manual test, flash split images
Change-Id: Iff56efdcb7bdfd5bc8dd7ff44e8234e091ac2346
parent 76cde417

fs_mgr/liblp/images.cpp

+138 −25
Original line number Diff line number Diff line
@@ -62,7 +62,7 @@ std::unique_ptr<LpMetadata> ReadFromImageBlob(const void* data, size_t bytes) { 
}



std::unique_ptr<LpMetadata> ReadFromImageFile(const char* file) {

    android::base::unique_fd fd(open(file, O_RDONLY));

    android::base::unique_fd fd(open(file, O_RDONLY | O_CLOEXEC));

    if (fd < 0) {

        PERROR << __PRETTY_FUNCTION__ << " open failed: " << file;

        return nullptr;
@@ -84,7 +84,7 @@ bool WriteToImageFile(int fd, const LpMetadata& input) { 
}



bool WriteToImageFile(const char* file, const LpMetadata& input) {

    android::base::unique_fd fd(open(file, O_CREAT | O_RDWR | O_TRUNC, 0644));

    android::base::unique_fd fd(open(file, O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC, 0644));

    if (fd < 0) {

        PERROR << __PRETTY_FUNCTION__ << " open failed: " << file;

        return false;
@@ -97,7 +97,6 @@ SparseBuilder::SparseBuilder(const LpMetadata& metadata, uint32_t block_size, 
    : metadata_(metadata),

      geometry_(metadata.geometry),

      block_size_(block_size),

      file_(nullptr, sparse_file_destroy),

      images_(images) {

    uint64_t total_size = GetTotalSuperPartitionSize(metadata);

    if (block_size % LP_SECTOR_SIZE != 0) {
@@ -129,34 +128,72 @@ SparseBuilder::SparseBuilder(const LpMetadata& metadata, uint32_t block_size, 
        return;

    }



    file_.reset(sparse_file_new(block_size_, total_size));

    if (!file_) {

        LERROR << "Could not allocate sparse file of size " << total_size;

    for (const auto& block_device : metadata.block_devices) {

        SparsePtr file(sparse_file_new(block_size_, block_device.size), sparse_file_destroy);

        if (!file) {

            LERROR << "Could not allocate sparse file of size " << block_device.size;

            return;

        }

        device_images_.emplace_back(std::move(file));

    }

}



bool SparseBuilder::IsValid() const {

    return device_images_.size() == metadata_.block_devices.size();

}



bool SparseBuilder::Export(const char* file) {

    android::base::unique_fd fd(open(file, O_CREAT | O_RDWR | O_TRUNC, 0644));

    android::base::unique_fd fd(open(file, O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC, 0644));

    if (fd < 0) {

        PERROR << "open failed: " << file;

        return false;

    }

    if (device_images_.size() > 1) {

        LERROR << "Cannot export to a single image on retrofit builds.";

        return false;

    }

    // No gzip compression; sparseify; no checksum.

    int ret = sparse_file_write(device_images_[0].get(), fd, false, true, false);

    if (ret != 0) {

        LERROR << "sparse_file_write failed (error code " << ret << ")";

        return false;

    }

    return true;

}



bool SparseBuilder::ExportFiles(const std::string& output_dir) {

    android::base::unique_fd dir(open(output_dir.c_str(), O_CLOEXEC | O_DIRECTORY | O_NOFOLLOW));

    if (dir < 0) {

        PERROR << "open dir failed: " << output_dir;

        return false;

    }



    for (size_t i = 0; i < device_images_.size(); i++) {

        std::string name = GetBlockDevicePartitionName(metadata_.block_devices[i]);

        std::string path = output_dir + "/super_" + name + ".img";

        android::base::unique_fd fd(openat(

                dir, path.c_str(), O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0644));

        if (fd < 0) {

            PERROR << "open failed: " << path;

            return false;

        }

        // No gzip compression; sparseify; no checksum.

    int ret = sparse_file_write(file_.get(), fd, false, true, false);

        int ret = sparse_file_write(device_images_[i].get(), fd, false, true, false);

        if (ret != 0) {

            LERROR << "sparse_file_write failed (error code " << ret << ")";

            return false;

        }

    }

    return true;

}



bool SparseBuilder::AddData(const std::string& blob, uint64_t sector) {

bool SparseBuilder::AddData(sparse_file* file, const std::string& blob, uint64_t sector) {

    uint32_t block;

    if (!SectorToBlock(sector, &block)) {

        return false;

    }

    void* data = const_cast<char*>(blob.data());

    int ret = sparse_file_add_data(file_.get(), data, blob.size(), block);

    int ret = sparse_file_add_data(file, data, blob.size(), block);

    if (ret != 0) {

        LERROR << "sparse_file_add_data failed (error code " << ret << ")";

        return false;
@@ -179,8 +216,12 @@ bool SparseBuilder::SectorToBlock(uint64_t sector, uint32_t* block) { 
    return true;

}



uint64_t SparseBuilder::BlockToSector(uint64_t block) const {

    return (block * block_size_) / LP_SECTOR_SIZE;

}



bool SparseBuilder::Build() {

    if (sparse_file_add_fill(file_.get(), 0, LP_PARTITION_RESERVED_BYTES, 0) < 0) {

    if (sparse_file_add_fill(device_images_[0].get(), 0, LP_PARTITION_RESERVED_BYTES, 0) < 0) {

        LERROR << "Could not add initial sparse block for reserved zeroes";

        return false;

    }
@@ -194,7 +235,13 @@ bool SparseBuilder::Build() { 
    for (size_t i = 0; i < geometry_.metadata_slot_count * 2; i++) {

        all_metadata_ += metadata_blob;

    }

    if (!AddData(all_metadata_, 0)) {



    uint64_t first_sector = LP_PARTITION_RESERVED_BYTES / LP_SECTOR_SIZE;

    if (!AddData(device_images_[0].get(), all_metadata_, first_sector)) {

        return false;

    }



    if (!CheckExtentOrdering()) {

        return false;

    }
@@ -228,13 +275,10 @@ static inline bool HasFillValue(uint32_t* buffer, size_t count) { 


bool SparseBuilder::AddPartitionImage(const LpMetadataPartition& partition,

                                      const std::string& file) {

    if (partition.num_extents != 1) {

        LERROR << "Partition for new tables should not have more than one extent: "

               << GetPartitionName(partition);

        return false;

    }

    // Track which extent we're processing.

    uint32_t extent_index = partition.first_extent_index;



    const LpMetadataExtent& extent = metadata_.extents[partition.first_extent_index];

    const LpMetadataExtent& extent = metadata_.extents[extent_index];

    if (extent.target_type != LP_TARGET_TYPE_LINEAR) {

        LERROR << "Partition should only have linear extents: " << GetPartitionName(partition);

        return false;
@@ -252,9 +296,11 @@ bool SparseBuilder::AddPartitionImage(const LpMetadataPartition& partition, 
        LERROR << "Could not compute image size";

        return false;

    }

    if (file_length > extent.num_sectors * LP_SECTOR_SIZE) {

    uint64_t partition_size = ComputePartitionSize(partition);

    if (file_length > partition_size) {

        LERROR << "Image for partition '" << GetPartitionName(partition)

               << "' is greater than its size";

               << "' is greater than its size (" << file_length << ", excepted " << partition_size

               << ")";

        return false;

    }

    if (SeekFile64(fd, 0, SEEK_SET)) {
@@ -262,14 +308,39 @@ bool SparseBuilder::AddPartitionImage(const LpMetadataPartition& partition, 
        return false;

    }



    // We track the current logical sector and the position the current extent

    // ends at.

    uint64_t output_sector = 0;

    uint64_t extent_last_sector = extent.num_sectors;



    // We also track the output device and the current output block within that

    // device.

    uint32_t output_block;

    if (!SectorToBlock(extent.target_data, &output_block)) {

        return false;

    }

    sparse_file* output_device = device_images_[extent.target_source].get();



    // Proceed to read the file and build sparse images.

    uint64_t pos = 0;

    uint64_t remaining = file_length;

    while (remaining) {

        // Check if we need to advance to the next extent.

        if (output_sector == extent_last_sector) {

            extent_index++;

            if (extent_index >= partition.first_extent_index + partition.num_extents) {

                LERROR << "image is larger than extent table";

                return false;

            }



            const LpMetadataExtent& extent = metadata_.extents[extent_index];

            extent_last_sector += extent.num_sectors;

            output_device = device_images_[extent.target_source].get();

            if (!SectorToBlock(extent.target_data, &output_block)) {

                return false;

            }

        }



        uint32_t buffer[block_size_ / sizeof(uint32_t)];

        size_t read_size = remaining >= sizeof(buffer) ? sizeof(buffer) : size_t(remaining);

        if (!android::base::ReadFully(fd, buffer, sizeof(buffer))) {
@@ -277,13 +348,13 @@ bool SparseBuilder::AddPartitionImage(const LpMetadataPartition& partition, 
            return false;

        }

        if (read_size != sizeof(buffer) || !HasFillValue(buffer, read_size / sizeof(uint32_t))) {

            int rv = sparse_file_add_fd(file_.get(), fd, pos, read_size, output_block);

            int rv = sparse_file_add_fd(output_device, fd, pos, read_size, output_block);

            if (rv) {

                LERROR << "sparse_file_add_fd failed with code: " << rv;

                return false;

            }

        } else {

            int rv = sparse_file_add_fill(file_.get(), buffer[0], read_size, output_block);

            int rv = sparse_file_add_fill(output_device, buffer[0], read_size, output_block);

            if (rv) {

                LERROR << "sparse_file_add_fill failed with code: " << rv;

                return false;
@@ -291,21 +362,57 @@ bool SparseBuilder::AddPartitionImage(const LpMetadataPartition& partition, 
        }

        pos += read_size;

        remaining -= read_size;

        output_sector += block_size_ / LP_SECTOR_SIZE;

        output_block++;

    }



    return true;

}



uint64_t SparseBuilder::ComputePartitionSize(const LpMetadataPartition& partition) const {

    uint64_t sectors = 0;

    for (size_t i = 0; i < partition.num_extents; i++) {

        sectors += metadata_.extents[partition.first_extent_index + i].num_sectors;

    }

    return sectors * LP_SECTOR_SIZE;

}



// For simplicity, we don't allow serializing any configuration: extents must

// be ordered, such that any extent at position I in the table occurs *before*

// any extent after position I, for the same block device. We validate that

// here.

//

// Without this, it would be more difficult to find the appropriate extent for

// an output block. With this guarantee it is a linear walk.

bool SparseBuilder::CheckExtentOrdering() {

    std::vector<uint64_t> last_sectors(metadata_.block_devices.size());



    for (const auto& extent : metadata_.extents) {

        if (extent.target_type != LP_TARGET_TYPE_LINEAR) {

            LERROR << "Extents must all be type linear.";

            return false;

        }

        if (extent.target_data <= last_sectors[extent.target_source]) {

            LERROR << "Extents must appear in increasing order.";

            return false;

        }

        if ((extent.num_sectors * LP_SECTOR_SIZE) % block_size_ != 0) {

            LERROR << "Extents must be aligned to the block size.";

            return false;

        }

        last_sectors[extent.target_source] = extent.target_data;

    }

    return true;

}



int SparseBuilder::OpenImageFile(const std::string& file) {

    android::base::unique_fd source_fd(open(file.c_str(), O_RDONLY));

    android::base::unique_fd source_fd(open(file.c_str(), O_RDONLY | O_CLOEXEC));

    if (source_fd < 0) {

        PERROR << "open image file failed: " << file;

        return -1;

    }



    std::unique_ptr<sparse_file, decltype(&sparse_file_destroy)> source(

            sparse_file_import(source_fd, true, true), sparse_file_destroy);

    SparsePtr source(sparse_file_import(source_fd, true, true), sparse_file_destroy);

    if (!source) {

        int fd = source_fd.get();

        temp_fds_.push_back(std::move(source_fd));
@@ -340,5 +447,11 @@ bool WriteToSparseFile(const char* file, const LpMetadata& metadata, uint32_t bl 
    return builder.IsValid() && builder.Build() && builder.Export(file);

}



bool WriteSplitSparseFiles(const std::string& output_dir, const LpMetadata& metadata,

                           uint32_t block_size, const std::map<std::string, std::string>& images) {

    SparseBuilder builder(metadata, block_size, images);

    return builder.IsValid() && builder.Build() && builder.ExportFiles(output_dir);

}



}  // namespace fs_mgr

}  // namespace android

fs_mgr/liblp/images.h

+10 −4
Original line number Diff line number Diff line
@@ -42,20 +42,26 @@ class SparseBuilder { 


    bool Build();

    bool Export(const char* file);

    bool IsValid() const { return file_ != nullptr; }

    bool ExportFiles(const std::string& dir);

    bool IsValid() const;



    sparse_file* file() const { return file_.get(); }

    using SparsePtr = std::unique_ptr<sparse_file, decltype(&sparse_file_destroy)>;

    const std::vector<SparsePtr>& device_images() const { return device_images_; }



  private:

    bool AddData(const std::string& blob, uint64_t sector);

    bool AddData(sparse_file* file, const std::string& blob, uint64_t sector);

    bool AddPartitionImage(const LpMetadataPartition& partition, const std::string& file);

    int OpenImageFile(const std::string& file);

    bool SectorToBlock(uint64_t sector, uint32_t* block);

    uint64_t BlockToSector(uint64_t block) const;

    bool CheckExtentOrdering();

    uint64_t ComputePartitionSize(const LpMetadataPartition& partition) const;



    const LpMetadata& metadata_;

    const LpMetadataGeometry& geometry_;

    uint32_t block_size_;

    std::unique_ptr<sparse_file, decltype(&sparse_file_destroy)> file_;



    std::vector<SparsePtr> device_images_;

    std::string all_metadata_;

    std::map<std::string, std::string> images_;

    std::vector<android::base::unique_fd> temp_fds_;

fs_mgr/liblp/include/liblp/liblp.h

+8 −0
Original line number Diff line number Diff line
@@ -78,6 +78,14 @@ bool WriteToImageFile(const char* file, const LpMetadata& metadata); 
std::unique_ptr<LpMetadata> ReadFromImageFile(const char* file);

std::unique_ptr<LpMetadata> ReadFromImageBlob(const void* data, size_t bytes);



// Similar to WriteToSparseFile, this will generate an image that can be

// flashed to a device directly. However unlike WriteToSparseFile, it

// is intended for retrofit devices, and will generate one sparse file per

// block device (each named super_<name>.img) and placed in the specified

// output folder.

bool WriteSplitSparseFiles(const std::string& output_dir, const LpMetadata& metadata,

                           uint32_t block_size, const std::map<std::string, std::string>& images);



// Helper to extract safe C++ strings from partition info.

std::string GetPartitionName(const LpMetadataPartition& partition);

std::string GetPartitionGroupName(const LpMetadataPartitionGroup& group);

fs_mgr/liblp/io_test.cpp

+4 −2
Original line number Diff line number Diff line
@@ -596,12 +596,14 @@ TEST(liblp, FlashSparseImage) { 
    // Build the sparse file.

    SparseBuilder sparse(*exported.get(), 512, {});

    ASSERT_TRUE(sparse.IsValid());

    sparse_file_verbose(sparse.file());

    ASSERT_TRUE(sparse.Build());



    const auto& images = sparse.device_images();

    ASSERT_EQ(images.size(), static_cast<size_t>(1));



    // Write it to the fake disk.

    ASSERT_NE(lseek(fd.get(), 0, SEEK_SET), -1);

    int ret = sparse_file_write(sparse.file(), fd.get(), false, false, false);

    int ret = sparse_file_write(images[0].get(), fd.get(), false, false, false);

    ASSERT_EQ(ret, 0);



    // Verify that we can read both sets of metadata.