Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b0a29ad8 authored by Tianjie Xu's avatar Tianjie Xu
Browse files

Generate brotli compressed {}.new.dat for full OTA 

Brotli has a better compression ratio than deflation. So for non-AB
full OTA, we can compress the new.dat with brotli first and store it
in the zip package.
This increase the OTA generation time by ~5 minutes for a full OTA
(measured based on 2.0G system.new.dat)

Bug: 34220646
Test: Generate a full OTA for bullhead
Change-Id: I9c0550af8eafcfa68711f74290c8e2d05a96648f
parent bc45a33c

core/Makefile

+7 −2
Original line number Original line Diff line number Diff line
@@ -2007,7 +2007,8 @@ OTATOOLS := $(HOST_OUT_EXECUTABLES)/minigzip \ 
  $(HOST_OUT_EXECUTABLES)/lib/shflags/shflags \

  $(HOST_OUT_EXECUTABLES)/lib/shflags/shflags \

  $(HOST_OUT_EXECUTABLES)/delta_generator \

  $(HOST_OUT_EXECUTABLES)/delta_generator \

  $(AVBTOOL) \

  $(AVBTOOL) \

  $(BLK_ALLOC_TO_BASE_FS)

  $(BLK_ALLOC_TO_BASE_FS) \

  $(BRO)





ifeq (true,$(PRODUCTS.$(INTERNAL_PRODUCT).PRODUCT_SUPPORTS_VBOOT))

ifeq (true,$(PRODUCTS.$(INTERNAL_PRODUCT).PRODUCT_SUPPORTS_VBOOT))

OTATOOLS += \

OTATOOLS += \
@@ -2041,7 +2042,9 @@ OTATOOLS += \ 
  $(HOST_LIBRARY_PATH)/libz-host$(HOST_SHLIB_SUFFIX) \

  $(HOST_LIBRARY_PATH)/libz-host$(HOST_SHLIB_SUFFIX) \

  $(HOST_LIBRARY_PATH)/libsparse-host$(HOST_SHLIB_SUFFIX) \

  $(HOST_LIBRARY_PATH)/libsparse-host$(HOST_SHLIB_SUFFIX) \

  $(HOST_LIBRARY_PATH)/libbase$(HOST_SHLIB_SUFFIX) \

  $(HOST_LIBRARY_PATH)/libbase$(HOST_SHLIB_SUFFIX) \

  $(HOST_LIBRARY_PATH)/libpcre2$(HOST_SHLIB_SUFFIX)

  $(HOST_LIBRARY_PATH)/libpcre2$(HOST_SHLIB_SUFFIX) \

  $(HOST_LIBRARY_PATH)/libbrotli$(HOST_SHLIB_SUFFIX)







.PHONY: otatools

.PHONY: otatools

otatools: $(OTATOOLS)

otatools: $(OTATOOLS)
@@ -2475,6 +2478,8 @@ $(INTERNAL_OTA_PACKAGE_TARGET): KEY_CERT_PAIR := $(DEFAULT_KEY_CERT_PAIR) 




ifeq ($(AB_OTA_UPDATER),true)

ifeq ($(AB_OTA_UPDATER),true)

$(INTERNAL_OTA_PACKAGE_TARGET): $(BRILLO_UPDATE_PAYLOAD)

$(INTERNAL_OTA_PACKAGE_TARGET): $(BRILLO_UPDATE_PAYLOAD)

else

$(INTERNAL_OTA_PACKAGE_TARGET): $(BRO)

endif

endif





$(INTERNAL_OTA_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE) \

$(INTERNAL_OTA_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE) \

core/config.mk

+1 −0
Original line number Original line Diff line number Diff line
@@ -592,6 +592,7 @@ NANOPB_SRCS := external/nanopb-c/generator/protoc-gen-nanopb \ 
VTSC := $(HOST_OUT_EXECUTABLES)/vtsc$(HOST_EXECUTABLE_SUFFIX)

VTSC := $(HOST_OUT_EXECUTABLES)/vtsc$(HOST_EXECUTABLE_SUFFIX)

MKBOOTFS := $(HOST_OUT_EXECUTABLES)/mkbootfs$(HOST_EXECUTABLE_SUFFIX)

MKBOOTFS := $(HOST_OUT_EXECUTABLES)/mkbootfs$(HOST_EXECUTABLE_SUFFIX)

MINIGZIP := $(HOST_OUT_EXECUTABLES)/minigzip$(HOST_EXECUTABLE_SUFFIX)

MINIGZIP := $(HOST_OUT_EXECUTABLES)/minigzip$(HOST_EXECUTABLE_SUFFIX)

BRO := $(HOST_OUT_EXECUTABLES)/bro$(HOST_EXECUTABLE_SUFFIX)

ifeq (,$(strip $(BOARD_CUSTOM_MKBOOTIMG)))

ifeq (,$(strip $(BOARD_CUSTOM_MKBOOTIMG)))

MKBOOTIMG := $(HOST_OUT_EXECUTABLES)/mkbootimg$(HOST_EXECUTABLE_SUFFIX)

MKBOOTIMG := $(HOST_OUT_EXECUTABLES)/mkbootimg$(HOST_EXECUTABLE_SUFFIX)

else

else

tools/releasetools/common.py

+31 −5
Original line number Original line Diff line number Diff line
@@ -1515,9 +1515,34 @@ class BlockDifference(object): 
    ZipWrite(output_zip,

    ZipWrite(output_zip,

             '{}.transfer.list'.format(self.path),

             '{}.transfer.list'.format(self.path),

             '{}.transfer.list'.format(self.partition))

             '{}.transfer.list'.format(self.partition))



    # For full OTA, compress the new.dat with brotli with quality 6 to reduce its size. Quailty 9

    # almost triples the compression time but doesn't further reduce the size too much.

    # For a typical 1.8G system.new.dat

    #                       zip  | brotli(quality 6)  | brotli(quality 9)

    #   compressed_size:    942M | 869M (~8% reduced) | 854M

    #   compression_time:   75s  | 265s               | 719s

    #   decompression_time: 15s  | 25s                | 25s



    if not self.src:

      bro_cmd = ['bro', '--quality', '6',

                 '--input', '{}.new.dat'.format(self.path),

                 '--output', '{}.new.dat.br'.format(self.path)]

      print("Compressing {}.new.dat with brotli".format(self.partition))

      p = Run(bro_cmd, stdout=subprocess.PIPE)

      p.communicate()

      assert p.returncode == 0,\

          'compression of {}.new.dat failed'.format(self.partition)



      new_data_name = '{}.new.dat.br'.format(self.partition)

      ZipWrite(output_zip,

      ZipWrite(output_zip,

             '{}.new.dat'.format(self.path),

               '{}.new.dat.br'.format(self.path),

             '{}.new.dat'.format(self.partition))

               new_data_name,

               compress_type=zipfile.ZIP_STORED)

    else:

      new_data_name = '{}.new.dat'.format(self.partition)

      ZipWrite(output_zip, '{}.new.dat'.format(self.path), new_data_name)



    ZipWrite(output_zip,

    ZipWrite(output_zip,

             '{}.patch.dat'.format(self.path),

             '{}.patch.dat'.format(self.path),

             '{}.patch.dat'.format(self.partition),

             '{}.patch.dat'.format(self.partition),
@@ -1530,9 +1555,10 @@ class BlockDifference(object): 




    call = ('block_image_update("{device}", '

    call = ('block_image_update("{device}", '

            'package_extract_file("{partition}.transfer.list"), '

            'package_extract_file("{partition}.transfer.list"), '

            '"{partition}.new.dat", "{partition}.patch.dat") ||\n'

            '"{new_data_name}", "{partition}.patch.dat") ||\n'

            '  abort("E{code}: Failed to update {partition} image.");'.format(

            '  abort("E{code}: Failed to update {partition} image.");'.format(

                device=self.device, partition=self.partition, code=code))

                device=self.device, partition=self.partition,

                new_data_name=new_data_name, code=code))

    script.AppendExtra(script.WordWrap(call))

    script.AppendExtra(script.WordWrap(call))





  def _HashBlocks(self, source, ranges): # pylint: disable=no-self-use

  def _HashBlocks(self, source, ranges): # pylint: disable=no-self-use