Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 3a0a1cfa authored by Steve Kondik's avatar Steve Kondik Committed by Brint E. Kriebel
Browse files

releasetools: Add support for LZMA in blockimgdiff 

 * Requires backports.lzma to be installed for Python2.
 * To enable, set WITH_LZMA_OTA to true in the build environment.
 * This results in significantly smaller OTA packages, which results
   in less $$$ being paid to deliver said OTAs. Unfortunately it is
   also significantly slower to decompress (parallelization may help).

Change-Id: If4b8092718aa6623cfff101030eabd24cde8763c
(cherry picked from commit e78b239c)
Ticket: CYNGNOS-326
parent 2283792f

core/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -1690,6 +1690,7 @@ $(INTERNAL_OTA_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE) $(DISTTOOLS) 
	$(hide) MKBOOTIMG=$(MKBOOTIMG) \

	   $(OTA_FROM_TARGET_SCRIPT) -v \

	   --block \

	   $(if $(WITH_LZMA_OTA), -z) \

	   -p $(HOST_OUT) \

	   -k $(KEY_CERT_PAIR) \

	   --backup=$(backuptool) \

tools/releasetools/blockimgdiff.py

+17 −2
Original line number Diff line number Diff line
@@ -27,6 +27,12 @@ import sys 
import threading

import tempfile



try:

  from backports import lzma

except ImportError:

  lzma = None

  pass



from rangelib import *



__all__ = ["EmptyImage", "DataImage", "BlockImageDiff"]
@@ -190,12 +196,13 @@ class Transfer(object): 
# original image.



class BlockImageDiff(object):

  def __init__(self, tgt, src=None, threads=None, version=2):

  def __init__(self, tgt, src=None, threads=None, version=2, use_lzma=False):

    if threads is None:

      threads = multiprocessing.cpu_count() // 2

      if threads == 0: threads = 1

    self.threads = threads

    self.version = version

    self.use_lzma = use_lzma



    assert version in (1, 2)
@@ -409,7 +416,15 @@ class BlockImageDiff(object): 
    print("Reticulating splines...")

    diff_q = []

    patch_num = 0

    with open(prefix + ".new.dat", "wb") as new_f:



    if lzma and self.use_lzma:

        open_patch = lzma.open

        new_file = ".new.dat.xz"

    else:

        open_patch = open

        new_file = ".new.dat"



    with open_patch(prefix + new_file, "wb") as new_f:

      for xf in self.transfers:

        if xf.style == "zero":

          pass

tools/releasetools/common.py

+20 −6
Original line number Diff line number Diff line
@@ -29,6 +29,11 @@ import threading 
import time

import zipfile



try:

  from backports import lzma;

except ImportError:

  lzma = None



import blockimgdiff

from rangelib import *
@@ -1100,11 +1105,12 @@ def ComputeDifferences(diffs): 




class BlockDifference:

  def __init__(self, partition, tgt, src=None, check_first_block=False):

  def __init__(self, partition, tgt, src=None, check_first_block=False, use_lzma=False):

    self.tgt = tgt

    self.src = src

    self.partition = partition

    self.check_first_block = check_first_block

    self.use_lzma = use_lzma



    version = 1

    if OPTIONS.info_dict:
@@ -1113,7 +1119,7 @@ class BlockDifference: 
          OPTIONS.info_dict.get("blockimgdiff_versions", "1").split(","))



    b = blockimgdiff.BlockImageDiff(tgt, src, threads=OPTIONS.worker_threads,

                                    version=version)

                                    version=version, use_lzma=use_lzma)

    tmpdir = tempfile.mkdtemp()

    OPTIONS.tempfiles.append(tmpdir)

    self.path = os.path.join(tmpdir, partition)
@@ -1151,18 +1157,26 @@ class BlockDifference: 


  def _WriteUpdate(self, script, output_zip):

    partition = self.partition

    suffix = ".new.dat"



    with open(self.path + ".transfer.list", "rb") as f:

      ZipWriteStr(output_zip, partition + ".transfer.list", f.read())

    with open(self.path + ".new.dat", "rb") as f:

      ZipWriteStr(output_zip, partition + ".new.dat", f.read())

    if lzma and self.use_lzma:

      suffix += ".xz"

      with open(self.path + suffix, "rb") as f:

        ZipWriteStr(output_zip, partition + suffix, f.read(),

                           compression=zipfile.ZIP_STORED)

    else:

      with open(self.path + suffix, "rb") as f:

        ZipWriteStr(output_zip, partition + suffix, f.read())

    with open(self.path + ".patch.dat", "rb") as f:

      ZipWriteStr(output_zip, partition + ".patch.dat", f.read(),

                         compression=zipfile.ZIP_STORED)



    call = (('block_image_update("%s", '

             'package_extract_file("%s.transfer.list"), '

             '"%s.new.dat", "%s.patch.dat");\n') %

            (self.device, partition, partition, partition))

             '"%s%s", "%s.patch.dat");\n') %

            (self.device, partition, partition, suffix, partition))

    script.AppendExtra(script._WordWrap(call))



  def _CheckFirstBlock(self, script):

tools/releasetools/ota_from_target_files

+16 −6
Original line number Diff line number Diff line
@@ -70,6 +70,10 @@ Usage: ota_from_target_files [flags] input_target_files output_ota_package 
      file-based OTA if the target_files is older and doesn't support

      block-based OTAs.



  -z  Compress the block-based image using LZMA. Results in substantial

      space reduction at the cost of longer compress/decompress time.

      Requires the "backports.lzma" module to be installed.



  -b  (--binary)  <file>

      Use the given binary as the update-binary in the output package,

      instead of the binary in the build's target_files.  Use for
@@ -139,6 +143,7 @@ OPTIONS.fallback_to_full = True 
OPTIONS.backuptool = False

OPTIONS.override_device = 'auto'

OPTIONS.override_prop = False

OPTIONS.use_lzma = False



def MostPopularKey(d, default):

  """Given a dict, return the key corresponding to the largest
@@ -616,7 +621,7 @@ else if get_stage("%(bcb_dev)s") == "3/3" then 
    # writes incrementals to do it.

    system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict)

    system_tgt.ResetFileMap()

    system_diff = common.BlockDifference("system", system_tgt, src=None)

    system_diff = common.BlockDifference("system", system_tgt, src=None, use_lzma=OPTIONS.use_lzma)

    system_diff.WriteScript(script, output_zip)

  else:

    script.FormatPartition("/system")
@@ -648,7 +653,7 @@ else if get_stage("%(bcb_dev)s") == "3/3" then 
    if block_based:

      vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict)

      vendor_tgt.ResetFileMap()

      vendor_diff = common.BlockDifference("vendor", vendor_tgt)

      vendor_diff = common.BlockDifference("vendor", vendor_tgt, use_lzma=OPTIONS.use_lzma)

      vendor_diff.WriteScript(script, output_zip)

    else:

      script.FormatPartition("/vendor")
@@ -812,7 +817,7 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 
  system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict)

  system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict)

  system_diff = common.BlockDifference("system", system_tgt, system_src,

                                       check_first_block=True)

                                       check_first_block=True, use_lzma=OPTIONS.use_lzma)



  if HasVendorPartition(target_zip):

    if not HasVendorPartition(source_zip):
@@ -820,7 +825,7 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): 
    vendor_src = GetImage("vendor", OPTIONS.source_tmp, OPTIONS.source_info_dict)

    vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, OPTIONS.target_info_dict)

    vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src,

                                         check_first_block=True)

                                         check_first_block=True, use_lzma=OPTIONS.use_lzma)

  else:

    vendor_diff = None
@@ -1517,12 +1522,16 @@ def main(argv): 
      OPTIONS.override_device = a

    elif o in ("--override_prop"):

      OPTIONS.override_prop = bool(a.lower() == 'true')

    elif o in ("-z", "--use_lzma"):

      OPTIONS.use_lzma = True

      # Import now, and bomb out if backports.lzma isn't installed

      from backports import lzma

    else:

      return False

    return True



  args = common.ParseOptions(argv, __doc__,

                             extra_opts="b:k:i:d:wne:t:a:2o:",

                             extra_opts="b:k:i:d:wne:t:a:2o:z",

                             extra_long_opts=["board_config=",

                                              "package_key=",

                                              "incremental_from=",
@@ -1540,7 +1549,8 @@ def main(argv): 
                                              "no_fallback_to_full",

                                              "backup=",

                                              "override_device=",

                                              "override_prop="],

                                              "override_prop=",

                                              "use_lzma"],

                             extra_option_handler=option_handler)



  if len(args) != 2: