Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f3282b4a authored by Tao Bao's avatar Tao Bao
Browse files

Wrap zipfile.write(), writestr() and close() 

In order to work around the zip 2GiB limit, we need to wrap the related
functions in zipfile. Calls to those functions should always be replaced
with calls to the wrappers instead.

Bug: 18015246
Change-Id: I499574cee51ec4804bc10cbefe0b17940afed918
(cherry picked from commit 2ed665a0)
parent 821a554c

tools/releasetools/add_img_to_target_files.py

+4 −8
Original line number Diff line number Diff line
@@ -33,10 +33,6 @@ import os 
import tempfile

import zipfile



# missing in Python 2.4 and before

if not hasattr(os, "SEEK_SET"):

  os.SEEK_SET = 0



import build_image

import common
@@ -189,7 +185,7 @@ def AddUserdata(output_zip, prefix="IMAGES/"): 
  assert succ, "build userdata.img image failed"



  common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict)

  output_zip.write(img.name, prefix + "userdata.img")

  common.ZipWrite(output_zip, img.name, prefix + "userdata.img")

  img.close()

  os.rmdir(user_dir)

  os.rmdir(temp_dir)
@@ -226,7 +222,7 @@ def AddCache(output_zip, prefix="IMAGES/"): 
  assert succ, "build cache.img image failed"



  common.CheckSize(img.name, "cache.img", OPTIONS.info_dict)

  output_zip.write(img.name, prefix + "cache.img")

  common.ZipWrite(output_zip, img.name, prefix + "cache.img")

  img.close()

  os.rmdir(user_dir)

  os.rmdir(temp_dir)
@@ -252,7 +248,7 @@ def AddImagesToTargetFiles(filename): 
    OPTIONS.info_dict["selinux_fc"] = os.path.join(

        OPTIONS.input_tmp, "BOOT", "RAMDISK", "file_contexts")



  input_zip.close()

  common.ZipClose(input_zip)

  output_zip = zipfile.ZipFile(filename, "a",

                               compression=zipfile.ZIP_DEFLATED)
@@ -297,7 +293,7 @@ def AddImagesToTargetFiles(filename): 
  banner("cache")

  AddCache(output_zip)



  output_zip.close()

  common.ZipClose(output_zip)



def main(argv):

  def option_handler(o, _):

tools/releasetools/build_image.py

+4 −4
Original line number Diff line number Diff line
@@ -205,8 +205,8 @@ def BuildImage(in_dir, prop_dict, out_file): 
  Returns:

    True iff the image is built successfully.

  """

  # system_root_image=true: build a system.img that combines the contents of /system

  # and the ramdisk, and can be mounted at the root of the file system.

  # system_root_image=true: build a system.img that combines the contents of

  # /system and the ramdisk, and can be mounted at the root of the file system.

  origin_in = in_dir

  fs_config = prop_dict.get("fs_config")

  if (prop_dict.get("system_root_image") == "true"

tools/releasetools/common.py

+46 −11
Original line number Diff line number Diff line
@@ -32,10 +32,7 @@ import zipfile 
import blockimgdiff

import rangelib



try:

from hashlib import sha1 as sha1

except ImportError:

  from sha import sha as sha1





class Options(object):
@@ -384,6 +381,10 @@ def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 
    p.communicate()

    assert p.returncode == 0, "vboot_signer of %s image failed" % path



    # Clean up the temp files.

    img_unsigned.close()

    img_keyblock.close()



  img.seek(os.SEEK_SET, 0)

  data = img.read()
@@ -861,16 +862,50 @@ def ZipWrite(zip_file, filename, arcname=None, perms=0o644, 
    zipfile.ZIP64_LIMIT = saved_zip64_limit





def ZipWriteStr(zip_file, filename, data, perms=0o644, compression=None):

  # use a fixed timestamp so the output is repeatable.

  zinfo = zipfile.ZipInfo(filename=filename,

                          date_time=(2009, 1, 1, 0, 0, 0))

  if compression is None:

def ZipWriteStr(zip_file, zinfo_or_arcname, data, perms=0o644,

                compress_type=None):

  """Wrap zipfile.writestr() function to work around the zip64 limit.



  Even with the ZIP64_LIMIT workaround, it won't allow writing a string

  longer than 2GiB. It gives 'OverflowError: size does not fit in an int'

  when calling crc32(bytes).



  But it still works fine to write a shorter string into a large zip file.

  We should use ZipWrite() whenever possible, and only use ZipWriteStr()

  when we know the string won't be too long.

  """



  saved_zip64_limit = zipfile.ZIP64_LIMIT

  zipfile.ZIP64_LIMIT = (1 << 32) - 1



  if not isinstance(zinfo_or_arcname, zipfile.ZipInfo):

    zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname)

    zinfo.compress_type = zip_file.compression

  else:

    zinfo.compress_type = compression

    zinfo = zinfo_or_arcname



  # If compress_type is given, it overrides the value in zinfo.

  if compress_type is not None:

    zinfo.compress_type = compress_type



  # Use a fixed timestamp so the output is repeatable.

  zinfo.external_attr = perms << 16

  zinfo.date_time = (2009, 1, 1, 0, 0, 0)



  zip_file.writestr(zinfo, data)

  zipfile.ZIP64_LIMIT = saved_zip64_limit





def ZipClose(zip_file):

  # http://b/18015246

  # zipfile also refers to ZIP64_LIMIT during close() when it writes out the

  # central directory.

  saved_zip64_limit = zipfile.ZIP64_LIMIT

  zipfile.ZIP64_LIMIT = (1 << 32) - 1



  zip_file.close()



  zipfile.ZIP64_LIMIT = saved_zip64_limit





class DeviceSpecificParams(object):
@@ -976,7 +1011,7 @@ class File(object): 
    return t



  def AddToZip(self, z, compression=None):

    ZipWriteStr(z, self.name, self.data, compression=compression)

    ZipWriteStr(z, self.name, self.data, compress_type=compression)



DIFF_PROGRAM_BY_EXT = {

    ".gz" : "imgdiff",

tools/releasetools/img_from_target_files.py

+4 −9
Original line number Diff line number Diff line
@@ -43,7 +43,8 @@ OPTIONS = common.OPTIONS 


def CopyInfo(output_zip):

  """Copy the android-info.txt file from the input to the output."""

  output_zip.write(os.path.join(OPTIONS.input_tmp, "OTA", "android-info.txt"),

  common.ZipWrite(

      output_zip, os.path.join(OPTIONS.input_tmp, "OTA", "android-info.txt"),

      "android-info.txt")
@@ -133,13 +134,7 @@ def main(argv): 


  finally:

    print "cleaning up..."

    # http://b/18015246

    # See common.py for context.  zipfile also refers to ZIP64_LIMIT during

    # close() when it writes out the central directory.

    saved_zip64_limit = zipfile.ZIP64_LIMIT

    zipfile.ZIP64_LIMIT = (1 << 32) - 1

    output_zip.close()

    zipfile.ZIP64_LIMIT = saved_zip64_limit

    common.ZipClose(output_zip)

    shutil.rmtree(OPTIONS.input_tmp)



  print "done."

tools/releasetools/ota_from_target_files.py

+5 −5
Original line number Diff line number Diff line
@@ -92,7 +92,6 @@ if sys.hexversion < 0x02070000: 
  print >> sys.stderr, "Python 2.7 or newer is required."

  sys.exit(1)



import copy

import multiprocessing

import os

import tempfile
@@ -371,6 +370,7 @@ def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 
        symlinks.append((input_zip.read(info.filename),

                         "/" + partition + "/" + basefilename))

      else:

        import copy

        info2 = copy.copy(info)

        fn = info2.filename = partition + "/" + basefilename

        if substitute and fn in substitute and substitute[fn] is None:
@@ -380,7 +380,7 @@ def CopyPartitionFiles(itemset, input_zip, output_zip=None, substitute=None): 
            data = substitute[fn]

          else:

            data = input_zip.read(info.filename)

          output_zip.writestr(info2, data)

          common.ZipWriteStr(output_zip, info2, data)

        if fn.endswith("/"):

          itemset.Get(fn[:-1], is_dir=True)

        else:
@@ -1581,6 +1581,7 @@ def main(argv): 
        OPTIONS.package_key = OPTIONS.info_dict.get(

            "default_system_dev_certificate",

            "build/target/product/security/testkey")

      common.ZipClose(output_zip)

      break



    else:
@@ -1601,15 +1602,14 @@ def main(argv): 
        common.DumpInfoDict(OPTIONS.source_info_dict)

      try:

        WriteIncrementalOTAPackage(input_zip, source_zip, output_zip)

        common.ZipClose(output_zip)

        break

      except ValueError:

        if not OPTIONS.fallback_to_full:

          raise

        print "--- failed to build incremental; falling back to full ---"

        OPTIONS.incremental_source = None

        output_zip.close()



  output_zip.close()

        common.ZipClose(output_zip)



  if not OPTIONS.no_signing:

    SignOutput(temp_zip_file.name, args[1])