Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2ee00d5f authored by Dan Willemsen's avatar Dan Willemsen
Browse files

Allow add_img_to_target_files to work on a directory 

In addition to the current behavior of add_img_to_target_files working
on an existing zip file, allow passing in a directory where the target
files have already been extracted. When in this mode, it writes the
images out to that directory instead of the zip file.

This allows us to call add_img_to_target_files on the temp directory
used during the build to create the target files package, saving the
time and space of unzipping what we just zipped. This also allows us to
use the parallel soong_zip, which compresses the images much faster.

Test: aosp_marlin target_files zip is the same before/after this change
Test: marlin target_files zip is the same before/after this change
Test: bullhead target_files zip is the same before/after this change
Change-Id: I155654cdc7ad7754ba4ef76ec69c31f504a58198
parent 7dc54e97

tools/releasetools/add_img_to_target_files.py

+124 −100
Original line number Diff line number Diff line
@@ -73,6 +73,25 @@ OPTIONS.replace_verity_public_key = False 
OPTIONS.replace_verity_private_key = False

OPTIONS.is_signing = False





class OutputFile(object):

  def __init__(self, output_zip, input_dir, prefix, name):

    self._output_zip = output_zip

    self.input_name = os.path.join(input_dir, prefix, name)



    if self._output_zip:

      self._zip_name = os.path.join(prefix, name)



      root, suffix = os.path.splitext(name)

      self.name = common.MakeTempFile(prefix=root + '-', suffix=suffix)

    else:

      self.name = self.input_name



  def Write(self):

    if self._output_zip:

      common.ZipWrite(self._output_zip, self.name, self._zip_name)





def GetCareMap(which, imgname):

  """Generate care_map of system (or vendor) partition"""
@@ -98,10 +117,10 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None): 
  """Turn the contents of SYSTEM into a system image and store it in

  output_zip. Returns the name of the system image file."""



  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system.img")

  if os.path.exists(prebuilt_path):

  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.img")

  if os.path.exists(img.input_name):

    print("system.img already exists in %s, no need to rebuild..." % (prefix,))

    return prebuilt_path

    return img.input_name



  def output_sink(fn, data):

    ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w")
@@ -113,74 +132,52 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None): 
    common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img,

                             boot_img, info_dict=OPTIONS.info_dict)



  block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map")

  imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict,

  block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.map")

  CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system", img,

              block_list=block_list)



  common.ZipWrite(output_zip, imgname, prefix + "system.img")

  common.ZipWrite(output_zip, block_list, prefix + "system.map")

  return imgname





def BuildSystem(input_dir, info_dict, block_list=None):

  """Build the (sparse) system image and return the name of a temp

  file containing it."""

  return CreateImage(input_dir, info_dict, "system", block_list=block_list)

  return img.name





def AddSystemOther(output_zip, prefix="IMAGES/"):

  """Turn the contents of SYSTEM_OTHER into a system_other image

  and store it in output_zip."""



  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system_other.img")

  if os.path.exists(prebuilt_path):

  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system_other.img")

  if os.path.exists(img.input_name):

    print("system_other.img already exists in %s, no need to rebuild..." % (

        prefix,))

    return



  imgname = BuildSystemOther(OPTIONS.input_tmp, OPTIONS.info_dict)

  common.ZipWrite(output_zip, imgname, prefix + "system_other.img")



def BuildSystemOther(input_dir, info_dict):

  """Build the (sparse) system_other image and return the name of a temp

  file containing it."""

  return CreateImage(input_dir, info_dict, "system_other", block_list=None)

  CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system_other", img)





def AddVendor(output_zip, prefix="IMAGES/"):

  """Turn the contents of VENDOR into a vendor image and store in it

  output_zip."""



  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "vendor.img")

  if os.path.exists(prebuilt_path):

  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.img")

  if os.path.exists(img.input_name):

    print("vendor.img already exists in %s, no need to rebuild..." % (prefix,))

    return prebuilt_path

    return img.input_name



  block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map")

  imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict,

  block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.map")

  CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "vendor", img,

              block_list=block_list)

  common.ZipWrite(output_zip, imgname, prefix + "vendor.img")

  common.ZipWrite(output_zip, block_list, prefix + "vendor.map")

  return imgname

  return img.name





def BuildVendor(input_dir, info_dict, block_list=None):

  """Build the (sparse) vendor image and return the name of a temp

  file containing it."""

  return CreateImage(input_dir, info_dict, "vendor", block_list=block_list)





def CreateImage(input_dir, info_dict, what, block_list=None):

def CreateImage(input_dir, info_dict, what, output_file, block_list=None):

  print("creating " + what + ".img...")



  img = common.MakeTempFile(prefix=what + "-", suffix=".img")



  # The name of the directory it is making an image out of matters to

  # mkyaffs2image.  It wants "system" but we have a directory named

  # "SYSTEM", so create a symlink.

  temp_dir = tempfile.mkdtemp()

  OPTIONS.tempfiles.append(temp_dir)

  try:

    os.symlink(os.path.join(input_dir, what.upper()),

               os.path.join(input_dir, what))

               os.path.join(temp_dir, what))

  except OSError as e:

    # bogus error on my mac version?

    #   File "./build/tools/releasetools/img_from_target_files"
@@ -215,12 +212,16 @@ def CreateImage(input_dir, info_dict, what, block_list=None): 
  if fs_config:

    image_props["fs_config"] = fs_config

  if block_list:

    image_props["block_list"] = block_list

    image_props["block_list"] = block_list.name



  succ = build_image.BuildImage(os.path.join(input_dir, what),

                                image_props, img)

  succ = build_image.BuildImage(os.path.join(temp_dir, what),

                                image_props, output_file.name)

  assert succ, "build " + what + ".img image failed"



  output_file.Write()

  if block_list:

    block_list.Write()



  is_verity_partition = "verity_block_device" in image_props

  verity_supported = image_props.get("verity") == "true"

  if is_verity_partition and verity_supported:
@@ -229,8 +230,6 @@ def CreateImage(input_dir, info_dict, what, block_list=None): 
      adjusted_blocks_key = what + "_adjusted_partition_size"

      info_dict[adjusted_blocks_key] = int(adjusted_blocks_value)/4096 - 1



  return img





def AddUserdata(output_zip, prefix="IMAGES/"):

  """Create a userdata image and store it in output_zip.
@@ -241,8 +240,8 @@ def AddUserdata(output_zip, prefix="IMAGES/"): 
  in OPTIONS.info_dict.

  """



  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "userdata.img")

  if os.path.exists(prebuilt_path):

  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "userdata.img")

  if os.path.exists(img.input_name):

    print("userdata.img already exists in %s, no need to rebuild..." % (

        prefix,))

    return
@@ -265,6 +264,7 @@ def AddUserdata(output_zip, prefix="IMAGES/"): 
  # empty dir named "data", or a symlink to the DATA dir,

  # and build the image from that.

  temp_dir = tempfile.mkdtemp()

  OPTIONS.tempfiles.append(temp_dir)

  user_dir = os.path.join(temp_dir, "data")

  empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true")

  if empty:
@@ -275,8 +275,6 @@ def AddUserdata(output_zip, prefix="IMAGES/"): 
    os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"),

               user_dir)



  img = tempfile.NamedTemporaryFile()



  fstab = OPTIONS.info_dict["fstab"]

  if fstab:

    image_props["fs_type"] = fstab["/data"].fs_type
@@ -284,17 +282,15 @@ def AddUserdata(output_zip, prefix="IMAGES/"): 
  assert succ, "build userdata.img image failed"



  common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict)

  common.ZipWrite(output_zip, img.name, prefix + "userdata.img")

  img.close()

  shutil.rmtree(temp_dir)

  img.Write()





def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"):

  """Create a VBMeta image and store it in output_zip."""

  _, img_file_name = tempfile.mkstemp()

  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vbmeta.img")

  avbtool = os.getenv('AVBTOOL') or "avbtool"

  cmd = [avbtool, "make_vbmeta_image",

         "--output", img_file_name,

         "--output", img.name,

         "--include_descriptors_from_image", boot_img_path,

         "--include_descriptors_from_image", system_img_path,

         "--generate_dm_verity_cmdline_from_hashtree", system_img_path]
@@ -305,19 +301,19 @@ def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"): 
  p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

  p.communicate()

  assert p.returncode == 0, "avbtool make_vbmeta_image failed"

  common.ZipWrite(output_zip, img_file_name, prefix + "vbmeta.img")

  img.Write()





def AddPartitionTable(output_zip, prefix="IMAGES/"):

  """Create a partition table image and store it in output_zip."""



  _, img_file_name = tempfile.mkstemp()

  _, bpt_file_name = tempfile.mkstemp()

  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.img")

  bpt = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.bpt")



  # use BPTTOOL from environ, or "bpttool" if empty or not set.

  bpttool = os.getenv("BPTTOOL") or "bpttool"

  cmd = [bpttool, "make_table", "--output_json", bpt_file_name,

         "--output_gpt", img_file_name]

  cmd = [bpttool, "make_table", "--output_json", bpt.name,

         "--output_gpt", img.name]

  input_files_str = OPTIONS.info_dict["board_bpt_input_files"]

  input_files = input_files_str.split(" ")

  for i in input_files:
@@ -333,15 +329,15 @@ def AddPartitionTable(output_zip, prefix="IMAGES/"): 
  p.communicate()

  assert p.returncode == 0, "bpttool make_table failed"



  common.ZipWrite(output_zip, img_file_name, prefix + "partition-table.img")

  common.ZipWrite(output_zip, bpt_file_name, prefix + "partition-table.bpt")

  img.Write()

  bpt.Write()





def AddCache(output_zip, prefix="IMAGES/"):

  """Create an empty cache image and store it in output_zip."""



  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "cache.img")

  if os.path.exists(prebuilt_path):

  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "cache.img")

  if os.path.exists(img.input_name):

    print("cache.img already exists in %s, no need to rebuild..." % (prefix,))

    return
@@ -362,9 +358,9 @@ def AddCache(output_zip, prefix="IMAGES/"): 
  # mkyaffs2image.  So we create a temp dir, and within it we create an

  # empty dir named "cache", and build the image from that.

  temp_dir = tempfile.mkdtemp()

  OPTIONS.tempfiles.append(temp_dir)

  user_dir = os.path.join(temp_dir, "cache")

  os.mkdir(user_dir)

  img = tempfile.NamedTemporaryFile()



  fstab = OPTIONS.info_dict["fstab"]

  if fstab:
@@ -373,35 +369,39 @@ def AddCache(output_zip, prefix="IMAGES/"): 
  assert succ, "build cache.img image failed"



  common.CheckSize(img.name, "cache.img", OPTIONS.info_dict)

  common.ZipWrite(output_zip, img.name, prefix + "cache.img")

  img.close()

  os.rmdir(user_dir)

  os.rmdir(temp_dir)

  img.Write()





def AddImagesToTargetFiles(filename):

  if os.path.isdir(filename):

    OPTIONS.input_tmp = os.path.abspath(filename)

    input_zip = None

  else:

    OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename)



  if not OPTIONS.add_missing:

    for n in input_zip.namelist():

      if n.startswith("IMAGES/"):

    if os.path.isdir(os.path.join(OPTIONS.input_tmp, "IMAGES")):

      print("target_files appears to already contain images.")

      sys.exit(1)



  try:

    input_zip.getinfo("VENDOR/")

    has_vendor = True

  except KeyError:

    has_vendor = False



  has_system_other = "SYSTEM_OTHER/" in input_zip.namelist()

  has_vendor = os.path.isdir(os.path.join(OPTIONS.input_tmp, "VENDOR"))

  has_system_other = os.path.isdir(os.path.join(OPTIONS.input_tmp,

                                                "SYSTEM_OTHER"))



  if input_zip:

    OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp)



    common.ZipClose(input_zip)

    output_zip = zipfile.ZipFile(filename, "a",

                                 compression=zipfile.ZIP_DEFLATED,

                                 allowZip64=True)

  else:

    OPTIONS.info_dict = common.LoadInfoDict(filename, filename)

    output_zip = None

    images_dir = os.path.join(OPTIONS.input_tmp, "IMAGES")

    if not os.path.isdir(images_dir):

      os.makedirs(images_dir)

    images_dir = None



  has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true")

  system_root_image = (OPTIONS.info_dict.get("system_root_image", None) == "true")
@@ -422,7 +422,10 @@ def AddImagesToTargetFiles(filename): 
    boot_image = common.GetBootableImage(

        "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT")

    if boot_image:

      if output_zip:

        boot_image.AddToZip(output_zip)

      else:

        boot_image.WriteToDir(OPTIONS.input_tmp)



  recovery_image = None

  if has_recovery:
@@ -438,7 +441,10 @@ def AddImagesToTargetFiles(filename): 
      recovery_image = common.GetBootableImage(

          "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY")

      if recovery_image:

        if output_zip:

          recovery_image.AddToZip(output_zip)

        else:

          recovery_image.WriteToDir(OPTIONS.input_tmp)



      banner("recovery (two-step image)")

      # The special recovery.img for two-step package use.
@@ -446,7 +452,10 @@ def AddImagesToTargetFiles(filename): 
          "IMAGES/recovery-two-step.img", "recovery-two-step.img",

          OPTIONS.input_tmp, "RECOVERY", two_step_image=True)

      if recovery_two_step_image:

        if output_zip:

          recovery_two_step_image.AddToZip(output_zip)

        else:

          recovery_two_step_image.WriteToDir(OPTIONS.input_tmp)



  banner("system")

  system_img_path = AddSystem(
@@ -502,23 +511,38 @@ def AddImagesToTargetFiles(filename): 
      img_vendor_dir = os.path.join(

        OPTIONS.input_tmp, "VENDOR_IMAGES")

      if os.path.exists(img_radio_path):

        if output_zip:

          common.ZipWrite(output_zip, img_radio_path,

                          os.path.join("IMAGES", img_name))

        else:

          shutil.copy(img_radio_path, prebuilt_path)

      else:

        for root, _, files in os.walk(img_vendor_dir):

          if img_name in files:

            if output_zip:

              common.ZipWrite(output_zip, os.path.join(root, img_name),

                os.path.join("IMAGES", img_name))

            else:

              shutil.copy(os.path.join(root, img_name), prebuilt_path)

            break



      if output_zip:

        # Zip spec says: All slashes MUST be forward slashes.

        img_path = 'IMAGES/' + img_name

        assert img_path in output_zip.namelist(), "cannot find " + img_name

      else:

        img_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name)

        assert os.path.exists(img_path), "cannot find " + img_name



    if care_map_list:

      file_path = "META/care_map.txt"

      if output_zip:

        common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list))

      else:

        with open(os.path.join(OPTIONS.input_tmp, file_path), 'w') as fp:

          fp.write('\n'.join(care_map_list))



  if output_zip:

    common.ZipClose(output_zip)



def main(argv):

tools/releasetools/common.py

+4 −0
Original line number Diff line number Diff line
@@ -1191,6 +1191,10 @@ class File(object): 
    t.flush()

    return t



  def WriteToDir(self, d):

    with open(os.path.join(d, self.name), "wb") as fp:

      fp.write(self.data)



  def AddToZip(self, z, compression=None):

    ZipWriteStr(z, self.name, self.data, compress_type=compression)