Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 185ce5ff authored by Tao Bao's avatar Tao Bao Committed by Gerrit Code Review
Browse files

Merge "releasetools: Switch to Python logging."

parents ac02d676 32fcdab5

tools/releasetools/add_img_to_target_files.py

+26 −24
Original line number Diff line number Diff line
@@ -46,6 +46,7 @@ Usage: add_img_to_target_files [flag] target_files 
from __future__ import print_function



import datetime

import logging

import os

import shlex

import shutil
@@ -62,8 +63,9 @@ if sys.hexversion < 0x02070000: 
  print("Python 2.7 or newer is required.", file=sys.stderr)

  sys.exit(1)



OPTIONS = common.OPTIONS

logger = logging.getLogger(__name__)



OPTIONS = common.OPTIONS

OPTIONS.add_missing = False

OPTIONS.rebuild_recovery = False

OPTIONS.replace_updated_files_list = []
@@ -127,7 +129,7 @@ def AddSystem(output_zip, recovery_img=None, boot_img=None): 


  img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "system.img")

  if os.path.exists(img.input_name):

    print("system.img already exists; no need to rebuild...")

    logger.info("system.img already exists; no need to rebuild...")

    return img.input_name



  def output_sink(fn, data):
@@ -142,7 +144,7 @@ def AddSystem(output_zip, recovery_img=None, boot_img=None): 
      common.ZipWrite(output_zip, ofile.name, arc_name)



  if OPTIONS.rebuild_recovery:

    print("Building new recovery patch")

    logger.info("Building new recovery patch")

    common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img,

                             boot_img, info_dict=OPTIONS.info_dict)
@@ -159,7 +161,7 @@ def AddSystemOther(output_zip): 


  img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "system_other.img")

  if os.path.exists(img.input_name):

    print("system_other.img already exists; no need to rebuild...")

    logger.info("system_other.img already exists; no need to rebuild...")

    return



  CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system_other", img)
@@ -171,7 +173,7 @@ def AddVendor(output_zip): 


  img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "vendor.img")

  if os.path.exists(img.input_name):

    print("vendor.img already exists; no need to rebuild...")

    logger.info("vendor.img already exists; no need to rebuild...")

    return img.input_name



  block_list = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "vendor.map")
@@ -186,7 +188,7 @@ def AddProduct(output_zip): 


  img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "product.img")

  if os.path.exists(img.input_name):

    print("product.img already exists; no need to rebuild...")

    logger.info("product.img already exists; no need to rebuild...")

    return img.input_name



  block_list = OutputFile(
@@ -204,7 +206,7 @@ def AddProductServices(output_zip): 
  img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES",

                   "product_services.img")

  if os.path.exists(img.input_name):

    print("product_services.img already exists; no need to rebuild...")

    logger.info("product_services.img already exists; no need to rebuild...")

    return img.input_name



  block_list = OutputFile(
@@ -220,7 +222,7 @@ def AddOdm(output_zip): 


  img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "odm.img")

  if os.path.exists(img.input_name):

    print("odm.img already exists; no need to rebuild...")

    logger.info("odm.img already exists; no need to rebuild...")

    return img.input_name



  block_list = OutputFile(
@@ -239,7 +241,7 @@ def AddDtbo(output_zip): 
  """

  img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "dtbo.img")

  if os.path.exists(img.input_name):

    print("dtbo.img already exists; no need to rebuild...")

    logger.info("dtbo.img already exists; no need to rebuild...")

    return img.input_name



  dtbo_prebuilt_path = os.path.join(
@@ -269,7 +271,7 @@ def AddDtbo(output_zip): 




def CreateImage(input_dir, info_dict, what, output_file, block_list=None):

  print("creating " + what + ".img...")

  logger.info("creating " + what + ".img...")



  image_props = build_image.ImagePropFromGlobalDict(info_dict, what)

  fstab = info_dict["fstab"]
@@ -340,7 +342,7 @@ def AddUserdata(output_zip): 


  img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "userdata.img")

  if os.path.exists(img.input_name):

    print("userdata.img already exists; no need to rebuild...")

    logger.info("userdata.img already exists; no need to rebuild...")

    return



  # Skip userdata.img if no size.
@@ -348,7 +350,7 @@ def AddUserdata(output_zip): 
  if not image_props.get("partition_size"):

    return



  print("creating userdata.img...")

  logger.info("creating userdata.img...")



  image_props["timestamp"] = FIXED_FILE_TIMESTAMP
@@ -411,7 +413,7 @@ def AddVBMeta(output_zip, partitions, name, needed_partitions): 
  img = OutputFile(

      output_zip, OPTIONS.input_tmp, "IMAGES", "{}.img".format(name))

  if os.path.exists(img.input_name):

    print("{}.img already exists; not rebuilding...".format(name))

    logger.info("%s.img already exists; not rebuilding...", name)

    return img.input_name



  avbtool = os.getenv('AVBTOOL') or OPTIONS.info_dict["avb_avbtool"]
@@ -495,7 +497,7 @@ def AddCache(output_zip): 


  img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "cache.img")

  if os.path.exists(img.input_name):

    print("cache.img already exists; no need to rebuild...")

    logger.info("cache.img already exists; no need to rebuild...")

    return



  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "cache")
@@ -503,7 +505,7 @@ def AddCache(output_zip): 
  if "fs_type" not in image_props:

    return



  print("creating cache.img...")

  logger.info("creating cache.img...")



  image_props["timestamp"] = FIXED_FILE_TIMESTAMP
@@ -580,8 +582,7 @@ def AddCareMapForAbOta(output_zip, ab_partitions, image_paths): 


      present_props = [x for x in prop_name_list if x in build_props]

      if not present_props:

        print("Warning: fingerprint is not present for partition {}".

              format(partition))

        logger.warning("fingerprint is not present for partition %s", partition)

        property_id, fingerprint = "unknown", "unknown"

      else:

        property_id = present_props[0]
@@ -633,7 +634,7 @@ def AddPackRadioImages(output_zip, images): 


    prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name)

    if os.path.exists(prebuilt_path):

      print("%s already exists, no need to overwrite..." % (img_name,))

      logger.info("%s already exists, no need to overwrite...", img_name)

      continue



    img_radio_path = os.path.join(OPTIONS.input_tmp, "RADIO", img_name)
@@ -698,7 +699,7 @@ def AddImagesToTargetFiles(filename): 


  if not OPTIONS.add_missing:

    if os.path.isdir(os.path.join(OPTIONS.input_tmp, "IMAGES")):

      print("target_files appears to already contain images.")

      logger.warning("target_files appears to already contain images.")

      sys.exit(1)



  OPTIONS.info_dict = common.LoadInfoDict(OPTIONS.input_tmp, repacking=True)
@@ -748,7 +749,7 @@ def AddImagesToTargetFiles(filename): 
  partitions = dict()



  def banner(s):

    print("\n\n++++ " + s + " ++++\n\n")

    logger.info("\n\n++++ " + s + " ++++\n\n")



  banner("boot")

  # common.GetBootableImage() returns the image directly if present.
@@ -912,20 +913,21 @@ def main(argv): 
                       "is_signing"],

      extra_option_handler=option_handler)





  if len(args) != 1:

    common.Usage(__doc__)

    sys.exit(1)



  common.InitLogging()



  AddImagesToTargetFiles(args[0])

  print("done.")

  logger.info("done.")



if __name__ == '__main__':

  try:

    common.CloseInheritedPipes()

    main(sys.argv[1:])

  except common.ExternalError as e:

    print("\n   ERROR: %s\n" % (e,))

  except common.ExternalError:

    logger.exception("\n   ERROR:\n")

    sys.exit(1)

  finally:

    common.Cleanup()

tools/releasetools/blockimgdiff.py

+57 −62
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ import copy 
import functools

import heapq

import itertools

import logging

import multiprocessing

import os

import os.path
@@ -33,6 +34,8 @@ from rangelib import RangeSet 


__all__ = ["EmptyImage", "DataImage", "BlockImageDiff"]



logger = logging.getLogger(__name__)





def compute_patch(srcfile, tgtfile, imgdiff=False):

  patchfile = common.MakeTempFile(prefix='patch-')
@@ -304,8 +307,8 @@ class ImgdiffStats(object): 
    """Prints a report of the collected imgdiff stats."""



    def print_header(header, separator):

      print(header)

      print(separator * len(header) + '\n')

      logger.info(header)

      logger.info(separator * len(header) + '\n')



    print_header('  Imgdiff Stats Report  ', '=')

    for key in self.REASONS:
@@ -314,7 +317,7 @@ class ImgdiffStats(object): 
      values = self.stats[key]

      section_header = ' {} (count: {}) '.format(key, len(values))

      print_header(section_header, '-')

      print(''.join(['  {}\n'.format(name) for name in values]))

      logger.info(''.join(['  {}\n'.format(name) for name in values]))





class BlockImageDiff(object):
@@ -482,7 +485,7 @@ class BlockImageDiff(object): 
    self.WriteTransfers(prefix)



    # Report the imgdiff stats.

    if common.OPTIONS.verbose and not self.disable_imgdiff:

    if not self.disable_imgdiff:

      self.imgdiff_stats.Report()



  def WriteTransfers(self, prefix):
@@ -692,16 +695,17 @@ class BlockImageDiff(object): 
    OPTIONS = common.OPTIONS

    if OPTIONS.cache_size is not None:

      max_allowed = OPTIONS.cache_size * OPTIONS.stash_threshold

      print("max stashed blocks: %d  (%d bytes), "

            "limit: %d bytes (%.2f%%)\n" % (

      logger.info(

          "max stashed blocks: %d  (%d bytes), limit: %d bytes (%.2f%%)\n",

          max_stashed_blocks, self._max_stashed_size, max_allowed,

                self._max_stashed_size * 100.0 / max_allowed))

          self._max_stashed_size * 100.0 / max_allowed)

    else:

      print("max stashed blocks: %d  (%d bytes), limit: <unknown>\n" % (

          max_stashed_blocks, self._max_stashed_size))

      logger.info(

          "max stashed blocks: %d  (%d bytes), limit: <unknown>\n",

          max_stashed_blocks, self._max_stashed_size)



  def ReviseStashSize(self):

    print("Revising stash size...")

    logger.info("Revising stash size...")

    stash_map = {}



    # Create the map between a stash and its def/use points. For example, for a
@@ -746,7 +750,7 @@ class BlockImageDiff(object): 
          # that will use this stash and replace the command with "new".

          use_cmd = stash_map[stash_raw_id][2]

          replaced_cmds.append(use_cmd)

          print("%10d  %9s  %s" % (sr.size(), "explicit", use_cmd))

          logger.info("%10d  %9s  %s", sr.size(), "explicit", use_cmd)

        else:

          # Update the stashes map.

          if sh in stashes:
@@ -762,7 +766,7 @@ class BlockImageDiff(object): 
        if xf.src_ranges.overlaps(xf.tgt_ranges):

          if stashed_blocks + xf.src_ranges.size() > max_allowed:

            replaced_cmds.append(xf)

            print("%10d  %9s  %s" % (xf.src_ranges.size(), "implicit", xf))

            logger.info("%10d  %9s  %s", xf.src_ranges.size(), "implicit", xf)



      # Replace the commands in replaced_cmds with "new"s.

      for cmd in replaced_cmds:
@@ -788,28 +792,29 @@ class BlockImageDiff(object): 
          stashes.pop(sh)



    num_of_bytes = new_blocks * self.tgt.blocksize

    print("  Total %d blocks (%d bytes) are packed as new blocks due to "

          "insufficient cache size." % (new_blocks, num_of_bytes))

    logger.info(

        "  Total %d blocks (%d bytes) are packed as new blocks due to "

        "insufficient cache size.", new_blocks, num_of_bytes)

    return new_blocks



  def ComputePatches(self, prefix):

    print("Reticulating splines...")

    logger.info("Reticulating splines...")

    diff_queue = []

    patch_num = 0

    with open(prefix + ".new.dat", "wb") as new_f:

      for index, xf in enumerate(self.transfers):

        if xf.style == "zero":

          tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize

          print("%10d %10d (%6.2f%%) %7s %s %s" % (

              tgt_size, tgt_size, 100.0, xf.style, xf.tgt_name,

              str(xf.tgt_ranges)))

          logger.info(

              "%10d %10d (%6.2f%%) %7s %s %s", tgt_size, tgt_size, 100.0,

              xf.style, xf.tgt_name, str(xf.tgt_ranges))



        elif xf.style == "new":

          self.tgt.WriteRangeDataToFd(xf.tgt_ranges, new_f)

          tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize

          print("%10d %10d (%6.2f%%) %7s %s %s" % (

              tgt_size, tgt_size, 100.0, xf.style,

              xf.tgt_name, str(xf.tgt_ranges)))

          logger.info(

              "%10d %10d (%6.2f%%) %7s %s %s", tgt_size, tgt_size, 100.0,

              xf.style, xf.tgt_name, str(xf.tgt_ranges))



        elif xf.style == "diff":

          # We can't compare src and tgt directly because they may have
@@ -827,11 +832,12 @@ class BlockImageDiff(object): 
            xf.patch = None

            tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize

            if xf.src_ranges != xf.tgt_ranges:

              print("%10d %10d (%6.2f%%) %7s %s %s (from %s)" % (

                  tgt_size, tgt_size, 100.0, xf.style,

              logger.info(

                  "%10d %10d (%6.2f%%) %7s %s %s (from %s)", tgt_size, tgt_size,

                  100.0, xf.style,

                  xf.tgt_name if xf.tgt_name == xf.src_name else (

                      xf.tgt_name + " (from " + xf.src_name + ")"),

                  str(xf.tgt_ranges), str(xf.src_ranges)))

                  str(xf.tgt_ranges), str(xf.src_ranges))

          else:

            if xf.patch:

              # We have already generated the patch with imgdiff, while
@@ -850,9 +856,9 @@ class BlockImageDiff(object): 


    if diff_queue:

      if self.threads > 1:

        print("Computing patches (using %d threads)..." % (self.threads,))

        logger.info("Computing patches (using %d threads)...", self.threads)

      else:

        print("Computing patches...")

        logger.info("Computing patches...")



      diff_total = len(diff_queue)

      patches = [None] * diff_total
@@ -874,13 +880,6 @@ class BlockImageDiff(object): 
            xf_index, imgdiff, patch_index = diff_queue.pop()

            xf = self.transfers[xf_index]



            if sys.stdout.isatty():

              diff_left = len(diff_queue)

              progress = (diff_total - diff_left) * 100 / diff_total

              # '\033[K' is to clear to EOL.

              print(' [%3d%%] %s\033[K' % (progress, xf.tgt_name), end='\r')

              sys.stdout.flush()



          patch = xf.patch

          if not patch:

            src_ranges = xf.src_ranges
@@ -918,13 +917,10 @@ class BlockImageDiff(object): 
      while threads:

        threads.pop().join()



      if sys.stdout.isatty():

        print('\n')



      if error_messages:

        print('ERROR:')

        print('\n'.join(error_messages))

        print('\n\n\n')

        logger.error('ERROR:')

        logger.error('\n'.join(error_messages))

        logger.error('\n\n\n')

        sys.exit(1)

    else:

      patches = []
@@ -938,14 +934,13 @@ class BlockImageDiff(object): 
        offset += xf.patch_len

        patch_fd.write(patch)



        if common.OPTIONS.verbose:

        tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize

          print("%10d %10d (%6.2f%%) %7s %s %s %s" % (

              xf.patch_len, tgt_size, xf.patch_len * 100.0 / tgt_size,

              xf.style,

        logger.info(

            "%10d %10d (%6.2f%%) %7s %s %s %s", xf.patch_len, tgt_size,

            xf.patch_len * 100.0 / tgt_size, xf.style,

            xf.tgt_name if xf.tgt_name == xf.src_name else (

                xf.tgt_name + " (from " + xf.src_name + ")"),

              xf.tgt_ranges, xf.src_ranges))

            xf.tgt_ranges, xf.src_ranges)



  def AssertSha1Good(self):

    """Check the SHA-1 of the src & tgt blocks in the transfer list.
@@ -1005,7 +1000,7 @@ class BlockImageDiff(object): 
        assert touched[i] == 1



  def ImproveVertexSequence(self):

    print("Improving vertex order...")

    logger.info("Improving vertex order...")



    # At this point our digraph is acyclic; we reversed any edges that

    # were backwards in the heuristically-generated sequence.  The
@@ -1057,7 +1052,7 @@ class BlockImageDiff(object): 
    blocks will be written to the same stash slot in WriteTransfers().

    """



    print("Reversing backward edges...")

    logger.info("Reversing backward edges...")

    in_order = 0

    out_of_order = 0

    stash_raw_id = 0
@@ -1089,15 +1084,15 @@ class BlockImageDiff(object): 
          xf.goes_after[u] = None    # value doesn't matter

          u.goes_before[xf] = None



    print(("  %d/%d dependencies (%.2f%%) were violated; "

           "%d source blocks stashed.") %

          (out_of_order, in_order + out_of_order,

           (out_of_order * 100.0 / (in_order + out_of_order))

           if (in_order + out_of_order) else 0.0,

           stash_size))

    logger.info(

        "  %d/%d dependencies (%.2f%%) were violated; %d source blocks "

        "stashed.", out_of_order, in_order + out_of_order,

        (out_of_order * 100.0 / (in_order + out_of_order)) if (

            in_order + out_of_order) else 0.0,

        stash_size)



  def FindVertexSequence(self):

    print("Finding vertex sequence...")

    logger.info("Finding vertex sequence...")



    # This is based on "A Fast & Effective Heuristic for the Feedback

    # Arc Set Problem" by P. Eades, X. Lin, and W.F. Smyth.  Think of
@@ -1210,7 +1205,7 @@ class BlockImageDiff(object): 
    self.transfers = new_transfers



  def GenerateDigraph(self):

    print("Generating digraph...")

    logger.info("Generating digraph...")



    # Each item of source_ranges will be:

    #   - None, if that block is not used as a source,
@@ -1376,9 +1371,9 @@ class BlockImageDiff(object): 


        if tgt_changed < tgt_size * crop_threshold:

          assert tgt_changed + tgt_skipped.size() == tgt_size

          print('%10d %10d (%6.2f%%) %s' % (

              tgt_skipped.size(), tgt_size,

              tgt_skipped.size() * 100.0 / tgt_size, tgt_name))

          logger.info(

              '%10d %10d (%6.2f%%) %s', tgt_skipped.size(), tgt_size,

              tgt_skipped.size() * 100.0 / tgt_size, tgt_name)

          AddSplitTransfers(

              "%s-skipped" % (tgt_name,),

              "%s-skipped" % (src_name,),
@@ -1519,7 +1514,7 @@ class BlockImageDiff(object): 
                                   split_src_ranges,

                                   patch_content))



    print("Finding transfers...")

    logger.info("Finding transfers...")



    large_apks = []

    split_large_apks = []

tools/releasetools/build_image.py

+14 −9
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ Usage: build_image.py input_directory properties_file output_image \\ 


from __future__ import print_function



import logging

import os

import os.path

import re
@@ -35,6 +36,8 @@ import sys 
import common

import verity_utils



logger = logging.getLogger(__name__)



OPTIONS = common.OPTIONS

BLOCK_SIZE = common.BLOCK_SIZE

BYTES_IN_MB = 1024 * 1024
@@ -228,8 +231,8 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): 
      "partition_size" not in prop_dict):

    # If partition_size is not defined, use output of `du' + reserved_size.

    size = GetDiskUsage(in_dir)

    if OPTIONS.verbose:

      print("The tree size of %s is %d MB." % (in_dir, size // BYTES_IN_MB))

    logger.info(

        "The tree size of %s is %d MB.", in_dir, size // BYTES_IN_MB)

    size += int(prop_dict.get("partition_reserved_size", 0))

    # Round this up to a multiple of 4K so that avbtool works

    size = common.RoundUpTo4K(size)
@@ -241,8 +244,8 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): 
          lambda x: verity_utils.AVBCalcMaxImageSize(

              avbtool, avb_footer_type, x, avb_signing_args))

    prop_dict["partition_size"] = str(size)

    if OPTIONS.verbose:

      print("Allocating %d MB for %s." % (size // BYTES_IN_MB, out_file))

    logger.info(

        "Allocating %d MB for %s.", size // BYTES_IN_MB, out_file)



  prop_dict["image_size"] = prop_dict["partition_size"]
@@ -350,8 +353,8 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): 
      du_str = "{} bytes ({} MB)".format(du, du // BYTES_IN_MB)

    # Suppress any errors from GetDiskUsage() to avoid hiding the real errors

    # from common.RunAndCheckOutput().

    except Exception as e:  # pylint: disable=broad-except

      print(e, file=sys.stderr)

    except Exception:  # pylint: disable=broad-except

      logger.exception("Failed to compute disk usage with du")

      du_str = "unknown"

    print(

        "Out of space? The tree size of {} is {}, with reserved space of {} "
@@ -664,6 +667,8 @@ def main(argv): 
    print(__doc__)

    sys.exit(1)



  common.InitLogging()



  in_dir = argv[0]

  glob_dict_file = argv[1]

  out_file = argv[2]
@@ -697,7 +702,7 @@ def main(argv): 
    elif image_filename == "product_services.img":

      mount_point = "product_services"

    else:

      print("error: unknown image file name ", image_filename, file=sys.stderr)

      logger.error("Unknown image file name %s", image_filename)

      sys.exit(1)



    image_properties = ImagePropFromGlobalDict(glob_dict, mount_point)
@@ -705,14 +710,14 @@ def main(argv): 
  try:

    BuildImage(in_dir, image_properties, out_file, target_out)

  except:

    print("Error: Failed to build {} from {}".format(out_file, in_dir),

          file=sys.stderr)

    logger.error("Failed to build %s from %s", out_file, in_dir)

    raise



  if prop_file_out:

    glob_dict_out = GlobalDictFromImageProp(image_properties, mount_point)

    SaveGlobalDict(prop_file_out, glob_dict_out)





if __name__ == '__main__':

  try:

    main(sys.argv[1:])

tools/releasetools/check_ota_package_signature.py

+5 −1
Original line number Diff line number Diff line
@@ -21,16 +21,18 @@ Verify a given OTA package with the specifed certificate. 
from __future__ import print_function



import argparse

import logging

import re

import subprocess

import sys

import zipfile



from hashlib import sha1

from hashlib import sha256



import common



logger = logging.getLogger(__name__)





def CertUsesSha256(cert):

  """Check if the cert uses SHA-256 hashing algorithm."""
@@ -181,6 +183,8 @@ def main(): 
  parser.add_argument('package', help='The OTA package to be verified.')

  args = parser.parse_args()



  common.InitLogging()



  VerifyPackage(args.certificate, args.package)

  VerifyAbOtaPayload(args.certificate, args.package)

tools/releasetools/common.py

+82 −40

File changed.

Preview size limit exceeded, changes collapsed.