Merge "releasetools: Switch to Python logging." am: 185ce5ffb3 am: c4c8c99741 (8b8062cc) · Commits · e / os / android_build

tools/releasetools/add_img_to_target_files.py

+26 −24

Original line number	Diff line number	Diff line
		@@ -46,6 +46,7 @@ Usage: add_img_to_target_files [flag] target_files
		from __future__ import print_function

		import datetime
		import logging
		import os
		import shlex
		import shutil
		@@ -62,8 +63,9 @@ if sys.hexversion < 0x02070000:
		print("Python 2.7 or newer is required.", file=sys.stderr)
		sys.exit(1)

		OPTIONS = common.OPTIONS
		logger = logging.getLogger(__name__)

		OPTIONS = common.OPTIONS
		OPTIONS.add_missing = False
		OPTIONS.rebuild_recovery = False
		OPTIONS.replace_updated_files_list = []
		@@ -127,7 +129,7 @@ def AddSystem(output_zip, recovery_img=None, boot_img=None):

		img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "system.img")
		if os.path.exists(img.input_name):
		print("system.img already exists; no need to rebuild...")
		logger.info("system.img already exists; no need to rebuild...")
		return img.input_name

		def output_sink(fn, data):
		@@ -142,7 +144,7 @@ def AddSystem(output_zip, recovery_img=None, boot_img=None):
		common.ZipWrite(output_zip, ofile.name, arc_name)

		if OPTIONS.rebuild_recovery:
		print("Building new recovery patch")
		logger.info("Building new recovery patch")
		common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img,
		boot_img, info_dict=OPTIONS.info_dict)

		@@ -159,7 +161,7 @@ def AddSystemOther(output_zip):

		img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "system_other.img")
		if os.path.exists(img.input_name):
		print("system_other.img already exists; no need to rebuild...")
		logger.info("system_other.img already exists; no need to rebuild...")
		return

		CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system_other", img)
		@@ -171,7 +173,7 @@ def AddVendor(output_zip):

		img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "vendor.img")
		if os.path.exists(img.input_name):
		print("vendor.img already exists; no need to rebuild...")
		logger.info("vendor.img already exists; no need to rebuild...")
		return img.input_name

		block_list = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "vendor.map")
		@@ -186,7 +188,7 @@ def AddProduct(output_zip):

		img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "product.img")
		if os.path.exists(img.input_name):
		print("product.img already exists; no need to rebuild...")
		logger.info("product.img already exists; no need to rebuild...")
		return img.input_name

		block_list = OutputFile(
		@@ -204,7 +206,7 @@ def AddProductServices(output_zip):
		img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES",
		"product_services.img")
		if os.path.exists(img.input_name):
		print("product_services.img already exists; no need to rebuild...")
		logger.info("product_services.img already exists; no need to rebuild...")
		return img.input_name

		block_list = OutputFile(
		@@ -220,7 +222,7 @@ def AddOdm(output_zip):

		img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "odm.img")
		if os.path.exists(img.input_name):
		print("odm.img already exists; no need to rebuild...")
		logger.info("odm.img already exists; no need to rebuild...")
		return img.input_name

		block_list = OutputFile(
		@@ -239,7 +241,7 @@ def AddDtbo(output_zip):
		"""
		img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "dtbo.img")
		if os.path.exists(img.input_name):
		print("dtbo.img already exists; no need to rebuild...")
		logger.info("dtbo.img already exists; no need to rebuild...")
		return img.input_name

		dtbo_prebuilt_path = os.path.join(
		@@ -269,7 +271,7 @@ def AddDtbo(output_zip):


		def CreateImage(input_dir, info_dict, what, output_file, block_list=None):
		print("creating " + what + ".img...")
		logger.info("creating " + what + ".img...")

		image_props = build_image.ImagePropFromGlobalDict(info_dict, what)
		fstab = info_dict["fstab"]
		@@ -340,7 +342,7 @@ def AddUserdata(output_zip):

		img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "userdata.img")
		if os.path.exists(img.input_name):
		print("userdata.img already exists; no need to rebuild...")
		logger.info("userdata.img already exists; no need to rebuild...")
		return

		# Skip userdata.img if no size.
		@@ -348,7 +350,7 @@ def AddUserdata(output_zip):
		if not image_props.get("partition_size"):
		return

		print("creating userdata.img...")
		logger.info("creating userdata.img...")

		image_props["timestamp"] = FIXED_FILE_TIMESTAMP

		@@ -411,7 +413,7 @@ def AddVBMeta(output_zip, partitions, name, needed_partitions):
		img = OutputFile(
		output_zip, OPTIONS.input_tmp, "IMAGES", "{}.img".format(name))
		if os.path.exists(img.input_name):
		print("{}.img already exists; not rebuilding...".format(name))
		logger.info("%s.img already exists; not rebuilding...", name)
		return img.input_name

		avbtool = os.getenv('AVBTOOL') or OPTIONS.info_dict["avb_avbtool"]
		@@ -495,7 +497,7 @@ def AddCache(output_zip):

		img = OutputFile(output_zip, OPTIONS.input_tmp, "IMAGES", "cache.img")
		if os.path.exists(img.input_name):
		print("cache.img already exists; no need to rebuild...")
		logger.info("cache.img already exists; no need to rebuild...")
		return

		image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "cache")
		@@ -503,7 +505,7 @@ def AddCache(output_zip):
		if "fs_type" not in image_props:
		return

		print("creating cache.img...")
		logger.info("creating cache.img...")

		image_props["timestamp"] = FIXED_FILE_TIMESTAMP

		@@ -580,8 +582,7 @@ def AddCareMapForAbOta(output_zip, ab_partitions, image_paths):

		present_props = [x for x in prop_name_list if x in build_props]
		if not present_props:
		print("Warning: fingerprint is not present for partition {}".
		format(partition))
		logger.warning("fingerprint is not present for partition %s", partition)
		property_id, fingerprint = "unknown", "unknown"
		else:
		property_id = present_props[0]
		@@ -633,7 +634,7 @@ def AddPackRadioImages(output_zip, images):

		prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name)
		if os.path.exists(prebuilt_path):
		print("%s already exists, no need to overwrite..." % (img_name,))
		logger.info("%s already exists, no need to overwrite...", img_name)
		continue

		img_radio_path = os.path.join(OPTIONS.input_tmp, "RADIO", img_name)
		@@ -698,7 +699,7 @@ def AddImagesToTargetFiles(filename):

		if not OPTIONS.add_missing:
		if os.path.isdir(os.path.join(OPTIONS.input_tmp, "IMAGES")):
		print("target_files appears to already contain images.")
		logger.warning("target_files appears to already contain images.")
		sys.exit(1)

		OPTIONS.info_dict = common.LoadInfoDict(OPTIONS.input_tmp, repacking=True)
		@@ -748,7 +749,7 @@ def AddImagesToTargetFiles(filename):
		partitions = dict()

		def banner(s):
		print("\n\n++++ " + s + " ++++\n\n")
		logger.info("\n\n++++ " + s + " ++++\n\n")

		banner("boot")
		# common.GetBootableImage() returns the image directly if present.
		@@ -912,20 +913,21 @@ def main(argv):
		"is_signing"],
		extra_option_handler=option_handler)


		if len(args) != 1:
		common.Usage(__doc__)
		sys.exit(1)

		common.InitLogging()

		AddImagesToTargetFiles(args[0])
		print("done.")
		logger.info("done.")

		if __name__ == '__main__':
		try:
		common.CloseInheritedPipes()
		main(sys.argv[1:])
		except common.ExternalError as e:
		print("\n ERROR: %s\n" % (e,))
		except common.ExternalError:
		logger.exception("\n ERROR:\n")
		sys.exit(1)
		finally:
		common.Cleanup()

tools/releasetools/blockimgdiff.py

+57 −62

Original line number	Diff line number	Diff line
		@@ -19,6 +19,7 @@ import copy
		import functools
		import heapq
		import itertools
		import logging
		import multiprocessing
		import os
		import os.path
		@@ -33,6 +34,8 @@ from rangelib import RangeSet

		__all__ = ["EmptyImage", "DataImage", "BlockImageDiff"]

		logger = logging.getLogger(__name__)


		def compute_patch(srcfile, tgtfile, imgdiff=False):
		patchfile = common.MakeTempFile(prefix='patch-')
		@@ -304,8 +307,8 @@ class ImgdiffStats(object):
		"""Prints a report of the collected imgdiff stats."""

		def print_header(header, separator):
		print(header)
		print(separator * len(header) + '\n')
		logger.info(header)
		logger.info(separator * len(header) + '\n')

		print_header(' Imgdiff Stats Report ', '=')
		for key in self.REASONS:
		@@ -314,7 +317,7 @@ class ImgdiffStats(object):
		values = self.stats[key]
		section_header = ' {} (count: {}) '.format(key, len(values))
		print_header(section_header, '-')
		print(''.join([' {}\n'.format(name) for name in values]))
		logger.info(''.join([' {}\n'.format(name) for name in values]))


		class BlockImageDiff(object):
		@@ -482,7 +485,7 @@ class BlockImageDiff(object):
		self.WriteTransfers(prefix)

		# Report the imgdiff stats.
		if common.OPTIONS.verbose and not self.disable_imgdiff:
		if not self.disable_imgdiff:
		self.imgdiff_stats.Report()

		def WriteTransfers(self, prefix):
		@@ -692,16 +695,17 @@ class BlockImageDiff(object):
		OPTIONS = common.OPTIONS
		if OPTIONS.cache_size is not None:
		max_allowed = OPTIONS.cache_size * OPTIONS.stash_threshold
		print("max stashed blocks: %d (%d bytes), "
		"limit: %d bytes (%.2f%%)\n" % (
		logger.info(
		"max stashed blocks: %d (%d bytes), limit: %d bytes (%.2f%%)\n",
		max_stashed_blocks, self._max_stashed_size, max_allowed,
		self._max_stashed_size * 100.0 / max_allowed))
		self._max_stashed_size * 100.0 / max_allowed)
		else:
		print("max stashed blocks: %d (%d bytes), limit: <unknown>\n" % (
		max_stashed_blocks, self._max_stashed_size))
		logger.info(
		"max stashed blocks: %d (%d bytes), limit: <unknown>\n",
		max_stashed_blocks, self._max_stashed_size)

		def ReviseStashSize(self):
		print("Revising stash size...")
		logger.info("Revising stash size...")
		stash_map = {}

		# Create the map between a stash and its def/use points. For example, for a
		@@ -746,7 +750,7 @@ class BlockImageDiff(object):
		# that will use this stash and replace the command with "new".
		use_cmd = stash_map[stash_raw_id][2]
		replaced_cmds.append(use_cmd)
		print("%10d %9s %s" % (sr.size(), "explicit", use_cmd))
		logger.info("%10d %9s %s", sr.size(), "explicit", use_cmd)
		else:
		# Update the stashes map.
		if sh in stashes:
		@@ -762,7 +766,7 @@ class BlockImageDiff(object):
		if xf.src_ranges.overlaps(xf.tgt_ranges):
		if stashed_blocks + xf.src_ranges.size() > max_allowed:
		replaced_cmds.append(xf)
		print("%10d %9s %s" % (xf.src_ranges.size(), "implicit", xf))
		logger.info("%10d %9s %s", xf.src_ranges.size(), "implicit", xf)

		# Replace the commands in replaced_cmds with "new"s.
		for cmd in replaced_cmds:
		@@ -788,28 +792,29 @@ class BlockImageDiff(object):
		stashes.pop(sh)

		num_of_bytes = new_blocks * self.tgt.blocksize
		print(" Total %d blocks (%d bytes) are packed as new blocks due to "
		"insufficient cache size." % (new_blocks, num_of_bytes))
		logger.info(
		" Total %d blocks (%d bytes) are packed as new blocks due to "
		"insufficient cache size.", new_blocks, num_of_bytes)
		return new_blocks

		def ComputePatches(self, prefix):
		print("Reticulating splines...")
		logger.info("Reticulating splines...")
		diff_queue = []
		patch_num = 0
		with open(prefix + ".new.dat", "wb") as new_f:
		for index, xf in enumerate(self.transfers):
		if xf.style == "zero":
		tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize
		print("%10d %10d (%6.2f%%) %7s %s %s" % (
		tgt_size, tgt_size, 100.0, xf.style, xf.tgt_name,
		str(xf.tgt_ranges)))
		logger.info(
		"%10d %10d (%6.2f%%) %7s %s %s", tgt_size, tgt_size, 100.0,
		xf.style, xf.tgt_name, str(xf.tgt_ranges))

		elif xf.style == "new":
		self.tgt.WriteRangeDataToFd(xf.tgt_ranges, new_f)
		tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize
		print("%10d %10d (%6.2f%%) %7s %s %s" % (
		tgt_size, tgt_size, 100.0, xf.style,
		xf.tgt_name, str(xf.tgt_ranges)))
		logger.info(
		"%10d %10d (%6.2f%%) %7s %s %s", tgt_size, tgt_size, 100.0,
		xf.style, xf.tgt_name, str(xf.tgt_ranges))

		elif xf.style == "diff":
		# We can't compare src and tgt directly because they may have
		@@ -827,11 +832,12 @@ class BlockImageDiff(object):
		xf.patch = None
		tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize
		if xf.src_ranges != xf.tgt_ranges:
		print("%10d %10d (%6.2f%%) %7s %s %s (from %s)" % (
		tgt_size, tgt_size, 100.0, xf.style,
		logger.info(
		"%10d %10d (%6.2f%%) %7s %s %s (from %s)", tgt_size, tgt_size,
		100.0, xf.style,
		xf.tgt_name if xf.tgt_name == xf.src_name else (
		xf.tgt_name + " (from " + xf.src_name + ")"),
		str(xf.tgt_ranges), str(xf.src_ranges)))
		str(xf.tgt_ranges), str(xf.src_ranges))
		else:
		if xf.patch:
		# We have already generated the patch with imgdiff, while
		@@ -850,9 +856,9 @@ class BlockImageDiff(object):

		if diff_queue:
		if self.threads > 1:
		print("Computing patches (using %d threads)..." % (self.threads,))
		logger.info("Computing patches (using %d threads)...", self.threads)
		else:
		print("Computing patches...")
		logger.info("Computing patches...")

		diff_total = len(diff_queue)
		patches = [None] * diff_total
		@@ -874,13 +880,6 @@ class BlockImageDiff(object):
		xf_index, imgdiff, patch_index = diff_queue.pop()
		xf = self.transfers[xf_index]

		if sys.stdout.isatty():
		diff_left = len(diff_queue)
		progress = (diff_total - diff_left) * 100 / diff_total
		# '\033[K' is to clear to EOL.
		print(' [%3d%%] %s\033[K' % (progress, xf.tgt_name), end='\r')
		sys.stdout.flush()

		patch = xf.patch
		if not patch:
		src_ranges = xf.src_ranges
		@@ -918,13 +917,10 @@ class BlockImageDiff(object):
		while threads:
		threads.pop().join()

		if sys.stdout.isatty():
		print('\n')

		if error_messages:
		print('ERROR:')
		print('\n'.join(error_messages))
		print('\n\n\n')
		logger.error('ERROR:')
		logger.error('\n'.join(error_messages))
		logger.error('\n\n\n')
		sys.exit(1)
		else:
		patches = []
		@@ -938,14 +934,13 @@ class BlockImageDiff(object):
		offset += xf.patch_len
		patch_fd.write(patch)

		if common.OPTIONS.verbose:
		tgt_size = xf.tgt_ranges.size() * self.tgt.blocksize
		print("%10d %10d (%6.2f%%) %7s %s %s %s" % (
		xf.patch_len, tgt_size, xf.patch_len * 100.0 / tgt_size,
		xf.style,
		logger.info(
		"%10d %10d (%6.2f%%) %7s %s %s %s", xf.patch_len, tgt_size,
		xf.patch_len * 100.0 / tgt_size, xf.style,
		xf.tgt_name if xf.tgt_name == xf.src_name else (
		xf.tgt_name + " (from " + xf.src_name + ")"),
		xf.tgt_ranges, xf.src_ranges))
		xf.tgt_ranges, xf.src_ranges)

		def AssertSha1Good(self):
		"""Check the SHA-1 of the src & tgt blocks in the transfer list.
		@@ -1005,7 +1000,7 @@ class BlockImageDiff(object):
		assert touched[i] == 1

		def ImproveVertexSequence(self):
		print("Improving vertex order...")
		logger.info("Improving vertex order...")

		# At this point our digraph is acyclic; we reversed any edges that
		# were backwards in the heuristically-generated sequence. The
		@@ -1057,7 +1052,7 @@ class BlockImageDiff(object):
		blocks will be written to the same stash slot in WriteTransfers().
		"""

		print("Reversing backward edges...")
		logger.info("Reversing backward edges...")
		in_order = 0
		out_of_order = 0
		stash_raw_id = 0
		@@ -1089,15 +1084,15 @@ class BlockImageDiff(object):
		xf.goes_after[u] = None # value doesn't matter
		u.goes_before[xf] = None

		print((" %d/%d dependencies (%.2f%%) were violated; "
		"%d source blocks stashed.") %
		(out_of_order, in_order + out_of_order,
		(out_of_order * 100.0 / (in_order + out_of_order))
		if (in_order + out_of_order) else 0.0,
		stash_size))
		logger.info(
		" %d/%d dependencies (%.2f%%) were violated; %d source blocks "
		"stashed.", out_of_order, in_order + out_of_order,
		(out_of_order * 100.0 / (in_order + out_of_order)) if (
		in_order + out_of_order) else 0.0,
		stash_size)

		def FindVertexSequence(self):
		print("Finding vertex sequence...")
		logger.info("Finding vertex sequence...")

		# This is based on "A Fast & Effective Heuristic for the Feedback
		# Arc Set Problem" by P. Eades, X. Lin, and W.F. Smyth. Think of
		@@ -1210,7 +1205,7 @@ class BlockImageDiff(object):
		self.transfers = new_transfers

		def GenerateDigraph(self):
		print("Generating digraph...")
		logger.info("Generating digraph...")

		# Each item of source_ranges will be:
		# - None, if that block is not used as a source,
		@@ -1376,9 +1371,9 @@ class BlockImageDiff(object):

		if tgt_changed < tgt_size * crop_threshold:
		assert tgt_changed + tgt_skipped.size() == tgt_size
		print('%10d %10d (%6.2f%%) %s' % (
		tgt_skipped.size(), tgt_size,
		tgt_skipped.size() * 100.0 / tgt_size, tgt_name))
		logger.info(
		'%10d %10d (%6.2f%%) %s', tgt_skipped.size(), tgt_size,
		tgt_skipped.size() * 100.0 / tgt_size, tgt_name)
		AddSplitTransfers(
		"%s-skipped" % (tgt_name,),
		"%s-skipped" % (src_name,),
		@@ -1519,7 +1514,7 @@ class BlockImageDiff(object):
		split_src_ranges,
		patch_content))

		print("Finding transfers...")
		logger.info("Finding transfers...")

		large_apks = []
		split_large_apks = []

tools/releasetools/build_image.py

+14 −9

Original line number	Diff line number	Diff line
		@@ -26,6 +26,7 @@ Usage: build_image.py input_directory properties_file output_image \\

		from __future__ import print_function

		import logging
		import os
		import os.path
		import re
		@@ -35,6 +36,8 @@ import sys
		import common
		import verity_utils

		logger = logging.getLogger(__name__)

		OPTIONS = common.OPTIONS
		BLOCK_SIZE = common.BLOCK_SIZE
		BYTES_IN_MB = 1024 * 1024
		@@ -228,8 +231,8 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None):
		"partition_size" not in prop_dict):
		# If partition_size is not defined, use output of `du' + reserved_size.
		size = GetDiskUsage(in_dir)
		if OPTIONS.verbose:
		print("The tree size of %s is %d MB." % (in_dir, size // BYTES_IN_MB))
		logger.info(
		"The tree size of %s is %d MB.", in_dir, size // BYTES_IN_MB)
		size += int(prop_dict.get("partition_reserved_size", 0))
		# Round this up to a multiple of 4K so that avbtool works
		size = common.RoundUpTo4K(size)
		@@ -241,8 +244,8 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None):
		lambda x: verity_utils.AVBCalcMaxImageSize(
		avbtool, avb_footer_type, x, avb_signing_args))
		prop_dict["partition_size"] = str(size)
		if OPTIONS.verbose:
		print("Allocating %d MB for %s." % (size // BYTES_IN_MB, out_file))
		logger.info(
		"Allocating %d MB for %s.", size // BYTES_IN_MB, out_file)

		prop_dict["image_size"] = prop_dict["partition_size"]

		@@ -350,8 +353,8 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None):
		du_str = "{} bytes ({} MB)".format(du, du // BYTES_IN_MB)
		# Suppress any errors from GetDiskUsage() to avoid hiding the real errors
		# from common.RunAndCheckOutput().
		except Exception as e: # pylint: disable=broad-except
		print(e, file=sys.stderr)
		except Exception: # pylint: disable=broad-except
		logger.exception("Failed to compute disk usage with du")
		du_str = "unknown"
		print(
		"Out of space? The tree size of {} is {}, with reserved space of {} "
		@@ -664,6 +667,8 @@ def main(argv):
		print(__doc__)
		sys.exit(1)

		common.InitLogging()

		in_dir = argv[0]
		glob_dict_file = argv[1]
		out_file = argv[2]
		@@ -697,7 +702,7 @@ def main(argv):
		elif image_filename == "product_services.img":
		mount_point = "product_services"
		else:
		print("error: unknown image file name ", image_filename, file=sys.stderr)
		logger.error("Unknown image file name %s", image_filename)
		sys.exit(1)

		image_properties = ImagePropFromGlobalDict(glob_dict, mount_point)
		@@ -705,14 +710,14 @@ def main(argv):
		try:
		BuildImage(in_dir, image_properties, out_file, target_out)
		except:
		print("Error: Failed to build {} from {}".format(out_file, in_dir),
		file=sys.stderr)
		logger.error("Failed to build %s from %s", out_file, in_dir)
		raise

		if prop_file_out:
		glob_dict_out = GlobalDictFromImageProp(image_properties, mount_point)
		SaveGlobalDict(prop_file_out, glob_dict_out)


		if __name__ == '__main__':
		try:
		main(sys.argv[1:])

tools/releasetools/check_ota_package_signature.py

+5 −1

Original line number	Diff line number	Diff line
		@@ -21,16 +21,18 @@ Verify a given OTA package with the specifed certificate.
		from __future__ import print_function

		import argparse
		import logging
		import re
		import subprocess
		import sys
		import zipfile

		from hashlib import sha1
		from hashlib import sha256

		import common

		logger = logging.getLogger(__name__)


		def CertUsesSha256(cert):
		"""Check if the cert uses SHA-256 hashing algorithm."""
		@@ -181,6 +183,8 @@ def main():
		parser.add_argument('package', help='The OTA package to be verified.')
		args = parser.parse_args()

		common.InitLogging()

		VerifyPackage(args.certificate, args.package)
		VerifyAbOtaPayload(args.certificate, args.package)

tools/releasetools/common.py

+82 −40

File changed.

Preview size limit exceeded, changes collapsed.