Merge "releasetools: Make scripts pylint clean." (23a40704) · Commits · e / os / android_build

tools/releasetools/blockimgdiff.py

+40 −41

Original line number	Diff line number	Diff line
		@@ -163,7 +163,7 @@ class DataImage(Image):

		def RangeSha1(self, ranges):
		h = sha1()
		for data in self._GetRangeData(ranges):
		for data in self._GetRangeData(ranges): # pylint: disable=not-an-iterable
		h.update(data)
		return h.hexdigest()

		@@ -177,7 +177,7 @@ class DataImage(Image):
		return sha1(self.data).hexdigest()

		def WriteRangeDataToFd(self, ranges, fd):
		for data in self._GetRangeData(ranges):
		for data in self._GetRangeData(ranges): # pylint: disable=not-an-iterable
		fd.write(data)


		@@ -320,46 +320,45 @@ class ImgdiffStats(object):
		print(''.join([' {}\n'.format(name) for name in values]))


		# BlockImageDiff works on two image objects. An image object is
		# anything that provides the following attributes:
		#
		# blocksize: the size in bytes of a block, currently must be 4096.
		#
		# total_blocks: the total size of the partition/image, in blocks.
		#
		# care_map: a RangeSet containing which blocks (in the range [0,
		# total_blocks) we actually care about; i.e. which blocks contain
		# data.
		#
		# file_map: a dict that partitions the blocks contained in care_map
		# into smaller domains that are useful for doing diffs on.
		# (Typically a domain is a file, and the key in file_map is the
		# pathname.)
		#
		# clobbered_blocks: a RangeSet containing which blocks contain data
		# but may be altered by the FS. They need to be excluded when
		# verifying the partition integrity.
		#
		# ReadRangeSet(): a function that takes a RangeSet and returns the
		# data contained in the image blocks of that RangeSet. The data
		# is returned as a list or tuple of strings; concatenating the
		# elements together should produce the requested data.
		# Implementations are free to break up the data into list/tuple
		# elements in any way that is convenient.
		#
		# RangeSha1(): a function that returns (as a hex string) the SHA-1
		# hash of all the data in the specified range.
		#
		# TotalSha1(): a function that returns (as a hex string) the SHA-1
		# hash of all the data in the image (ie, all the blocks in the
		# care_map minus clobbered_blocks, or including the clobbered
		# blocks if include_clobbered_blocks is True).
		#
		# When creating a BlockImageDiff, the src image may be None, in which
		# case the list of transfers produced will never read from the
		# original image.

		class BlockImageDiff(object):
		"""Generates the diff of two block image objects.

		BlockImageDiff works on two image objects. An image object is anything that
		provides the following attributes:

		blocksize: the size in bytes of a block, currently must be 4096.

		total_blocks: the total size of the partition/image, in blocks.

		care_map: a RangeSet containing which blocks (in the range [0,
		total_blocks) we actually care about; i.e. which blocks contain data.

		file_map: a dict that partitions the blocks contained in care_map into
		smaller domains that are useful for doing diffs on. (Typically a domain
		is a file, and the key in file_map is the pathname.)

		clobbered_blocks: a RangeSet containing which blocks contain data but may
		be altered by the FS. They need to be excluded when verifying the
		partition integrity.

		ReadRangeSet(): a function that takes a RangeSet and returns the data
		contained in the image blocks of that RangeSet. The data is returned as
		a list or tuple of strings; concatenating the elements together should
		produce the requested data. Implementations are free to break up the
		data into list/tuple elements in any way that is convenient.

		RangeSha1(): a function that returns (as a hex string) the SHA-1 hash of
		all the data in the specified range.

		TotalSha1(): a function that returns (as a hex string) the SHA-1 hash of
		all the data in the image (ie, all the blocks in the care_map minus
		clobbered_blocks, or including the clobbered blocks if
		include_clobbered_blocks is True).

		When creating a BlockImageDiff, the src image may be None, in which case the
		list of transfers produced will never read from the original image.
		"""

		def __init__(self, tgt, src=None, threads=None, version=4,
		disable_imgdiff=False):
		if threads is None:

tools/releasetools/common.py

+64 −51

Original line number	Diff line number	Diff line
		@@ -44,7 +44,7 @@ class Options(object):
		"darwin": "out/host/darwin-x86",
		}

		self.search_path = platform_search_path.get(sys.platform, None)
		self.search_path = platform_search_path.get(sys.platform)
		self.signapk_path = "framework/signapk.jar" # Relative to search_path
		self.signapk_shared_library_path = "lib64" # Relative to search_path
		self.extra_signapk_args = []
		@@ -237,15 +237,15 @@ def LoadInfoDict(input_file, input_dir=None):
		makeint("boot_size")
		makeint("fstab_version")

		system_root_image = d.get("system_root_image", None) == "true"
		if d.get("no_recovery", None) != "true":
		system_root_image = d.get("system_root_image") == "true"
		if d.get("no_recovery") != "true":
		recovery_fstab_path = "RECOVERY/RAMDISK/etc/recovery.fstab"
		d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"],
		recovery_fstab_path, system_root_image)
		elif d.get("recovery_as_boot", None) == "true":
		d["fstab"] = LoadRecoveryFSTab(
		read_helper, d["fstab_version"], recovery_fstab_path, system_root_image)
		elif d.get("recovery_as_boot") == "true":
		recovery_fstab_path = "BOOT/RAMDISK/etc/recovery.fstab"
		d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"],
		recovery_fstab_path, system_root_image)
		d["fstab"] = LoadRecoveryFSTab(
		read_helper, d["fstab_version"], recovery_fstab_path, system_root_image)
		else:
		d["fstab"] = None

		@@ -441,11 +441,11 @@ def _BuildBootableImage(sourcedir, fs_config_file, info_dict=None,
		cmd.append("--pagesize")
		cmd.append(open(fn).read().rstrip("\n"))

		args = info_dict.get("mkbootimg_args", None)
		args = info_dict.get("mkbootimg_args")
		if args and args.strip():
		cmd.extend(shlex.split(args))

		args = info_dict.get("mkbootimg_version_args", None)
		args = info_dict.get("mkbootimg_version_args")
		if args and args.strip():
		cmd.extend(shlex.split(args))

		@@ -453,7 +453,7 @@ def _BuildBootableImage(sourcedir, fs_config_file, info_dict=None,
		cmd.extend(["--ramdisk", ramdisk_img.name])

		img_unsigned = None
		if info_dict.get("vboot", None):
		if info_dict.get("vboot"):
		img_unsigned = tempfile.NamedTemporaryFile()
		cmd.extend(["--output", img_unsigned.name])
		else:
		@@ -471,8 +471,8 @@ def _BuildBootableImage(sourcedir, fs_config_file, info_dict=None,
		p.communicate()
		assert p.returncode == 0, "mkbootimg of %s image failed" % (partition_name,)

		if (info_dict.get("boot_signer", None) == "true" and
		info_dict.get("verity_key", None)):
		if (info_dict.get("boot_signer") == "true" and
		info_dict.get("verity_key")):
		# Hard-code the path as "/boot" for two-step special recovery image (which
		# will be loaded into /boot during the two-step OTA).
		if two_step_image:
		@@ -489,7 +489,7 @@ def _BuildBootableImage(sourcedir, fs_config_file, info_dict=None,
		assert p.returncode == 0, "boot_signer of %s image failed" % path

		# Sign the image if vboot is non-empty.
		elif info_dict.get("vboot", None):
		elif info_dict.get("vboot"):
		path = "/" + partition_name
		img_keyblock = tempfile.NamedTemporaryFile()
		# We have switched from the prebuilt futility binary to using the tool
		@@ -578,9 +578,9 @@ def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir,


		def Gunzip(in_filename, out_filename):
		"""Gunzip the given gzip compressed file to a given output file.
		"""
		with gzip.open(in_filename, "rb") as in_file, open(out_filename, "wb") as out_file:
		"""Gunzips the given gzip compressed file to a given output file."""
		with gzip.open(in_filename, "rb") as in_file, \
		open(out_filename, "wb") as out_file:
		shutil.copyfileobj(in_file, out_file)


		@@ -734,7 +734,7 @@ def GetKeyPasswords(keylist):
		devnull.close()

		key_passwords.update(PasswordManager().GetPasswords(need_passwords))
		key_passwords.update(dict.fromkeys(no_passwords, None))
		key_passwords.update(dict.fromkeys(no_passwords))
		return key_passwords


		@@ -799,8 +799,7 @@ def GetMinSdkVersionInt(apk_name, codename_to_api_level_map):


		def SignFile(input_name, output_name, key, password, min_api_level=None,
		codename_to_api_level_map=dict(),
		whole_file=False):
		codename_to_api_level_map=None, whole_file=False):
		"""Sign the input_name zip/jar/apk, producing output_name. Use the
		given key and password (the latter may be None if the key does not
		have a password.
		@@ -816,6 +815,8 @@ def SignFile(input_name, output_name, key, password, min_api_level=None,
		codename_to_api_level_map is needed to translate the codename which may be
		encountered as the APK's minSdkVersion.
		"""
		if codename_to_api_level_map is None:
		codename_to_api_level_map = {}

		java_library_path = os.path.join(
		OPTIONS.search_path, OPTIONS.signapk_shared_library_path)
		@@ -877,7 +878,7 @@ def CheckSize(data, target, info_dict):
		device = p.device
		if "/" in device:
		device = device[device.rfind("/")+1:]
		limit = info_dict.get(device + "_size", None)
		limit = info_dict.get(device + "_size")
		if not fs_type or not limit:
		return

		@@ -1107,8 +1108,8 @@ def Cleanup():

		class PasswordManager(object):
		def __init__(self):
		self.editor = os.getenv("EDITOR", None)
		self.pwfile = os.getenv("ANDROID_PW_FILE", None)
		self.editor = os.getenv("EDITOR")
		self.pwfile = os.getenv("ANDROID_PW_FILE")

		def GetPasswords(self, items):
		"""Get passwords corresponding to each string in 'items',
		@@ -1358,7 +1359,7 @@ class DeviceSpecificParams(object):
		module does not define the function, return the value of the
		'default' kwarg (which itself defaults to None)."""
		if self.module is None or not hasattr(self.module, function_name):
		return kwargs.get("default", None)
		return kwargs.get("default")
		return getattr(self.module, function_name)(((self,) + args), *kwargs)

		def FullOTA_Assertions(self):
		@@ -1408,6 +1409,7 @@ class DeviceSpecificParams(object):
		def VerifyOTA_Assertions(self):
		return self._DoCall("VerifyOTA_Assertions")


		class File(object):
		def __init__(self, name, data, compress_size=None):
		self.name = name
		@@ -1436,6 +1438,7 @@ class File(object):
		def AddToZip(self, z, compression=None):
		ZipWriteStr(z, self.name, self.data, compress_type=compression)


		DIFF_PROGRAM_BY_EXT = {
		".gz" : "imgdiff",
		".zip" : ["imgdiff", "-z"],
		@@ -1444,6 +1447,7 @@ DIFF_PROGRAM_BY_EXT = {
		".img" : "imgdiff",
		}


		class Difference(object):
		def __init__(self, tf, sf, diff_program=None):
		self.tf = tf
		@@ -1511,9 +1515,11 @@ class Difference(object):


		def GetPatch(self):
		"""Return a tuple (target_file, source_file, patch_data).
		"""Returns a tuple of (target_file, source_file, patch_data).

		patch_data may be None if ComputePatch hasn't been called, or if
		computing the patch failed."""
		computing the patch failed.
		"""
		return self.tf, self.sf, self.patch


		@@ -1545,7 +1551,8 @@ def ComputeDifferences(diffs):
		else:
		name = "%s (%s)" % (tf.name, sf.name)
		if patch is None:
		print("patching failed! %s" % (name,))
		print(
		"patching failed! %s" % (name,))
		else:
		print("%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
		dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name))
		@@ -1598,7 +1605,8 @@ class BlockDifference(object):
		def required_cache(self):
		return self._required_cache

		def WriteScript(self, script, output_zip, progress=None):
		def WriteScript(self, script, output_zip, progress=None,
		write_verify_script=False):
		if not self.src:
		# write the output unconditionally
		script.Print("Patching %s image unconditionally..." % (self.partition,))
		@@ -1608,7 +1616,8 @@ class BlockDifference(object):
		if progress:
		script.ShowProgress(progress, 0)
		self._WriteUpdate(script, output_zip)
		if OPTIONS.verify:

		if write_verify_script:
		self._WritePostInstallVerifyScript(script)

		def WriteStrictVerifyScript(self, script):
		@@ -1622,8 +1631,8 @@ class BlockDifference(object):
		script.Print("Verifying %s..." % (partition,))
		ranges = self.tgt.care_map
		ranges_str = ranges.to_string_raw()
		script.AppendExtra('range_sha1("%s", "%s") == "%s" && '
		'ui_print(" Verified.") \|\| '
		script.AppendExtra(
		'range_sha1("%s", "%s") == "%s" && ui_print(" Verified.") \|\| '
		'ui_print("\\"%s\\" has unexpected contents.");' % (
		self.device, ranges_str,
		self.tgt.TotalSha1(include_clobbered_blocks=True),
		@@ -1651,10 +1660,10 @@ class BlockDifference(object):
		return

		ranges_str = ranges.to_string_raw()
		script.AppendExtra(('if (range_sha1("%s", "%s") == "%s" \|\| '
		'block_image_verify("%s", '
		'package_extract_file("%s.transfer.list"), '
		'"%s.new.dat", "%s.patch.dat")) then') % (
		script.AppendExtra(
		'if (range_sha1("%s", "%s") == "%s" \|\| block_image_verify("%s", '
		'package_extract_file("%s.transfer.list"), "%s.new.dat", '
		'"%s.patch.dat")) then' % (
		self.device, ranges_str, expected_sha1,
		self.device, partition, partition, partition))
		script.Print('Verified %s image...' % (partition,))
		@@ -1706,7 +1715,8 @@ class BlockDifference(object):
		# Unlike pre-install verification, clobbered_blocks should not be ignored.
		ranges = self.tgt.care_map
		ranges_str = ranges.to_string_raw()
		script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % (
		script.AppendExtra(
		'if range_sha1("%s", "%s") == "%s" then' % (
		self.device, ranges_str,
		self.tgt.TotalSha1(include_clobbered_blocks=True)))

		@@ -1714,7 +1724,8 @@ class BlockDifference(object):
		# Verify that extended blocks are really zeroed out.
		if self.tgt.extended:
		ranges_str = self.tgt.extended.to_string_raw()
		script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % (
		script.AppendExtra(
		'if range_sha1("%s", "%s") == "%s" then' % (
		self.device, ranges_str,
		self._HashZeroBlocks(self.tgt.extended.size())))
		script.Print('Verified the updated %s image.' % (partition,))
		@@ -1746,9 +1757,9 @@ class BlockDifference(object):
		'{}.transfer.list'.format(self.path),
		'{}.transfer.list'.format(self.partition))

		# For full OTA, compress the new.dat with brotli with quality 6 to reduce its size. Quailty 9
		# almost triples the compression time but doesn't further reduce the size too much.
		# For a typical 1.8G system.new.dat
		# For full OTA, compress the new.dat with brotli with quality 6 to reduce
		# its size. Quailty 9 almost triples the compression time but doesn't
		# further reduce the size too much. For a typical 1.8G system.new.dat
		# zip \| brotli(quality 6) \| brotli(quality 9)
		# compressed_size: 942M \| 869M (~8% reduced) \| 854M
		# compression_time: 75s \| 265s \| 719s
		@@ -1813,6 +1824,7 @@ class BlockDifference(object):

		DataImage = blockimgdiff.DataImage


		# map recovery.fstab's fs_types to mount/format "partition types"
		PARTITION_TYPES = {
		"ext4": "EMMC",
		@@ -1821,6 +1833,7 @@ PARTITION_TYPES = {
		"squashfs": "EMMC"
		}


		def GetTypeAndDevice(mount_point, info):
		fstab = info["fstab"]
		if fstab:

tools/releasetools/edify_generator.py

+4 −4

Original line number	Diff line number	Diff line
		@@ -132,8 +132,8 @@ class EdifyGenerator(object):
		self.script.append(
		('(!less_than_int(%s, getprop("ro.build.date.utc"))) \|\| '
		'abort("E%d: Can\'t install this package (%s) over newer '
		'build (" + getprop("ro.build.date") + ").");') % (timestamp,
		common.ErrorCode.OLDER_BUILD, timestamp_text))
		'build (" + getprop("ro.build.date") + ").");') % (
		timestamp, common.ErrorCode.OLDER_BUILD, timestamp_text))

		def AssertDevice(self, device):
		"""Assert that the device identifier is the given string."""
		@@ -260,8 +260,8 @@ class EdifyGenerator(object):
		cmd.append(',\0%s,\0package_extract_file("%s")' % patchpairs[i:i+2])
		cmd.append(') \|\|\n abort("E%d: Failed to apply patch to %s");' % (
		common.ErrorCode.APPLY_PATCH_FAILURE, srcfile))
		cmd = "".join(cmd)
		self.script.append(self.WordWrap(cmd))
		cmd_str = "".join(cmd)
		self.script.append(self.WordWrap(cmd_str))

		def WriteRawImage(self, mount_point, fn, mapfn=None):
		"""Write the given package file into the partition for the given

tools/releasetools/img_from_target_files.py

+9 −8

Original line number	Diff line number	Diff line
		@@ -28,17 +28,17 @@ Usage: img_from_target_files [flags] input_target_files output_image_zip

		from __future__ import print_function

		import os
		import shutil
		import sys
		import zipfile

		import common

		if sys.hexversion < 0x02070000:
		print("Python 2.7 or newer is required.", file=sys.stderr)
		sys.exit(1)

		import os
		import shutil
		import zipfile

		import common

		OPTIONS = common.OPTIONS

		@@ -51,11 +51,12 @@ def CopyInfo(output_zip):


		def main(argv):
		bootable_only = [False]
		# This allows modifying the value from inner function.
		bootable_only_array = [False]

		def option_handler(o, _):
		if o in ("-z", "--bootable_zip"):
		bootable_only[0] = True
		bootable_only_array[0] = True
		else:
		return False
		return True
		@@ -65,7 +66,7 @@ def main(argv):
		extra_long_opts=["bootable_zip"],
		extra_option_handler=option_handler)

		bootable_only = bootable_only[0]
		bootable_only = bootable_only_array[0]

		if len(args) != 2:
		common.Usage(__doc__)

tools/releasetools/ota_from_target_files.py

+8 −4

Original line number	Diff line number	Diff line
		@@ -827,7 +827,8 @@ else if get_stage("%(bcb_dev)s") == "3/3" then
		allow_shared_blocks)
		system_tgt.ResetFileMap()
		system_diff = common.BlockDifference("system", system_tgt, src=None)
		system_diff.WriteScript(script, output_zip)
		system_diff.WriteScript(script, output_zip,
		write_verify_script=OPTIONS.verify)

		boot_img = common.GetBootableImage(
		"boot.img", "boot.img", OPTIONS.input_tmp, "BOOT")
		@@ -839,7 +840,8 @@ else if get_stage("%(bcb_dev)s") == "3/3" then
		allow_shared_blocks)
		vendor_tgt.ResetFileMap()
		vendor_diff = common.BlockDifference("vendor", vendor_tgt)
		vendor_diff.WriteScript(script, output_zip)
		vendor_diff.WriteScript(script, output_zip,
		write_verify_script=OPTIONS.verify)

		AddCompatibilityArchiveIfTrebleEnabled(input_zip, output_zip, target_info)

		@@ -1559,10 +1561,12 @@ else
		device_specific.IncrementalOTA_InstallBegin()

		system_diff.WriteScript(script, output_zip,
		progress=0.8 if vendor_diff else 0.9)
		progress=0.8 if vendor_diff else 0.9,
		write_verify_script=OPTIONS.verify)

		if vendor_diff:
		vendor_diff.WriteScript(script, output_zip, progress=0.1)
		vendor_diff.WriteScript(script, output_zip, progress=0.1,
		write_verify_script=OPTIONS.verify)

		if OPTIONS.two_step:
		common.ZipWriteStr(output_zip, "boot.img", target_boot.data)