am ab7ca1d2: refactor BlockDifference into common (caec135f) · Commits · e / os / android_build

tools/releasetools/blockimgdiff.py

+67 −2

Original line number	Diff line number	Diff line
		@@ -14,6 +14,8 @@ import tempfile

		from rangelib import *

		__all__ = ["EmptyImage", "DataImage", "BlockImageDiff"]

		def compute_patch(src, tgt, imgdiff=False):
		srcfd, srcfile = tempfile.mkstemp(prefix="src-")
		tgtfd, tgtfile = tempfile.mkstemp(prefix="tgt-")
		@@ -60,6 +62,59 @@ class EmptyImage(object):
		file_map = {}
		def ReadRangeSet(self, ranges):
		return ()
		def TotalSha1(self):
		return sha1().hexdigest()


		class DataImage(object):
		"""An image wrapped around a single string of data."""

		def __init__(self, data, trim=False, pad=False):
		self.data = data
		self.blocksize = 4096

		assert not (trim and pad)

		partial = len(self.data) % self.blocksize
		if partial > 0:
		if trim:
		self.data = self.data[:-partial]
		elif pad:
		self.data += '\0' * (self.blocksize - partial)
		else:
		raise ValueError(("data for DataImage must be multiple of %d bytes "
		"unless trim or pad is specified") %
		(self.blocksize,))

		assert len(self.data) % self.blocksize == 0

		self.total_blocks = len(self.data) / self.blocksize
		self.care_map = RangeSet(data=(0, self.total_blocks))

		zero_blocks = []
		nonzero_blocks = []
		reference = '\0' * self.blocksize

		for i in range(self.total_blocks):
		d = self.data[iself.blocksize : (i+1)self.blocksize]
		if d == reference:
		zero_blocks.append(i)
		zero_blocks.append(i+1)
		else:
		nonzero_blocks.append(i)
		nonzero_blocks.append(i+1)

		self.file_map = {"__ZERO": RangeSet(zero_blocks),
		"__NONZERO": RangeSet(nonzero_blocks)}

		def ReadRangeSet(self, ranges):
		return [self.data[sself.blocksize:eself.blocksize] for (s, e) in ranges]

		def TotalSha1(self):
		if not hasattr(self, "sha1"):
		self.sha1 = sha1(self.data).hexdigest()
		return self.sha1


		class Transfer(object):
		def __init__(self, tgt_name, src_name, tgt_ranges, src_ranges, style, by_id):
		@@ -104,6 +159,10 @@ class Transfer(object):
		# Implementations are free to break up the data into list/tuple
		# elements in any way that is convenient.
		#
		# TotalSha1(): a function that returns (as a hex string) the SHA-1
		# hash of all the data in the image (ie, all the blocks in the
		# care_map)
		#
		# When creating a BlockImageDiff, the src image may be None, in which
		# case the list of transfers produced will never read from the
		# original image.
		@@ -478,6 +537,11 @@ class BlockImageDiff(object):
		# If the blocks written by A are read by B, then B needs to go before A.
		i = a.tgt_ranges.intersect(b.src_ranges)
		if i:
		if b.src_name == "__ZERO":
		# the cost of removing source blocks for the __ZERO domain
		# is (nearly) zero.
		size = 0
		else:
		size = i.size()
		b.goes_before[a] = size
		a.goes_after[b] = size
		@@ -491,7 +555,8 @@ class BlockImageDiff(object):
		# in any file and that are filled with zeros. We have a
		# special transfer style for zero blocks.
		src_ranges = self.src.file_map.get("__ZERO", empty)
		Transfer(tgt_fn, None, tgt_ranges, src_ranges, "zero", self.transfers)
		Transfer(tgt_fn, "__ZERO", tgt_ranges, src_ranges,
		"zero", self.transfers)
		continue

		elif tgt_fn in self.src.file_map:

tools/releasetools/common.py

+56 −0

Original line number	Diff line number	Diff line
		@@ -29,6 +29,8 @@ import threading
		import time
		import zipfile

		import blockimgdiff

		try:
		from hashlib import sha1 as sha1
		except ImportError:
		@@ -1010,6 +1012,60 @@ def ComputeDifferences(diffs):
		threads.pop().join()


		class BlockDifference:
		def __init__(self, partition, tgt, src=None):
		self.tgt = tgt
		self.src = src
		self.partition = partition

		b = blockimgdiff.BlockImageDiff(tgt, src, threads=OPTIONS.worker_threads)
		tmpdir = tempfile.mkdtemp()
		OPTIONS.tempfiles.append(tmpdir)
		self.path = os.path.join(tmpdir, partition)
		b.Compute(self.path)

		_, self.device = GetTypeAndDevice("/" + partition, OPTIONS.info_dict)

		def WriteScript(self, script, output_zip, progress=None):
		if not self.src:
		# write the output unconditionally
		if progress: script.ShowProgress(progress, 0)
		self._WriteUpdate(script, output_zip)

		else:
		script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' %
		(self.device, self.src.care_map.to_string_raw(),
		self.src.TotalSha1()))
		script.Print("Patching %s image..." % (self.partition,))
		if progress: script.ShowProgress(progress, 0)
		self._WriteUpdate(script, output_zip)
		script.AppendExtra(('else\n'
		' (range_sha1("%s", "%s") == "%s") \|\|\n'
		' abort("%s partition has unexpected contents");\n'
		'endif;') %
		(self.device, self.tgt.care_map.to_string_raw(),
		self.tgt.TotalSha1(), self.partition))

		def _WriteUpdate(self, script, output_zip):
		partition = self.partition
		with open(self.path + ".transfer.list", "rb") as f:
		ZipWriteStr(output_zip, partition + ".transfer.list", f.read())
		with open(self.path + ".new.dat", "rb") as f:
		ZipWriteStr(output_zip, partition + ".new.dat", f.read())
		with open(self.path + ".patch.dat", "rb") as f:
		ZipWriteStr(output_zip, partition + ".patch.dat", f.read(),
		compression=zipfile.ZIP_STORED)

		call = (('block_image_update("%s", '
		'package_extract_file("%s.transfer.list"), '
		'"%s.new.dat", "%s.patch.dat");\n') %
		(self.device, partition, partition, partition))
		script.AppendExtra(script._WordWrap(call))


		DataImage = blockimgdiff.DataImage


		# map recovery.fstab's fs_types to mount/format "partition types"
		PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD",
		"ext4": "EMMC", "emmc": "EMMC",

tools/releasetools/ota_from_target_files

+7 −58

Original line number	Diff line number	Diff line
		@@ -455,35 +455,6 @@ def GetImage(which, tmpdir, info_dict):
		return sparse_img.SparseImage(path, mappath)


		class BlockDifference:
		def __init__(self, partition, tgt, src=None):
		self.partition = partition

		b = blockimgdiff.BlockImageDiff(tgt, src, threads=OPTIONS.worker_threads)
		tmpdir = tempfile.mkdtemp()
		OPTIONS.tempfiles.append(tmpdir)
		self.path = os.path.join(tmpdir, partition)
		b.Compute(self.path)

		_, self.device = common.GetTypeAndDevice("/" + partition, OPTIONS.info_dict)

		def WriteScript(self, script, output_zip):
		partition = self.partition
		with open(self.path + ".transfer.list", "rb") as f:
		common.ZipWriteStr(output_zip, partition + ".transfer.list", f.read())
		with open(self.path + ".new.dat", "rb") as f:
		common.ZipWriteStr(output_zip, partition + ".new.dat", f.read())
		with open(self.path + ".patch.dat", "rb") as f:
		common.ZipWriteStr(output_zip, partition + ".patch.dat", f.read(),
		compression=zipfile.ZIP_STORED)

		call = (('block_image_update("%s", '
		'package_extract_file("%s.transfer.list"), '
		'"%s.new.dat", "%s.patch.dat");\n') %
		(self.device, partition, partition, partition))
		script.AppendExtra(script._WordWrap(call))


		def WriteFullOTAPackage(input_zip, output_zip):
		# TODO: how to determine this? We don't know what version it will
		# be installed on top of. For now, we expect the API just won't
		@@ -586,7 +557,7 @@ else if get_stage("%(bcb_dev)s", "stage") == "3/3" then
		# writes incrementals to do it.
		system_tgt = GetImage("system", OPTIONS.input_tmp, OPTIONS.info_dict)
		system_tgt.ResetFileMap()
		system_diff = BlockDifference("system", system_tgt, src=None)
		system_diff = common.BlockDifference("system", system_tgt, src=None)
		system_diff.WriteScript(script, output_zip)
		else:
		script.FormatPartition("/system")
		@@ -619,7 +590,7 @@ else if get_stage("%(bcb_dev)s", "stage") == "3/3" then
		if block_based:
		vendor_tgt = GetImage("vendor", OPTIONS.input_tmp, OPTIONS.info_dict)
		vendor_tgt.ResetFileMap()
		vendor_diff = BlockDifference("vendor", vendor_tgt)
		vendor_diff = common.BlockDifference("vendor", vendor_tgt)
		vendor_diff.WriteScript(script, output_zip)
		else:
		script.FormatPartition("/vendor")
		@@ -760,14 +731,14 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip):

		system_src = GetImage("system", OPTIONS.source_tmp, OPTIONS.source_info_dict)
		system_tgt = GetImage("system", OPTIONS.target_tmp, OPTIONS.target_info_dict)
		system_diff = BlockDifference("system", system_tgt, system_src)
		system_diff = common.BlockDifference("system", system_tgt, system_src)

		if HasVendorPartition(target_zip):
		if not HasVendorPartition(source_zip):
		raise RuntimeError("can't generate incremental that adds /vendor")
		vendor_src = GetImage("vendor", OPTIONS.source_tmp, OPTIONS.source_info_dict)
		vendor_tgt = GetImage("vendor", OPTIONS.target_tmp, OPTIONS.target_info_dict)
		vendor_diff = BlockDifference("vendor", vendor_tgt, vendor_src)
		vendor_diff = common.BlockDifference("vendor", vendor_tgt, vendor_src)
		else:
		vendor_diff = None

		@@ -867,32 +838,10 @@ else

		device_specific.IncrementalOTA_InstallBegin()

		script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' %
		(system_diff.device, system_src.care_map.to_string_raw(),
		system_src.TotalSha1()))
		script.Print("Patching system image...")
		script.ShowProgress(0.8 if vendor_diff else 0.9, 0)
		system_diff.WriteScript(script, output_zip)
		script.AppendExtra(('else\n'
		' (range_sha1("%s", "%s") == "%s") \|\|\n'
		' abort("system partition has unexpected contents");\n'
		'endif;') %
		(system_diff.device, system_tgt.care_map.to_string_raw(),
		system_tgt.TotalSha1()))

		system_diff.WriteScript(script, output_zip,
		progress=0.8 if vendor_diff else 0.9)
		if vendor_diff:
		script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' %
		(vendor_diff.device, vendor_src.care_map.to_string_raw(),
		vendor_src.TotalSha1()))
		script.Print("Patching vendor image...")
		script.ShowProgress(0.1, 0)
		vendor_diff.WriteScript(script, output_zip)
		script.AppendExtra(('else\n'
		' (range_sha1("%s", "%s") == "%s") \|\|\n'
		' abort("vendor partition has unexpected contents");\n'
		'endif;') %
		(vendor_diff.device, vendor_tgt.care_map.to_string_raw(),
		vendor_tgt.TotalSha1()))
		vendor_diff.WriteScript(script, output_zip, progress=0.1)

		if OPTIONS.two_step:
		common.ZipWriteStr(output_zip, "boot.img", target_boot.data)