Merge "Generate the compute_hash_tree command" am: 08fff56237 am: ebf3195528 (e85994c1) · Commits · e / os / android_build

tools/releasetools/blockimgdiff.py

+17 −1

Original line number	Diff line number	Diff line
		@@ -32,7 +32,6 @@ from hashlib import sha1
		import common
		from rangelib import RangeSet


		__all__ = ["EmptyImage", "DataImage", "BlockImageDiff"]


		@@ -649,6 +648,14 @@ class BlockImageDiff(object):

		self.touched_src_sha1 = self.src.RangeSha1(self.touched_src_ranges)

		if self.tgt.hashtree_info:
		out.append("compute_hash_tree {} {} {} {} {}\n".format(
		self.tgt.hashtree_info.hashtree_range.to_string_raw(),
		self.tgt.hashtree_info.filesystem_range.to_string_raw(),
		self.tgt.hashtree_info.hash_algorithm,
		self.tgt.hashtree_info.salt,
		self.tgt.hashtree_info.root_hash))

		# Zero out extended blocks as a workaround for bug 20881595.
		if self.tgt.extended:
		assert (WriteSplitTransfers(out, "zero", self.tgt.extended) ==
		@@ -988,6 +995,12 @@ class BlockImageDiff(object):
		assert touched[i] == 0
		touched[i] = 1

		if self.tgt.hashtree_info:
		for s, e in self.tgt.hashtree_info.hashtree_range:
		for i in range(s, e):
		assert touched[i] == 0
		touched[i] = 1

		# Check that we've written every target block.
		for s, e in self.tgt.care_map:
		for i in range(s, e):
		@@ -1533,6 +1546,9 @@ class BlockImageDiff(object):
		AddTransfer(tgt_fn, None, tgt_ranges, empty, "new", self.transfers)
		continue

		elif tgt_fn == "__HASHTREE":
		continue

		elif tgt_fn in self.src.file_map:
		# Look for an exact pathname match in the source.
		AddTransfer(tgt_fn, tgt_fn, tgt_ranges, self.src.file_map[tgt_fn],

tools/releasetools/common.py

+7 −4

Original line number	Diff line number	Diff line
		@@ -701,7 +701,8 @@ def UnzipTemp(filename, pattern=None):
		return tmp


		def GetSparseImage(which, tmpdir, input_zip, allow_shared_blocks):
		def GetSparseImage(which, tmpdir, input_zip, allow_shared_blocks,
		hashtree_info_generator=None):
		"""Returns a SparseImage object suitable for passing to BlockImageDiff.

		This function loads the specified sparse image from the given path, and
		@@ -714,7 +715,8 @@ def GetSparseImage(which, tmpdir, input_zip, allow_shared_blocks):
		tmpdir: The directory that contains the prebuilt image and block map file.
		input_zip: The target-files ZIP archive.
		allow_shared_blocks: Whether having shared blocks is allowed.

		hashtree_info_generator: If present, generates the hashtree_info for this
		sparse image.
		Returns:
		A SparseImage object, with file_map info loaded.
		"""
		@@ -732,8 +734,9 @@ def GetSparseImage(which, tmpdir, input_zip, allow_shared_blocks):
		# unconditionally. Note that they are still part of care_map. (Bug: 20939131)
		clobbered_blocks = "0"

		image = sparse_img.SparseImage(path, mappath, clobbered_blocks,
		allow_shared_blocks=allow_shared_blocks)
		image = sparse_img.SparseImage(
		path, mappath, clobbered_blocks, allow_shared_blocks=allow_shared_blocks,
		hashtree_info_generator=hashtree_info_generator)

		# block.map may contain less blocks, because mke2fs may skip allocating blocks
		# if they contain all zeros. We can't reconstruct such a file from its block

tools/releasetools/ota_from_target_files.py

+11 −3

Original line number	Diff line number	Diff line
		@@ -176,6 +176,7 @@ import zipfile

		import common
		import edify_generator
		import verity_utils

		if sys.hexversion < 0x02070000:
		print("Python 2.7 or newer is required.", file=sys.stderr)
		@@ -1411,8 +1412,12 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
		target_info.get('ext4_share_dup_blocks') == "true")
		system_src = common.GetSparseImage("system", OPTIONS.source_tmp, source_zip,
		allow_shared_blocks)

		hashtree_info_generator = verity_utils.CreateHashtreeInfoGenerator(
		"system", 4096, target_info)
		system_tgt = common.GetSparseImage("system", OPTIONS.target_tmp, target_zip,
		allow_shared_blocks)
		allow_shared_blocks,
		hashtree_info_generator)

		blockimgdiff_version = max(
		int(i) for i in target_info.get("blockimgdiff_versions", "1").split(","))
		@@ -1439,8 +1444,11 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_file):
		raise RuntimeError("can't generate incremental that adds /vendor")
		vendor_src = common.GetSparseImage("vendor", OPTIONS.source_tmp, source_zip,
		allow_shared_blocks)
		vendor_tgt = common.GetSparseImage("vendor", OPTIONS.target_tmp, target_zip,
		allow_shared_blocks)
		hashtree_info_generator = verity_utils.CreateHashtreeInfoGenerator(
		"vendor", 4096, target_info)
		vendor_tgt = common.GetSparseImage(
		"vendor", OPTIONS.target_tmp, target_zip, allow_shared_blocks,
		hashtree_info_generator)

		# Check first block of vendor partition for remount R/W only if
		# disk type is ext4

tools/releasetools/sparse_img.py

+24 −3

Original line number	Diff line number	Diff line
		@@ -33,7 +33,8 @@ class SparseImage(object):
		"""

		def __init__(self, simg_fn, file_map_fn=None, clobbered_blocks=None,
		mode="rb", build_map=True, allow_shared_blocks=False):
		mode="rb", build_map=True, allow_shared_blocks=False,
		hashtree_info_generator=None):
		self.simg_f = f = open(simg_fn, mode)

		header_bin = f.read(28)
		@@ -64,6 +65,8 @@ class SparseImage(object):
		% (total_blks, blk_sz, total_chunks))

		if not build_map:
		assert not hashtree_info_generator, \
		"Cannot generate the hashtree info without building the offset map."
		return

		pos = 0 # in blocks
		@@ -102,7 +105,17 @@ class SparseImage(object):
		if data_sz != 0:
		raise ValueError("Don't care chunk input size is non-zero (%u)" %
		(data_sz))
		else:
		# Fills the don't care data ranges with zeros.
		# TODO(xunchang) pass the care_map to hashtree info generator.
		if hashtree_info_generator:
		fill_data = '\x00' * 4
		# In order to compute verity hashtree on device, we need to write
		# zeros explicitly to the don't care ranges. Because these ranges may
		# contain non-zero data from the previous build.
		care_data.append(pos)
		care_data.append(pos + chunk_sz)
		offset_map.append((pos, chunk_sz, None, fill_data))

		pos += chunk_sz

		elif chunk_type == 0xCAC4:
		@@ -128,6 +141,10 @@ class SparseImage(object):
		extended = extended.intersect(all_blocks).subtract(self.care_map)
		self.extended = extended

		self.hashtree_info = None
		if hashtree_info_generator:
		self.hashtree_info = hashtree_info_generator.Generate(self)

		if file_map_fn:
		self.LoadFileBlockMap(file_map_fn, self.clobbered_blocks,
		allow_shared_blocks)
		@@ -246,6 +263,8 @@ class SparseImage(object):
		remaining = remaining.subtract(ranges)

		remaining = remaining.subtract(clobbered_blocks)
		if self.hashtree_info:
		remaining = remaining.subtract(self.hashtree_info.hashtree_range)

		# For all the remaining blocks in the care_map (ie, those that
		# aren't part of the data for any file nor part of the clobbered_blocks),
		@@ -308,6 +327,8 @@ class SparseImage(object):
		out["__NONZERO-%d" % i] = rangelib.RangeSet(data=blocks)
		if clobbered_blocks:
		out["__COPY"] = clobbered_blocks
		if self.hashtree_info:
		out["__HASHTREE"] = self.hashtree_info.hashtree_range

		def ResetFileMap(self):
		"""Throw away the file map and treat the entire image as

tools/releasetools/test_verity_utils.py

0 → 100644

+168 −0

Original line number	Diff line number	Diff line
		#
		# Copyright (C) 2018 The Android Open Source Project
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.
		#

		"""Unittests for verity_utils.py."""

		from __future__ import print_function

		import os
		import os.path
		import unittest

		import build_image
		import common
		import sparse_img
		import test_utils
		import verity_utils
		from rangelib import RangeSet


		class VerityUtilsTest(unittest.TestCase):
		def setUp(self):
		self.testdata_dir = test_utils.get_testdata_dir()

		self.partition_size = 1024 * 1024
		self.prop_dict = {
		'verity': 'true',
		'verity_fec': 'true',
		'system_verity_block_device': '/dev/block/system',
		'system_size': self.partition_size
		}

		self.hash_algorithm = "sha256"
		self.fixed_salt = \
		"aee087a5be3b982978c923f566a94613496b417f2af592639bc80d141e34dfe7"
		self.expected_root_hash = \
		"0b7c4565e87b1026e11fbab91c0bc29e185c847a5b44d40e6e86e461e8adf80d"

		def tearDown(self):
		common.Cleanup()

		def _create_simg(self, raw_data):
		output_file = common.MakeTempFile()
		raw_image = common.MakeTempFile()
		with open(raw_image, 'wb') as f:
		f.write(raw_data)

		cmd = ["img2simg", raw_image, output_file, '4096']
		p = common.Run(cmd)
		p.communicate()
		self.assertEqual(0, p.returncode)

		return output_file

		def _generate_image(self):
		partition_size = 1024 * 1024
		adjusted_size, verity_size = build_image.AdjustPartitionSizeForVerity(
		partition_size, True)

		raw_image = ""
		for i in range(adjusted_size):
		raw_image += str(i % 10)

		output_file = self._create_simg(raw_image)

		# Append the verity metadata.
		prop_dict = {
		'partition_size': str(partition_size),
		'image_size': str(adjusted_size),
		'verity_block_device': '/dev/block/system',
		'verity_key': os.path.join(self.testdata_dir, 'testkey'),
		'verity_signer_cmd': 'verity_signer',
		'verity_size': str(verity_size),
		}
		build_image.MakeVerityEnabledImage(output_file, True, prop_dict)

		return output_file

		def test_VerifiedBootVersion1HashtreeInfoGenerator_create(self):
		image_file = sparse_img.SparseImage(self._generate_image())

		generator = verity_utils.CreateHashtreeInfoGenerator(
		'system', image_file, self.prop_dict)
		self.assertEqual(
		verity_utils.VerifiedBootVersion1HashtreeInfoGenerator, type(generator))
		self.assertEqual(self.partition_size, generator.partition_size)
		self.assertTrue(generator.fec_supported)

		def test_VerifiedBootVersion1HashtreeInfoGenerator_decomposeImage(self):
		image_file = sparse_img.SparseImage(self._generate_image())

		generator = verity_utils.VerifiedBootVersion1HashtreeInfoGenerator(
		self.partition_size, 4096, True)
		generator.DecomposeSparseImage(image_file)
		self.assertEqual(991232, generator.filesystem_size)
		self.assertEqual(12288, generator.hashtree_size)
		self.assertEqual(32768, generator.metadata_size)

		def test_VerifiedBootVersion1HashtreeInfoGenerator_parseHashtreeMetadata(
		self):
		image_file = sparse_img.SparseImage(self._generate_image())
		generator = verity_utils.VerifiedBootVersion1HashtreeInfoGenerator(
		self.partition_size, 4096, True)
		generator.DecomposeSparseImage(image_file)

		generator._ParseHashtreeMetadata()

		self.assertEqual(
		self.hash_algorithm, generator.hashtree_info.hash_algorithm)
		self.assertEqual(self.fixed_salt, generator.hashtree_info.salt)
		self.assertEqual(self.expected_root_hash, generator.hashtree_info.root_hash)

		def test_VerifiedBootVersion1HashtreeInfoGenerator_validateHashtree_smoke(
		self):
		generator = verity_utils.VerifiedBootVersion1HashtreeInfoGenerator(
		self.partition_size, 4096, True)
		generator.image = sparse_img.SparseImage(self._generate_image())

		generator.hashtree_info = info = verity_utils.HashtreeInfo()
		info.filesystem_range = RangeSet(data=[0, 991232 / 4096])
		info.hashtree_range = RangeSet(
		data=[991232 / 4096, (991232 + 12288) / 4096])
		info.hash_algorithm = self.hash_algorithm
		info.salt = self.fixed_salt
		info.root_hash = self.expected_root_hash

		self.assertTrue(generator.ValidateHashtree())

		def test_VerifiedBootVersion1HashtreeInfoGenerator_validateHashtree_failure(
		self):
		generator = verity_utils.VerifiedBootVersion1HashtreeInfoGenerator(
		self.partition_size, 4096, True)
		generator.image = sparse_img.SparseImage(self._generate_image())

		generator.hashtree_info = info = verity_utils.HashtreeInfo()
		info.filesystem_range = RangeSet(data=[0, 991232 / 4096])
		info.hashtree_range = RangeSet(
		data=[991232 / 4096, (991232 + 12288) / 4096])
		info.hash_algorithm = self.hash_algorithm
		info.salt = self.fixed_salt
		info.root_hash = "a" + self.expected_root_hash[1:]

		self.assertFalse(generator.ValidateHashtree())

		def test_VerifiedBootVersion1HashtreeInfoGenerator_generate(self):
		image_file = sparse_img.SparseImage(self._generate_image())
		generator = verity_utils.CreateHashtreeInfoGenerator(
		'system', 4096, self.prop_dict)
		info = generator.Generate(image_file)

		self.assertEqual(RangeSet(data=[0, 991232 / 4096]), info.filesystem_range)
		self.assertEqual(RangeSet(data=[991232 / 4096, (991232 + 12288) / 4096]),
		info.hashtree_range)
		self.assertEqual(self.hash_algorithm, info.hash_algorithm)
		self.assertEqual(self.fixed_salt, info.salt)
		self.assertEqual(self.expected_root_hash, info.root_hash)