Allow system images larger than 2GiB. (cd082d4b) · Commits · e / os / android_build

tools/releasetools/add_img_to_target_files.py

+4 −12

Original line number	Original line	Diff line number	Diff line
	@@ -30,9 +30,6 @@ if sys.hexversion < 0x02070000:

	import errno		import errno
	import os		import os
	import re
	import shutil
	import subprocess
	import tempfile		import tempfile
	import zipfile		import zipfile

	@@ -70,10 +67,8 @@ def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None):
	block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map")		block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map")
	imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict,		imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict,
	block_list=block_list)		block_list=block_list)
	with open(imgname, "rb") as f:		common.ZipWrite(output_zip, imgname, prefix + "system.img")
	common.ZipWriteStr(output_zip, prefix + "system.img", f.read())		common.ZipWrite(output_zip, block_list, prefix + "system.map")
	with open(block_list, "rb") as f:
	common.ZipWriteStr(output_zip, prefix + "system.map", f.read())


	def BuildSystem(input_dir, info_dict, block_list=None):		def BuildSystem(input_dir, info_dict, block_list=None):
	@@ -94,10 +89,8 @@ def AddVendor(output_zip, prefix="IMAGES/"):
	block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map")		block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map")
	imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict,		imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict,
	block_list=block_list)		block_list=block_list)
	with open(imgname, "rb") as f:		common.ZipWrite(output_zip, imgname, prefix + "vendor.img")
	common.ZipWriteStr(output_zip, prefix + "vendor.img", f.read())		common.ZipWrite(output_zip, block_list, prefix + "vendor.map")
	with open(block_list, "rb") as f:
	common.ZipWriteStr(output_zip, prefix + "vendor.map", f.read())


	def BuildVendor(input_dir, info_dict, block_list=None):		def BuildVendor(input_dir, info_dict, block_list=None):
	@@ -296,7 +289,6 @@ def AddImagesToTargetFiles(filename):
	output_zip.close()		output_zip.close()

	def main(argv):		def main(argv):

	def option_handler(o, a):		def option_handler(o, a):
	if o in ("-a", "--add_missing"):		if o in ("-a", "--add_missing"):
	OPTIONS.add_missing = True		OPTIONS.add_missing = True

tools/releasetools/common.py

+53 −13

Original line number	Original line	Diff line number	Diff line
	@@ -781,6 +781,44 @@ class PasswordManager(object):
	return result		return result


			def ZipWrite(zip_file, filename, arcname=None, perms=0o644,
			compress_type=None):
			import datetime

			# http://b/18015246
			# Python 2.7's zipfile implementation wrongly thinks that zip64 is required
			# for files larger than 2GiB. We can work around this by adjusting their
			# limit. Note that `zipfile.writestr()` will not work for strings larger than
			# 2GiB. The Python interpreter sometimes rejects strings that large (though
			# it isn't clear to me exactly what circumstances cause this).
			# `zipfile.write()` must be used directly to work around this.
			#
			# This mess can be avoided if we port to python3.
			saved_zip64_limit = zipfile.ZIP64_LIMIT
			zipfile.ZIP64_LIMIT = (1 << 32) - 1

			compress_type = compress_type or zip_file.compression
			arcname = arcname or filename

			saved_stat = os.stat(filename)

			try:
			# `zipfile.write()` doesn't allow us to pass ZipInfo, so just modify the
			# file to be zipped and reset it when we're done.
			os.chmod(filename, perms)

			# Use a fixed timestamp so the output is repeatable.
			epoch = datetime.datetime.fromtimestamp(0)
			timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds()
			os.utime(filename, (timestamp, timestamp))

			zip_file.write(filename, arcname=arcname, compress_type=compress_type)
			finally:
			os.chmod(filename, saved_stat.st_mode)
			os.utime(filename, (saved_stat.st_atime, saved_stat.st_mtime))
			zipfile.ZIP64_LIMIT = saved_zip64_limit


	def ZipWriteStr(zip, filename, data, perms=0644, compression=None):		def ZipWriteStr(zip, filename, data, perms=0644, compression=None):
	# use a fixed timestamp so the output is repeatable.		# use a fixed timestamp so the output is repeatable.
	zinfo = zipfile.ZipInfo(filename=filename,		zinfo = zipfile.ZipInfo(filename=filename,
	@@ -1074,19 +1112,21 @@ class BlockDifference:
	self.tgt.TotalSha1(), self.partition))		self.tgt.TotalSha1(), self.partition))

	def _WriteUpdate(self, script, output_zip):		def _WriteUpdate(self, script, output_zip):
	partition = self.partition		ZipWrite(output_zip,
	with open(self.path + ".transfer.list", "rb") as f:		'{}.transfer.list'.format(self.path),
	ZipWriteStr(output_zip, partition + ".transfer.list", f.read())		'{}.transfer.list'.format(self.partition))
	with open(self.path + ".new.dat", "rb") as f:		ZipWrite(output_zip,
	ZipWriteStr(output_zip, partition + ".new.dat", f.read())		'{}.new.dat'.format(self.path),
	with open(self.path + ".patch.dat", "rb") as f:		'{}.new.dat'.format(self.partition))
	ZipWriteStr(output_zip, partition + ".patch.dat", f.read(),		ZipWrite(output_zip,
	compression=zipfile.ZIP_STORED)		'{}.patch.dat'.format(self.path),
			'{}.patch.dat'.format(self.partition),
	call = (('block_image_update("%s", '		compress_type=zipfile.ZIP_STORED)
	'package_extract_file("%s.transfer.list"), '
	'"%s.new.dat", "%s.patch.dat");\n') %		call = ('block_image_update("{device}", '
	(self.device, partition, partition, partition))		'package_extract_file("{partition}.transfer.list"), '
			'"{partition}.new.dat", "{partition}.patch.dat");\n'.format(
			device=self.device, partition=self.partition))
	script.AppendExtra(script._WordWrap(call))		script.AppendExtra(script._WordWrap(call))

	def _CheckFirstBlock(self, script):		def _CheckFirstBlock(self, script):

tools/releasetools/img_from_target_files.py

+7 −5

Original line number	Original line	Diff line number	Diff line
	@@ -88,11 +88,13 @@ def main(argv):
	# and all we have to do is copy them to the output zip.		# and all we have to do is copy them to the output zip.
	images = os.listdir(images_path)		images = os.listdir(images_path)
	if images:		if images:
	for i in images:		for image in images:
	if bootable_only and i not in ("boot.img", "recovery.img"): continue		if bootable_only and image not in ("boot.img", "recovery.img"):
	if not i.endswith(".img"): continue		continue
	with open(os.path.join(images_path, i), "r") as f:		if not image.endswith(".img"):
	common.ZipWriteStr(output_zip, i, f.read())		continue
			common.ZipWrite(
			output_zip, os.path.join(images_path, image), image)
	done = True		done = True

	if not done:		if not done:

tools/releasetools/ota_from_target_files

+2 −4

Original line number	Original line	Diff line number	Diff line
	@@ -646,10 +646,8 @@ endif;
	WriteMetadata(metadata, output_zip)		WriteMetadata(metadata, output_zip)


	def WritePolicyConfig(file_context, output_zip):		def WritePolicyConfig(file_name, output_zip):
	f = open(file_context, 'r');		common.ZipWrite(output_zip, file_name, os.path.basename(file_name))
	basename = os.path.basename(file_context)
	common.ZipWriteStr(output_zip, basename, f.read())


	def WriteMetadata(metadata, output_zip):		def WriteMetadata(metadata, output_zip):

tools/releasetools/test_common.py

0 → 100644

+108 −0

Original line number	Original line	Diff line number	Diff line
			#
			# Copyright (C) 2015 The Android Open Source Project
			#
			# Licensed under the Apache License, Version 2.0 (the "License");
			# you may not use this file except in compliance with the License.
			# You may obtain a copy of the License at
			#
			# http://www.apache.org/licenses/LICENSE-2.0
			#
			# Unless required by applicable law or agreed to in writing, software
			# distributed under the License is distributed on an "AS IS" BASIS,
			# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			# See the License for the specific language governing permissions and
			# limitations under the License.
			#
			import os
			import tempfile
			import time
			import unittest
			import zipfile

			import common


			def random_string_with_holes(size, block_size, step_size):
			data = ["\0"] * size
			for begin in range(0, size, step_size):
			end = begin + block_size
			data[begin:end] = os.urandom(block_size)
			return "".join(data)


			class CommonZipTest(unittest.TestCase):
			def _test_ZipWrite(self, contents, extra_zipwrite_args=None):
			extra_zipwrite_args = dict(extra_zipwrite_args or {})

			test_file = tempfile.NamedTemporaryFile(delete=False)
			zip_file = tempfile.NamedTemporaryFile(delete=False)

			test_file_name = test_file.name
			zip_file_name = zip_file.name

			# File names within an archive strip the leading slash.
			arcname = extra_zipwrite_args.get("arcname", test_file_name)
			if arcname[0] == "/":
			arcname = arcname[1:]

			zip_file.close()
			zip_file = zipfile.ZipFile(zip_file_name, "w")

			try:
			test_file.write(contents)
			test_file.close()

			old_stat = os.stat(test_file_name)
			expected_mode = extra_zipwrite_args.get("perms", 0o644)

			time.sleep(5) # Make sure the atime/mtime will change measurably.

			common.ZipWrite(zip_file, test_file_name, **extra_zipwrite_args)

			new_stat = os.stat(test_file_name)
			self.assertEqual(int(old_stat.st_mode), int(new_stat.st_mode))
			self.assertEqual(int(old_stat.st_mtime), int(new_stat.st_mtime))

			zip_file.close()
			zip_file = zipfile.ZipFile(zip_file_name, "r")
			info = zip_file.getinfo(arcname)

			self.assertEqual(info.date_time, (2009, 1, 1, 0, 0, 0))
			mode = (info.external_attr >> 16) & 0o777
			self.assertEqual(mode, expected_mode)
			self.assertEqual(zip_file.read(arcname), contents)
			finally:
			os.remove(test_file_name)
			os.remove(zip_file_name)

			def test_ZipWrite(self):
			file_contents = os.urandom(1024)
			self._test_ZipWrite(file_contents)

			def test_ZipWrite_with_opts(self):
			file_contents = os.urandom(1024)
			self._test_ZipWrite(file_contents, {
			"arcname": "foobar",
			"perms": 0o777,
			"compress_type": zipfile.ZIP_DEFLATED,
			})

			def test_ZipWrite_large_file(self):
			kilobytes = 1024
			megabytes = 1024 * kilobytes
			gigabytes = 1024 * megabytes

			size = int(2 * gigabytes + 1)
			block_size = 4 * kilobytes
			step_size = 4 * megabytes
			file_contents = random_string_with_holes(
			size, block_size, step_size)
			self._test_ZipWrite(file_contents, {
			"compress_type": zipfile.ZIP_DEFLATED,
			})

			def test_ZipWrite_resets_ZIP64_LIMIT(self):
			default_limit = (1 << 31) - 1
			self.assertEqual(default_limit, zipfile.ZIP64_LIMIT)
			self._test_ZipWrite('')
			self.assertEqual(default_limit, zipfile.ZIP64_LIMIT)