Split the huge merge_target_files script into multiple files. (2465fc85) · Commits · e / os / android_build

tools/releasetools/merge/Android.bp

+7 −1

Original line number	Diff line number	Diff line
		@@ -19,14 +19,20 @@ package {
		filegroup {
		name: "releasetools_merge_sources",
		srcs: [
		"merge_compatibility_checks.py",
		"merge_dexopt.py",
		"merge_meta.py",
		"merge_target_files.py",
		"merge_utils.py",
		],
		}

		filegroup {
		name: "releasetools_merge_tests",
		srcs: [
		"test_merge_target_files.py",
		"test_merge_compatibility_checks.py",
		"test_merge_meta.py",
		"test_merge_utils.py",
		],
		}

tools/releasetools/merge/merge_compatibility_checks.py

0 → 100644

+206 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python
		#
		# Copyright (C) 2022 The Android Open Source Project
		#
		# Licensed under the Apache License, Version 2.0 (the "License"); you may not
		# use this file except in compliance with the License. You may obtain a copy of
		# the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
		# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
		# License for the specific language governing permissions and limitations under
		# the License.
		#
		"""Compatibility checks that should be performed on merged target_files."""

		import json
		import logging
		import os
		from xml.etree import ElementTree

		import apex_utils
		import check_target_files_vintf
		import common
		import find_shareduid_violation

		logger = logging.getLogger(__name__)
		OPTIONS = common.OPTIONS


		def CheckCompatibility(target_files_dir, partition_map):
		"""Runs various compatibility checks.

		Returns a possibly-empty list of error messages.
		"""
		errors = []

		errors.extend(CheckVintf(target_files_dir))
		errors.extend(CheckShareduidViolation(target_files_dir, partition_map))
		errors.extend(CheckApexDuplicatePackages(target_files_dir, partition_map))

		# The remaining checks only use the following partitions:
		partition_map = {
		partition: path
		for partition, path in partition_map.items()
		if partition in ('system', 'system_ext', 'product', 'vendor', 'odm')
		}

		errors.extend(CheckInitRcFiles(target_files_dir, partition_map))
		errors.extend(CheckCombinedSepolicy(target_files_dir, partition_map))

		return errors


		def CheckVintf(target_files_dir):
		"""Check for any VINTF issues using check_vintf."""
		errors = []
		try:
		if not check_target_files_vintf.CheckVintf(target_files_dir):
		errors.append('Incompatible VINTF.')
		except RuntimeError as err:
		errors.append(str(err))
		return errors


		def CheckShareduidViolation(target_files_dir, partition_map):
		"""Check for any APK sharedUserId violations across partition sets.

		Writes results to META/shareduid_violation_modules.json to help
		with followup debugging.
		"""
		errors = []
		violation = find_shareduid_violation.FindShareduidViolation(
		target_files_dir, partition_map)
		shareduid_violation_modules = os.path.join(
		target_files_dir, 'META', 'shareduid_violation_modules.json')
		with open(shareduid_violation_modules, 'w') as f:
		# Write the output to a file to enable debugging.
		f.write(violation)

		# Check for violations across the partition sets.
		shareduid_errors = common.SharedUidPartitionViolations(
		json.loads(violation),
		[OPTIONS.framework_partition_set, OPTIONS.vendor_partition_set])
		if shareduid_errors:
		for error in shareduid_errors:
		errors.append('APK sharedUserId error: %s' % error)
		errors.append('See APK sharedUserId violations file: %s' %
		shareduid_violation_modules)
		return errors


		def CheckInitRcFiles(target_files_dir, partition_map):
		"""Check for any init.rc issues using host_init_verifier."""
		try:
		common.RunHostInitVerifier(
		product_out=target_files_dir, partition_map=partition_map)
		except RuntimeError as err:
		return [str(err)]
		return []


		def CheckCombinedSepolicy(target_files_dir, partition_map, execute=True):
		"""Uses secilc to compile a split sepolicy file.

		Depends on various /etc/selinux/ and /etc/vintf/ files within partitions.
		"""
		errors = []

		def get_file(partition, path):
		if partition not in partition_map:
		logger.warning('Cannot load SEPolicy files for missing partition %s',
		partition)
		return None
		file_path = os.path.join(target_files_dir, partition_map[partition], path)
		if os.path.exists(file_path):
		return file_path
		return None

		# Load the kernel sepolicy version from the FCM. This is normally provided
		# directly to selinux.cpp as a build flag, but is also available in this file.
		fcm_file = get_file('system', 'etc/vintf/compatibility_matrix.device.xml')
		if not fcm_file:
		errors.append('Missing required file for loading sepolicy: '
		'/system/etc/vintf/compatibility_matrix.device.xml')
		return errors
		kernel_sepolicy_version = ElementTree.parse(fcm_file).getroot().find(
		'sepolicy/kernel-sepolicy-version').text

		# Load the vendor's plat sepolicy version. This is the version used for
		# locating sepolicy mapping files.
		vendor_plat_version_file = get_file('vendor',
		'etc/selinux/plat_sepolicy_vers.txt')
		if not vendor_plat_version_file:
		errors.append('Missing required sepolicy file %s' %
		vendor_plat_version_file)
		return errors
		with open(vendor_plat_version_file) as f:
		vendor_plat_version = f.read().strip()

		# Use the same flags and arguments as selinux.cpp OpenSplitPolicy().
		cmd = ['secilc', '-m', '-M', 'true', '-G', '-N']
		cmd.extend(['-c', kernel_sepolicy_version])
		cmd.extend(['-o', os.path.join(target_files_dir, 'META/combined_sepolicy')])
		cmd.extend(['-f', '/dev/null'])

		required_policy_files = (
		('system', 'etc/selinux/plat_sepolicy.cil'),
		('system', 'etc/selinux/mapping/%s.cil' % vendor_plat_version),
		('vendor', 'etc/selinux/vendor_sepolicy.cil'),
		('vendor', 'etc/selinux/plat_pub_versioned.cil'),
		)
		for policy in (map(lambda partition_and_path: get_file(*partition_and_path),
		required_policy_files)):
		if not policy:
		errors.append('Missing required sepolicy file %s' % policy)
		return errors
		cmd.append(policy)

		optional_policy_files = (
		('system', 'etc/selinux/mapping/%s.compat.cil' % vendor_plat_version),
		('system_ext', 'etc/selinux/system_ext_sepolicy.cil'),
		('system_ext', 'etc/selinux/mapping/%s.cil' % vendor_plat_version),
		('product', 'etc/selinux/product_sepolicy.cil'),
		('product', 'etc/selinux/mapping/%s.cil' % vendor_plat_version),
		('odm', 'etc/selinux/odm_sepolicy.cil'),
		)
		for policy in (map(lambda partition_and_path: get_file(*partition_and_path),
		optional_policy_files)):
		if policy:
		cmd.append(policy)

		try:
		if execute:
		common.RunAndCheckOutput(cmd)
		else:
		return cmd
		except RuntimeError as err:
		errors.append(str(err))

		return errors


		def CheckApexDuplicatePackages(target_files_dir, partition_map):
		"""Checks if the same APEX package name is provided by multiple partitions."""
		errors = []

		apex_packages = set()
		for partition in partition_map.keys():
		try:
		apex_info = apex_utils.GetApexInfoFromTargetFiles(
		target_files_dir, partition, compressed_only=False)
		except RuntimeError as err:
		errors.append(str(err))
		apex_info = []
		partition_apex_packages = set([info.package_name for info in apex_info])
		duplicates = apex_packages.intersection(partition_apex_packages)
		if duplicates:
		errors.append(
		'Duplicate APEX package_names found in multiple partitions: %s' %
		' '.join(duplicates))
		apex_packages.update(partition_apex_packages)

		return errors

tools/releasetools/merge/merge_dexopt.py

0 → 100644

+322 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python
		#
		# Copyright (C) 2022 The Android Open Source Project
		#
		# Licensed under the Apache License, Version 2.0 (the "License"); you may not
		# use this file except in compliance with the License. You may obtain a copy of
		# the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
		# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
		# License for the specific language governing permissions and limitations under
		# the License.
		#
		"""Generates dexopt files for vendor apps, from a merged target_files.

		Expects items in OPTIONS prepared by merge_target_files.py.
		"""

		import glob
		import json
		import logging
		import os
		import shutil
		import subprocess

		import common

		logger = logging.getLogger(__name__)
		OPTIONS = common.OPTIONS


		def MergeDexopt(temp_dir, output_target_files_dir):
		"""If needed, generates dexopt files for vendor apps.

		Args:
		temp_dir: Location containing an 'output' directory where target files have
		been extracted, e.g. <temp_dir>/output/SYSTEM, <temp_dir>/output/IMAGES,
		etc.
		output_target_files_dir: The name of a directory that will be used to create
		the output target files package after all the special cases are processed.
		"""
		# Load vendor and framework META/misc_info.txt.
		if (OPTIONS.vendor_misc_info.get('building_with_vsdk') != 'true' or
		OPTIONS.framework_dexpreopt_tools is None or
		OPTIONS.framework_dexpreopt_config is None or
		OPTIONS.vendor_dexpreopt_config is None):
		return

		logger.info('applying dexpreopt')

		# The directory structure to apply dexpreopt is:
		#
		# <temp_dir>/
		# framework_meta/
		# META/
		# vendor_meta/
		# META/
		# output/
		# SYSTEM/
		# VENDOR/
		# IMAGES/
		# <other items extracted from system and vendor target files>
		# tools/
		# <contents of dexpreopt_tools.zip>
		# system_config/
		# <contents of system dexpreopt_config.zip>
		# vendor_config/
		# <contents of vendor dexpreopt_config.zip>
		# system -> output/SYSTEM
		# vendor -> output/VENDOR
		# apex -> output/SYSTEM/apex (only for flattened APEX builds)
		# apex/ (extracted updatable APEX)
		# <apex 1>/
		# ...
		# <apex 2>/
		# ...
		# ...
		# out/dex2oat_result/vendor/
		# <app>
		# oat/arm64/
		# package.vdex
		# package.odex
		# <priv-app>
		# oat/arm64/
		# package.vdex
		# package.odex
		dexpreopt_tools_files_temp_dir = os.path.join(temp_dir, 'tools')
		dexpreopt_framework_config_files_temp_dir = os.path.join(
		temp_dir, 'system_config')
		dexpreopt_vendor_config_files_temp_dir = os.path.join(temp_dir,
		'vendor_config')

		extract_items(
		input_zip=OPTIONS.framework_dexpreopt_tools,
		output_dir=dexpreopt_tools_files_temp_dir,
		extract_item_list=('*',))
		extract_items(
		input_zip=OPTIONS.framework_dexpreopt_config,
		output_dir=dexpreopt_framework_config_files_temp_dir,
		extract_item_list=('*',))
		extract_items(
		input_zip=OPTIONS.vendor_dexpreopt_config,
		output_dir=dexpreopt_vendor_config_files_temp_dir,
		extract_item_list=('*',))

		os.symlink(
		os.path.join(output_target_files_dir, 'SYSTEM'),
		os.path.join(temp_dir, 'system'))
		os.symlink(
		os.path.join(output_target_files_dir, 'VENDOR'),
		os.path.join(temp_dir, 'vendor'))

		# The directory structure for flatteded APEXes is:
		#
		# SYSTEM
		# apex
		# <APEX name, e.g., com.android.wifi>
		# apex_manifest.pb
		# apex_pubkey
		# etc/
		# javalib/
		# lib/
		# lib64/
		# priv-app/
		#
		# The directory structure for updatable APEXes is:
		#
		# SYSTEM
		# apex
		# com.android.adbd.apex
		# com.android.appsearch.apex
		# com.android.art.apex
		# ...
		apex_root = os.path.join(output_target_files_dir, 'SYSTEM', 'apex')

		# Check for flattended versus updatable APEX.
		if OPTIONS.framework_misc_info.get('target_flatten_apex') == 'false':
		# Extract APEX.
		logging.info('extracting APEX')

		apex_extract_root_dir = os.path.join(temp_dir, 'apex')
		os.makedirs(apex_extract_root_dir)

		for apex in (glob.glob(os.path.join(apex_root, '*.apex')) +
		glob.glob(os.path.join(apex_root, '*.capex'))):
		logging.info(' apex: %s', apex)
		# deapexer is in the same directory as the merge_target_files binary extracted
		# from otatools.zip.
		apex_json_info = subprocess.check_output(['deapexer', 'info', apex])
		logging.info(' info: %s', apex_json_info)
		apex_info = json.loads(apex_json_info)
		apex_name = apex_info['name']
		logging.info(' name: %s', apex_name)

		apex_extract_dir = os.path.join(apex_extract_root_dir, apex_name)
		os.makedirs(apex_extract_dir)

		# deapexer uses debugfs_static, which is part of otatools.zip.
		command = [
		'deapexer',
		'--debugfs_path',
		'debugfs_static',
		'extract',
		apex,
		apex_extract_dir,
		]
		logging.info(' running %s', command)
		subprocess.check_call(command)
		else:
		# Flattened APEXes don't need to be extracted since they have the necessary
		# directory structure.
		os.symlink(os.path.join(apex_root), os.path.join(temp_dir, 'apex'))

		# Modify system config to point to the tools that have been extracted.
		# Absolute or .. paths are not allowed by the dexpreopt_gen tool in
		# dexpreopt_soong.config.
		dexpreopt_framework_soon_config = os.path.join(
		dexpreopt_framework_config_files_temp_dir, 'dexpreopt_soong.config')
		with open(dexpreopt_framework_soon_config, 'w') as f:
		dexpreopt_soong_config = {
		'Profman': 'tools/profman',
		'Dex2oat': 'tools/dex2oatd',
		'Aapt': 'tools/aapt2',
		'SoongZip': 'tools/soong_zip',
		'Zip2zip': 'tools/zip2zip',
		'ManifestCheck': 'tools/manifest_check',
		'ConstructContext': 'tools/construct_context',
		}
		json.dump(dexpreopt_soong_config, f)

		# TODO(b/188179859): Make *dex location configurable to vendor or system_other.
		use_system_other_odex = False

		if use_system_other_odex:
		dex_img = 'SYSTEM_OTHER'
		else:
		dex_img = 'VENDOR'
		# Open vendor_filesystem_config to append the items generated by dexopt.
		vendor_file_system_config = open(
		os.path.join(temp_dir, 'output', 'META',
		'vendor_filesystem_config.txt'), 'a')

		# Dexpreopt vendor apps.
		dexpreopt_config_suffix = '_dexpreopt.config'
		for config in glob.glob(
		os.path.join(dexpreopt_vendor_config_files_temp_dir,
		'*' + dexpreopt_config_suffix)):
		app = os.path.basename(config)[:-len(dexpreopt_config_suffix)]
		logging.info('dexpreopt config: %s %s', config, app)

		apk_dir = 'app'
		apk_path = os.path.join(temp_dir, 'vendor', apk_dir, app, app + '.apk')
		if not os.path.exists(apk_path):
		apk_dir = 'priv-app'
		apk_path = os.path.join(temp_dir, 'vendor', apk_dir, app, app + '.apk')
		if not os.path.exists(apk_path):
		logging.warning(
		'skipping dexpreopt for %s, no apk found in vendor/app '
		'or vendor/priv-app', app)
		continue

		# Generate dexpreopting script. Note 'out_dir' is not the output directory
		# where the script is generated, but the OUT_DIR at build time referenced
		# in the dexpreot config files, e.g., "out/.../core-oj.jar", so the tool knows
		# how to adjust the path.
		command = [
		os.path.join(dexpreopt_tools_files_temp_dir, 'dexpreopt_gen'),
		'-global',
		os.path.join(dexpreopt_framework_config_files_temp_dir,
		'dexpreopt.config'),
		'-global_soong',
		os.path.join(dexpreopt_framework_config_files_temp_dir,
		'dexpreopt_soong.config'),
		'-module',
		config,
		'-dexpreopt_script',
		'dexpreopt_app.sh',
		'-out_dir',
		'out',
		'-base_path',
		'.',
		'--uses_target_files',
		]

		# Run the command from temp_dir so all tool paths are its descendants.
		logging.info('running %s', command)
		subprocess.check_call(command, cwd=temp_dir)

		# Call the generated script.
		command = ['sh', 'dexpreopt_app.sh', apk_path]
		logging.info('running %s', command)
		subprocess.check_call(command, cwd=temp_dir)

		# Output files are in:
		#
		# <temp_dir>/out/dex2oat_result/vendor/priv-app/<app>/oat/arm64/package.vdex
		# <temp_dir>/out/dex2oat_result/vendor/priv-app/<app>/oat/arm64/package.odex
		# <temp_dir>/out/dex2oat_result/vendor/app/<app>/oat/arm64/package.vdex
		# <temp_dir>/out/dex2oat_result/vendor/app/<app>/oat/arm64/package.odex
		#
		# Copy the files to their destination. The structure of system_other is:
		#
		# system_other/
		# system-other-odex-marker
		# system/
		# app/
		# <app>/oat/arm64/
		# <app>.odex
		# <app>.vdex
		# ...
		# priv-app/
		# <app>/oat/arm64/
		# <app>.odex
		# <app>.vdex
		# ...

		# TODO(b/188179859): Support for other architectures.
		arch = 'arm64'

		dex_destination = os.path.join(temp_dir, 'output', dex_img, apk_dir, app,
		'oat', arch)
		os.makedirs(dex_destination)
		dex2oat_path = os.path.join(temp_dir, 'out', 'dex2oat_result', 'vendor',
		apk_dir, app, 'oat', arch)
		shutil.copy(
		os.path.join(dex2oat_path, 'package.vdex'),
		os.path.join(dex_destination, app + '.vdex'))
		shutil.copy(
		os.path.join(dex2oat_path, 'package.odex'),
		os.path.join(dex_destination, app + '.odex'))

		# Append entries to vendor_file_system_config.txt, such as:
		#
		# vendor/app/<app>/oat 0 2000 755 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
		# vendor/app/<app>/oat/arm64 0 2000 755 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
		# vendor/app/<app>/oat/arm64/<app>.odex 0 0 644 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
		# vendor/app/<app>/oat/arm64/<app>.vdex 0 0 644 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
		if not use_system_other_odex:
		vendor_app_prefix = 'vendor/' + apk_dir + '/' + app + '/oat'
		selabel = 'selabel=u:object_r:vendor_app_file:s0 capabilities=0x0'
		vendor_file_system_config.writelines([
		vendor_app_prefix + ' 0 2000 755 ' + selabel + '\n',
		vendor_app_prefix + '/' + arch + ' 0 2000 755 ' + selabel + '\n',
		vendor_app_prefix + '/' + arch + '/' + app + '.odex 0 0 644 ' +
		selabel + '\n',
		vendor_app_prefix + '/' + arch + '/' + app + '.vdex 0 0 644 ' +
		selabel + '\n',
		])

		if not use_system_other_odex:
		vendor_file_system_config.close()
		# Delete vendor.img so that it will be regenerated.
		# TODO(b/188179859): Rebuilding a vendor image in GRF mode (e.g., T(framework)
		# and S(vendor) may require logic similar to that in
		# rebuild_image_with_sepolicy.
		vendor_img = os.path.join(output_target_files_dir, 'IMAGES', 'vendor.img')
		if os.path.exists(vendor_img):
		logging.info('Deleting %s', vendor_img)
		os.remove(vendor_img)