Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d5d93874 authored by Daniel Norman's avatar Daniel Norman Committed by Gerrit Code Review
Browse files

Merge changes from topic "mtf-cleanup-infer"

* changes:
  Infer merge configs if not provided.
  Split the huge merge_target_files script into multiple files.
parents 344245b2 5f47677f
Loading
Loading
Loading
Loading
+7 −1
Original line number Diff line number Diff line
@@ -19,14 +19,20 @@ package {
filegroup {
    name: "releasetools_merge_sources",
    srcs: [
        "merge_compatibility_checks.py",
        "merge_dexopt.py",
        "merge_meta.py",
        "merge_target_files.py",
        "merge_utils.py",
    ],
}

filegroup {
    name: "releasetools_merge_tests",
    srcs: [
        "test_merge_target_files.py",
        "test_merge_compatibility_checks.py",
        "test_merge_meta.py",
        "test_merge_utils.py",
    ],
}

+206 −0
Original line number Diff line number Diff line
#!/usr/bin/env python
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
#
"""Compatibility checks that should be performed on merged target_files."""

import json
import logging
import os
from xml.etree import ElementTree

import apex_utils
import check_target_files_vintf
import common
import find_shareduid_violation

logger = logging.getLogger(__name__)
OPTIONS = common.OPTIONS


def CheckCompatibility(target_files_dir, partition_map):
  """Runs various compatibility checks.

  Returns a possibly-empty list of error messages.
  """
  errors = []

  errors.extend(CheckVintf(target_files_dir))
  errors.extend(CheckShareduidViolation(target_files_dir, partition_map))
  errors.extend(CheckApexDuplicatePackages(target_files_dir, partition_map))

  # The remaining checks only use the following partitions:
  partition_map = {
      partition: path
      for partition, path in partition_map.items()
      if partition in ('system', 'system_ext', 'product', 'vendor', 'odm')
  }

  errors.extend(CheckInitRcFiles(target_files_dir, partition_map))
  errors.extend(CheckCombinedSepolicy(target_files_dir, partition_map))

  return errors


def CheckVintf(target_files_dir):
  """Check for any VINTF issues using check_vintf."""
  errors = []
  try:
    if not check_target_files_vintf.CheckVintf(target_files_dir):
      errors.append('Incompatible VINTF.')
  except RuntimeError as err:
    errors.append(str(err))
  return errors


def CheckShareduidViolation(target_files_dir, partition_map):
  """Check for any APK sharedUserId violations across partition sets.

  Writes results to META/shareduid_violation_modules.json to help
  with followup debugging.
  """
  errors = []
  violation = find_shareduid_violation.FindShareduidViolation(
      target_files_dir, partition_map)
  shareduid_violation_modules = os.path.join(
      target_files_dir, 'META', 'shareduid_violation_modules.json')
  with open(shareduid_violation_modules, 'w') as f:
    # Write the output to a file to enable debugging.
    f.write(violation)

    # Check for violations across the partition sets.
    shareduid_errors = common.SharedUidPartitionViolations(
        json.loads(violation),
        [OPTIONS.framework_partition_set, OPTIONS.vendor_partition_set])
    if shareduid_errors:
      for error in shareduid_errors:
        errors.append('APK sharedUserId error: %s' % error)
      errors.append('See APK sharedUserId violations file: %s' %
                    shareduid_violation_modules)
  return errors


def CheckInitRcFiles(target_files_dir, partition_map):
  """Check for any init.rc issues using host_init_verifier."""
  try:
    common.RunHostInitVerifier(
        product_out=target_files_dir, partition_map=partition_map)
  except RuntimeError as err:
    return [str(err)]
  return []


def CheckCombinedSepolicy(target_files_dir, partition_map, execute=True):
  """Uses secilc to compile a split sepolicy file.

  Depends on various */etc/selinux/* and */etc/vintf/* files within partitions.
  """
  errors = []

  def get_file(partition, path):
    if partition not in partition_map:
      logger.warning('Cannot load SEPolicy files for missing partition %s',
                     partition)
      return None
    file_path = os.path.join(target_files_dir, partition_map[partition], path)
    if os.path.exists(file_path):
      return file_path
    return None

  # Load the kernel sepolicy version from the FCM. This is normally provided
  # directly to selinux.cpp as a build flag, but is also available in this file.
  fcm_file = get_file('system', 'etc/vintf/compatibility_matrix.device.xml')
  if not fcm_file:
    errors.append('Missing required file for loading sepolicy: '
                  '/system/etc/vintf/compatibility_matrix.device.xml')
    return errors
  kernel_sepolicy_version = ElementTree.parse(fcm_file).getroot().find(
      'sepolicy/kernel-sepolicy-version').text

  # Load the vendor's plat sepolicy version. This is the version used for
  # locating sepolicy mapping files.
  vendor_plat_version_file = get_file('vendor',
                                      'etc/selinux/plat_sepolicy_vers.txt')
  if not vendor_plat_version_file:
    errors.append('Missing required sepolicy file %s' %
                  vendor_plat_version_file)
    return errors
  with open(vendor_plat_version_file) as f:
    vendor_plat_version = f.read().strip()

  # Use the same flags and arguments as selinux.cpp OpenSplitPolicy().
  cmd = ['secilc', '-m', '-M', 'true', '-G', '-N']
  cmd.extend(['-c', kernel_sepolicy_version])
  cmd.extend(['-o', os.path.join(target_files_dir, 'META/combined_sepolicy')])
  cmd.extend(['-f', '/dev/null'])

  required_policy_files = (
      ('system', 'etc/selinux/plat_sepolicy.cil'),
      ('system', 'etc/selinux/mapping/%s.cil' % vendor_plat_version),
      ('vendor', 'etc/selinux/vendor_sepolicy.cil'),
      ('vendor', 'etc/selinux/plat_pub_versioned.cil'),
  )
  for policy in (map(lambda partition_and_path: get_file(*partition_and_path),
                     required_policy_files)):
    if not policy:
      errors.append('Missing required sepolicy file %s' % policy)
      return errors
    cmd.append(policy)

  optional_policy_files = (
      ('system', 'etc/selinux/mapping/%s.compat.cil' % vendor_plat_version),
      ('system_ext', 'etc/selinux/system_ext_sepolicy.cil'),
      ('system_ext', 'etc/selinux/mapping/%s.cil' % vendor_plat_version),
      ('product', 'etc/selinux/product_sepolicy.cil'),
      ('product', 'etc/selinux/mapping/%s.cil' % vendor_plat_version),
      ('odm', 'etc/selinux/odm_sepolicy.cil'),
  )
  for policy in (map(lambda partition_and_path: get_file(*partition_and_path),
                     optional_policy_files)):
    if policy:
      cmd.append(policy)

  try:
    if execute:
      common.RunAndCheckOutput(cmd)
    else:
      return cmd
  except RuntimeError as err:
    errors.append(str(err))

  return errors


def CheckApexDuplicatePackages(target_files_dir, partition_map):
  """Checks if the same APEX package name is provided by multiple partitions."""
  errors = []

  apex_packages = set()
  for partition in partition_map.keys():
    try:
      apex_info = apex_utils.GetApexInfoFromTargetFiles(
          target_files_dir, partition, compressed_only=False)
    except RuntimeError as err:
      errors.append(str(err))
      apex_info = []
    partition_apex_packages = set([info.package_name for info in apex_info])
    duplicates = apex_packages.intersection(partition_apex_packages)
    if duplicates:
      errors.append(
          'Duplicate APEX package_names found in multiple partitions: %s' %
          ' '.join(duplicates))
    apex_packages.update(partition_apex_packages)

  return errors
+322 −0
Original line number Diff line number Diff line
#!/usr/bin/env python
#
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
#
"""Generates dexopt files for vendor apps, from a merged target_files.

Expects items in OPTIONS prepared by merge_target_files.py.
"""

import glob
import json
import logging
import os
import shutil
import subprocess

import common

logger = logging.getLogger(__name__)
OPTIONS = common.OPTIONS


def MergeDexopt(temp_dir, output_target_files_dir):
  """If needed, generates dexopt files for vendor apps.

  Args:
    temp_dir: Location containing an 'output' directory where target files have
      been extracted, e.g. <temp_dir>/output/SYSTEM, <temp_dir>/output/IMAGES,
      etc.
    output_target_files_dir: The name of a directory that will be used to create
      the output target files package after all the special cases are processed.
  """
  # Load vendor and framework META/misc_info.txt.
  if (OPTIONS.vendor_misc_info.get('building_with_vsdk') != 'true' or
      OPTIONS.framework_dexpreopt_tools is None or
      OPTIONS.framework_dexpreopt_config is None or
      OPTIONS.vendor_dexpreopt_config is None):
    return

  logger.info('applying dexpreopt')

  # The directory structure to apply dexpreopt is:
  #
  # <temp_dir>/
  #     framework_meta/
  #         META/
  #     vendor_meta/
  #         META/
  #     output/
  #         SYSTEM/
  #         VENDOR/
  #         IMAGES/
  #         <other items extracted from system and vendor target files>
  #     tools/
  #         <contents of dexpreopt_tools.zip>
  #     system_config/
  #         <contents of system dexpreopt_config.zip>
  #     vendor_config/
  #         <contents of vendor dexpreopt_config.zip>
  #     system -> output/SYSTEM
  #     vendor -> output/VENDOR
  #     apex -> output/SYSTEM/apex (only for flattened APEX builds)
  #     apex/ (extracted updatable APEX)
  #         <apex 1>/
  #             ...
  #         <apex 2>/
  #             ...
  #         ...
  #     out/dex2oat_result/vendor/
  #         <app>
  #             oat/arm64/
  #                 package.vdex
  #                 package.odex
  #         <priv-app>
  #             oat/arm64/
  #                 package.vdex
  #                 package.odex
  dexpreopt_tools_files_temp_dir = os.path.join(temp_dir, 'tools')
  dexpreopt_framework_config_files_temp_dir = os.path.join(
      temp_dir, 'system_config')
  dexpreopt_vendor_config_files_temp_dir = os.path.join(temp_dir,
                                                        'vendor_config')

  extract_items(
      input_zip=OPTIONS.framework_dexpreopt_tools,
      output_dir=dexpreopt_tools_files_temp_dir,
      extract_item_list=('*',))
  extract_items(
      input_zip=OPTIONS.framework_dexpreopt_config,
      output_dir=dexpreopt_framework_config_files_temp_dir,
      extract_item_list=('*',))
  extract_items(
      input_zip=OPTIONS.vendor_dexpreopt_config,
      output_dir=dexpreopt_vendor_config_files_temp_dir,
      extract_item_list=('*',))

  os.symlink(
      os.path.join(output_target_files_dir, 'SYSTEM'),
      os.path.join(temp_dir, 'system'))
  os.symlink(
      os.path.join(output_target_files_dir, 'VENDOR'),
      os.path.join(temp_dir, 'vendor'))

  # The directory structure for flatteded APEXes is:
  #
  # SYSTEM
  #     apex
  #         <APEX name, e.g., com.android.wifi>
  #             apex_manifest.pb
  #             apex_pubkey
  #             etc/
  #             javalib/
  #             lib/
  #             lib64/
  #             priv-app/
  #
  # The directory structure for updatable APEXes is:
  #
  # SYSTEM
  #     apex
  #         com.android.adbd.apex
  #         com.android.appsearch.apex
  #         com.android.art.apex
  #         ...
  apex_root = os.path.join(output_target_files_dir, 'SYSTEM', 'apex')

  # Check for flattended versus updatable APEX.
  if OPTIONS.framework_misc_info.get('target_flatten_apex') == 'false':
    # Extract APEX.
    logging.info('extracting APEX')

    apex_extract_root_dir = os.path.join(temp_dir, 'apex')
    os.makedirs(apex_extract_root_dir)

    for apex in (glob.glob(os.path.join(apex_root, '*.apex')) +
                 glob.glob(os.path.join(apex_root, '*.capex'))):
      logging.info('  apex: %s', apex)
      # deapexer is in the same directory as the merge_target_files binary extracted
      # from otatools.zip.
      apex_json_info = subprocess.check_output(['deapexer', 'info', apex])
      logging.info('    info: %s', apex_json_info)
      apex_info = json.loads(apex_json_info)
      apex_name = apex_info['name']
      logging.info('    name: %s', apex_name)

      apex_extract_dir = os.path.join(apex_extract_root_dir, apex_name)
      os.makedirs(apex_extract_dir)

      # deapexer uses debugfs_static, which is part of otatools.zip.
      command = [
          'deapexer',
          '--debugfs_path',
          'debugfs_static',
          'extract',
          apex,
          apex_extract_dir,
      ]
      logging.info('    running %s', command)
      subprocess.check_call(command)
  else:
    # Flattened APEXes don't need to be extracted since they have the necessary
    # directory structure.
    os.symlink(os.path.join(apex_root), os.path.join(temp_dir, 'apex'))

  # Modify system config to point to the tools that have been extracted.
  # Absolute or .. paths are not allowed  by the dexpreopt_gen tool in
  # dexpreopt_soong.config.
  dexpreopt_framework_soon_config = os.path.join(
      dexpreopt_framework_config_files_temp_dir, 'dexpreopt_soong.config')
  with open(dexpreopt_framework_soon_config, 'w') as f:
    dexpreopt_soong_config = {
        'Profman': 'tools/profman',
        'Dex2oat': 'tools/dex2oatd',
        'Aapt': 'tools/aapt2',
        'SoongZip': 'tools/soong_zip',
        'Zip2zip': 'tools/zip2zip',
        'ManifestCheck': 'tools/manifest_check',
        'ConstructContext': 'tools/construct_context',
    }
    json.dump(dexpreopt_soong_config, f)

  # TODO(b/188179859): Make *dex location configurable to vendor or system_other.
  use_system_other_odex = False

  if use_system_other_odex:
    dex_img = 'SYSTEM_OTHER'
  else:
    dex_img = 'VENDOR'
    # Open vendor_filesystem_config to append the items generated by dexopt.
    vendor_file_system_config = open(
        os.path.join(temp_dir, 'output', 'META',
                     'vendor_filesystem_config.txt'), 'a')

  # Dexpreopt vendor apps.
  dexpreopt_config_suffix = '_dexpreopt.config'
  for config in glob.glob(
      os.path.join(dexpreopt_vendor_config_files_temp_dir,
                   '*' + dexpreopt_config_suffix)):
    app = os.path.basename(config)[:-len(dexpreopt_config_suffix)]
    logging.info('dexpreopt config: %s %s', config, app)

    apk_dir = 'app'
    apk_path = os.path.join(temp_dir, 'vendor', apk_dir, app, app + '.apk')
    if not os.path.exists(apk_path):
      apk_dir = 'priv-app'
      apk_path = os.path.join(temp_dir, 'vendor', apk_dir, app, app + '.apk')
      if not os.path.exists(apk_path):
        logging.warning(
            'skipping dexpreopt for %s, no apk found in vendor/app '
            'or vendor/priv-app', app)
        continue

    # Generate dexpreopting script. Note 'out_dir' is not the output directory
    # where the script is generated, but the OUT_DIR at build time referenced
    # in the dexpreot config files, e.g., "out/.../core-oj.jar", so the tool knows
    # how to adjust the path.
    command = [
        os.path.join(dexpreopt_tools_files_temp_dir, 'dexpreopt_gen'),
        '-global',
        os.path.join(dexpreopt_framework_config_files_temp_dir,
                     'dexpreopt.config'),
        '-global_soong',
        os.path.join(dexpreopt_framework_config_files_temp_dir,
                     'dexpreopt_soong.config'),
        '-module',
        config,
        '-dexpreopt_script',
        'dexpreopt_app.sh',
        '-out_dir',
        'out',
        '-base_path',
        '.',
        '--uses_target_files',
    ]

    # Run the command from temp_dir so all tool paths are its descendants.
    logging.info('running %s', command)
    subprocess.check_call(command, cwd=temp_dir)

    # Call the generated script.
    command = ['sh', 'dexpreopt_app.sh', apk_path]
    logging.info('running %s', command)
    subprocess.check_call(command, cwd=temp_dir)

    # Output files are in:
    #
    # <temp_dir>/out/dex2oat_result/vendor/priv-app/<app>/oat/arm64/package.vdex
    # <temp_dir>/out/dex2oat_result/vendor/priv-app/<app>/oat/arm64/package.odex
    # <temp_dir>/out/dex2oat_result/vendor/app/<app>/oat/arm64/package.vdex
    # <temp_dir>/out/dex2oat_result/vendor/app/<app>/oat/arm64/package.odex
    #
    # Copy the files to their destination. The structure of system_other is:
    #
    # system_other/
    #     system-other-odex-marker
    #     system/
    #         app/
    #             <app>/oat/arm64/
    #                 <app>.odex
    #                 <app>.vdex
    #             ...
    #         priv-app/
    #             <app>/oat/arm64/
    #                 <app>.odex
    #                 <app>.vdex
    #             ...

    # TODO(b/188179859): Support for other architectures.
    arch = 'arm64'

    dex_destination = os.path.join(temp_dir, 'output', dex_img, apk_dir, app,
                                   'oat', arch)
    os.makedirs(dex_destination)
    dex2oat_path = os.path.join(temp_dir, 'out', 'dex2oat_result', 'vendor',
                                apk_dir, app, 'oat', arch)
    shutil.copy(
        os.path.join(dex2oat_path, 'package.vdex'),
        os.path.join(dex_destination, app + '.vdex'))
    shutil.copy(
        os.path.join(dex2oat_path, 'package.odex'),
        os.path.join(dex_destination, app + '.odex'))

    # Append entries to vendor_file_system_config.txt, such as:
    #
    # vendor/app/<app>/oat 0 2000 755 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
    # vendor/app/<app>/oat/arm64 0 2000 755 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
    # vendor/app/<app>/oat/arm64/<app>.odex 0 0 644 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
    # vendor/app/<app>/oat/arm64/<app>.vdex 0 0 644 selabel=u:object_r:vendor_app_file:s0 capabilities=0x0
    if not use_system_other_odex:
      vendor_app_prefix = 'vendor/' + apk_dir + '/' + app + '/oat'
      selabel = 'selabel=u:object_r:vendor_app_file:s0 capabilities=0x0'
      vendor_file_system_config.writelines([
          vendor_app_prefix + ' 0 2000 755 ' + selabel + '\n',
          vendor_app_prefix + '/' + arch + ' 0 2000 755 ' + selabel + '\n',
          vendor_app_prefix + '/' + arch + '/' + app + '.odex 0 0 644 ' +
          selabel + '\n',
          vendor_app_prefix + '/' + arch + '/' + app + '.vdex 0 0 644 ' +
          selabel + '\n',
      ])

  if not use_system_other_odex:
    vendor_file_system_config.close()
    # Delete vendor.img so that it will be regenerated.
    # TODO(b/188179859): Rebuilding a vendor image in GRF mode (e.g., T(framework)
    #                    and S(vendor) may require logic similar to that in
    #                    rebuild_image_with_sepolicy.
    vendor_img = os.path.join(output_target_files_dir, 'IMAGES', 'vendor.img')
    if os.path.exists(vendor_img):
      logging.info('Deleting %s', vendor_img)
      os.remove(vendor_img)
+287 −0

File added.

Preview size limit exceeded, changes collapsed.

+55 −985

File changed.

Preview size limit exceeded, changes collapsed.

Loading