Merge "Runs python formatter and linter over merge_target_files."

# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the

# License for the specific language governing permissions and limitations under

# the License.

"""This script merges two partial target files packages.

"""

This script merges two partial target files packages (one of which contains

system files, and the other contains non-system files) together, producing a

complete target files package that can be used to generate an OTA package.

One package contains system files, and the other contains non-system files.

It produces a complete target files package that can be used to generate an

OTA package.

Usage: merge_target_files.py [args]

    'META/*',

]

# default_system_misc_info_keys is a list of keys to obtain from the system instance of

# META/misc_info.txt. The remaining keys from the other instance.

# default_system_misc_info_keys is a list of keys to obtain from the system

# instance of META/misc_info.txt. The remaining keys from the other instance.

default_system_misc_info_keys = [

    'avb_system_hashtree_enable',

  Args:

    target_files: The target files zip archive from which to extract items.

    target_files_temp_dir: The temporary directory where the extracted items

      will land.

    extract_item_list: A list of items to extract.

  """

  # zip file. Otherwise, the extraction step will fail.

  with zipfile.ZipFile(

      target_files,

      'r',

      allowZip64=True) as target_files_zipfile:

      target_files, 'r', allowZip64=True) as target_files_zipfile:

    target_files_namelist = target_files_zipfile.namelist()

  filtered_extract_item_list = []

  # Extract from target_files into target_files_temp_dir the

  # filtered_extract_item_list.

  common.UnzipToDir(

      target_files,

      target_files_temp_dir,

  common.UnzipToDir(target_files, target_files_temp_dir,

                    filtered_extract_item_list)

  """Similar to extract_items() except uses an input dir instead of zip."""

  file_paths = []

  for dirpath, _, filenames in os.walk(from_dir):

    file_paths.extend(os.path.relpath(path=os.path.join(dirpath, filename),

                                      start=from_dir) for filename in filenames)

    file_paths.extend(

        os.path.relpath(path=os.path.join(dirpath, filename), start=from_dir)

        for filename in filenames)

  filtered_file_paths = set()

  for pattern in patterns:

  Args:

    config_file_path: The path to the config file to open and read.

  Returns:

    The list of strings in the config file.

  """

  with open(config_file_path) as config_file:

    return config_file.read().splitlines()

def validate_config_lists(

    system_item_list,

    system_misc_info_keys,

def validate_config_lists(system_item_list, system_misc_info_keys,

                          other_item_list):

  """Performs validations on the merge config lists.

  Args:

    system_item_list: The list of items to extract from the partial

    system target files package as is.

    system_misc_info_keys: A list of keys to obtain from the system instance

    of META/misc_info.txt. The remaining keys from the other instance.

    other_item_list: The list of items to extract from the partial

    other target files package as is.

    system_item_list: The list of items to extract from the partial system

      target files package as is.

    system_misc_info_keys: A list of keys to obtain from the system instance of

      META/misc_info.txt. The remaining keys from the other instance.

    other_item_list: The list of items to extract from the partial other target

      files package as is.

  Returns:

    False if a validation fails, otherwise true.

  # by the default config lists.

  difference = default_combined_item_set.difference(combined_item_set)

  if difference:

    logger.error('Missing merge config items: %s' % list(difference))

    logger.error('Missing merge config items: %s', list(difference))

    logger.error('Please ensure missing items are in either the '

                 'system-item-list or other-item-list files provided to '

                 'this script.')

  return True

def process_ab_partitions_txt(

    system_target_files_temp_dir,

def process_ab_partitions_txt(system_target_files_temp_dir,

                              other_target_files_temp_dir,

                              output_target_files_temp_dir):

  """Perform special processing for META/ab_partitions.txt

  """Perform special processing for META/ab_partitions.txt.

  This function merges the contents of the META/ab_partitions.txt files from

  the system directory and the other directory, placing the merged result in

  names.

  Args:

    system_target_files_temp_dir: The name of a directory containing the

    special items extracted from the system target files package.

    other_target_files_temp_dir: The name of a directory containing the

    special items extracted from the other target files package.

    output_target_files_temp_dir: The name of a directory that will be used

    to create the output target files package after all the special cases

    are processed.

    system_target_files_temp_dir: The name of a directory containing the special

      items extracted from the system target files package.

    other_target_files_temp_dir: The name of a directory containing the special

      items extracted from the other target files package.

    output_target_files_temp_dir: The name of a directory that will be used to

      create the output target files package after all the special cases are

      processed.

  """

  system_ab_partitions_txt = os.path.join(

      system_target_files_temp_dir, 'META', 'ab_partitions.txt')

  system_ab_partitions_txt = os.path.join(system_target_files_temp_dir, 'META',

                                          'ab_partitions.txt')

  other_ab_partitions_txt = os.path.join(

      other_target_files_temp_dir, 'META', 'ab_partitions.txt')

  other_ab_partitions_txt = os.path.join(other_target_files_temp_dir, 'META',

                                         'ab_partitions.txt')

  with open(system_ab_partitions_txt) as f:

    system_ab_partitions = f.read().splitlines()

  output_ab_partitions = set(system_ab_partitions + other_ab_partitions)

  output_ab_partitions_txt = os.path.join(

      output_target_files_temp_dir, 'META', 'ab_partitions.txt')

  output_ab_partitions_txt = os.path.join(output_target_files_temp_dir, 'META',

                                          'ab_partitions.txt')

  with open(output_ab_partitions_txt, 'w') as output:

    for partition in sorted(output_ab_partitions):

def append_recovery_to_filesystem_config(output_target_files_temp_dir):

  """Perform special processing for META/filesystem_config.txt

  """Perform special processing for META/filesystem_config.txt.

  This function appends recovery information to META/filesystem_config.txt

  so that recovery patch regeneration will succeed.

  Args:

    output_target_files_temp_dir: The name of a directory that will be used

    to create the output target files package after all the special cases

    are processed. We find filesystem_config.txt here.

    output_target_files_temp_dir: The name of a directory that will be used to

      create the output target files package after all the special cases are

      processed. We find filesystem_config.txt here.

  """

  filesystem_config_txt = os.path.join(

      output_target_files_temp_dir,

      'META',

  filesystem_config_txt = os.path.join(output_target_files_temp_dir, 'META',

                                       'filesystem_config.txt')

  with open(filesystem_config_txt, 'a') as f:

    # TODO(bpeckham) this data is hard coded. It should be generated

    # programmatically.

    f.write(

        'system/bin/install-recovery.sh 0 0 750 '

    f.write('system/bin/install-recovery.sh 0 0 750 '

            'selabel=u:object_r:install_recovery_exec:s0 capabilities=0x0\n')

    f.write(

        'system/recovery-from-boot.p 0 0 644 '

    f.write('system/recovery-from-boot.p 0 0 644 '

            'selabel=u:object_r:system_file:s0 capabilities=0x0\n')

    f.write(

        'system/etc/recovery.img 0 0 440 '

    f.write('system/etc/recovery.img 0 0 440 '

            'selabel=u:object_r:install_recovery_exec:s0 capabilities=0x0\n')

def process_misc_info_txt(

    system_target_files_temp_dir,

def process_misc_info_txt(system_target_files_temp_dir,

                          other_target_files_temp_dir,

    output_target_files_temp_dir,

    system_misc_info_keys):

  """Perform special processing for META/misc_info.txt

                          output_target_files_temp_dir, system_misc_info_keys):

  """Perform special processing for META/misc_info.txt.

  This function merges the contents of the META/misc_info.txt files from the

  system directory and the other directory, placing the merged result in the

  content.

  Args:

    system_target_files_temp_dir: The name of a directory containing the

    special items extracted from the system target files package.

    other_target_files_temp_dir: The name of a directory containing the

    special items extracted from the other target files package.

    output_target_files_temp_dir: The name of a directory that will be used

    to create the output target files package after all the special cases

    are processed.

    system_misc_info_keys: A list of keys to obtain from the system instance

    of META/misc_info.txt. The remaining keys from the other instance.

    system_target_files_temp_dir: The name of a directory containing the special

      items extracted from the system target files package.

    other_target_files_temp_dir: The name of a directory containing the special

      items extracted from the other target files package.

    output_target_files_temp_dir: The name of a directory that will be used to

      create the output target files package after all the special cases are

      processed.

    system_misc_info_keys: A list of keys to obtain from the system instance of

      META/misc_info.txt. The remaining keys from the other instance.

  """

  def read_helper(d):

    # Partition groups and group sizes are defined by the other (non-system)

    # misc info file because these values may vary for each board that uses

    # a shared system image.

    for partition_group in merged_info_dict['super_partition_groups'].split(' '):

    for partition_group in merged_info_dict['super_partition_groups'].split(

        ' '):

      if ('super_%s_group_size' % partition_group) not in merged_info_dict:

        raise ValueError(

            'Other META/misc_info.txt does not contain required key '

            'super_%s_group_size.' % partition_group)

      key = 'super_%s_partition_list' % partition_group

      merged_info_dict[key] = '%s %s' % (

        system_info_dict.get(key, ''),

        merged_info_dict.get(key, ''))

      merged_info_dict[key] = '%s %s' % (system_info_dict.get(

          key, ''), merged_info_dict.get(key, ''))

  output_misc_info_txt = os.path.join(

      output_target_files_temp_dir,

      'META', 'misc_info.txt')

  output_misc_info_txt = os.path.join(output_target_files_temp_dir, 'META',

                                      'misc_info.txt')

  sorted_keys = sorted(merged_info_dict.keys())

  Args:

    temp_dir: The name of a scratch directory that this function can use for

      intermediate files generated during processing.

    output_target_files_temp_dir: The name of the working directory that must

      already contain plat_file_contexts and vendor_file_contexts (in the

      appropriate sub directories), and to which META/file_contexts.bin will be

  for partition in ['SYSTEM', 'VENDOR', 'PRODUCT', 'ODM']:

    prefix = 'plat' if partition == 'SYSTEM' else partition.lower()

    file_contexts = os.path.join(

        output_target_files_temp_dir,

        partition, 'etc', 'selinux', prefix + '_file_contexts')

    file_contexts = os.path.join(output_target_files_temp_dir, partition, 'etc',

                                 'selinux', prefix + '_file_contexts')

    mandatory = partition in ['SYSTEM', 'VENDOR']

  # Finally, the compile step creates the final META/file_contexts.bin.

  file_contexts_bin = os.path.join(

      output_target_files_temp_dir,

      'META', 'file_contexts.bin')

  file_contexts_bin = os.path.join(output_target_files_temp_dir, 'META',

                                   'file_contexts.bin')

  command = [

      'sefcontext_compile',

      '-o', file_contexts_bin,

      '-o',

      file_contexts_bin,

      sorted_file_contexts_txt,

  ]

  common.RunAndWait(command, verbose=True)

def process_special_cases(

    temp_dir,

    system_target_files_temp_dir,

def process_special_cases(temp_dir, system_target_files_temp_dir,

                          other_target_files_temp_dir,

    output_target_files_temp_dir,

    system_misc_info_keys,

    rebuild_recovery

):

                          output_target_files_temp_dir, system_misc_info_keys,

                          rebuild_recovery):

  """Perform special-case processing for certain target files items.

  Certain files in the output target files package require special-case

  Args:

    temp_dir: The name of a scratch directory that this function can use for

      intermediate files generated during processing.

    system_target_files_temp_dir: The name of a directory containing the

    special items extracted from the system target files package.

    other_target_files_temp_dir: The name of a directory containing the

    special items extracted from the other target files package.

    output_target_files_temp_dir: The name of a directory that will be used

    to create the output target files package after all the special cases

    are processed.

    system_misc_info_keys: A list of keys to obtain from the system instance

    of META/misc_info.txt. The remaining keys from the other instance.

    system_target_files_temp_dir: The name of a directory containing the special

      items extracted from the system target files package.

    other_target_files_temp_dir: The name of a directory containing the special

      items extracted from the other target files package.

    output_target_files_temp_dir: The name of a directory that will be used to

      create the output target files package after all the special cases are

      processed.

    system_misc_info_keys: A list of keys to obtain from the system instance of

      META/misc_info.txt. The remaining keys from the other instance.

    rebuild_recovery: If true, rebuild the recovery patch used by non-A/B

      devices and write it to the system image.

  """

      output_target_files_temp_dir=output_target_files_temp_dir)

def merge_target_files(

    temp_dir,

    system_target_files,

    system_item_list,

    system_misc_info_keys,

    other_target_files,

    other_item_list,

    output_target_files,

    output_dir,

    output_item_list,

    output_ota,

    output_super_empty,

def merge_target_files(temp_dir, system_target_files, system_item_list,

                       system_misc_info_keys, other_target_files,

                       other_item_list, output_target_files, output_dir,

                       output_item_list, output_ota, output_super_empty,

                       rebuild_recovery):

  """Merge two target files packages together.

    temp_dir: The name of a directory we use when we extract items from the

      input target files packages, and also a scratch directory that we use for

      temporary files.

    system_target_files: The name of the zip archive containing the system

      partial target files package.

    system_item_list: The list of items to extract from the partial system

      target files package as is, meaning these items will land in the output

      target files package exactly as they appear in the input partial system

      target files package.

    system_misc_info_keys: The list of keys to obtain from the system instance

      of META/misc_info.txt. The remaining keys from the other instance.

    other_target_files: The name of the zip archive containing the other

    partial target files package.

    other_item_list: The list of items to extract from the partial other

    target files package as is, meaning these items will land in the output

    target files package exactly as they appear in the input partial other

    other_target_files: The name of the zip archive containing the other partial

      target files package.

    output_target_files: The name of the output zip archive target files

    package created by merging system and other.

    other_item_list: The list of items to extract from the partial other target

      files package as is, meaning these items will land in the output target

      files package exactly as they appear in the input partial other target

      files package.

    output_target_files: The name of the output zip archive target files package

      created by merging system and other.

    output_dir: The destination directory for saving merged files.

    output_item_list: The list of items to copy into the output_dir.

    output_ota: The name of the output zip archive ota package.

    output_super_empty: If provided, creates a super_empty.img file from the

      merged target files package and saves it at this path.

    rebuild_recovery: If true, rebuild the recovery patch used by non-A/B

      devices and write it to the system image.

  """

  logger.info(

      'starting: merge system %s and other %s into output %s',

      system_target_files,

      other_target_files,

      output_target_files)

  logger.info('starting: merge system %s and other %s into output %s',

              system_target_files, other_target_files, output_target_files)

  # Create directory names that we'll use when we extract files from system,

  # and other, and for zipping the final output.

  # Create super_empty.img using the merged misc_info.txt.

  if output_super_empty:

    misc_info_txt = os.path.join(output_target_files_temp_dir,

                                 'META', 'misc_info.txt')

    misc_info_txt = os.path.join(output_target_files_temp_dir, 'META',

                                 'misc_info.txt')

    def read_helper():

      with open(misc_info_txt) as f:

        return list(f.read().splitlines())

  output_zip = os.path.abspath(output_target_files)

  output_target_files_list = os.path.join(temp_dir, 'output.list')

  output_target_files_meta_dir = os.path.join(

      output_target_files_temp_dir, 'META')

  output_target_files_meta_dir = os.path.join(output_target_files_temp_dir,

                                              'META')

  command = [

      'find',

  # TODO(bpeckham): sort this to be more like build.

  meta_content = common.RunAndCheckOutput(command, verbose=False)

  command = [

      'find',

      output_target_files_temp_dir,

      '-path',

      output_target_files_meta_dir,

      '-prune',

      '-o',

      '-print'

      'find', output_target_files_temp_dir, '-path',

      output_target_files_meta_dir, '-prune', '-o', '-print'

  ]

  # TODO(bpeckham): sort this to be more like build.

  other_content = common.RunAndCheckOutput(command, verbose=False)

  command = [

      'soong_zip',

      '-d',

      '-o', output_zip,

      '-C', output_target_files_temp_dir,

      '-l', output_target_files_list,

      '-o',

      output_zip,

      '-C',

      output_target_files_temp_dir,

      '-l',

      output_target_files_list,

  ]

  logger.info('creating %s', output_target_files)

  common.RunAndWait(command, verbose=True)

  directory. It also cleans up the temporary directory.

  Args:

    func: The function to call. Should accept one parameter, the path to

    the temporary directory.

    func: The function to call. Should accept one parameter, the path to the

      temporary directory.

    keep_tmp: Keep the temporary directory after processing is complete.

  """

    return True

  args = common.ParseOptions(

      sys.argv[1:], __doc__,

      sys.argv[1:],

      __doc__,

      extra_long_opts=[

          'system-target-files=',

          'system-item-list=',

      ],

      extra_option_handler=option_handler)

  if (len(args) != 0 or

      OPTIONS.system_target_files is None or

      OPTIONS.other_target_files is None or (

        OPTIONS.output_target_files is None and

        OPTIONS.output_dir is None) or (

        OPTIONS.output_dir is not None and

        OPTIONS.output_item_list is None)):

  if (args or OPTIONS.system_target_files is None or

      OPTIONS.other_target_files is None or

      (OPTIONS.output_target_files is None and OPTIONS.output_dir is None) or

      (OPTIONS.output_dir is not None and OPTIONS.output_item_list is None)):

    common.Usage(__doc__)

    sys.exit(1)

          output_item_list=output_item_list,

          output_ota=OPTIONS.output_ota,

          output_super_empty=OPTIONS.output_super_empty,

          rebuild_recovery=OPTIONS.rebuild_recovery),

      OPTIONS.keep_tmp)

          rebuild_recovery=OPTIONS.rebuild_recovery), OPTIONS.keep_tmp)

if __name__ == '__main__':