Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d12bc882 authored by Nishith Khanna's avatar Nishith Khanna
Browse files

Merge remote-tracking branch 'origin/lineage-21.0' into v1-u

parents 0f6afad7 2ba7f00a

ci/build_test_suites

0 → 100755
+23 −0
Original line number Diff line number Diff line 
#!prebuilts/build-tools/linux-x86/bin/py3-cmd

# Copyright 2024, The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



import sys



import build_test_suites



if __name__ == '__main__':

  sys.dont_write_bytecode = True



  build_test_suites.main(sys.argv)

ci/build_test_suites.py

0 → 100644
+408 −0
Original line number Diff line number Diff line 
# Copyright 2024, The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



"""Script to build only the necessary modules for general-tests along



with whatever other targets are passed in.

"""



import argparse

from collections.abc import Sequence

import json

import os

import pathlib

import re

import subprocess

import sys

from typing import Any



import test_mapping_module_retriever





# List of modules that are always required to be in general-tests.zip

REQUIRED_MODULES = frozenset(

    ['cts-tradefed', 'vts-tradefed', 'compatibility-host-util', 'soong_zip']

)





def build_test_suites(argv):

  args = parse_args(argv)



  if not os.environ.get('BUILD_NUMBER')[0] == 'P':

    build_everything(args)

    return



  # Call the class to map changed files to modules to build.

  # TODO(lucafarsi): Move this into a replaceable class.

  build_affected_modules(args)





def parse_args(argv):

  argparser = argparse.ArgumentParser()

  argparser.add_argument(

      'extra_targets', nargs='*', help='Extra test suites to build.'

  )

  argparser.add_argument('--target_product')

  argparser.add_argument('--target_release')

  argparser.add_argument(

      '--with_dexpreopt_boot_img_and_system_server_only', action='store_true'

  )

  argparser.add_argument('--dist_dir')

  argparser.add_argument('--change_info', nargs='?')

  argparser.add_argument('--extra_required_modules', nargs='*')



  return argparser.parse_args()





def build_everything(args: argparse.Namespace):

  build_command = base_build_command(args)

  build_command.append('general-tests')



  run_command(build_command, print_output=True)





def build_affected_modules(args: argparse.Namespace):

  modules_to_build = find_modules_to_build(

      pathlib.Path(args.change_info), args.extra_required_modules

  )



  # Call the build command with everything.

  build_command = base_build_command(args)

  build_command.extend(modules_to_build)

  # When not building general-tests we also have to build the general tests

  # shared libs.

  build_command.append('general-tests-shared-libs')



  run_command(build_command, print_output=True)



  zip_build_outputs(modules_to_build, args.dist_dir, args.target_release)





def base_build_command(args: argparse.Namespace) -> list:

  build_command = []

  build_command.append('time')

  build_command.append('./build/soong/soong_ui.bash')

  build_command.append('--make-mode')

  build_command.append('dist')

  build_command.append('DIST_DIR=' + args.dist_dir)

  build_command.append('TARGET_PRODUCT=' + args.target_product)

  build_command.append('TARGET_RELEASE=' + args.target_release)

  if args.with_dexpreopt_boot_img_and_system_server_only:

    build_command.append('WITH_DEXPREOPT_BOOT_IMG_AND_SYSTEM_SERVER_ONLY=true')

  build_command.extend(args.extra_targets)



  return build_command





def run_command(

    args: list[str],

    env: dict[str, str] = os.environ,

    print_output: bool = False,

) -> str:

  result = subprocess.run(

      args=args,

      text=True,

      capture_output=True,

      check=False,

      env=env,

  )

  # If the process failed, print its stdout and propagate the exception.

  if not result.returncode == 0:

    print('Build command failed! output:')

    print('stdout: ' + result.stdout)

    print('stderr: ' + result.stderr)



  result.check_returncode()



  if print_output:

    print(result.stdout)



  return result.stdout





def find_modules_to_build(

    change_info: pathlib.Path, extra_required_modules: list[str]

) -> set[str]:

  changed_files = find_changed_files(change_info)



  test_mappings = test_mapping_module_retriever.GetTestMappings(

      changed_files, set()

  )



  # Soong_zip is required to generate the output zip so always build it.

  modules_to_build = set(REQUIRED_MODULES)

  if extra_required_modules:

    modules_to_build.update(extra_required_modules)



  modules_to_build.update(find_affected_modules(test_mappings, changed_files))



  return modules_to_build





def find_changed_files(change_info: pathlib.Path) -> set[str]:

  with open(change_info) as change_info_file:

    change_info_contents = json.load(change_info_file)



  changed_files = set()



  for change in change_info_contents['changes']:

    project_path = change.get('projectPath') + '/'



    for revision in change.get('revisions'):

      for file_info in revision.get('fileInfos'):

        changed_files.add(project_path + file_info.get('path'))



  return changed_files





def find_affected_modules(

    test_mappings: dict[str, Any], changed_files: set[str]

) -> set[str]:

  modules = set()



  # The test_mappings object returned by GetTestMappings is organized as

  # follows:

  # {

  #   'test_mapping_file_path': {

  #     'group_name' : [

  #       'name': 'module_name',

  #     ],

  #   }

  # }

  for test_mapping in test_mappings.values():

    for group in test_mapping.values():

      for entry in group:

        module_name = entry.get('name', None)



        if not module_name:

          continue



        file_patterns = entry.get('file_patterns')

        if not file_patterns:

          modules.add(module_name)

          continue



        if matches_file_patterns(file_patterns, changed_files):

          modules.add(module_name)

          continue



  return modules





# TODO(lucafarsi): Share this logic with the original logic in

# test_mapping_test_retriever.py

def matches_file_patterns(

    file_patterns: list[set], changed_files: set[str]

) -> bool:

  for changed_file in changed_files:

    for pattern in file_patterns:

      if re.search(pattern, changed_file):

        return True



  return False





def zip_build_outputs(

    modules_to_build: set[str], dist_dir: str, target_release: str

):

  src_top = os.environ.get('TOP', os.getcwd())



  # Call dumpvars to get the necessary things.

  # TODO(lucafarsi): Don't call soong_ui 4 times for this, --dumpvars-mode can

  # do it but it requires parsing.

  host_out_testcases = pathlib.Path(

      get_soong_var('HOST_OUT_TESTCASES', target_release)

  )

  target_out_testcases = pathlib.Path(

      get_soong_var('TARGET_OUT_TESTCASES', target_release)

  )

  product_out = pathlib.Path(get_soong_var('PRODUCT_OUT', target_release))

  soong_host_out = pathlib.Path(get_soong_var('SOONG_HOST_OUT', target_release))

  host_out = pathlib.Path(get_soong_var('HOST_OUT', target_release))



  # Call the class to package the outputs.

  # TODO(lucafarsi): Move this code into a replaceable class.

  host_paths = []

  target_paths = []

  host_config_files = []

  target_config_files = []

  for module in modules_to_build:

    host_path = os.path.join(host_out_testcases, module)

    if os.path.exists(host_path):

      host_paths.append(host_path)

      collect_config_files(src_top, host_path, host_config_files)



    target_path = os.path.join(target_out_testcases, module)

    if os.path.exists(target_path):

      target_paths.append(target_path)

      collect_config_files(src_top, target_path, target_config_files)



  zip_test_configs_zips(

      dist_dir, host_out, product_out, host_config_files, target_config_files

  )



  zip_command = base_zip_command(host_out, dist_dir, 'general-tests.zip')



  # Add host testcases.

  zip_command.append('-C')

  zip_command.append(os.path.join(src_top, soong_host_out))

  zip_command.append('-P')

  zip_command.append('host/')

  for path in host_paths:

    zip_command.append('-D')

    zip_command.append(path)



  # Add target testcases.

  zip_command.append('-C')

  zip_command.append(os.path.join(src_top, product_out))

  zip_command.append('-P')

  zip_command.append('target')

  for path in target_paths:

    zip_command.append('-D')

    zip_command.append(path)



  # TODO(lucafarsi): Push this logic into a general-tests-minimal build command

  # Add necessary tools. These are also hardcoded in general-tests.mk.

  framework_path = os.path.join(soong_host_out, 'framework')



  zip_command.append('-C')

  zip_command.append(framework_path)

  zip_command.append('-P')

  zip_command.append('host/tools')

  zip_command.append('-f')

  zip_command.append(os.path.join(framework_path, 'cts-tradefed.jar'))

  zip_command.append('-f')

  zip_command.append(

      os.path.join(framework_path, 'compatibility-host-util.jar')

  )

  zip_command.append('-f')

  zip_command.append(os.path.join(framework_path, 'vts-tradefed.jar'))



  run_command(zip_command, print_output=True)





def collect_config_files(

    src_top: pathlib.Path, root_dir: pathlib.Path, config_files: list[str]

):

  for root, dirs, files in os.walk(os.path.join(src_top, root_dir)):

    for file in files:

      if file.endswith('.config'):

        config_files.append(os.path.join(root_dir, file))





def base_zip_command(

    host_out: pathlib.Path, dist_dir: pathlib.Path, name: str

) -> list[str]:

  return [

      'time',

      os.path.join(host_out, 'bin', 'soong_zip'),

      '-d',

      '-o',

      os.path.join(dist_dir, name),

  ]





# generate general-tests_configs.zip which contains all of the .config files

# that were built and general-tests_list.zip which contains a text file which

# lists all of the .config files that are in general-tests_configs.zip.

#

# general-tests_comfigs.zip is organized as follows:

# /

#   host/

#     testcases/

#       test_1.config

#       test_2.config

#       ...

#   target/

#     testcases/

#       test_1.config

#       test_2.config

#       ...

#

# So the process is we write out the paths to all the host config files into one

# file and all the paths to the target config files in another. We also write

# the paths to all the config files into a third file to use for

# general-tests_list.zip.

def zip_test_configs_zips(

    dist_dir: pathlib.Path,

    host_out: pathlib.Path,

    product_out: pathlib.Path,

    host_config_files: list[str],

    target_config_files: list[str],

):

  with open(

      os.path.join(host_out, 'host_general-tests_list'), 'w'

  ) as host_list_file, open(

      os.path.join(product_out, 'target_general-tests_list'), 'w'

  ) as target_list_file, open(

      os.path.join(host_out, 'general-tests_list'), 'w'

  ) as list_file:



    for config_file in host_config_files:

      host_list_file.write(config_file + '\n')

      list_file.write('host/' + os.path.relpath(config_file, host_out) + '\n')



    for config_file in target_config_files:

      target_list_file.write(config_file + '\n')

      list_file.write(

          'target/' + os.path.relpath(config_file, product_out) + '\n'

      )



  tests_config_zip_command = base_zip_command(

      host_out, dist_dir, 'general-tests_configs.zip'

  )

  tests_config_zip_command.append('-P')

  tests_config_zip_command.append('host')

  tests_config_zip_command.append('-C')

  tests_config_zip_command.append(host_out)

  tests_config_zip_command.append('-l')

  tests_config_zip_command.append(

      os.path.join(host_out, 'host_general-tests_list')

  )

  tests_config_zip_command.append('-P')

  tests_config_zip_command.append('target')

  tests_config_zip_command.append('-C')

  tests_config_zip_command.append(product_out)

  tests_config_zip_command.append('-l')

  tests_config_zip_command.append(

      os.path.join(product_out, 'target_general-tests_list')

  )

  run_command(tests_config_zip_command, print_output=True)



  tests_list_zip_command = base_zip_command(

      host_out, dist_dir, 'general-tests_list.zip'

  )

  tests_list_zip_command.append('-C')

  tests_list_zip_command.append(host_out)

  tests_list_zip_command.append('-f')

  tests_list_zip_command.append(os.path.join(host_out, 'general-tests_list'))

  run_command(tests_list_zip_command, print_output=True)





def get_soong_var(var: str, target_release: str) -> str:

  new_env = os.environ.copy()

  new_env['TARGET_RELEASE'] = target_release



  value = run_command(

      ['./build/soong/soong_ui.bash', '--dumpvar-mode', '--abs', var],

      env=new_env,

  ).strip()

  if not value:

    raise RuntimeError('Necessary soong variable ' + var + ' not found.')



  return value





def main(argv):

  build_test_suites(sys.argv)

ci/test_mapping_module_retriever.py

0 → 100644
+125 −0
Original line number Diff line number Diff line 
# Copyright 2024, The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



"""

Simple parsing code to scan test_mapping files and determine which

modules are needed to build for the given list of changed files.

TODO(lucafarsi): Deduplicate from artifact_helper.py

"""



from typing import Any, Dict, Set, Text

import json

import os

import re



# Regex to extra test name from the path of test config file.

TEST_NAME_REGEX = r'(?:^|.*/)([^/]+)\.config'



# Key name for TEST_MAPPING imports

KEY_IMPORTS = 'imports'

KEY_IMPORT_PATH = 'path'



# Name of TEST_MAPPING file.

TEST_MAPPING = 'TEST_MAPPING'



# Pattern used to identify double-quoted strings and '//'-format comments in

# TEST_MAPPING file, but only double-quoted strings are included within the

# matching group.

_COMMENTS_RE = re.compile(r'(\"(?:[^\"\\]|\\.)*\"|(?=//))(?://.*)?')





def FilterComments(test_mapping_file: Text) -> Text:

  """Remove comments in TEST_MAPPING file to valid format.



  Only '//' is regarded as comments.



  Args:

    test_mapping_file: Path to a TEST_MAPPING file.



  Returns:

    Valid json string without comments.

  """

  return re.sub(_COMMENTS_RE, r'\1', test_mapping_file)



def GetTestMappings(paths: Set[Text],

                    checked_paths: Set[Text]) -> Dict[Text, Dict[Text, Any]]:

  """Get the affected TEST_MAPPING files.



  TEST_MAPPING files in source code are packaged into a build artifact

  `test_mappings.zip`. Inside the zip file, the path of each TEST_MAPPING file

  is preserved. From all TEST_MAPPING files in the source code, this method

  locates the affected TEST_MAPPING files based on the given paths list.



  A TEST_MAPPING file may also contain `imports` that import TEST_MAPPING files

  from a different location, e.g.,

    "imports": [

      {

        "path": "../folder2"

      }

    ]

  In that example, TEST_MAPPING files inside ../folder2 (relative to the

  TEST_MAPPING file containing that imports section) and its parent directories

  will also be included.



  Args:

    paths: A set of paths with related TEST_MAPPING files for given changes.

    checked_paths: A set of paths that have been checked for TEST_MAPPING file

      already. The set is updated after processing each TEST_MAPPING file. It's

      used to prevent infinite loop when the method is called recursively.



  Returns:

    A dictionary of Test Mapping containing the content of the affected

      TEST_MAPPING files, indexed by the path containing the TEST_MAPPING file.

  """

  test_mappings = {}



  # Search for TEST_MAPPING files in each modified path and its parent

  # directories.

  all_paths = set()

  for path in paths:

    dir_names = path.split(os.path.sep)

    all_paths |= set(

        [os.path.sep.join(dir_names[:i + 1]) for i in range(len(dir_names))])

  # Add root directory to the paths to search for TEST_MAPPING file.

  all_paths.add('')



  all_paths.difference_update(checked_paths)

  checked_paths |= all_paths

  # Try to load TEST_MAPPING file in each possible path.

  for path in all_paths:

    try:

      test_mapping_file = os.path.join(os.path.join(os.getcwd(), path), 'TEST_MAPPING')

      # Read content of TEST_MAPPING file.

      content = FilterComments(open(test_mapping_file, "r").read())

      test_mapping = json.loads(content)

      test_mappings[path] = test_mapping



      import_paths = set()

      for import_detail in test_mapping.get(KEY_IMPORTS, []):

        import_path = import_detail[KEY_IMPORT_PATH]

        # Try the import path as absolute path.

        import_paths.add(import_path)

        # Try the import path as relative path based on the test mapping file

        # containing the import.

        norm_import_path = os.path.normpath(os.path.join(path, import_path))

        import_paths.add(norm_import_path)

      import_paths.difference_update(checked_paths)

      if import_paths:

        import_test_mappings = GetTestMappings(import_paths, checked_paths)

        test_mappings.update(import_test_mappings)

    except (KeyError, FileNotFoundError, NotADirectoryError):

      # TEST_MAPPING file doesn't exist in path

      pass



  return test_mappings

cogsetup.sh

+9 −32
Original line number Diff line number Diff line
@@ -21,43 +21,21 @@ function _create_out_symlink_for_cog() { 
    OUT_DIR="out"

  fi



  if [[ -L "${OUT_DIR}" ]]; then

  # getoutdir ensures paths are absolute. envsetup could be called from a

  # directory other than the root of the source tree

  local outdir=$(getoutdir)

  if [[ -L "${outdir}" ]]; then

    return

  fi

  if [ -d "${OUT_DIR}" ]; then

    echo -e "\tOutput directory ${OUT_DIR} cannot be present in a Cog workspace."

    echo -e "\tDelete \"${OUT_DIR}\" or create a symlink from \"${OUT_DIR}\" to a directory outside your workspace."

  if [ -d "${outdir}" ]; then

    echo -e "\tOutput directory ${outdir} cannot be present in a Cog workspace."

    echo -e "\tDelete \"${outdir}\" or create a symlink from \"${outdir}\" to a directory outside your workspace."

    return 1

  fi



  DEFAULT_OUTPUT_DIR="${HOME}/.cog/android-build-out"

  mkdir -p ${DEFAULT_OUTPUT_DIR}

  ln -s ${DEFAULT_OUTPUT_DIR} `pwd`/out

}



# This function moves the reclient binaries into a directory that exists in a

# non-cog part of the overall filesystem.  This is to workaround the problem

# described in b/289391270.

function _copy_reclient_binaries_from_cog() {

  if [[ "${OUT_DIR}" == "" ]]; then

    OUT_DIR="out"

  fi

  local RECLIENT_VERSION=`readlink prebuilts/remoteexecution-client/live`



  local NONCOG_RECLIENT_BIN_DIR_BASE="${OUT_DIR}/.reclient"

  local NONCOG_RECLIENT_BIN_DIR="${NONCOG_RECLIENT_BIN_DIR_BASE}/${RECLIENT_VERSION}"



  # Create the non cog directory and setup live symlink.

  mkdir -p ${NONCOG_RECLIENT_BIN_DIR}



  if [ `ls ${NONCOG_RECLIENT_BIN_DIR} | wc -l` -lt 8 ]; then

    # Not all binaries exist, copy them from the Cog directory.

    local TOP=$(gettop)

    cp ${TOP}/prebuilts/remoteexecution-client/live/* ${NONCOG_RECLIENT_BIN_DIR}

  fi



  ln -sfn ${RECLIENT_VERSION} ${NONCOG_RECLIENT_BIN_DIR_BASE}/live

  export RBE_DIR="${NONCOG_RECLIENT_BIN_DIR_BASE}/live"

  ln -s ${DEFAULT_OUTPUT_DIR} ${outdir}

}



# This function sets up the build environment to be appropriate for Cog.
@@ -67,7 +45,6 @@ function _setup_cog_env() { 
    echo -e "\e[0;33mWARNING:\e[00m Cog environment setup failed!"

    return 1

  fi

  _copy_reclient_binaries_from_cog



  export ANDROID_BUILD_ENVIRONMENT_CONFIG="googler-cog"

core/Makefile

+71 −26

File changed.

Preview size limit exceeded, changes collapsed.