Merge "Stream build process output" into main (ad64a4d2) · Commits · e / os / android_build

ci/Android.bp

0 → 100644

+85 −0

Original line number	Diff line number	Diff line
		// Copyright 2024 Google Inc. All rights reserved.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		package {
		default_applicable_licenses: ["Android-Apache-2.0"],
		}

		python_test_host {
		name: "build_test_suites_test",
		main: "build_test_suites_test.py",
		pkg_path: "testdata",
		srcs: [
		"build_test_suites_test.py",
		],
		libs: [
		"build_test_suites",
		"pyfakefs",
		"ci_test_lib",
		],
		test_options: {
		unit_test: true,
		},
		data: [
		":py3-cmd",
		],
		version: {
		py3: {
		embedded_launcher: true,
		},
		},
		}

		// This test is only intended to be run locally since it's slow, not hermetic,
		// and requires a lot of system state. It is therefore not marked as `unit_test`
		// and is not part of any test suite. Note that we also don't want to run this
		// test with Bazel since that would require disabling sandboxing and explicitly
		// passing in all the env vars we depend on via the command-line. The test
		// target could be configured to do so but it's not worth doing seeing that
		// we're moving away from Bazel.
		python_test_host {
		name: "build_test_suites_local_test",
		main: "build_test_suites_local_test.py",
		srcs: [
		"build_test_suites_local_test.py",
		],
		libs: [
		"build_test_suites",
		"pyfakefs",
		"ci_test_lib",
		],
		test_config_template: "AndroidTest.xml.template",
		test_options: {
		unit_test: false,
		},
		version: {
		py3: {
		embedded_launcher: true,
		},
		},
		}

		python_library_host {
		name: "build_test_suites",
		srcs: [
		"build_test_suites.py",
		],
		}

		python_library_host {
		name: "ci_test_lib",
		srcs: [
		"ci_test_lib.py",
		],
		}

ci/AndroidTest.xml.template

0 → 100644

+22 −0

Original line number	Diff line number	Diff line
		<?xml version="1.0" encoding="utf-8"?>
		<!-- Copyright (C) 2020 The Android Open Source Project

		Licensed under the Apache License, Version 2.0 (the "License");
		you may not use this file except in compliance with the License.
		You may obtain a copy of the License at

		http://www.apache.org/licenses/LICENSE-2.0

		Unless required by applicable law or agreed to in writing, software
		distributed under the License is distributed on an "AS IS" BASIS,
		WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		See the License for the specific language governing permissions and
		limitations under the License.
		-->
		<configuration>
		<test class="com.android.tradefed.testtype.python.PythonBinaryHostTest">
		<option name="par-file-name" value="{MODULE}"/>
		<option name="use-test-output-file" value="false"/>
		<option name="test-timeout" value="5m"/>
		</test>
		</configuration>

ci/build_test_suites

+3 −2

Original line number	Diff line number	Diff line
		#!prebuilts/build-tools/linux-x86/bin/py3-cmd -B
		#
		# Copyright 2024, The Android Open Source Project
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		@@ -13,7 +14,7 @@
		# See the License for the specific language governing permissions and
		# limitations under the License.

		import sys
		import build_test_suites
		import sys

		build_test_suites.main(sys.argv)
		build_test_suites.main(sys.argv[1:])

ci/build_test_suites.py

+65 −354

Original line number	Diff line number	Diff line
		@@ -12,404 +12,115 @@
		# See the License for the specific language governing permissions and
		# limitations under the License.

		"""Script to build only the necessary modules for general-tests along

		with whatever other targets are passed in.
		"""
		"""Build script for the CI `test_suites` target."""

		import argparse
		from collections.abc import Sequence
		import json
		import logging
		import os
		import pathlib
		import re
		import subprocess
		import sys
		from typing import Any

		import test_mapping_module_retriever


		# List of modules that are always required to be in general-tests.zip
		REQUIRED_MODULES = frozenset(
		['cts-tradefed', 'vts-tradefed', 'compatibility-host-util', 'soong_zip']
		)


		def build_test_suites(argv):
		args = parse_args(argv)

		if is_optimization_enabled():
		# Call the class to map changed files to modules to build.
		# TODO(lucafarsi): Move this into a replaceable class.
		build_affected_modules(args)
		else:
		build_everything(args)


		def parse_args(argv):
		argparser = argparse.ArgumentParser()
		argparser.add_argument(
		'extra_targets', nargs='*', help='Extra test suites to build.'
		)
		argparser.add_argument('--target_product')
		argparser.add_argument('--target_release')
		argparser.add_argument(
		'--with_dexpreopt_boot_img_and_system_server_only', action='store_true'
		)
		argparser.add_argument('--change_info', nargs='?')
		class Error(Exception):

		return argparser.parse_args()
		def __init__(self, message):
		super().__init__(message)


		def is_optimization_enabled() -> bool:
		# TODO(lucafarsi): switch back to building only affected general-tests modules
		# in presubmit once ready.
		# if os.environ.get('BUILD_NUMBER')[0] == 'P':
		# return True
		return False
		class BuildFailureError(Error):

		def __init__(self, return_code):
		super().__init__(f'Build command failed with return code: f{return_code}')
		self.return_code = return_code

		def build_everything(args: argparse.Namespace):
		build_command = base_build_command(args, args.extra_targets)
		build_command.append('general-tests')

		run_command(build_command, print_output=True)
		REQUIRED_ENV_VARS = frozenset(['TARGET_PRODUCT', 'TARGET_RELEASE', 'TOP'])
		SOONG_UI_EXE_REL_PATH = 'build/soong/soong_ui.bash'


		def build_affected_modules(args: argparse.Namespace):
		modules_to_build = find_modules_to_build(
		pathlib.Path(args.change_info), args.extra_required_modules
		)
		def get_top() -> pathlib.Path:
		return pathlib.Path(os.environ['TOP'])

		# Call the build command with everything.
		build_command = base_build_command(args, args.extra_targets)
		build_command.extend(modules_to_build)
		# When not building general-tests we also have to build the general tests
		# shared libs.
		build_command.append('general-tests-shared-libs')

		run_command(build_command, print_output=True)
		def build_test_suites(argv: list[str]) -> int:
		"""Builds the general-tests and any other test suites passed in.

		zip_build_outputs(modules_to_build, args.target_release)
		Args:
		argv: The command line arguments passed in.

		Returns:
		The exit code of the build.
		"""
		args = parse_args(argv)
		check_required_env()

		def base_build_command(
		args: argparse.Namespace, extra_targets: set[str]
		) -> list:
		build_command = []
		build_command.append('time')
		build_command.append('./build/soong/soong_ui.bash')
		build_command.append('--make-mode')
		build_command.append('dist')
		build_command.append('TARGET_PRODUCT=' + args.target_product)
		build_command.append('TARGET_RELEASE=' + args.target_release)
		if args.with_dexpreopt_boot_img_and_system_server_only:
		build_command.append('WITH_DEXPREOPT_BOOT_IMG_AND_SYSTEM_SERVER_ONLY=true')
		build_command.extend(extra_targets)
		try:
		build_everything(args)
		except BuildFailureError as e:
		logging.error('Build command failed! Check build_log for details.')
		return e.return_code

		return build_command
		return 0


		def run_command(
		args: list[str],
		env: dict[str, str] = os.environ,
		print_output: bool = False,
		) -> str:
		result = subprocess.run(
		args=args,
		text=True,
		capture_output=True,
		check=False,
		env=env,
		)
		# If the process failed, print its stdout and propagate the exception.
		if not result.returncode == 0:
		print('Build command failed! output:')
		print('stdout: ' + result.stdout)
		print('stderr: ' + result.stderr)
		def check_required_env():
		"""Check for required env vars.

		result.check_returncode()
		Raises:
		RuntimeError: If any required env vars are not found.
		"""
		missing_env_vars = sorted(v for v in REQUIRED_ENV_VARS if v not in os.environ)

		if print_output:
		print(result.stdout)
		if not missing_env_vars:
		return

		return result.stdout
		t = ','.join(missing_env_vars)
		raise Error(f'Missing required environment variables: {t}')


		def find_modules_to_build(
		change_info: pathlib.Path, extra_required_modules: list[str]
		) -> set[str]:
		changed_files = find_changed_files(change_info)
		def parse_args(argv):
		argparser = argparse.ArgumentParser()

		test_mappings = test_mapping_module_retriever.GetTestMappings(
		changed_files, set()
		argparser.add_argument(
		'extra_targets', nargs='*', help='Extra test suites to build.'
		)

		# Soong_zip is required to generate the output zip so always build it.
		modules_to_build = set(REQUIRED_MODULES)
		if extra_required_modules:
		modules_to_build.update(extra_required_modules)

		modules_to_build.update(find_affected_modules(test_mappings, changed_files))

		return modules_to_build

		return argparser.parse_args(argv)

		def find_changed_files(change_info: pathlib.Path) -> set[str]:
		with open(change_info) as change_info_file:
		change_info_contents = json.load(change_info_file)

		changed_files = set()

		for change in change_info_contents['changes']:
		project_path = change.get('projectPath') + '/'

		for revision in change.get('revisions'):
		for file_info in revision.get('fileInfos'):
		changed_files.add(project_path + file_info.get('path'))

		return changed_files


		def find_affected_modules(
		test_mappings: dict[str, Any], changed_files: set[str]
		) -> set[str]:
		modules = set()

		# The test_mappings object returned by GetTestMappings is organized as
		# follows:
		# {
		# 'test_mapping_file_path': {
		# 'group_name' : [
		# 'name': 'module_name',
		# ],
		# }
		# }
		for test_mapping in test_mappings.values():
		for group in test_mapping.values():
		for entry in group:
		module_name = entry.get('name', None)

		if not module_name:
		continue

		file_patterns = entry.get('file_patterns')
		if not file_patterns:
		modules.add(module_name)
		continue

		if matches_file_patterns(file_patterns, changed_files):
		modules.add(module_name)
		continue

		return modules


		# TODO(lucafarsi): Share this logic with the original logic in
		# test_mapping_test_retriever.py
		def matches_file_patterns(
		file_patterns: list[set], changed_files: set[str]
		) -> bool:
		for changed_file in changed_files:
		for pattern in file_patterns:
		if re.search(pattern, changed_file):
		return True

		return False


		def zip_build_outputs(
		modules_to_build: set[str], target_release: str
		):
		src_top = os.environ.get('TOP', os.getcwd())

		# Call dumpvars to get the necessary things.
		# TODO(lucafarsi): Don't call soong_ui 4 times for this, --dumpvars-mode can
		# do it but it requires parsing.
		host_out_testcases = pathlib.Path(
		get_soong_var('HOST_OUT_TESTCASES', target_release)
		)
		target_out_testcases = pathlib.Path(
		get_soong_var('TARGET_OUT_TESTCASES', target_release)
		)
		product_out = pathlib.Path(get_soong_var('PRODUCT_OUT', target_release))
		soong_host_out = pathlib.Path(get_soong_var('SOONG_HOST_OUT', target_release))
		host_out = pathlib.Path(get_soong_var('HOST_OUT', target_release))
		dist_dir = pathlib.Path(get_soong_var('DIST_DIR', target_release))

		# Call the class to package the outputs.
		# TODO(lucafarsi): Move this code into a replaceable class.
		host_paths = []
		target_paths = []
		host_config_files = []
		target_config_files = []
		for module in modules_to_build:
		host_path = os.path.join(host_out_testcases, module)
		if os.path.exists(host_path):
		host_paths.append(host_path)
		collect_config_files(src_top, host_path, host_config_files)

		target_path = os.path.join(target_out_testcases, module)
		if os.path.exists(target_path):
		target_paths.append(target_path)
		collect_config_files(src_top, target_path, target_config_files)

		zip_test_configs_zips(
		dist_dir, host_out, product_out, host_config_files, target_config_files
		)

		zip_command = base_zip_command(host_out, dist_dir, 'general-tests.zip')

		# Add host testcases.
		zip_command.append('-C')
		zip_command.append(os.path.join(src_top, soong_host_out))
		zip_command.append('-P')
		zip_command.append('host/')
		for path in host_paths:
		zip_command.append('-D')
		zip_command.append(path)

		# Add target testcases.
		zip_command.append('-C')
		zip_command.append(os.path.join(src_top, product_out))
		zip_command.append('-P')
		zip_command.append('target')
		for path in target_paths:
		zip_command.append('-D')
		zip_command.append(path)

		# TODO(lucafarsi): Push this logic into a general-tests-minimal build command
		# Add necessary tools. These are also hardcoded in general-tests.mk.
		framework_path = os.path.join(soong_host_out, 'framework')

		zip_command.append('-C')
		zip_command.append(framework_path)
		zip_command.append('-P')
		zip_command.append('host/tools')
		zip_command.append('-f')
		zip_command.append(os.path.join(framework_path, 'cts-tradefed.jar'))
		zip_command.append('-f')
		zip_command.append(
		os.path.join(framework_path, 'compatibility-host-util.jar')
		)
		zip_command.append('-f')
		zip_command.append(os.path.join(framework_path, 'vts-tradefed.jar'))
		def build_everything(args: argparse.Namespace):
		"""Builds all tests (regardless of whether they are needed).

		run_command(zip_command, print_output=True)
		Args:
		args: The parsed arguments.

		Raises:
		BuildFailure: If the build command fails.
		"""
		build_command = base_build_command(args, args.extra_targets)

		def collect_config_files(
		src_top: pathlib.Path, root_dir: pathlib.Path, config_files: list[str]
		):
		for root, dirs, files in os.walk(os.path.join(src_top, root_dir)):
		for file in files:
		if file.endswith('.config'):
		config_files.append(os.path.join(root_dir, file))
		try:
		run_command(build_command)
		except subprocess.CalledProcessError as e:
		raise BuildFailureError(e.returncode) from e


		def base_zip_command(
		host_out: pathlib.Path, dist_dir: pathlib.Path, name: str
		def base_build_command(
		args: argparse.Namespace, extra_targets: set[str]
		) -> list[str]:
		return [
		'time',
		os.path.join(host_out, 'bin', 'soong_zip'),
		'-d',
		'-o',
		os.path.join(dist_dir, name),
		]


		# generate general-tests_configs.zip which contains all of the .config files
		# that were built and general-tests_list.zip which contains a text file which
		# lists all of the .config files that are in general-tests_configs.zip.
		#
		# general-tests_comfigs.zip is organized as follows:
		# /
		# host/
		# testcases/
		# test_1.config
		# test_2.config
		# ...
		# target/
		# testcases/
		# test_1.config
		# test_2.config
		# ...
		#
		# So the process is we write out the paths to all the host config files into one
		# file and all the paths to the target config files in another. We also write
		# the paths to all the config files into a third file to use for
		# general-tests_list.zip.
		def zip_test_configs_zips(
		dist_dir: pathlib.Path,
		host_out: pathlib.Path,
		product_out: pathlib.Path,
		host_config_files: list[str],
		target_config_files: list[str],
		):
		with open(
		os.path.join(host_out, 'host_general-tests_list'), 'w'
		) as host_list_file, open(
		os.path.join(product_out, 'target_general-tests_list'), 'w'
		) as target_list_file, open(
		os.path.join(host_out, 'general-tests_list'), 'w'
		) as list_file:

		for config_file in host_config_files:
		host_list_file.write(config_file + '\n')
		list_file.write('host/' + os.path.relpath(config_file, host_out) + '\n')

		for config_file in target_config_files:
		target_list_file.write(config_file + '\n')
		list_file.write(
		'target/' + os.path.relpath(config_file, product_out) + '\n'
		)

		tests_config_zip_command = base_zip_command(
		host_out, dist_dir, 'general-tests_configs.zip'
		)
		tests_config_zip_command.append('-P')
		tests_config_zip_command.append('host')
		tests_config_zip_command.append('-C')
		tests_config_zip_command.append(host_out)
		tests_config_zip_command.append('-l')
		tests_config_zip_command.append(
		os.path.join(host_out, 'host_general-tests_list')
		)
		tests_config_zip_command.append('-P')
		tests_config_zip_command.append('target')
		tests_config_zip_command.append('-C')
		tests_config_zip_command.append(product_out)
		tests_config_zip_command.append('-l')
		tests_config_zip_command.append(
		os.path.join(product_out, 'target_general-tests_list')
		)
		run_command(tests_config_zip_command, print_output=True)

		tests_list_zip_command = base_zip_command(
		host_out, dist_dir, 'general-tests_list.zip'
		)
		tests_list_zip_command.append('-C')
		tests_list_zip_command.append(host_out)
		tests_list_zip_command.append('-f')
		tests_list_zip_command.append(os.path.join(host_out, 'general-tests_list'))
		run_command(tests_list_zip_command, print_output=True)

		build_command = []
		build_command.append(get_top().joinpath(SOONG_UI_EXE_REL_PATH))
		build_command.append('--make-mode')
		build_command.extend(extra_targets)

		def get_soong_var(var: str, target_release: str) -> str:
		new_env = os.environ.copy()
		new_env['TARGET_RELEASE'] = target_release
		return build_command

		value = run_command(
		['./build/soong/soong_ui.bash', '--dumpvar-mode', '--abs', var],
		env=new_env,
		).strip()
		if not value:
		raise RuntimeError('Necessary soong variable ' + var + ' not found.')

		return value
		def run_command(args: list[str], stdout=None):
		subprocess.run(args=args, check=True, stdout=stdout)


		def main(argv):
		build_test_suites(argv)
		sys.exit(build_test_suites(argv))

ci/build_test_suites_local_test.py

0 → 100644

+123 −0

Original line number	Diff line number	Diff line
		# Copyright 2024, The Android Open Source Project
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.

		"""Integration tests for build_test_suites that require a local build env."""

		import os
		import pathlib
		import shutil
		import signal
		import subprocess
		import tempfile
		import time
		import ci_test_lib


		class BuildTestSuitesLocalTest(ci_test_lib.TestCase):

		def setUp(self):
		self.top_dir = pathlib.Path(os.environ['ANDROID_BUILD_TOP']).resolve()
		self.executable = self.top_dir.joinpath('build/make/ci/build_test_suites')
		self.process_session = ci_test_lib.TemporaryProcessSession(self)
		self.temp_dir = ci_test_lib.TestTemporaryDirectory.create(self)

		def build_subprocess_args(self, build_args: list[str]):
		env = os.environ.copy()
		env['TOP'] = str(self.top_dir)
		env['OUT_DIR'] = self.temp_dir

		args = ([self.executable] + build_args,)
		kwargs = {
		'cwd': self.top_dir,
		'env': env,
		'text': True,
		}

		return (args, kwargs)

		def run_build(self, build_args: list[str]) -> subprocess.CompletedProcess:
		args, kwargs = self.build_subprocess_args(build_args)

		return subprocess.run(
		*args,
		**kwargs,
		check=True,
		capture_output=True,
		timeout=5 * 60,
		)

		def assert_children_alive(self, children: list[int]):
		for c in children:
		self.assertTrue(ci_test_lib.process_alive(c))

		def assert_children_dead(self, children: list[int]):
		for c in children:
		self.assertFalse(ci_test_lib.process_alive(c))

		def test_fails_for_invalid_arg(self):
		invalid_arg = '--invalid-arg'

		with self.assertRaises(subprocess.CalledProcessError) as cm:
		self.run_build([invalid_arg])

		self.assertIn(invalid_arg, cm.exception.stderr)

		def test_builds_successfully(self):
		self.run_build(['nothing'])

		def test_can_interrupt_build(self):
		args, kwargs = self.build_subprocess_args(['general-tests'])
		p = self.process_session.create(args, kwargs)

		# TODO(lucafarsi): Replace this (and other instances) with a condition.
		time.sleep(5) # Wait for the build to get going.
		self.assertIsNone(p.poll()) # Check that the process is still alive.
		children = query_child_pids(p.pid)
		self.assert_children_alive(children)

		p.send_signal(signal.SIGINT)
		p.wait()

		time.sleep(5) # Wait for things to die out.
		self.assert_children_dead(children)

		def test_can_kill_build_process_group(self):
		args, kwargs = self.build_subprocess_args(['general-tests'])
		p = self.process_session.create(args, kwargs)

		time.sleep(5) # Wait for the build to get going.
		self.assertIsNone(p.poll()) # Check that the process is still alive.
		children = query_child_pids(p.pid)
		self.assert_children_alive(children)

		os.killpg(os.getpgid(p.pid), signal.SIGKILL)
		p.wait()

		time.sleep(5) # Wait for things to die out.
		self.assert_children_dead(children)


		# TODO(hzalek): Replace this with `psutils` once available in the tree.
		def query_child_pids(parent_pid: int) -> set[int]:
		p = subprocess.run(
		['pgrep', '-P', str(parent_pid)],
		check=True,
		capture_output=True,
		text=True,
		)
		return {int(pid) for pid in p.stdout.splitlines()}


		if __name__ == '__main__':
		ci_test_lib.main()