Merge changes I958e0349,Id3711ce0,Id3d09557,I6b6b726b into main (04198dc1) · Commits · e / os / android_build

ci/Android.bp

+7 −5

Original line number	Diff line number	Diff line
		@@ -103,13 +103,10 @@ python_binary_host {
		"test_mapping_module_retriever.py",
		"build_context.py",
		"test_discovery_agent.py",
		"metrics_agent.py",
		"buildbot.py",
		],
		main: "build_test_suites.py",
		version: {
		py3: {
		embedded_launcher: true,
		},
		},
		libs: [
		"soong-metrics-proto-py",
		],
		@@ -123,6 +120,11 @@ python_library_host {
		"test_mapping_module_retriever.py",
		"build_context.py",
		"test_discovery_agent.py",
		"metrics_agent.py",
		"buildbot.py",
		],
		libs: [
		"soong-metrics-proto-py",
		],
		}

ci/build_context.py

+3 −0

Original line number	Diff line number	Diff line
		@@ -47,6 +47,9 @@ class BuildContext:
		self.is_test_mapping = False
		self.test_mapping_test_groups = set()
		self.file_download_options = set()
		self.name = test_info_dict.get('name')
		self.command = test_info_dict.get('command')
		self.extra_options = test_info_dict.get('extraOptions')
		for opt in test_info_dict.get('extraOptions', []):
		key = opt.get('key')
		if key == 'test-mapping-test-group':

ci/build_test_suites

+3 −5

Original line number	Diff line number	Diff line
		#!prebuilts/build-tools/linux-x86/bin/py3-cmd -B
		#!/usr/bin/env bash
		#
		# Copyright 2024, The Android Open Source Project
		#
		@@ -14,7 +14,5 @@
		# See the License for the specific language governing permissions and
		# limitations under the License.

		import build_test_suites
		import sys

		build_test_suites.main(sys.argv[1:])
		build/soong/soong_ui.bash --make-mode build_test_suites \|\| exit $?
		$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites $@ \|\| exit $?

ci/build_test_suites.py

+74 −11

Original line number	Diff line number	Diff line
		@@ -20,11 +20,14 @@ import json
		import logging
		import os
		import pathlib
		import re
		import subprocess
		import sys
		from typing import Callable
		from build_context import BuildContext
		import optimized_targets
		import metrics_agent
		import test_discovery_agent


		REQUIRED_ENV_VARS = frozenset(['TARGET_PRODUCT', 'TARGET_RELEASE', 'TOP'])
		@@ -70,7 +73,24 @@ class BuildPlanner:

		build_targets = set()
		packaging_commands_getters = []
		test_discovery_zip_regexes = set()
		optimization_rationale = ''
		try:
		# Do not use these regexes for now, only run this to collect data on what
		# would be optimized.
		test_discovery_zip_regexes = self._get_test_discovery_zip_regexes()
		logging.info(f'Discovered test discovery regexes: {test_discovery_zip_regexes}')
		except test_discovery_agent.TestDiscoveryError as e:
		optimization_rationale = e.message
		logging.warning(f'Unable to perform test discovery: {optimization_rationale}')
		for target in self.args.extra_targets:
		if optimization_rationale:
		get_metrics_agent().report_unoptimized_target(target, optimization_rationale)
		else:
		regex = r'\b(%s)\b' % re.escape(target)
		if any(re.search(regex, opt) for opt in test_discovery_zip_regexes):
		get_metrics_agent().report_optimized_target(target)

		if self._unused_target_exclusion_enabled(
		target
		) and not self.build_context.build_target_used(target):
		@@ -97,6 +117,34 @@ class BuildPlanner:
		in self.build_context.enabled_build_features
		)

		def _get_test_discovery_zip_regexes(self) -> set[str]:
		build_target_regexes = set()
		for test_info in self.build_context.test_infos:
		tf_command = self._build_tf_command(test_info)
		discovery_agent = test_discovery_agent.TestDiscoveryAgent(tradefed_args=tf_command)
		for regex in discovery_agent.discover_test_zip_regexes():
		build_target_regexes.add(regex)
		return build_target_regexes


		def _build_tf_command(self, test_info) -> list[str]:
		command = [test_info.command]
		for extra_option in test_info.extra_options:
		if not extra_option.get('key'):
		continue
		arg_key = '--' + extra_option.get('key')
		if arg_key == '--build-id':
		command.append(arg_key)
		command.append(os.environ.get('BUILD_NUMBER'))
		continue
		if extra_option.get('values'):
		for value in extra_option.get('values'):
		command.append(arg_key)
		command.append(value)
		else:
		command.append(arg_key)

		return command

		@dataclass(frozen=True)
		class BuildPlan:
		@@ -113,6 +161,8 @@ def build_test_suites(argv: list[str]) -> int:
		Returns:
		The exit code of the build.
		"""
		get_metrics_agent().analysis_start()
		try:
		args = parse_args(argv)
		check_required_env()
		build_context = BuildContext(load_build_context())
		@@ -120,12 +170,18 @@ def build_test_suites(argv: list[str]) -> int:
		build_context, args, optimized_targets.OPTIMIZED_BUILD_TARGETS
		)
		build_plan = build_planner.create_build_plan()
		except:
		raise
		finally:
		get_metrics_agent().analysis_end()

		try:
		execute_build_plan(build_plan)
		except BuildFailureError as e:
		logging.error('Build command failed! Check build_log for details.')
		return e.return_code
		finally:
		get_metrics_agent().end_reporting()

		return 0

		@@ -183,12 +239,15 @@ def execute_build_plan(build_plan: BuildPlan):
		except subprocess.CalledProcessError as e:
		raise BuildFailureError(e.returncode) from e

		for packaging_commands_getter in build_plan.packaging_commands_getters:
		get_metrics_agent().packaging_start()
		try:
		for packaging_commands_getter in build_plan.packaging_commands_getters:
		for packaging_command in packaging_commands_getter():
		run_command(packaging_command)
		except subprocess.CalledProcessError as e:
		raise BuildFailureError(e.returncode) from e
		finally:
		get_metrics_agent().packaging_end()


		def get_top() -> pathlib.Path:
		@@ -199,6 +258,10 @@ def run_command(args: list[str], stdout=None):
		subprocess.run(args=args, check=True, stdout=stdout)


		def get_metrics_agent():
		return metrics_agent.MetricsAgent.instance()


		def main(argv):
		dist_dir = os.environ.get('DIST_DIR')
		if dist_dir:

ci/build_test_suites_test.py

+12 −0

Original line number	Diff line number	Diff line
		@@ -37,6 +37,8 @@ import build_test_suites
		import ci_test_lib
		import optimized_targets
		from pyfakefs import fake_filesystem_unittest
		import metrics_agent
		import test_discovery_agent


		class BuildTestSuitesTest(fake_filesystem_unittest.TestCase):
		@@ -52,6 +54,10 @@ class BuildTestSuitesTest(fake_filesystem_unittest.TestCase):
		self.addCleanup(subprocess_run_patcher.stop)
		self.mock_subprocess_run = subprocess_run_patcher.start()

		metrics_agent_finalize_patcher = mock.patch('metrics_agent.MetricsAgent.end_reporting')
		self.addCleanup(metrics_agent_finalize_patcher.stop)
		self.mock_metrics_agent_end = metrics_agent_finalize_patcher.start()

		self._setup_working_build_env()

		def test_missing_target_release_env_var_raises(self):
		@@ -256,6 +262,12 @@ class BuildPlannerTest(unittest.TestCase):
		def get_enabled_flag(self):
		return f'{self.target}_enabled'

		def setUp(self):
		test_discovery_agent_patcher = mock.patch('test_discovery_agent.TestDiscoveryAgent.discover_test_zip_regexes')
		self.addCleanup(test_discovery_agent_patcher.stop)
		self.mock_test_discovery_agent_end = test_discovery_agent_patcher.start()


		def test_build_optimization_off_builds_everything(self):
		build_targets = {'target_1', 'target_2'}
		build_planner = self.create_build_planner(