Merge "Don't build targets if they're not used." into main (1ef9d680) · Commits · e / os / android_build

ci/build_test_suites.py

+73 −2

Original line number	Diff line number	Diff line
		@@ -20,6 +20,7 @@ import json
		import logging
		import os
		import pathlib
		import re
		import subprocess
		import sys
		from typing import Callable
		@@ -52,6 +53,15 @@ class BuildPlanner:
		any output zip files needed by the build.
		"""

		_DOWNLOAD_OPTS = {
		'test-config-only-zip',
		'test-zip-file-filter',
		'extra-host-shared-lib-zip',
		'sandbox-tests-zips',
		'additional-files-filter',
		'cts-package-name',
		}

		def __init__(
		self,
		build_context: dict[str, any],
		@@ -64,12 +74,19 @@ class BuildPlanner:

		def create_build_plan(self):

		if 'optimized_build' not in self.build_context['enabled_build_features']:
		if 'optimized_build' not in self.build_context.get(
		'enabledBuildFeatures', []
		):
		return BuildPlan(set(self.args.extra_targets), set())

		build_targets = set()
		packaging_functions = set()
		for target in self.args.extra_targets:
		if self._unused_target_exclusion_enabled(
		target
		) and not self._build_target_used(target):
		continue

		target_optimizer_getter = self.target_optimizations.get(target, None)
		if not target_optimizer_getter:
		build_targets.add(target)
		@@ -83,6 +100,60 @@ class BuildPlanner:

		return BuildPlan(build_targets, packaging_functions)

		def _unused_target_exclusion_enabled(self, target: str) -> bool:
		return f'{target}_unused_exclusion' in self.build_context.get(
		'enabledBuildFeatures', []
		)

		def _build_target_used(self, target: str) -> bool:
		"""Determines whether this target's outputs are used by the test configurations listed in the build context."""
		file_download_regexes = self._aggregate_file_download_regexes()
		# For all of a targets' outputs, check if any of the regexes used by tests
		# to download artifacts would match it. If any of them do then this target
		# is necessary.
		for artifact in self._get_target_potential_outputs(target):
		for regex in file_download_regexes:
		if re.match(regex, artifact):
		return True
		return False

		def _get_target_potential_outputs(self, target: str) -> set[str]:
		tests_suffix = '-tests'
		if target.endswith('tests'):
		tests_suffix = ''
		# This is a list of all the potential zips output by the test suite targets.
		# If the test downloads artifacts from any of these zips, we will be
		# conservative and avoid skipping the tests.
		return {
		f'{target}.zip',
		f'android-{target}.zip',
		f'android-{target}-verifier.zip',
		f'{target}{tests_suffix}_list.zip',
		f'android-{target}{tests_suffix}_list.zip',
		f'{target}{tests_suffix}_host-shared-libs.zip',
		f'android-{target}{tests_suffix}_host-shared-libs.zip',
		f'{target}{tests_suffix}_configs.zip',
		f'android-{target}{tests_suffix}_configs.zip',
		}

		def _aggregate_file_download_regexes(self) -> set[re.Pattern]:
		"""Lists out all test config options to specify targets to download.

		These come in the form of regexes.
		"""
		all_regexes = set()
		for test_info in self._get_test_infos():
		for opt in test_info.get('extraOptions', []):
		# check the known list of options for downloading files.
		if opt.get('key') in self._DOWNLOAD_OPTS:
		all_regexes.update(
		re.compile(value) for value in opt.get('values', [])
		)
		return all_regexes

		def _get_test_infos(self):
		return self.build_context.get('testContext', dict()).get('testInfos', [])


		@dataclass(frozen=True)
		class BuildPlan:
		@@ -155,7 +226,7 @@ def load_build_context():


		def empty_build_context():
		return {'enabled_build_features': []}
		return {'enabledBuildFeatures': []}


		def execute_build_plan(build_plan: BuildPlan):

ci/build_test_suites_test.py

+71 −3

Original line number	Diff line number	Diff line
		@@ -324,6 +324,55 @@ class BuildPlannerTest(unittest.TestCase):
		self.run_packaging_functions(build_plan),
		)

		def test_target_output_used_target_built(self):
		build_target = 'test_target'
		build_planner = self.create_build_planner(
		build_targets={build_target},
		build_context=self.create_build_context(
		test_context=self.get_test_context(build_target),
		enabled_build_features={'test_target_unused_exclusion'},
		),
		)

		build_plan = build_planner.create_build_plan()

		self.assertSetEqual(build_plan.build_targets, {build_target})

		def test_target_regex_used_target_built(self):
		build_target = 'test_target'
		test_context = self.get_test_context(build_target)
		test_context['testInfos'][0]['extraOptions'] = [{
		'key': 'additional-files-filter',
		'values': [f'.{build_target}.\.zip'],
		}]
		build_planner = self.create_build_planner(
		build_targets={build_target},
		build_context=self.create_build_context(
		test_context=test_context,
		enabled_build_features={'test_target_unused_exclusion'},
		),
		)

		build_plan = build_planner.create_build_plan()

		self.assertSetEqual(build_plan.build_targets, {build_target})

		def test_target_output_not_used_target_not_built(self):
		build_target = 'test_target'
		test_context = self.get_test_context(build_target)
		test_context['testInfos'][0]['extraOptions'] = []
		build_planner = self.create_build_planner(
		build_targets={build_target},
		build_context=self.create_build_context(
		test_context=test_context,
		enabled_build_features={'test_target_unused_exclusion'},
		),
		)

		build_plan = build_planner.create_build_plan()

		self.assertSetEqual(build_plan.build_targets, set())

		def create_build_planner(
		self,
		build_targets: set[str],
		@@ -352,10 +401,10 @@ class BuildPlannerTest(unittest.TestCase):
		test_context: dict[str, any] = {},
		) -> dict[str, any]:
		build_context = {}
		build_context['enabled_build_features'] = enabled_build_features
		build_context['enabledBuildFeatures'] = enabled_build_features
		if optimized_build_enabled:
		build_context['enabled_build_features'].add('optimized_build')
		build_context['test_context'] = test_context
		build_context['enabledBuildFeatures'].add('optimized_build')
		build_context['testContext'] = test_context
		return build_context

		def create_args(
		@@ -398,6 +447,25 @@ class BuildPlannerTest(unittest.TestCase):

		return output

		def get_test_context(self, target: str):
		return {
		'testInfos': [
		{
		'name': 'atp_test',
		'target': 'test_target',
		'branch': 'branch',
		'extraOptions': [{
		'key': 'additional-files-filter',
		'values': [f'{target}.zip'],
		}],
		'command': '/tf/command',
		'extraBuildTargets': [
		'extra_build_target',
		],
		},
		],
		}


		def wait_until(
		condition_function: Callable[[], bool],

ci/optimized_targets.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -53,7 +53,7 @@ class NullOptimizer(OptimizedBuildTarget):


		def get_target_optimizer(target, enabled_flag, build_context, optimizer):
		if enabled_flag in build_context['enabled_build_features']:
		if enabled_flag in build_context['enabledBuildFeatures']:
		return optimizer

		return NullOptimizer(target)