Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b24c1c3d authored by Luca Farsi's avatar Luca Farsi
Browse files

Don't build targets if they're not used. 

Add functionality in build_test_suites.py to not build targets if
their outputs are not used in the test configurations saved in the build
context. If none of the tests reference the targets' outputs they will
not be built at all.

Note that the corresponding flags will need to be enabled for these
optimizations to take place.

Test: atest optimized_targets_test
Bug: 348489774
Change-Id: I8f0ac90e75552ae80073f13229b026c7f23476a6
parent 8afaf50f

ci/build_test_suites.py

+73 −2
Original line number Original line Diff line number Diff line
@@ -20,6 +20,7 @@ import json 
import logging

import logging

import os

import os

import pathlib

import pathlib

import re

import subprocess

import subprocess

import sys

import sys

from typing import Callable

from typing import Callable
@@ -51,6 +52,15 @@ class BuildPlanner: 
  any output zip files needed by the build.

  any output zip files needed by the build.

  """

  """





  _DOWNLOAD_OPTS = {

      'test-config-only-zip',

      'test-zip-file-filter',

      'extra-host-shared-lib-zip',

      'sandbox-tests-zips',

      'additional-files-filter',

      'cts-package-name',

  }



  def __init__(

  def __init__(

      self,

      self,

      build_context: dict[str, any],

      build_context: dict[str, any],
@@ -63,12 +73,19 @@ class BuildPlanner: 




  def create_build_plan(self):

  def create_build_plan(self):





    if 'optimized_build' not in self.build_context['enabled_build_features']:

    if 'optimized_build' not in self.build_context.get(

        'enabledBuildFeatures', []

    ):

      return BuildPlan(set(self.args.extra_targets), set())

      return BuildPlan(set(self.args.extra_targets), set())





    build_targets = set()

    build_targets = set()

    packaging_functions = set()

    packaging_functions = set()

    for target in self.args.extra_targets:

    for target in self.args.extra_targets:

      if self._unused_target_exclusion_enabled(

          target

      ) and not self._build_target_used(target):

        continue



      target_optimizer_getter = self.target_optimizations.get(target, None)

      target_optimizer_getter = self.target_optimizations.get(target, None)

      if not target_optimizer_getter:

      if not target_optimizer_getter:

        build_targets.add(target)

        build_targets.add(target)
@@ -82,6 +99,60 @@ class BuildPlanner: 




    return BuildPlan(build_targets, packaging_functions)

    return BuildPlan(build_targets, packaging_functions)





  def _unused_target_exclusion_enabled(self, target: str) -> bool:

    return f'{target}_unused_exclusion' in self.build_context.get(

        'enabledBuildFeatures', []

    )



  def _build_target_used(self, target: str) -> bool:

    """Determines whether this target's outputs are used by the test configurations listed in the build context."""

    file_download_regexes = self._aggregate_file_download_regexes()

    # For all of a targets' outputs, check if any of the regexes used by tests

    # to download artifacts would match it. If any of them do then this target

    # is necessary.

    for artifact in self._get_target_potential_outputs(target):

      for regex in file_download_regexes:

        if re.match(regex, artifact):

          return True

    return False



  def _get_target_potential_outputs(self, target: str) -> set[str]:

    tests_suffix = '-tests'

    if target.endswith('tests'):

      tests_suffix = ''

    # This is a list of all the potential zips output by the test suite targets.

    # If the test downloads artifacts from any of these zips, we will be

    # conservative and avoid skipping the tests.

    return {

        f'{target}.zip',

        f'android-{target}.zip',

        f'android-{target}-verifier.zip',

        f'{target}{tests_suffix}_list.zip',

        f'android-{target}{tests_suffix}_list.zip',

        f'{target}{tests_suffix}_host-shared-libs.zip',

        f'android-{target}{tests_suffix}_host-shared-libs.zip',

        f'{target}{tests_suffix}_configs.zip',

        f'android-{target}{tests_suffix}_configs.zip',

    }



  def _aggregate_file_download_regexes(self) -> set[re.Pattern]:

    """Lists out all test config options to specify targets to download.



    These come in the form of regexes.

    """

    all_regexes = set()

    for test_info in self._get_test_infos():

      for opt in test_info.get('extraOptions', []):

        # check the known list of options for downloading files.

        if opt.get('key') in self._DOWNLOAD_OPTS:

          all_regexes.update(

              re.compile(value) for value in opt.get('values', [])

          )

    return all_regexes



  def _get_test_infos(self):

    return self.build_context.get('testContext', dict()).get('testInfos', [])







@dataclass(frozen=True)

@dataclass(frozen=True)

class BuildPlan:

class BuildPlan:
@@ -154,7 +225,7 @@ def load_build_context(): 








def empty_build_context():

def empty_build_context():

  return {'enabled_build_features': []}

  return {'enabledBuildFeatures': []}









def execute_build_plan(build_plan: BuildPlan):

def execute_build_plan(build_plan: BuildPlan):

ci/build_test_suites_test.py

+71 −3
Original line number Original line Diff line number Diff line
@@ -324,6 +324,55 @@ class BuildPlannerTest(unittest.TestCase): 
        self.run_packaging_functions(build_plan),

        self.run_packaging_functions(build_plan),

    )

    )





  def test_target_output_used_target_built(self):

    build_target = 'test_target'

    build_planner = self.create_build_planner(

        build_targets={build_target},

        build_context=self.create_build_context(

            test_context=self.get_test_context(build_target),

            enabled_build_features={'test_target_unused_exclusion'},

        ),

    )



    build_plan = build_planner.create_build_plan()



    self.assertSetEqual(build_plan.build_targets, {build_target})



  def test_target_regex_used_target_built(self):

    build_target = 'test_target'

    test_context = self.get_test_context(build_target)

    test_context['testInfos'][0]['extraOptions'] = [{

        'key': 'additional-files-filter',

        'values': [f'.*{build_target}.*\.zip'],

    }]

    build_planner = self.create_build_planner(

        build_targets={build_target},

        build_context=self.create_build_context(

            test_context=test_context,

            enabled_build_features={'test_target_unused_exclusion'},

        ),

    )



    build_plan = build_planner.create_build_plan()



    self.assertSetEqual(build_plan.build_targets, {build_target})



  def test_target_output_not_used_target_not_built(self):

    build_target = 'test_target'

    test_context = self.get_test_context(build_target)

    test_context['testInfos'][0]['extraOptions'] = []

    build_planner = self.create_build_planner(

        build_targets={build_target},

        build_context=self.create_build_context(

            test_context=test_context,

            enabled_build_features={'test_target_unused_exclusion'},

        ),

    )



    build_plan = build_planner.create_build_plan()



    self.assertSetEqual(build_plan.build_targets, set())



  def create_build_planner(

  def create_build_planner(

      self,

      self,

      build_targets: set[str],

      build_targets: set[str],
@@ -352,10 +401,10 @@ class BuildPlannerTest(unittest.TestCase): 
      test_context: dict[str, any] = {},

      test_context: dict[str, any] = {},

  ) -> dict[str, any]:

  ) -> dict[str, any]:

    build_context = {}

    build_context = {}

    build_context['enabled_build_features'] = enabled_build_features

    build_context['enabledBuildFeatures'] = enabled_build_features

    if optimized_build_enabled:

    if optimized_build_enabled:

      build_context['enabled_build_features'].add('optimized_build')

      build_context['enabledBuildFeatures'].add('optimized_build')

    build_context['test_context'] = test_context

    build_context['testContext'] = test_context

    return build_context

    return build_context





  def create_args(

  def create_args(
@@ -398,6 +447,25 @@ class BuildPlannerTest(unittest.TestCase): 




    return output

    return output





  def get_test_context(self, target: str):

    return {

        'testInfos': [

            {

                'name': 'atp_test',

                'target': 'test_target',

                'branch': 'branch',

                'extraOptions': [{

                    'key': 'additional-files-filter',

                    'values': [f'{target}.zip'],

                }],

                'command': '/tf/command',

                'extraBuildTargets': [

                    'extra_build_target',

                ],

            },

        ],

    }







def wait_until(

def wait_until(

    condition_function: Callable[[], bool],

    condition_function: Callable[[], bool],

ci/optimized_targets.py

+1 −1
Original line number Original line Diff line number Diff line
@@ -53,7 +53,7 @@ class NullOptimizer(OptimizedBuildTarget): 








def get_target_optimizer(target, enabled_flag, build_context, optimizer):

def get_target_optimizer(target, enabled_flag, build_context, optimizer):

  if enabled_flag in build_context['enabled_build_features']:

  if enabled_flag in build_context['enabledBuildFeatures']:

    return optimizer

    return optimizer





  return NullOptimizer(target)

  return NullOptimizer(target)