Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5f64e580 authored by Julien Desprez's avatar Julien Desprez Committed by Automerger Merge Worker
Browse files

Merge changes I6937dd0c,I203645ab into main am: 5f080131 am: 4e57299a 

Original change: https://android-review.googlesource.com/c/platform/build/+/3369439



Change-Id: I5ccfcc29113973a9c8e7d4a0a69e8bad36e0299a
Signed-off-by: default avatarAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
parents 1eabb056 4e57299a

ci/build_device_and_tests

+3 −2
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

set -euo pipefail



build/soong/soong_ui.bash --make-mode build_test_suites || exit $?

$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites $@ || exit $?
 
build/soong/soong_ui.bash --make-mode build_test_suites

$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites --device-build $@

ci/build_test_suites

+3 −2
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

set -euo pipefail



build/soong/soong_ui.bash --make-mode build_test_suites || exit $?

$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites $@ || exit $?
 
build/soong/soong_ui.bash --make-mode build_test_suites

$(build/soong/soong_ui.bash --dumpvar-mode HOST_OUT)/bin/build_test_suites $@

ci/build_test_suites.py

+86 −23
Original line number Diff line number Diff line
@@ -30,9 +30,10 @@ import metrics_agent 
import test_discovery_agent





REQUIRED_ENV_VARS = frozenset(['TARGET_PRODUCT', 'TARGET_RELEASE', 'TOP'])

REQUIRED_ENV_VARS = frozenset(['TARGET_PRODUCT', 'TARGET_RELEASE', 'TOP', 'DIST_DIR'])

SOONG_UI_EXE_REL_PATH = 'build/soong/soong_ui.bash'

LOG_PATH = 'logs/build_test_suites.log'

REQUIRED_BUILD_TARGETS = frozenset(['dist'])





class Error(Exception):
@@ -73,34 +74,46 @@ class BuildPlanner: 


    build_targets = set()

    packaging_commands_getters = []

    test_discovery_zip_regexes = set()

    # In order to roll optimizations out differently between test suites and

    # device builds, we have separate flags.

    if (

        'test_suites_zip_test_discovery'

        in self.build_context.enabled_build_features

        and not self.args.device_build

    ) or (

        'device_zip_test_discovery'

        in self.build_context.enabled_build_features

        and self.args.device_build

    ):

      preliminary_build_targets = self._collect_preliminary_build_targets()

    else:

      preliminary_build_targets = self._legacy_collect_preliminary_build_targets()



      # Keep reporting metrics when test discovery is disabled.

      # To be removed once test discovery is fully rolled out.

      optimization_rationale = ''

      test_discovery_zip_regexes = set()

      try:

      # Do not use these regexes for now, only run this to collect data on what

      # would be optimized.

        test_discovery_zip_regexes = self._get_test_discovery_zip_regexes()

        logging.info(f'Discovered test discovery regexes: {test_discovery_zip_regexes}')

      except test_discovery_agent.TestDiscoveryError as e:

        optimization_rationale = e.message

        logging.warning(f'Unable to perform test discovery: {optimization_rationale}')



      for target in self.args.extra_targets:

        if optimization_rationale:

          get_metrics_agent().report_unoptimized_target(target, optimization_rationale)

      else:

          continue

        try:

          regex = r'\b(%s)\b' % re.escape(target)

          regex = r'\b(%s.*)\b' % re.escape(target)

          if any(re.search(regex, opt) for opt in test_discovery_zip_regexes):

            get_metrics_agent().report_unoptimized_target(target, 'Test artifact used.')

          else:

            continue

          get_metrics_agent().report_optimized_target(target)

        except Exception as e:

          logging.error(f'unable to parse test discovery output: {repr(e)}')



      if self._unused_target_exclusion_enabled(

          target

      ) and not self.build_context.build_target_used(target):

        continue



    for target in preliminary_build_targets:

      target_optimizer_getter = self.target_optimizations.get(target, None)

      if not target_optimizer_getter:

        build_targets.add(target)
@@ -116,6 +129,51 @@ class BuildPlanner: 


    return BuildPlan(build_targets, packaging_commands_getters)



  def _collect_preliminary_build_targets(self):

    build_targets = set()

    try:

      test_discovery_zip_regexes = self._get_test_discovery_zip_regexes()

      logging.info(f'Discovered test discovery regexes: {test_discovery_zip_regexes}')

    except test_discovery_agent.TestDiscoveryError as e:

      optimization_rationale = e.message

      logging.warning(f'Unable to perform test discovery: {optimization_rationale}')



      for target in self.args.extra_targets:

        get_metrics_agent().report_unoptimized_target(target, optimization_rationale)

      return self._legacy_collect_preliminary_build_targets()



    for target in self.args.extra_targets:

      if target in REQUIRED_BUILD_TARGETS:

        build_targets.add(target)

        continue



      regex = r'\b(%s.*)\b' % re.escape(target)

      for opt in test_discovery_zip_regexes:

        try:

          if re.search(regex, opt):

            get_metrics_agent().report_unoptimized_target(target, 'Test artifact used.')

            build_targets.add(target)

            continue

          get_metrics_agent().report_optimized_target(target)

        except Exception as e:

          # In case of exception report as unoptimized

          build_targets.add(target)

          get_metrics_agent().report_unoptimized_target(target, f'Error in parsing test discovery output for {target}: {repr(e)}')

          logging.error(f'unable to parse test discovery output: {repr(e)}')



    return build_targets



  def _legacy_collect_preliminary_build_targets(self):

    build_targets = set()

    for target in self.args.extra_targets:

      if self._unused_target_exclusion_enabled(

          target

      ) and not self.build_context.build_target_used(target):

        continue



      build_targets.add(target)

    return build_targets



  def _unused_target_exclusion_enabled(self, target: str) -> bool:

    return (

        f'{target}_unused_exclusion'
@@ -197,6 +255,11 @@ def parse_args(argv: list[str]) -> argparse.Namespace: 
  argparser.add_argument(

      'extra_targets', nargs='*', help='Extra test suites to build.'

  )

  argparser.add_argument(

      '--device-build',

      action='store_true',

      help='Flag to indicate running a device build.',

  )



  return argparser.parse_args(argv)

ci/build_test_suites_test.py

+10 −0
Original line number Diff line number Diff line
@@ -78,6 +78,12 @@ class BuildTestSuitesTest(fake_filesystem_unittest.TestCase): 
    with self.assert_raises_word(build_test_suites.Error, 'TOP'):

      build_test_suites.main([])



  def test_missing_dist_dir_env_var_raises(self):

    del os.environ['DIST_DIR']



    with self.assert_raises_word(build_test_suites.Error, 'DIST_DIR'):

      build_test_suites.main([])



  def test_invalid_arg_raises(self):

    invalid_args = ['--invalid_arg']
@@ -114,6 +120,9 @@ class BuildTestSuitesTest(fake_filesystem_unittest.TestCase): 
    self.soong_ui_dir = self.fake_top.joinpath('build/soong')

    self.soong_ui_dir.mkdir(parents=True, exist_ok=True)



    self.logs_dir = self.fake_top.joinpath('dist/logs')

    self.logs_dir.mkdir(parents=True, exist_ok=True)



    self.soong_ui = self.soong_ui_dir.joinpath('soong_ui.bash')

    self.soong_ui.touch()
@@ -121,6 +130,7 @@ class BuildTestSuitesTest(fake_filesystem_unittest.TestCase): 
        'TARGET_RELEASE': 'release',

        'TARGET_PRODUCT': 'product',

        'TOP': str(self.fake_top),

        'DIST_DIR': str(self.fake_top.joinpath('dist')),

    })



    self.mock_subprocess_run.return_value = 0