Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 368af498 authored by Jack He's avatar Jack He
Browse files

Cert: generate coverage report in teardown_class 

* Generate coverage report in teardown class to same time
* Merge coverage data after each test case
* But only generate coverage report after running all tests in a class
* It takes about 1 second to merge coverage data and 2 seconds to
  generate test reports. Hence this CL would roughly save 2 seconds for
  each test case
* This CL assumes that all test cases in the same test class use the
  same underlying stack binary
* An assert is added to make sure this is true

Bug: 194826691
Test: gd/cert/run
Tag: #gd-refactor
Change-Id: I108c580ea8755fa8a161bef1ddda90a49bc7a430
parent 332bec32

system/blueberry/tests/gd/cert/gd_base_test.py

+27 −1
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ from cert.gd_base_test_lib import teardown_rootcanal 
from cert.gd_base_test_lib import setup_test_core

from cert.gd_base_test_lib import teardown_test_core

from cert.gd_base_test_lib import dump_crashes_core

from cert.gd_device_lib import generate_coverage_report_for_host



from blueberry.tests.gd.cert.context import get_current_context

from blueberry.tests.gd.cert.gd_device import MOBLY_CONTROLLER_CONFIG_NAME as CONTROLLER_CONFIG_NAME
@@ -43,9 +44,18 @@ class GdBaseTestClass(base_test.BaseTestClass): 
        self.dut_module = dut_module

        self.cert_module = cert_module

        self.log = TraceLogger(logging.getLogger())

        self.dut_coverage_info = None

        self.cert_coverage_info = None



    def teardown_class(self):

        pass

        # assume each test runs the same binary for dut and cert

        # generate coverage report after running all tests in a class

        if self.dut_coverage_info:

            generate_coverage_report_for_host(self.dut_coverage_info)

            self.dut_coverage_info = None

        if self.cert_coverage_info:

            generate_coverage_report_for_host(self.cert_coverage_info)

            self.cert_coverage_info = None



    def setup_test(self):

        self.log_path_base = get_current_context().get_full_output_path()
@@ -82,6 +92,22 @@ class GdBaseTestClass(base_test.BaseTestClass): 
        self.register_controller(importlib.import_module('blueberry.tests.gd.cert.gd_device'), builtin=True)

        self.dut = self.gd_device[1]

        self.cert = self.gd_device[0]

        if self.dut.host_only_device:

            new_dut_coverage_info = self.dut.get_coverage_info()

            if self.dut_coverage_info:

                asserts.assert_true(

                    self.dut_coverage_info == new_dut_coverage_info,

                    msg="DUT coverage info must be the same for each test run, old: {}, new: {}".format(

                        self.dut_coverage_info, new_dut_coverage_info))

            self.dut_coverage_info = new_dut_coverage_info

        if self.cert.host_only_device:

            new_cert_coverage_info = self.cert.get_coverage_info()

            if self.cert_coverage_info:

                asserts.assert_true(

                    self.cert_coverage_info == new_cert_coverage_info,

                    msg="CERT coverage info must be the same for each test run, old: {}, new: {}".format(

                        self.cert_coverage_info, new_cert_coverage_info))

            self.cert_coverage_info = new_cert_coverage_info



        setup_test_core(dut=self.dut, cert=self.cert, dut_module=self.dut_module, cert_module=self.cert_module)

system/blueberry/tests/gd/cert/gd_device.py

+18 −5
Original line number Diff line number Diff line
@@ -35,7 +35,8 @@ from cert.gd_device_lib import destroy_core 
from cert.gd_device_lib import get_info

from cert.gd_device_lib import replace_vars

from cert.gd_device_lib import GdDeviceBaseCore

from cert.gd_device_lib import GdHostOnlyDeviceCore

from cert.gd_device_lib import get_coverage_profdata_path_for_host

from cert.gd_device_lib import merge_coverage_profdata_for_host

from cert.gd_device_lib import MOBLY_CONTROLLER_CONFIG_NAME

from cert.logging_client_interceptor import LoggingClientInterceptor

from cert.os_utils import get_gd_root
@@ -167,6 +168,7 @@ class GdHostOnlyDevice(GdDeviceBase): 
                 type_identifier: str, name: str, verbose_mode: bool):

        super().__init__(grpc_port, grpc_root_server_port, signal_port, cmd, label, MOBLY_CONTROLLER_CONFIG_NAME, name,

                         verbose_mode)

        self.host_only_device = True

        # Enable LLVM code coverage output for host only tests

        self.backing_process_profraw_path = pathlib.Path(self.log_path_base).joinpath(

            "%s_%s_backing_coverage.profraw" % (self.type_identifier, self.label))
@@ -177,14 +179,25 @@ class GdHostOnlyDevice(GdDeviceBase): 
            self.environment["ASAN_SYMBOLIZER_PATH"] = llvm_symbolizer

        else:

            logging.warning("[%s] Cannot find LLVM symbolizer at %s" % (self.label, str(llvm_symbolizer)))

        self.profdata_path = get_coverage_profdata_path_for_host(self.test_runner_base_path, self.type_identifier,

                                                                 self.label)



    def teardown(self):

        super().teardown()

        self.generate_coverage_report()

        merge_coverage_profdata_for_host(self.backing_process_profraw_path, self.profdata_path, self.label)



    def generate_coverage_report(self):

        GdHostOnlyDeviceCore.generate_coverage_report(self, self.backing_process_profraw_path, self.label,

                                                      self.test_runner_base_path, self.type_identifier, self.cmd)

    def get_coverage_info(self):

        """

        Get information needed for coverage reporting

        :return: a dictionary with all information needed for coverage reporting

        """

        return {

            "profdata_path": self.profdata_path,

            "label": self.label,

            "test_runner_base_path": self.test_runner_base_path,

            "type_identifier": self.type_identifier,

            "stack_bin": self.cmd[0]

        }



    def setup(self):

        # Ensure ports are available

system/gd/cert/gd_base_test.py

+27 −1
Original line number Diff line number Diff line
@@ -41,6 +41,7 @@ from cert.gd_base_test_lib import teardown_rootcanal 
from cert.gd_base_test_lib import setup_test_core

from cert.gd_base_test_lib import teardown_test_core

from cert.gd_base_test_lib import dump_crashes_core

from cert.gd_device_lib import generate_coverage_report_for_host





class GdBaseTestClass(BaseTestClass):
@@ -50,9 +51,18 @@ class GdBaseTestClass(BaseTestClass): 
    def setup_class(self, dut_module, cert_module):

        self.dut_module = dut_module

        self.cert_module = cert_module

        self.dut_coverage_info = None

        self.cert_coverage_info = None



    def teardown_class(self):

        pass

        # assume each test runs the same binary for dut and cert

        # generate coverage report after running all tests in a class

        if self.dut_coverage_info:

            generate_coverage_report_for_host(self.dut_coverage_info)

            self.dut_coverage_info = None

        if self.cert_coverage_info:

            generate_coverage_report_for_host(self.cert_coverage_info)

            self.cert_coverage_info = None



    def setup_test(self):

        self.log_path_base = get_current_context().get_full_output_path()
@@ -89,6 +99,22 @@ class GdBaseTestClass(BaseTestClass): 
        self.register_controller(importlib.import_module('cert.gd_device'), builtin=True)

        self.dut = self.gd_devices[1]

        self.cert = self.gd_devices[0]

        if self.dut.host_only_device:

            new_dut_coverage_info = self.dut.get_coverage_info()

            if self.dut_coverage_info:

                asserts.assert_true(

                    self.dut_coverage_info == new_dut_coverage_info,

                    msg="DUT coverage info must be the same for each test run, old: {}, new: {}".format(

                        self.dut_coverage_info, new_dut_coverage_info))

            self.dut_coverage_info = new_dut_coverage_info

        if self.cert.host_only_device:

            new_cert_coverage_info = self.cert.get_coverage_info()

            if self.cert_coverage_info:

                asserts.assert_true(

                    self.cert_coverage_info == new_cert_coverage_info,

                    msg="CERT coverage info must be the same for each test run, old: {}, new: {}".format(

                        self.cert_coverage_info, new_cert_coverage_info))

            self.cert_coverage_info = new_cert_coverage_info



        setup_test_core(dut=self.dut, cert=self.cert, dut_module=self.dut_module, cert_module=self.cert_module)

system/gd/cert/gd_device.py

+18 −5
Original line number Diff line number Diff line
@@ -42,7 +42,8 @@ from cert.gd_device_lib import destroy_core 
from cert.gd_device_lib import get_info

from cert.gd_device_lib import replace_vars

from cert.gd_device_lib import GdDeviceBaseCore

from cert.gd_device_lib import GdHostOnlyDeviceCore

from cert.gd_device_lib import get_coverage_profdata_path_for_host

from cert.gd_device_lib import merge_coverage_profdata_for_host

from cert.gd_device_lib import MOBLY_CONTROLLER_CONFIG_NAME

from cert.gd_device_lib import ACTS_CONTROLLER_REFERENCE_NAME

from cert.logging_client_interceptor import LoggingClientInterceptor
@@ -170,6 +171,7 @@ class GdHostOnlyDevice(GdDeviceBase): 
                 type_identifier: str, name: str, verbose_mode: bool):

        super().__init__(grpc_port, grpc_root_server_port, signal_port, cmd, label, MOBLY_CONTROLLER_CONFIG_NAME, name,

                         verbose_mode)

        self.host_only_device = True

        # Enable LLVM code coverage output for host only tests

        self.backing_process_profraw_path = pathlib.Path(self.log_path_base).joinpath(

            "%s_%s_backing_coverage.profraw" % (self.type_identifier, self.label))
@@ -180,14 +182,25 @@ class GdHostOnlyDevice(GdDeviceBase): 
            self.environment["ASAN_SYMBOLIZER_PATH"] = llvm_symbolizer

        else:

            logging.warning("[%s] Cannot find LLVM symbolizer at %s" % (self.label, str(llvm_symbolizer)))

        self.profdata_path = get_coverage_profdata_path_for_host(self.test_runner_base_path, self.type_identifier,

                                                                 self.label)



    def teardown(self):

        super().teardown()

        self.generate_coverage_report()

        merge_coverage_profdata_for_host(self.backing_process_profraw_path, self.profdata_path, self.label)



    def generate_coverage_report(self):

        GdHostOnlyDeviceCore.generate_coverage_report(self, self.backing_process_profraw_path, self.label,

                                                      self.test_runner_base_path, self.type_identifier, self.cmd)

    def get_coverage_info(self):

        """

        Get information needed for coverage reporting

        :return: a dictionary with all information needed for coverage reporting

        """

        return {

            "profdata_path": self.profdata_path,

            "label": self.label,

            "test_runner_base_path": self.test_runner_base_path,

            "type_identifier": self.type_identifier,

            "stack_bin": self.cmd[0]

        }



    def setup(self):

        # Ensure ports are available

system/gd/cert/gd_device_lib.py

+79 −85
Original line number Diff line number Diff line
@@ -116,6 +116,7 @@ class GdDeviceBaseCore(ABC): 
        :param test_runner_base_path: path to test run logs

        """

        self.verbose_mode = verbose_mode

        self.host_only_device = False

        self.grpc_root_server_port = int(grpc_root_server_port)

        self.grpc_port = int(grpc_port)

        self.signal_port = int(signal_port)
@@ -246,92 +247,85 @@ class GdDeviceBaseCore(ABC): 
            raise





class GdHostOnlyDeviceCore():

    """

    Provide core methods of GdHostOnlyDevice class unbound of ACTS dependency

    """

def get_coverage_profdata_path_for_host(test_runner_base_path, type_identifier, label) -> pathlib.Path:

    return pathlib.Path(test_runner_base_path).joinpath(

        "%s_%s_backing_process_coverage.profdata" % (type_identifier, label))



    def generate_coverage_report(self, backing_process_profraw_path, label, test_runner_base_path, type_identifier,

                                 cmd):

        self.backing_process_profraw_path = backing_process_profraw_path

        self.label = label

        self.test_runner_base_path = test_runner_base_path

        self.type_identifier = type_identifier

        self.cmd = cmd



        if not self.backing_process_profraw_path.is_file():

            logging.info("[%s] Skip coverage report as there is no profraw file at %s" %

                         (self.label, str(self.backing_process_profraw_path)))

def merge_coverage_profdata_for_host(backing_process_profraw_path, profdata_path: pathlib.Path, label):

    if not backing_process_profraw_path.is_file():

        logging.info(

            "[%s] Skip coverage report as there is no profraw file at %s" % (label, str(backing_process_profraw_path)))

        return

    try:

            if self.backing_process_profraw_path.stat().st_size <= 0:

                logging.info("[%s] Skip coverage report as profraw file is empty at %s" %

                             (self.label, str(self.backing_process_profraw_path)))

        if backing_process_profraw_path.stat().st_size <= 0:

            logging.info(

                "[%s] Skip coverage report as profraw file is empty at %s" % (label, str(backing_process_profraw_path)))

            return

    except OSError:

        logging.info("[%s] Skip coverage report as profraw file is inaccessible at %s" %

                         (self.label, str(self.backing_process_profraw_path)))

                     (label, str(backing_process_profraw_path)))

        return

    llvm_binutils = pathlib.Path(get_gd_root()).joinpath("llvm_binutils").joinpath("bin")

    llvm_profdata = llvm_binutils.joinpath("llvm-profdata")

    if not llvm_profdata.is_file():

            logging.info(

                "[%s] Skip coverage report as llvm-profdata is not found at %s" % (self.label, str(llvm_profdata)))

        logging.info("[%s] Skip coverage report as llvm-profdata is not found at %s" % (label, str(llvm_profdata)))

        return

        llvm_cov = llvm_binutils.joinpath("llvm-cov")

        if not llvm_cov.is_file():

            logging.info("[%s] Skip coverage report as llvm-cov is not found at %s" % (self.label, str(llvm_cov)))

            return

        logging.info("[%s] Generating coverage report" % self.label)

        profdata_path = pathlib.Path(self.test_runner_base_path).joinpath(

            "%s_%s_backing_process_coverage.profdata" % (self.type_identifier, self.label))

        profdata_path_tmp = pathlib.Path(self.test_runner_base_path).joinpath(

            "%s_%s_backing_process_coverage_tmp.profdata" % (self.type_identifier, self.label))

    logging.info("[%s] Merging coverage profdata" % label)

    profdata_path_tmp = profdata_path.parent / (profdata_path.stem + "_tmp." + profdata_path.suffix)

    # Merge with existing profdata if possible

        profdata_cmd = [str(llvm_profdata), "merge", "-sparse", str(self.backing_process_profraw_path)]

    profdata_cmd = [str(llvm_profdata), "merge", "-sparse", str(backing_process_profraw_path)]

    if profdata_path.is_file():

        profdata_cmd.append(str(profdata_path))

    profdata_cmd += ["-o", str(profdata_path_tmp)]

    logging.debug("Running llvm_profdata: %s" % " ".join(profdata_cmd))

    result = subprocess.run(profdata_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

    if result.returncode != 0:

            logging.warning("[%s] Failed to index profdata, cmd result: %r" % (self.label, result))

        logging.warning("[%s] Failed to index profdata, cmd result: %r" % (label, result))

        profdata_path.unlink(missing_ok=True)

        return

    shutil.move(profdata_path_tmp, profdata_path)

        coverage_result_path = pathlib.Path(self.test_runner_base_path).joinpath(

            "%s_%s_backing_process_coverage.json" % (self.type_identifier, self.label))





def generate_coverage_report_for_host(coverage_info):

    label = coverage_info["label"]

    test_runner_base_path = coverage_info["test_runner_base_path"]

    type_identifier = coverage_info["type_identifier"]

    profdata_path = coverage_info["profdata_path"]

    stack_bin = coverage_info["stack_bin"]

    llvm_binutils = pathlib.Path(get_gd_root()).joinpath("llvm_binutils").joinpath("bin")

    llvm_cov = llvm_binutils.joinpath("llvm-cov")

    if not llvm_cov.is_file():

        logging.info("[%s] Skip coverage report as llvm-cov is not found at %s" % (label, str(llvm_cov)))

        return

    logging.info("[%s] Generating coverage report in JSON" % label)

    coverage_result_path = pathlib.Path(test_runner_base_path).joinpath(

        "%s_%s_backing_process_coverage.json" % (type_identifier, label))

    with coverage_result_path.open("w") as coverage_result_file:

        llvm_cov_export_cmd = [

                str(llvm_cov), "export", "--format=text", "--ignore-filename-regex", "(external|out).*",

                "--instr-profile",

            str(llvm_cov), "export", "--format=text", "--ignore-filename-regex", "(external|out).*", "--instr-profile",

            str(profdata_path),

                str(self.cmd[0])

            str(stack_bin)

        ]

        logging.debug("Running llvm_cov export: %s" % " ".join(llvm_cov_export_cmd))

        result = subprocess.run(

                llvm_cov_export_cmd,

                stderr=subprocess.PIPE,

                stdout=coverage_result_file,

                cwd=os.path.join(get_gd_root()))

            llvm_cov_export_cmd, stderr=subprocess.PIPE, stdout=coverage_result_file, cwd=os.path.join(get_gd_root()))

    if result.returncode != 0:

            logging.warning("[%s] Failed to generated coverage report, cmd result: %r" % (self.label, result))

        logging.warning("[%s] Failed to generated coverage report, cmd result: %r" % (label, result))

        coverage_result_path.unlink(missing_ok=True)

        return

        coverage_summary_path = pathlib.Path(self.test_runner_base_path).joinpath(

            "%s_%s_backing_process_coverage_summary.txt" % (self.type_identifier, self.label))

    logging.info("[%s] Generating coverage summary in text" % label)

    coverage_summary_path = pathlib.Path(test_runner_base_path).joinpath(

        "%s_%s_backing_process_coverage_summary.txt" % (type_identifier, label))

    with coverage_summary_path.open("w") as coverage_summary_file:

        llvm_cov_report_cmd = [

            str(llvm_cov), "report", "--ignore-filename-regex", "(external|out).*", "--instr-profile",

            str(profdata_path),

                str(self.cmd[0])

            str(stack_bin)

        ]

        logging.debug("Running llvm_cov report: %s" % " ".join(llvm_cov_report_cmd))

        result = subprocess.run(

                llvm_cov_report_cmd,

                stderr=subprocess.PIPE,

                stdout=coverage_summary_file,

                cwd=os.path.join(get_gd_root()))

            llvm_cov_report_cmd, stderr=subprocess.PIPE, stdout=coverage_summary_file, cwd=os.path.join(get_gd_root()))

    if result.returncode != 0:

            logging.warning("[%s] Failed to generated coverage summary, cmd result: %r" % (self.label, result))

        logging.warning("[%s] Failed to generated coverage summary, cmd result: %r" % (label, result))

        coverage_summary_path.unlink(missing_ok=True)