Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ebd72a96 authored by Jack He's avatar Jack He
Browse files

Cert: Free resources occupying required ports during setup 

* For ports that are supposed to be listening on host, kill
  processes that occupy these ports using psutil
* For ports that are supposed to be adb and maybe ssh forward
  from Android device to host machine:
  1. Try to cancel any existing port forwarding or reversal
     for those ports
  2. If port portwarding or reversal still fails, try rebooting
     Android once and try again before failing the test

Bug: 153274925
Test: gd/cert/run --host
Change-Id: I28b430366f4a241ce07a79155f6260db1e8b57f9
parent 1ac34f73

system/gd/cert/gd_base_test.py

+28 −7
Original line number Diff line number Diff line
@@ -23,7 +23,10 @@ import subprocess 
from acts import asserts

from acts.context import get_current_context

from acts.base_test import BaseTestClass

from cert.os_utils import get_gd_root, is_subprocess_alive



from cert.os_utils import get_gd_root

from cert.os_utils import is_subprocess_alive

from cert.os_utils import make_ports_available

from facade import rootservice_pb2 as facade_rootservice
@@ -40,18 +43,35 @@ class GdBaseTestClass(BaseTestClass): 
        self.rootcanal_running = False

        if 'rootcanal' in self.controller_configs:

            self.rootcanal_running = True

            # Get root canal binary

            rootcanal = os.path.join(get_gd_root(), "root-canal")

            asserts.assert_true(

                os.path.isfile(rootcanal),

                "Root canal does not exist at %s" % rootcanal)



            # Get root canal log

            rootcanal_logpath = os.path.join(self.log_path_base,

                                             'rootcanal_logs.txt')

            self.rootcanal_logs = open(rootcanal_logpath, 'w')



            # Make sure ports are available

            rootcanal_config = self.controller_configs['rootcanal']

            rootcanal_hci_port = str(rootcanal_config.get("hci_port", "6402"))

            rootcanal = os.path.join(get_gd_root(), "root-canal")

            rootcanal_test_port = int(rootcanal_config.get("test_port", "6401"))

            rootcanal_hci_port = int(rootcanal_config.get("hci_port", "6402"))

            rootcanal_link_layer_port = int(

                rootcanal_config.get("link_layer_port", "6403"))

            asserts.assert_true(

                make_ports_available((rootcanal_test_port, rootcanal_hci_port,

                                      rootcanal_link_layer_port)),

                "Failed to make root canal ports available")



            # Start root canal process

            self.rootcanal_process = subprocess.Popen(

                [

                    rootcanal,

                    str(rootcanal_config.get("test_port", "6401")),

                    rootcanal_hci_port,

                    str(rootcanal_config.get("link_layer_port", "6403"))

                    str(rootcanal_test_port),

                    str(rootcanal_hci_port),

                    str(rootcanal_link_layer_port)

                ],

                cwd=get_gd_root(),

                env=os.environ.copy(),
@@ -63,9 +83,10 @@ class GdBaseTestClass(BaseTestClass): 
            asserts.assert_true(

                is_subprocess_alive(self.rootcanal_process),

                msg="root-canal stopped immediately after running")



            # Modify the device config to include the correct root-canal port

            for gd_device_config in self.controller_configs.get("GdDevice"):

                gd_device_config["rootcanal_port"] = rootcanal_hci_port

                gd_device_config["rootcanal_port"] = str(rootcanal_hci_port)



        # Parse and construct GD device objects

        self.register_controller(

system/gd/cert/gd_device.py

+44 −2
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ from google.protobuf import empty_pb2 as empty_proto 


from cert.os_utils import get_gd_root

from cert.os_utils import is_subprocess_alive

from cert.os_utils import make_ports_available

from facade import rootservice_pb2_grpc as facade_rootservice_pb2_grpc

from hal import facade_pb2_grpc as hal_facade_pb2_grpc

from hci.facade import facade_pb2_grpc as hci_facade_pb2_grpc
@@ -174,6 +175,11 @@ class GdDeviceBase(ABC): 
        - Should be executed after children classes' setup() methods

        :return:

        """

        # Ensure signal port is available

        # signal port is the only port that always listen on the host machine

        asserts.assert_true(

            make_ports_available([self.signal_port]),

            "[%s] Failed to make signal port available" % self.label)

        # Start backing process

        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as signal_socket:

            # Setup signaling socket
@@ -289,6 +295,16 @@ class GdHostOnlyDevice(GdDeviceBase): 
            self.log_path_base, "%s_%s_backing_coverage.profraw" %

            (self.type_identifier, self.label))



    def setup(self):

        # Ensure ports are available

        # Only check on host only test, for Android devices, these ports will

        # be opened on Android device and host machine ports will be occupied

        # by sshd or adb forwarding

        asserts.assert_true(

            make_ports_available((self.grpc_port, self.grpc_root_server_port)),

            "[%s] Failed to make backing process ports available" % self.label)

        super().setup()





class GdAndroidDevice(GdDeviceBase):

    """Real Android device where the backing process is running on it
@@ -313,10 +329,16 @@ class GdAndroidDevice(GdDeviceBase): 
            msg="device %s cannot run as root after enabling verity" %

            self.serial_number)

        self.adb.shell("date " + time.strftime("%m%d%H%M%Y.%S"))

        # Try freeing ports and ignore results

        self.adb.remove_tcp_forward(self.grpc_port)

        self.adb.remove_tcp_forward(self.grpc_root_server_port)

        self.adb.reverse("--remove tcp:%d" % self.signal_port)

        # Set up port forwarding or reverse or die

        self.tcp_forward_or_die(self.grpc_port, self.grpc_port)

        self.tcp_forward_or_die(self.grpc_root_server_port,

                                self.grpc_root_server_port)

        self.tcp_reverse_or_die(self.signal_port, self.signal_port)

        # Puh test binaries

        self.push_or_die(

            os.path.join(get_gd_root(), "target",

                         "bluetooth_stack_with_facade"), "system/bin")
@@ -374,11 +396,12 @@ class GdAndroidDevice(GdDeviceBase): 
                (src_file_path, dst_file_path, e),

                extras=e)



    def tcp_forward_or_die(self, host_port, device_port):

    def tcp_forward_or_die(self, host_port, device_port, num_retry=1):

        """

        Forward a TCP port from host to device or fail

        :param host_port: host port, int, 0 for adb to assign one

        :param device_port: device port, int

        :param num_retry: number of times to reboot and retry this before dying

        :return: host port int

        """

        error_or_port = self.adb.tcp_forward(host_port, device_port)
@@ -386,16 +409,26 @@ class GdAndroidDevice(GdDeviceBase): 
            logging.debug("host port %d was already forwarded" % host_port)

            return host_port

        if not isinstance(error_or_port, int):

            if num_retry > 0:

                # If requested, reboot an retry

                num_retry -= 1

                logging.warning("[%s] Failed to TCP forward host port %d to "

                                "device port %d, num_retries left is %d" %

                                (self.label, host_port, device_port, num_retry))

                self.reboot()

                return self.tcp_forward_or_die(

                    host_port, device_port, num_retry=num_retry)

            asserts.fail(

                'Unable to forward host port %d to device port %d, error %s' %

                (host_port, device_port, error_or_port))

        return error_or_port



    def tcp_reverse_or_die(self, device_port, host_port):

    def tcp_reverse_or_die(self, device_port, host_port, num_retry=1):

        """

        Forward a TCP port from device to host or fail

        :param device_port: device port, int, 0 for adb to assign one

        :param host_port: host port, int

        :param num_retry: number of times to reboot and retry this before dying

        :return: device port int

        """

        error_or_port = self.adb.reverse(
@@ -406,6 +439,15 @@ class GdAndroidDevice(GdDeviceBase): 
        try:

            error_or_port = int(error_or_port)

        except ValueError:

            if num_retry > 0:

                # If requested, reboot an retry

                num_retry -= 1

                logging.warning("[%s] Failed to TCP reverse device port %d to "

                                "host port %d, num_retries left is %d" %

                                (self.label, device_port, host_port, num_retry))

                self.reboot()

                return self.tcp_reverse_or_die(

                    device_port, host_port, num_retry=num_retry)

            asserts.fail(

                'Unable to reverse device port %d to host port %d, error %s' %

                (device_port, host_port, error_or_port))

system/gd/cert/os_utils.py

+41 −0
Original line number Diff line number Diff line
@@ -14,8 +14,11 @@ 
#   See the License for the specific language governing permissions and

#   limitations under the License.



import logging

from pathlib import Path

import psutil

import subprocess

from typing import Container





def is_subprocess_alive(process, timeout_seconds=1):
@@ -41,3 +44,41 @@ def get_gd_root(): 
    :return: root directory string of gd test library

    """

    return str(Path(__file__).absolute().parents[1])





def make_ports_available(ports: Container[int], timeout_seconds=10):

    """Make sure a list of ports are available

    kill occupying process if possible

    :param ports: list of target ports

    :param timeout_seconds: number of seconds to wait when killing processes

    :return: True on success, False on failure

    """

    if not ports:

        logging.warning("Empty ports is given to make_ports_available()")

        return True

    # Get connections whose state are in LISTEN only

    # Connections in other states won't affect binding as SO_REUSEADDR is used

    listening_conns_for_port = filter(

        lambda conn: (conn and conn.status == psutil.CONN_LISTEN and conn.laddr and conn.laddr.port in ports),

        psutil.net_connections())

    success = True

    for conn in listening_conns_for_port:

        logging.warning(

            "Freeing port %d used by %s" % (conn.laddr.port, str(conn)))

        if not conn.pid:

            logging.error(

                "Failed to kill process occupying port %d due to lack of pid" %

                conn.laddr.port)

            success = False

            continue

        logging.warning("Killing pid %d that is using port port %d" %

                        (conn.pid, conn.laddr.port))

        process = psutil.Process(conn.pid)

        process.kill()

        try:

            process.wait(timeout=timeout_seconds)

        except psutil.TimeoutExpired:

            logging.error("SIGKILL timeout after %d seconds for pid %d" %

                          (timeout_seconds, conn.pid))

            continue

    return success

system/gd/setup.py

+1 −0
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ import sys 


install_requires = [

    'grpcio',

    'psutil',

]



host_executables = [