Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 585b4344 authored by Zhuoyao Zhang's avatar Zhuoyao Zhang
Browse files

Make edit monitor error type clearer 

Before this change, edit monitor will send a EXCEEDED_RESOURCE_USAGE
error to clearcut, however, it does not tell whether it is due to memory
or CPU usage overload. This CL separate the errory type to
EXCEEDED_MEMORY_USAGE and EXCEEDED_CPU_USAGE to make it easier to debug

Test: atest daemon_manager_test
Bug: 365617369
Change-Id: Ib0605249d87544c0a2ec82d72f05b2cc9aaa17c1
parent 4e9d7b3c

tools/edit_monitor/daemon_manager.py

+19 −11
Original line number Diff line number Diff line
@@ -141,17 +141,11 @@ class DaemonManager: 
        # Logging the error and continue.

        logging.warning("Failed to monitor daemon process with error: %s", e)



      if (

          self.max_memory_usage >= memory_threshold

          or self.max_cpu_usage >= cpu_threshold

      ):

        logging.error(

            "Daemon process is consuming too much resource, killing..."

        ),

        self._send_error_event_to_clearcut(

            edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE

        )

        self._terminate_process(self.daemon_process.pid)

      if self.max_memory_usage >= memory_threshold:

        self._handle_resource_exhausted_error("memory")



      if self.max_cpu_usage >= cpu_threshold:

        self._handle_resource_exhausted_error("cpu")



    logging.info(

        "Daemon process %d terminated. Max memory usage: %f, Max cpu"
@@ -392,6 +386,20 @@ class DaemonManager: 


    return pids



  def _handle_resource_exhausted_error(self, resource_type:str):

    if resource_type == "memory":

      self._send_error_event_to_clearcut(

          edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE

      )

    else:

      self._send_error_event_to_clearcut(

          edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE

      )

    logging.error(

        "Daemon process is consuming too much %s, killing...", resource_type

    ),

    self._terminate_process(self.daemon_process.pid)



  def _send_error_event_to_clearcut(self, error_type):

    edit_monitor_error_event_proto = edit_event_pb2.EditEvent(

        user_name=self.user_name,

tools/edit_monitor/daemon_manager_test.py

+2 −2
Original line number Diff line number Diff line
@@ -217,7 +217,7 @@ class DaemonManagerTest(unittest.TestCase): 
    self.assert_no_subprocess_running()

    self._assert_error_event_logged(

        fake_cclient,

        edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE,

        edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE,

    )



  def test_monitor_daemon_subprocess_killed_high_cpu_usage(self):
@@ -235,7 +235,7 @@ class DaemonManagerTest(unittest.TestCase): 
    self.assert_no_subprocess_running()

    self._assert_error_event_logged(

        fake_cclient,

        edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE,

        edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE,

    )



  @mock.patch('subprocess.check_output')

tools/edit_monitor/proto/edit_event.proto

+2 −1
Original line number Diff line number Diff line
@@ -16,8 +16,9 @@ message EditEvent { 
    FAILED_TO_START_EDIT_MONITOR = 1;

    FAILED_TO_STOP_EDIT_MONITOR = 2;

    FAILED_TO_REBOOT_EDIT_MONITOR = 3;

    KILLED_DUE_TO_EXCEEDED_RESOURCE_USAGE = 4;

    KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE = 4;

    FORCE_CLEANUP = 5;

    KILLED_DUE_TO_EXCEEDED_CPU_USAGE = 6;

  }



  // Event that logs a single edit