Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 78fd076d authored by Zhuoyao Zhang's avatar Zhuoyao Zhang
Browse files

Reboot edit monitor when memory exhausted 

Instead of teminating, reboot the edit monitor when memory usage exceeds
the threshold

Test: atest daemon_manager_test
Bug: 379747487
Change-Id: If90309e720aa0d095bc1eca93521a01ed63575da
parent 6988272f

tools/edit_monitor/daemon_manager.py

+12 −16
Original line number Diff line number Diff line
@@ -145,10 +145,20 @@ class DaemonManager: 
        logging.warning("Failed to monitor daemon process with error: %s", e)



      if self.max_memory_usage >= memory_threshold:

        self._handle_resource_exhausted_error("memory")

        self._send_error_event_to_clearcut(

            edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE

        )

        logging.error(

            "Daemon process is consuming too much memory, rebooting...")

        self.reboot()



      if self.max_cpu_usage >= cpu_threshold:

        self._handle_resource_exhausted_error("cpu")

        self._send_error_event_to_clearcut(

            edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE

        )

        logging.error(

            "Daemon process is consuming too much cpu, killing...")

        self._terminate_process(self.daemon_process.pid)



    logging.info(

        "Daemon process %d terminated. Max memory usage: %f, Max cpu"
@@ -395,20 +405,6 @@ class DaemonManager: 


    return pids



  def _handle_resource_exhausted_error(self, resource_type:str):

    if resource_type == "memory":

      self._send_error_event_to_clearcut(

          edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE

      )

    else:

      self._send_error_event_to_clearcut(

          edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE

      )

    logging.error(

        "Daemon process is consuming too much %s, killing...", resource_type

    ),

    self._terminate_process(self.daemon_process.pid)



  def _send_error_event_to_clearcut(self, error_type):

    edit_monitor_error_event_proto = edit_event_pb2.EditEvent(

        user_name=self.user_name,

tools/edit_monitor/daemon_manager_test.py

+7 −2
Original line number Diff line number Diff line
@@ -202,11 +202,15 @@ class DaemonManagerTest(unittest.TestCase): 
        fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR

    )



  def test_monitor_daemon_subprocess_killed_high_memory_usage(self):

  @mock.patch('os.execv')

  def test_monitor_reboot_with_high_memory_usage(self, mock_execv):

    fake_cclient = FakeClearcutClient()

    binary_file = tempfile.NamedTemporaryFile(

        dir=self.working_dir.name, delete=False

    )



    dm = daemon_manager.DaemonManager(

        TEST_BINARY_FILE,

        binary_file.name,

        daemon_target=memory_consume_daemon_target,

        daemon_args=(2,),

        cclient=fake_cclient,
@@ -222,6 +226,7 @@ class DaemonManagerTest(unittest.TestCase): 
        fake_cclient,

        edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE,

    )

    mock_execv.assert_called_once()



  def test_monitor_daemon_subprocess_killed_high_cpu_usage(self):

    fake_cclient = FakeClearcutClient()