Merge "Reboot edit monitor when memory exhausted" into main (bdf5eff5) · Commits · e / os / android_build

tools/edit_monitor/daemon_manager.py

+12 −16

Original line number	Diff line number	Diff line
		@@ -145,10 +145,20 @@ class DaemonManager:
		logging.warning("Failed to monitor daemon process with error: %s", e)

		if self.max_memory_usage >= memory_threshold:
		self._handle_resource_exhausted_error("memory")
		self._send_error_event_to_clearcut(
		edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE
		)
		logging.error(
		"Daemon process is consuming too much memory, rebooting...")
		self.reboot()

		if self.max_cpu_usage >= cpu_threshold:
		self._handle_resource_exhausted_error("cpu")
		self._send_error_event_to_clearcut(
		edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE
		)
		logging.error(
		"Daemon process is consuming too much cpu, killing...")
		self._terminate_process(self.daemon_process.pid)

		logging.info(
		"Daemon process %d terminated. Max memory usage: %f, Max cpu"
		@@ -395,20 +405,6 @@ class DaemonManager:

		return pids

		def _handle_resource_exhausted_error(self, resource_type:str):
		if resource_type == "memory":
		self._send_error_event_to_clearcut(
		edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE
		)
		else:
		self._send_error_event_to_clearcut(
		edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_CPU_USAGE
		)
		logging.error(
		"Daemon process is consuming too much %s, killing...", resource_type
		),
		self._terminate_process(self.daemon_process.pid)

		def _send_error_event_to_clearcut(self, error_type):
		edit_monitor_error_event_proto = edit_event_pb2.EditEvent(
		user_name=self.user_name,

+7 −2

Original line number	Diff line number	Diff line
		@@ -202,11 +202,15 @@ class DaemonManagerTest(unittest.TestCase):
		fake_cclient, edit_event_pb2.EditEvent.FAILED_TO_START_EDIT_MONITOR
		)

		def test_monitor_daemon_subprocess_killed_high_memory_usage(self):
		@mock.patch('os.execv')
		def test_monitor_reboot_with_high_memory_usage(self, mock_execv):
		fake_cclient = FakeClearcutClient()
		binary_file = tempfile.NamedTemporaryFile(
		dir=self.working_dir.name, delete=False
		)

		dm = daemon_manager.DaemonManager(
		TEST_BINARY_FILE,
		binary_file.name,
		daemon_target=memory_consume_daemon_target,
		daemon_args=(2,),
		cclient=fake_cclient,
		@@ -222,6 +226,7 @@ class DaemonManagerTest(unittest.TestCase):
		fake_cclient,
		edit_event_pb2.EditEvent.KILLED_DUE_TO_EXCEEDED_MEMORY_USAGE,
		)
		mock_execv.assert_called_once()

		def test_monitor_daemon_subprocess_killed_high_cpu_usage(self):
		fake_cclient = FakeClearcutClient()