Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c09a173d authored by Zhuoyao Zhang's avatar Zhuoyao Zhang
Browse files

Edit monitor do not monitor the build out directory 

We want to monitor the edits under source tree, while the build out
directory is only used to store build outputs. During the build time,
there will be a lot of file changes under the out directory, although
the edit monitor will not send any of the file changes under the out
dir to clearcut, it needs to dispatch all the file change events
which consumes the CUP usage.

This cl will skip the build out directory when scheduling the watchdog
observer in edit monitor.

Test: atest edit_monitor_test
Bug: 381095876
Change-Id: I6759d5f7c7fd37c153bcb1fff29bd8fc37581774
parent 7e2362bb

tools/edit_monitor/edit_monitor.py

+18 −4
Original line number Diff line number Diff line
@@ -46,7 +46,6 @@ class ClearcutEventHandler(PatternMatchingEventHandler): 
      is_dry_run: bool = False,

      cclient: clearcut_client.Clearcut | None = None,

  ):



    super().__init__(patterns=["*"], ignore_directories=True)

    self.root_monitoring_path = path

    self.flush_interval_sec = flush_interval_sec
@@ -199,11 +198,26 @@ def start( 
    conn: the sender of the pipe to communicate with the deamon manager.

  """

  event_handler = ClearcutEventHandler(

      path, flush_interval_sec, single_events_size_threshold, is_dry_run, cclient)

      path,

      flush_interval_sec,

      single_events_size_threshold,

      is_dry_run,

      cclient,

  )

  observer = Observer()



  logging.info("Starting observer on path %s.", path)

  observer.schedule(event_handler, path, recursive=True)

  out_dir = os.environ.get("OUT_DIR", "out")

  sub_dirs = [

      os.path.join(path, name)

      for name in os.listdir(path)

      if name != out_dir

      and not name.startswith(".")

      and os.path.isdir(os.path.join(path, name))

  ]

  for sub_dir_name in sub_dirs:

      logging.info("Starting observer on path %s.", sub_dir_name)

      observer.schedule(event_handler, sub_dir_name, recursive=True)



  observer.start()

  logging.info("Observer started.")

  if pipe_sender:

tools/edit_monitor/edit_monitor_integration_test.py

+6 −4
Original line number Diff line number Diff line
@@ -58,18 +58,20 @@ class EditMonitorIntegrationTest(unittest.TestCase): 
    super().tearDown()



  def test_log_single_edit_event_success(self):

    p = self._start_edit_monitor_process()



    # Create the .git file under the monitoring dir.

    self.root_monitoring_path.joinpath(".git").touch()

    test_dir = self.root_monitoring_path.joinpath('test')

    test_dir.mkdir()



    p = self._start_edit_monitor_process()



    # Create and modify a file.

    test_file = self.root_monitoring_path.joinpath("test.txt")

    test_file = test_dir.joinpath("test.txt")

    with open(test_file, "w") as f:

      f.write("something")



    # Move the file.

    test_file_moved = self.root_monitoring_path.joinpath("new_test.txt")

    test_file_moved = test_dir.joinpath("new_test.txt")

    test_file.rename(test_file_moved)



    # Delete the file.

tools/edit_monitor/edit_monitor_test.py

+55 −22
Original line number Diff line number Diff line
@@ -56,17 +56,19 @@ class EditMonitorTest(unittest.TestCase): 
  def test_log_single_edit_event_success(self):

    # Create the .git file under the monitoring dir.

    self.root_monitoring_path.joinpath('.git').touch()

    test_dir = self.root_monitoring_path.joinpath('test')

    test_dir.mkdir()

    fake_cclient = FakeClearcutClient(

        log_output_file=self.log_event_dir.joinpath('logs.output')

    )

    p = self._start_test_edit_monitor_process(fake_cclient)



    # Create and modify a file.

    test_file = self.root_monitoring_path.joinpath('test.txt')

    test_file = test_dir.joinpath('test.txt')

    with open(test_file, 'w') as f:

      f.write('something')

    # Move the file.

    test_file_moved = self.root_monitoring_path.joinpath('new_test.txt')

    test_file_moved = test_dir.joinpath('new_test.txt')

    test_file.rename(test_file_moved)

    # Delete the file.

    test_file_moved.unlink()
@@ -79,27 +81,19 @@ class EditMonitorTest(unittest.TestCase): 
    logged_events = self._get_logged_events()

    self.assertEqual(len(logged_events), 4)

    expected_create_event = edit_event_pb2.EditEvent.SingleEditEvent(

        file_path=str(

            self.root_monitoring_path.joinpath('test.txt').resolve()

        ),

        file_path=str(test_dir.joinpath('test.txt').resolve()),

        edit_type=edit_event_pb2.EditEvent.CREATE,

    )

    expected_modify_event = edit_event_pb2.EditEvent.SingleEditEvent(

        file_path=str(

            self.root_monitoring_path.joinpath('test.txt').resolve()

        ),

        file_path=str(test_dir.joinpath('test.txt').resolve()),

        edit_type=edit_event_pb2.EditEvent.MODIFY,

    )

    expected_move_event = edit_event_pb2.EditEvent.SingleEditEvent(

        file_path=str(

            self.root_monitoring_path.joinpath('test.txt').resolve()

        ),

        file_path=str(test_dir.joinpath('test.txt').resolve()),

        edit_type=edit_event_pb2.EditEvent.MOVE,

    )

    expected_delete_event = edit_event_pb2.EditEvent.SingleEditEvent(

        file_path=str(

            self.root_monitoring_path.joinpath('new_test.txt').resolve()

        ),

        file_path=str(test_dir.joinpath('new_test.txt').resolve()),

        edit_type=edit_event_pb2.EditEvent.DELETE,

    )

    self.assertEqual(
@@ -127,10 +121,11 @@ class EditMonitorTest(unittest.TestCase): 
        ).single_edit_event,

    )





  def test_log_aggregated_edit_event_success(self):

    # Create the .git file under the monitoring dir.

    self.root_monitoring_path.joinpath('.git').touch()

    test_dir = self.root_monitoring_path.joinpath('test')

    test_dir.mkdir()

    fake_cclient = FakeClearcutClient(

        log_output_file=self.log_event_dir.joinpath('logs.output')

    )
@@ -138,7 +133,7 @@ class EditMonitorTest(unittest.TestCase): 


    # Create 6 test files

    for i in range(6):

      test_file = self.root_monitoring_path.joinpath('test_' + str(i))

      test_file = test_dir.joinpath('test_' + str(i))

      test_file.touch()



    # Give some time for the edit monitor to receive the edit event.
@@ -163,9 +158,34 @@ class EditMonitorTest(unittest.TestCase): 
        ).aggregated_edit_event,

    )



  def test_do_not_log_edit_event_under_out_dir(self):

    # Create the .git file under the monitoring dir.

    self.root_monitoring_path.joinpath('.git').touch()

    fake_cclient = FakeClearcutClient(

        log_output_file=self.log_event_dir.joinpath('logs.output')

    )

    p = self._start_test_edit_monitor_process(fake_cclient)



    # Create out directory

    self.root_monitoring_path.joinpath('out').mkdir()

    # Create a file under out directory

    test_file = self.root_monitoring_path.joinpath('out', 'test.txt')

    with open(test_file, 'w') as f:

      f.write('something')

    # Give some time for the edit monitor to receive the edit event.

    time.sleep(1)

    # Stop the edit monitor and flush all events.

    os.kill(p.pid, signal.SIGINT)

    p.join()



    logged_events = self._get_logged_events()

    self.assertEqual(len(logged_events), 0)



  def test_do_not_log_edit_event_for_directory_change(self):

    # Create the .git file under the monitoring dir.

    self.root_monitoring_path.joinpath('.git').touch()

    test_dir = self.root_monitoring_path.joinpath('test')

    test_dir.mkdir()

    fake_cclient = FakeClearcutClient(

        log_output_file=self.log_event_dir.joinpath('logs.output')

    )
@@ -185,15 +205,17 @@ class EditMonitorTest(unittest.TestCase): 
  def test_do_not_log_edit_event_for_hidden_file(self):

    # Create the .git file under the monitoring dir.

    self.root_monitoring_path.joinpath('.git').touch()

    test_dir = self.root_monitoring_path.joinpath('test')

    test_dir.mkdir()

    fake_cclient = FakeClearcutClient(

        log_output_file=self.log_event_dir.joinpath('logs.output')

    )

    p = self._start_test_edit_monitor_process(fake_cclient)



    # Create a hidden file.

    self.root_monitoring_path.joinpath('.test.txt').touch()

    test_dir.joinpath('.test.txt').touch()

    # Create a hidden dir.

    hidden_dir = self.root_monitoring_path.joinpath('.test')

    hidden_dir = test_dir.joinpath('.test')

    hidden_dir.mkdir()

    hidden_dir.joinpath('test.txt').touch()

    # Give some time for the edit monitor to receive the edit event.
@@ -206,15 +228,17 @@ class EditMonitorTest(unittest.TestCase): 
    self.assertEqual(len(logged_events), 0)



  def test_do_not_log_edit_event_for_non_git_project_file(self):

    test_dir = self.root_monitoring_path.joinpath('test')

    test_dir.mkdir()

    fake_cclient = FakeClearcutClient(

        log_output_file=self.log_event_dir.joinpath('logs.output')

    )

    p = self._start_test_edit_monitor_process(fake_cclient)



    # Create a file.

    self.root_monitoring_path.joinpath('test.txt').touch()

    test_dir.joinpath('test.txt').touch()

    # Create a file under a sub dir.

    sub_dir = self.root_monitoring_path.joinpath('.test')

    sub_dir = test_dir.joinpath('.test')

    sub_dir.mkdir()

    sub_dir.joinpath('test.txt').touch()

    # Give some time for the edit monitor to receive the edit event.
@@ -229,6 +253,8 @@ class EditMonitorTest(unittest.TestCase): 
  def test_log_edit_event_fail(self):

    # Create the .git file under the monitoring dir.

    self.root_monitoring_path.joinpath('.git').touch()

    test_dir = self.root_monitoring_path.joinpath('test')

    test_dir.mkdir()

    fake_cclient = FakeClearcutClient(

        log_output_file=self.log_event_dir.joinpath('logs.output'),

        raise_log_exception=True,
@@ -236,7 +262,7 @@ class EditMonitorTest(unittest.TestCase): 
    p = self._start_test_edit_monitor_process(fake_cclient)



    # Create a file.

    self.root_monitoring_path.joinpath('test.txt').touch()

    test_dir.joinpath('test.txt').touch()

    # Give some time for the edit monitor to receive the edit event.

    time.sleep(1)

    # Stop the edit monitor and flush all events.
@@ -253,7 +279,14 @@ class EditMonitorTest(unittest.TestCase): 
    # Start edit monitor in a subprocess.

    p = multiprocessing.Process(

        target=edit_monitor.start,

        args=(str(self.root_monitoring_path.resolve()), False, 0.5, 5, cclient, sender),

        args=(

            str(self.root_monitoring_path.resolve()),

            False,

            0.5,

            5,

            cclient,

            sender,

        ),

    )

    p.daemon = True

    p.start()