Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e1f5b7b9 authored by John Wu's avatar John Wu
Browse files

Update toggle-test.py 

* Rename toggle_test.py to toggle-test.py
* Extract common functionality into ravenlib.py for future script usage
* Update test result CSV parser to be compatible with the new format
  that includes method-level results
* Fix a bug in RavenwoodTestStats.java
* Remove outdated script

Bug: 292141694
Flag: EXEMPT host side change only
Test: manually check script works
Change-Id: I2b4620652cfb07601cfe40d2b5573ebc87448c63
parent e804eaa6

ravenwood/junit-impl-src/android/platform/test/ravenwood/RavenwoodTestStats.java

+1 −1
Original line number Diff line number Diff line
@@ -303,7 +303,7 @@ public class RavenwoodTestStats { 
            var description = failure.getDescription();

            addResult(description.getClassName(),

                    description.getMethodName(),

                    Result.Passed,

                    Result.Failed,

                    "  testFailure: ",

                    failure);

        }

ravenwood/scripts/bulk_enable.py

deleted100644 → 0
+0 −77
Original line number Diff line number Diff line 
#!/usr/bin/env python3

#

# Copyright (C) 2024 The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#   http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



"""

Tool to bulk-enable tests that are now passing on Ravenwood.



Currently only offers to include classes which are fully passing; ignores

classes that have partial success.



Typical usage:

$ RAVENWOOD_RUN_DISABLED_TESTS=1 atest MyTestsRavenwood

$ cd /path/to/tests/root

$ python bulk_enable.py /path/to/atest/output/host_log.txt

"""



import collections

import os

import re

import subprocess

import sys



re_result = re.compile("I/ModuleListener.+?null-device-0 (.+?)#(.+?) ([A-Z_]+)(.*)$")



DRY_RUN = "-n" in sys.argv



ANNOTATION = "@android.platform.test.annotations.EnabledOnRavenwood"

SED_ARG = "s/^((public )?class )/%s\\n\\1/g" % (ANNOTATION)



STATE_PASSED = "PASSED"

STATE_FAILURE = "FAILURE"

STATE_ASSUMPTION_FAILURE = "ASSUMPTION_FAILURE"

STATE_CANDIDATE = "CANDIDATE"



stats_total = collections.defaultdict(int)

stats_class = collections.defaultdict(lambda: collections.defaultdict(int))

stats_method = collections.defaultdict()



with open(sys.argv[-1]) as f:

    for line in f.readlines():

        result = re_result.search(line)

        if result:

            clazz, method, state, msg = result.groups()

            if state == STATE_FAILURE and "actually passed under Ravenwood" in msg:

                state = STATE_CANDIDATE

            stats_total[state] += 1

            stats_class[clazz][state] += 1

            stats_method[(clazz, method)] = state



# Find classes who are fully "candidates" (would be entirely green if enabled)

num_enabled = 0

for clazz in stats_class.keys():

    stats = stats_class[clazz]

    if STATE_CANDIDATE in stats and len(stats) == 1:

        num_enabled += stats[STATE_CANDIDATE]

        print("Enabling fully-passing class", clazz)

        clazz_match = re.compile("%s\.(kt|java)" % (clazz.split(".")[-1]))

        for root, dirs, files in os.walk("."):

            for f in files:

                if clazz_match.match(f) and not DRY_RUN:

                    path = os.path.join(root, f)

                    subprocess.run(["sed", "-i", "-E", SED_ARG, path])



print("Overall stats", stats_total)

print("Candidates actually enabled", num_enabled)

ravenwood/scripts/toggle_test.py→ravenwood/scripts/ravenlib.py

100755 → 100644
+167 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3

#

# Copyright (C) 2025 The Android Open Source Project

#
@@ -14,52 +13,73 @@ 
# See the License for the specific language governing permissions and

# limitations under the License.



"""

Tool to bulk-toggle entire test classes for Ravenwood.



To enable all tests that passed:

$ RAVENWOOD_RUN_DISABLED_TESTS=1 atest SystemUiRavenTests

$ ./toggle_test.py enable /path/to/src /tmp/Ravenwood-stats_SystemUiRavenTestsRavenwood_latest.csv



To disable all tests that failed:

$ atest SystemUiRavenTests

$ ./toggle_test.py disable /path/to/src /tmp/Ravenwood-stats_SystemUiRavenTestsRavenwood_latest.csv

"""

"""Library for Ravenwood python scripts."""



import csv

import pathlib

import sys



import re

from typing import Callable



Path = pathlib.Path

ENABLED_ANNOTATION = "android.platform.test.annotations.EnabledOnRavenwood"

DISABLED_ANNOTATION = "android.platform.test.annotations.DisabledOnRavenwood"





class SourceFile:

  """A source file. We assume the filename's stem is the class name."""

  """A Java or Kotlin source file."""



  path: Path

  lines: list[str]

  class_name: str

  modified: bool = False



  def __init__(self, path: Path):

    self.path = path

    self.class_name = path.stem

    with open(path, "r") as f:

      self.lines = f.readlines()



  def find_annotation(self) -> (int, [(str, int)]):

    """

    Find the class level annotations in the file, and returns

    the indexes (index = line number - 1) of them.

  def get_package(self) -> str:

    """Returns the package name of the source file."""

    for line in self.lines:

      if line.startswith("package "):

        return line.split(" ", 1)[1].strip()

    return ""



  def get_class_idx(self, class_name: str) -> int:

    """Returns the index of the class in the source file."""

    simple_class_name = class_name.split(".")[-1]

    for idx, line in enumerate(self.lines):

      if f"class {simple_class_name}" in line:

        return idx

    return -1



  def list_classes(self) -> list[(str, int)]:

    """Finds the classes and their line numbers in the source file."""

    if self.path.name.endswith(".java"):

      pattern = re.compile(

          r"^(?:(?:public|protected|private|abstract|static|final)\s+)*"

          r"class\s+"

          r"(\w+)"

      )

    elif self.path.name.endswith(".kt"):

      pattern = re.compile(

          r"^(?:(?:public|protected|private|internal|data|open|abstract|sealed)\s+)*"

          r"class\s+"

          r"(\w+)"

      )

    else:

      return []



    results = []

    package = self.get_package()

    for idx, line in enumerate(self.lines):

      class_name = pattern.findall(line)

      if class_name:

        results.append((f"{package}.{class_name[0]}", idx))



    (We assume class level annotations don't have any indents.)

    """

    return results



  def list_annotations(self, class_idx: int) -> list[(str, int)]:

    """Finds the annotations and their line numbers in the source file."""

    result = []

    for idx in range(len(self.lines)):

      # Find the class line

      if f"class {self.class_name}" in self.lines[idx]:

        curr_idx = idx - 1

    curr_idx = class_idx - 1

    while True:

      line = self.lines[curr_idx].strip()

      if line.startswith("@"):
@@ -68,26 +88,31 @@ class SourceFile: 
        curr_idx -= 1

      else:

        break

        return (idx, result)

    return result



  def remove_annotation(self, annotation: str):

  def remove_annotation(self, class_name: str, annotation: str):

    """Removes an annotation (ignoring the package name) from the source file."""

    for (annot, idx) in self.find_annotation()[1]:

    class_idx = self.get_class_idx(class_name)

    for annot, idx in self.list_annotations(class_idx):

      if annot.split(".")[-1] == annotation.split(".")[-1]:

        self.lines.pop(idx)

        self.modified = True

        break



  def add_annotation(self, annotation: str):

  def add_annotation(self, class_name: str, annotation: str):

    """Adds an annotation to the source file, if it doesn't have it already."""

    (class_idx, annot_list) = self.find_annotation()

    for annot, _ in annot_list:

    class_idx = self.get_class_idx(class_name)

    for annot, _ in self.list_annotations(class_idx):

      if annot.split(".")[-1] == annotation.split(".")[-1]:

        # The annotation is already present.

        return

    self.lines.insert(class_idx, f"@{annotation}\n")

    self.modified = True



  def write(self):

    """Writes the source file to disk."""

    if not self.modified:

      return

    with open(self.path, "w") as f:

      f.writelines(self.lines)
@@ -97,72 +122,46 @@ class SourceFile: 
      print(line, end="")





def find_passed_tests(csv_file: str) -> list[str]:

  """Finds all test classes that passed from a test result CSV file."""

  test = []

  with open(csv_file) as f:

    reader = csv.DictReader(f)

    for row in reader:

      if int(row["Failed"]) == 0:

        test.append(row["Class"])

  return test

def load_source_files(src_root: str) -> list[SourceFile]:

  """Loads all the source files from the given root directory."""

  files = []

  for java in Path(src_root).glob("**/*.java"):

    files.append(SourceFile(java))

  for kt in Path(src_root).glob("**/*.kt"):

    files.append(SourceFile(kt))

  return files





def find_failed_tests(csv_file: str) -> list[str]:

  """Finds all test classes with at least one failure from a test result CSV file."""

def load_source_map(src_root: str) -> dict[str, SourceFile]:

  """Loads all the source files from the given root directory, and returns a map from class name to source file."""

  files = load_source_files(src_root)

  result = {}

  for src in files:

    for clazz, _ in src.list_classes():

      result[clazz] = src

  return result





def _find_tests(

    csv_file: str, select_func: Callable[[[str]], bool]

) -> list[str]:

  """Finds all test classes from a test result CSV file."""

  test = []

  with open(csv_file) as f:

    reader = csv.DictReader(f)

    for row in reader:

      if int(row["Failed"]) > 0:

      if row["ClassOrMethod"] == "c" and select_func(row):

        test.append(row["Class"])

  return test





def load_test_files(src_root: str, tests: list[str]) -> list[SourceFile]:

  """

  Find all source files of the given list of classes e.g. ["com.android.mytest.MyTestClass", ...]

  in a root directory, and load each of as SourceFile and return a list of them.

  """

  files = []

  src_root = Path(src_root)

  for test in tests:

    components = test.split(".")

    src_dir = Path(src_root, *components[:-1])

    java_src = src_dir / f"{components[-1]}.java"

    kt_src = src_dir / f"{components[-1]}.kt"

    if java_src.exists():

      files.append(SourceFile(java_src))

    elif kt_src.exists():

      files.append(SourceFile(kt_src))

    else:

      print("Cannot find source for test", test)

  return files





def main():

  enable = False

  if "enable" in sys.argv:

    enable = True

    test_classes = find_passed_tests(sys.argv[-1])

  elif "disable" in sys.argv:

    test_classes = find_failed_tests(sys.argv[-1])

  else:

    print("Usage: toggle_test.py <enable|disable> <src_root> <csv_file>")

    exit(1)



  test_sources = load_test_files(sys.argv[-2], test_classes)

def find_passed_tests(csv_file: str) -> list[str]:

  """Finds all test classes that passed from a test result CSV file."""

  return _find_tests(

      csv_file, lambda row: int(row["Failed"]) == 0 and int(row["Skipped"]) == 0

  )



  if enable:

    for src in test_sources:

      src.remove_annotation(DISABLED_ANNOTATION)

      src.add_annotation(ENABLED_ANNOTATION)

      src.write()

  else:

    for src in test_sources:

      src.remove_annotation(ENABLED_ANNOTATION)

      src.add_annotation(DISABLED_ANNOTATION)

      src.write()



if __name__ == "__main__":

  main()
 
def find_failed_tests(csv_file: str) -> list[str]:

  """Finds all test classes with at least one failure from a test result CSV file."""

  return _find_tests(csv_file, lambda row: int(row["Failed"]) > 0)

ravenwood/scripts/toggle-test.py

0 → 100755
+85 −0
Original line number Diff line number Diff line 
#!/usr/bin/env -S python3 -B

#

# Copyright (C) 2025 The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#   http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.



"""Tool to manage Ravenwood annotations for entire test classes.



To enable all tests that passed:

$ RAVENWOOD_RUN_DISABLED_TESTS=1 atest SystemUiRavenTests

$ ./toggle-test.py enable /path/to/src \

    /tmp/Ravenwood-stats_SystemUiRavenTestsRavenwood_latest.csv



To disable all tests that failed:

$ atest SystemUiRavenTests

$ ./toggle-test.py disable /path/to/src \

    /tmp/Ravenwood-stats_SystemUiRavenTestsRavenwood_latest.csv

"""



import pathlib

import sys

import ravenlib





Path = pathlib.Path

ENABLED_ANNOTATION = "android.platform.test.annotations.EnabledOnRavenwood"

DISABLED_ANNOTATION = "android.platform.test.annotations.DisabledOnRavenwood"





def usage():

  print("Usage: toggle-test.py <enable|disable> <src_root> <csv_file>")

  exit(1)





def enable_tests(src_root: str, csv_file: str):

  test_sources = ravenlib.load_source_map(src_root)

  test_classes = ravenlib.find_passed_tests(csv_file)

  for test_class in test_classes:

    if test_class not in test_sources:

      # Cannot find source for test

      continue

    src = test_sources[test_class]

    src.remove_annotation(test_class, DISABLED_ANNOTATION)

    src.add_annotation(test_class, ENABLED_ANNOTATION)

    src.write()





def disable_tests(src_root: str, csv_file: str):

  test_sources = ravenlib.load_source_map(src_root)

  test_classes = ravenlib.find_failed_tests(csv_file)

  for test_class in test_classes:

    if test_class not in test_sources:

      # Cannot find source for test

      continue

    src = test_sources[test_class]

    src.remove_annotation(test_class, ENABLED_ANNOTATION)

    src.add_annotation(test_class, DISABLED_ANNOTATION)

    src.write()





def main():

  action = sys.argv[1]

  if action == "enable":

    enable_tests(sys.argv[2], sys.argv[3])

  elif action == "disable":

    disable_tests(sys.argv[2], sys.argv[3])

  else:

    usage()





if __name__ == "__main__":

  try:

    main()

  except IndexError:

    usage()