Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit f9eb7b94 authored by Pedro Loureiro's avatar Pedro Loureiro Committed by Gerrit Code Review
Browse files

Merge "Add lint database filtering tool"

parents 33d67f0b dea35e10
Loading
Loading
Loading
Loading
+69 −0
Original line number Diff line number Diff line
@@ -24,6 +24,41 @@ package {
    default_applicable_licenses: ["frameworks_base_license"],
}

python_binary_host {
    name: "api_versions_trimmer",
    srcs: ["api_versions_trimmer.py"],
    version: {
        py2: {
            enabled: false,
        },
        py3: {
            enabled: true,
            embedded_launcher: false,
        },
    },
}

python_test_host {
    name: "api_versions_trimmer_unittests",
    main: "api_versions_trimmer_unittests.py",
    srcs: [
        "api_versions_trimmer_unittests.py",
        "api_versions_trimmer.py",
    ],
    test_options: {
        unit_test: true,
    },
    version: {
        py2: {
            enabled: false,
        },
        py3: {
            enabled: true,
            embedded_launcher: false,
        },
    },
}

metalava_cmd = "$(location metalava)"
// Silence reflection warnings. See b/168689341
metalava_cmd += " -J--add-opens=java.base/java.util=ALL-UNNAMED "
@@ -401,3 +436,37 @@ genrule {
        },
    ],
}

// This rule will filter classes present in the jar files of mainline modules
// from the lint database in api-versions.xml.
// This is done to reduce the number of false positive NewApi findings in
// java libraries that compile against the module SDK
genrule {
    name: "api-versions-xml-public-filtered",
    srcs: [
        // Note: order matters: first parameter is the full api-versions.xml
        // after that the stubs files in any order
        // stubs files are all modules that export API surfaces EXCEPT ART
        ":framework-doc-stubs{.api_versions.xml}",
        ":android.net.ipsec.ike.stubs{.jar}",
        ":conscrypt.module.public.api.stubs{.jar}",
        ":framework-connectivity.stubs{.jar}",
        ":framework-media.stubs{.jar}",
        ":framework-mediaprovider.stubs{.jar}",
        ":framework-permission.stubs{.jar}",
        ":framework-sdkextensions.stubs{.jar}",
        ":framework-statsd.stubs{.jar}",
        ":framework-tethering.stubs{.jar}",
        ":framework-wifi.stubs{.jar}",
        ":i18n.module.public.api.stubs{.jar}",
    ],
    out: ["api-versions-public-filtered.xml"],
    tools: ["api_versions_trimmer"],
    cmd: "$(location api_versions_trimmer) $(out) $(in)",
    dist: {
        targets: [
            "sdk",
            "win_sdk",
        ],
    },
}
+136 −0
Original line number Diff line number Diff line
#!/usr/bin/env python3
#
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Script to remove mainline APIs from the api-versions.xml."""

import argparse
import re
import xml.etree.ElementTree as ET
import zipfile


def read_classes(stubs):
  """Read classes from the stubs file.

  Args:
    stubs: argument can be a path to a file (a string), a file-like object or a
    path-like object

  Returns:
    a set of the classes found in the file (set of strings)
  """
  classes = set()
  with zipfile.ZipFile(stubs) as z:
    for info in z.infolist():
      if (not info.is_dir()
          and info.filename.endswith(".class")
          and not info.filename.startswith("META-INF")):
        # drop ".class" extension
        classes.add(info.filename[:-6])
  return classes


def filter_method_tag(method, classes_to_remove):
  """Updates the signature of this method by calling filter_method_signature.

  Updates the method passed into this function.

  Args:
    method: xml element that represents a method
    classes_to_remove: set of classes you to remove
  """
  filtered = filter_method_signature(method.get("name"), classes_to_remove)
  method.set("name", filtered)


def filter_method_signature(signature, classes_to_remove):
  """Removes mentions of certain classes from this method signature.

  Replaces any existing classes that need to be removed, with java/lang/Object

  Args:
    signature: string that is a java representation of a method signature
    classes_to_remove: set of classes you to remove
  """
  regex = re.compile("L.*?;")
  start = signature.find("(")
  matches = set(regex.findall(signature[start:]))
  for m in matches:
    # m[1:-1] to drop the leading `L` and `;` ending
    if m[1:-1] in classes_to_remove:
      signature = signature.replace(m, "Ljava/lang/Object;")
  return signature


def filter_lint_database(database, classes_to_remove, output):
  """Reads a lint database and writes a filtered version without some classes.

  Reads database from api-versions.xml and removes any references to classes
  in the second argument. Writes the result (another xml with the same format
  of the database) to output.

  Args:
    database: path to xml with lint database to read
    classes_to_remove: iterable (ideally a set or similar for quick
    lookups) that enumerates the classes that should be removed
    output: path to write the filtered database
  """
  xml = ET.parse(database)
  root = xml.getroot()
  for c in xml.findall("class"):
    cname = c.get("name")
    if cname in classes_to_remove:
      root.remove(c)
    else:
      # find the <extends /> tag inside this class to see if the parent
      # has been removed from the known classes (attribute called name)
      super_classes = c.findall("extends")
      for super_class in super_classes:
        super_class_name = super_class.get("name")
        if super_class_name in classes_to_remove:
          super_class.set("name", "java/lang/Object")
      interfaces = c.findall("implements")
      for interface in interfaces:
        interface_name = interface.get("name")
        if interface_name in classes_to_remove:
          c.remove(interface)
      for method in c.findall("method"):
        filter_method_tag(method, classes_to_remove)
  xml.write(output)


def main():
  """Run the program."""
  parser = argparse.ArgumentParser(
      description=
      ("Read a lint database (api-versions.xml) and many stubs jar files. "
       "Produce another database file that doesn't include the classes present "
       "in the stubs file(s)."))
  parser.add_argument("output", help="Destination of the result (xml file).")
  parser.add_argument(
      "api_versions",
      help="The lint database (api-versions.xml file) to read data from"
  )
  parser.add_argument("stubs", nargs="+", help="The stubs jar file(s)")
  parsed = parser.parse_args()
  classes = set()
  for stub in parsed.stubs:
    classes.update(read_classes(stub))
  filter_lint_database(parsed.api_versions, classes, parsed.output)


if __name__ == "__main__":
  main()
+307 −0
Original line number Diff line number Diff line
#!/usr/bin/env python3
#
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import io
import re
import unittest
import xml.etree.ElementTree as ET
import zipfile

import api_versions_trimmer


def create_in_memory_zip_file(files):
  f = io.BytesIO()
  with zipfile.ZipFile(f, "w") as z:
    for fname in files:
      with z.open(fname, mode="w") as class_file:
        class_file.write(b"")
  return f


def indent(elem, level=0):
  i = "\n" + level * "  "
  j = "\n" + (level - 1) * "  "
  if len(elem):
    if not elem.text or not elem.text.strip():
      elem.text = i + "  "
      if not elem.tail or not elem.tail.strip():
        elem.tail = i
        for subelem in elem:
          indent(subelem, level + 1)
        if not elem.tail or not elem.tail.strip():
          elem.tail = j
    else:
      if level and (not elem.tail or not elem.tail.strip()):
        elem.tail = j
    return elem


def pretty_print(s):
  tree = ET.parse(io.StringIO(s))
  el = indent(tree.getroot())
  res = ET.tostring(el).decode("utf-8")
  # remove empty lines inside the result because this still breaks some
  # comparisons
  return re.sub(r"\n\s*\n", "\n", res, re.MULTILINE)


class ApiVersionsTrimmerUnittests(unittest.TestCase):

  def setUp(self):
    # so it prints diffs in long strings (xml files)
    self.maxDiff = None

  def test_read_classes(self):
    f = create_in_memory_zip_file(
        ["a/b/C.class",
         "a/b/D.class",
        ]
    )
    res = api_versions_trimmer.read_classes(f)
    self.assertEqual({"a/b/C", "a/b/D"}, res)

  def test_read_classes_ignore_dex(self):
    f = create_in_memory_zip_file(
        ["a/b/C.class",
         "a/b/D.class",
         "a/b/E.dex",
         "f.dex",
        ]
    )
    res = api_versions_trimmer.read_classes(f)
    self.assertEqual({"a/b/C", "a/b/D"}, res)

  def test_read_classes_ignore_manifest(self):
    f = create_in_memory_zip_file(
        ["a/b/C.class",
         "a/b/D.class",
         "META-INFO/G.class"
        ]
    )
    res = api_versions_trimmer.read_classes(f)
    self.assertEqual({"a/b/C", "a/b/D"}, res)

  def test_filter_method_signature(self):
    xml = """
    <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z" since="24"/>
    """
    method = ET.fromstring(xml)
    classes_to_remove = {"android/accessibilityservice/GestureDescription"}
    expected = "dispatchGesture(Ljava/lang/Object;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z"
    api_versions_trimmer.filter_method_tag(method, classes_to_remove)
    self.assertEqual(expected, method.get("name"))

  def test_filter_method_signature_with_L_in_method(self):
    xml = """
    <method name="dispatchLeftGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z" since="24"/>
    """
    method = ET.fromstring(xml)
    classes_to_remove = {"android/accessibilityservice/GestureDescription"}
    expected = "dispatchLeftGesture(Ljava/lang/Object;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z"
    api_versions_trimmer.filter_method_tag(method, classes_to_remove)
    self.assertEqual(expected, method.get("name"))

  def test_filter_method_signature_with_L_in_class(self):
    xml = """
    <method name="dispatchGesture(Landroid/accessibilityservice/LeftGestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z" since="24"/>
    """
    method = ET.fromstring(xml)
    classes_to_remove = {"android/accessibilityservice/LeftGestureDescription"}
    expected = "dispatchGesture(Ljava/lang/Object;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z"
    api_versions_trimmer.filter_method_tag(method, classes_to_remove)
    self.assertEqual(expected, method.get("name"))

  def test_filter_method_signature_with_inner_class(self):
    xml = """
    <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription$Inner;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z" since="24"/>
    """
    method = ET.fromstring(xml)
    classes_to_remove = {"android/accessibilityservice/GestureDescription$Inner"}
    expected = "dispatchGesture(Ljava/lang/Object;Landroid/accessibilityservice/AccessibilityService$GestureResultCallback;Landroid/os/Handler;)Z"
    api_versions_trimmer.filter_method_tag(method, classes_to_remove)
    self.assertEqual(expected, method.get("name"))

  def _run_filter_db_test(self, database_str, expected):
    """Performs the pattern of testing the filter_lint_database method.

    Filters instances of the class "a/b/C" (hard-coded) from the database string
    and compares the result with the expected result (performs formatting of
    the xml of both inputs)

    Args:
      database_str: string, the contents of the lint database (api-versions.xml)
      expected: string, the expected result after filtering the original
    database
    """
    database = io.StringIO(database_str)
    classes_to_remove = {"a/b/C"}
    output = io.BytesIO()
    api_versions_trimmer.filter_lint_database(
        database,
        classes_to_remove,
        output
    )
    expected = pretty_print(expected)
    res = pretty_print(output.getvalue().decode("utf-8"))
    self.assertEqual(expected, res)

  def test_filter_lint_database_updates_method_signature_params(self):
    self._run_filter_db_test(
        database_str="""
    <api version="2">
      <!-- will be removed -->
      <class name="a/b/C" since="1">
        <extends name="java/lang/Object"/>
      </class>

      <class name="a/b/E" since="1">
        <!-- extends will be modified -->
        <extends name="a/b/C"/>
        <!-- first parameter will be modified -->
        <method name="dispatchGesture(La/b/C;Landroid/os/Handler;)Z" since="24"/>
        <!-- second should remain untouched -->
        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
      </class>
    </api>
    """,
        expected="""
    <api version="2">
      <class name="a/b/E" since="1">
        <extends name="java/lang/Object"/>
        <method name="dispatchGesture(Ljava/lang/Object;Landroid/os/Handler;)Z" since="24"/>
        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
      </class>
    </api>
    """)

  def test_filter_lint_database_updates_method_signature_return(self):
    self._run_filter_db_test(
        database_str="""
    <api version="2">
      <!-- will be removed -->
      <class name="a/b/C" since="1">
        <extends name="java/lang/Object"/>
      </class>

      <class name="a/b/E" since="1">
        <!-- extends will be modified -->
        <extends name="a/b/C"/>
        <!-- return type should be changed -->
        <method name="gestureIdToString(I)La/b/C;" since="24"/>
      </class>
    </api>
    """,
        expected="""
    <api version="2">
      <class name="a/b/E" since="1">

        <extends name="java/lang/Object"/>

        <method name="gestureIdToString(I)Ljava/lang/Object;" since="24"/>
      </class>
    </api>
    """)

  def test_filter_lint_database_removes_implements(self):
    self._run_filter_db_test(
        database_str="""
    <api version="2">
      <!-- will be removed -->
      <class name="a/b/C" since="1">
        <extends name="java/lang/Object"/>
      </class>

      <class name="a/b/D" since="1">
        <extends name="java/lang/Object"/>
        <implements name="a/b/C"/>
        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
      </class>
    </api>
    """,
        expected="""
    <api version="2">

      <class name="a/b/D" since="1">
        <extends name="java/lang/Object"/>
        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
      </class>
    </api>
    """)

  def test_filter_lint_database_updates_extends(self):
    self._run_filter_db_test(
        database_str="""
    <api version="2">
      <!-- will be removed -->
      <class name="a/b/C" since="1">
        <extends name="java/lang/Object"/>
      </class>

      <class name="a/b/E" since="1">
        <!-- extends will be modified -->
        <extends name="a/b/C"/>
        <method name="dispatchGesture(Ljava/lang/Object;Landroid/os/Handler;)Z" since="24"/>
        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
      </class>
    </api>
    """,
        expected="""
    <api version="2">
      <class name="a/b/E" since="1">
        <extends name="java/lang/Object"/>
        <method name="dispatchGesture(Ljava/lang/Object;Landroid/os/Handler;)Z" since="24"/>
        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
      </class>
    </api>
    """)

  def test_filter_lint_database_removes_class(self):
    self._run_filter_db_test(
        database_str="""
    <api version="2">
      <!-- will be removed -->
      <class name="a/b/C" since="1">
        <extends name="java/lang/Object"/>
      </class>

      <class name="a/b/D" since="1">
        <extends name="java/lang/Object"/>
        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
      </class>
    </api>
    """,
        expected="""
    <api version="2">

      <class name="a/b/D" since="1">
        <extends name="java/lang/Object"/>
        <method name="dispatchGesture(Landroid/accessibilityservice/GestureDescription;Landroid/accessibilityservice/AccessibilityService$GestureRe
sultCallback;Landroid/os/Handler;)Z" since="24"/>
      </class>
    </api>
    """)


if __name__ == "__main__":
  unittest.main()