Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit a6e09b42 authored by Andrei Onea's avatar Andrei Onea
Browse files

Automatically greylist code in 3P packages 

generate_hidden_api_lists now receives a file containing package names
which need to be greylisted (although it could be made to work with any
api list required).
Also took the opportunity to clean up the tests to reflect the more
strict code.

Bug: 129387816
Test: m appcompat
Test: frameworks/base/tools/hiddenapi/generate_hiddenapi_lists_test.py
Change-Id: I619f8581d166aa48eda572bc0053d8739d6420eb
parent f3736d67

config/hiddenapi-greylist-packages.txt

0 → 100644
+2 −0
Original line number Diff line number Diff line 
org.ccil.cowan.tagsoup

org.ccil.cowan.tagsoup.jaxp

config/hiddenapi-greylist.txt

+0 −105
Original line number Diff line number Diff line
@@ -2873,108 +2873,3 @@ Lorg/apache/xpath/XPathContext;->setCurrentExpressionNodeStack(Lorg/apache/xml/u 
Lorg/apache/xpath/XPathContext;->setCurrentNodeStack(Lorg/apache/xml/utils/IntStack;)V

Lorg/apache/xpath/XPathContext;->setSecureProcessing(Z)V

Lorg/apache/xpath/XPathContext;->setVarStack(Lorg/apache/xpath/VariableStack;)V
 
Lorg/ccil/cowan/tagsoup/AttributesImpl;-><init>(Lorg/xml/sax/Attributes;)V

Lorg/ccil/cowan/tagsoup/AttributesImpl;->addAttribute(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V

Lorg/ccil/cowan/tagsoup/AttributesImpl;->data:[Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/AttributesImpl;->length:I

Lorg/ccil/cowan/tagsoup/AttributesImpl;->setAttribute(ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V

Lorg/ccil/cowan/tagsoup/AttributesImpl;->setValue(ILjava/lang/String;)V

Lorg/ccil/cowan/tagsoup/AutoDetector;->autoDetectingReader(Ljava/io/InputStream;)Ljava/io/Reader;

Lorg/ccil/cowan/tagsoup/Element;-><init>(Lorg/ccil/cowan/tagsoup/ElementType;Z)V

Lorg/ccil/cowan/tagsoup/Element;->anonymize()V

Lorg/ccil/cowan/tagsoup/Element;->atts()Lorg/ccil/cowan/tagsoup/AttributesImpl;

Lorg/ccil/cowan/tagsoup/Element;->canContain(Lorg/ccil/cowan/tagsoup/Element;)Z

Lorg/ccil/cowan/tagsoup/Element;->clean()V

Lorg/ccil/cowan/tagsoup/Element;->flags()I

Lorg/ccil/cowan/tagsoup/Element;->localName()Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Element;->name()Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Element;->namespace()Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Element;->next()Lorg/ccil/cowan/tagsoup/Element;

Lorg/ccil/cowan/tagsoup/Element;->parent()Lorg/ccil/cowan/tagsoup/ElementType;

Lorg/ccil/cowan/tagsoup/Element;->preclosed:Z

Lorg/ccil/cowan/tagsoup/Element;->setAttribute(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V

Lorg/ccil/cowan/tagsoup/Element;->setNext(Lorg/ccil/cowan/tagsoup/Element;)V

Lorg/ccil/cowan/tagsoup/Element;->theAtts:Lorg/ccil/cowan/tagsoup/AttributesImpl;

Lorg/ccil/cowan/tagsoup/Element;->theNext:Lorg/ccil/cowan/tagsoup/Element;

Lorg/ccil/cowan/tagsoup/Element;->theType:Lorg/ccil/cowan/tagsoup/ElementType;

Lorg/ccil/cowan/tagsoup/ElementType;-><init>(Ljava/lang/String;IIILorg/ccil/cowan/tagsoup/Schema;)V

Lorg/ccil/cowan/tagsoup/ElementType;->atts()Lorg/ccil/cowan/tagsoup/AttributesImpl;

Lorg/ccil/cowan/tagsoup/ElementType;->setAttribute(Lorg/ccil/cowan/tagsoup/AttributesImpl;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V

Lorg/ccil/cowan/tagsoup/ElementType;->theAtts:Lorg/ccil/cowan/tagsoup/AttributesImpl;

Lorg/ccil/cowan/tagsoup/ElementType;->theFlags:I

Lorg/ccil/cowan/tagsoup/ElementType;->theLocalName:Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/ElementType;->theMemberOf:I

Lorg/ccil/cowan/tagsoup/ElementType;->theModel:I

Lorg/ccil/cowan/tagsoup/ElementType;->theName:Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/ElementType;->theNamespace:Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/ElementType;->theParent:Lorg/ccil/cowan/tagsoup/ElementType;

Lorg/ccil/cowan/tagsoup/ElementType;->theSchema:Lorg/ccil/cowan/tagsoup/Schema;

Lorg/ccil/cowan/tagsoup/HTMLScanner;-><init>()V

Lorg/ccil/cowan/tagsoup/HTMLSchema;-><init>()V

Lorg/ccil/cowan/tagsoup/jaxp/SAXFactoryImpl;-><init>()V

Lorg/ccil/cowan/tagsoup/jaxp/SAXParserImpl;-><init>()V

Lorg/ccil/cowan/tagsoup/jaxp/SAXParserImpl;->newInstance(Ljava/util/Map;)Lorg/ccil/cowan/tagsoup/jaxp/SAXParserImpl;

Lorg/ccil/cowan/tagsoup/Parser;-><init>()V

Lorg/ccil/cowan/tagsoup/Parser;->bogonsEmpty:Z

Lorg/ccil/cowan/tagsoup/Parser;->CDATAElements:Z

Lorg/ccil/cowan/tagsoup/Parser;->cleanPublicid(Ljava/lang/String;)Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Parser;->defaultAttributes:Z

Lorg/ccil/cowan/tagsoup/Parser;->etagchars:[C

Lorg/ccil/cowan/tagsoup/Parser;->expandEntities(Ljava/lang/String;)Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Parser;->getInputStream(Ljava/lang/String;Ljava/lang/String;)Ljava/io/InputStream;

Lorg/ccil/cowan/tagsoup/Parser;->ignorableWhitespace:Z

Lorg/ccil/cowan/tagsoup/Parser;->ignoreBogons:Z

Lorg/ccil/cowan/tagsoup/Parser;->lookupEntity([CII)I

Lorg/ccil/cowan/tagsoup/Parser;->makeName([CII)Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Parser;->pop()V

Lorg/ccil/cowan/tagsoup/Parser;->push(Lorg/ccil/cowan/tagsoup/Element;)V

Lorg/ccil/cowan/tagsoup/Parser;->rectify(Lorg/ccil/cowan/tagsoup/Element;)V

Lorg/ccil/cowan/tagsoup/Parser;->restart(Lorg/ccil/cowan/tagsoup/Element;)V

Lorg/ccil/cowan/tagsoup/Parser;->restartablyPop()V

Lorg/ccil/cowan/tagsoup/Parser;->rootBogons:Z

Lorg/ccil/cowan/tagsoup/Parser;->schemaProperty:Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Parser;->split(Ljava/lang/String;)[Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Parser;->theAttributeName:Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Parser;->theAutoDetector:Lorg/ccil/cowan/tagsoup/AutoDetector;

Lorg/ccil/cowan/tagsoup/Parser;->theContentHandler:Lorg/xml/sax/ContentHandler;

Lorg/ccil/cowan/tagsoup/Parser;->theDoctypeIsPresent:Z

Lorg/ccil/cowan/tagsoup/Parser;->theDoctypeSystemId:Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Parser;->theFeatures:Ljava/util/HashMap;

Lorg/ccil/cowan/tagsoup/Parser;->theLexicalHandler:Lorg/xml/sax/ext/LexicalHandler;

Lorg/ccil/cowan/tagsoup/Parser;->theNewElement:Lorg/ccil/cowan/tagsoup/Element;

Lorg/ccil/cowan/tagsoup/Parser;->thePCDATA:Lorg/ccil/cowan/tagsoup/Element;

Lorg/ccil/cowan/tagsoup/Parser;->thePITarget:Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Parser;->theSaved:Lorg/ccil/cowan/tagsoup/Element;

Lorg/ccil/cowan/tagsoup/Parser;->theScanner:Lorg/ccil/cowan/tagsoup/Scanner;

Lorg/ccil/cowan/tagsoup/Parser;->theSchema:Lorg/ccil/cowan/tagsoup/Schema;

Lorg/ccil/cowan/tagsoup/Parser;->theStack:Lorg/ccil/cowan/tagsoup/Element;

Lorg/ccil/cowan/tagsoup/Parser;->trimquotes(Ljava/lang/String;)Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Parser;->virginStack:Z

Lorg/ccil/cowan/tagsoup/PYXScanner;-><init>()V

Lorg/ccil/cowan/tagsoup/PYXWriter;-><init>(Ljava/io/Writer;)V

Lorg/ccil/cowan/tagsoup/ScanHandler;->aname([CII)V

Lorg/ccil/cowan/tagsoup/ScanHandler;->aval([CII)V

Lorg/ccil/cowan/tagsoup/ScanHandler;->entity([CII)V

Lorg/ccil/cowan/tagsoup/ScanHandler;->eof([CII)V

Lorg/ccil/cowan/tagsoup/ScanHandler;->etag([CII)V

Lorg/ccil/cowan/tagsoup/ScanHandler;->gi([CII)V

Lorg/ccil/cowan/tagsoup/ScanHandler;->pcdata([CII)V

Lorg/ccil/cowan/tagsoup/ScanHandler;->pi([CII)V

Lorg/ccil/cowan/tagsoup/ScanHandler;->stagc([CII)V

Lorg/ccil/cowan/tagsoup/Scanner;->startCDATA()V

Lorg/ccil/cowan/tagsoup/Schema;->elementType(Ljava/lang/String;III)V

Lorg/ccil/cowan/tagsoup/Schema;->getElementType(Ljava/lang/String;)Lorg/ccil/cowan/tagsoup/ElementType;

Lorg/ccil/cowan/tagsoup/Schema;->getEntity(Ljava/lang/String;)I

Lorg/ccil/cowan/tagsoup/Schema;->getPrefix()Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Schema;->getURI()Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Schema;->parent(Ljava/lang/String;Ljava/lang/String;)V

Lorg/ccil/cowan/tagsoup/Schema;->theElementTypes:Ljava/util/HashMap;

Lorg/ccil/cowan/tagsoup/Schema;->theEntities:Ljava/util/HashMap;

Lorg/ccil/cowan/tagsoup/Schema;->thePrefix:Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/Schema;->theRoot:Lorg/ccil/cowan/tagsoup/ElementType;

Lorg/ccil/cowan/tagsoup/Schema;->theURI:Ljava/lang/String;

Lorg/ccil/cowan/tagsoup/XMLWriter;-><init>(Ljava/io/Writer;)V

Lorg/ccil/cowan/tagsoup/XMLWriter;->htmlMode:Z

Lorg/ccil/cowan/tagsoup/XMLWriter;->setOutput(Ljava/io/Writer;)V

Lorg/ccil/cowan/tagsoup/XMLWriter;->setOutputProperty(Ljava/lang/String;Ljava/lang/String;)V

Lorg/ccil/cowan/tagsoup/XMLWriter;->setPrefix(Ljava/lang/String;Ljava/lang/String;)V

tools/hiddenapi/generate_hiddenapi_lists.py

+35 −1
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ from collections import defaultdict 
import os

import sys

import re

import functools



# Names of flags recognized by the `hiddenapi` tool.

FLAG_WHITELIST = "whitelist"
@@ -58,6 +59,10 @@ ALL_FLAGS_SET = set(ALL_FLAGS) 
# script to skip any entries which do not exist any more.

FLAG_IGNORE_CONFLICTS_SUFFIX = "-ignore-conflicts"



# Suffix used in command line args to express that all apis within a given set

# of packages should be assign the given flag.

FLAG_PACKAGES_SUFFIX = "-packages"



# Regex patterns of fields/methods used in serialization. These are

# considered public API despite being hidden.

SERIALIZATION_PATTERNS = [
@@ -91,12 +96,16 @@ def get_args(): 


    for flag in ALL_FLAGS:

        ignore_conflicts_flag = flag + FLAG_IGNORE_CONFLICTS_SUFFIX

        packages_flag = flag + FLAG_PACKAGES_SUFFIX

        parser.add_argument('--' + flag, dest=flag, nargs='*', default=[], metavar='TXT_FILE',

            help='lists of entries with flag "' + flag + '"')

        parser.add_argument('--' + ignore_conflicts_flag, dest=ignore_conflicts_flag, nargs='*',

            default=[], metavar='TXT_FILE',

            help='lists of entries with flag "' + flag +

                 '". skip entry if missing or flag conflict.')

        parser.add_argument('--' + packages_flag, dest=packages_flag, nargs='*',

            default=[], metavar='TXT_FILE',

            help='lists of packages to be added to ' + flag + ' list')



    return parser.parse_args()
@@ -128,6 +137,19 @@ def write_lines(filename, lines): 
    with open(filename, 'w') as f:

        f.writelines(lines)



def extract_package(signature):

    """Extracts the package from a signature.



    Args:

        signature (string): JNI signature of a method or field.



    Returns:

        The package name of the class containing the field/method.

    """

    full_class_name = signature.split(";->")[0]

    package_name = full_class_name[1:full_class_name.rindex("/")]

    return package_name.replace('/', '.')



class FlagsDict:

    def __init__(self):

        self._dict_keyset = set()
@@ -206,7 +228,10 @@ class FlagsDict: 
        self._dict_keyset.update([ csv[0] for csv in csv_values ])



        # Check that all flags are known.

        csv_flags = set(reduce(lambda x, y: set(x).union(y), [ csv[1:] for csv in csv_values ], []))

        csv_flags = set(functools.reduce(

            lambda x, y: set(x).union(y),

            [ csv[1:] for csv in csv_values ],

            []))

        self._check_flags_set(csv_flags, source)



        # Iterate over all CSV lines, find entry in dict and append flags to it.
@@ -273,6 +298,15 @@ def main(argv): 
            valid_entries = flags.get_valid_subset_of_unassigned_apis(read_lines(filename))

            flags.assign_flag(flag, valid_entries, filename)



    # All members in the specified packages will be assigned the appropriate flag.

    for flag in ALL_FLAGS:

        for filename in args[flag + FLAG_PACKAGES_SUFFIX]:

            packages_needing_list = set(read_lines(filename))

            should_add_signature_to_list = lambda sig,lists: extract_package(

                sig) in packages_needing_list and not lists

            valid_entries = flags.filter_apis(should_add_signature_to_list)

            flags.assign_flag(flag, valid_entries)



    # Assign all remaining entries to the blacklist.

    flags.assign_flag(FLAG_BLACKLIST, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED))

tools/hiddenapi/generate_hiddenapi_lists_test.py

+26 −37
Original line number Diff line number Diff line
@@ -18,33 +18,23 @@ import unittest 
from generate_hiddenapi_lists import *



class TestHiddenapiListGeneration(unittest.TestCase):

    def test_init(self):

        # Check empty lists

        flags = FlagsDict([], [])

        self.assertEquals(flags.generate_csv(), [])



        # Check valid input - two public and two private API signatures.

        flags = FlagsDict(['A', 'B'], ['C', 'D'])

        self.assertEquals(flags.generate_csv(),

                          [ 'A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST, 'C', 'D' ])



        # Check invalid input - overlapping public/private API signatures.

        with self.assertRaises(AssertionError):

            flags = FlagsDict(['A', 'B'], ['B', 'C', 'D'])



    def test_filter_apis(self):

        # Initialize flags so that A and B are put on the whitelist and

        # C, D, E are left unassigned. Try filtering for the unassigned ones.

        flags = FlagsDict(['A', 'B'], ['C', 'D', 'E'])

        flags = FlagsDict()

        flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST,

                        'C', 'D', 'E'])

        filter_set = flags.filter_apis(lambda api, flags: not flags)

        self.assertTrue(isinstance(filter_set, set))

        self.assertEqual(filter_set, set([ 'C', 'D', 'E' ]))



    def test_get_valid_subset_of_unassigned_keys(self):

        # Create flags where only A is unassigned.

        flags = FlagsDict(['A'], ['B', 'C'])

        flags = FlagsDict()

        flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B', 'C'])

        flags.assign_flag(FLAG_GREYLIST, set(['C']))

        self.assertEquals(flags.generate_csv(),

        self.assertEqual(flags.generate_csv(),

            [ 'A,' + FLAG_WHITELIST, 'B', 'C,' + FLAG_GREYLIST ])



        # Check three things:
@@ -55,44 +45,30 @@ class TestHiddenapiListGeneration(unittest.TestCase): 
            flags.get_valid_subset_of_unassigned_apis(set(['A', 'B', 'D'])), set([ 'B' ]))



    def test_parse_and_merge_csv(self):

        flags = FlagsDict(['A'], ['B'])

        self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])

        flags = FlagsDict()



        # Test empty CSV entry.

        flags.parse_and_merge_csv(['B'])

        self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])



        # Test assigning an already assigned flag.

        flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST])

        self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])

        self.assertEqual(flags.generate_csv(), [])



        # Test new additions.

        flags.parse_and_merge_csv([

            'A,' + FLAG_GREYLIST,

            'B,' + FLAG_BLACKLIST + ',' + FLAG_GREYLIST_MAX_O ])

        self.assertEqual(flags.generate_csv(),

            [ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST,

            [ 'A,' + FLAG_GREYLIST,

              'B,' + FLAG_BLACKLIST + "," + FLAG_GREYLIST_MAX_O ])



        # Test unknown API signature.

        with self.assertRaises(AssertionError):

            flags.parse_and_merge_csv([ 'C' ])



        # Test unknown flag.

        with self.assertRaises(AssertionError):

            flags.parse_and_merge_csv([ 'A,foo' ])

            flags.parse_and_merge_csv([ 'C,foo' ])



    def test_assign_flag(self):

        flags = FlagsDict(['A'], ['B'])

        self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])



        # Test assigning an already assigned flag.

        flags.assign_flag(FLAG_WHITELIST, set([ 'A' ]))

        self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])

        flags = FlagsDict()

        flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B'])



        # Test new additions.

        flags.assign_flag(FLAG_GREYLIST, set([ 'A', 'B' ]))

        self.assertEquals(flags.generate_csv(),

        self.assertEqual(flags.generate_csv(),

            [ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST, 'B,' + FLAG_GREYLIST ])



        # Test invalid API signature.
@@ -103,5 +79,18 @@ class TestHiddenapiListGeneration(unittest.TestCase): 
        with self.assertRaises(AssertionError):

            flags.assign_flag('foo', set([ 'A' ]))



    def test_extract_package(self):

        signature = 'Lcom/foo/bar/Baz;->method1()Lcom/bar/Baz;'

        expected_package = 'com.foo.bar'

        self.assertEqual(extract_package(signature), expected_package)



        signature = 'Lcom/foo1/bar/MyClass;->method2()V'

        expected_package = 'com.foo1.bar'

        self.assertEqual(extract_package(signature), expected_package)



        signature = 'Lcom/foo_bar/baz/MyClass;->method3()V'

        expected_package = 'com.foo_bar.baz'

        self.assertEqual(extract_package(signature), expected_package)



if __name__ == '__main__':

    unittest.main()