Merge "Automatically greylist code in 3P packages" (1f80714c) · Commits · e / os / android_frameworks_base

config/hiddenapi-greylist-packages.txt

0 → 100644

+2 −0

Original line number	Diff line number	Diff line
		org.ccil.cowan.tagsoup
		org.ccil.cowan.tagsoup.jaxp

config/hiddenapi-greylist.txt

+0 −105

Original line number	Diff line number	Diff line
		@@ -2604,108 +2604,3 @@ Lorg/apache/xpath/XPathContext;->setCurrentExpressionNodeStack(Lorg/apache/xml/u
		Lorg/apache/xpath/XPathContext;->setCurrentNodeStack(Lorg/apache/xml/utils/IntStack;)V
		Lorg/apache/xpath/XPathContext;->setSecureProcessing(Z)V
		Lorg/apache/xpath/XPathContext;->setVarStack(Lorg/apache/xpath/VariableStack;)V
		Lorg/ccil/cowan/tagsoup/AttributesImpl;-><init>(Lorg/xml/sax/Attributes;)V
		Lorg/ccil/cowan/tagsoup/AttributesImpl;->addAttribute(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
		Lorg/ccil/cowan/tagsoup/AttributesImpl;->data:[Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/AttributesImpl;->length:I
		Lorg/ccil/cowan/tagsoup/AttributesImpl;->setAttribute(ILjava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
		Lorg/ccil/cowan/tagsoup/AttributesImpl;->setValue(ILjava/lang/String;)V
		Lorg/ccil/cowan/tagsoup/AutoDetector;->autoDetectingReader(Ljava/io/InputStream;)Ljava/io/Reader;
		Lorg/ccil/cowan/tagsoup/Element;-><init>(Lorg/ccil/cowan/tagsoup/ElementType;Z)V
		Lorg/ccil/cowan/tagsoup/Element;->anonymize()V
		Lorg/ccil/cowan/tagsoup/Element;->atts()Lorg/ccil/cowan/tagsoup/AttributesImpl;
		Lorg/ccil/cowan/tagsoup/Element;->canContain(Lorg/ccil/cowan/tagsoup/Element;)Z
		Lorg/ccil/cowan/tagsoup/Element;->clean()V
		Lorg/ccil/cowan/tagsoup/Element;->flags()I
		Lorg/ccil/cowan/tagsoup/Element;->localName()Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Element;->name()Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Element;->namespace()Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Element;->next()Lorg/ccil/cowan/tagsoup/Element;
		Lorg/ccil/cowan/tagsoup/Element;->parent()Lorg/ccil/cowan/tagsoup/ElementType;
		Lorg/ccil/cowan/tagsoup/Element;->preclosed:Z
		Lorg/ccil/cowan/tagsoup/Element;->setAttribute(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
		Lorg/ccil/cowan/tagsoup/Element;->setNext(Lorg/ccil/cowan/tagsoup/Element;)V
		Lorg/ccil/cowan/tagsoup/Element;->theAtts:Lorg/ccil/cowan/tagsoup/AttributesImpl;
		Lorg/ccil/cowan/tagsoup/Element;->theNext:Lorg/ccil/cowan/tagsoup/Element;
		Lorg/ccil/cowan/tagsoup/Element;->theType:Lorg/ccil/cowan/tagsoup/ElementType;
		Lorg/ccil/cowan/tagsoup/ElementType;-><init>(Ljava/lang/String;IIILorg/ccil/cowan/tagsoup/Schema;)V
		Lorg/ccil/cowan/tagsoup/ElementType;->atts()Lorg/ccil/cowan/tagsoup/AttributesImpl;
		Lorg/ccil/cowan/tagsoup/ElementType;->setAttribute(Lorg/ccil/cowan/tagsoup/AttributesImpl;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
		Lorg/ccil/cowan/tagsoup/ElementType;->theAtts:Lorg/ccil/cowan/tagsoup/AttributesImpl;
		Lorg/ccil/cowan/tagsoup/ElementType;->theFlags:I
		Lorg/ccil/cowan/tagsoup/ElementType;->theLocalName:Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/ElementType;->theMemberOf:I
		Lorg/ccil/cowan/tagsoup/ElementType;->theModel:I
		Lorg/ccil/cowan/tagsoup/ElementType;->theName:Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/ElementType;->theNamespace:Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/ElementType;->theParent:Lorg/ccil/cowan/tagsoup/ElementType;
		Lorg/ccil/cowan/tagsoup/ElementType;->theSchema:Lorg/ccil/cowan/tagsoup/Schema;
		Lorg/ccil/cowan/tagsoup/HTMLScanner;-><init>()V
		Lorg/ccil/cowan/tagsoup/HTMLSchema;-><init>()V
		Lorg/ccil/cowan/tagsoup/jaxp/SAXFactoryImpl;-><init>()V
		Lorg/ccil/cowan/tagsoup/jaxp/SAXParserImpl;-><init>()V
		Lorg/ccil/cowan/tagsoup/jaxp/SAXParserImpl;->newInstance(Ljava/util/Map;)Lorg/ccil/cowan/tagsoup/jaxp/SAXParserImpl;
		Lorg/ccil/cowan/tagsoup/Parser;-><init>()V
		Lorg/ccil/cowan/tagsoup/Parser;->bogonsEmpty:Z
		Lorg/ccil/cowan/tagsoup/Parser;->CDATAElements:Z
		Lorg/ccil/cowan/tagsoup/Parser;->cleanPublicid(Ljava/lang/String;)Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Parser;->defaultAttributes:Z
		Lorg/ccil/cowan/tagsoup/Parser;->etagchars:[C
		Lorg/ccil/cowan/tagsoup/Parser;->expandEntities(Ljava/lang/String;)Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Parser;->getInputStream(Ljava/lang/String;Ljava/lang/String;)Ljava/io/InputStream;
		Lorg/ccil/cowan/tagsoup/Parser;->ignorableWhitespace:Z
		Lorg/ccil/cowan/tagsoup/Parser;->ignoreBogons:Z
		Lorg/ccil/cowan/tagsoup/Parser;->lookupEntity([CII)I
		Lorg/ccil/cowan/tagsoup/Parser;->makeName([CII)Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Parser;->pop()V
		Lorg/ccil/cowan/tagsoup/Parser;->push(Lorg/ccil/cowan/tagsoup/Element;)V
		Lorg/ccil/cowan/tagsoup/Parser;->rectify(Lorg/ccil/cowan/tagsoup/Element;)V
		Lorg/ccil/cowan/tagsoup/Parser;->restart(Lorg/ccil/cowan/tagsoup/Element;)V
		Lorg/ccil/cowan/tagsoup/Parser;->restartablyPop()V
		Lorg/ccil/cowan/tagsoup/Parser;->rootBogons:Z
		Lorg/ccil/cowan/tagsoup/Parser;->schemaProperty:Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Parser;->split(Ljava/lang/String;)[Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Parser;->theAttributeName:Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Parser;->theAutoDetector:Lorg/ccil/cowan/tagsoup/AutoDetector;
		Lorg/ccil/cowan/tagsoup/Parser;->theContentHandler:Lorg/xml/sax/ContentHandler;
		Lorg/ccil/cowan/tagsoup/Parser;->theDoctypeIsPresent:Z
		Lorg/ccil/cowan/tagsoup/Parser;->theDoctypeSystemId:Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Parser;->theFeatures:Ljava/util/HashMap;
		Lorg/ccil/cowan/tagsoup/Parser;->theLexicalHandler:Lorg/xml/sax/ext/LexicalHandler;
		Lorg/ccil/cowan/tagsoup/Parser;->theNewElement:Lorg/ccil/cowan/tagsoup/Element;
		Lorg/ccil/cowan/tagsoup/Parser;->thePCDATA:Lorg/ccil/cowan/tagsoup/Element;
		Lorg/ccil/cowan/tagsoup/Parser;->thePITarget:Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Parser;->theSaved:Lorg/ccil/cowan/tagsoup/Element;
		Lorg/ccil/cowan/tagsoup/Parser;->theScanner:Lorg/ccil/cowan/tagsoup/Scanner;
		Lorg/ccil/cowan/tagsoup/Parser;->theSchema:Lorg/ccil/cowan/tagsoup/Schema;
		Lorg/ccil/cowan/tagsoup/Parser;->theStack:Lorg/ccil/cowan/tagsoup/Element;
		Lorg/ccil/cowan/tagsoup/Parser;->trimquotes(Ljava/lang/String;)Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Parser;->virginStack:Z
		Lorg/ccil/cowan/tagsoup/PYXScanner;-><init>()V
		Lorg/ccil/cowan/tagsoup/PYXWriter;-><init>(Ljava/io/Writer;)V
		Lorg/ccil/cowan/tagsoup/ScanHandler;->aname([CII)V
		Lorg/ccil/cowan/tagsoup/ScanHandler;->aval([CII)V
		Lorg/ccil/cowan/tagsoup/ScanHandler;->entity([CII)V
		Lorg/ccil/cowan/tagsoup/ScanHandler;->eof([CII)V
		Lorg/ccil/cowan/tagsoup/ScanHandler;->etag([CII)V
		Lorg/ccil/cowan/tagsoup/ScanHandler;->gi([CII)V
		Lorg/ccil/cowan/tagsoup/ScanHandler;->pcdata([CII)V
		Lorg/ccil/cowan/tagsoup/ScanHandler;->pi([CII)V
		Lorg/ccil/cowan/tagsoup/ScanHandler;->stagc([CII)V
		Lorg/ccil/cowan/tagsoup/Scanner;->startCDATA()V
		Lorg/ccil/cowan/tagsoup/Schema;->elementType(Ljava/lang/String;III)V
		Lorg/ccil/cowan/tagsoup/Schema;->getElementType(Ljava/lang/String;)Lorg/ccil/cowan/tagsoup/ElementType;
		Lorg/ccil/cowan/tagsoup/Schema;->getEntity(Ljava/lang/String;)I
		Lorg/ccil/cowan/tagsoup/Schema;->getPrefix()Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Schema;->getURI()Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Schema;->parent(Ljava/lang/String;Ljava/lang/String;)V
		Lorg/ccil/cowan/tagsoup/Schema;->theElementTypes:Ljava/util/HashMap;
		Lorg/ccil/cowan/tagsoup/Schema;->theEntities:Ljava/util/HashMap;
		Lorg/ccil/cowan/tagsoup/Schema;->thePrefix:Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/Schema;->theRoot:Lorg/ccil/cowan/tagsoup/ElementType;
		Lorg/ccil/cowan/tagsoup/Schema;->theURI:Ljava/lang/String;
		Lorg/ccil/cowan/tagsoup/XMLWriter;-><init>(Ljava/io/Writer;)V
		Lorg/ccil/cowan/tagsoup/XMLWriter;->htmlMode:Z
		Lorg/ccil/cowan/tagsoup/XMLWriter;->setOutput(Ljava/io/Writer;)V
		Lorg/ccil/cowan/tagsoup/XMLWriter;->setOutputProperty(Ljava/lang/String;Ljava/lang/String;)V
		Lorg/ccil/cowan/tagsoup/XMLWriter;->setPrefix(Ljava/lang/String;Ljava/lang/String;)V

tools/hiddenapi/generate_hiddenapi_lists.py

+35 −1

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@ from collections import defaultdict
		import os
		import sys
		import re
		import functools

		# Names of flags recognized by the `hiddenapi` tool.
		FLAG_WHITELIST = "whitelist"
		@@ -58,6 +59,10 @@ ALL_FLAGS_SET = set(ALL_FLAGS)
		# script to skip any entries which do not exist any more.
		FLAG_IGNORE_CONFLICTS_SUFFIX = "-ignore-conflicts"

		# Suffix used in command line args to express that all apis within a given set
		# of packages should be assign the given flag.
		FLAG_PACKAGES_SUFFIX = "-packages"

		# Regex patterns of fields/methods used in serialization. These are
		# considered public API despite being hidden.
		SERIALIZATION_PATTERNS = [
		@@ -91,12 +96,16 @@ def get_args():

		for flag in ALL_FLAGS:
		ignore_conflicts_flag = flag + FLAG_IGNORE_CONFLICTS_SUFFIX
		packages_flag = flag + FLAG_PACKAGES_SUFFIX
		parser.add_argument('--' + flag, dest=flag, nargs='*', default=[], metavar='TXT_FILE',
		help='lists of entries with flag "' + flag + '"')
		parser.add_argument('--' + ignore_conflicts_flag, dest=ignore_conflicts_flag, nargs='*',
		default=[], metavar='TXT_FILE',
		help='lists of entries with flag "' + flag +
		'". skip entry if missing or flag conflict.')
		parser.add_argument('--' + packages_flag, dest=packages_flag, nargs='*',
		default=[], metavar='TXT_FILE',
		help='lists of packages to be added to ' + flag + ' list')

		return parser.parse_args()

		@@ -128,6 +137,19 @@ def write_lines(filename, lines):
		with open(filename, 'w') as f:
		f.writelines(lines)

		def extract_package(signature):
		"""Extracts the package from a signature.

		Args:
		signature (string): JNI signature of a method or field.

		Returns:
		The package name of the class containing the field/method.
		"""
		full_class_name = signature.split(";->")[0]
		package_name = full_class_name[1:full_class_name.rindex("/")]
		return package_name.replace('/', '.')

		class FlagsDict:
		def __init__(self):
		self._dict_keyset = set()
		@@ -206,7 +228,10 @@ class FlagsDict:
		self._dict_keyset.update([ csv[0] for csv in csv_values ])

		# Check that all flags are known.
		csv_flags = set(reduce(lambda x, y: set(x).union(y), [ csv[1:] for csv in csv_values ], []))
		csv_flags = set(functools.reduce(
		lambda x, y: set(x).union(y),
		[ csv[1:] for csv in csv_values ],
		[]))
		self._check_flags_set(csv_flags, source)

		# Iterate over all CSV lines, find entry in dict and append flags to it.
		@@ -273,6 +298,15 @@ def main(argv):
		valid_entries = flags.get_valid_subset_of_unassigned_apis(read_lines(filename))
		flags.assign_flag(flag, valid_entries, filename)

		# All members in the specified packages will be assigned the appropriate flag.
		for flag in ALL_FLAGS:
		for filename in args[flag + FLAG_PACKAGES_SUFFIX]:
		packages_needing_list = set(read_lines(filename))
		should_add_signature_to_list = lambda sig,lists: extract_package(
		sig) in packages_needing_list and not lists
		valid_entries = flags.filter_apis(should_add_signature_to_list)
		flags.assign_flag(flag, valid_entries)

		# Assign all remaining entries to the blacklist.
		flags.assign_flag(FLAG_BLACKLIST, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED))

tools/hiddenapi/generate_hiddenapi_lists_test.py

+26 −37

Original line number	Diff line number	Diff line
		@@ -18,33 +18,23 @@ import unittest
		from generate_hiddenapi_lists import *

		class TestHiddenapiListGeneration(unittest.TestCase):
		def test_init(self):
		# Check empty lists
		flags = FlagsDict([], [])
		self.assertEquals(flags.generate_csv(), [])

		# Check valid input - two public and two private API signatures.
		flags = FlagsDict(['A', 'B'], ['C', 'D'])
		self.assertEquals(flags.generate_csv(),
		[ 'A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST, 'C', 'D' ])

		# Check invalid input - overlapping public/private API signatures.
		with self.assertRaises(AssertionError):
		flags = FlagsDict(['A', 'B'], ['B', 'C', 'D'])

		def test_filter_apis(self):
		# Initialize flags so that A and B are put on the whitelist and
		# C, D, E are left unassigned. Try filtering for the unassigned ones.
		flags = FlagsDict(['A', 'B'], ['C', 'D', 'E'])
		flags = FlagsDict()
		flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST,
		'C', 'D', 'E'])
		filter_set = flags.filter_apis(lambda api, flags: not flags)
		self.assertTrue(isinstance(filter_set, set))
		self.assertEqual(filter_set, set([ 'C', 'D', 'E' ]))

		def test_get_valid_subset_of_unassigned_keys(self):
		# Create flags where only A is unassigned.
		flags = FlagsDict(['A'], ['B', 'C'])
		flags = FlagsDict()
		flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B', 'C'])
		flags.assign_flag(FLAG_GREYLIST, set(['C']))
		self.assertEquals(flags.generate_csv(),
		self.assertEqual(flags.generate_csv(),
		[ 'A,' + FLAG_WHITELIST, 'B', 'C,' + FLAG_GREYLIST ])

		# Check three things:
		@@ -55,44 +45,30 @@ class TestHiddenapiListGeneration(unittest.TestCase):
		flags.get_valid_subset_of_unassigned_apis(set(['A', 'B', 'D'])), set([ 'B' ]))

		def test_parse_and_merge_csv(self):
		flags = FlagsDict(['A'], ['B'])
		self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
		flags = FlagsDict()

		# Test empty CSV entry.
		flags.parse_and_merge_csv(['B'])
		self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])

		# Test assigning an already assigned flag.
		flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST])
		self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
		self.assertEqual(flags.generate_csv(), [])

		# Test new additions.
		flags.parse_and_merge_csv([
		'A,' + FLAG_GREYLIST,
		'B,' + FLAG_BLACKLIST + ',' + FLAG_GREYLIST_MAX_O ])
		self.assertEqual(flags.generate_csv(),
		[ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST,
		[ 'A,' + FLAG_GREYLIST,
		'B,' + FLAG_BLACKLIST + "," + FLAG_GREYLIST_MAX_O ])

		# Test unknown API signature.
		with self.assertRaises(AssertionError):
		flags.parse_and_merge_csv([ 'C' ])

		# Test unknown flag.
		with self.assertRaises(AssertionError):
		flags.parse_and_merge_csv([ 'A,foo' ])
		flags.parse_and_merge_csv([ 'C,foo' ])

		def test_assign_flag(self):
		flags = FlagsDict(['A'], ['B'])
		self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])

		# Test assigning an already assigned flag.
		flags.assign_flag(FLAG_WHITELIST, set([ 'A' ]))
		self.assertEquals(flags.generate_csv(), [ 'A,' + FLAG_WHITELIST, 'B' ])
		flags = FlagsDict()
		flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B'])

		# Test new additions.
		flags.assign_flag(FLAG_GREYLIST, set([ 'A', 'B' ]))
		self.assertEquals(flags.generate_csv(),
		self.assertEqual(flags.generate_csv(),
		[ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST, 'B,' + FLAG_GREYLIST ])

		# Test invalid API signature.
		@@ -103,5 +79,18 @@ class TestHiddenapiListGeneration(unittest.TestCase):
		with self.assertRaises(AssertionError):
		flags.assign_flag('foo', set([ 'A' ]))

		def test_extract_package(self):
		signature = 'Lcom/foo/bar/Baz;->method1()Lcom/bar/Baz;'
		expected_package = 'com.foo.bar'
		self.assertEqual(extract_package(signature), expected_package)

		signature = 'Lcom/foo1/bar/MyClass;->method2()V'
		expected_package = 'com.foo1.bar'
		self.assertEqual(extract_package(signature), expected_package)

		signature = 'Lcom/foo_bar/baz/MyClass;->method3()V'
		expected_package = 'com.foo_bar.baz'
		self.assertEqual(extract_package(signature), expected_package)

		if __name__ == '__main__':
		unittest.main()