Merge "Update language to comply with Android's inclusive language guidance" am: 31be55091f (df5b6605) · Commits · e / os / android_frameworks_base

tools/hiddenapi/generate_hiddenapi_lists.py

+93 −34

Original line number	Diff line number	Diff line
		@@ -13,9 +13,7 @@
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.
		"""
		Generate API lists for non-SDK API enforcement.
		"""
		"""Generate API lists for non-SDK API enforcement."""
		import argparse
		from collections import defaultdict
		import functools
		@@ -24,27 +22,47 @@ import re
		import sys

		# Names of flags recognized by the `hiddenapi` tool.
		FLAG_WHITELIST = "whitelist"
		FLAG_GREYLIST = "greylist"
		FLAG_BLACKLIST = "blacklist"
		FLAG_GREYLIST_MAX_O = "greylist-max-o"
		FLAG_GREYLIST_MAX_P = "greylist-max-p"
		FLAG_GREYLIST_MAX_Q = "greylist-max-q"
		FLAG_GREYLIST_MAX_R = "greylist-max-r"
		FLAG_CORE_PLATFORM_API = "core-platform-api"
		FLAG_PUBLIC_API = "public-api"
		FLAG_SYSTEM_API = "system-api"
		FLAG_TEST_API = "test-api"
		FLAG_SDK = 'sdk'
		FLAG_UNSUPPORTED = 'unsupported'
		FLAG_BLOCKED = 'blocked'
		FLAG_MAX_TARGET_O = 'max-target-o'
		FLAG_MAX_TARGET_P = 'max-target-p'
		FLAG_MAX_TARGET_Q = 'max-target-q'
		FLAG_MAX_TARGET_R = 'max-target-r'
		FLAG_CORE_PLATFORM_API = 'core-platform-api'
		FLAG_PUBLIC_API = 'public-api'
		FLAG_SYSTEM_API = 'system-api'
		FLAG_TEST_API = 'test-api'

		OLD_FLAG_SDK = "whitelist"
		OLD_FLAG_UNSUPPORTED = "greylist"
		OLD_FLAG_BLOCKED = "blacklist"
		OLD_FLAG_MAX_TARGET_O = "greylist-max-o"
		OLD_FLAG_MAX_TARGET_P = "greylist-max-p"
		OLD_FLAG_MAX_TARGET_Q = "greylist-max-q"
		OLD_FLAG_MAX_TARGET_R = "greylist-max-r"

		OLD_FLAGS_TO_NEW = {
		OLD_FLAG_SDK: FLAG_SDK,
		OLD_FLAG_UNSUPPORTED: FLAG_UNSUPPORTED,
		OLD_FLAG_BLOCKED: FLAG_BLOCKED,
		OLD_FLAG_MAX_TARGET_O: FLAG_MAX_TARGET_O,
		OLD_FLAG_MAX_TARGET_P: FLAG_MAX_TARGET_P,
		OLD_FLAG_MAX_TARGET_Q: FLAG_MAX_TARGET_Q,
		OLD_FLAG_MAX_TARGET_R: FLAG_MAX_TARGET_R,
		}

		NEW_FLAGS_TO_OLD = dict(zip(OLD_FLAGS_TO_NEW.values(), OLD_FLAGS_TO_NEW.keys()))

		# List of all known flags.
		FLAGS_API_LIST = [
		FLAG_WHITELIST,
		FLAG_GREYLIST,
		FLAG_BLACKLIST,
		FLAG_GREYLIST_MAX_O,
		FLAG_GREYLIST_MAX_P,
		FLAG_GREYLIST_MAX_Q,
		FLAG_GREYLIST_MAX_R,
		FLAG_SDK,
		FLAG_UNSUPPORTED,
		FLAG_BLOCKED,
		FLAG_MAX_TARGET_O,
		FLAG_MAX_TARGET_P,
		FLAG_MAX_TARGET_Q,
		FLAG_MAX_TARGET_R,
		]
		ALL_FLAGS = FLAGS_API_LIST + [
		FLAG_CORE_PLATFORM_API,
		@@ -58,7 +76,7 @@ ALL_FLAGS_SET = set(ALL_FLAGS)

		# Suffix used in command line args to express that only known and
		# otherwise unassigned entries should be assign the given flag.
		# For example, the P dark greylist is checked in as it was in P,
		# For example, the max-target-P list is checked in as it was in P,
		# but signatures have changes since then. The flag instructs this
		# script to skip any entries which do not exist any more.
		FLAG_IGNORE_CONFLICTS_SUFFIX = "-ignore-conflicts"
		@@ -87,6 +105,7 @@ SERIALIZATION_REGEX = re.compile(r'.*->(' + '\|'.join(SERIALIZATION_PATTERNS) + r
		HAS_NO_API_LIST_ASSIGNED = lambda api, flags: not FLAGS_API_LIST_SET.intersection(flags)
		IS_SERIALIZATION = lambda api, flags: SERIALIZATION_REGEX.match(api)


		def get_args():
		"""Parses command line arguments.

		@@ -113,6 +132,7 @@ def get_args():

		return parser.parse_args()


		def read_lines(filename):
		"""Reads entire file and return it as a list of lines.

		@@ -130,8 +150,9 @@ def read_lines(filename):
		lines = map(lambda line: line.strip(), lines)
		return set(lines)


		def write_lines(filename, lines):
		"""Writes list of lines into a file, overwriting the file it it exists.
		"""Writes list of lines into a file, overwriting the file if it exists.

		Args:
		filename (string): Path to the file to be writting into.
		@@ -141,6 +162,7 @@ def write_lines(filename, lines):
		with open(filename, 'w') as f:
		f.writelines(lines)


		def extract_package(signature):
		"""Extracts the package from a signature.

		@@ -159,6 +181,7 @@ def extract_package(signature):
		package_name = full_class_name.rpartition("/")[0]
		return package_name.replace('/', '.')


		class FlagsDict:
		def __init__(self):
		self._dict_keyset = set()
		@@ -182,6 +205,36 @@ class FlagsDict:
		"Please visit go/hiddenapi for more information.").format(
		source, "\n".join(flags_subset - ALL_FLAGS_SET))

		def convert_to_new_flag(self, flag):
		"""Converts old flag to a new variant.

		Flags that are considered old are replaced with new versions.
		Otherwise, it is a no-op.

		Args:
		flag: a string, representing SDK flag.

		Returns:
		A string. Result of conversion.

		"""
		return OLD_FLAGS_TO_NEW.get(flag, flag)

		def convert_to_old_flag(self, flag):
		"""Converts a new flag to a old variant.

		No-op if there is no suitable old flag.
		Only used to support backwards compatibility.

		Args:
		flag: a string, representing SDK flag.

		Returns:
		A string. Result of conversion.

		"""
		return NEW_FLAGS_TO_OLD.get(flag, flag)

		def filter_apis(self, filter_fn):
		"""Returns APIs which match a given predicate.

		@@ -212,10 +265,16 @@ class FlagsDict:
		def generate_csv(self):
		"""Constructs CSV entries from a dictionary.

		Old versions of flags are used to generate the file.

		Returns:
		List of lines comprising a CSV file. See "parse_and_merge_csv" for format description.
		"""
		return sorted(map(lambda api: ",".join([api] + sorted(self._dict[api])), self._dict))
		lines = []
		for api in self._dict:
		flags = sorted([self.convert_to_old_flag(flag) for flag in self._dict[api]])
		lines.append(",".join([api] + flags))
		return sorted(lines)

		def parse_and_merge_csv(self, csv_lines, source = "<unknown>"):
		"""Parses CSV entries and merges them into a given dictionary.
		@@ -237,17 +296,16 @@ class FlagsDict:
		self._dict_keyset.update([ csv[0] for csv in csv_values ])

		# Check that all flags are known.
		csv_flags = set(functools.reduce(
		lambda x, y: set(x).union(y),
		[ csv[1:] for csv in csv_values ],
		[]))
		csv_flags = set()
		for csv in csv_values:
		csv_flags.update([self.convert_to_new_flag(flag) for flag in csv[1:]])
		self._check_flags_set(csv_flags, source)

		# Iterate over all CSV lines, find entry in dict and append flags to it.
		for csv in csv_values:
		flags = csv[1:]
		flags = [self.convert_to_new_flag(flag) for flag in csv[1:]]
		if (FLAG_PUBLIC_API in flags) or (FLAG_SYSTEM_API in flags):
		flags.append(FLAG_WHITELIST)
		flags.append(FLAG_SDK)
		self._dict[csv[0]].update(flags)

		def assign_flag(self, flag, apis, source="<unknown>"):
		@@ -271,6 +329,7 @@ class FlagsDict:
		for api in apis:
		self._dict[api].add(flag)


		def main(argv):
		# Parse arguments.
		args = vars(get_args())
		@@ -287,8 +346,8 @@ def main(argv):
		flags.parse_and_merge_csv(read_lines(filename), filename)

		# Combine inputs which do not require any particular order.
		# (1) Assign serialization API to whitelist.
		flags.assign_flag(FLAG_WHITELIST, flags.filter_apis(IS_SERIALIZATION))
		# (1) Assign serialization API to SDK.
		flags.assign_flag(FLAG_SDK, flags.filter_apis(IS_SERIALIZATION))

		# (2) Merge text files with a known flag into the dictionary.
		for flag in ALL_FLAGS:
		@@ -314,8 +373,8 @@ def main(argv):
		valid_entries = flags.filter_apis(should_add_signature_to_list)
		flags.assign_flag(flag, valid_entries)

		# Assign all remaining entries to the blacklist.
		flags.assign_flag(FLAG_BLACKLIST, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED))
		# Mark all remaining entries as blocked.
		flags.assign_flag(FLAG_BLOCKED, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED))

		# Write output.
		write_lines(args["output"], flags.generate_csv())

tools/hiddenapi/generate_hiddenapi_lists_test.py

+20 −19

Original line number	Diff line number	Diff line
		@@ -23,7 +23,7 @@ class TestHiddenapiListGeneration(unittest.TestCase):
		# Initialize flags so that A and B are put on the whitelist and
		# C, D, E are left unassigned. Try filtering for the unassigned ones.
		flags = FlagsDict()
		flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST,
		flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B,' + FLAG_SDK,
		'C', 'D', 'E'])
		filter_set = flags.filter_apis(lambda api, flags: not flags)
		self.assertTrue(isinstance(filter_set, set))
		@@ -32,10 +32,10 @@ class TestHiddenapiListGeneration(unittest.TestCase):
		def test_get_valid_subset_of_unassigned_keys(self):
		# Create flags where only A is unassigned.
		flags = FlagsDict()
		flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B', 'C'])
		flags.assign_flag(FLAG_GREYLIST, set(['C']))
		flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B', 'C'])
		flags.assign_flag(FLAG_UNSUPPORTED, set(['C']))
		self.assertEqual(flags.generate_csv(),
		[ 'A,' + FLAG_WHITELIST, 'B', 'C,' + FLAG_GREYLIST ])
		[ 'A,' + OLD_FLAG_SDK, 'B', 'C,' + OLD_FLAG_UNSUPPORTED ])

		# Check three things:
		# (1) B is selected as valid unassigned
		@@ -50,20 +50,21 @@ class TestHiddenapiListGeneration(unittest.TestCase):
		# Test empty CSV entry.
		self.assertEqual(flags.generate_csv(), [])

		# Test new additions.
		# Test new additions. CSV generator produces values with old flags
		# to be backwards compatible.
		flags.parse_and_merge_csv([
		'A,' + FLAG_GREYLIST,
		'B,' + FLAG_BLACKLIST + ',' + FLAG_GREYLIST_MAX_O,
		'C,' + FLAG_SYSTEM_API + ',' + FLAG_WHITELIST,
		'D,' + FLAG_GREYLIST+ ',' + FLAG_TEST_API,
		'E,' + FLAG_BLACKLIST+ ',' + FLAG_TEST_API,
		'A,' + FLAG_UNSUPPORTED,
		'B,' + FLAG_BLOCKED + ',' + FLAG_MAX_TARGET_O,
		'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API,
		'D,' + FLAG_UNSUPPORTED + ',' + FLAG_TEST_API,
		'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API,
		])
		self.assertEqual(flags.generate_csv(), [
		'A,' + FLAG_GREYLIST,
		'B,' + FLAG_BLACKLIST + "," + FLAG_GREYLIST_MAX_O,
		'C,' + FLAG_SYSTEM_API + ',' + FLAG_WHITELIST,
		'D,' + FLAG_GREYLIST+ ',' + FLAG_TEST_API,
		'E,' + FLAG_BLACKLIST+ ',' + FLAG_TEST_API,
		'A,' + OLD_FLAG_UNSUPPORTED,
		'B,' + OLD_FLAG_BLOCKED + "," + OLD_FLAG_MAX_TARGET_O,
		'C,' + FLAG_SYSTEM_API + ',' + OLD_FLAG_SDK,
		'D,' + OLD_FLAG_UNSUPPORTED + ',' + FLAG_TEST_API,
		'E,' + OLD_FLAG_BLOCKED + ',' + FLAG_TEST_API,
		])

		# Test unknown flag.
		@@ -72,16 +73,16 @@ class TestHiddenapiListGeneration(unittest.TestCase):

		def test_assign_flag(self):
		flags = FlagsDict()
		flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B'])
		flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B'])

		# Test new additions.
		flags.assign_flag(FLAG_GREYLIST, set([ 'A', 'B' ]))
		flags.assign_flag(FLAG_UNSUPPORTED, set([ 'A', 'B' ]))
		self.assertEqual(flags.generate_csv(),
		[ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST, 'B,' + FLAG_GREYLIST ])
		[ 'A,' + OLD_FLAG_UNSUPPORTED + "," + OLD_FLAG_SDK, 'B,' + OLD_FLAG_UNSUPPORTED ])

		# Test invalid API signature.
		with self.assertRaises(AssertionError):
		flags.assign_flag(FLAG_WHITELIST, set([ 'C' ]))
		flags.assign_flag(FLAG_SDK, set([ 'C' ]))

		# Test invalid flag.
		with self.assertRaises(AssertionError):