Merge "Cleanup verify_overlaps*.py" (6f041b56) · Commits · e / os / android_build_soong

scripts/hiddenapi/verify_overlaps.py

+48 −48

Original line number	Diff line number	Diff line
		@@ -23,13 +23,13 @@ from itertools import chain
		from signature_trie import signature_trie


		def dict_reader(csvfile):
		def dict_reader(csv_file):
		return csv.DictReader(
		csvfile, delimiter=",", quotechar="\|", fieldnames=["signature"])
		csv_file, delimiter=",", quotechar="\|", fieldnames=["signature"])


		def read_flag_trie_from_file(file):
		with open(file, "r") as stream:
		with open(file, "r", encoding="utf8") as stream:
		return read_flag_trie_from_stream(stream)


		@@ -43,24 +43,24 @@ def read_flag_trie_from_stream(stream):


		def extract_subset_from_monolithic_flags_as_dict_from_file(
		monolithicTrie, patternsFile):
		monolithic_trie, patterns_file):
		"""Extract a subset of flags from the dict of monolithic flags.

		:param monolithicFlagsDict: the dict containing all the monolithic flags.
		:param patternsFile: a file containing a list of signature patterns that
		:param monolithic_trie: the trie containing all the monolithic flags.
		:param patterns_file: a file containing a list of signature patterns that
		define the subset.
		:return: the dict from signature to row.
		"""
		with open(patternsFile, "r") as stream:
		with open(patterns_file, "r", encoding="utf8") as stream:
		return extract_subset_from_monolithic_flags_as_dict_from_stream(
		monolithicTrie, stream)
		monolithic_trie, stream)


		def extract_subset_from_monolithic_flags_as_dict_from_stream(
		monolithicTrie, stream):
		monolithic_trie, stream):
		"""Extract a subset of flags from the trie of monolithic flags.

		:param monolithicTrie: the trie containing all the monolithic flags.
		:param monolithic_trie: the trie containing all the monolithic flags.
		:param stream: a stream containing a list of signature patterns that define
		the subset.
		:return: the dict from signature to row.
		@@ -68,7 +68,7 @@ def extract_subset_from_monolithic_flags_as_dict_from_stream(
		dict_signature_to_row = {}
		for pattern in stream:
		pattern = pattern.rstrip()
		rows = monolithicTrie.get_matching_rows(pattern)
		rows = monolithic_trie.get_matching_rows(pattern)
		for row in rows:
		signature = row["signature"]
		dict_signature_to_row[signature] = row
		@@ -93,48 +93,48 @@ def read_signature_csv_from_stream_as_dict(stream):
		return dict_signature_to_row


		def read_signature_csv_from_file_as_dict(csvFile):
		def read_signature_csv_from_file_as_dict(csv_file):
		"""Read the csvFile into a dict.

		The first column is assumed to be the signature and used as the
		key.

		The whole row is stored as the value.
		:param csvFile: the csv file to read
		:param csv_file: the csv file to read
		:return: the dict from signature to row.
		"""
		with open(csvFile, "r") as f:
		with open(csv_file, "r", encoding="utf8") as f:
		return read_signature_csv_from_stream_as_dict(f)


		def compare_signature_flags(monolithicFlagsDict, modularFlagsDict):
		def compare_signature_flags(monolithic_flags_dict, modular_flags_dict):
		"""Compare the signature flags between the two dicts.

		:param monolithicFlagsDict: the dict containing the subset of the monolithic
		flags that should be equal to the modular flags.
		:param modularFlagsDict:the dict containing the flags produced by a single
		:param monolithic_flags_dict: the dict containing the subset of the
		monolithic flags that should be equal to the modular flags.
		:param modular_flags_dict:the dict containing the flags produced by a single
		bootclasspath_fragment module.
		:return: list of mismatches., each mismatch is a tuple where the first item
		is the signature, and the second and third items are lists of the flags from
		modular dict, and monolithic dict respectively.
		"""
		mismatchingSignatures = []
		mismatching_signatures = []
		# Create a sorted set of all the signatures from both the monolithic and
		# modular dicts.
		allSignatures = sorted(
		set(chain(monolithicFlagsDict.keys(), modularFlagsDict.keys())))
		for signature in allSignatures:
		monolithicRow = monolithicFlagsDict.get(signature, {})
		monolithicFlags = monolithicRow.get(None, [])
		if signature in modularFlagsDict:
		modularRow = modularFlagsDict.get(signature, {})
		modularFlags = modularRow.get(None, [])
		all_signatures = sorted(
		set(chain(monolithic_flags_dict.keys(), modular_flags_dict.keys())))
		for signature in all_signatures:
		monolithic_row = monolithic_flags_dict.get(signature, {})
		monolithic_flags = monolithic_row.get(None, [])
		if signature in modular_flags_dict:
		modular_row = modular_flags_dict.get(signature, {})
		modular_flags = modular_row.get(None, [])
		else:
		modularFlags = ["blocked"]
		if monolithicFlags != modularFlags:
		mismatchingSignatures.append(
		(signature, modularFlags, monolithicFlags))
		return mismatchingSignatures
		modular_flags = ["blocked"]
		if monolithic_flags != modular_flags:
		mismatching_signatures.append(
		(signature, modular_flags, monolithic_flags))
		return mismatching_signatures


		def main(argv):
		@@ -149,30 +149,30 @@ def main(argv):
		args = args_parser.parse_args(argv[1:])

		# Read in all the flags into the trie
		monolithicFlagsPath = args.monolithicFlags
		monolithicTrie = read_flag_trie_from_file(monolithicFlagsPath)
		monolithic_flags_path = args.monolithicFlags
		monolithic_trie = read_flag_trie_from_file(monolithic_flags_path)

		# For each subset specified on the command line, create dicts for the flags
		# provided by the subset and the corresponding flags from the complete set
		# of flags and compare them.
		failed = False
		for modularPair in args.modularFlags:
		parts = modularPair.split(":")
		modularFlagsPath = parts[0]
		modularPatternsPath = parts[1]
		modularFlagsDict = read_signature_csv_from_file_as_dict(
		modularFlagsPath)
		monolithicFlagsSubsetDict = \
		for modular_pair in args.modularFlags:
		parts = modular_pair.split(":")
		modular_flags_path = parts[0]
		modular_patterns_path = parts[1]
		modular_flags_dict = read_signature_csv_from_file_as_dict(
		modular_flags_path)
		monolithic_flags_subset_dict = \
		extract_subset_from_monolithic_flags_as_dict_from_file(
		monolithicTrie, modularPatternsPath)
		mismatchingSignatures = compare_signature_flags(
		monolithicFlagsSubsetDict, modularFlagsDict)
		if mismatchingSignatures:
		monolithic_trie, modular_patterns_path)
		mismatching_signatures = compare_signature_flags(
		monolithic_flags_subset_dict, modular_flags_dict)
		if mismatching_signatures:
		failed = True
		print("ERROR: Hidden API flags are inconsistent:")
		print("< " + modularFlagsPath)
		print("> " + monolithicFlagsPath)
		for mismatch in mismatchingSignatures:
		print("< " + modular_flags_path)
		print("> " + monolithic_flags_path)
		for mismatch in mismatching_signatures:
		signature = mismatch[0]
		print()
		print("< " + ",".join([signature] + mismatch[1]))

scripts/hiddenapi/verify_overlaps_test.py

+105 −105

Original line number	Diff line number	Diff line
		@@ -17,24 +17,26 @@
		import io
		import unittest

		from verify_overlaps import * #pylint: disable=unused-wildcard-import,wildcard-import
		import verify_overlaps as vo


		#pylint: disable=line-too-long
		class TestDetectOverlaps(unittest.TestCase):

		def read_flag_trie_from_string(self, csvdata):
		@staticmethod
		def read_flag_trie_from_string(csvdata):
		with io.StringIO(csvdata) as f:
		return read_flag_trie_from_stream(f)
		return vo.read_flag_trie_from_stream(f)

		def read_signature_csv_from_string_as_dict(self, csvdata):
		@staticmethod
		def read_signature_csv_from_string_as_dict(csvdata):
		with io.StringIO(csvdata) as f:
		return read_signature_csv_from_stream_as_dict(f)
		return vo.read_signature_csv_from_stream_as_dict(f)

		@staticmethod
		def extract_subset_from_monolithic_flags_as_dict_from_string(
		self, monolithic, patterns):
		monolithic, patterns):
		with io.StringIO(patterns) as f:
		return extract_subset_from_monolithic_flags_as_dict_from_stream(
		return vo.extract_subset_from_monolithic_flags_as_dict_from_stream(
		monolithic, f)

		extractInput = """
		@@ -50,14 +52,14 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
		monolithic = self.read_flag_trie_from_string(
		TestDetectOverlaps.extractInput)

		patterns = 'Ljava/lang/Object;->hashCode()I'
		patterns = "Ljava/lang/Object;->hashCode()I"

		subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(
		monolithic, patterns)
		expected = {
		'Ljava/lang/Object;->hashCode()I': {
		None: ['public-api', 'system-api', 'test-api'],
		'signature': 'Ljava/lang/Object;->hashCode()I',
		"Ljava/lang/Object;->hashCode()I": {
		None: ["public-api", "system-api", "test-api"],
		"signature": "Ljava/lang/Object;->hashCode()I",
		},
		}
		self.assertEqual(expected, subset)
		@@ -66,18 +68,18 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
		monolithic = self.read_flag_trie_from_string(
		TestDetectOverlaps.extractInput)

		patterns = 'java/lang/Object'
		patterns = "java/lang/Object"

		subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(
		monolithic, patterns)
		expected = {
		'Ljava/lang/Object;->hashCode()I': {
		None: ['public-api', 'system-api', 'test-api'],
		'signature': 'Ljava/lang/Object;->hashCode()I',
		"Ljava/lang/Object;->hashCode()I": {
		None: ["public-api", "system-api", "test-api"],
		"signature": "Ljava/lang/Object;->hashCode()I",
		},
		'Ljava/lang/Object;->toString()Ljava/lang/String;': {
		None: ['blocked'],
		'signature': 'Ljava/lang/Object;->toString()Ljava/lang/String;',
		"Ljava/lang/Object;->toString()Ljava/lang/String;": {
		None: ["blocked"],
		"signature": "Ljava/lang/Object;->toString()Ljava/lang/String;",
		},
		}
		self.assertEqual(expected, subset)
		@@ -86,20 +88,20 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
		monolithic = self.read_flag_trie_from_string(
		TestDetectOverlaps.extractInput)

		patterns = 'java/lang/Character'
		patterns = "java/lang/Character"

		subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(
		monolithic, patterns)
		expected = {
		'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;':
		{
		None: ['blocked'],
		'signature':
		'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;',
		"Ljava/lang/Character$UnicodeScript;"
		"->of(I)Ljava/lang/Character$UnicodeScript;": {
		None: ["blocked"],
		"signature": "Ljava/lang/Character$UnicodeScript;"
		"->of(I)Ljava/lang/Character$UnicodeScript;",
		},
		'Ljava/lang/Character;->serialVersionUID:J': {
		None: ['sdk'],
		'signature': 'Ljava/lang/Character;->serialVersionUID:J',
		"Ljava/lang/Character;->serialVersionUID:J": {
		None: ["sdk"],
		"signature": "Ljava/lang/Character;->serialVersionUID:J",
		},
		}
		self.assertEqual(expected, subset)
		@@ -108,16 +110,16 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
		monolithic = self.read_flag_trie_from_string(
		TestDetectOverlaps.extractInput)

		patterns = 'java/lang/Character$UnicodeScript'
		patterns = "java/lang/Character$UnicodeScript"

		subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(
		monolithic, patterns)
		expected = {
		'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;':
		{
		None: ['blocked'],
		'signature':
		'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;',
		"Ljava/lang/Character$UnicodeScript;"
		"->of(I)Ljava/lang/Character$UnicodeScript;": {
		None: ["blocked"],
		"signature": "Ljava/lang/Character$UnicodeScript;"
		"->of(I)Ljava/lang/Character$UnicodeScript;",
		},
		}
		self.assertEqual(expected, subset)
		@@ -126,32 +128,32 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
		monolithic = self.read_flag_trie_from_string(
		TestDetectOverlaps.extractInput)

		patterns = 'java/lang/*'
		patterns = "java/lang/*"

		subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(
		monolithic, patterns)
		expected = {
		'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;':
		{
		None: ['blocked'],
		'signature':
		'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;',
		"Ljava/lang/Character$UnicodeScript;"
		"->of(I)Ljava/lang/Character$UnicodeScript;": {
		None: ["blocked"],
		"signature": "Ljava/lang/Character$UnicodeScript;"
		"->of(I)Ljava/lang/Character$UnicodeScript;",
		},
		'Ljava/lang/Character;->serialVersionUID:J': {
		None: ['sdk'],
		'signature': 'Ljava/lang/Character;->serialVersionUID:J',
		"Ljava/lang/Character;->serialVersionUID:J": {
		None: ["sdk"],
		"signature": "Ljava/lang/Character;->serialVersionUID:J",
		},
		'Ljava/lang/Object;->hashCode()I': {
		None: ['public-api', 'system-api', 'test-api'],
		'signature': 'Ljava/lang/Object;->hashCode()I',
		"Ljava/lang/Object;->hashCode()I": {
		None: ["public-api", "system-api", "test-api"],
		"signature": "Ljava/lang/Object;->hashCode()I",
		},
		'Ljava/lang/Object;->toString()Ljava/lang/String;': {
		None: ['blocked'],
		'signature': 'Ljava/lang/Object;->toString()Ljava/lang/String;',
		"Ljava/lang/Object;->toString()Ljava/lang/String;": {
		None: ["blocked"],
		"signature": "Ljava/lang/Object;->toString()Ljava/lang/String;",
		},
		'Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V': {
		None: ['blocked'],
		'signature': 'Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V',
		"Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V": {
		None: ["blocked"],
		"signature": "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V",
		},
		}
		self.assertEqual(expected, subset)
		@@ -160,36 +162,36 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
		monolithic = self.read_flag_trie_from_string(
		TestDetectOverlaps.extractInput)

		patterns = 'java/**'
		patterns = "java/**"

		subset = self.extract_subset_from_monolithic_flags_as_dict_from_string(
		monolithic, patterns)
		expected = {
		'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;':
		{
		None: ['blocked'],
		'signature':
		'Ljava/lang/Character$UnicodeScript;->of(I)Ljava/lang/Character$UnicodeScript;',
		"Ljava/lang/Character$UnicodeScript;"
		"->of(I)Ljava/lang/Character$UnicodeScript;": {
		None: ["blocked"],
		"signature": "Ljava/lang/Character$UnicodeScript;"
		"->of(I)Ljava/lang/Character$UnicodeScript;",
		},
		'Ljava/lang/Character;->serialVersionUID:J': {
		None: ['sdk'],
		'signature': 'Ljava/lang/Character;->serialVersionUID:J',
		"Ljava/lang/Character;->serialVersionUID:J": {
		None: ["sdk"],
		"signature": "Ljava/lang/Character;->serialVersionUID:J",
		},
		'Ljava/lang/Object;->hashCode()I': {
		None: ['public-api', 'system-api', 'test-api'],
		'signature': 'Ljava/lang/Object;->hashCode()I',
		"Ljava/lang/Object;->hashCode()I": {
		None: ["public-api", "system-api", "test-api"],
		"signature": "Ljava/lang/Object;->hashCode()I",
		},
		'Ljava/lang/Object;->toString()Ljava/lang/String;': {
		None: ['blocked'],
		'signature': 'Ljava/lang/Object;->toString()Ljava/lang/String;',
		"Ljava/lang/Object;->toString()Ljava/lang/String;": {
		None: ["blocked"],
		"signature": "Ljava/lang/Object;->toString()Ljava/lang/String;",
		},
		'Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V': {
		None: ['blocked'],
		'signature': 'Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V',
		"Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V": {
		None: ["blocked"],
		"signature": "Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V",
		},
		'Ljava/util/zip/ZipFile;-><clinit>()V': {
		None: ['blocked'],
		'signature': 'Ljava/util/zip/ZipFile;-><clinit>()V',
		"Ljava/util/zip/ZipFile;-><clinit>()V": {
		None: ["blocked"],
		"signature": "Ljava/util/zip/ZipFile;-><clinit>()V",
		},
		}
		self.assertEqual(expected, subset)
		@@ -200,7 +202,7 @@ Ljava/lang/ProcessBuilder$Redirect$1;-><init>()V,blocked
		Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
		Ljava/lang/Object;->hashCode()I,blocked
		""")
		self.assertTrue('Duplicate signature: Ljava/lang/Object;->hashCode()I'
		self.assertTrue("Duplicate signature: Ljava/lang/Object;->hashCode()I"
		in str(context.exception))

		def test_read_trie_missing_member(self):
		@@ -209,8 +211,8 @@ Ljava/lang/Object;->hashCode()I,blocked
		Ljava/lang/Object,public-api,system-api,test-api
		""")
		self.assertTrue(
		'Invalid signature: Ljava/lang/Object, does not identify a specific member'
		in str(context.exception))
		"Invalid signature: Ljava/lang/Object, "
		"does not identify a specific member" in str(context.exception))

		def test_match(self):
		monolithic = self.read_signature_csv_from_string_as_dict("""
		@@ -219,7 +221,7 @@ Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
		modular = self.read_signature_csv_from_string_as_dict("""
		Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
		""")
		mismatches = compare_signature_flags(monolithic, modular)
		mismatches = vo.compare_signature_flags(monolithic, modular)
		expected = []
		self.assertEqual(expected, mismatches)

		@@ -230,12 +232,12 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,public-api
		modular = self.read_signature_csv_from_string_as_dict("""
		Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api
		""")
		mismatches = compare_signature_flags(monolithic, modular)
		mismatches = vo.compare_signature_flags(monolithic, modular)
		expected = [
		(
		'Ljava/lang/Object;->toString()Ljava/lang/String;',
		['public-api', 'system-api', 'test-api'],
		['public-api'],
		"Ljava/lang/Object;->toString()Ljava/lang/String;",
		["public-api", "system-api", "test-api"],
		["public-api"],
		),
		]
		self.assertEqual(expected, mismatches)
		@@ -247,12 +249,12 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,blocked
		modular = self.read_signature_csv_from_string_as_dict("""
		Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api
		""")
		mismatches = compare_signature_flags(monolithic, modular)
		mismatches = vo.compare_signature_flags(monolithic, modular)
		expected = [
		(
		'Ljava/lang/Object;->toString()Ljava/lang/String;',
		['public-api', 'system-api', 'test-api'],
		['blocked'],
		"Ljava/lang/Object;->toString()Ljava/lang/String;",
		["public-api", "system-api", "test-api"],
		["blocked"],
		),
		]
		self.assertEqual(expected, mismatches)
		@@ -264,26 +266,26 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api
		modular = self.read_signature_csv_from_string_as_dict("""
		Ljava/lang/Object;->toString()Ljava/lang/String;,blocked
		""")
		mismatches = compare_signature_flags(monolithic, modular)
		mismatches = vo.compare_signature_flags(monolithic, modular)
		expected = [
		(
		'Ljava/lang/Object;->toString()Ljava/lang/String;',
		['blocked'],
		['public-api', 'system-api', 'test-api'],
		"Ljava/lang/Object;->toString()Ljava/lang/String;",
		["blocked"],
		["public-api", "system-api", "test-api"],
		),
		]
		self.assertEqual(expected, mismatches)

		def test_match_treat_missing_from_modular_as_blocked(self):
		monolithic = self.read_signature_csv_from_string_as_dict('')
		monolithic = self.read_signature_csv_from_string_as_dict("")
		modular = self.read_signature_csv_from_string_as_dict("""
		Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api
		""")
		mismatches = compare_signature_flags(monolithic, modular)
		mismatches = vo.compare_signature_flags(monolithic, modular)
		expected = [
		(
		'Ljava/lang/Object;->toString()Ljava/lang/String;',
		['public-api', 'system-api', 'test-api'],
		"Ljava/lang/Object;->toString()Ljava/lang/String;",
		["public-api", "system-api", "test-api"],
		[],
		),
		]
		@@ -294,12 +296,12 @@ Ljava/lang/Object;->toString()Ljava/lang/String;,public-api,system-api,test-api
		Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
		""")
		modular = {}
		mismatches = compare_signature_flags(monolithic, modular)
		mismatches = vo.compare_signature_flags(monolithic, modular)
		expected = [
		(
		'Ljava/lang/Object;->hashCode()I',
		['blocked'],
		['public-api', 'system-api', 'test-api'],
		"Ljava/lang/Object;->hashCode()I",
		["blocked"],
		["public-api", "system-api", "test-api"],
		),
		]
		self.assertEqual(expected, mismatches)
		@@ -309,12 +311,10 @@ Ljava/lang/Object;->hashCode()I,public-api,system-api,test-api
		Ljava/lang/Object;->hashCode()I,blocked
		""")
		modular = {}
		mismatches = compare_signature_flags(monolithic, modular)
		mismatches = vo.compare_signature_flags(monolithic, modular)
		expected = []
		self.assertEqual(expected, mismatches)


		#pylint: enable=line-too-long

		if __name__ == '__main__':
		if __name__ == "__main__":
		unittest.main(verbosity=2)