Loading Android.mk +28 −125 Original line number Diff line number Diff line Loading @@ -322,131 +322,34 @@ $(OUT_DOCS)/offline-sdk-timestamp: $(OUT_DOCS)/offline-sdk-docs-docs.zip ( unzip -qo $< -d $(OUT_DOCS)/offline-sdk && touch -f $@ ) || exit 1 # ==== hiddenapi lists ======================================= include $(CLEAR_VARS) # File names of final API lists LOCAL_WHITELIST := $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST) LOCAL_LIGHT_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) LOCAL_DARK_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) LOCAL_BLACKLIST := $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) # File names of source files we will use to generate the final API lists. LOCAL_SRC_GREYLIST := frameworks/base/config/hiddenapi-light-greylist.txt LOCAL_SRC_VENDOR_LIST := frameworks/base/config/hiddenapi-vendor-list.txt LOCAL_SRC_FORCE_BLACKLIST := frameworks/base/config/hiddenapi-force-blacklist.txt LOCAL_SRC_PUBLIC_API := $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) LOCAL_SRC_PRIVATE_API := $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) LOCAL_SRC_REMOVED_API := $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) LOCAL_SRC_ALL := \ $(LOCAL_SRC_GREYLIST) \ $(LOCAL_SRC_VENDOR_LIST) \ $(LOCAL_SRC_FORCE_BLACKLIST) \ $(LOCAL_SRC_PUBLIC_API) \ $(LOCAL_SRC_PRIVATE_API) \ $(LOCAL_SRC_REMOVED_API) define assert-has-no-overlap if [ ! -z "`comm -12 <(sort $(1)) <(sort $(2))`" ]; then \ echo "$(1) and $(2) should not overlap" 1>&2; \ comm -12 <(sort $(1)) <(sort $(2)) 1>&2; \ exit 1; \ fi endef define assert-is-subset if [ ! -z "`comm -23 <(sort $(1)) <(sort $(2))`" ]; then \ echo "$(1) must be a subset of $(2)" 1>&2; \ comm -23 <(sort $(1)) <(sort $(2)) 1>&2; \ exit 1; \ fi endef define assert-has-no-duplicates if [ ! -z "`sort $(1) | uniq -D`" ]; then \ echo "$(1) has duplicate entries" 1>&2; \ sort $(1) | uniq -D 1>&2; \ exit 1; \ fi endef # The following rules build API lists in the build folder. # By not using files from the source tree, ART buildbots can mock these lists # or have alternative rules for building them. Other rules in the build system # should depend on the files in the build folder. # Merge whitelist from: # (1) public API stubs # (2) whitelist entries generated by class2greylist (PRIVATE_WHITELIST_INPUTS) $(LOCAL_WHITELIST): $(LOCAL_SRC_PUBLIC_API) sort $(LOCAL_SRC_PUBLIC_API) $(PRIVATE_WHITELIST_INPUTS) > $@ $(call assert-has-no-duplicates,$@) # Merge light greylist from multiple files: # (1) manual greylist LOCAL_SRC_GREYLIST # (2) list of usages from vendor apps LOCAL_SRC_VENDOR_LIST # (3) list of removed APIs in LOCAL_SRC_REMOVED_API # @removed does not imply private in Doclava. We must take the subset also # in LOCAL_SRC_PRIVATE_API. # (4) list of serialization APIs # Automatically adds all methods which match the signatures in # REGEX_SERIALIZATION. These are greylisted in order to allow applications # to write their own serializers. # (5) greylist entries generated by class2greylist (PRIVATE_GREYLIST_INPUTS) $(LOCAL_LIGHT_GREYLIST): REGEX_SERIALIZATION := \ "readObject\(Ljava/io/ObjectInputStream;\)V" \ "readObjectNoData\(\)V" \ "readResolve\(\)Ljava/lang/Object;" \ "serialVersionUID:J" \ "serialPersistentFields:\[Ljava/io/ObjectStreamField;" \ "writeObject\(Ljava/io/ObjectOutputStream;\)V" \ "writeReplace\(\)Ljava/lang/Object;" $(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) sort $(LOCAL_SRC_GREYLIST) $(LOCAL_SRC_VENDOR_LIST) $(PRIVATE_GREYLIST_INPUTS) \ <(grep -E "\->("$(subst $(space),"|",$(REGEX_SERIALIZATION))")$$" \ $(LOCAL_SRC_PRIVATE_API)) \ <(comm -12 <(sort $(LOCAL_SRC_REMOVED_API)) <(sort $(LOCAL_SRC_PRIVATE_API))) \ > $@ $(call assert-has-no-duplicates,$@) $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) $(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST)) $(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST)) # Generate dark greylist as remaining classes and class members in the same # package as classes listed in the light greylist. # The algorithm is as follows: # (1) extract the class descriptor from each entry in LOCAL_LIGHT_GREYLIST # (2) strip everything after the last forward-slash, # e.g. 'Lpackage/subpackage/class$inner;' turns into 'Lpackage/subpackage/' # (3) insert all entries from LOCAL_SRC_PRIVATE_API which begin with the package # name but do not contain another forward-slash in the class name, e.g. # matching '^Lpackage/subpackage/[^/;]*;' # (4) subtract entries shared with LOCAL_LIGHT_GREYLIST $(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) comm -13 <(sort $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_SRC_FORCE_BLACKLIST)) \ <(cat $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) | \ sed 's/\->.*//' | sed 's/\(.*\/\).*/\1/' | sort | uniq | \ while read PKG_NAME; do \ grep -E "^$${PKG_NAME}[^/;]*;" $(LOCAL_SRC_PRIVATE_API); \ done | sort | uniq) \ > $@ $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) $(call assert-has-no-duplicates,$@) $(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST)) $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) $(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST)) # Generate blacklist as private API minus (light greylist plus dark greylist). $(LOCAL_BLACKLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST) comm -13 <(sort $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST)) \ <(sort $(LOCAL_SRC_PRIVATE_API)) \ > $@ $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) $(call assert-has-no-duplicates,$@) $(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST)) $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) $(call assert-has-no-overlap,$@,$(LOCAL_DARK_GREYLIST)) $(call assert-is-subset,$(LOCAL_SRC_FORCE_BLACKLIST),$@) $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \ .KATI_IMPLICIT_OUTPUTS := \ $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \ $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \ $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \ frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \ frameworks/base/config/hiddenapi-light-greylist.txt \ frameworks/base/config/hiddenapi-vendor-list.txt \ frameworks/base/config/hiddenapi-force-blacklist.txt \ $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \ $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \ $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \ --input-public $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \ --input-private $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \ --input-whitelists $(PRIVATE_WHITELIST_INPUTS) \ --input-greylists \ frameworks/base/config/hiddenapi-light-greylist.txt \ frameworks/base/config/hiddenapi-vendor-list.txt \ <(comm -12 <(sort $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)) \ $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST)) \ $(PRIVATE_GREYLIST_INPUTS) \ --input-blacklists frameworks/base/config/hiddenapi-force-blacklist.txt \ --output-whitelist $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST) \ --output-light-greylist $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \ --output-dark-greylist $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \ --output-blacklist $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) # Include subdirectory makefiles # ============================================================ Loading tools/hiddenapi/generate_hiddenapi_lists.py 0 → 100755 +241 −0 Original line number Diff line number Diff line #!/usr/bin/env python # # Copyright (C) 2018 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Generate API lists for non-SDK API enforcement. usage: generate-hiddenapi-lists.py [-h] --input-public INPUT_PUBLIC --input-private INPUT_PRIVATE [--input-whitelists [INPUT_WHITELISTS [INPUT_WHITELISTS ...]]] [--input-greylists [INPUT_GREYLISTS [INPUT_GREYLISTS ...]]] [--input-blacklists [INPUT_BLACKLISTS [INPUT_BLACKLISTS ...]]] --output-whitelist OUTPUT_WHITELIST --output-light-greylist OUTPUT_LIGHT_GREYLIST --output-dark-greylist OUTPUT_DARK_GREYLIST --output-blacklist OUTPUT_BLACKLIST """ import argparse import os import sys import re def get_args(): """Parses command line arguments. Returns: Namespace: dictionary of parsed arguments """ parser = argparse.ArgumentParser() parser.add_argument('--input-public', required=True, help='List of all public members') parser.add_argument('--input-private', required=True, help='List of all private members') parser.add_argument( '--input-whitelists', nargs='*', help='Lists of members to force on whitelist') parser.add_argument( '--input-greylists', nargs='*', help='Lists of members to force on light greylist') parser.add_argument( '--input-blacklists', nargs='*', help='Lists of members to force on blacklist') parser.add_argument('--output-whitelist', required=True) parser.add_argument('--output-light-greylist', required=True) parser.add_argument('--output-dark-greylist', required=True) parser.add_argument('--output-blacklist', required=True) return parser.parse_args() def read_lines(filename): """Reads entire file and return it as a list of lines. Args: filename (string): Path to the file to read from. Returns: list: Lines of the loaded file as a list of strings. """ with open(filename, 'r') as f: return f.readlines() def write_lines(filename, lines): """Writes list of lines into a file, overwriting the file it it exists. Args: filename (string): Path to the file to be writting into. lines (list): List of strings to write into the file. """ with open(filename, 'w') as f: f.writelines(lines) def move_between_sets(subset, src, dst, source = "<unknown>"): """Removes a subset of elements from one set and add it to another. Args: subset (set): The subset of `src` to be moved from `src` to `dst`. src (set): Source set. Must be a superset of `subset`. dst (set): Destination set. Must be disjoint with `subset`. """ assert src.issuperset(subset), ( "Error processing: {}\n" "The following entries were not found:\n" "{}" "Please visit go/hiddenapi for more information.").format( source, "".join(map(lambda x: " " + str(x), subset.difference(src)))) assert dst.isdisjoint(subset) # Order matters if `src` and `subset` are the same object. dst.update(subset) src.difference_update(subset) def get_package_name(signature): """Returns the package name prefix of a class member signature. Example: "Ljava/lang/String;->hashCode()J" --> "Ljava/lang/" Args: signature (string): Member signature Returns string: Package name of the given member """ class_name_end = signature.find("->") assert class_name_end != -1, "Invalid signature: {}".format(signature) package_name_end = signature.rfind("/", 0, class_name_end) assert package_name_end != -1, "Invalid signature: {}".format(signature) return signature[:package_name_end + 1] def all_package_names(*args): """Returns a set of packages names in given lists of member signatures. Example: args = [ set([ "Lpkg1/ClassA;->foo()V", "Lpkg2/ClassB;->bar()J" ]), set([ "Lpkg1/ClassC;->baz()Z" ]) ] return value = set([ "Lpkg1/", "Lpkg2" ]) Args: *args (list): List of sets to iterate over and extract the package names of its elements (member signatures) Returns: set: All package names extracted from the given lists of signatures. """ packages = set() for arg in args: packages = packages.union(map(get_package_name, arg)) return packages def move_all(src, dst): """Moves all elements of one set to another. Args: src (set): Source set. Will become empty. dst (set): Destination set. Will contain all elements of `src`. """ move_between_sets(src, src, dst) def move_from_files(filenames, src, dst): """Loads member signatures from a list of files and moves them to a given set. Opens files in `filenames`, reads all their lines and moves those from `src` set to `dst` set. Args: filenames (list): List of paths to files to be loaded. src (set): Set that loaded lines should be moved from. dst (set): Set that loaded lines should be moved to. """ if filenames: for filename in filenames: move_between_sets(set(read_lines(filename)), src, dst, filename) def move_serialization(src, dst): """Moves all members matching serialization API signatures between given sets. Args: src (set): Set that will be searched for serialization API and that API will be removed from it. dst (set): Set that serialization API will be moved to. """ serialization_patterns = [ r'readObject\(Ljava/io/ObjectInputStream;\)V', r'readObjectNoData\(\)V', r'readResolve\(\)Ljava/lang/Object;', r'serialVersionUID:J', r'serialPersistentFields:\[Ljava/io/ObjectStreamField;', r'writeObject\(Ljava/io/ObjectOutputStream;\)V', r'writeReplace\(\)Ljava/lang/Object;', ] regex = re.compile(r'.*->(' + '|'.join(serialization_patterns) + r')$') move_between_sets(filter(lambda api: regex.match(api), src), src, dst) def move_from_packages(packages, src, dst): """Moves all members of given package names from one set to another. Args: packages (list): List of string package names. src (set): Set that will be searched for API matching one of the given package names. Surch API will be removed from the set. dst (set): Set that matching API will be moved to. """ move_between_sets(filter(lambda api: get_package_name(api) in packages, src), src, dst) def main(argv): args = get_args() # Initialize API sets by loading lists of public and private API. Public API # are all members resolvable from SDK API stubs, other members are private. # As an optimization, skip the step of moving public API from a full set of # members and start with a populated whitelist. whitelist = set(read_lines(args.input_public)) uncategorized = set(read_lines(args.input_private)) light_greylist = set() dark_greylist = set() blacklist = set() # Assert that there is no overlap between public and private API. assert whitelist.isdisjoint(uncategorized) num_all_api = len(whitelist) + len(uncategorized) # Read all files which manually assign members to specific lists. move_from_files(args.input_whitelists, uncategorized, whitelist) move_from_files(args.input_greylists, uncategorized, light_greylist) move_from_files(args.input_blacklists, uncategorized, blacklist) # Iterate over all uncategorized members and move serialization API to light greylist. move_serialization(uncategorized, light_greylist) # Extract package names of members from whitelist and light greylist, which # are assumed to have been finalized at this point. Assign all uncategorized # members from the same packages to the dark greylist. dark_greylist_packages = all_package_names(whitelist, light_greylist) move_from_packages(dark_greylist_packages, uncategorized, dark_greylist) # Assign all uncategorized members to the blacklist. move_all(uncategorized, blacklist) # Assert we have not missed anything. assert whitelist.isdisjoint(light_greylist) assert whitelist.isdisjoint(dark_greylist) assert whitelist.isdisjoint(blacklist) assert light_greylist.isdisjoint(dark_greylist) assert light_greylist.isdisjoint(blacklist) assert dark_greylist.isdisjoint(blacklist) assert num_all_api == len(whitelist) + len(light_greylist) + len(dark_greylist) + len(blacklist) # Write final lists to disk. write_lines(args.output_whitelist, whitelist) write_lines(args.output_light_greylist, light_greylist) write_lines(args.output_dark_greylist, dark_greylist) write_lines(args.output_blacklist, blacklist) if __name__ == "__main__": main(sys.argv) tools/hiddenapi/generate_hiddenapi_lists_test.py 0 → 100755 +89 −0 Original line number Diff line number Diff line #!/usr/bin/env python # # Copyright (C) 2018 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for Hidden API list generation.""" import unittest from generate_hiddenapi_lists import * class TestHiddenapiListGeneration(unittest.TestCase): def test_move_between_sets(self): A = set([1, 2, 3, 4]) B = set([5, 6, 7, 8]) move_between_sets(set([2, 4]), A, B) self.assertEqual(A, set([1, 3])) self.assertEqual(B, set([2, 4, 5, 6, 7, 8])) def test_move_between_sets_fail_not_superset(self): A = set([1, 2, 3, 4]) B = set([5, 6, 7, 8]) with self.assertRaises(AssertionError) as ar: move_between_sets(set([0, 2]), A, B) def test_move_between_sets_fail_not_disjoint(self): A = set([1, 2, 3, 4]) B = set([4, 5, 6, 7, 8]) with self.assertRaises(AssertionError) as ar: move_between_sets(set([1, 4]), A, B) def test_get_package_name(self): self.assertEqual(get_package_name("Ljava/lang/String;->clone()V"), "Ljava/lang/") def test_get_package_name_fail_no_arrow(self): with self.assertRaises(AssertionError) as ar: get_package_name("Ljava/lang/String;-clone()V") with self.assertRaises(AssertionError) as ar: get_package_name("Ljava/lang/String;>clone()V") with self.assertRaises(AssertionError) as ar: get_package_name("Ljava/lang/String;__clone()V") def test_get_package_name_fail_no_package(self): with self.assertRaises(AssertionError) as ar: get_package_name("LString;->clone()V") def test_all_package_names(self): self.assertEqual(all_package_names(), set()) self.assertEqual(all_package_names(set(["Lfoo/Bar;->baz()V"])), set(["Lfoo/"])) self.assertEqual( all_package_names(set(["Lfoo/Bar;->baz()V", "Lfoo/BarX;->bazx()I"])), set(["Lfoo/"])) self.assertEqual( all_package_names( set(["Lfoo/Bar;->baz()V"]), set(["Lfoo/BarX;->bazx()I", "Labc/xyz/Mno;->ijk()J"])), set(["Lfoo/", "Labc/xyz/"])) def test_move_all(self): src = set([ "abc", "xyz" ]) dst = set([ "def" ]) move_all(src, dst) self.assertEqual(src, set()) self.assertEqual(dst, set([ "abc", "def", "xyz" ])) def test_move_from_packages(self): src = set([ "Lfoo/bar/ClassA;->abc()J", # will be moved "Lfoo/bar/ClassA;->def()J", # will be moved "Lcom/pkg/example/ClassD;->ijk:J", # not moved: different package "Lfoo/bar/xyz/ClassC;->xyz()Z" ]) # not moved: subpackage dst = set() packages = set([ "Lfoo/bar/" ]) move_from_packages(packages, src, dst) self.assertEqual( src, set([ "Lfoo/bar/xyz/ClassC;->xyz()Z", "Lcom/pkg/example/ClassD;->ijk:J" ])) self.assertEqual( dst, set([ "Lfoo/bar/ClassA;->abc()J", "Lfoo/bar/ClassA;->def()J" ])) if __name__ == '__main__': unittest.main() Loading
Android.mk +28 −125 Original line number Diff line number Diff line Loading @@ -322,131 +322,34 @@ $(OUT_DOCS)/offline-sdk-timestamp: $(OUT_DOCS)/offline-sdk-docs-docs.zip ( unzip -qo $< -d $(OUT_DOCS)/offline-sdk && touch -f $@ ) || exit 1 # ==== hiddenapi lists ======================================= include $(CLEAR_VARS) # File names of final API lists LOCAL_WHITELIST := $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST) LOCAL_LIGHT_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) LOCAL_DARK_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) LOCAL_BLACKLIST := $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) # File names of source files we will use to generate the final API lists. LOCAL_SRC_GREYLIST := frameworks/base/config/hiddenapi-light-greylist.txt LOCAL_SRC_VENDOR_LIST := frameworks/base/config/hiddenapi-vendor-list.txt LOCAL_SRC_FORCE_BLACKLIST := frameworks/base/config/hiddenapi-force-blacklist.txt LOCAL_SRC_PUBLIC_API := $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) LOCAL_SRC_PRIVATE_API := $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) LOCAL_SRC_REMOVED_API := $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) LOCAL_SRC_ALL := \ $(LOCAL_SRC_GREYLIST) \ $(LOCAL_SRC_VENDOR_LIST) \ $(LOCAL_SRC_FORCE_BLACKLIST) \ $(LOCAL_SRC_PUBLIC_API) \ $(LOCAL_SRC_PRIVATE_API) \ $(LOCAL_SRC_REMOVED_API) define assert-has-no-overlap if [ ! -z "`comm -12 <(sort $(1)) <(sort $(2))`" ]; then \ echo "$(1) and $(2) should not overlap" 1>&2; \ comm -12 <(sort $(1)) <(sort $(2)) 1>&2; \ exit 1; \ fi endef define assert-is-subset if [ ! -z "`comm -23 <(sort $(1)) <(sort $(2))`" ]; then \ echo "$(1) must be a subset of $(2)" 1>&2; \ comm -23 <(sort $(1)) <(sort $(2)) 1>&2; \ exit 1; \ fi endef define assert-has-no-duplicates if [ ! -z "`sort $(1) | uniq -D`" ]; then \ echo "$(1) has duplicate entries" 1>&2; \ sort $(1) | uniq -D 1>&2; \ exit 1; \ fi endef # The following rules build API lists in the build folder. # By not using files from the source tree, ART buildbots can mock these lists # or have alternative rules for building them. Other rules in the build system # should depend on the files in the build folder. # Merge whitelist from: # (1) public API stubs # (2) whitelist entries generated by class2greylist (PRIVATE_WHITELIST_INPUTS) $(LOCAL_WHITELIST): $(LOCAL_SRC_PUBLIC_API) sort $(LOCAL_SRC_PUBLIC_API) $(PRIVATE_WHITELIST_INPUTS) > $@ $(call assert-has-no-duplicates,$@) # Merge light greylist from multiple files: # (1) manual greylist LOCAL_SRC_GREYLIST # (2) list of usages from vendor apps LOCAL_SRC_VENDOR_LIST # (3) list of removed APIs in LOCAL_SRC_REMOVED_API # @removed does not imply private in Doclava. We must take the subset also # in LOCAL_SRC_PRIVATE_API. # (4) list of serialization APIs # Automatically adds all methods which match the signatures in # REGEX_SERIALIZATION. These are greylisted in order to allow applications # to write their own serializers. # (5) greylist entries generated by class2greylist (PRIVATE_GREYLIST_INPUTS) $(LOCAL_LIGHT_GREYLIST): REGEX_SERIALIZATION := \ "readObject\(Ljava/io/ObjectInputStream;\)V" \ "readObjectNoData\(\)V" \ "readResolve\(\)Ljava/lang/Object;" \ "serialVersionUID:J" \ "serialPersistentFields:\[Ljava/io/ObjectStreamField;" \ "writeObject\(Ljava/io/ObjectOutputStream;\)V" \ "writeReplace\(\)Ljava/lang/Object;" $(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) sort $(LOCAL_SRC_GREYLIST) $(LOCAL_SRC_VENDOR_LIST) $(PRIVATE_GREYLIST_INPUTS) \ <(grep -E "\->("$(subst $(space),"|",$(REGEX_SERIALIZATION))")$$" \ $(LOCAL_SRC_PRIVATE_API)) \ <(comm -12 <(sort $(LOCAL_SRC_REMOVED_API)) <(sort $(LOCAL_SRC_PRIVATE_API))) \ > $@ $(call assert-has-no-duplicates,$@) $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) $(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST)) $(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST)) # Generate dark greylist as remaining classes and class members in the same # package as classes listed in the light greylist. # The algorithm is as follows: # (1) extract the class descriptor from each entry in LOCAL_LIGHT_GREYLIST # (2) strip everything after the last forward-slash, # e.g. 'Lpackage/subpackage/class$inner;' turns into 'Lpackage/subpackage/' # (3) insert all entries from LOCAL_SRC_PRIVATE_API which begin with the package # name but do not contain another forward-slash in the class name, e.g. # matching '^Lpackage/subpackage/[^/;]*;' # (4) subtract entries shared with LOCAL_LIGHT_GREYLIST $(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) comm -13 <(sort $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_SRC_FORCE_BLACKLIST)) \ <(cat $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) | \ sed 's/\->.*//' | sed 's/\(.*\/\).*/\1/' | sort | uniq | \ while read PKG_NAME; do \ grep -E "^$${PKG_NAME}[^/;]*;" $(LOCAL_SRC_PRIVATE_API); \ done | sort | uniq) \ > $@ $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) $(call assert-has-no-duplicates,$@) $(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST)) $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) $(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST)) # Generate blacklist as private API minus (light greylist plus dark greylist). $(LOCAL_BLACKLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST) comm -13 <(sort $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST)) \ <(sort $(LOCAL_SRC_PRIVATE_API)) \ > $@ $(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API)) $(call assert-has-no-duplicates,$@) $(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST)) $(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST)) $(call assert-has-no-overlap,$@,$(LOCAL_DARK_GREYLIST)) $(call assert-is-subset,$(LOCAL_SRC_FORCE_BLACKLIST),$@) $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \ .KATI_IMPLICIT_OUTPUTS := \ $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \ $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \ $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \ frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \ frameworks/base/config/hiddenapi-light-greylist.txt \ frameworks/base/config/hiddenapi-vendor-list.txt \ frameworks/base/config/hiddenapi-force-blacklist.txt \ $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \ $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \ $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE) frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \ --input-public $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \ --input-private $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \ --input-whitelists $(PRIVATE_WHITELIST_INPUTS) \ --input-greylists \ frameworks/base/config/hiddenapi-light-greylist.txt \ frameworks/base/config/hiddenapi-vendor-list.txt \ <(comm -12 <(sort $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)) \ $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST)) \ $(PRIVATE_GREYLIST_INPUTS) \ --input-blacklists frameworks/base/config/hiddenapi-force-blacklist.txt \ --output-whitelist $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST) \ --output-light-greylist $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \ --output-dark-greylist $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \ --output-blacklist $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST) # Include subdirectory makefiles # ============================================================ Loading
tools/hiddenapi/generate_hiddenapi_lists.py 0 → 100755 +241 −0 Original line number Diff line number Diff line #!/usr/bin/env python # # Copyright (C) 2018 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Generate API lists for non-SDK API enforcement. usage: generate-hiddenapi-lists.py [-h] --input-public INPUT_PUBLIC --input-private INPUT_PRIVATE [--input-whitelists [INPUT_WHITELISTS [INPUT_WHITELISTS ...]]] [--input-greylists [INPUT_GREYLISTS [INPUT_GREYLISTS ...]]] [--input-blacklists [INPUT_BLACKLISTS [INPUT_BLACKLISTS ...]]] --output-whitelist OUTPUT_WHITELIST --output-light-greylist OUTPUT_LIGHT_GREYLIST --output-dark-greylist OUTPUT_DARK_GREYLIST --output-blacklist OUTPUT_BLACKLIST """ import argparse import os import sys import re def get_args(): """Parses command line arguments. Returns: Namespace: dictionary of parsed arguments """ parser = argparse.ArgumentParser() parser.add_argument('--input-public', required=True, help='List of all public members') parser.add_argument('--input-private', required=True, help='List of all private members') parser.add_argument( '--input-whitelists', nargs='*', help='Lists of members to force on whitelist') parser.add_argument( '--input-greylists', nargs='*', help='Lists of members to force on light greylist') parser.add_argument( '--input-blacklists', nargs='*', help='Lists of members to force on blacklist') parser.add_argument('--output-whitelist', required=True) parser.add_argument('--output-light-greylist', required=True) parser.add_argument('--output-dark-greylist', required=True) parser.add_argument('--output-blacklist', required=True) return parser.parse_args() def read_lines(filename): """Reads entire file and return it as a list of lines. Args: filename (string): Path to the file to read from. Returns: list: Lines of the loaded file as a list of strings. """ with open(filename, 'r') as f: return f.readlines() def write_lines(filename, lines): """Writes list of lines into a file, overwriting the file it it exists. Args: filename (string): Path to the file to be writting into. lines (list): List of strings to write into the file. """ with open(filename, 'w') as f: f.writelines(lines) def move_between_sets(subset, src, dst, source = "<unknown>"): """Removes a subset of elements from one set and add it to another. Args: subset (set): The subset of `src` to be moved from `src` to `dst`. src (set): Source set. Must be a superset of `subset`. dst (set): Destination set. Must be disjoint with `subset`. """ assert src.issuperset(subset), ( "Error processing: {}\n" "The following entries were not found:\n" "{}" "Please visit go/hiddenapi for more information.").format( source, "".join(map(lambda x: " " + str(x), subset.difference(src)))) assert dst.isdisjoint(subset) # Order matters if `src` and `subset` are the same object. dst.update(subset) src.difference_update(subset) def get_package_name(signature): """Returns the package name prefix of a class member signature. Example: "Ljava/lang/String;->hashCode()J" --> "Ljava/lang/" Args: signature (string): Member signature Returns string: Package name of the given member """ class_name_end = signature.find("->") assert class_name_end != -1, "Invalid signature: {}".format(signature) package_name_end = signature.rfind("/", 0, class_name_end) assert package_name_end != -1, "Invalid signature: {}".format(signature) return signature[:package_name_end + 1] def all_package_names(*args): """Returns a set of packages names in given lists of member signatures. Example: args = [ set([ "Lpkg1/ClassA;->foo()V", "Lpkg2/ClassB;->bar()J" ]), set([ "Lpkg1/ClassC;->baz()Z" ]) ] return value = set([ "Lpkg1/", "Lpkg2" ]) Args: *args (list): List of sets to iterate over and extract the package names of its elements (member signatures) Returns: set: All package names extracted from the given lists of signatures. """ packages = set() for arg in args: packages = packages.union(map(get_package_name, arg)) return packages def move_all(src, dst): """Moves all elements of one set to another. Args: src (set): Source set. Will become empty. dst (set): Destination set. Will contain all elements of `src`. """ move_between_sets(src, src, dst) def move_from_files(filenames, src, dst): """Loads member signatures from a list of files and moves them to a given set. Opens files in `filenames`, reads all their lines and moves those from `src` set to `dst` set. Args: filenames (list): List of paths to files to be loaded. src (set): Set that loaded lines should be moved from. dst (set): Set that loaded lines should be moved to. """ if filenames: for filename in filenames: move_between_sets(set(read_lines(filename)), src, dst, filename) def move_serialization(src, dst): """Moves all members matching serialization API signatures between given sets. Args: src (set): Set that will be searched for serialization API and that API will be removed from it. dst (set): Set that serialization API will be moved to. """ serialization_patterns = [ r'readObject\(Ljava/io/ObjectInputStream;\)V', r'readObjectNoData\(\)V', r'readResolve\(\)Ljava/lang/Object;', r'serialVersionUID:J', r'serialPersistentFields:\[Ljava/io/ObjectStreamField;', r'writeObject\(Ljava/io/ObjectOutputStream;\)V', r'writeReplace\(\)Ljava/lang/Object;', ] regex = re.compile(r'.*->(' + '|'.join(serialization_patterns) + r')$') move_between_sets(filter(lambda api: regex.match(api), src), src, dst) def move_from_packages(packages, src, dst): """Moves all members of given package names from one set to another. Args: packages (list): List of string package names. src (set): Set that will be searched for API matching one of the given package names. Surch API will be removed from the set. dst (set): Set that matching API will be moved to. """ move_between_sets(filter(lambda api: get_package_name(api) in packages, src), src, dst) def main(argv): args = get_args() # Initialize API sets by loading lists of public and private API. Public API # are all members resolvable from SDK API stubs, other members are private. # As an optimization, skip the step of moving public API from a full set of # members and start with a populated whitelist. whitelist = set(read_lines(args.input_public)) uncategorized = set(read_lines(args.input_private)) light_greylist = set() dark_greylist = set() blacklist = set() # Assert that there is no overlap between public and private API. assert whitelist.isdisjoint(uncategorized) num_all_api = len(whitelist) + len(uncategorized) # Read all files which manually assign members to specific lists. move_from_files(args.input_whitelists, uncategorized, whitelist) move_from_files(args.input_greylists, uncategorized, light_greylist) move_from_files(args.input_blacklists, uncategorized, blacklist) # Iterate over all uncategorized members and move serialization API to light greylist. move_serialization(uncategorized, light_greylist) # Extract package names of members from whitelist and light greylist, which # are assumed to have been finalized at this point. Assign all uncategorized # members from the same packages to the dark greylist. dark_greylist_packages = all_package_names(whitelist, light_greylist) move_from_packages(dark_greylist_packages, uncategorized, dark_greylist) # Assign all uncategorized members to the blacklist. move_all(uncategorized, blacklist) # Assert we have not missed anything. assert whitelist.isdisjoint(light_greylist) assert whitelist.isdisjoint(dark_greylist) assert whitelist.isdisjoint(blacklist) assert light_greylist.isdisjoint(dark_greylist) assert light_greylist.isdisjoint(blacklist) assert dark_greylist.isdisjoint(blacklist) assert num_all_api == len(whitelist) + len(light_greylist) + len(dark_greylist) + len(blacklist) # Write final lists to disk. write_lines(args.output_whitelist, whitelist) write_lines(args.output_light_greylist, light_greylist) write_lines(args.output_dark_greylist, dark_greylist) write_lines(args.output_blacklist, blacklist) if __name__ == "__main__": main(sys.argv)
tools/hiddenapi/generate_hiddenapi_lists_test.py 0 → 100755 +89 −0 Original line number Diff line number Diff line #!/usr/bin/env python # # Copyright (C) 2018 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for Hidden API list generation.""" import unittest from generate_hiddenapi_lists import * class TestHiddenapiListGeneration(unittest.TestCase): def test_move_between_sets(self): A = set([1, 2, 3, 4]) B = set([5, 6, 7, 8]) move_between_sets(set([2, 4]), A, B) self.assertEqual(A, set([1, 3])) self.assertEqual(B, set([2, 4, 5, 6, 7, 8])) def test_move_between_sets_fail_not_superset(self): A = set([1, 2, 3, 4]) B = set([5, 6, 7, 8]) with self.assertRaises(AssertionError) as ar: move_between_sets(set([0, 2]), A, B) def test_move_between_sets_fail_not_disjoint(self): A = set([1, 2, 3, 4]) B = set([4, 5, 6, 7, 8]) with self.assertRaises(AssertionError) as ar: move_between_sets(set([1, 4]), A, B) def test_get_package_name(self): self.assertEqual(get_package_name("Ljava/lang/String;->clone()V"), "Ljava/lang/") def test_get_package_name_fail_no_arrow(self): with self.assertRaises(AssertionError) as ar: get_package_name("Ljava/lang/String;-clone()V") with self.assertRaises(AssertionError) as ar: get_package_name("Ljava/lang/String;>clone()V") with self.assertRaises(AssertionError) as ar: get_package_name("Ljava/lang/String;__clone()V") def test_get_package_name_fail_no_package(self): with self.assertRaises(AssertionError) as ar: get_package_name("LString;->clone()V") def test_all_package_names(self): self.assertEqual(all_package_names(), set()) self.assertEqual(all_package_names(set(["Lfoo/Bar;->baz()V"])), set(["Lfoo/"])) self.assertEqual( all_package_names(set(["Lfoo/Bar;->baz()V", "Lfoo/BarX;->bazx()I"])), set(["Lfoo/"])) self.assertEqual( all_package_names( set(["Lfoo/Bar;->baz()V"]), set(["Lfoo/BarX;->bazx()I", "Labc/xyz/Mno;->ijk()J"])), set(["Lfoo/", "Labc/xyz/"])) def test_move_all(self): src = set([ "abc", "xyz" ]) dst = set([ "def" ]) move_all(src, dst) self.assertEqual(src, set()) self.assertEqual(dst, set([ "abc", "def", "xyz" ])) def test_move_from_packages(self): src = set([ "Lfoo/bar/ClassA;->abc()J", # will be moved "Lfoo/bar/ClassA;->def()J", # will be moved "Lcom/pkg/example/ClassD;->ijk:J", # not moved: different package "Lfoo/bar/xyz/ClassC;->xyz()Z" ]) # not moved: subpackage dst = set() packages = set([ "Lfoo/bar/" ]) move_from_packages(packages, src, dst) self.assertEqual( src, set([ "Lfoo/bar/xyz/ClassC;->xyz()Z", "Lcom/pkg/example/ClassD;->ijk:J" ])) self.assertEqual( dst, set([ "Lfoo/bar/ClassA;->abc()J", "Lfoo/bar/ClassA;->def()J" ])) if __name__ == '__main__': unittest.main()
