Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 8503b904 authored by David Brazdil's avatar David Brazdil
Browse files

Rewrite hidden API list generation in Python 

Generating hidden API lists has grown in complexity and the original
Makefile rule has become a bottleneck for build times. Rewrite the
logic in Python.

Bug: 113278235
Bug: 73736106
Test: frameworks/base/tools/hiddenapi/generate_hiddenapi_lists_test.py
Change-Id: I63f03133d70260d06c55f482b844a4980dc6f734
parent 6717a6f9

Android.mk

+28 −125
Original line number Diff line number Diff line
@@ -322,131 +322,34 @@ $(OUT_DOCS)/offline-sdk-timestamp: $(OUT_DOCS)/offline-sdk-docs-docs.zip 
	( unzip -qo $< -d $(OUT_DOCS)/offline-sdk && touch -f $@ ) || exit 1



# ==== hiddenapi lists =======================================

include $(CLEAR_VARS)



# File names of final API lists

LOCAL_WHITELIST := $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST)

LOCAL_LIGHT_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST)

LOCAL_DARK_GREYLIST := $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST)

LOCAL_BLACKLIST := $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST)



# File names of source files we will use to generate the final API lists.

LOCAL_SRC_GREYLIST := frameworks/base/config/hiddenapi-light-greylist.txt

LOCAL_SRC_VENDOR_LIST := frameworks/base/config/hiddenapi-vendor-list.txt

LOCAL_SRC_FORCE_BLACKLIST := frameworks/base/config/hiddenapi-force-blacklist.txt

LOCAL_SRC_PUBLIC_API := $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST)

LOCAL_SRC_PRIVATE_API := $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST)

LOCAL_SRC_REMOVED_API := $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)



LOCAL_SRC_ALL := \

	$(LOCAL_SRC_GREYLIST) \

	$(LOCAL_SRC_VENDOR_LIST) \

	$(LOCAL_SRC_FORCE_BLACKLIST) \

	$(LOCAL_SRC_PUBLIC_API) \

	$(LOCAL_SRC_PRIVATE_API) \

	$(LOCAL_SRC_REMOVED_API)



define assert-has-no-overlap

if [ ! -z "`comm -12 <(sort $(1)) <(sort $(2))`" ]; then \

	echo "$(1) and $(2) should not overlap" 1>&2; \

	comm -12 <(sort $(1)) <(sort $(2)) 1>&2; \

	exit 1; \

fi

endef



define assert-is-subset

if [ ! -z "`comm -23 <(sort $(1)) <(sort $(2))`" ]; then \

	echo "$(1) must be a subset of $(2)" 1>&2; \

	comm -23 <(sort $(1)) <(sort $(2)) 1>&2; \

	exit 1; \

fi

endef



define assert-has-no-duplicates

if [ ! -z "`sort $(1) | uniq -D`" ]; then \

	echo "$(1) has duplicate entries" 1>&2; \

	sort $(1) | uniq -D 1>&2; \

	exit 1; \

fi

endef



# The following rules build API lists in the build folder.

# By not using files from the source tree, ART buildbots can mock these lists

# or have alternative rules for building them. Other rules in the build system

# should depend on the files in the build folder.



# Merge whitelist from:

# (1) public API stubs

# (2) whitelist entries generated by class2greylist (PRIVATE_WHITELIST_INPUTS)

$(LOCAL_WHITELIST): $(LOCAL_SRC_PUBLIC_API)

	sort $(LOCAL_SRC_PUBLIC_API) $(PRIVATE_WHITELIST_INPUTS) > $@

	$(call assert-has-no-duplicates,$@)



# Merge light greylist from multiple files:

#  (1) manual greylist LOCAL_SRC_GREYLIST

#  (2) list of usages from vendor apps LOCAL_SRC_VENDOR_LIST

#  (3) list of removed APIs in LOCAL_SRC_REMOVED_API

#      @removed does not imply private in Doclava. We must take the subset also

#      in LOCAL_SRC_PRIVATE_API.

#  (4) list of serialization APIs

#      Automatically adds all methods which match the signatures in

#      REGEX_SERIALIZATION. These are greylisted in order to allow applications

#      to write their own serializers.

#  (5) greylist entries generated by class2greylist (PRIVATE_GREYLIST_INPUTS)

$(LOCAL_LIGHT_GREYLIST): REGEX_SERIALIZATION := \

    "readObject\(Ljava/io/ObjectInputStream;\)V" \

    "readObjectNoData\(\)V" \

    "readResolve\(\)Ljava/lang/Object;" \

    "serialVersionUID:J" \

    "serialPersistentFields:\[Ljava/io/ObjectStreamField;" \

    "writeObject\(Ljava/io/ObjectOutputStream;\)V" \

    "writeReplace\(\)Ljava/lang/Object;"

$(LOCAL_LIGHT_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST)

	sort $(LOCAL_SRC_GREYLIST) $(LOCAL_SRC_VENDOR_LIST) $(PRIVATE_GREYLIST_INPUTS) \

	     <(grep -E "\->("$(subst $(space),"|",$(REGEX_SERIALIZATION))")$$" \

	               $(LOCAL_SRC_PRIVATE_API)) \

	     <(comm -12 <(sort $(LOCAL_SRC_REMOVED_API)) <(sort $(LOCAL_SRC_PRIVATE_API))) \

	     > $@

	$(call assert-has-no-duplicates,$@)

	$(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API))

	$(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST))

	$(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST))



# Generate dark greylist as remaining classes and class members in the same

# package as classes listed in the light greylist.

# The algorithm is as follows:

#   (1) extract the class descriptor from each entry in LOCAL_LIGHT_GREYLIST

#   (2) strip everything after the last forward-slash,

#       e.g. 'Lpackage/subpackage/class$inner;' turns into 'Lpackage/subpackage/'

#   (3) insert all entries from LOCAL_SRC_PRIVATE_API which begin with the package

#       name but do not contain another forward-slash in the class name, e.g.

#       matching '^Lpackage/subpackage/[^/;]*;'

#   (4) subtract entries shared with LOCAL_LIGHT_GREYLIST

$(LOCAL_DARK_GREYLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST)

	comm -13 <(sort $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_SRC_FORCE_BLACKLIST)) \

	         <(cat $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) | \

	               sed 's/\->.*//' | sed 's/\(.*\/\).*/\1/' | sort | uniq | \

	               while read PKG_NAME; do \

	                   grep -E "^$${PKG_NAME}[^/;]*;" $(LOCAL_SRC_PRIVATE_API); \

	               done | sort | uniq) \

	         > $@

	$(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API))

	$(call assert-has-no-duplicates,$@)

	$(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST))

	$(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST))

	$(call assert-has-no-overlap,$@,$(LOCAL_SRC_FORCE_BLACKLIST))



# Generate blacklist as private API minus (light greylist plus dark greylist).

$(LOCAL_BLACKLIST): $(LOCAL_SRC_ALL) $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST)

	comm -13 <(sort $(LOCAL_WHITELIST) $(LOCAL_LIGHT_GREYLIST) $(LOCAL_DARK_GREYLIST)) \

	         <(sort $(LOCAL_SRC_PRIVATE_API)) \

	         > $@

	$(call assert-is-subset,$@,$(LOCAL_SRC_PRIVATE_API))

	$(call assert-has-no-duplicates,$@)

	$(call assert-has-no-overlap,$@,$(LOCAL_WHITELIST))

	$(call assert-has-no-overlap,$@,$(LOCAL_LIGHT_GREYLIST))

	$(call assert-has-no-overlap,$@,$(LOCAL_DARK_GREYLIST))

	$(call assert-is-subset,$(LOCAL_SRC_FORCE_BLACKLIST),$@)

$(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \

    .KATI_IMPLICIT_OUTPUTS := \

        $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \

        $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \

        $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST)

$(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST): \

    frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \

    frameworks/base/config/hiddenapi-light-greylist.txt \

    frameworks/base/config/hiddenapi-vendor-list.txt \

    frameworks/base/config/hiddenapi-force-blacklist.txt \

    $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \

    $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \

    $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)

	frameworks/base/tools/hiddenapi/generate_hiddenapi_lists.py \

	    --input-public $(INTERNAL_PLATFORM_HIDDENAPI_PUBLIC_LIST) \

	    --input-private $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST) \

	    --input-whitelists $(PRIVATE_WHITELIST_INPUTS) \

	    --input-greylists \

	        frameworks/base/config/hiddenapi-light-greylist.txt \

	        frameworks/base/config/hiddenapi-vendor-list.txt \

	        <(comm -12 <(sort $(INTERNAL_PLATFORM_REMOVED_DEX_API_FILE)) \

	                   $(INTERNAL_PLATFORM_HIDDENAPI_PRIVATE_LIST)) \

	        $(PRIVATE_GREYLIST_INPUTS) \

	    --input-blacklists frameworks/base/config/hiddenapi-force-blacklist.txt \

	    --output-whitelist $(INTERNAL_PLATFORM_HIDDENAPI_WHITELIST) \

	    --output-light-greylist $(INTERNAL_PLATFORM_HIDDENAPI_LIGHT_GREYLIST) \

	    --output-dark-greylist $(INTERNAL_PLATFORM_HIDDENAPI_DARK_GREYLIST) \

	    --output-blacklist $(INTERNAL_PLATFORM_HIDDENAPI_BLACKLIST)



# Include subdirectory makefiles

# ============================================================

tools/hiddenapi/generate_hiddenapi_lists.py

0 → 100755
+241 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python

#

# Copyright (C) 2018 The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

"""

Generate API lists for non-SDK API enforcement.



usage: generate-hiddenapi-lists.py [-h]

                                   --input-public INPUT_PUBLIC

                                   --input-private INPUT_PRIVATE

                                   [--input-whitelists [INPUT_WHITELISTS [INPUT_WHITELISTS ...]]]

                                   [--input-greylists [INPUT_GREYLISTS [INPUT_GREYLISTS ...]]]

                                   [--input-blacklists [INPUT_BLACKLISTS [INPUT_BLACKLISTS ...]]]

                                   --output-whitelist OUTPUT_WHITELIST

                                   --output-light-greylist OUTPUT_LIGHT_GREYLIST

                                   --output-dark-greylist OUTPUT_DARK_GREYLIST

                                   --output-blacklist OUTPUT_BLACKLIST

"""

import argparse

import os

import sys

import re



def get_args():

    """Parses command line arguments.



    Returns:

        Namespace: dictionary of parsed arguments

    """

    parser = argparse.ArgumentParser()

    parser.add_argument('--input-public', required=True, help='List of all public members')

    parser.add_argument('--input-private', required=True, help='List of all private members')

    parser.add_argument(

        '--input-whitelists', nargs='*',

        help='Lists of members to force on whitelist')

    parser.add_argument(

        '--input-greylists', nargs='*',

        help='Lists of members to force on light greylist')

    parser.add_argument(

        '--input-blacklists', nargs='*',

        help='Lists of members to force on blacklist')

    parser.add_argument('--output-whitelist', required=True)

    parser.add_argument('--output-light-greylist', required=True)

    parser.add_argument('--output-dark-greylist', required=True)

    parser.add_argument('--output-blacklist', required=True)

    return parser.parse_args()



def read_lines(filename):

    """Reads entire file and return it as a list of lines.



    Args:

        filename (string): Path to the file to read from.



    Returns:

        list: Lines of the loaded file as a list of strings.

    """

    with open(filename, 'r') as f:

        return f.readlines()



def write_lines(filename, lines):

    """Writes list of lines into a file, overwriting the file it it exists.



    Args:

        filename (string): Path to the file to be writting into.

        lines (list): List of strings to write into the file.

    """

    with open(filename, 'w') as f:

        f.writelines(lines)



def move_between_sets(subset, src, dst, source = "<unknown>"):

    """Removes a subset of elements from one set and add it to another.



    Args:

        subset (set): The subset of `src` to be moved from `src` to `dst`.

        src (set): Source set. Must be a superset of `subset`.

        dst (set): Destination set. Must be disjoint with `subset`.

    """

    assert src.issuperset(subset), (

        "Error processing: {}\n"

        "The following entries were not found:\n"

        "{}"

        "Please visit go/hiddenapi for more information.").format(

            source, "".join(map(lambda x: "  " + str(x), subset.difference(src))))

    assert dst.isdisjoint(subset)

    # Order matters if `src` and `subset` are the same object.

    dst.update(subset)

    src.difference_update(subset)



def get_package_name(signature):

    """Returns the package name prefix of a class member signature.



    Example: "Ljava/lang/String;->hashCode()J" --> "Ljava/lang/"



    Args:

        signature (string): Member signature



    Returns

        string: Package name of the given member

    """

    class_name_end = signature.find("->")

    assert class_name_end != -1, "Invalid signature: {}".format(signature)

    package_name_end = signature.rfind("/", 0, class_name_end)

    assert package_name_end != -1, "Invalid signature: {}".format(signature)

    return signature[:package_name_end + 1]



def all_package_names(*args):

    """Returns a set of packages names in given lists of member signatures.



    Example: args = [ set([ "Lpkg1/ClassA;->foo()V", "Lpkg2/ClassB;->bar()J" ]),

                      set([ "Lpkg1/ClassC;->baz()Z" ]) ]

             return value = set([ "Lpkg1/", "Lpkg2" ])



    Args:

        *args (list): List of sets to iterate over and extract the package names

                      of its elements (member signatures)



    Returns:

        set: All package names extracted from the given lists of signatures.

    """

    packages = set()

    for arg in args:

        packages = packages.union(map(get_package_name, arg))

    return packages



def move_all(src, dst):

    """Moves all elements of one set to another.



    Args:

        src (set): Source set. Will become empty.

        dst (set): Destination set. Will contain all elements of `src`.

    """

    move_between_sets(src, src, dst)



def move_from_files(filenames, src, dst):

    """Loads member signatures from a list of files and moves them to a given set.



    Opens files in `filenames`, reads all their lines and moves those from `src`

    set to `dst` set.



    Args:

        filenames (list): List of paths to files to be loaded.

        src (set): Set that loaded lines should be moved from.

        dst (set): Set that loaded lines should be moved to.

    """

    if filenames:

        for filename in filenames:

            move_between_sets(set(read_lines(filename)), src, dst, filename)



def move_serialization(src, dst):

    """Moves all members matching serialization API signatures between given sets.



    Args:

        src (set): Set that will be searched for serialization API and that API

                   will be removed from it.

        dst (set): Set that serialization API will be moved to.

    """

    serialization_patterns = [

        r'readObject\(Ljava/io/ObjectInputStream;\)V',

        r'readObjectNoData\(\)V',

        r'readResolve\(\)Ljava/lang/Object;',

        r'serialVersionUID:J',

        r'serialPersistentFields:\[Ljava/io/ObjectStreamField;',

        r'writeObject\(Ljava/io/ObjectOutputStream;\)V',

        r'writeReplace\(\)Ljava/lang/Object;',

    ]

    regex = re.compile(r'.*->(' + '|'.join(serialization_patterns) + r')$')

    move_between_sets(filter(lambda api: regex.match(api), src), src, dst)



def move_from_packages(packages, src, dst):

    """Moves all members of given package names from one set to another.



    Args:

        packages (list): List of string package names.

        src (set): Set that will be searched for API matching one of the given

                   package names. Surch API will be removed from the set.

        dst (set): Set that matching API will be moved to.

    """

    move_between_sets(filter(lambda api: get_package_name(api) in packages, src), src, dst)



def main(argv):

    args = get_args()



    # Initialize API sets by loading lists of public and private API. Public API

    # are all members resolvable from SDK API stubs, other members are private.

    # As an optimization, skip the step of moving public API from a full set of

    # members and start with a populated whitelist.

    whitelist = set(read_lines(args.input_public))

    uncategorized = set(read_lines(args.input_private))

    light_greylist = set()

    dark_greylist = set()

    blacklist = set()



    # Assert that there is no overlap between public and private API.

    assert whitelist.isdisjoint(uncategorized)

    num_all_api = len(whitelist) + len(uncategorized)



    # Read all files which manually assign members to specific lists.

    move_from_files(args.input_whitelists, uncategorized, whitelist)

    move_from_files(args.input_greylists, uncategorized, light_greylist)

    move_from_files(args.input_blacklists, uncategorized, blacklist)



    # Iterate over all uncategorized members and move serialization API to light greylist.

    move_serialization(uncategorized, light_greylist)



    # Extract package names of members from whitelist and light greylist, which

    # are assumed to have been finalized at this point. Assign all uncategorized

    # members from the same packages to the dark greylist.

    dark_greylist_packages = all_package_names(whitelist, light_greylist)

    move_from_packages(dark_greylist_packages, uncategorized, dark_greylist)



    # Assign all uncategorized members to the blacklist.

    move_all(uncategorized, blacklist)



    # Assert we have not missed anything.

    assert whitelist.isdisjoint(light_greylist)

    assert whitelist.isdisjoint(dark_greylist)

    assert whitelist.isdisjoint(blacklist)

    assert light_greylist.isdisjoint(dark_greylist)

    assert light_greylist.isdisjoint(blacklist)

    assert dark_greylist.isdisjoint(blacklist)

    assert num_all_api == len(whitelist) + len(light_greylist) + len(dark_greylist) + len(blacklist)



    # Write final lists to disk.

    write_lines(args.output_whitelist, whitelist)

    write_lines(args.output_light_greylist, light_greylist)

    write_lines(args.output_dark_greylist, dark_greylist)

    write_lines(args.output_blacklist, blacklist)



if __name__ == "__main__":

    main(sys.argv)

tools/hiddenapi/generate_hiddenapi_lists_test.py

0 → 100755
+89 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python

#

# Copyright (C) 2018 The Android Open Source Project

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

"""Unit tests for Hidden API list generation."""

import unittest

from generate_hiddenapi_lists import *



class TestHiddenapiListGeneration(unittest.TestCase):



    def test_move_between_sets(self):

        A = set([1, 2, 3, 4])

        B = set([5, 6, 7, 8])

        move_between_sets(set([2, 4]), A, B)

        self.assertEqual(A, set([1, 3]))

        self.assertEqual(B, set([2, 4, 5, 6, 7, 8]))



    def test_move_between_sets_fail_not_superset(self):

        A = set([1, 2, 3, 4])

        B = set([5, 6, 7, 8])

        with self.assertRaises(AssertionError) as ar:

            move_between_sets(set([0, 2]), A, B)



    def test_move_between_sets_fail_not_disjoint(self):

        A = set([1, 2, 3, 4])

        B = set([4, 5, 6, 7, 8])

        with self.assertRaises(AssertionError) as ar:

            move_between_sets(set([1, 4]), A, B)



    def test_get_package_name(self):

        self.assertEqual(get_package_name("Ljava/lang/String;->clone()V"), "Ljava/lang/")



    def test_get_package_name_fail_no_arrow(self):

        with self.assertRaises(AssertionError) as ar:

            get_package_name("Ljava/lang/String;-clone()V")

        with self.assertRaises(AssertionError) as ar:

            get_package_name("Ljava/lang/String;>clone()V")

        with self.assertRaises(AssertionError) as ar:

            get_package_name("Ljava/lang/String;__clone()V")



    def test_get_package_name_fail_no_package(self):

        with self.assertRaises(AssertionError) as ar:

            get_package_name("LString;->clone()V")



    def test_all_package_names(self):

        self.assertEqual(all_package_names(), set())

        self.assertEqual(all_package_names(set(["Lfoo/Bar;->baz()V"])), set(["Lfoo/"]))

        self.assertEqual(

            all_package_names(set(["Lfoo/Bar;->baz()V", "Lfoo/BarX;->bazx()I"])),

            set(["Lfoo/"]))

        self.assertEqual(

            all_package_names(

                set(["Lfoo/Bar;->baz()V"]),

                set(["Lfoo/BarX;->bazx()I", "Labc/xyz/Mno;->ijk()J"])),

            set(["Lfoo/", "Labc/xyz/"]))



    def test_move_all(self):

        src = set([ "abc", "xyz" ])

        dst = set([ "def" ])

        move_all(src, dst)

        self.assertEqual(src, set())

        self.assertEqual(dst, set([ "abc", "def", "xyz" ]))



    def test_move_from_packages(self):

        src = set([ "Lfoo/bar/ClassA;->abc()J",        # will be moved

                    "Lfoo/bar/ClassA;->def()J",        # will be moved

                    "Lcom/pkg/example/ClassD;->ijk:J", # not moved: different package

                    "Lfoo/bar/xyz/ClassC;->xyz()Z" ])  # not moved: subpackage

        dst = set()

        packages = set([ "Lfoo/bar/" ])

        move_from_packages(packages, src, dst)

        self.assertEqual(

            src, set([ "Lfoo/bar/xyz/ClassC;->xyz()Z", "Lcom/pkg/example/ClassD;->ijk:J" ]))

        self.assertEqual(

            dst, set([ "Lfoo/bar/ClassA;->abc()J", "Lfoo/bar/ClassA;->def()J" ]))



if __name__ == '__main__':

    unittest.main()