Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c57b3c2b authored by Aleksei Kalinov's avatar Aleksei Kalinov
Browse files

Update language to comply with Android's inclusive language guidance 

The binary uses flags to mean both command-line parameter names
and the API names, used in files and internally in the class.

Internal representation and command-line parameter names are updated to
use more inclusive language. However, the output file produced by
`generate_csv` function still uses old flags. Format update in
the input/output files is out of scope for this change
and will be updated in the follow-up changelist.

See https://source.android.com/setup/contribute/respectful-code for
reference.

Bug: 161896447

Test: python3 generate_hiddenapi_lists_test.py

Change-Id: I36c85ce04b89ba3e4eee319f95511110d2c4374e
parent 9e9ce9f9

tools/hiddenapi/generate_hiddenapi_lists.py

+93 −34
Original line number Diff line number Diff line
@@ -13,9 +13,7 @@ 
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

"""

Generate API lists for non-SDK API enforcement.

"""

"""Generate API lists for non-SDK API enforcement."""

import argparse

from collections import defaultdict

import functools
@@ -24,27 +22,47 @@ import re 
import sys



# Names of flags recognized by the `hiddenapi` tool.

FLAG_WHITELIST = "whitelist"

FLAG_GREYLIST = "greylist"

FLAG_BLACKLIST = "blacklist"

FLAG_GREYLIST_MAX_O = "greylist-max-o"

FLAG_GREYLIST_MAX_P = "greylist-max-p"

FLAG_GREYLIST_MAX_Q = "greylist-max-q"

FLAG_GREYLIST_MAX_R = "greylist-max-r"

FLAG_CORE_PLATFORM_API = "core-platform-api"

FLAG_PUBLIC_API = "public-api"

FLAG_SYSTEM_API = "system-api"

FLAG_TEST_API = "test-api"

FLAG_SDK = 'sdk'

FLAG_UNSUPPORTED = 'unsupported'

FLAG_BLOCKED = 'blocked'

FLAG_MAX_TARGET_O = 'max-target-o'

FLAG_MAX_TARGET_P = 'max-target-p'

FLAG_MAX_TARGET_Q = 'max-target-q'

FLAG_MAX_TARGET_R = 'max-target-r'

FLAG_CORE_PLATFORM_API = 'core-platform-api'

FLAG_PUBLIC_API = 'public-api'

FLAG_SYSTEM_API = 'system-api'

FLAG_TEST_API = 'test-api'



OLD_FLAG_SDK = "whitelist"

OLD_FLAG_UNSUPPORTED = "greylist"

OLD_FLAG_BLOCKED = "blacklist"

OLD_FLAG_MAX_TARGET_O = "greylist-max-o"

OLD_FLAG_MAX_TARGET_P = "greylist-max-p"

OLD_FLAG_MAX_TARGET_Q = "greylist-max-q"

OLD_FLAG_MAX_TARGET_R = "greylist-max-r"



OLD_FLAGS_TO_NEW = {

    OLD_FLAG_SDK: FLAG_SDK,

    OLD_FLAG_UNSUPPORTED: FLAG_UNSUPPORTED,

    OLD_FLAG_BLOCKED: FLAG_BLOCKED,

    OLD_FLAG_MAX_TARGET_O: FLAG_MAX_TARGET_O,

    OLD_FLAG_MAX_TARGET_P: FLAG_MAX_TARGET_P,

    OLD_FLAG_MAX_TARGET_Q: FLAG_MAX_TARGET_Q,

    OLD_FLAG_MAX_TARGET_R: FLAG_MAX_TARGET_R,

}



NEW_FLAGS_TO_OLD = dict(zip(OLD_FLAGS_TO_NEW.values(), OLD_FLAGS_TO_NEW.keys()))



# List of all known flags.

FLAGS_API_LIST = [

    FLAG_WHITELIST,

    FLAG_GREYLIST,

    FLAG_BLACKLIST,

    FLAG_GREYLIST_MAX_O,

    FLAG_GREYLIST_MAX_P,

    FLAG_GREYLIST_MAX_Q,

    FLAG_GREYLIST_MAX_R,

    FLAG_SDK,

    FLAG_UNSUPPORTED,

    FLAG_BLOCKED,

    FLAG_MAX_TARGET_O,

    FLAG_MAX_TARGET_P,

    FLAG_MAX_TARGET_Q,

    FLAG_MAX_TARGET_R,

]

ALL_FLAGS = FLAGS_API_LIST + [

    FLAG_CORE_PLATFORM_API,
@@ -58,7 +76,7 @@ ALL_FLAGS_SET = set(ALL_FLAGS) 


# Suffix used in command line args to express that only known and

# otherwise unassigned entries should be assign the given flag.

# For example, the P dark greylist is checked in as it was in P,

# For example, the max-target-P list is checked in as it was in P,

# but signatures have changes since then. The flag instructs this

# script to skip any entries which do not exist any more.

FLAG_IGNORE_CONFLICTS_SUFFIX = "-ignore-conflicts"
@@ -87,6 +105,7 @@ SERIALIZATION_REGEX = re.compile(r'.*->(' + '|'.join(SERIALIZATION_PATTERNS) + r 
HAS_NO_API_LIST_ASSIGNED = lambda api, flags: not FLAGS_API_LIST_SET.intersection(flags)

IS_SERIALIZATION = lambda api, flags: SERIALIZATION_REGEX.match(api)





def get_args():

    """Parses command line arguments.
@@ -113,6 +132,7 @@ def get_args(): 


    return parser.parse_args()





def read_lines(filename):

    """Reads entire file and return it as a list of lines.
@@ -130,8 +150,9 @@ def read_lines(filename): 
    lines = map(lambda line: line.strip(), lines)

    return set(lines)





def write_lines(filename, lines):

    """Writes list of lines into a file, overwriting the file it it exists.

    """Writes list of lines into a file, overwriting the file if it exists.



    Args:

        filename (string): Path to the file to be writting into.
@@ -141,6 +162,7 @@ def write_lines(filename, lines): 
    with open(filename, 'w') as f:

        f.writelines(lines)





def extract_package(signature):

    """Extracts the package from a signature.
@@ -159,6 +181,7 @@ def extract_package(signature): 
    package_name = full_class_name.rpartition("/")[0]

    return package_name.replace('/', '.')





class FlagsDict:

    def __init__(self):

        self._dict_keyset = set()
@@ -182,6 +205,36 @@ class FlagsDict: 
            "Please visit go/hiddenapi for more information.").format(

                source, "\n".join(flags_subset - ALL_FLAGS_SET))



    def convert_to_new_flag(self, flag):

      """Converts old flag to a new variant.



      Flags that are considered old are replaced with new versions.

      Otherwise, it is a no-op.



      Args:

        flag: a string, representing SDK flag.



      Returns:

         A string. Result of conversion.



      """

      return OLD_FLAGS_TO_NEW.get(flag, flag)



    def convert_to_old_flag(self, flag):

      """Converts a new flag to a old variant.



      No-op if there is no suitable old flag.

      Only used to support backwards compatibility.



      Args:

        flag: a string, representing SDK flag.



      Returns:

         A string. Result of conversion.



      """

      return NEW_FLAGS_TO_OLD.get(flag, flag)



    def filter_apis(self, filter_fn):

        """Returns APIs which match a given predicate.
@@ -212,10 +265,16 @@ class FlagsDict: 
    def generate_csv(self):

        """Constructs CSV entries from a dictionary.



        Old versions of flags are used to generate the file.



        Returns:

            List of lines comprising a CSV file. See "parse_and_merge_csv" for format description.

        """

        return sorted(map(lambda api: ",".join([api] + sorted(self._dict[api])), self._dict))

        lines = []

        for api in self._dict:

          flags = sorted([self.convert_to_old_flag(flag) for flag in self._dict[api]])

          lines.append(",".join([api] + flags))

        return sorted(lines)



    def parse_and_merge_csv(self, csv_lines, source = "<unknown>"):

        """Parses CSV entries and merges them into a given dictionary.
@@ -237,17 +296,16 @@ class FlagsDict: 
        self._dict_keyset.update([ csv[0] for csv in csv_values ])



        # Check that all flags are known.

        csv_flags = set(functools.reduce(

            lambda x, y: set(x).union(y),

            [ csv[1:] for csv in csv_values ],

            []))

        csv_flags = set()

        for csv in csv_values:

          csv_flags.update([self.convert_to_new_flag(flag) for flag in csv[1:]])

        self._check_flags_set(csv_flags, source)



        # Iterate over all CSV lines, find entry in dict and append flags to it.

        for csv in csv_values:

            flags = csv[1:]

            flags = [self.convert_to_new_flag(flag) for flag in csv[1:]]

            if (FLAG_PUBLIC_API in flags) or (FLAG_SYSTEM_API in flags):

                flags.append(FLAG_WHITELIST)

                flags.append(FLAG_SDK)

            self._dict[csv[0]].update(flags)



    def assign_flag(self, flag, apis, source="<unknown>"):
@@ -271,6 +329,7 @@ class FlagsDict: 
        for api in apis:

            self._dict[api].add(flag)





def main(argv):

    # Parse arguments.

    args = vars(get_args())
@@ -287,8 +346,8 @@ def main(argv): 
        flags.parse_and_merge_csv(read_lines(filename), filename)



    # Combine inputs which do not require any particular order.

    # (1) Assign serialization API to whitelist.

    flags.assign_flag(FLAG_WHITELIST, flags.filter_apis(IS_SERIALIZATION))

    # (1) Assign serialization API to SDK.

    flags.assign_flag(FLAG_SDK, flags.filter_apis(IS_SERIALIZATION))



    # (2) Merge text files with a known flag into the dictionary.

    for flag in ALL_FLAGS:
@@ -314,8 +373,8 @@ def main(argv): 
            valid_entries = flags.filter_apis(should_add_signature_to_list)

            flags.assign_flag(flag, valid_entries)



    # Assign all remaining entries to the blacklist.

    flags.assign_flag(FLAG_BLACKLIST, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED))

    # Mark all remaining entries as blocked.

    flags.assign_flag(FLAG_BLOCKED, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED))



    # Write output.

    write_lines(args["output"], flags.generate_csv())

tools/hiddenapi/generate_hiddenapi_lists_test.py

+20 −19
Original line number Diff line number Diff line
@@ -23,7 +23,7 @@ class TestHiddenapiListGeneration(unittest.TestCase): 
        # Initialize flags so that A and B are put on the whitelist and

        # C, D, E are left unassigned. Try filtering for the unassigned ones.

        flags = FlagsDict()

        flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B,' + FLAG_WHITELIST,

        flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B,' + FLAG_SDK,

                        'C', 'D', 'E'])

        filter_set = flags.filter_apis(lambda api, flags: not flags)

        self.assertTrue(isinstance(filter_set, set))
@@ -32,10 +32,10 @@ class TestHiddenapiListGeneration(unittest.TestCase): 
    def test_get_valid_subset_of_unassigned_keys(self):

        # Create flags where only A is unassigned.

        flags = FlagsDict()

        flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B', 'C'])

        flags.assign_flag(FLAG_GREYLIST, set(['C']))

        flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B', 'C'])

        flags.assign_flag(FLAG_UNSUPPORTED, set(['C']))

        self.assertEqual(flags.generate_csv(),

            [ 'A,' + FLAG_WHITELIST, 'B', 'C,' + FLAG_GREYLIST ])

            [ 'A,' + OLD_FLAG_SDK, 'B', 'C,' + OLD_FLAG_UNSUPPORTED ])



        # Check three things:

        # (1) B is selected as valid unassigned
@@ -50,20 +50,21 @@ class TestHiddenapiListGeneration(unittest.TestCase): 
        # Test empty CSV entry.

        self.assertEqual(flags.generate_csv(), [])



        # Test new additions.

        # Test new additions. CSV generator produces values with old flags

        # to be backwards compatible.

        flags.parse_and_merge_csv([

            'A,' + FLAG_GREYLIST,

            'B,' + FLAG_BLACKLIST + ',' + FLAG_GREYLIST_MAX_O,

            'C,' + FLAG_SYSTEM_API + ',' + FLAG_WHITELIST,

            'D,' + FLAG_GREYLIST+ ',' + FLAG_TEST_API,

            'E,' + FLAG_BLACKLIST+ ',' + FLAG_TEST_API,

            'A,' + FLAG_UNSUPPORTED,

            'B,' + FLAG_BLOCKED + ',' + FLAG_MAX_TARGET_O,

            'C,' + FLAG_SDK + ',' + FLAG_SYSTEM_API,

            'D,' + FLAG_UNSUPPORTED + ',' + FLAG_TEST_API,

            'E,' + FLAG_BLOCKED + ',' + FLAG_TEST_API,

        ])

        self.assertEqual(flags.generate_csv(), [

            'A,' + FLAG_GREYLIST,

            'B,' + FLAG_BLACKLIST + "," + FLAG_GREYLIST_MAX_O,

            'C,' + FLAG_SYSTEM_API + ',' + FLAG_WHITELIST,

            'D,' + FLAG_GREYLIST+ ',' + FLAG_TEST_API,

            'E,' + FLAG_BLACKLIST+ ',' + FLAG_TEST_API,

            'A,' + OLD_FLAG_UNSUPPORTED,

            'B,' + OLD_FLAG_BLOCKED + "," + OLD_FLAG_MAX_TARGET_O,

            'C,' + FLAG_SYSTEM_API + ',' + OLD_FLAG_SDK,

            'D,' + OLD_FLAG_UNSUPPORTED + ',' + FLAG_TEST_API,

            'E,' + OLD_FLAG_BLOCKED + ',' + FLAG_TEST_API,

        ])



        # Test unknown flag.
@@ -72,16 +73,16 @@ class TestHiddenapiListGeneration(unittest.TestCase): 


    def test_assign_flag(self):

        flags = FlagsDict()

        flags.parse_and_merge_csv(['A,' + FLAG_WHITELIST, 'B'])

        flags.parse_and_merge_csv(['A,' + FLAG_SDK, 'B'])



        # Test new additions.

        flags.assign_flag(FLAG_GREYLIST, set([ 'A', 'B' ]))

        flags.assign_flag(FLAG_UNSUPPORTED, set([ 'A', 'B' ]))

        self.assertEqual(flags.generate_csv(),

            [ 'A,' + FLAG_GREYLIST + "," + FLAG_WHITELIST, 'B,' + FLAG_GREYLIST ])

            [ 'A,' + OLD_FLAG_UNSUPPORTED + "," + OLD_FLAG_SDK, 'B,' + OLD_FLAG_UNSUPPORTED ])



        # Test invalid API signature.

        with self.assertRaises(AssertionError):

            flags.assign_flag(FLAG_WHITELIST, set([ 'C' ]))

            flags.assign_flag(FLAG_SDK, set([ 'C' ]))



        # Test invalid flag.

        with self.assertRaises(AssertionError):