Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e7bc60a9 authored by Igor Viarheichyk's avatar Igor Viarheichyk
Browse files

Parse BCP 47 locale names with Unicode extensions 

Locale parser is modified to handle BCP 47 locale extensions and
recognize numbering system specifications. This fixes incorrect
interpretation of an extension key as a region code.

Numbering system specification is stored in the configuration data,
however it's not used for best configuration selection yet.

Bug: 67508876
Test: build and run libandroidfw_tests

Change-Id: If4072b9f39e80f60d4567841afe9ce7a92979574
parent 4749fdcf

libs/androidfw/ResourceTypes.cpp

+115 −13
Original line number Diff line number Diff line
@@ -2847,14 +2847,111 @@ void ResTable_config::getBcp47Locale(char str[RESTABLE_MAX_LOCALE_LEN], bool can 
        }

        memcpy(str + charsWritten, localeVariant, sizeof(localeVariant));

    }

}



/* static */ inline bool assignLocaleComponent(ResTable_config* config,

        const char* start, size_t size) {

    /* TODO: Add BCP47 extension. It requires RESTABLE_MAX_LOCALE_LEN

     * increase from 28 to 42 bytes (-u-nu-xxxxxxxx) */

}



struct LocaleParserState {

    enum State : uint8_t {

        BASE, UNICODE_EXTENSION, IGNORE_THE_REST

    } parserState;

    enum UnicodeState : uint8_t {

        /* Initial state after the Unicode singleton is detected. Either a keyword

         * or an attribute is expected. */

        NO_KEY,

        /* Unicode extension key (but not attribute) is expected. Next states:

         * NO_KEY, IGNORE_KEY or NUMBERING_SYSTEM. */

        EXPECT_KEY,

        /* A key is detected, however it is not supported for now. Ignore its

         * value. Next states: IGNORE_KEY or NUMBERING_SYSTEM. */

        IGNORE_KEY,

        /* Numbering system key was detected. Store its value in the configuration

         * localeNumberingSystem field. Next state: EXPECT_KEY */

        NUMBERING_SYSTEM

    } unicodeState;



    LocaleParserState(): parserState(BASE), unicodeState(NO_KEY) {}

};



/* static */ inline LocaleParserState assignLocaleComponent(ResTable_config* config,

        const char* start, size_t size, LocaleParserState state) {



    /* It is assumed that this function is not invoked with state.parserState

     * set to IGNORE_THE_REST. The condition is checked by setBcp47Locale

     * function. */



    if (state.parserState == LocaleParserState::UNICODE_EXTENSION) {

        switch (size) {

            case 1:

                /* Other BCP 47 extensions are not supported at the moment */

                state.parserState = LocaleParserState::IGNORE_THE_REST;

                break;

            case 2:

                if (state.unicodeState == LocaleParserState::NO_KEY ||

                    state.unicodeState == LocaleParserState::EXPECT_KEY) {

                    /* Analyze Unicode extension key. Currently only 'nu'

                     * (numbering system) is supported.*/

                    if ((start[0] == 'n' || start[0] == 'N') &&

                        (start[1] == 'u' || start[1] == 'U')) {

                        state.unicodeState = LocaleParserState::NUMBERING_SYSTEM;

                    } else {

                        state.unicodeState = LocaleParserState::IGNORE_KEY;

                    }

                } else {

                    /* Keys are not allowed in other state allowed, ignore the rest. */

                    state.parserState = LocaleParserState::IGNORE_THE_REST;

                }

                break;

            case 3:

            case 4:

            case 5:

            case 6:

            case 7:

            case 8:

                switch (state.unicodeState) {

                    case LocaleParserState::NUMBERING_SYSTEM:

                        /* Accept only the first occurrence of the numbering system. */

                        if (config->localeNumberingSystem[0] == '\0') {

                            for (size_t i = 0; i < size; ++i) {

                               config->localeNumberingSystem[i] = tolower(start[i]);

                            }

                            state.unicodeState = LocaleParserState::EXPECT_KEY;

                        } else {

                            state.parserState = LocaleParserState::IGNORE_THE_REST;

                        }

                        break;

                    case LocaleParserState::IGNORE_KEY:

                        /* Unsupported Unicode keyword. Ignore. */

                        state.unicodeState = LocaleParserState::EXPECT_KEY;

                        break;

                    case LocaleParserState::EXPECT_KEY:

                        /* A keyword followed by an attribute is not allowed. */

                        state.parserState = LocaleParserState::IGNORE_THE_REST;

                        break;

                    case LocaleParserState::NO_KEY:

                        /* Extension attribute. Do nothing. */

                        break;

                    default:

                        break;

                }

                break;

            default:

                /* Unexpected field length - ignore the rest and treat as an error */

                state.parserState = LocaleParserState::IGNORE_THE_REST;

        }

        return state;

    }



  switch (size) {

       case 0:

           return false;

           state.parserState = LocaleParserState::IGNORE_THE_REST;

           break;

       case 1:

           state.parserState = (start[0] == 'u' || start[0] == 'U')

                   ? LocaleParserState::UNICODE_EXTENSION

                   : LocaleParserState::IGNORE_THE_REST;

           break;

       case 2:

       case 3:

           config->language[0] ? config->packRegion(start) : config->packLanguage(start);
@@ -2878,30 +2975,35 @@ void ResTable_config::getBcp47Locale(char str[RESTABLE_MAX_LOCALE_LEN], bool can 
           }

           break;

       default:

           return false;

           state.parserState = LocaleParserState::IGNORE_THE_REST;

  }



  return true;

  return state;

}



void ResTable_config::setBcp47Locale(const char* in) {

    locale = 0;

    memset(localeScript, 0, sizeof(localeScript));

    memset(localeVariant, 0, sizeof(localeVariant));

    memset(localeNumberingSystem, 0, sizeof(localeNumberingSystem));



    const char* separator = in;

    const char* start = in;

    while ((separator = strchr(start, '-')) != NULL) {

    LocaleParserState state;

    while (const char* separator = strchr(start, '-')) {

        const size_t size = separator - start;

        if (!assignLocaleComponent(this, start, size)) {

            fprintf(stderr, "Invalid BCP-47 locale string: %s", in);

        state = assignLocaleComponent(this, start, size, state);

        if (state.parserState == LocaleParserState::IGNORE_THE_REST) {

            fprintf(stderr, "Invalid BCP-47 locale string: %s\n", in);

            break;

        }



        start = (separator + 1);

    }



    const size_t size = in + strlen(in) - start;

    assignLocaleComponent(this, start, size);

    if (state.parserState != LocaleParserState::IGNORE_THE_REST) {

        const size_t size = strlen(start);

        assignLocaleComponent(this, start, size, state);

    }



    localeScriptWasComputed = (localeScript[0] == '\0');

    if (localeScriptWasComputed) {

        computeScript();

libs/androidfw/include/androidfw/ResourceTypes.h

+8 −3
Original line number Diff line number Diff line
@@ -1179,6 +1179,10 @@ struct ResTable_config 
    // tried but could not compute a script.

    bool localeScriptWasComputed;



    // The value of BCP 47 Unicode extension for key 'nu' (numbering system).

    // Varies in length from 3 to 8 chars. Zero-filled value.

    char localeNumberingSystem[8];



    void copyFromDeviceNoSwap(const ResTable_config& o);

    

    void copyFromDtoH(const ResTable_config& o);
@@ -1256,9 +1260,9 @@ struct ResTable_config 
    // variants, it will be a modified bcp47 tag: b+en+Latn+US.

    void appendDirLocale(String8& str) const;



    // Sets the values of language, region, script and variant to the

    // well formed BCP-47 locale contained in |in|. The input locale is

    // assumed to be valid and no validation is performed.

    // Sets the values of language, region, script, variant and numbering

    // system to the well formed BCP 47 locale contained in |in|.

    // The input locale is assumed to be valid and no validation is performed.

    void setBcp47Locale(const char* in);



    inline void clearLocale() {
@@ -1266,6 +1270,7 @@ struct ResTable_config 
        localeScriptWasComputed = false;

        memset(localeScript, 0, sizeof(localeScript));

        memset(localeVariant, 0, sizeof(localeVariant));

        memset(localeNumberingSystem, 0, sizeof(localeNumberingSystem));

    }



    inline void computeScript() {

libs/androidfw/tests/ConfigLocale_test.cpp

+42 −0
Original line number Diff line number Diff line
@@ -185,6 +185,7 @@ TEST(ConfigLocaleTest, setLocale) { 
    EXPECT_TRUE(test.localeScriptWasComputed);

    EXPECT_EQ(0, memcmp("Latn", test.localeScript, 4));

    EXPECT_EQ(0, test.localeVariant[0]);

    EXPECT_EQ(0, test.localeNumberingSystem[0]);



    test.setBcp47Locale("eng-419");

    char out[4] = {1, 1, 1, 1};
@@ -198,6 +199,7 @@ TEST(ConfigLocaleTest, setLocale) { 
    EXPECT_EQ('4', out[0]);

    EXPECT_EQ('1', out[1]);

    EXPECT_EQ('9', out[2]);

    EXPECT_EQ(0, test.localeNumberingSystem[0]);



    test.setBcp47Locale("en-Latn-419");

    EXPECT_EQ('e', test.language[0]);
@@ -209,6 +211,7 @@ TEST(ConfigLocaleTest, setLocale) { 
    EXPECT_EQ('4', out[0]);

    EXPECT_EQ('1', out[1]);

    EXPECT_EQ('9', out[2]);

    EXPECT_EQ(0, test.localeNumberingSystem[0]);



    test.setBcp47Locale("de-1901");

    memset(out, 1, 4);
@@ -222,6 +225,7 @@ TEST(ConfigLocaleTest, setLocale) { 
    test.unpackRegion(out);

    EXPECT_EQ('\0', out[0]);

    EXPECT_EQ(0, strcmp("1901", test.localeVariant));

    EXPECT_EQ(0, test.localeNumberingSystem[0]);



    test.setBcp47Locale("de-Latn-1901");

    memset(out, 1, 4);
@@ -235,6 +239,44 @@ TEST(ConfigLocaleTest, setLocale) { 
    test.unpackRegion(out);

    EXPECT_EQ('\0', out[0]);

    EXPECT_EQ(0, strcmp("1901", test.localeVariant));

    EXPECT_EQ(0, test.localeNumberingSystem[0]);



    test.setBcp47Locale("ar-EG-u-nu-latn");

    EXPECT_EQ('a', test.language[0]);

    EXPECT_EQ('r', test.language[1]);

    EXPECT_EQ('E', test.country[0]);

    EXPECT_EQ('G', test.country[1]);

    EXPECT_TRUE(test.localeScriptWasComputed);

    EXPECT_EQ(0, memcmp("Arab", test.localeScript, 4));

    EXPECT_EQ(0, test.localeVariant[0]);

    EXPECT_EQ(0, memcmp("latn", test.localeNumberingSystem, 4));



    test.setBcp47Locale("ar-EG-u");

    EXPECT_EQ(0, test.localeNumberingSystem[0]);



    test.setBcp47Locale("ar-EG-u-nu");

    EXPECT_EQ(0, test.localeNumberingSystem[0]);



    test.setBcp47Locale("ar-EG-u-attr-nu-latn");

    EXPECT_EQ(0, memcmp("latn", test.localeNumberingSystem, 4));



    test.setBcp47Locale("ar-EG-u-ca-gregory-nu-latn");

    EXPECT_EQ(0, memcmp("latn", test.localeNumberingSystem, 4));



    test.setBcp47Locale("ar-EG-u-nu-latn-ca-gregory");

    EXPECT_EQ(0, memcmp("latn", test.localeNumberingSystem, 4));



    test.setBcp47Locale("ar-EG-u-nu-toolongnumsys");

    EXPECT_EQ(0, test.localeNumberingSystem[0]);



    test.setBcp47Locale("ar-EG-u-nu-latn-nu-arab");

    EXPECT_EQ(0, memcmp("latn", test.localeNumberingSystem, 4));



    test.setBcp47Locale("ar-EG-u-co-nu-latn");

    EXPECT_EQ(0, test.localeNumberingSystem[0]);



    test.setBcp47Locale("ar-u-co-abcd-attr-nu-latn");

    EXPECT_EQ(0, test.localeNumberingSystem[0]);

}



TEST(ConfigLocaleTest, computeScript) {