Loading tools/aapt/Android.mk +2 −1 Original line number Original line Diff line number Diff line Loading @@ -50,9 +50,11 @@ aaptSources := \ aaptTests := \ aaptTests := \ tests/AaptConfig_test.cpp \ tests/AaptConfig_test.cpp \ tests/AaptGroupEntry_test.cpp \ tests/AaptGroupEntry_test.cpp \ tests/Pseudolocales_test.cpp \ tests/ResourceFilter_test.cpp tests/ResourceFilter_test.cpp aaptCIncludes := \ aaptCIncludes := \ system/core/base/include \ external/libpng \ external/libpng \ external/zlib external/zlib Loading Loading @@ -99,7 +101,6 @@ LOCAL_SRC_FILES := $(aaptSources) include $(BUILD_HOST_STATIC_LIBRARY) include $(BUILD_HOST_STATIC_LIBRARY) # ========================================================== # ========================================================== # Build the host executable: aapt # Build the host executable: aapt # ========================================================== # ========================================================== Loading tools/aapt/XMLNode.cpp +6 −31 Original line number Original line Diff line number Diff line Loading @@ -213,16 +213,14 @@ status_t parseStyledString(Bundle* /* bundle */, Vector<StringPool::entry_style_span> spanStack; Vector<StringPool::entry_style_span> spanStack; String16 curString; String16 curString; String16 rawString; String16 rawString; Pseudolocalizer pseudo(pseudolocalize); const char* errorMsg; const char* errorMsg; int xliffDepth = 0; int xliffDepth = 0; bool firstTime = true; bool firstTime = true; size_t len; size_t len; ResXMLTree::event_code_t code; ResXMLTree::event_code_t code; // Bracketing if pseudolocalization accented method specified. curString.append(pseudo.start()); if (pseudolocalize == PSEUDO_ACCENTED) { curString.append(String16(String8("["))); } while ((code=inXml->next()) != ResXMLTree::END_DOCUMENT && code != ResXMLTree::BAD_DOCUMENT) { while ((code=inXml->next()) != ResXMLTree::END_DOCUMENT && code != ResXMLTree::BAD_DOCUMENT) { if (code == ResXMLTree::TEXT) { if (code == ResXMLTree::TEXT) { String16 text(inXml->getText(&len)); String16 text(inXml->getText(&len)); Loading @@ -231,18 +229,12 @@ status_t parseStyledString(Bundle* /* bundle */, if (text.string()[0] == '@') { if (text.string()[0] == '@') { // If this is a resource reference, don't do the pseudoloc. // If this is a resource reference, don't do the pseudoloc. pseudolocalize = NO_PSEUDOLOCALIZATION; pseudolocalize = NO_PSEUDOLOCALIZATION; pseudo.setMethod(pseudolocalize); curString = String16(); } } } } if (xliffDepth == 0 && pseudolocalize > 0) { if (xliffDepth == 0 && pseudolocalize > 0) { String16 pseudo; curString.append(pseudo.text(text)); if (pseudolocalize == PSEUDO_ACCENTED) { pseudo = pseudolocalize_string(text); } else if (pseudolocalize == PSEUDO_BIDI) { pseudo = pseudobidi_string(text); } else { pseudo = text; } curString.append(pseudo); } else { } else { if (isFormatted && hasSubstitutionErrors(fileName, inXml, text) != NO_ERROR) { if (isFormatted && hasSubstitutionErrors(fileName, inXml, text) != NO_ERROR) { return UNKNOWN_ERROR; return UNKNOWN_ERROR; Loading Loading @@ -382,24 +374,7 @@ moveon: } } } } // Bracketing if pseudolocalization accented method specified. curString.append(pseudo.end()); if (pseudolocalize == PSEUDO_ACCENTED) { const char16_t* str = outString->string(); const char16_t* p = str; const char16_t* e = p + outString->size(); int words_cnt = 0; while (p < e) { if (isspace(*p)) { words_cnt++; } p++; } unsigned int length = words_cnt > 3 ? outString->size() : outString->size() / 2; curString.append(String16(String8(" "))); curString.append(pseudo_generate_expansion(length)); curString.append(String16(String8("]"))); } if (code == ResXMLTree::BAD_DOCUMENT) { if (code == ResXMLTree::BAD_DOCUMENT) { SourcePos(String8(fileName), inXml->getLineNumber()).error( SourcePos(String8(fileName), inXml->getLineNumber()).error( Loading tools/aapt/pseudolocalize.cpp +145 −26 Original line number Original line Diff line number Diff line Loading @@ -16,6 +16,80 @@ static const String16 k_pdf = String16("\xE2\x80\xac"); static const String16 k_placeholder_open = String16("\xc2\xbb"); static const String16 k_placeholder_open = String16("\xc2\xbb"); static const String16 k_placeholder_close = String16("\xc2\xab"); static const String16 k_placeholder_close = String16("\xc2\xab"); static const char16_t k_arg_start = '{'; static const char16_t k_arg_end = '}'; Pseudolocalizer::Pseudolocalizer(PseudolocalizationMethod m) : mImpl(nullptr), mLastDepth(0) { setMethod(m); } void Pseudolocalizer::setMethod(PseudolocalizationMethod m) { if (mImpl) { delete mImpl; } if (m == PSEUDO_ACCENTED) { mImpl = new PseudoMethodAccent(); } else if (m == PSEUDO_BIDI) { mImpl = new PseudoMethodBidi(); } else { mImpl = new PseudoMethodNone(); } } String16 Pseudolocalizer::text(const String16& text) { String16 out; size_t depth = mLastDepth; size_t lastpos, pos; const size_t length= text.size(); const char16_t* str = text.string(); bool escaped = false; for (lastpos = pos = 0; pos < length; pos++) { char16_t c = str[pos]; if (escaped) { escaped = false; continue; } if (c == '\'') { escaped = true; continue; } if (c == k_arg_start) { depth++; } else if (c == k_arg_end && depth) { depth--; } if (mLastDepth != depth || pos == length - 1) { bool pseudo = ((mLastDepth % 2) == 0); size_t nextpos = pos; if (!pseudo || depth == mLastDepth) { nextpos++; } size_t size = nextpos - lastpos; if (size) { String16 chunk = String16(text, size, lastpos); if (pseudo) { chunk = mImpl->text(chunk); } else if (str[lastpos] == k_arg_start && str[nextpos - 1] == k_arg_end) { chunk = mImpl->placeholder(chunk); } out.append(chunk); } if (pseudo && depth < mLastDepth) { // End of message out.append(mImpl->end()); } else if (!pseudo && depth > mLastDepth) { // Start of message out.append(mImpl->start()); } lastpos = nextpos; mLastDepth = depth; } } return out; } static const char* static const char* pseudolocalize_char(const char16_t c) pseudolocalize_char(const char16_t c) { { Loading Loading @@ -78,8 +152,7 @@ pseudolocalize_char(const char16_t c) } } } } static bool static bool is_possible_normal_placeholder_end(const char16_t c) { is_possible_normal_placeholder_end(const char16_t c) { switch (c) { switch (c) { case 's': return true; case 's': return true; case 'S': return true; case 'S': return true; Loading @@ -106,8 +179,7 @@ is_possible_normal_placeholder_end(const char16_t c) { } } } } String16 static String16 pseudo_generate_expansion(const unsigned int length) { pseudo_generate_expansion(const unsigned int length) { String16 result = k_expansion_string; String16 result = k_expansion_string; const char16_t* s = result.string(); const char16_t* s = result.string(); if (result.size() < length) { if (result.size() < length) { Loading @@ -127,18 +199,47 @@ pseudo_generate_expansion(const unsigned int length) { return result; return result; } } static bool is_space(const char16_t c) { return (c == ' ' || c == '\t' || c == '\n'); } String16 PseudoMethodAccent::start() { String16 result; if (mDepth == 0) { result = String16(String8("[")); } mWordCount = mLength = 0; mDepth++; return result; } String16 PseudoMethodAccent::end() { String16 result; if (mLength) { result.append(String16(String8(" "))); result.append(pseudo_generate_expansion( mWordCount > 3 ? mLength : mLength / 2)); } mWordCount = mLength = 0; mDepth--; if (mDepth == 0) { result.append(String16(String8("]"))); } return result; } /** /** * Converts characters so they look like they've been localized. * Converts characters so they look like they've been localized. * * * Note: This leaves escape sequences untouched so they can later be * Note: This leaves escape sequences untouched so they can later be * processed by ResTable::collectString in the normal way. * processed by ResTable::collectString in the normal way. */ */ String16 String16 PseudoMethodAccent::text(const String16& source) pseudolocalize_string(const String16& source) { { const char16_t* s = source.string(); const char16_t* s = source.string(); String16 result; String16 result; const size_t I = source.size(); const size_t I = source.size(); bool lastspace = true; for (size_t i=0; i<I; i++) { for (size_t i=0; i<I; i++) { char16_t c = s[i]; char16_t c = s[i]; if (c == '\\') { if (c == '\\') { Loading Loading @@ -170,23 +271,24 @@ pseudolocalize_string(const String16& source) } } } else if (c == '%') { } else if (c == '%') { // Placeholder syntax, no need to pseudolocalize // Placeholder syntax, no need to pseudolocalize result += k_placeholder_open; String16 chunk; bool end = false; bool end = false; result.append(&c, 1); chunk.append(&c, 1); while (!end && i < I) { while (!end && i < I) { ++i; ++i; c = s[i]; c = s[i]; result.append(&c, 1); chunk.append(&c, 1); if (is_possible_normal_placeholder_end(c)) { if (is_possible_normal_placeholder_end(c)) { end = true; end = true; } else if (c == 't') { } else if (c == 't') { ++i; ++i; c = s[i]; c = s[i]; result.append(&c, 1); chunk.append(&c, 1); end = true; end = true; } } } } result += k_placeholder_close; // Treat chunk as a placeholder unless it ends with %. result += ((c == '%') ? chunk : placeholder(chunk)); } else if (c == '<' || c == '&') { } else if (c == '<' || c == '&') { // html syntax, no need to pseudolocalize // html syntax, no need to pseudolocalize bool tag_closed = false; bool tag_closed = false; Loading Loading @@ -234,35 +336,52 @@ pseudolocalize_string(const String16& source) if (p != NULL) { if (p != NULL) { result += String16(p); result += String16(p); } else { } else { bool space = is_space(c); if (lastspace && !space) { mWordCount++; } lastspace = space; result.append(&c, 1); result.append(&c, 1); } } // Count only pseudolocalizable chars and delimiters mLength++; } } } } return result; return result; } } String16 PseudoMethodAccent::placeholder(const String16& source) { // Surround a placeholder with brackets return k_placeholder_open + source + k_placeholder_close; } String16 String16 PseudoMethodBidi::text(const String16& source) pseudobidi_string(const String16& source) { { const char16_t* s = source.string(); const char16_t* s = source.string(); String16 result; String16 result; result += k_rlm; bool lastspace = true; result += k_rlo; bool space = true; for (size_t i=0; i<source.size(); i++) { for (size_t i=0; i<source.size(); i++) { char16_t c = s[i]; char16_t c = s[i]; switch(c) { space = is_space(c); case ' ': result += k_pdf; if (lastspace && !space) { result += k_rlm; // Word start result += k_rlm + k_rlo; } else if (!lastspace && space) { // Word end result += k_pdf + k_rlm; } lastspace = space; result.append(&c, 1); result.append(&c, 1); result += k_rlm; result += k_rlo; break; default: result.append(&c, 1); break; } } if (!lastspace) { // End of last word result += k_pdf + k_rlm; } } result += k_pdf; result += k_rlm; return result; return result; } } String16 PseudoMethodBidi::placeholder(const String16& source) { // Surround a placeholder with directionality change sequence return k_rlm + k_rlo + source + k_pdf + k_rlm; } tools/aapt/pseudolocalize.h +49 −9 Original line number Original line Diff line number Diff line #ifndef HOST_PSEUDOLOCALIZE_H #ifndef HOST_PSEUDOLOCALIZE_H #define HOST_PSEUDOLOCALIZE_H #define HOST_PSEUDOLOCALIZE_H #include <base/macros.h> #include "StringPool.h" #include "StringPool.h" #include <string> class PseudoMethodImpl { public: virtual ~PseudoMethodImpl() {} virtual String16 start() { return String16(); } virtual String16 end() { return String16(); } virtual String16 text(const String16& text) = 0; virtual String16 placeholder(const String16& text) = 0; }; String16 pseudolocalize_string(const String16& source); class PseudoMethodNone : public PseudoMethodImpl { // Surrounds every word in the sentance with specific characters that makes public: // the word directionality RTL. PseudoMethodNone() {} String16 pseudobidi_string(const String16& source); String16 text(const String16& text) { return text; } // Generates expansion string based on the specified lenght. String16 placeholder(const String16& text) { return text; } // Generated string could not be shorter that length, but it could be slightly private: // longer. DISALLOW_COPY_AND_ASSIGN(PseudoMethodNone); String16 pseudo_generate_expansion(const unsigned int length); }; class PseudoMethodBidi : public PseudoMethodImpl { public: String16 text(const String16& text); String16 placeholder(const String16& text); }; class PseudoMethodAccent : public PseudoMethodImpl { public: PseudoMethodAccent() : mDepth(0), mWordCount(0), mLength(0) {} String16 start(); String16 end(); String16 text(const String16& text); String16 placeholder(const String16& text); private: size_t mDepth; size_t mWordCount; size_t mLength; }; class Pseudolocalizer { public: Pseudolocalizer(PseudolocalizationMethod m); ~Pseudolocalizer() { if (mImpl) delete mImpl; } void setMethod(PseudolocalizationMethod m); String16 start() { return mImpl->start(); } String16 end() { return mImpl->end(); } String16 text(const String16& text); private: PseudoMethodImpl *mImpl; size_t mLastDepth; }; #endif // HOST_PSEUDOLOCALIZE_H #endif // HOST_PSEUDOLOCALIZE_H tools/aapt/tests/Pseudolocales_test.cpp 0 → 100644 +217 −0 Original line number Original line Diff line number Diff line /* * Copyright (C) 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include <androidfw/ResourceTypes.h> #include <utils/String8.h> #include <gtest/gtest.h> #include "Bundle.h" #include "pseudolocalize.h" using android::String8; // In this context, 'Axis' represents a particular field in the configuration, // such as language or density. static void simple_helper(const char* input, const char* expected, PseudolocalizationMethod method) { Pseudolocalizer pseudo(method); String16 result = pseudo.start() + pseudo.text(String16(String8(input))) + pseudo.end(); //std::cout << String8(result).string() << std::endl; ASSERT_EQ(String8(expected), String8(result)); } static void compound_helper(const char* in1, const char* in2, const char *in3, const char* expected, PseudolocalizationMethod method) { Pseudolocalizer pseudo(method); String16 result = pseudo.start() + \ pseudo.text(String16(String8(in1))) + \ pseudo.text(String16(String8(in2))) + \ pseudo.text(String16(String8(in3))) + \ pseudo.end(); ASSERT_EQ(String8(expected), String8(result)); } TEST(Pseudolocales, NoPseudolocalization) { simple_helper("", "", NO_PSEUDOLOCALIZATION); simple_helper("Hello, world", "Hello, world", NO_PSEUDOLOCALIZATION); compound_helper("Hello,", " world", "", "Hello, world", NO_PSEUDOLOCALIZATION); } TEST(Pseudolocales, PlaintextAccent) { simple_helper("", "[]", PSEUDO_ACCENTED); simple_helper("Hello, world", "[Ĥéļļö, ŵöŕļð one two]", PSEUDO_ACCENTED); simple_helper("Hello, %1d", "[Ĥéļļö, »%1d« one two]", PSEUDO_ACCENTED); simple_helper("Battery %1d%%", "[βåţţéŕý »%1d«%% one two]", PSEUDO_ACCENTED); compound_helper("", "", "", "[]", PSEUDO_ACCENTED); compound_helper("Hello,", " world", "", "[Ĥéļļö, ŵöŕļð one two]", PSEUDO_ACCENTED); } TEST(Pseudolocales, PlaintextBidi) { simple_helper("", "", PSEUDO_BIDI); simple_helper("word", "\xe2\x80\x8f\xE2\x80\xaeword\xE2\x80\xac\xe2\x80\x8f", PSEUDO_BIDI); simple_helper(" word ", " \xe2\x80\x8f\xE2\x80\xaeword\xE2\x80\xac\xe2\x80\x8f ", PSEUDO_BIDI); simple_helper(" word ", " \xe2\x80\x8f\xE2\x80\xaeword\xE2\x80\xac\xe2\x80\x8f ", PSEUDO_BIDI); simple_helper("hello\n world\n", "\xe2\x80\x8f\xE2\x80\xaehello\xE2\x80\xac\xe2\x80\x8f\n" \ " \xe2\x80\x8f\xE2\x80\xaeworld\xE2\x80\xac\xe2\x80\x8f\n", PSEUDO_BIDI); compound_helper("hello", "\n ", " world\n", "\xe2\x80\x8f\xE2\x80\xaehello\xE2\x80\xac\xe2\x80\x8f\n" \ " \xe2\x80\x8f\xE2\x80\xaeworld\xE2\x80\xac\xe2\x80\x8f\n", PSEUDO_BIDI); } TEST(Pseudolocales, SimpleICU) { // Single-fragment messages simple_helper("{placeholder}", "[»{placeholder}«]", PSEUDO_ACCENTED); simple_helper("{USER} is offline", "[»{USER}« îš öƒƒļîñé one two]", PSEUDO_ACCENTED); simple_helper("Copy from {path1} to {path2}", "[Çöþý ƒŕöḿ »{path1}« ţö »{path2}« one two three]", PSEUDO_ACCENTED); simple_helper("Today is {1,date} {1,time}", "[Ţöðåý îš »{1,date}« »{1,time}« one two]", PSEUDO_ACCENTED); // Multi-fragment messages compound_helper("{USER}", " ", "is offline", "[»{USER}« îš öƒƒļîñé one two]", PSEUDO_ACCENTED); compound_helper("Copy from ", "{path1}", " to {path2}", "[Çöþý ƒŕöḿ »{path1}« ţö »{path2}« one two three]", PSEUDO_ACCENTED); } TEST(Pseudolocales, ICUBidi) { // Single-fragment messages simple_helper("{placeholder}", "\xe2\x80\x8f\xE2\x80\xae{placeholder}\xE2\x80\xac\xe2\x80\x8f", PSEUDO_BIDI); simple_helper( "{COUNT, plural, one {one} other {other}}", "{COUNT, plural, " \ "one {\xe2\x80\x8f\xE2\x80\xaeone\xE2\x80\xac\xe2\x80\x8f} " \ "other {\xe2\x80\x8f\xE2\x80\xaeother\xE2\x80\xac\xe2\x80\x8f}}", PSEUDO_BIDI ); } TEST(Pseudolocales, Escaping) { // Single-fragment messages simple_helper("'{USER'} is offline", "['{ÛŠÉŔ'} îš öƒƒļîñé one two three]", PSEUDO_ACCENTED); // Multi-fragment messages compound_helper("'{USER}", " ", "''is offline", "['{ÛŠÉŔ} ''îš öƒƒļîñé one two three]", PSEUDO_ACCENTED); } TEST(Pseudolocales, PluralsAndSelects) { simple_helper( "{COUNT, plural, one {Delete a file} other {Delete {COUNT} files}}", "[{COUNT, plural, one {Ðéļéţé å ƒîļé one two} " \ "other {Ðéļéţé »{COUNT}« ƒîļéš one two}}]", PSEUDO_ACCENTED ); simple_helper( "Distance is {COUNT, plural, one {# mile} other {# miles}}", "[Ðîšţåñçé îš {COUNT, plural, one {# ḿîļé one two} " \ "other {# ḿîļéš one two}}]", PSEUDO_ACCENTED ); simple_helper( "{1, select, female {{1} added you} " \ "male {{1} added you} other {{1} added you}}", "[{1, select, female {»{1}« åððéð ýöû one two} " \ "male {»{1}« åððéð ýöû one two} other {»{1}« åððéð ýöû one two}}]", PSEUDO_ACCENTED ); compound_helper( "{COUNT, plural, one {Delete a file} " \ "other {Delete ", "{COUNT}", " files}}", "[{COUNT, plural, one {Ðéļéţé å ƒîļé one two} " \ "other {Ðéļéţé »{COUNT}« ƒîļéš one two}}]", PSEUDO_ACCENTED ); } TEST(Pseudolocales, NestedICU) { simple_helper( "{person, select, " \ "female {" \ "{num_circles, plural," \ "=0{{person} didn't add you to any of her circles.}" \ "=1{{person} added you to one of her circles.}" \ "other{{person} added you to her # circles.}}}" \ "male {" \ "{num_circles, plural," \ "=0{{person} didn't add you to any of his circles.}" \ "=1{{person} added you to one of his circles.}" \ "other{{person} added you to his # circles.}}}" \ "other {" \ "{num_circles, plural," \ "=0{{person} didn't add you to any of their circles.}" \ "=1{{person} added you to one of their circles.}" \ "other{{person} added you to their # circles.}}}}", "[{person, select, " \ "female {" \ "{num_circles, plural," \ "=0{»{person}« ðîðñ'ţ åðð ýöû ţö åñý öƒ ĥéŕ çîŕçļéš." \ " one two three four five}" \ "=1{»{person}« åððéð ýöû ţö öñé öƒ ĥéŕ çîŕçļéš." \ " one two three four}" \ "other{»{person}« åððéð ýöû ţö ĥéŕ # çîŕçļéš." \ " one two three four}}}" \ "male {" \ "{num_circles, plural," \ "=0{»{person}« ðîðñ'ţ åðð ýöû ţö åñý öƒ ĥîš çîŕçļéš." \ " one two three four five}" \ "=1{»{person}« åððéð ýöû ţö öñé öƒ ĥîš çîŕçļéš." \ " one two three four}" \ "other{»{person}« åððéð ýöû ţö ĥîš # çîŕçļéš." \ " one two three four}}}" \ "other {{num_circles, plural," \ "=0{»{person}« ðîðñ'ţ åðð ýöû ţö åñý öƒ ţĥéîŕ çîŕçļéš." \ " one two three four five}" \ "=1{»{person}« åððéð ýöû ţö öñé öƒ ţĥéîŕ çîŕçļéš." \ " one two three four}" \ "other{»{person}« åððéð ýöû ţö ţĥéîŕ # çîŕçļéš." \ " one two three four}}}}]", PSEUDO_ACCENTED ); } TEST(Pseudolocales, RedefineMethod) { Pseudolocalizer pseudo(PSEUDO_ACCENTED); String16 result = pseudo.text(String16(String8("Hello, "))); pseudo.setMethod(NO_PSEUDOLOCALIZATION); result.append(pseudo.text(String16(String8("world!")))); ASSERT_EQ(String8("Ĥéļļö, world!"), String8(result)); } Loading
tools/aapt/Android.mk +2 −1 Original line number Original line Diff line number Diff line Loading @@ -50,9 +50,11 @@ aaptSources := \ aaptTests := \ aaptTests := \ tests/AaptConfig_test.cpp \ tests/AaptConfig_test.cpp \ tests/AaptGroupEntry_test.cpp \ tests/AaptGroupEntry_test.cpp \ tests/Pseudolocales_test.cpp \ tests/ResourceFilter_test.cpp tests/ResourceFilter_test.cpp aaptCIncludes := \ aaptCIncludes := \ system/core/base/include \ external/libpng \ external/libpng \ external/zlib external/zlib Loading Loading @@ -99,7 +101,6 @@ LOCAL_SRC_FILES := $(aaptSources) include $(BUILD_HOST_STATIC_LIBRARY) include $(BUILD_HOST_STATIC_LIBRARY) # ========================================================== # ========================================================== # Build the host executable: aapt # Build the host executable: aapt # ========================================================== # ========================================================== Loading
tools/aapt/XMLNode.cpp +6 −31 Original line number Original line Diff line number Diff line Loading @@ -213,16 +213,14 @@ status_t parseStyledString(Bundle* /* bundle */, Vector<StringPool::entry_style_span> spanStack; Vector<StringPool::entry_style_span> spanStack; String16 curString; String16 curString; String16 rawString; String16 rawString; Pseudolocalizer pseudo(pseudolocalize); const char* errorMsg; const char* errorMsg; int xliffDepth = 0; int xliffDepth = 0; bool firstTime = true; bool firstTime = true; size_t len; size_t len; ResXMLTree::event_code_t code; ResXMLTree::event_code_t code; // Bracketing if pseudolocalization accented method specified. curString.append(pseudo.start()); if (pseudolocalize == PSEUDO_ACCENTED) { curString.append(String16(String8("["))); } while ((code=inXml->next()) != ResXMLTree::END_DOCUMENT && code != ResXMLTree::BAD_DOCUMENT) { while ((code=inXml->next()) != ResXMLTree::END_DOCUMENT && code != ResXMLTree::BAD_DOCUMENT) { if (code == ResXMLTree::TEXT) { if (code == ResXMLTree::TEXT) { String16 text(inXml->getText(&len)); String16 text(inXml->getText(&len)); Loading @@ -231,18 +229,12 @@ status_t parseStyledString(Bundle* /* bundle */, if (text.string()[0] == '@') { if (text.string()[0] == '@') { // If this is a resource reference, don't do the pseudoloc. // If this is a resource reference, don't do the pseudoloc. pseudolocalize = NO_PSEUDOLOCALIZATION; pseudolocalize = NO_PSEUDOLOCALIZATION; pseudo.setMethod(pseudolocalize); curString = String16(); } } } } if (xliffDepth == 0 && pseudolocalize > 0) { if (xliffDepth == 0 && pseudolocalize > 0) { String16 pseudo; curString.append(pseudo.text(text)); if (pseudolocalize == PSEUDO_ACCENTED) { pseudo = pseudolocalize_string(text); } else if (pseudolocalize == PSEUDO_BIDI) { pseudo = pseudobidi_string(text); } else { pseudo = text; } curString.append(pseudo); } else { } else { if (isFormatted && hasSubstitutionErrors(fileName, inXml, text) != NO_ERROR) { if (isFormatted && hasSubstitutionErrors(fileName, inXml, text) != NO_ERROR) { return UNKNOWN_ERROR; return UNKNOWN_ERROR; Loading Loading @@ -382,24 +374,7 @@ moveon: } } } } // Bracketing if pseudolocalization accented method specified. curString.append(pseudo.end()); if (pseudolocalize == PSEUDO_ACCENTED) { const char16_t* str = outString->string(); const char16_t* p = str; const char16_t* e = p + outString->size(); int words_cnt = 0; while (p < e) { if (isspace(*p)) { words_cnt++; } p++; } unsigned int length = words_cnt > 3 ? outString->size() : outString->size() / 2; curString.append(String16(String8(" "))); curString.append(pseudo_generate_expansion(length)); curString.append(String16(String8("]"))); } if (code == ResXMLTree::BAD_DOCUMENT) { if (code == ResXMLTree::BAD_DOCUMENT) { SourcePos(String8(fileName), inXml->getLineNumber()).error( SourcePos(String8(fileName), inXml->getLineNumber()).error( Loading
tools/aapt/pseudolocalize.cpp +145 −26 Original line number Original line Diff line number Diff line Loading @@ -16,6 +16,80 @@ static const String16 k_pdf = String16("\xE2\x80\xac"); static const String16 k_placeholder_open = String16("\xc2\xbb"); static const String16 k_placeholder_open = String16("\xc2\xbb"); static const String16 k_placeholder_close = String16("\xc2\xab"); static const String16 k_placeholder_close = String16("\xc2\xab"); static const char16_t k_arg_start = '{'; static const char16_t k_arg_end = '}'; Pseudolocalizer::Pseudolocalizer(PseudolocalizationMethod m) : mImpl(nullptr), mLastDepth(0) { setMethod(m); } void Pseudolocalizer::setMethod(PseudolocalizationMethod m) { if (mImpl) { delete mImpl; } if (m == PSEUDO_ACCENTED) { mImpl = new PseudoMethodAccent(); } else if (m == PSEUDO_BIDI) { mImpl = new PseudoMethodBidi(); } else { mImpl = new PseudoMethodNone(); } } String16 Pseudolocalizer::text(const String16& text) { String16 out; size_t depth = mLastDepth; size_t lastpos, pos; const size_t length= text.size(); const char16_t* str = text.string(); bool escaped = false; for (lastpos = pos = 0; pos < length; pos++) { char16_t c = str[pos]; if (escaped) { escaped = false; continue; } if (c == '\'') { escaped = true; continue; } if (c == k_arg_start) { depth++; } else if (c == k_arg_end && depth) { depth--; } if (mLastDepth != depth || pos == length - 1) { bool pseudo = ((mLastDepth % 2) == 0); size_t nextpos = pos; if (!pseudo || depth == mLastDepth) { nextpos++; } size_t size = nextpos - lastpos; if (size) { String16 chunk = String16(text, size, lastpos); if (pseudo) { chunk = mImpl->text(chunk); } else if (str[lastpos] == k_arg_start && str[nextpos - 1] == k_arg_end) { chunk = mImpl->placeholder(chunk); } out.append(chunk); } if (pseudo && depth < mLastDepth) { // End of message out.append(mImpl->end()); } else if (!pseudo && depth > mLastDepth) { // Start of message out.append(mImpl->start()); } lastpos = nextpos; mLastDepth = depth; } } return out; } static const char* static const char* pseudolocalize_char(const char16_t c) pseudolocalize_char(const char16_t c) { { Loading Loading @@ -78,8 +152,7 @@ pseudolocalize_char(const char16_t c) } } } } static bool static bool is_possible_normal_placeholder_end(const char16_t c) { is_possible_normal_placeholder_end(const char16_t c) { switch (c) { switch (c) { case 's': return true; case 's': return true; case 'S': return true; case 'S': return true; Loading @@ -106,8 +179,7 @@ is_possible_normal_placeholder_end(const char16_t c) { } } } } String16 static String16 pseudo_generate_expansion(const unsigned int length) { pseudo_generate_expansion(const unsigned int length) { String16 result = k_expansion_string; String16 result = k_expansion_string; const char16_t* s = result.string(); const char16_t* s = result.string(); if (result.size() < length) { if (result.size() < length) { Loading @@ -127,18 +199,47 @@ pseudo_generate_expansion(const unsigned int length) { return result; return result; } } static bool is_space(const char16_t c) { return (c == ' ' || c == '\t' || c == '\n'); } String16 PseudoMethodAccent::start() { String16 result; if (mDepth == 0) { result = String16(String8("[")); } mWordCount = mLength = 0; mDepth++; return result; } String16 PseudoMethodAccent::end() { String16 result; if (mLength) { result.append(String16(String8(" "))); result.append(pseudo_generate_expansion( mWordCount > 3 ? mLength : mLength / 2)); } mWordCount = mLength = 0; mDepth--; if (mDepth == 0) { result.append(String16(String8("]"))); } return result; } /** /** * Converts characters so they look like they've been localized. * Converts characters so they look like they've been localized. * * * Note: This leaves escape sequences untouched so they can later be * Note: This leaves escape sequences untouched so they can later be * processed by ResTable::collectString in the normal way. * processed by ResTable::collectString in the normal way. */ */ String16 String16 PseudoMethodAccent::text(const String16& source) pseudolocalize_string(const String16& source) { { const char16_t* s = source.string(); const char16_t* s = source.string(); String16 result; String16 result; const size_t I = source.size(); const size_t I = source.size(); bool lastspace = true; for (size_t i=0; i<I; i++) { for (size_t i=0; i<I; i++) { char16_t c = s[i]; char16_t c = s[i]; if (c == '\\') { if (c == '\\') { Loading Loading @@ -170,23 +271,24 @@ pseudolocalize_string(const String16& source) } } } else if (c == '%') { } else if (c == '%') { // Placeholder syntax, no need to pseudolocalize // Placeholder syntax, no need to pseudolocalize result += k_placeholder_open; String16 chunk; bool end = false; bool end = false; result.append(&c, 1); chunk.append(&c, 1); while (!end && i < I) { while (!end && i < I) { ++i; ++i; c = s[i]; c = s[i]; result.append(&c, 1); chunk.append(&c, 1); if (is_possible_normal_placeholder_end(c)) { if (is_possible_normal_placeholder_end(c)) { end = true; end = true; } else if (c == 't') { } else if (c == 't') { ++i; ++i; c = s[i]; c = s[i]; result.append(&c, 1); chunk.append(&c, 1); end = true; end = true; } } } } result += k_placeholder_close; // Treat chunk as a placeholder unless it ends with %. result += ((c == '%') ? chunk : placeholder(chunk)); } else if (c == '<' || c == '&') { } else if (c == '<' || c == '&') { // html syntax, no need to pseudolocalize // html syntax, no need to pseudolocalize bool tag_closed = false; bool tag_closed = false; Loading Loading @@ -234,35 +336,52 @@ pseudolocalize_string(const String16& source) if (p != NULL) { if (p != NULL) { result += String16(p); result += String16(p); } else { } else { bool space = is_space(c); if (lastspace && !space) { mWordCount++; } lastspace = space; result.append(&c, 1); result.append(&c, 1); } } // Count only pseudolocalizable chars and delimiters mLength++; } } } } return result; return result; } } String16 PseudoMethodAccent::placeholder(const String16& source) { // Surround a placeholder with brackets return k_placeholder_open + source + k_placeholder_close; } String16 String16 PseudoMethodBidi::text(const String16& source) pseudobidi_string(const String16& source) { { const char16_t* s = source.string(); const char16_t* s = source.string(); String16 result; String16 result; result += k_rlm; bool lastspace = true; result += k_rlo; bool space = true; for (size_t i=0; i<source.size(); i++) { for (size_t i=0; i<source.size(); i++) { char16_t c = s[i]; char16_t c = s[i]; switch(c) { space = is_space(c); case ' ': result += k_pdf; if (lastspace && !space) { result += k_rlm; // Word start result += k_rlm + k_rlo; } else if (!lastspace && space) { // Word end result += k_pdf + k_rlm; } lastspace = space; result.append(&c, 1); result.append(&c, 1); result += k_rlm; result += k_rlo; break; default: result.append(&c, 1); break; } } if (!lastspace) { // End of last word result += k_pdf + k_rlm; } } result += k_pdf; result += k_rlm; return result; return result; } } String16 PseudoMethodBidi::placeholder(const String16& source) { // Surround a placeholder with directionality change sequence return k_rlm + k_rlo + source + k_pdf + k_rlm; }
tools/aapt/pseudolocalize.h +49 −9 Original line number Original line Diff line number Diff line #ifndef HOST_PSEUDOLOCALIZE_H #ifndef HOST_PSEUDOLOCALIZE_H #define HOST_PSEUDOLOCALIZE_H #define HOST_PSEUDOLOCALIZE_H #include <base/macros.h> #include "StringPool.h" #include "StringPool.h" #include <string> class PseudoMethodImpl { public: virtual ~PseudoMethodImpl() {} virtual String16 start() { return String16(); } virtual String16 end() { return String16(); } virtual String16 text(const String16& text) = 0; virtual String16 placeholder(const String16& text) = 0; }; String16 pseudolocalize_string(const String16& source); class PseudoMethodNone : public PseudoMethodImpl { // Surrounds every word in the sentance with specific characters that makes public: // the word directionality RTL. PseudoMethodNone() {} String16 pseudobidi_string(const String16& source); String16 text(const String16& text) { return text; } // Generates expansion string based on the specified lenght. String16 placeholder(const String16& text) { return text; } // Generated string could not be shorter that length, but it could be slightly private: // longer. DISALLOW_COPY_AND_ASSIGN(PseudoMethodNone); String16 pseudo_generate_expansion(const unsigned int length); }; class PseudoMethodBidi : public PseudoMethodImpl { public: String16 text(const String16& text); String16 placeholder(const String16& text); }; class PseudoMethodAccent : public PseudoMethodImpl { public: PseudoMethodAccent() : mDepth(0), mWordCount(0), mLength(0) {} String16 start(); String16 end(); String16 text(const String16& text); String16 placeholder(const String16& text); private: size_t mDepth; size_t mWordCount; size_t mLength; }; class Pseudolocalizer { public: Pseudolocalizer(PseudolocalizationMethod m); ~Pseudolocalizer() { if (mImpl) delete mImpl; } void setMethod(PseudolocalizationMethod m); String16 start() { return mImpl->start(); } String16 end() { return mImpl->end(); } String16 text(const String16& text); private: PseudoMethodImpl *mImpl; size_t mLastDepth; }; #endif // HOST_PSEUDOLOCALIZE_H #endif // HOST_PSEUDOLOCALIZE_H
tools/aapt/tests/Pseudolocales_test.cpp 0 → 100644 +217 −0 Original line number Original line Diff line number Diff line /* * Copyright (C) 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include <androidfw/ResourceTypes.h> #include <utils/String8.h> #include <gtest/gtest.h> #include "Bundle.h" #include "pseudolocalize.h" using android::String8; // In this context, 'Axis' represents a particular field in the configuration, // such as language or density. static void simple_helper(const char* input, const char* expected, PseudolocalizationMethod method) { Pseudolocalizer pseudo(method); String16 result = pseudo.start() + pseudo.text(String16(String8(input))) + pseudo.end(); //std::cout << String8(result).string() << std::endl; ASSERT_EQ(String8(expected), String8(result)); } static void compound_helper(const char* in1, const char* in2, const char *in3, const char* expected, PseudolocalizationMethod method) { Pseudolocalizer pseudo(method); String16 result = pseudo.start() + \ pseudo.text(String16(String8(in1))) + \ pseudo.text(String16(String8(in2))) + \ pseudo.text(String16(String8(in3))) + \ pseudo.end(); ASSERT_EQ(String8(expected), String8(result)); } TEST(Pseudolocales, NoPseudolocalization) { simple_helper("", "", NO_PSEUDOLOCALIZATION); simple_helper("Hello, world", "Hello, world", NO_PSEUDOLOCALIZATION); compound_helper("Hello,", " world", "", "Hello, world", NO_PSEUDOLOCALIZATION); } TEST(Pseudolocales, PlaintextAccent) { simple_helper("", "[]", PSEUDO_ACCENTED); simple_helper("Hello, world", "[Ĥéļļö, ŵöŕļð one two]", PSEUDO_ACCENTED); simple_helper("Hello, %1d", "[Ĥéļļö, »%1d« one two]", PSEUDO_ACCENTED); simple_helper("Battery %1d%%", "[βåţţéŕý »%1d«%% one two]", PSEUDO_ACCENTED); compound_helper("", "", "", "[]", PSEUDO_ACCENTED); compound_helper("Hello,", " world", "", "[Ĥéļļö, ŵöŕļð one two]", PSEUDO_ACCENTED); } TEST(Pseudolocales, PlaintextBidi) { simple_helper("", "", PSEUDO_BIDI); simple_helper("word", "\xe2\x80\x8f\xE2\x80\xaeword\xE2\x80\xac\xe2\x80\x8f", PSEUDO_BIDI); simple_helper(" word ", " \xe2\x80\x8f\xE2\x80\xaeword\xE2\x80\xac\xe2\x80\x8f ", PSEUDO_BIDI); simple_helper(" word ", " \xe2\x80\x8f\xE2\x80\xaeword\xE2\x80\xac\xe2\x80\x8f ", PSEUDO_BIDI); simple_helper("hello\n world\n", "\xe2\x80\x8f\xE2\x80\xaehello\xE2\x80\xac\xe2\x80\x8f\n" \ " \xe2\x80\x8f\xE2\x80\xaeworld\xE2\x80\xac\xe2\x80\x8f\n", PSEUDO_BIDI); compound_helper("hello", "\n ", " world\n", "\xe2\x80\x8f\xE2\x80\xaehello\xE2\x80\xac\xe2\x80\x8f\n" \ " \xe2\x80\x8f\xE2\x80\xaeworld\xE2\x80\xac\xe2\x80\x8f\n", PSEUDO_BIDI); } TEST(Pseudolocales, SimpleICU) { // Single-fragment messages simple_helper("{placeholder}", "[»{placeholder}«]", PSEUDO_ACCENTED); simple_helper("{USER} is offline", "[»{USER}« îš öƒƒļîñé one two]", PSEUDO_ACCENTED); simple_helper("Copy from {path1} to {path2}", "[Çöþý ƒŕöḿ »{path1}« ţö »{path2}« one two three]", PSEUDO_ACCENTED); simple_helper("Today is {1,date} {1,time}", "[Ţöðåý îš »{1,date}« »{1,time}« one two]", PSEUDO_ACCENTED); // Multi-fragment messages compound_helper("{USER}", " ", "is offline", "[»{USER}« îš öƒƒļîñé one two]", PSEUDO_ACCENTED); compound_helper("Copy from ", "{path1}", " to {path2}", "[Çöþý ƒŕöḿ »{path1}« ţö »{path2}« one two three]", PSEUDO_ACCENTED); } TEST(Pseudolocales, ICUBidi) { // Single-fragment messages simple_helper("{placeholder}", "\xe2\x80\x8f\xE2\x80\xae{placeholder}\xE2\x80\xac\xe2\x80\x8f", PSEUDO_BIDI); simple_helper( "{COUNT, plural, one {one} other {other}}", "{COUNT, plural, " \ "one {\xe2\x80\x8f\xE2\x80\xaeone\xE2\x80\xac\xe2\x80\x8f} " \ "other {\xe2\x80\x8f\xE2\x80\xaeother\xE2\x80\xac\xe2\x80\x8f}}", PSEUDO_BIDI ); } TEST(Pseudolocales, Escaping) { // Single-fragment messages simple_helper("'{USER'} is offline", "['{ÛŠÉŔ'} îš öƒƒļîñé one two three]", PSEUDO_ACCENTED); // Multi-fragment messages compound_helper("'{USER}", " ", "''is offline", "['{ÛŠÉŔ} ''îš öƒƒļîñé one two three]", PSEUDO_ACCENTED); } TEST(Pseudolocales, PluralsAndSelects) { simple_helper( "{COUNT, plural, one {Delete a file} other {Delete {COUNT} files}}", "[{COUNT, plural, one {Ðéļéţé å ƒîļé one two} " \ "other {Ðéļéţé »{COUNT}« ƒîļéš one two}}]", PSEUDO_ACCENTED ); simple_helper( "Distance is {COUNT, plural, one {# mile} other {# miles}}", "[Ðîšţåñçé îš {COUNT, plural, one {# ḿîļé one two} " \ "other {# ḿîļéš one two}}]", PSEUDO_ACCENTED ); simple_helper( "{1, select, female {{1} added you} " \ "male {{1} added you} other {{1} added you}}", "[{1, select, female {»{1}« åððéð ýöû one two} " \ "male {»{1}« åððéð ýöû one two} other {»{1}« åððéð ýöû one two}}]", PSEUDO_ACCENTED ); compound_helper( "{COUNT, plural, one {Delete a file} " \ "other {Delete ", "{COUNT}", " files}}", "[{COUNT, plural, one {Ðéļéţé å ƒîļé one two} " \ "other {Ðéļéţé »{COUNT}« ƒîļéš one two}}]", PSEUDO_ACCENTED ); } TEST(Pseudolocales, NestedICU) { simple_helper( "{person, select, " \ "female {" \ "{num_circles, plural," \ "=0{{person} didn't add you to any of her circles.}" \ "=1{{person} added you to one of her circles.}" \ "other{{person} added you to her # circles.}}}" \ "male {" \ "{num_circles, plural," \ "=0{{person} didn't add you to any of his circles.}" \ "=1{{person} added you to one of his circles.}" \ "other{{person} added you to his # circles.}}}" \ "other {" \ "{num_circles, plural," \ "=0{{person} didn't add you to any of their circles.}" \ "=1{{person} added you to one of their circles.}" \ "other{{person} added you to their # circles.}}}}", "[{person, select, " \ "female {" \ "{num_circles, plural," \ "=0{»{person}« ðîðñ'ţ åðð ýöû ţö åñý öƒ ĥéŕ çîŕçļéš." \ " one two three four five}" \ "=1{»{person}« åððéð ýöû ţö öñé öƒ ĥéŕ çîŕçļéš." \ " one two three four}" \ "other{»{person}« åððéð ýöû ţö ĥéŕ # çîŕçļéš." \ " one two three four}}}" \ "male {" \ "{num_circles, plural," \ "=0{»{person}« ðîðñ'ţ åðð ýöû ţö åñý öƒ ĥîš çîŕçļéš." \ " one two three four five}" \ "=1{»{person}« åððéð ýöû ţö öñé öƒ ĥîš çîŕçļéš." \ " one two three four}" \ "other{»{person}« åððéð ýöû ţö ĥîš # çîŕçļéš." \ " one two three four}}}" \ "other {{num_circles, plural," \ "=0{»{person}« ðîðñ'ţ åðð ýöû ţö åñý öƒ ţĥéîŕ çîŕçļéš." \ " one two three four five}" \ "=1{»{person}« åððéð ýöû ţö öñé öƒ ţĥéîŕ çîŕçļéš." \ " one two three four}" \ "other{»{person}« åððéð ýöû ţö ţĥéîŕ # çîŕçļéš." \ " one two three four}}}}]", PSEUDO_ACCENTED ); } TEST(Pseudolocales, RedefineMethod) { Pseudolocalizer pseudo(PSEUDO_ACCENTED); String16 result = pseudo.text(String16(String8("Hello, "))); pseudo.setMethod(NO_PSEUDOLOCALIZATION); result.append(pseudo.text(String16(String8("world!")))); ASSERT_EQ(String8("Ĥéļļö, world!"), String8(result)); }