Merge "AAPT2: Fix pseudolocalization to respect <xliff:g>" (024d22fd) · Commits · e / os / android_frameworks_base

libs/androidfw/include/androidfw/StringPiece.h

+28 −0

Original line number	Diff line number	Diff line
		@@ -271,8 +271,36 @@ inline ::std::ostream& operator<<(::std::ostream& out, const BasicStringPiece<ch
		return out.write(str.data(), str.size());
		}

		template <typename TChar>
		inline ::std::basic_string<TChar>& operator+=(::std::basic_string<TChar>& lhs,
		const BasicStringPiece<TChar>& rhs) {
		return lhs.append(rhs.data(), rhs.size());
		}

		template <typename TChar>
		inline bool operator==(const ::std::basic_string<TChar>& lhs, const BasicStringPiece<TChar>& rhs) {
		return rhs == lhs;
		}

		template <typename TChar>
		inline bool operator!=(const ::std::basic_string<TChar>& lhs, const BasicStringPiece<TChar>& rhs) {
		return rhs != lhs;
		}

		} // namespace android

		inline ::std::ostream& operator<<(::std::ostream& out, const std::u16string& str) {
		ssize_t utf8_len = utf16_to_utf8_length(str.data(), str.size());
		if (utf8_len < 0) {
		return out << "???";
		}

		std::string utf8;
		utf8.resize(static_cast<size_t>(utf8_len));
		utf16_to_utf8(str.data(), str.size(), &*utf8.begin(), utf8_len + 1);
		return out << utf8;
		}

		namespace std {

		template <typename TChar>

tools/aapt2/Main.cpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -25,7 +25,7 @@ namespace aapt {
		static const char* sMajorVersion = "2";

		// Update minor version whenever a feature or flag is added.
		static const char* sMinorVersion = "6";
		static const char* sMinorVersion = "7";

		int PrintVersion() {
		std::cerr << "Android Asset Packaging Tool (aapt) " << sMajorVersion << "."

tools/aapt2/ResourceParser.cpp

+105 −78

Original line number	Diff line number	Diff line
		@@ -26,6 +26,7 @@
		#include "ResourceValues.h"
		#include "ValueVisitor.h"
		#include "util/ImmutableMap.h"
		#include "util/Maybe.h"
		#include "util/Util.h"
		#include "xml/XmlPullParser.h"

		@@ -150,82 +151,108 @@ ResourceParser::ResourceParser(IDiagnostics* diag, ResourceTable* table,
		/**
		* Build a string from XML that converts nested elements into Span objects.
		*/
		bool ResourceParser::FlattenXmlSubtree(xml::XmlPullParser* parser,
		std::string* out_raw_string,
		StyleString* out_style_string) {
		bool ResourceParser::FlattenXmlSubtree(
		xml::XmlPullParser* parser, std::string* out_raw_string, StyleString* out_style_string,
		std::vector<UntranslatableSection>* out_untranslatable_sections) {
		// Keeps track of formatting tags (<b>, <i>) and the range of characters for which they apply.
		std::vector<Span> span_stack;

		bool error = false;
		// Clear the output variables.
		out_raw_string->clear();
		out_style_string->spans.clear();
		out_untranslatable_sections->clear();

		// The StringBuilder will concatenate the various segments of text which are initially
		// separated by tags. It also handles unicode escape codes and quotations.
		util::StringBuilder builder;

		// The first occurrence of a <xliff:g> tag. Nested <xliff:g> tags are illegal.
		Maybe<size_t> untranslatable_start_depth;

		size_t depth = 1;
		while (xml::XmlPullParser::IsGoodEvent(parser->Next())) {
		const xml::XmlPullParser::Event event = parser->event();
		if (event == xml::XmlPullParser::Event::kEndElement) {
		if (!parser->element_namespace().empty()) {
		// We already warned and skipped the start element, so just skip here
		// too
		continue;
		}

		depth--;
		if (depth == 0) {
		break;
		}

		span_stack.back().last_char = builder.Utf16Len() - 1;
		out_style_string->spans.push_back(span_stack.back());
		span_stack.pop_back();

		} else if (event == xml::XmlPullParser::Event::kText) {
		out_raw_string->append(parser->text());
		builder.Append(parser->text());

		} else if (event == xml::XmlPullParser::Event::kStartElement) {
		if (!parser->element_namespace().empty()) {
		if (parser->element_namespace() != sXliffNamespaceUri) {
		// Only warn if this isn't an xliff namespace.
		diag_->Warn(DiagMessage(source_.WithLine(parser->line_number()))
		<< "skipping element '" << parser->element_name()
		<< "' with unknown namespace '"
		<< parser->element_namespace() << "'");
		}
		continue;
		}
		depth++;

		// Build a span object out of the nested element.
		if (event == xml::XmlPullParser::Event::kStartElement) {
		if (parser->element_namespace().empty()) {
		// This is an HTML tag which we encode as a span. Add it to the span stack.
		std::string span_name = parser->element_name();
		const auto end_attr_iter = parser->end_attributes();
		for (auto attr_iter = parser->begin_attributes();
		attr_iter != end_attr_iter; ++attr_iter) {
		for (auto attr_iter = parser->begin_attributes(); attr_iter != end_attr_iter; ++attr_iter) {
		span_name += ";";
		span_name += attr_iter->name;
		span_name += "=";
		span_name += attr_iter->value;
		}

		// Make sure the string is representable in our binary format.
		if (builder.Utf16Len() > std::numeric_limits<uint32_t>::max()) {
		diag_->Error(DiagMessage(source_.WithLine(parser->line_number()))
		<< "style string '" << builder.ToString()
		<< "' is too long");
		error = true;
		<< "style string '" << builder.ToString() << "' is too long");
		return false;
		}

		span_stack.push_back(Span{std::move(span_name), static_cast<uint32_t>(builder.Utf16Len())});
		} else if (parser->element_namespace() == sXliffNamespaceUri) {
		if (parser->element_name() == "g") {
		if (untranslatable_start_depth) {
		// We've already encountered an <xliff:g> tag, and nested <xliff:g> tags are illegal.
		diag_->Error(DiagMessage(source_.WithLine(parser->line_number()))
		<< "illegal nested XLIFF 'g' tag");
		return false;
		} else {
		span_stack.push_back(
		Span{span_name, static_cast<uint32_t>(builder.Utf16Len())});
		// Mark the start of an untranslatable section. Use UTF8 indices/lengths.
		untranslatable_start_depth = depth;
		const size_t current_idx = builder.ToString().size();
		out_untranslatable_sections->push_back(UntranslatableSection{current_idx, current_idx});
		}
		}
		// Ignore other xliff tags, they get handled by other tools.

		} else {
		// Besides XLIFF, any other namespaced tag is unsupported and ignored.
		diag_->Warn(DiagMessage(source_.WithLine(parser->line_number()))
		<< "ignoring element '" << parser->element_name()
		<< "' with unknown namespace '" << parser->element_namespace() << "'");
		}

		// Enter one level inside the element.
		depth++;
		} else if (event == xml::XmlPullParser::Event::kText) {
		// Record both the raw text and append to the builder to deal with escape sequences
		// and quotations.
		out_raw_string->append(parser->text());
		builder.Append(parser->text());
		} else if (event == xml::XmlPullParser::Event::kEndElement) {
		// Return one level from within the element.
		depth--;
		if (depth == 0) {
		break;
		}

		if (parser->element_namespace().empty()) {
		// This is an HTML tag which we encode as a span. Update the span
		// stack and pop the top entry.
		Span& top_span = span_stack.back();
		top_span.last_char = builder.Utf16Len() - 1;
		out_style_string->spans.push_back(std::move(top_span));
		span_stack.pop_back();
		} else if (untranslatable_start_depth == make_value(depth)) {
		// This is the end of an untranslatable section. Use UTF8 indices/lengths.
		UntranslatableSection& untranslatable_section = out_untranslatable_sections->back();
		untranslatable_section.end = builder.ToString().size();
		untranslatable_start_depth = {};
		}
		} else if (event == xml::XmlPullParser::Event::kComment) {
		// Skip
		// Ignore.
		} else {
		LOG(FATAL) << "unhandled XML event";
		}
		}
		CHECK(span_stack.empty()) << "spans haven't been fully processed";

		CHECK(span_stack.empty()) << "spans haven't been fully processed";
		out_style_string->str = builder.ToString();
		return !error;
		return true;
		}

		bool ResourceParser::Parse(xml::XmlPullParser* parser) {
		@@ -548,15 +575,18 @@ std::unique_ptr<Item> ResourceParser::ParseXml(xml::XmlPullParser* parser,

		std::string raw_value;
		StyleString style_string;
		if (!FlattenXmlSubtree(parser, &raw_value, &style_string)) {
		std::vector<UntranslatableSection> untranslatable_sections;
		if (!FlattenXmlSubtree(parser, &raw_value, &style_string, &untranslatable_sections)) {
		return {};
		}

		if (!style_string.spans.empty()) {
		// This can only be a StyledString.
		return util::make_unique<StyledString>(table_->string_pool.MakeRef(
		style_string,
		StringPool::Context(StringPool::Context::kStylePriority, config_)));
		std::unique_ptr<StyledString> styled_string =
		util::make_unique<StyledString>(table_->string_pool.MakeRef(
		style_string, StringPool::Context(StringPool::Context::kStylePriority, config_)));
		styled_string->untranslatable_sections = std::move(untranslatable_sections);
		return std::move(styled_string);
		}

		auto on_create_reference = [&](const ResourceName& name) {
		@@ -582,8 +612,10 @@ std::unique_ptr<Item> ResourceParser::ParseXml(xml::XmlPullParser* parser,
		// Try making a regular string.
		if (type_mask & android::ResTable_map::TYPE_STRING) {
		// Use the trimmed, escaped string.
		return util::make_unique<String>(table_->string_pool.MakeRef(
		style_string.str, StringPool::Context(config_)));
		std::unique_ptr<String> string = util::make_unique<String>(
		table_->string_pool.MakeRef(style_string.str, StringPool::Context(config_)));
		string->untranslatable_sections = std::move(untranslatable_sections);
		return std::move(string);
		}

		if (allow_raw_value) {
		@@ -609,17 +641,15 @@ bool ResourceParser::ParseString(xml::XmlPullParser* parser,
		formatted = maybe_formatted.value();
		}

		bool translateable = options_.translatable;
		if (Maybe<StringPiece> translateable_attr =
		xml::FindAttribute(parser, "translatable")) {
		Maybe<bool> maybe_translateable =
		ResourceUtils::ParseBool(translateable_attr.value());
		if (!maybe_translateable) {
		bool translatable = options_.translatable;
		if (Maybe<StringPiece> translatable_attr = xml::FindAttribute(parser, "translatable")) {
		Maybe<bool> maybe_translatable = ResourceUtils::ParseBool(translatable_attr.value());
		if (!maybe_translatable) {
		diag_->Error(DiagMessage(out_resource->source)
		<< "invalid value for 'translatable'. Must be a boolean");
		return false;
		}
		translateable = maybe_translateable.value();
		translatable = maybe_translatable.value();
		}

		out_resource->value =
		@@ -630,9 +660,9 @@ bool ResourceParser::ParseString(xml::XmlPullParser* parser,
		}

		if (String* string_value = ValueCast<String>(out_resource->value.get())) {
		string_value->SetTranslateable(translateable);
		string_value->SetTranslatable(translatable);

		if (formatted && translateable) {
		if (formatted && translatable) {
		if (!util::VerifyJavaStringFormat(*string_value->value)) {
		DiagMessage msg(out_resource->source);
		msg << "multiple substitutions specified in non-positional format; "
		@@ -646,9 +676,8 @@ bool ResourceParser::ParseString(xml::XmlPullParser* parser,
		}
		}

		} else if (StyledString* string_value =
		ValueCast<StyledString>(out_resource->value.get())) {
		string_value->SetTranslateable(translateable);
		} else if (StyledString* string_value = ValueCast<StyledString>(out_resource->value.get())) {
		string_value->SetTranslatable(translatable);
		}
		return true;
		}
		@@ -1151,19 +1180,17 @@ bool ResourceParser::ParseArrayImpl(xml::XmlPullParser* parser,

		std::unique_ptr<Array> array = util::make_unique<Array>();

		bool translateable = options_.translatable;
		if (Maybe<StringPiece> translateable_attr =
		xml::FindAttribute(parser, "translatable")) {
		Maybe<bool> maybe_translateable =
		ResourceUtils::ParseBool(translateable_attr.value());
		if (!maybe_translateable) {
		bool translatable = options_.translatable;
		if (Maybe<StringPiece> translatable_attr = xml::FindAttribute(parser, "translatable")) {
		Maybe<bool> maybe_translatable = ResourceUtils::ParseBool(translatable_attr.value());
		if (!maybe_translatable) {
		diag_->Error(DiagMessage(out_resource->source)
		<< "invalid value for 'translatable'. Must be a boolean");
		return false;
		}
		translateable = maybe_translateable.value();
		translatable = maybe_translatable.value();
		}
		array->SetTranslateable(translateable);
		array->SetTranslatable(translatable);

		bool error = false;
		const size_t depth = parser->depth();

tools/aapt2/ResourceParser.h

+10 −10

Original line number	Diff line number	Diff line
		@@ -60,16 +60,16 @@ class ResourceParser {
		private:
		DISALLOW_COPY_AND_ASSIGN(ResourceParser);

		/*
		* Parses the XML subtree as a StyleString (flattened XML representation for
		* strings
		* with formatting). If successful, `out_style_string`
		* contains the escaped and whitespace trimmed text, while `out_raw_string`
		* contains the unescaped text. Returns true on success.
		*/
		bool FlattenXmlSubtree(xml::XmlPullParser* parser,
		std::string* out_raw_string,
		StyleString* out_style_string);
		// Parses the XML subtree as a StyleString (flattened XML representation for strings with
		// formatting). If parsing fails, false is returned and the out parameters are left in an
		// unspecified state. Otherwise,
		// `out_style_string` contains the escaped and whitespace trimmed text.
		// `out_raw_string` contains the un-escaped text.
		// `out_untranslatable_sections` contains the sections of the string that should not be
		// translated.
		bool FlattenXmlSubtree(xml::XmlPullParser* parser, std::string* out_raw_string,
		StyleString* out_style_string,
		std::vector<UntranslatableSection>* out_untranslatable_sections);

		/*
		* Parses the XML subtree and returns an Item.

tools/aapt2/ResourceParser_test.cpp

+58 −14

Original line number	Diff line number	Diff line
		@@ -76,6 +76,7 @@ TEST_F(ResourceParserTest, ParseQuotedString) {
		String* str = test::GetValue<String>(&table_, "string/foo");
		ASSERT_NE(nullptr, str);
		EXPECT_EQ(std::string(" hey there "), *str->value);
		EXPECT_TRUE(str->untranslatable_sections.empty());
		}

		TEST_F(ResourceParserTest, ParseEscapedString) {
		@@ -85,6 +86,7 @@ TEST_F(ResourceParserTest, ParseEscapedString) {
		String* str = test::GetValue<String>(&table_, "string/foo");
		ASSERT_NE(nullptr, str);
		EXPECT_EQ(std::string("?123"), *str->value);
		EXPECT_TRUE(str->untranslatable_sections.empty());
		}

		TEST_F(ResourceParserTest, ParseFormattedString) {
		@@ -97,8 +99,7 @@ TEST_F(ResourceParserTest, ParseFormattedString) {

		TEST_F(ResourceParserTest, ParseStyledString) {
		// Use a surrogate pair unicode point so that we can verify that the span
		// indices
		// use UTF-16 length and not UTF-18 length.
		// indices use UTF-16 length and not UTF-8 length.
		std::string input =
		"<string name=\"foo\">This is my aunt\u2019s <b>string</b></string>";
		ASSERT_TRUE(TestParse(input));
		@@ -109,6 +110,7 @@ TEST_F(ResourceParserTest, ParseStyledString) {
		const std::string expected_str = "This is my aunt\u2019s string";
		EXPECT_EQ(expected_str, *str->value->str);
		EXPECT_EQ(1u, str->value->spans.size());
		EXPECT_TRUE(str->untranslatable_sections.empty());

		EXPECT_EQ(std::string("b"), *str->value->spans[0].name);
		EXPECT_EQ(17u, str->value->spans[0].first_char);
		@@ -122,6 +124,7 @@ TEST_F(ResourceParserTest, ParseStringWithWhitespace) {
		String* str = test::GetValue<String>(&table_, "string/foo");
		ASSERT_NE(nullptr, str);
		EXPECT_EQ(std::string("This is what I think"), *str->value);
		EXPECT_TRUE(str->untranslatable_sections.empty());

		input = "<string name=\"foo2\">\" This is what I think \"</string>";
		ASSERT_TRUE(TestParse(input));
		@@ -131,16 +134,61 @@ TEST_F(ResourceParserTest, ParseStringWithWhitespace) {
		EXPECT_EQ(std::string(" This is what I think "), *str->value);
		}

		TEST_F(ResourceParserTest, IgnoreXliffTags) {
		std::string input =
		"<string name=\"foo\" \n"
		" xmlns:xliff=\"urn:oasis:names:tc:xliff:document:1.2\">\n"
		" There are <xliff:g id=\"count\">%1$d</xliff:g> apples</string>";
		TEST_F(ResourceParserTest, IgnoreXliffTagsOtherThanG) {
		std::string input = R"EOF(
		<string name="foo" xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
		There are <xliff:source>no</xliff:source> apples</string>)EOF";
		ASSERT_TRUE(TestParse(input));

		String* str = test::GetValue<String>(&table_, "string/foo");
		ASSERT_NE(nullptr, str);
		EXPECT_EQ(StringPiece("There are no apples"), StringPiece(*str->value));
		EXPECT_TRUE(str->untranslatable_sections.empty());
		}

		TEST_F(ResourceParserTest, NestedXliffGTagsAreIllegal) {
		std::string input = R"EOF(
		<string name="foo" xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
		Do not <xliff:g>translate <xliff:g>this</xliff:g></xliff:g></string>)EOF";
		EXPECT_FALSE(TestParse(input));
		}

		TEST_F(ResourceParserTest, RecordUntranslateableXliffSectionsInString) {
		std::string input = R"EOF(
		<string name="foo" xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
		There are <xliff:g id="count">%1$d</xliff:g> apples</string>)EOF";
		ASSERT_TRUE(TestParse(input));

		String* str = test::GetValue<String>(&table_, "string/foo");
		ASSERT_NE(nullptr, str);
		EXPECT_EQ(StringPiece("There are %1$d apples"), StringPiece(*str->value));

		ASSERT_EQ(1u, str->untranslatable_sections.size());

		// We expect indices and lengths that span to include the whitespace
		// before %1$d. This is due to how the StringBuilder withholds whitespace unless
		// needed (to deal with line breaks, etc.).
		EXPECT_EQ(9u, str->untranslatable_sections[0].start);
		EXPECT_EQ(14u, str->untranslatable_sections[0].end);
		}

		TEST_F(ResourceParserTest, RecordUntranslateableXliffSectionsInStyledString) {
		std::string input = R"EOF(
		<string name="foo" xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
		There are <b><xliff:g id="count">%1$d</xliff:g></b> apples</string>)EOF";
		ASSERT_TRUE(TestParse(input));

		StyledString* str = test::GetValue<StyledString>(&table_, "string/foo");
		ASSERT_NE(nullptr, str);
		EXPECT_EQ(StringPiece("There are %1$d apples"), StringPiece(*str->value->str));

		ASSERT_EQ(1u, str->untranslatable_sections.size());

		// We expect indices and lengths that span to include the whitespace
		// before %1$d. This is due to how the StringBuilder withholds whitespace unless
		// needed (to deal with line breaks, etc.).
		EXPECT_EQ(9u, str->untranslatable_sections[0].start);
		EXPECT_EQ(14u, str->untranslatable_sections[0].end);
		}

		TEST_F(ResourceParserTest, ParseNull) {
		@@ -149,15 +197,11 @@ TEST_F(ResourceParserTest, ParseNull) {

		// The Android runtime treats a value of android::Res_value::TYPE_NULL as
		// a non-existing value, and this causes problems in styles when trying to
		// resolve
		// an attribute. Null values must be encoded as
		// android::Res_value::TYPE_REFERENCE
		// resolve an attribute. Null values must be encoded as android::Res_value::TYPE_REFERENCE
		// with a data value of 0.
		BinaryPrimitive* integer =
		test::GetValue<BinaryPrimitive>(&table_, "integer/foo");
		BinaryPrimitive* integer = test::GetValue<BinaryPrimitive>(&table_, "integer/foo");
		ASSERT_NE(nullptr, integer);
		EXPECT_EQ(uint16_t(android::Res_value::TYPE_REFERENCE),
		integer->value.dataType);
		EXPECT_EQ(uint16_t(android::Res_value::TYPE_REFERENCE), integer->value.dataType);
		EXPECT_EQ(0u, integer->value.data);
		}