Merge "AAPT2: Encode 4-byte strings in Modified UTF-8" (aa4ea499) · Commits · e / os / android_frameworks_base

tools/aapt2/StringPool.cpp

+1 −1

Original line number	Original line	Diff line number	Diff line
	@@ -367,7 +367,7 @@ const std::string kStringTooLarge = "STRING_TOO_LARGE";
	static bool EncodeString(const std::string& str, const bool utf8, BigBuffer* out,		static bool EncodeString(const std::string& str, const bool utf8, BigBuffer* out,
	IDiagnostics* diag) {		IDiagnostics* diag) {
	if (utf8) {		if (utf8) {
	const std::string& encoded = str;		const std::string& encoded = util::Utf8ToModifiedUtf8(str);
	const ssize_t utf16_length = utf8_to_utf16_length(		const ssize_t utf16_length = utf8_to_utf16_length(
	reinterpret_cast<const uint8_t*>(encoded.data()), encoded.size());		reinterpret_cast<const uint8_t*>(encoded.data()), encoded.size());
	CHECK(utf16_length >= 0);		CHECK(utf16_length >= 0);

tools/aapt2/StringPool_test.cpp

+19 −0

Original line number	Original line	Diff line number	Diff line
	@@ -303,6 +303,25 @@ TEST(StringPoolTest, Flatten) {
	}		}
	}		}

			TEST(StringPoolTest, FlattenModifiedUTF8) {
			using namespace android; // For NO_ERROR on Windows.
			StdErrDiagnostics diag;
			StringPool pool;
			StringPool::Ref ref_a = pool.MakeRef("\xF0\x90\x90\x80"); // 𐐀 (U+10400)
			StringPool::Ref ref_b = pool.MakeRef("foo \xF0\x90\x90\xB7 bar"); // 𐐷 (U+10437)
			StringPool::Ref ref_c = pool.MakeRef("\xF0\x90\x90\x80\xF0\x90\x90\xB7");

			BigBuffer buffer(1024);
			StringPool::FlattenUtf8(&buffer, pool, &diag);
			std::unique_ptr<uint8_t[]> data = util::Copy(buffer);

			// Check that the 4 byte utf-8 codepoint is encoded using 2 3 byte surrogate pairs
			ResStringPool test;
			ASSERT_EQ(test.setTo(data.get(), buffer.size()), NO_ERROR);
			EXPECT_THAT(util::GetString(test, 0), Eq("\xED\xA0\x81\xED\xB0\x80"));
			EXPECT_THAT(util::GetString(test, 1), Eq("foo \xED\xA0\x81\xED\xB0\xB7 bar"));
			EXPECT_THAT(util::GetString(test, 2), Eq("\xED\xA0\x81\xED\xB0\x80\xED\xA0\x81\xED\xB0\xB7"));
			}

	TEST(StringPoolTest, MaxEncodingLength) {		TEST(StringPoolTest, MaxEncodingLength) {
	StdErrDiagnostics diag;		StdErrDiagnostics diag;

tools/aapt2/util/Util.cpp

+47 −0

Original line number	Original line	Diff line number	Diff line
	@@ -297,6 +297,53 @@ bool VerifyJavaStringFormat(const StringPiece& str) {
	return true;		return true;
	}		}

			std::string Utf8ToModifiedUtf8(const std::string& utf8) {
			// Java uses Modified UTF-8 which only supports the 1, 2, and 3 byte formats of UTF-8. To encode
			// 4 byte UTF-8 codepoints, Modified UTF-8 allows the use of surrogate pairs in the same format
			// of CESU-8 surrogate pairs. Calculate the size of the utf8 string with all 4 byte UTF-8
			// codepoints replaced with 2 3 byte surrogate pairs
			size_t modified_size = 0;
			const size_t size = utf8.size();
			for (size_t i = 0; i < size; i++) {
			if (((uint8_t) utf8[i] >> 4) == 0xF) {
			modified_size += 6;
			i += 3;
			} else {
			modified_size++;
			}
			}

			// Early out if no 4 byte codepoints are found
			if (size == modified_size) {
			return utf8;
			}

			std::string output;
			output.reserve(modified_size);
			for (size_t i = 0; i < size; i++) {
			if (((uint8_t) utf8[i] >> 4) == 0xF) {
			auto codepoint = (char32_t) utf32_from_utf8_at(utf8.data(), size, i, nullptr);

			// Calculate the high and low surrogates as UTF-16 would
			char32_t high = ((codepoint - 0x10000) / 0x400) + 0xD800;
			char32_t low = ((codepoint - 0x10000) % 0x400) + 0xDC00;

			// Encode each surrogate in UTF-8
			output.push_back((char) (0xE4 \| ((high >> 12) & 0xF)));
			output.push_back((char) (0x80 \| ((high >> 6) & 0x3F)));
			output.push_back((char) (0x80 \| (high & 0x3F)));
			output.push_back((char) (0xE4 \| ((low >> 12) & 0xF)));
			output.push_back((char) (0x80 \| ((low >> 6) & 0x3F)));
			output.push_back((char) (0x80 \| (low & 0x3F)));
			i += 3;
			} else {
			output.push_back(utf8[i]);
			}
			}

			return output;
			}

	std::u16string Utf8ToUtf16(const StringPiece& utf8) {		std::u16string Utf8ToUtf16(const StringPiece& utf8) {
	ssize_t utf16_length = utf8_to_utf16_length(		ssize_t utf16_length = utf8_to_utf16_length(
	reinterpret_cast<const uint8_t*>(utf8.data()), utf8.length());		reinterpret_cast<const uint8_t*>(utf8.data()), utf8.length());

tools/aapt2/util/Util.h

+3 −0

Original line number	Original line	Diff line number	Diff line
	@@ -197,6 +197,9 @@ inline StringBuilder::operator bool() const {
	return error_.empty();		return error_.empty();
	}		}

			// Converts a UTF8 string into Modified UTF8
			std::string Utf8ToModifiedUtf8(const std::string& utf8);

	// Converts a UTF8 string to a UTF16 string.		// Converts a UTF8 string to a UTF16 string.
	std::u16string Utf8ToUtf16(const android::StringPiece& utf8);		std::u16string Utf8ToUtf16(const android::StringPiece& utf8);
	std::string Utf16ToUtf8(const android::StringPiece16& utf16);		std::string Utf16ToUtf8(const android::StringPiece16& utf16);