Split UTF functions from String8/16 (c412dcb3) · Commits · e / os / android_frameworks_native-old

include/utils/String16.h

+1 −28

Original line number	Diff line number	Diff line
		@@ -19,39 +19,12 @@

		#include <utils/Errors.h>
		#include <utils/SharedBuffer.h>

		#include <stdint.h>
		#include <sys/types.h>
		#include <utils/Unicode.h>

		// ---------------------------------------------------------------------------

		extern "C" {

		typedef uint16_t char16_t;

		// Standard string functions on char16 strings.
		int strcmp16(const char16_t , const char16_t );
		int strncmp16(const char16_t s1, const char16_t s2, size_t n);
		size_t strlen16(const char16_t *);
		size_t strnlen16(const char16_t *, size_t);
		char16_t strcpy16(char16_t , const char16_t *);
		char16_t strncpy16(char16_t , const char16_t *, size_t);

		// Version of comparison that supports embedded nulls.
		// This is different than strncmp() because we don't stop
		// at a nul character and consider the strings to be different
		// if the lengths are different (thus we need to supply the
		// lengths of both strings). This can also be used when
		// your string is not nul-terminated as it will have the
		// equivalent result as strcmp16 (unlike strncmp16).
		int strzcmp16(const char16_t s1, size_t n1, const char16_t s2, size_t n2);

		// Version of strzcmp16 for comparing strings in different endianness.
		int strzcmp16_h_n(const char16_t s1H, size_t n1, const char16_t s2N, size_t n2);

		// Convert UTF-8 to UTF-16 including surrogate pairs
		void utf8_to_utf16(const uint8_t src, size_t srcLen, char16_t dst, const size_t dstLen);

		}

		// ---------------------------------------------------------------------------

include/utils/String8.h

+5 −111

Original line number	Diff line number	Diff line
		@@ -18,122 +18,16 @@
		#define ANDROID_STRING8_H

		#include <utils/Errors.h>
		#include <utils/SharedBuffer.h>
		#include <utils/Unicode.h>

		// Need this for the char16_t type; String8.h should not
		// be depedent on the String16 class.
		#include <utils/String16.h>

		#include <stdint.h>
		#include <string.h>
		#include <sys/types.h>

		// ---------------------------------------------------------------------------

		extern "C" {

		typedef uint32_t char32_t;

		size_t strlen32(const char32_t *);
		size_t strnlen32(const char32_t *, size_t);

		/*
		* Returns the length of "src" when "src" is valid UTF-8 string.
		* Returns 0 if src is NULL, 0-length string or non UTF-8 string.
		* This function should be used to determine whether "src" is valid UTF-8
		* characters with valid unicode codepoints. "src" must be null-terminated.
		*
		* If you are going to use other GetUtf... functions defined in this header
		* with string which may not be valid UTF-8 with valid codepoint (form 0 to
		* 0x10FFFF), you should use this function before calling others, since the
		* other functions do not check whether the string is valid UTF-8 or not.
		*
		* If you do not care whether "src" is valid UTF-8 or not, you should use
		* strlen() as usual, which should be much faster.
		*/
		size_t utf8_length(const char *src);

		/*
		* Returns the UTF-32 length of "src".
		*/
		size_t utf32_length(const char *src, size_t src_len);

		/*
		* Returns the UTF-8 length of "src".
		*/
		size_t utf8_length_from_utf16(const char16_t *src, size_t src_len);

		/*
		* Returns the UTF-8 length of "src".
		*/
		size_t utf8_length_from_utf32(const char32_t *src, size_t src_len);

		/*
		* Returns the unicode value at "index".
		* Returns -1 when the index is invalid (equals to or more than "src_len").
		* If returned value is positive, it is able to be converted to char32_t, which
		* is unsigned. Then, if "next_index" is not NULL, the next index to be used is
		* stored in "next_index". "next_index" can be NULL.
		*/
		int32_t utf32_at(const char *src, size_t src_len,
		size_t index, size_t *next_index);

		/*
		* Stores a UTF-32 string converted from "src" in "dst", if "dst_length" is not
		* large enough to store the string, the part of the "src" string is stored
		* into "dst".
		* Returns the size actually used for storing the string.
		* "dst" is not null-terminated when dst_len is fully used (like strncpy).
		*/
		size_t utf8_to_utf32(const char* src, size_t src_len,
		char32_t* dst, size_t dst_len);

		/*
		* Stores a UTF-8 string converted from "src" in "dst", if "dst_length" is not
		* large enough to store the string, the part of the "src" string is stored
		* into "dst" as much as possible. See the examples for more detail.
		* Returns the size actually used for storing the string.
		* dst" is not null-terminated when dst_len is fully used (like strncpy).
		*
		* Example 1
		* "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
		* "src_len" == 2
		* "dst_len" >= 7
		* ->
		* Returned value == 6
		* "dst" becomes \xE3\x81\x82\xE3\x81\x84\0
		* (note that "dst" is null-terminated)
		*
		* Example 2
		* "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
		* "src_len" == 2
		* "dst_len" == 5
		* ->
		* Returned value == 3
		* "dst" becomes \xE3\x81\x82\0
		* (note that "dst" is null-terminated, but \u3044 is not stored in "dst"
		* since "dst" does not have enough size to store the character)
		*
		* Example 3
		* "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
		* "src_len" == 2
		* "dst_len" == 6
		* ->
		* Returned value == 6
		* "dst" becomes \xE3\x81\x82\xE3\x81\x84
		* (note that "dst" is NOT null-terminated, like strncpy)
		*/
		size_t utf32_to_utf8(const char32_t* src, size_t src_len,
		char* dst, size_t dst_len);

		size_t utf16_to_utf8(const char16_t* src, size_t src_len,
		char* dst, size_t dst_len);

		}
		#include <string.h> // for strcmp

		// ---------------------------------------------------------------------------

		namespace android {

		class String16;
		class TextOutput;

		//! This is a string holding UTF-8 characters. Does not allow the value more
		@@ -182,7 +76,7 @@ public:
		size_t getUtf32Length() const;
		int32_t getUtf32At(size_t index,
		size_t *next_index) const;
		size_t getUtf32(char32_t* dst, size_t dst_len) const;
		void getUtf32(char32_t* dst) const;

		inline String8& operator=(const String8& other);
		inline String8& operator=(const char* other);

include/utils/Unicode.h

0 → 100644

+161 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2005 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#ifndef ANDROID_UNICODE_H
		#define ANDROID_UNICODE_H

		#include <sys/types.h>
		#include <stdint.h>

		extern "C" {

		typedef uint32_t char32_t;
		typedef uint16_t char16_t;

		// Standard string functions on char16_t strings.
		int strcmp16(const char16_t , const char16_t );
		int strncmp16(const char16_t s1, const char16_t s2, size_t n);
		size_t strlen16(const char16_t *);
		size_t strnlen16(const char16_t *, size_t);
		char16_t strcpy16(char16_t , const char16_t *);
		char16_t strncpy16(char16_t , const char16_t *, size_t);

		// Version of comparison that supports embedded nulls.
		// This is different than strncmp() because we don't stop
		// at a nul character and consider the strings to be different
		// if the lengths are different (thus we need to supply the
		// lengths of both strings). This can also be used when
		// your string is not nul-terminated as it will have the
		// equivalent result as strcmp16 (unlike strncmp16).
		int strzcmp16(const char16_t s1, size_t n1, const char16_t s2, size_t n2);

		// Version of strzcmp16 for comparing strings in different endianness.
		int strzcmp16_h_n(const char16_t s1H, size_t n1, const char16_t s2N, size_t n2);

		// Standard string functions on char32_t strings.
		size_t strlen32(const char32_t *);
		size_t strnlen32(const char32_t *, size_t);

		/**
		* Measure the length of a UTF-32 string in UTF-8. If the string is invalid
		* such as containing a surrogate character, -1 will be returned.
		*/
		ssize_t utf32_to_utf8_length(const char32_t *src, size_t src_len);

		/**
		* Stores a UTF-8 string converted from "src" in "dst", if "dst_length" is not
		* large enough to store the string, the part of the "src" string is stored
		* into "dst" as much as possible. See the examples for more detail.
		* Returns the size actually used for storing the string.
		* dst" is not null-terminated when dst_len is fully used (like strncpy).
		*
		* Example 1
		* "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
		* "src_len" == 2
		* "dst_len" >= 7
		* ->
		* Returned value == 6
		* "dst" becomes \xE3\x81\x82\xE3\x81\x84\0
		* (note that "dst" is null-terminated)
		*
		* Example 2
		* "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
		* "src_len" == 2
		* "dst_len" == 5
		* ->
		* Returned value == 3
		* "dst" becomes \xE3\x81\x82\0
		* (note that "dst" is null-terminated, but \u3044 is not stored in "dst"
		* since "dst" does not have enough size to store the character)
		*
		* Example 3
		* "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
		* "src_len" == 2
		* "dst_len" == 6
		* ->
		* Returned value == 6
		* "dst" becomes \xE3\x81\x82\xE3\x81\x84
		* (note that "dst" is NOT null-terminated, like strncpy)
		*/
		void utf32_to_utf8(const char32_t* src, size_t src_len, char* dst);

		/**
		* Returns the unicode value at "index".
		* Returns -1 when the index is invalid (equals to or more than "src_len").
		* If returned value is positive, it is able to be converted to char32_t, which
		* is unsigned. Then, if "next_index" is not NULL, the next index to be used is
		* stored in "next_index". "next_index" can be NULL.
		*/
		int32_t utf32_from_utf8_at(const char src, size_t src_len, size_t index, size_t next_index);


		/**
		* Returns the UTF-8 length of UTF-16 string "src".
		*/
		ssize_t utf16_to_utf8_length(const char16_t *src, size_t src_len);

		/**
		* Converts a UTF-16 string to UTF-8. The destination buffer must be large
		* enough to fit the UTF-16 as measured by utf16_to_utf8_length with an added
		* NULL terminator.
		*/
		void utf16_to_utf8(const char16_t* src, size_t src_len, char* dst);

		/**
		* Returns the length of "src" when "src" is valid UTF-8 string.
		* Returns 0 if src is NULL or 0-length string. Returns -1 when the source
		* is an invalid string.
		*
		* This function should be used to determine whether "src" is valid UTF-8
		* characters with valid unicode codepoints. "src" must be null-terminated.
		*
		* If you are going to use other utf8_to_... functions defined in this header
		* with string which may not be valid UTF-8 with valid codepoint (form 0 to
		* 0x10FFFF), you should use this function before calling others, since the
		* other functions do not check whether the string is valid UTF-8 or not.
		*
		* If you do not care whether "src" is valid UTF-8 or not, you should use
		* strlen() as usual, which should be much faster.
		*/
		ssize_t utf8_length(const char *src);

		/**
		* Measure the length of a UTF-32 string.
		*/
		size_t utf8_to_utf32_length(const char *src, size_t src_len);

		/**
		* Stores a UTF-32 string converted from "src" in "dst". "dst" must be large
		* enough to store the entire converted string as measured by
		* utf8_to_utf32_length plus space for a NULL terminator.
		*/
		void utf8_to_utf32(const char* src, size_t src_len, char32_t* dst);

		/**
		* Returns the UTF-16 length of UTF-8 string "src".
		*/
		ssize_t utf8_to_utf16_length(const uint8_t* src, size_t srcLen);

		/**
		* Convert UTF-8 to UTF-16 including surrogate pairs. The destination buffer
		* must be large enough to hold the result as measured by utf8_to_utf16_length
		* plus an added NULL terminator.
		*/
		void utf8_to_utf16(const uint8_t* src, size_t srcLen, char16_t* dst);

		}

		#endif

libs/utils/Android.mk

+1 −0

Original line number	Diff line number	Diff line
		@@ -41,6 +41,7 @@ commonSources:= \
		TextOutput.cpp \
		Threads.cpp \
		Timers.cpp \
		Unicode.cpp \
		VectorImpl.cpp \
		ZipFileCRO.cpp \
		ZipFileRO.cpp \

libs/utils/ResourceTypes.cpp

+74 −25

Original line number	Diff line number	Diff line
		@@ -444,15 +444,51 @@ void ResStringPool::uninit()
		}
		}

		#define DECODE_LENGTH(str, chrsz, len) \
		len = *(str); \
		if ((str)&(1<<(chrsz8-1))) { \
		(str)++; \
		len = (((len)&((1<<(chrsz8-1))-1))<<(chrsz8)) + *(str); \
		} \
		(str)++;
		/**
		* Strings in UTF-16 format have length indicated by a length encoded in the
		* stored data. It is either 1 or 2 characters of length data. This allows a
		* maximum length of 0x7FFFFFF (2147483647 bytes), but if you're storing that
		* much data in a string, you're abusing them.
		*
		* If the high bit is set, then there are two characters or 4 bytes of length
		* data encoded. In that case, drop the high bit of the first character and
		* add it together with the next character.
		*/
		static inline size_t
		decodeLength(const char16_t** str)
		{
		size_t len = **str;
		if ((len & 0x8000) != 0) {
		(*str)++;
		len = ((len & 0x7FFF) << 16) \| **str;
		}
		(*str)++;
		return len;
		}

		/**
		* Strings in UTF-8 format have length indicated by a length encoded in the
		* stored data. It is either 1 or 2 characters of length data. This allows a
		* maximum length of 0x7FFF (32767 bytes), but you should consider storing
		* text in another way if you're using that much data in a single string.
		*
		* If the high bit is set, then there are two characters or 2 bytes of length
		* data encoded. In that case, drop the high bit of the first character and
		* add it together with the next character.
		*/
		static inline size_t
		decodeLength(const uint8_t** str)
		{
		size_t len = **str;
		if ((len & 0x80) != 0) {
		(*str)++;
		len = ((len & 0x7F) << 8) \| **str;
		}
		(*str)++;
		return len;
		}

		const uint16_t* ResStringPool::stringAt(size_t idx, size_t* outLen) const
		const uint16_t* ResStringPool::stringAt(size_t idx, size_t* u16len) const
		{
		if (mError == NO_ERROR && idx < mHeader->stringCount) {
		const bool isUTF8 = (mHeader->flags&ResStringPool_header::UTF8_FLAG) != 0;
		@@ -461,37 +497,51 @@ const uint16_t* ResStringPool::stringAt(size_t idx, size_t* outLen) const
		if (!isUTF8) {
		const char16_t* strings = (char16_t*)mStrings;
		const char16_t* str = strings+off;
		DECODE_LENGTH(str, sizeof(char16_t), *outLen)
		if ((uint32_t)(str+*outLen-strings) < mStringPoolSize) {

		*u16len = decodeLength(&str);
		if ((uint32_t)(str+*u16len-strings) < mStringPoolSize) {
		return str;
		} else {
		LOGW("Bad string block: string #%d extends to %d, past end at %d\n",
		(int)idx, (int)(str+*outLen-strings), (int)mStringPoolSize);
		(int)idx, (int)(str+*u16len-strings), (int)mStringPoolSize);
		}
		} else {
		const uint8_t* strings = (uint8_t*)mStrings;
		const uint8_t* str = strings+off;
		DECODE_LENGTH(str, sizeof(uint8_t), *outLen)
		size_t encLen;
		DECODE_LENGTH(str, sizeof(uint8_t), encLen)
		if ((uint32_t)(str+encLen-strings) < mStringPoolSize) {
		const uint8_t* u8str = strings+off;

		*u16len = decodeLength(&u8str);
		size_t u8len = decodeLength(&u8str);

		// encLen must be less than 0x7FFF due to encoding.
		if ((uint32_t)(u8str+u8len-strings) < mStringPoolSize) {
		AutoMutex lock(mDecodeLock);

		if (mCache[idx] != NULL) {
		return mCache[idx];
		}
		char16_t u16str = (char16_t )calloc(*outLen+1, sizeof(char16_t));

		ssize_t actualLen = utf8_to_utf16_length(u8str, u8len);
		if (actualLen < 0 \|\| (size_t)actualLen != *u16len) {
		LOGW("Bad string block: string #%lld decoded length is not correct "
		"%lld vs %llu\n",
		(long long)idx, (long long)actualLen, (long long)*u16len);
		return NULL;
		}

		char16_t u16str = (char16_t )calloc(*u16len+1, sizeof(char16_t));
		if (!u16str) {
		LOGW("No memory when trying to allocate decode cache for string #%d\n",
		(int)idx);
		return NULL;
		}
		const unsigned char u8src = reinterpret_cast<const unsigned char >(str);
		utf8_to_utf16(u8src, encLen, u16str, *outLen);

		utf8_to_utf16(u8str, u8len, u16str);
		mCache[idx] = u16str;
		return u16str;
		} else {
		LOGW("Bad string block: string #%d extends to %d, past end at %d\n",
		(int)idx, (int)(str+encLen-strings), (int)mStringPoolSize);
		LOGW("Bad string block: string #%lld extends to %lld, past end at %lld\n",
		(long long)idx, (long long)(u8str+u8len-strings),
		(long long)mStringPoolSize);
		}
		}
		} else {
		@@ -512,9 +562,8 @@ const char* ResStringPool::string8At(size_t idx, size_t* outLen) const
		if (isUTF8) {
		const uint8_t* strings = (uint8_t*)mStrings;
		const uint8_t* str = strings+off;
		DECODE_LENGTH(str, sizeof(uint8_t), *outLen)
		size_t encLen;
		DECODE_LENGTH(str, sizeof(uint8_t), encLen)
		*outLen = decodeLength(&str);
		size_t encLen = decodeLength(&str);
		if ((uint32_t)(str+encLen-strings) < mStringPoolSize) {
		return (const char*)str;
		} else {