Merge "Unicode: specify destination length in utf8_to_utf16 methods"

char16_t *strncpy16(char16_t *, const char16_t *, size_t);

char16_t *strstr16(const char16_t*, const char16_t*);

// Version of comparison that supports embedded nulls.

// Version of comparison that supports embedded NULs.

// This is different than strncmp() because we don't stop

// at a nul character and consider the strings to be different

// if the lengths are different (thus we need to supply the

 * large enough to store the string, the part of the "src" string is stored

 * into "dst" as much as possible. See the examples for more detail.

 * Returns the size actually used for storing the string.

 * dst" is not null-terminated when dst_len is fully used (like strncpy).

 * dst" is not nul-terminated when dst_len is fully used (like strncpy).

 *

 * Example 1

 * "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)

 * ->

 * Returned value == 6

 * "dst" becomes \xE3\x81\x82\xE3\x81\x84\0

 * (note that "dst" is null-terminated)

 * (note that "dst" is nul-terminated)

 *

 * Example 2

 * "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)

 * ->

 * Returned value == 3

 * "dst" becomes \xE3\x81\x82\0

 * (note that "dst" is null-terminated, but \u3044 is not stored in "dst"

 * (note that "dst" is nul-terminated, but \u3044 is not stored in "dst"

 * since "dst" does not have enough size to store the character)

 *

 * Example 3

 * ->

 * Returned value == 6

 * "dst" becomes \xE3\x81\x82\xE3\x81\x84

 * (note that "dst" is NOT null-terminated, like strncpy)

 * (note that "dst" is NOT nul-terminated, like strncpy)

 */

void utf32_to_utf8(const char32_t* src, size_t src_len, char* dst, size_t dst_len);

/**

 * Converts a UTF-16 string to UTF-8. The destination buffer must be large

 * enough to fit the UTF-16 as measured by utf16_to_utf8_length with an added

 * NULL terminator.

 * NUL terminator.

 */

void utf16_to_utf8(const char16_t* src, size_t src_len, char* dst, size_t dst_len);

 * is an invalid string.

 *

 * This function should be used to determine whether "src" is valid UTF-8

 * characters with valid unicode codepoints. "src" must be null-terminated.

 * characters with valid unicode codepoints. "src" must be nul-terminated.

 *

 * If you are going to use other utf8_to_... functions defined in this header

 * with string which may not be valid UTF-8 with valid codepoint (form 0 to

/**

 * Stores a UTF-32 string converted from "src" in "dst". "dst" must be large

 * enough to store the entire converted string as measured by

 * utf8_to_utf32_length plus space for a NULL terminator.

 * utf8_to_utf32_length plus space for a NUL terminator.

 */

void utf8_to_utf32(const char* src, size_t src_len, char32_t* dst);

/**

 * Returns the UTF-16 length of UTF-8 string "src".

 * Returns the UTF-16 length of UTF-8 string "src". Returns -1 in case

 * it's invalid utf8. No buffer over-read occurs because of bound checks. Using overreadIsFatal you

 * can ask to log a message and fail in case the invalid utf8 could have caused an override if no

 * bound checks were used (otherwise -1 is returned).

 */

ssize_t utf8_to_utf16_length(const uint8_t* src, size_t srcLen);

ssize_t utf8_to_utf16_length(const uint8_t* src, size_t srcLen, bool overreadIsFatal = false);

/**

 * Convert UTF-8 to UTF-16 including surrogate pairs.

 * Returns a pointer to the end of the string (where a null terminator might go

 * if you wanted to add one).

 * Returns a pointer to the end of the string (where a NUL terminator might go

 * if you wanted to add one). At most dstLen characters are written; it won't emit half a surrogate

 * pair. If dstLen == 0 nothing is written and dst is returned. If dstLen > SSIZE_MAX it aborts

 * (this being probably a negative number returned as an error and casted to unsigned).

 */

char16_t* utf8_to_utf16_no_null_terminator(const uint8_t* src, size_t srcLen, char16_t* dst);

char16_t* utf8_to_utf16_no_null_terminator(

        const uint8_t* src, size_t srcLen, char16_t* dst, size_t dstLen);

/**

 * Convert UTF-8 to UTF-16 including surrogate pairs. The destination buffer

 * must be large enough to hold the result as measured by utf8_to_utf16_length

 * plus an added NULL terminator.

 * Convert UTF-8 to UTF-16 including surrogate pairs. At most dstLen - 1

 * characters are written; it won't emit half a surrogate pair; and a NUL terminator is appended

 * after. dstLen - 1 can be measured beforehand using utf8_to_utf16_length. Aborts if dstLen == 0

 * (at least one character is needed for the NUL terminator) or dstLen > SSIZE_MAX (the latter

 * case being likely a negative number returned as an error and casted to unsigned) . Returns a

 * pointer to the NUL terminator.

 */

void utf8_to_utf16(const uint8_t* src, size_t srcLen, char16_t* dst);

/**

 * Like utf8_to_utf16_no_null_terminator, but you can supply a maximum length of the

 * decoded string.  The decoded string will fill up to that length; if it is longer

 * the returned pointer will be to the character after dstLen.

 */

char16_t* utf8_to_utf16_n(const uint8_t* src, size_t srcLen, char16_t* dst, size_t dstLen);

char16_t *utf8_to_utf16(

        const uint8_t* src, size_t srcLen, char16_t* dst, size_t dstLen);

}

        u8cur = (const uint8_t*) u8str;

        char16_t* u16str = (char16_t*)buf->data();

        utf8_to_utf16(u8cur, u8len, u16str);

        utf8_to_utf16(u8cur, u8len, u16str, ((size_t) u16len) + 1);

        //printf("Created UTF-16 string from UTF-8 \"%s\":", in);

        //printHexData(1, str, buf->size(), 16, 1);

#include <log/log.h>

#include <utils/Unicode.h>

#include <limits.h>

#include <stddef.h>

#if defined(_WIN32)

    //printf("Char at %p: len=%d, utf-16=%p\n", src, length, (void*)result);

}

ssize_t utf8_to_utf16_length(const uint8_t* u8str, size_t u8len)

ssize_t utf8_to_utf16_length(const uint8_t* u8str, size_t u8len, bool overreadIsFatal)

{

    const uint8_t* const u8end = u8str + u8len;

    const uint8_t* u8cur = u8str;

    while (u8cur < u8end) {

        u16measuredLen++;

        int u8charLen = utf8_codepoint_len(*u8cur);

        // Malformed utf8, some characters are beyond the end.

        // Cases:

        // If u8charLen == 1, this becomes u8cur >= u8end, which cannot happen as u8cur < u8end,

        // then this condition fail and we continue, as expected.

        // If u8charLen == 2, this becomes u8cur + 1 >= u8end, which fails only if

        // u8cur == u8end - 1, that is, there was only one remaining character to read but we need

        // 2 of them. This condition holds and we return -1, as expected.

        if (u8cur + u8charLen - 1 >= u8end) {

            if (overreadIsFatal) {

                LOG_ALWAYS_FATAL("Attempt to overread computing length of utf8 string");

            } else {

                return -1;

            }

        }

        uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8charLen);

        if (codepoint > 0xFFFF) u16measuredLen++; // this will be a surrogate pair in utf16

        u8cur += u8charLen;

    return u16measuredLen;

}

char16_t* utf8_to_utf16_no_null_terminator(const uint8_t* u8str, size_t u8len, char16_t* u16str)

{

    const uint8_t* const u8end = u8str + u8len;

    const uint8_t* u8cur = u8str;

    char16_t* u16cur = u16str;

    while (u8cur < u8end) {

        size_t u8len = utf8_codepoint_len(*u8cur);

        uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8len);

        // Convert the UTF32 codepoint to one or more UTF16 codepoints

        if (codepoint <= 0xFFFF) {

            // Single UTF16 character

            *u16cur++ = (char16_t) codepoint;

        } else {

            // Multiple UTF16 characters with surrogates

            codepoint = codepoint - 0x10000;

            *u16cur++ = (char16_t) ((codepoint >> 10) + 0xD800);

            *u16cur++ = (char16_t) ((codepoint & 0x3FF) + 0xDC00);

        }

        u8cur += u8len;

    }

    return u16cur;

}

void utf8_to_utf16(const uint8_t* u8str, size_t u8len, char16_t* u16str) {

    char16_t* end = utf8_to_utf16_no_null_terminator(u8str, u8len, u16str);

char16_t* utf8_to_utf16(const uint8_t* u8str, size_t u8len, char16_t* u16str, size_t u16len) {

    // A value > SSIZE_MAX is probably a negative value returned as an error and casted.

    LOG_ALWAYS_FATAL_IF(u16len == 0 || u16len > SSIZE_MAX, "u16len is %zu", u16len);

    char16_t* end = utf8_to_utf16_no_null_terminator(u8str, u8len, u16str, u16len - 1);

    *end = 0;

    return end;

}

char16_t* utf8_to_utf16_n(const uint8_t* src, size_t srcLen, char16_t* dst, size_t dstLen) {

char16_t* utf8_to_utf16_no_null_terminator(

        const uint8_t* src, size_t srcLen, char16_t* dst, size_t dstLen) {

    if (dstLen == 0) {

        return dst;

    }

    // A value > SSIZE_MAX is probably a negative value returned as an error and casted.

    LOG_ALWAYS_FATAL_IF(dstLen > SSIZE_MAX, "dstLen is %zu", dstLen);

    const uint8_t* const u8end = src + srcLen;

    const uint8_t* u8cur = src;

    const char16_t* const u16end = dst + dstLen;

    char16_t output[1 + 1 + 1 + 2 + 1]; // Room for NULL

    utf8_to_utf16(str, sizeof(str), output);

    utf8_to_utf16(str, sizeof(str), output, sizeof(output) / sizeof(output[0]));

    EXPECT_EQ(0x0030, output[0])

            << "should be U+0030";

    EXPECT_EQ(nullptr, result);

}

// http://b/29267949

// Test that overreading in utf8_to_utf16_length is detected

TEST_F(UnicodeTest, InvalidUtf8OverreadDetected) {

    // An utf8 char starting with \xc4 is two bytes long.

    // Add extra zeros so no extra memory is read in case the code doesn't

    // work as expected.

    static char utf8[] = "\xc4\x00\x00\x00";

    ASSERT_DEATH(utf8_to_utf16_length((uint8_t *) utf8, strlen(utf8),

            true /* overreadIsFatal */), "" /* regex for ASSERT_DEATH */);

}

}