adb/base: fix adb push of Unicode filenames on Win32 (ac9514a4) · Commits · e / os / android_system_core

adb/sysdeps_win32.cpp

+45 −80

Original line number	Diff line number	Diff line
		@@ -35,6 +35,7 @@
		#include <base/logging.h>
		#include <base/stringprintf.h>
		#include <base/strings.h>
		#include <base/utf8.h>

		#include "adb.h"

		@@ -3516,100 +3517,61 @@ static void _widen_fatal(const char *fmt, ...) {
		exit(-1);
		}

		// TODO: Consider implementing widen() and narrow() out of std::wstring_convert
		// once libcxx is supported on Windows. Or, consider libutils/Unicode.cpp.
		// Convert size number of UTF-8 char's to UTF-16. Fatal exit on error.
		std::wstring widen(const char* utf8, const size_t size) {
		std::wstring utf16;
		if (!android::base::UTF8ToWide(utf8, size, &utf16)) {
		// If we call fatal() here and fatal() calls widen(), then there may be
		// infinite recursion. To avoid this, call _widen_fatal() instead.
		_widen_fatal("cannot convert from UTF-8 to UTF-16");
		}

		// Convert from UTF-8 to UTF-16. A size of -1 specifies a NULL terminated
		// string. Any other size specifies the number of chars to convert, excluding
		// any NULL terminator (if you're passing an explicit size, you probably don't
		// have a NULL terminated string in the first place).
		std::wstring widen(const char* utf8, const int size) {
		// Note: Do not call SystemErrorCodeToString() from widen() because
		// SystemErrorCodeToString() calls narrow() which may call fatal() which
		// calls adb_vfprintf() which calls widen(), potentially causing infinite
		// recursion.
		const int chars_to_convert = MultiByteToWideChar(CP_UTF8, 0, utf8, size,
		NULL, 0);
		if (chars_to_convert <= 0) {
		// UTF-8 to UTF-16 should be lossless, so we don't expect this to fail.
		_widen_fatal("MultiByteToWideChar failed counting: %d, "
		"GetLastError: %lu", chars_to_convert, GetLastError());
		return utf16;
		}

		// Convert a NULL-terminated string of UTF-8 characters to UTF-16. Fatal exit
		// on error.
		std::wstring widen(const char* utf8) {
		std::wstring utf16;
		size_t chars_to_allocate = chars_to_convert;
		if (size == -1) {
		// chars_to_convert includes a NULL terminator, so subtract space
		// for that because resize() includes that itself.
		--chars_to_allocate;
		}
		utf16.resize(chars_to_allocate);

		// This uses &string[0] to get write-access to the entire string buffer
		// which may be assuming that the chars are all contiguous, but it seems
		// to work and saves us the hassle of using a temporary
		// std::vector<wchar_t>.
		const int result = MultiByteToWideChar(CP_UTF8, 0, utf8, size, &utf16[0],
		chars_to_convert);
		if (result != chars_to_convert) {
		// UTF-8 to UTF-16 should be lossless, so we don't expect this to fail.
		_widen_fatal("MultiByteToWideChar failed conversion: %d, "
		"GetLastError: %lu", result, GetLastError());
		}

		// If a size was passed in (size != -1), then the string is NULL terminated
		// by a NULL char that was written by std::string::resize(). If size == -1,
		// then MultiByteToWideChar() read a NULL terminator from the original
		// string and converted it to a NULL UTF-16 char in the output.
		if (!android::base::UTF8ToWide(utf8, &utf16)) {
		// If we call fatal() here and fatal() calls widen(), then there may be
		// infinite recursion. To avoid this, call _widen_fatal() instead.
		_widen_fatal("cannot convert from UTF-8 to UTF-16");
		}

		return utf16;
		}

		// Convert a NULL terminated string from UTF-8 to UTF-16.
		std::wstring widen(const char* utf8) {
		// Pass -1 to let widen() determine the string length.
		return widen(utf8, -1);
		// Convert a UTF-8 std::string (including any embedded NULL characters) to
		// UTF-16. Fatal exit on error.
		std::wstring widen(const std::string& utf8) {
		std::wstring utf16;
		if (!android::base::UTF8ToWide(utf8, &utf16)) {
		// If we call fatal() here and fatal() calls widen(), then there may be
		// infinite recursion. To avoid this, call _widen_fatal() instead.
		_widen_fatal("cannot convert from UTF-8 to UTF-16");
		}

		// Convert from UTF-8 to UTF-16.
		std::wstring widen(const std::string& utf8) {
		return widen(utf8.c_str(), utf8.length());
		return utf16;
		}

		// Convert from UTF-16 to UTF-8.
		// Convert a UTF-16 std::wstring (including any embedded NULL characters) to
		// UTF-8. Fatal exit on error.
		std::string narrow(const std::wstring& utf16) {
		return narrow(utf16.c_str());
		std::string utf8;
		if (!android::base::WideToUTF8(utf16, &utf8)) {
		fatal("cannot convert from UTF-16 to UTF-8");
		}

		// Convert from UTF-16 to UTF-8.
		std::string narrow(const wchar_t* utf16) {
		// Note: Do not call SystemErrorCodeToString() from narrow() because
		// SystemErrorCodeToString() calls narrow() and we don't want potential
		// infinite recursion.
		const int chars_required = WideCharToMultiByte(CP_UTF8, 0, utf16, -1, NULL,
		0, NULL, NULL);
		if (chars_required <= 0) {
		// UTF-16 to UTF-8 should be lossless, so we don't expect this to fail.
		fatal("WideCharToMultiByte failed counting: %d, GetLastError: %lu",
		chars_required, GetLastError());
		return utf8;
		}

		// Convert a NULL-terminated string of UTF-16 characters to UTF-8. Fatal exit
		// on error.
		std::string narrow(const wchar_t* utf16) {
		std::string utf8;
		// Subtract space for the NULL terminator because resize() includes
		// that itself. Note that this could potentially throw a std::bad_alloc
		// exception.
		utf8.resize(chars_required - 1);

		// This uses &string[0] to get write-access to the entire string buffer
		// which may be assuming that the chars are all contiguous, but it seems
		// to work and saves us the hassle of using a temporary
		// std::vector<char>.
		const int result = WideCharToMultiByte(CP_UTF8, 0, utf16, -1, &utf8[0],
		chars_required, NULL, NULL);
		if (result != chars_required) {
		// UTF-16 to UTF-8 should be lossless, so we don't expect this to fail.
		fatal("WideCharToMultiByte failed conversion: %d, GetLastError: %lu",
		result, GetLastError());
		if (!android::base::WideToUTF8(utf16, &utf8)) {
		fatal("cannot convert from UTF-16 to UTF-8");
		}

		return utf8;
		@@ -3752,9 +3714,12 @@ int adb_chmod(const char* path, int mode) {
		// on error.
		static int _console_write_utf8(const char* buf, size_t size, FILE* stream,
		HANDLE console) {
		// Convert from UTF-8 to UTF-16.
		std::wstring output;

		// Try to convert from data that might be UTF-8 to UTF-16, ignoring errors.
		// Data might not be UTF-8 if the user cat's random data, runs dmesg, etc.
		// This could throw std::bad_alloc.
		const std::wstring output(widen(buf, size));
		(void)android::base::UTF8ToWide(buf, size, &output);

		// Note that this does not do \n => \r\n translation because that
		// doesn't seem necessary for the Windows console. For the Windows

base/Android.mk

+18 −0

Original line number	Diff line number	Diff line
		@@ -23,6 +23,9 @@ libbase_src_files := \
		strings.cpp \
		test_utils.cpp \

		libbase_windows_src_files := \
		utf8.cpp \

		libbase_test_src_files := \
		file_test.cpp \
		logging_test.cpp \
		@@ -31,6 +34,9 @@ libbase_test_src_files := \
		strings_test.cpp \
		test_main.cpp \

		libbase_test_windows_src_files := \
		utf8_test.cpp \

		libbase_cppflags := \
		-Wall \
		-Wextra \
		@@ -42,6 +48,9 @@ include $(CLEAR_VARS)
		LOCAL_MODULE := libbase
		LOCAL_CLANG := true
		LOCAL_SRC_FILES := $(libbase_src_files)
		LOCAL_SRC_FILES_darwin := $(libbase_darwin_src_files)
		LOCAL_SRC_FILES_linux := $(libbase_linux_src_files)
		LOCAL_SRC_FILES_windows := $(libbase_windows_src_files)
		LOCAL_C_INCLUDES := $(LOCAL_PATH)/include
		LOCAL_CPPFLAGS := $(libbase_cppflags)
		LOCAL_EXPORT_C_INCLUDE_DIRS := $(LOCAL_PATH)/include
		@@ -64,6 +73,9 @@ include $(BUILD_SHARED_LIBRARY)
		include $(CLEAR_VARS)
		LOCAL_MODULE := libbase
		LOCAL_SRC_FILES := $(libbase_src_files)
		LOCAL_SRC_FILES_darwin := $(libbase_darwin_src_files)
		LOCAL_SRC_FILES_linux := $(libbase_linux_src_files)
		LOCAL_SRC_FILES_windows := $(libbase_windows_src_files)
		LOCAL_C_INCLUDES := $(LOCAL_PATH)/include
		LOCAL_CPPFLAGS := $(libbase_cppflags)
		LOCAL_EXPORT_C_INCLUDE_DIRS := $(LOCAL_PATH)/include
		@@ -88,6 +100,9 @@ include $(CLEAR_VARS)
		LOCAL_MODULE := libbase_test
		LOCAL_CLANG := true
		LOCAL_SRC_FILES := $(libbase_test_src_files)
		LOCAL_SRC_FILES_darwin := $(libbase_test_darwin_src_files)
		LOCAL_SRC_FILES_linux := $(libbase_test_linux_src_files)
		LOCAL_SRC_FILES_windows := $(libbase_test_windows_src_files)
		LOCAL_C_INCLUDES := $(LOCAL_PATH)
		LOCAL_CPPFLAGS := $(libbase_cppflags)
		LOCAL_SHARED_LIBRARIES := libbase
		@@ -100,6 +115,9 @@ include $(CLEAR_VARS)
		LOCAL_MODULE := libbase_test
		LOCAL_MODULE_HOST_OS := darwin linux windows
		LOCAL_SRC_FILES := $(libbase_test_src_files)
		LOCAL_SRC_FILES_darwin := $(libbase_test_darwin_src_files)
		LOCAL_SRC_FILES_linux := $(libbase_test_linux_src_files)
		LOCAL_SRC_FILES_windows := $(libbase_test_windows_src_files)
		LOCAL_C_INCLUDES := $(LOCAL_PATH)
		LOCAL_CPPFLAGS := $(libbase_cppflags)
		LOCAL_SHARED_LIBRARIES := libbase

base/file.cpp

+4 −0

Original line number	Diff line number	Diff line
		@@ -24,6 +24,7 @@
		#include <string>

		#include "base/macros.h" // For TEMP_FAILURE_RETRY on Darwin.
		#include "base/utf8.h"
		#define LOG_TAG "base.file"
		#include "cutils/log.h"
		#include "utils/Compat.h"
		@@ -35,6 +36,9 @@
		namespace android {
		namespace base {

		// Versions of standard library APIs that support UTF-8 strings.
		using namespace android::base::utf8;

		bool ReadFdToString(int fd, std::string* content) {
		content->clear();

base/include/base/utf8.h

0 → 100755

+83 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2015 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#ifndef BASE_UTF8_H
		#define BASE_UTF8_H

		#ifdef _WIN32
		#include <string>
		#endif

		namespace android {
		namespace base {

		// Only available on Windows because this is only needed on Windows.
		#ifdef _WIN32
		// Convert size number of UTF-16 wchar_t's to UTF-8. Returns whether the
		// conversion was done successfully.
		bool WideToUTF8(const wchar_t* utf16, const size_t size, std::string* utf8);

		// Convert a NULL-terminated string of UTF-16 characters to UTF-8. Returns
		// whether the conversion was done successfully.
		bool WideToUTF8(const wchar_t* utf16, std::string* utf8);

		// Convert a UTF-16 std::wstring (including any embedded NULL characters) to
		// UTF-8. Returns whether the conversion was done successfully.
		bool WideToUTF8(const std::wstring& utf16, std::string* utf8);

		// Convert size number of UTF-8 char's to UTF-16. Returns whether the conversion
		// was done successfully.
		bool UTF8ToWide(const char* utf8, const size_t size, std::wstring* utf16);

		// Convert a NULL-terminated string of UTF-8 characters to UTF-16. Returns
		// whether the conversion was done successfully.
		bool UTF8ToWide(const char* utf8, std::wstring* utf16);

		// Convert a UTF-8 std::string (including any embedded NULL characters) to
		// UTF-16. Returns whether the conversion was done successfully.
		bool UTF8ToWide(const std::string& utf8, std::wstring* utf16);
		#endif

		// The functions in the utf8 namespace take UTF-8 strings. For Windows, these
		// are wrappers, for non-Windows these just expose existing APIs. To call these
		// functions, use:
		//
		// // anonymous namespace to avoid conflict with existing open(), unlink(), etc.
		// namespace {
		// // Import functions into anonymous namespace.
		// using namespace android::base::utf8;
		//
		// void SomeFunction(const char* name) {
		// int fd = open(name, ...); // Calls android::base::utf8::open().
		// ...
		// unlink(name); // Calls android::base::utf8::unlink().
		// }
		// }
		namespace utf8 {

		#ifdef _WIN32
		int open(const char* name, int flags, ...);
		int unlink(const char* name);
		#else
		using ::open;
		using ::unlink;
		#endif

		} // namespace utf8
		} // namespace base
		} // namespace android

		#endif // BASE_UTF8_H

base/utf8.cpp

0 → 100755

+170 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2015 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#include <windows.h>

		#include "base/utf8.h"

		#include <fcntl.h>

		#include <string>

		#include "base/logging.h"

		namespace android {
		namespace base {

		bool WideToUTF8(const wchar_t* utf16, const size_t size, std::string* utf8) {
		utf8->clear();

		if (size == 0) {
		return true;
		}

		// TODO: Consider using std::wstring_convert once libcxx is supported on
		// Windows.

		// Only Vista or later has this flag that causes WideCharToMultiByte() to
		// return an error on invalid characters.
		const DWORD flags =
		#if (WINVER >= 0x0600)
		WC_ERR_INVALID_CHARS;
		#else
		0;
		#endif

		const int chars_required = WideCharToMultiByte(CP_UTF8, flags, utf16, size,
		NULL, 0, NULL, NULL);
		if (chars_required <= 0) {
		return false;
		}

		// This could potentially throw a std::bad_alloc exception.
		utf8->resize(chars_required);

		const int result = WideCharToMultiByte(CP_UTF8, flags, utf16, size,
		&(*utf8)[0], chars_required, NULL,
		NULL);
		if (result != chars_required) {
		CHECK_LE(result, chars_required) << "WideCharToMultiByte wrote " << result
		<< " chars to buffer of " << chars_required << " chars";
		utf8->clear();
		return false;
		}

		return true;
		}

		bool WideToUTF8(const wchar_t* utf16, std::string* utf8) {
		// Compute string length of NULL-terminated string with wcslen().
		return WideToUTF8(utf16, wcslen(utf16), utf8);
		}

		bool WideToUTF8(const std::wstring& utf16, std::string* utf8) {
		// Use the stored length of the string which allows embedded NULL characters
		// to be converted.
		return WideToUTF8(utf16.c_str(), utf16.length(), utf8);
		}

		// Internal helper function that takes MultiByteToWideChar() flags.
		static bool _UTF8ToWideWithFlags(const char* utf8, const size_t size,
		std::wstring* utf16, const DWORD flags) {
		utf16->clear();

		if (size == 0) {
		return true;
		}

		// TODO: Consider using std::wstring_convert once libcxx is supported on
		// Windows.
		const int chars_required = MultiByteToWideChar(CP_UTF8, flags, utf8, size,
		NULL, 0);
		if (chars_required <= 0) {
		return false;
		}

		// This could potentially throw a std::bad_alloc exception.
		utf16->resize(chars_required);

		const int result = MultiByteToWideChar(CP_UTF8, flags, utf8, size,
		&(*utf16)[0], chars_required);
		if (result != chars_required) {
		CHECK_LE(result, chars_required) << "MultiByteToWideChar wrote " << result
		<< " chars to buffer of " << chars_required << " chars";
		utf16->clear();
		return false;
		}

		return true;
		}

		bool UTF8ToWide(const char* utf8, const size_t size, std::wstring* utf16) {
		// If strictly interpreting as UTF-8 succeeds, return success.
		if (_UTF8ToWideWithFlags(utf8, size, utf16, MB_ERR_INVALID_CHARS)) {
		return true;
		}

		// Fallback to non-strict interpretation, allowing invalid characters and
		// converting as best as possible, and return false to signify a problem.
		(void)_UTF8ToWideWithFlags(utf8, size, utf16, 0);
		return false;
		}

		bool UTF8ToWide(const char* utf8, std::wstring* utf16) {
		// Compute string length of NULL-terminated string with strlen().
		return UTF8ToWide(utf8, strlen(utf8), utf16);
		}

		bool UTF8ToWide(const std::string& utf8, std::wstring* utf16) {
		// Use the stored length of the string which allows embedded NULL characters
		// to be converted.
		return UTF8ToWide(utf8.c_str(), utf8.length(), utf16);
		}

		// Versions of standard library APIs that support UTF-8 strings.
		namespace utf8 {

		int open(const char* name, int flags, ...) {
		std::wstring name_utf16;
		if (!UTF8ToWide(name, &name_utf16)) {
		errno = EINVAL;
		return -1;
		}

		int mode = 0;
		if ((flags & O_CREAT) != 0) {
		va_list args;
		va_start(args, flags);
		mode = va_arg(args, int);
		va_end(args);
		}

		return _wopen(name_utf16.c_str(), flags, mode);
		}

		int unlink(const char* name) {
		std::wstring name_utf16;
		if (!UTF8ToWide(name, &name_utf16)) {
		errno = EINVAL;
		return -1;
		}

		return _wunlink(name_utf16.c_str());
		}

		} // namespace utf8
		} // namespace base
		} // namespace android