Loading native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +26 −27 Original line number Diff line number Diff line Loading @@ -37,7 +37,17 @@ namespace latinime { class ProximityInfo; static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd); // Helper method static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd) { int ret = munmap(const_cast<void *>(dictBuf), length); if (ret != 0) { AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno); } ret = close(fd); if (ret != 0) { AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno); } } static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir, jlong dictOffset, jlong dictSize, jboolean isUpdatable) { Loading Loading @@ -77,8 +87,8 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s return 0; } Dictionary *dictionary = 0; if (BinaryDictionaryFormat::UNKNOWN_VERSION == BinaryDictionaryFormat::detectFormatVersion(static_cast<uint8_t *>(dictBuf), if (BinaryDictionaryFormatUtils::UNKNOWN_VERSION == BinaryDictionaryFormatUtils::detectFormatVersion(static_cast<uint8_t *>(dictBuf), static_cast<int>(dictSize))) { AKLOGE("DICT: dictionary format is unknown, bad magic number"); releaseDictBuf(static_cast<const char *>(dictBuf) - offset, adjDictSize, fd); Loading @@ -91,6 +101,19 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s return reinterpret_cast<jlong>(dictionary); } static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); if (!dictionary) return; const BinaryDictionaryInfo *const binaryDictionaryInfo = dictionary->getBinaryDictionaryInfo(); const int dictBufOffset = binaryDictionaryInfo->getDictBufOffset(); const void *dictBuf = binaryDictionaryInfo->getDictBuf(); if (!dictBuf) return; releaseDictBuf(static_cast<const char *>(dictBuf) - dictBufOffset, binaryDictionaryInfo->getDictSize() + dictBufOffset, binaryDictionaryInfo->getMmapFd()); delete dictionary; } static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz, jlong dict, jlong proximityInfo, jlong dicTraverseSession, jintArray xCoordinatesArray, jintArray yCoordinatesArray, jintArray timesArray, jintArray pointerIdsArray, Loading Loading @@ -222,30 +245,6 @@ static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jclass clazz, ji afterCodePoints, afterLength); } static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); if (!dictionary) return; const BinaryDictionaryInfo *const binaryDictionaryInfo = dictionary->getBinaryDictionaryInfo(); const int dictBufOffset = binaryDictionaryInfo->getDictBufOffset(); const void *dictBuf = binaryDictionaryInfo->getDictBuf(); if (!dictBuf) return; releaseDictBuf(static_cast<const char *>(dictBuf) - dictBufOffset, binaryDictionaryInfo->getDictSize() + dictBufOffset, binaryDictionaryInfo->getMmapFd()); delete dictionary; } static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd) { int ret = munmap(const_cast<void *>(dictBuf), length); if (ret != 0) { AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno); } ret = close(fd); if (ret != 0) { AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno); } } static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz, jlong dict, jintArray word, jint probability) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); Loading native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp +7 −6 Original line number Diff line number Diff line Loading @@ -22,7 +22,7 @@ namespace latinime { * Dictionary size */ // Any file smaller than this is not a dictionary. const int BinaryDictionaryFormat::DICTIONARY_MINIMUM_SIZE = 4; const int BinaryDictionaryFormatUtils::DICTIONARY_MINIMUM_SIZE = 4; /** * Format versions Loading @@ -30,17 +30,18 @@ const int BinaryDictionaryFormat::DICTIONARY_MINIMUM_SIZE = 4; // Originally, format version 1 had a 16-bit magic number, then the version number `01' // then options that must be 0. Hence the first 32-bits of the format are always as follow // and it's okay to consider them a magic number as a whole. const uint32_t BinaryDictionaryFormat::FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B10100; const uint32_t BinaryDictionaryFormatUtils::FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B10100; // The versions of Latin IME that only handle format version 1 only test for the magic // number, so we had to change it so that version 2 files would be rejected by older // implementations. On this occasion, we made the magic number 32 bits long. const uint32_t BinaryDictionaryFormat::FORMAT_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; const uint32_t BinaryDictionaryFormatUtils::FORMAT_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; // Magic number (4 bytes), version (2 bytes), options (2 bytes), header size (4 bytes) = 12 const int BinaryDictionaryFormat::FORMAT_VERSION_2_MINIMUM_SIZE = 12; const int BinaryDictionaryFormatUtils::FORMAT_VERSION_2_MINIMUM_SIZE = 12; /* static */ BinaryDictionaryFormat::FORMAT_VERSION BinaryDictionaryFormat::detectFormatVersion( const uint8_t *const dict, const int dictSize) { /* static */ BinaryDictionaryFormatUtils::FORMAT_VERSION BinaryDictionaryFormatUtils::detectFormatVersion(const uint8_t *const dict, const int dictSize) { // The magic number is stored big-endian. // If the dictionary is less than 4 bytes, we can't even read the magic number, so we don't // understand this format. Loading native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h +2 −2 Original line number Diff line number Diff line Loading @@ -31,7 +31,7 @@ namespace latinime { * reading methods and utility methods for various purposes. * On the other hand, this file deals with only about dictionary format version. */ class BinaryDictionaryFormat { class BinaryDictionaryFormatUtils { public: // TODO: Remove obsolete version logic enum FORMAT_VERSION { Loading @@ -43,7 +43,7 @@ class BinaryDictionaryFormat { static FORMAT_VERSION detectFormatVersion(const uint8_t *const dict, const int dictSize); private: DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormat); DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormatUtils); static const int DICTIONARY_MINIMUM_SIZE; static const uint32_t FORMAT_VERSION_1_MAGIC_NUMBER; Loading native/jni/src/suggest/core/dictionary/binary_dictionary_header.cpp +3 −3 Original line number Diff line number Diff line Loading @@ -29,12 +29,12 @@ const float BinaryDictionaryHeader::MULTI_WORD_COST_MULTIPLIER_SCALE = 100.0f; BinaryDictionaryHeader::BinaryDictionaryHeader( const BinaryDictionaryInfo *const binaryDictionaryInfo) : mBinaryDictionaryInfo(binaryDictionaryInfo), mDictionaryFlags(BinaryDictionaryHeaderReader::getFlags(binaryDictionaryInfo)), mSize(BinaryDictionaryHeaderReader::getHeaderSize(binaryDictionaryInfo)), mDictionaryFlags(BinaryDictionaryHeaderReadingUtils::getFlags(binaryDictionaryInfo)), mSize(BinaryDictionaryHeaderReadingUtils::getHeaderSize(binaryDictionaryInfo)), mMultiWordCostMultiplier(readMultiWordCostMultiplier()) {} float BinaryDictionaryHeader::readMultiWordCostMultiplier() const { const int headerValue = BinaryDictionaryHeaderReader::readHeaderValueInt( const int headerValue = BinaryDictionaryHeaderReadingUtils::readHeaderValueInt( mBinaryDictionaryInfo, MULTIPLE_WORDS_DEMOTION_RATE_KEY); if (headerValue == S_INT_MIN) { // not found Loading native/jni/src/suggest/core/dictionary/binary_dictionary_header.h +5 −4 Original line number Diff line number Diff line Loading @@ -37,15 +37,16 @@ class BinaryDictionaryHeader { } AK_FORCE_INLINE bool supportsDynamicUpdate() const { return BinaryDictionaryHeaderReader::supportsDynamicUpdate(mDictionaryFlags); return BinaryDictionaryHeaderReadingUtils::supportsDynamicUpdate(mDictionaryFlags); } AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const { return BinaryDictionaryHeaderReader::requiresGermanUmlautProcessing(mDictionaryFlags); return BinaryDictionaryHeaderReadingUtils::requiresGermanUmlautProcessing(mDictionaryFlags); } AK_FORCE_INLINE bool requiresFrenchLigatureProcessing() const { return BinaryDictionaryHeaderReader::requiresFrenchLigatureProcessing(mDictionaryFlags); return BinaryDictionaryHeaderReadingUtils::requiresFrenchLigatureProcessing( mDictionaryFlags); } AK_FORCE_INLINE float getMultiWordCostMultiplier() const { Loading @@ -60,7 +61,7 @@ class BinaryDictionaryHeader { static const float MULTI_WORD_COST_MULTIPLIER_SCALE; const BinaryDictionaryInfo *const mBinaryDictionaryInfo; const BinaryDictionaryHeaderReader::DictionaryFlags mDictionaryFlags; const BinaryDictionaryHeaderReadingUtils::DictionaryFlags mDictionaryFlags; const int mSize; const float mMultiWordCostMultiplier; Loading Loading
native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp +26 −27 Original line number Diff line number Diff line Loading @@ -37,7 +37,17 @@ namespace latinime { class ProximityInfo; static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd); // Helper method static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd) { int ret = munmap(const_cast<void *>(dictBuf), length); if (ret != 0) { AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno); } ret = close(fd); if (ret != 0) { AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno); } } static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir, jlong dictOffset, jlong dictSize, jboolean isUpdatable) { Loading Loading @@ -77,8 +87,8 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s return 0; } Dictionary *dictionary = 0; if (BinaryDictionaryFormat::UNKNOWN_VERSION == BinaryDictionaryFormat::detectFormatVersion(static_cast<uint8_t *>(dictBuf), if (BinaryDictionaryFormatUtils::UNKNOWN_VERSION == BinaryDictionaryFormatUtils::detectFormatVersion(static_cast<uint8_t *>(dictBuf), static_cast<int>(dictSize))) { AKLOGE("DICT: dictionary format is unknown, bad magic number"); releaseDictBuf(static_cast<const char *>(dictBuf) - offset, adjDictSize, fd); Loading @@ -91,6 +101,19 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s return reinterpret_cast<jlong>(dictionary); } static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); if (!dictionary) return; const BinaryDictionaryInfo *const binaryDictionaryInfo = dictionary->getBinaryDictionaryInfo(); const int dictBufOffset = binaryDictionaryInfo->getDictBufOffset(); const void *dictBuf = binaryDictionaryInfo->getDictBuf(); if (!dictBuf) return; releaseDictBuf(static_cast<const char *>(dictBuf) - dictBufOffset, binaryDictionaryInfo->getDictSize() + dictBufOffset, binaryDictionaryInfo->getMmapFd()); delete dictionary; } static int latinime_BinaryDictionary_getSuggestions(JNIEnv *env, jclass clazz, jlong dict, jlong proximityInfo, jlong dicTraverseSession, jintArray xCoordinatesArray, jintArray yCoordinatesArray, jintArray timesArray, jintArray pointerIdsArray, Loading Loading @@ -222,30 +245,6 @@ static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jclass clazz, ji afterCodePoints, afterLength); } static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); if (!dictionary) return; const BinaryDictionaryInfo *const binaryDictionaryInfo = dictionary->getBinaryDictionaryInfo(); const int dictBufOffset = binaryDictionaryInfo->getDictBufOffset(); const void *dictBuf = binaryDictionaryInfo->getDictBuf(); if (!dictBuf) return; releaseDictBuf(static_cast<const char *>(dictBuf) - dictBufOffset, binaryDictionaryInfo->getDictSize() + dictBufOffset, binaryDictionaryInfo->getMmapFd()); delete dictionary; } static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd) { int ret = munmap(const_cast<void *>(dictBuf), length); if (ret != 0) { AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno); } ret = close(fd); if (ret != 0) { AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno); } } static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz, jlong dict, jintArray word, jint probability) { Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict); Loading
native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.cpp +7 −6 Original line number Diff line number Diff line Loading @@ -22,7 +22,7 @@ namespace latinime { * Dictionary size */ // Any file smaller than this is not a dictionary. const int BinaryDictionaryFormat::DICTIONARY_MINIMUM_SIZE = 4; const int BinaryDictionaryFormatUtils::DICTIONARY_MINIMUM_SIZE = 4; /** * Format versions Loading @@ -30,17 +30,18 @@ const int BinaryDictionaryFormat::DICTIONARY_MINIMUM_SIZE = 4; // Originally, format version 1 had a 16-bit magic number, then the version number `01' // then options that must be 0. Hence the first 32-bits of the format are always as follow // and it's okay to consider them a magic number as a whole. const uint32_t BinaryDictionaryFormat::FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B10100; const uint32_t BinaryDictionaryFormatUtils::FORMAT_VERSION_1_MAGIC_NUMBER = 0x78B10100; // The versions of Latin IME that only handle format version 1 only test for the magic // number, so we had to change it so that version 2 files would be rejected by older // implementations. On this occasion, we made the magic number 32 bits long. const uint32_t BinaryDictionaryFormat::FORMAT_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; const uint32_t BinaryDictionaryFormatUtils::FORMAT_VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; // Magic number (4 bytes), version (2 bytes), options (2 bytes), header size (4 bytes) = 12 const int BinaryDictionaryFormat::FORMAT_VERSION_2_MINIMUM_SIZE = 12; const int BinaryDictionaryFormatUtils::FORMAT_VERSION_2_MINIMUM_SIZE = 12; /* static */ BinaryDictionaryFormat::FORMAT_VERSION BinaryDictionaryFormat::detectFormatVersion( const uint8_t *const dict, const int dictSize) { /* static */ BinaryDictionaryFormatUtils::FORMAT_VERSION BinaryDictionaryFormatUtils::detectFormatVersion(const uint8_t *const dict, const int dictSize) { // The magic number is stored big-endian. // If the dictionary is less than 4 bytes, we can't even read the magic number, so we don't // understand this format. Loading
native/jni/src/suggest/core/dictionary/binary_dictionary_format_utils.h +2 −2 Original line number Diff line number Diff line Loading @@ -31,7 +31,7 @@ namespace latinime { * reading methods and utility methods for various purposes. * On the other hand, this file deals with only about dictionary format version. */ class BinaryDictionaryFormat { class BinaryDictionaryFormatUtils { public: // TODO: Remove obsolete version logic enum FORMAT_VERSION { Loading @@ -43,7 +43,7 @@ class BinaryDictionaryFormat { static FORMAT_VERSION detectFormatVersion(const uint8_t *const dict, const int dictSize); private: DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormat); DISALLOW_IMPLICIT_CONSTRUCTORS(BinaryDictionaryFormatUtils); static const int DICTIONARY_MINIMUM_SIZE; static const uint32_t FORMAT_VERSION_1_MAGIC_NUMBER; Loading
native/jni/src/suggest/core/dictionary/binary_dictionary_header.cpp +3 −3 Original line number Diff line number Diff line Loading @@ -29,12 +29,12 @@ const float BinaryDictionaryHeader::MULTI_WORD_COST_MULTIPLIER_SCALE = 100.0f; BinaryDictionaryHeader::BinaryDictionaryHeader( const BinaryDictionaryInfo *const binaryDictionaryInfo) : mBinaryDictionaryInfo(binaryDictionaryInfo), mDictionaryFlags(BinaryDictionaryHeaderReader::getFlags(binaryDictionaryInfo)), mSize(BinaryDictionaryHeaderReader::getHeaderSize(binaryDictionaryInfo)), mDictionaryFlags(BinaryDictionaryHeaderReadingUtils::getFlags(binaryDictionaryInfo)), mSize(BinaryDictionaryHeaderReadingUtils::getHeaderSize(binaryDictionaryInfo)), mMultiWordCostMultiplier(readMultiWordCostMultiplier()) {} float BinaryDictionaryHeader::readMultiWordCostMultiplier() const { const int headerValue = BinaryDictionaryHeaderReader::readHeaderValueInt( const int headerValue = BinaryDictionaryHeaderReadingUtils::readHeaderValueInt( mBinaryDictionaryInfo, MULTIPLE_WORDS_DEMOTION_RATE_KEY); if (headerValue == S_INT_MIN) { // not found Loading
native/jni/src/suggest/core/dictionary/binary_dictionary_header.h +5 −4 Original line number Diff line number Diff line Loading @@ -37,15 +37,16 @@ class BinaryDictionaryHeader { } AK_FORCE_INLINE bool supportsDynamicUpdate() const { return BinaryDictionaryHeaderReader::supportsDynamicUpdate(mDictionaryFlags); return BinaryDictionaryHeaderReadingUtils::supportsDynamicUpdate(mDictionaryFlags); } AK_FORCE_INLINE bool requiresGermanUmlautProcessing() const { return BinaryDictionaryHeaderReader::requiresGermanUmlautProcessing(mDictionaryFlags); return BinaryDictionaryHeaderReadingUtils::requiresGermanUmlautProcessing(mDictionaryFlags); } AK_FORCE_INLINE bool requiresFrenchLigatureProcessing() const { return BinaryDictionaryHeaderReader::requiresFrenchLigatureProcessing(mDictionaryFlags); return BinaryDictionaryHeaderReadingUtils::requiresFrenchLigatureProcessing( mDictionaryFlags); } AK_FORCE_INLINE float getMultiWordCostMultiplier() const { Loading @@ -60,7 +61,7 @@ class BinaryDictionaryHeader { static const float MULTI_WORD_COST_MULTIPLIER_SCALE; const BinaryDictionaryInfo *const mBinaryDictionaryInfo; const BinaryDictionaryHeaderReader::DictionaryFlags mDictionaryFlags; const BinaryDictionaryHeaderReadingUtils::DictionaryFlags mDictionaryFlags; const int mSize; const float mMultiWordCostMultiplier; Loading