Merge "Support bigram historical information migration." (61fc3299) · Commits · e / os / android_packages_inputmethods_LatinIME

native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp

+16 −10

Original line number	Diff line number	Diff line
		@@ -335,7 +335,7 @@ static void latinime_BinaryDictionary_addUnigramWord(JNIEnv *env, jclass clazz,
		if (!shortcutTargetCodePoints.empty()) {
		shortcuts.emplace_back(&shortcutTargetCodePoints, shortcutProbability);
		}
		// Use 1 for count to indicate the word has inputed.
		// Use 1 for count to indicate the word has inputted.
		const UnigramProperty unigramProperty(isNotAWord, isBlacklisted,
		probability, timestamp, 0 /* level /, 1 / count */, &shortcuts);
		dictionary->addUnigramWord(codePoints, codePointCount, &unigramProperty);
		@@ -353,8 +353,12 @@ static void latinime_BinaryDictionary_addBigramWords(JNIEnv *env, jclass clazz,
		jsize word1Length = env->GetArrayLength(word1);
		int word1CodePoints[word1Length];
		env->GetIntArrayRegion(word1, 0, word1Length, word1CodePoints);
		dictionary->addBigramWords(word0CodePoints, word0Length, word1CodePoints,
		word1Length, probability, timestamp);
		const std::vector<int> bigramTargetCodePoints(
		word1CodePoints, word1CodePoints + word1Length);
		// Use 1 for count to indicate the bigram has inputted.
		const BigramProperty bigramProperty(&bigramTargetCodePoints, probability,
		timestamp, 0 /* level /, 1 / count */);
		dictionary->addBigramWords(word0CodePoints, word0Length, &bigramProperty);
		}

		static void latinime_BinaryDictionary_removeBigramWords(JNIEnv *env, jclass clazz, jlong dict,
		@@ -437,14 +441,18 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j
		env->GetIntField(languageModelParam, shortcutProbabilityFieldId);
		shortcuts.emplace_back(&shortcutTargetCodePoints, shortcutProbability);
		}
		// Use 1 for count to indicate the word has inputed.
		// Use 1 for count to indicate the word has inputted.
		const UnigramProperty unigramProperty(isNotAWord, isBlacklisted,
		unigramProbability, timestamp, 0 /* level /, 1 / count */, &shortcuts);
		dictionary->addUnigramWord(word1CodePoints, word1Length, &unigramProperty);
		if (word0) {
		jint bigramProbability = env->GetIntField(languageModelParam, bigramProbabilityFieldId);
		dictionary->addBigramWords(word0CodePoints, word0Length, word1CodePoints, word1Length,
		bigramProbability, timestamp);
		const std::vector<int> bigramTargetCodePoints(
		word1CodePoints, word1CodePoints + word1Length);
		// Use 1 for count to indicate the bigram has inputted.
		const BigramProperty bigramProperty(&bigramTargetCodePoints, bigramProbability,
		timestamp, 0 /* level /, 1 / count */);
		dictionary->addBigramWords(word0CodePoints, word0Length, &bigramProperty);
		}
		if (dictionary->needsToRunGC(true /* mindsBlockByGC */)) {
		return i + 1;
		@@ -558,11 +566,9 @@ static bool latinime_BinaryDictionary_migrateNative(JNIEnv *env, jclass clazz, j
		return false;
		}
		}
		for (const BigramProperty &bigarmProperty : *wordProperty.getBigramProperties()) {
		const std::vector<int> *targetCodePoints = bigarmProperty.getTargetCodePoints();
		for (const BigramProperty &bigramProperty : *wordProperty.getBigramProperties()) {
		if (!dictionaryStructureWithBufferPolicy->addBigramWords(wordCodePoints, wordLength,
		targetCodePoints->data(), targetCodePoints->size(),
		bigarmProperty.getProbability(), bigarmProperty.getTimestamp())) {
		&bigramProperty)) {
		LogUtils::logToJava(env, "Cannot add bigram to the new dict.");
		return false;
		}

native/jni/src/suggest/core/dictionary/dictionary.cpp

+3 −4

Original line number	Diff line number	Diff line
		@@ -88,11 +88,10 @@ void Dictionary::addUnigramWord(const int *const word, const int length,
		mDictionaryStructureWithBufferPolicy->addUnigramWord(word, length, unigramProperty);
		}

		void Dictionary::addBigramWords(const int const word0, const int length0, const int const word1,
		const int length1, const int probability, const int timestamp) {
		void Dictionary::addBigramWords(const int *const word0, const int length0,
		const BigramProperty *const bigramProperty) {
		TimeKeeper::setCurrentTime();
		mDictionaryStructureWithBufferPolicy->addBigramWords(word0, length0, word1, length1,
		probability, timestamp);
		mDictionaryStructureWithBufferPolicy->addBigramWords(word0, length0, bigramProperty);
		}

		void Dictionary::removeBigramWords(const int *const word0, const int length0,

native/jni/src/suggest/core/dictionary/dictionary.h

+2 −2

Original line number	Diff line number	Diff line
		@@ -76,8 +76,8 @@ class Dictionary {
		void addUnigramWord(const int *const codePoints, const int codePointCount,
		const UnigramProperty *const unigramProperty);

		void addBigramWords(const int const word0, const int length0, const int const word1,
		const int length1, const int probability, const int timestamp);
		void addBigramWords(const int *const word0, const int length0,
		const BigramProperty *const bigramProperty);

		void removeBigramWords(const int const word0, const int length0, const int const word1,
		const int length1);

native/jni/src/suggest/core/policy/dictionary_structure_with_buffer_policy.h

+2 −2

Original line number	Diff line number	Diff line
		@@ -73,8 +73,8 @@ class DictionaryStructureWithBufferPolicy {
		const UnigramProperty *const unigramProperty) = 0;

		// Returns whether the update was success or not.
		virtual bool addBigramWords(const int const word0, const int length0, const int const word1,
		const int length1, const int probability, const int timestamp) = 0;
		virtual bool addBigramWords(const int *const word0, const int length0,
		const BigramProperty *const bigramProperty) = 0;

		// Returns whether the update was success or not.
		virtual bool removeBigramWords(const int *const word0, const int length0,

native/jni/src/suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.cpp

+11 −11

Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@

		#include "suggest/policyimpl/dictionary/bigram/ver4_bigram_list_policy.h"

		#include "suggest/core/dictionary/property/bigram_property.h"
		#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h"
		#include "suggest/policyimpl/dictionary/header/header_policy.h"
		#include "suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h"
		@@ -49,13 +50,12 @@ void Ver4BigramListPolicy::getNextBigram(int const outBigramPos, int const out
		}

		bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTargetTerminalId,
		const int newProbability, const int timestamp, bool *const outAddedNewEntry) {
		const BigramProperty const bigramProperty, bool const outAddedNewEntry) {
		// 1. The word has no bigrams yet.
		// 2. The word has bigrams, and there is the target in the list.
		// 3. The word has bigrams, and there is an invalid entry that can be reclaimed.
		// 4. The word has bigrams. We have to append new bigram entry to the list.
		// 5. Same as 4, but the list is the last entry of the content file.

		if (outAddedNewEntry) {
		*outAddedNewEntry = false;
		}
		@@ -69,7 +69,7 @@ bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTarget
		const BigramEntry newBigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
		newTargetTerminalId);
		const BigramEntry bigramEntryToWrite = createUpdatedBigramEntryFrom(&newBigramEntry,
		newProbability, timestamp);
		bigramProperty);
		// Write an entry.
		const int writingPos = mBigramDictContent->getBigramListHeadPos(terminalId);
		if (!mBigramDictContent->writeBigramEntry(&bigramEntryToWrite, writingPos)) {
		@@ -102,7 +102,7 @@ bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTarget
		const BigramEntry newBigramEntry(false /* hasNext */, NOT_A_PROBABILITY,
		newTargetTerminalId);
		const BigramEntry bigramEntryToWrite = createUpdatedBigramEntryFrom(
		&newBigramEntry, newProbability, timestamp);
		&newBigramEntry, bigramProperty);
		if (!mBigramDictContent->writeBigramEntryAtTail(&bigramEntryToWrite)) {
		return false;
		}
		@@ -128,7 +128,7 @@ bool Ver4BigramListPolicy::addNewEntry(const int terminalId, const int newTarget
		const BigramEntry updatedBigramEntry =
		originalBigramEntry.updateTargetTerminalIdAndGetEntry(newTargetTerminalId);
		const BigramEntry bigramEntryToWrite = createUpdatedBigramEntryFrom(
		&updatedBigramEntry, newProbability, timestamp);
		&updatedBigramEntry, bigramProperty);
		return mBigramDictContent->writeBigramEntry(&bigramEntryToWrite, entryPosToUpdate);
		}

		@@ -253,19 +253,19 @@ int Ver4BigramListPolicy::getEntryPosToUpdate(const int targetTerminalIdToFind,
		}

		const BigramEntry Ver4BigramListPolicy::createUpdatedBigramEntryFrom(
		const BigramEntry *const originalBigramEntry, const int newProbability,
		const int timestamp) const {
		const BigramEntry *const originalBigramEntry,
		const BigramProperty *const bigramProperty) const {
		// TODO: Consolidate historical info and probability.
		if (mHeaderPolicy->hasHistoricalInfoOfWords()) {
		// Use 1 for count to indicate the bigram has inputed.
		const HistoricalInfo historicalInfoForUpdate(timestamp, 0 /* level /, 1 / count */);
		const HistoricalInfo historicalInfoForUpdate(bigramProperty->getTimestamp(),
		bigramProperty->getLevel(), bigramProperty->getCount());
		const HistoricalInfo updatedHistoricalInfo =
		ForgettingCurveUtils::createUpdatedHistoricalInfo(
		originalBigramEntry->getHistoricalInfo(), newProbability,
		originalBigramEntry->getHistoricalInfo(), bigramProperty->getProbability(),
		&historicalInfoForUpdate, mHeaderPolicy);
		return originalBigramEntry->updateHistoricalInfoAndGetEntry(&updatedHistoricalInfo);
		} else {
		return originalBigramEntry->updateProbabilityAndGetEntry(newProbability);
		return originalBigramEntry->updateProbabilityAndGetEntry(bigramProperty->getProbability());
		}
		}