Quit use bigram probability diff for ver4 dict. (64341927) · Commits · e / os / android_packages_inputmethods_LatinIME

java/src/com/android/inputmethod/latin/BinaryDictionary.java

+0 −8

Original line number	Diff line number	Diff line
		@@ -214,8 +214,6 @@ public final class BinaryDictionary extends Dictionary {
		private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1);
		private static native int addMultipleDictionaryEntriesNative(long dict,
		LanguageModelParam[] languageModelParams, int startIndex);
		private static native int calculateProbabilityNative(long dict, int unigramProbability,
		int bigramProbability);
		private static native String getPropertyNative(long dict, String query);
		private static native boolean isCorruptedNative(long dict);
		private static native boolean migrateNative(long dict, String dictFilePath,
		@@ -551,12 +549,6 @@ public final class BinaryDictionary extends Dictionary {
		return true;
		}

		@UsedForTesting
		public int calculateProbability(final int unigramProbability, final int bigramProbability) {
		if (!isValidDictionary()) return NOT_A_PROBABILITY;
		return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability);
		}

		@UsedForTesting
		public String getPropertyForTest(final String query) {
		if (!isValidDictionary()) return "";

native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp

+0 −15

Original line number	Diff line number	Diff line
		@@ -467,16 +467,6 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j
		return languageModelParamCount;
		}

		static int latinime_BinaryDictionary_calculateProbabilityNative(JNIEnv *env, jclass clazz,
		jlong dict, jint unigramProbability, jint bigramProbability) {
		Dictionary dictionary = reinterpret_cast<Dictionary >(dict);
		if (!dictionary) {
		return NOT_A_PROBABILITY;
		}
		return dictionary->getDictionaryStructurePolicy()->getProbability(unigramProbability,
		bigramProbability);
		}

		static jstring latinime_BinaryDictionary_getProperty(JNIEnv *env, jclass clazz, jlong dict,
		jstring query) {
		Dictionary dictionary = reinterpret_cast<Dictionary >(dict);
		@@ -669,11 +659,6 @@ static const JNINativeMethod sMethods[] = {
		"(J[Lcom/android/inputmethod/latin/utils/LanguageModelParam;I)I"),
		reinterpret_cast<void *>(latinime_BinaryDictionary_addMultipleDictionaryEntries)
		},
		{
		const_cast<char *>("calculateProbabilityNative"),
		const_cast<char *>("(JII)I"),
		reinterpret_cast<void *>(latinime_BinaryDictionary_calculateProbabilityNative)
		},
		{
		const_cast<char *>("getPropertyNative"),
		const_cast<char *>("(JLjava/lang/String;)Ljava/lang/String;"),

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.cpp

+10 −12

Original line number	Diff line number	Diff line
		@@ -38,8 +38,6 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
		int level = 0;
		int count = 0;
		if (mHasHistoricalInfo) {
		probability = bigramListBuffer->readUintAndAdvancePosition(
		Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
		timestamp = bigramListBuffer->readUintAndAdvancePosition(
		Ver4DictConstants::TIME_STAMP_FIELD_SIZE, bigramEntryPos);
		level = bigramListBuffer->readUintAndAdvancePosition(
		@@ -47,7 +45,8 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
		count = bigramListBuffer->readUintAndAdvancePosition(
		Ver4DictConstants::WORD_COUNT_FIELD_SIZE, bigramEntryPos);
		} else {
		probability = bigramFlags & Ver4DictConstants::BIGRAM_PROBABILITY_MASK;
		probability = bigramListBuffer->readUintAndAdvancePosition(
		Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
		}
		const int encodedTargetTerminalId = bigramListBuffer->readUintAndAdvancePosition(
		Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, bigramEntryPos);
		@@ -65,21 +64,13 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
		bool BigramDictContent::writeBigramEntryAndAdvancePosition(
		const BigramEntry const bigramEntryToWrite, int const entryWritingPos) {
		BufferWithExtendableBuffer *const bigramListBuffer = getWritableContentBuffer();
		const int bigramFlags = createAndGetBigramFlags(
		mHasHistoricalInfo ? 0 : bigramEntryToWrite->getProbability(),
		bigramEntryToWrite->hasNext());
		const int bigramFlags = createAndGetBigramFlags(bigramEntryToWrite->hasNext());
		if (!bigramListBuffer->writeUintAndAdvancePosition(bigramFlags,
		Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, entryWritingPos)) {
		AKLOGE("Cannot write bigram flags. pos: %d, flags: %x", *entryWritingPos, bigramFlags);
		return false;
		}
		if (mHasHistoricalInfo) {
		if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
		Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
		AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
		bigramEntryToWrite->getProbability());
		return false;
		}
		const HistoricalInfo *const historicalInfo = bigramEntryToWrite->getHistoricalInfo();
		if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getTimeStamp(),
		Ver4DictConstants::TIME_STAMP_FIELD_SIZE, entryWritingPos)) {
		@@ -99,6 +90,13 @@ bool BigramDictContent::writeBigramEntryAndAdvancePosition(
		historicalInfo->getCount());
		return false;
		}
		} else {
		if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
		Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
		AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
		bigramEntryToWrite->getProbability());
		return false;
		}
		}
		const int targetTerminalIdToWrite =
		(bigramEntryToWrite->getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) ?

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/bigram_dict_content.h

+2 −3

Original line number	Diff line number	Diff line
		@@ -95,9 +95,8 @@ class BigramDictContent : public SparseTableDictContent {
		private:
		DISALLOW_COPY_AND_ASSIGN(BigramDictContent);

		int createAndGetBigramFlags(const int probability, const bool hasNext) const {
		return (probability & Ver4DictConstants::BIGRAM_PROBABILITY_MASK)
		\| (hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0);
		int createAndGetBigramFlags(const bool hasNext) const {
		return hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0;
		}

		bool runGCBigramList(const int bigramListPos,

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp

+2 −4

Original line number	Diff line number	Diff line
		@@ -115,9 +115,7 @@ int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability,
		} else if (bigramProbability == NOT_A_PROBABILITY) {
		return ProbabilityUtils::backoff(unigramProbability);
		} else {
		// bigramProbability is a bigram probability delta.
		return ProbabilityUtils::computeProbabilityForBigram(unigramProbability,
		bigramProbability);
		return bigramProbability;
		}
		}
		}
		@@ -398,7 +396,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
		const int probability = bigramEntry.hasHistoricalInfo() ?
		ForgettingCurveUtils::decodeProbability(
		bigramEntry.getHistoricalInfo(), mHeaderPolicy) :
		getProbability(word1Probability, bigramEntry.getProbability());
		bigramEntry.getProbability();
		bigrams.emplace_back(&word1, probability,
		historicalInfo->getTimeStamp(), historicalInfo->getLevel(),
		historicalInfo->getCount());