Compute probability using structure policy. (65d19946) · Commits · e / os / android_packages_inputmethods_LatinIME

native/jni/src/suggest/core/dicnode/dic_node_utils.cpp

+4 −3

Original line number	Diff line number	Diff line
		@@ -21,7 +21,6 @@
		#include "suggest/core/dicnode/dic_node.h"
		#include "suggest/core/dicnode/dic_node_vector.h"
		#include "suggest/core/dictionary/multi_bigram_map.h"
		#include "suggest/core/dictionary/probability_utils.h"
		#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
		#include "utils/char_utils.h"

		@@ -93,13 +92,15 @@ namespace latinime {
		if (NOT_A_VALID_WORD_POS == wordPos \|\| NOT_A_VALID_WORD_POS == prevWordPos) {
		// Note: Normally wordPos comes from the dictionary and should never equal
		// NOT_A_VALID_WORD_POS.
		return ProbabilityUtils::backoff(unigramProbability);
		return dictionaryStructurePolicy->getProbability(unigramProbability,
		NOT_A_PROBABILITY);
		}
		if (multiBigramMap) {
		return multiBigramMap->getBigramProbability(dictionaryStructurePolicy, prevWordPos,
		wordPos, unigramProbability);
		}
		return ProbabilityUtils::backoff(unigramProbability);
		return dictionaryStructurePolicy->getProbability(unigramProbability,
		NOT_A_PROBABILITY);
		}

		////////////////

native/jni/src/suggest/core/dicnode/internal/dic_node_state_prevword.h

+0 −4

Original line number	Diff line number	Diff line
		@@ -116,10 +116,6 @@ class DicNodeStatePrevWord {
		return mPrevWordStart;
		}

		int16_t getPrevWordProbability() const {
		return mPrevWordProbability;
		}

		int getPrevWordNodePos() const {
		return mPrevWordNodePos;
		}

native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp

+1 −2

Original line number	Diff line number	Diff line
		@@ -23,7 +23,6 @@
		#include "defines.h"
		#include "suggest/core/dictionary/binary_dictionary_bigrams_iterator.h"
		#include "suggest/core/dictionary/dictionary.h"
		#include "suggest/core/dictionary/probability_utils.h"
		#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
		#include "utils/char_utils.h"

		@@ -131,7 +130,7 @@ int BigramDictionary::getPredictions(const int *prevWord, const int prevWordLeng
		// resulting probability is 8 - although in the practice it's never bigger than 3 or 4
		// in very bad cases. This means that sometimes, we'll see some bigrams interverted
		// here, but it can't get too bad.
		const int probability = ProbabilityUtils::computeProbabilityForBigram(
		const int probability = mDictionaryStructurePolicy->getProbability(
		unigramProbability, bigramsIt.getProbability());
		addWordBigram(bigramBuffer, codePointCount, probability, outBigramProbability,
		outBigramCodePoints, outputTypes);

native/jni/src/suggest/core/dictionary/dictionary.cpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -90,7 +90,7 @@ int Dictionary::getProbability(const int *word, int length) const {
		if (NOT_A_VALID_WORD_POS == pos) {
		return NOT_A_PROBABILITY;
		}
		return getDictionaryStructurePolicy()->getUnigramProbability(pos);
		return getDictionaryStructurePolicy()->getUnigramProbabilityOfPtNode(pos);
		}

		bool Dictionary::isValidBigram(const int word0, int length0, const int word1, int length1) const {

native/jni/src/suggest/core/dictionary/multi_bigram_map.h

+11 −10

Original line number	Diff line number	Diff line
		@@ -22,7 +22,6 @@
		#include "defines.h"
		#include "suggest/core/dictionary/binary_dictionary_bigrams_iterator.h"
		#include "suggest/core/dictionary/bloom_filter.h"
		#include "suggest/core/dictionary/probability_utils.h"
		#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
		#include "utils/hash_map_compat.h"

		@@ -43,11 +42,12 @@ class MultiBigramMap {
		hash_map_compat<int, BigramMap>::const_iterator mapPosition =
		mBigramMaps.find(wordPosition);
		if (mapPosition != mBigramMaps.end()) {
		return mapPosition->second.getBigramProbability(nextWordPosition, unigramProbability);
		return mapPosition->second.getBigramProbability(structurePolicy, nextWordPosition,
		unigramProbability);
		}
		if (mBigramMaps.size() < MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP) {
		addBigramsForWordPosition(structurePolicy, wordPosition);
		return mBigramMaps[wordPosition].getBigramProbability(
		return mBigramMaps[wordPosition].getBigramProbability(structurePolicy,
		nextWordPosition, unigramProbability);
		}
		return readBigramProbabilityFromBinaryDictionary(structurePolicy, wordPosition,
		@@ -82,17 +82,17 @@ class MultiBigramMap {
		}

		AK_FORCE_INLINE int getBigramProbability(
		const DictionaryStructureWithBufferPolicy *const structurePolicy,
		const int nextWordPosition, const int unigramProbability) const {
		int bigramProbability = NOT_A_PROBABILITY;
		if (mBloomFilter.isInFilter(nextWordPosition)) {
		const hash_map_compat<int, int>::const_iterator bigramProbabilityIt =
		mBigramMap.find(nextWordPosition);
		if (bigramProbabilityIt != mBigramMap.end()) {
		const int bigramProbability = bigramProbabilityIt->second;
		return ProbabilityUtils::computeProbabilityForBigram(
		unigramProbability, bigramProbability);
		bigramProbability = bigramProbabilityIt->second;
		}
		}
		return ProbabilityUtils::backoff(unigramProbability);
		return structurePolicy->getProbability(unigramProbability, bigramProbability);
		}

		private:
		@@ -111,17 +111,18 @@ class MultiBigramMap {
		AK_FORCE_INLINE int readBigramProbabilityFromBinaryDictionary(
		const DictionaryStructureWithBufferPolicy *const structurePolicy, const int nodePos,
		const int nextWordPosition, const int unigramProbability) {
		int bigramProbability = NOT_A_PROBABILITY;
		const int bigramsListPos = structurePolicy->getBigramsPositionOfNode(nodePos);
		BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(),
		bigramsListPos);
		while (bigramsIt.hasNext()) {
		bigramsIt.next();
		if (bigramsIt.getBigramPos() == nextWordPosition) {
		return ProbabilityUtils::computeProbabilityForBigram(
		unigramProbability, bigramsIt.getProbability());
		bigramProbability = bigramsIt.getProbability();
		break;
		}
		}
		return ProbabilityUtils::backoff(unigramProbability);
		return structurePolicy->getProbability(unigramProbability, bigramProbability);
		}

		static const size_t MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP;