am d9f450ef: Implement removeBigramWords() for DynamicPatriciaTrie. (7c954268) · Commits · e / os / android_packages_inputmethods_LatinIME

native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp

+4 −1

Original line number	Diff line number	Diff line
		@@ -54,11 +54,13 @@ void DynamicBigramListPolicy::skipAllBigrams(int *const pos) const {
		}
		}

		bool DynamicBigramListPolicy::copyAllBigrams(int const fromPos, int const toPos) {
		bool DynamicBigramListPolicy::copyAllBigrams(int const fromPos, int const toPos,
		int *outBigramsCount) {
		const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos);
		if (usesAdditionalBuffer) {
		*fromPos -= mBuffer->getOriginalBufferSize();
		}
		*outBigramsCount = 0;
		BigramListReadWriteUtils::BigramFlags flags;
		do {
		// The buffer address can be changed after calling buffer writing methods.
		@@ -91,6 +93,7 @@ bool DynamicBigramListPolicy::copyAllBigrams(int const fromPos, int const toPo
		toPos)) {
		return false;
		}
		(*outBigramsCount)++;
		} while(BigramListReadWriteUtils::hasNext(flags));
		if (usesAdditionalBuffer) {
		*fromPos += mBuffer->getOriginalBufferSize();

native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h

+3 −2

Original line number	Diff line number	Diff line
		@@ -45,8 +45,9 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy {
		void skipAllBigrams(int *const pos) const;

		// Copy bigrams from the bigram list that starts at fromPos to toPos and advance these
		// positions after bigram lists. This method skips invalid bigram entries.
		bool copyAllBigrams(int const fromPos, int const toPos);
		// positions after bigram lists. This method skips invalid bigram entries and write the valid
		// bigram entry count to outBigramsCount.
		bool copyAllBigrams(int const fromPos, int const toPos, int *outBigramsCount);

		bool addNewBigramEntryToBigramList(const int bigramPos, const int probability, int *const pos);

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp

+20 −12

Original line number	Diff line number	Diff line
		@@ -125,11 +125,10 @@ bool DynamicPatriciaTrieWritingHelper::addBigramWords(const int word0Pos, const
		bool DynamicPatriciaTrieWritingHelper::removeBigramWords(const int word0Pos, const int word1Pos) {
		DynamicPatriciaTrieNodeReader nodeReader(mBuffer, mBigramPolicy, mShortcutPolicy);
		nodeReader.fetchNodeInfoFromBuffer(word0Pos);
		if (nodeReader.isDeleted() \|\| nodeReader.getBigramsPos() == NOT_A_DICT_POS) {
		if (nodeReader.getBigramsPos() == NOT_A_DICT_POS) {
		return false;
		}
		// TODO: Implement.
		return false;
		return mBigramPolicy->removeBigram(nodeReader.getBigramsPos(), word1Pos);
		}

		bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition(
		@@ -193,13 +192,9 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo
		const int originalBigramListPos, const int originalShortcutListPos,
		int *const writingPos) {
		const int nodePos = *writingPos;
		// Create node flags and write them.
		const PatriciaTrieReadingUtils::NodeFlags nodeFlags =
		PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord,
		probability != NOT_A_PROBABILITY, originalShortcutListPos != NOT_A_DICT_POS,
		originalBigramListPos != NOT_A_DICT_POS, codePointCount > 1,
		3 /* childrenPositionFieldSize */);
		if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, nodeFlags,
		// Write dummy flags. The Node flags are updated with appropriate flags at the last step of the
		// PtNode writing.
		if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, 0 /* nodeFlags */,
		writingPos)) {
		return false;
		}
		@@ -212,7 +207,7 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo
		// Write code points
		if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(mBuffer, codePoints,
		codePointCount, writingPos)) {
		return false;;
		return false;
		}
		// Write probability when the probability is a valid probability, which means this node is
		// terminal.
		@@ -235,12 +230,25 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo
		}
		}
		// Copy bigram list when the originalBigramListPos is valid dictionary position.
		int bigramCount = 0;
		if (originalBigramListPos != NOT_A_DICT_POS) {
		int fromPos = originalBigramListPos;
		if (!mBigramPolicy->copyAllBigrams(&fromPos, writingPos)) {
		if (!mBigramPolicy->copyAllBigrams(&fromPos, writingPos, &bigramCount)) {
		return false;
		}
		}
		// Create node flags and write them.
		PatriciaTrieReadingUtils::NodeFlags nodeFlags =
		PatriciaTrieReadingUtils::createAndGetFlags(isBlacklisted, isNotAWord,
		probability != NOT_A_PROBABILITY /* isTerminal */,
		originalShortcutListPos != NOT_A_DICT_POS /* hasShortcutTargets */,
		bigramCount > 0 /* hasBigrams /, codePointCount > 1 / hasMultipleChars */,
		CHILDREN_POSITION_FIELD_SIZE);
		int flagsFieldPos = nodePos;
		if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, nodeFlags,
		&flagsFieldPos)) {
		return false;
		}
		return true;
		}

tests/src/com/android/inputmethod/latin/BinaryDictionaryTests.java

+50 −1

Original line number	Diff line number	Diff line
		@@ -206,6 +206,7 @@ public class BinaryDictionaryTests extends AndroidTestCase {
		final int bigramCount = 1000;
		final int codePointSetSize = 50;
		final int seed = 11111;

		File dictFile = null;
		try {
		dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
		@@ -217,7 +218,6 @@ public class BinaryDictionaryTests extends AndroidTestCase {
		BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
		0 /* offset /, dictFile.length(), true / useFullEditDistance */,
		Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);

		final ArrayList<String> words = new ArrayList<String>();
		// Test a word that isn't contained within the dictionary.
		final Random random = new Random(seed);
		@@ -250,4 +250,53 @@ public class BinaryDictionaryTests extends AndroidTestCase {

		dictFile.delete();
		}

		public void testRemoveBigramWords() {
		File dictFile = null;
		try {
		dictFile = createEmptyDictionaryAndGetFile("TestBinaryDictionary");
		} catch (IOException e) {
		fail("IOException while writing an initial dictionary : " + e);
		} catch (UnsupportedFormatException e) {
		fail("UnsupportedFormatException while writing an initial dictionary : " + e);
		}
		BinaryDictionary binaryDictionary = new BinaryDictionary(dictFile.getAbsolutePath(),
		0 /* offset /, dictFile.length(), true / useFullEditDistance */,
		Locale.getDefault(), TEST_LOCALE, true /* isUpdatable */);

		final int unigramProbability = 100;
		final int bigramProbability = 10;
		binaryDictionary.addUnigramWord("aaa", unigramProbability);
		binaryDictionary.addUnigramWord("abb", unigramProbability);
		binaryDictionary.addUnigramWord("bcc", unigramProbability);
		binaryDictionary.addBigramWords("aaa", "abb", bigramProbability);
		binaryDictionary.addBigramWords("aaa", "bcc", bigramProbability);
		binaryDictionary.addBigramWords("abb", "aaa", bigramProbability);
		binaryDictionary.addBigramWords("abb", "bcc", bigramProbability);

		assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb"));
		assertEquals(true, binaryDictionary.isValidBigram("aaa", "bcc"));
		assertEquals(true, binaryDictionary.isValidBigram("abb", "aaa"));
		assertEquals(true, binaryDictionary.isValidBigram("abb", "bcc"));

		binaryDictionary.removeBigramWords("aaa", "abb");
		assertEquals(false, binaryDictionary.isValidBigram("aaa", "abb"));
		binaryDictionary.addBigramWords("aaa", "abb", bigramProbability);
		assertEquals(true, binaryDictionary.isValidBigram("aaa", "abb"));


		binaryDictionary.removeBigramWords("aaa", "bcc");
		assertEquals(false, binaryDictionary.isValidBigram("aaa", "bcc"));
		binaryDictionary.removeBigramWords("abb", "aaa");
		assertEquals(false, binaryDictionary.isValidBigram("abb", "aaa"));
		binaryDictionary.removeBigramWords("abb", "bcc");
		assertEquals(false, binaryDictionary.isValidBigram("abb", "bcc"));

		binaryDictionary.removeBigramWords("aaa", "abb");
		// Test remove non-existing bigram operation.
		binaryDictionary.removeBigramWords("aaa", "abb");
		binaryDictionary.removeBigramWords("bcc", "aaa");

		dictFile.delete();
		}
		}