Loading java/src/com/android/inputmethod/latin/BinaryDictionary.java +20 −0 Original line number Original line Diff line number Diff line Loading @@ -199,6 +199,7 @@ public final class BinaryDictionary extends Dictionary { int[] word, int probability, int timestamp); int[] word, int probability, int timestamp); private static native boolean removeNgramEntryNative(long dict, private static native boolean removeNgramEntryNative(long dict, int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word); int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word); // TODO: Rename to updateEntriesForWordWithNgramContextNative. private static native boolean updateCounterNative(long dict, private static native boolean updateCounterNative(long dict, int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word, boolean isValidWord, int count, int timestamp); int[] word, boolean isValidWord, int count, int timestamp); Loading Loading @@ -493,6 +494,25 @@ public final class BinaryDictionary extends Dictionary { return true; return true; } } // Update entries for the word occurrence with the ngramContext. @UsedForTesting public boolean updateEntriesForWordWithNgramContext(final NgramContext ngramContext, final String word, final boolean isValidWord, final int count, final int timestamp) { if (TextUtils.isEmpty(word)) { return false; } final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); final int[] wordCodePoints = StringUtils.toCodePointArray(word); if (!updateCounterNative(mNativeDict, prevWordCodePointArrays, isBeginningOfSentenceArray, wordCodePoints, isValidWord, count, timestamp)) { return false; } mHasUpdated = true; return true; } @UsedForTesting @UsedForTesting public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) { public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) { if (!isValidDictionary()) return; if (!isValidDictionary()) return; Loading tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +73 −84 Original line number Original line Diff line number Diff line Loading @@ -75,20 +75,25 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { return formatVersion > FormatSpec.VERSION401; return formatVersion > FormatSpec.VERSION401; } } private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, private void onInputWord(final BinaryDictionary binaryDictionary, final String word, final int probability) { final boolean isValidWord) { binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */, binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.EMPTY_PREV_WORDS_INFO, BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); false /* isBeginningOfSentence */, false /* isNotAWord */, false /* isBlacklisted */, mCurrentTime /* timestamp */); } } private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, private void onInputWordWithPrevWord(final BinaryDictionary binaryDictionary, final String word, final String word1, final int probability) { final boolean isValidWord, final String prevWord) { binaryDictionary.addNgramEntry(new NgramContext(new WordInfo(word0)), word1, probability, binaryDictionary.updateEntriesForWordWithNgramContext( new NgramContext(new WordInfo(prevWord)), word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); mCurrentTime /* timestamp */); } } private void onInputWordWithBeginningOfSentenceContext( final BinaryDictionary binaryDictionary, final String word, final boolean isValidWord) { binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.BEGINNING_OF_SENTENCE, word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); } private static boolean isValidBigram(final BinaryDictionary binaryDictionary, private static boolean isValidBigram(final BinaryDictionary binaryDictionary, final String word0, final String word1) { final String word0, final String word1) { return binaryDictionary.isValidNgram(new NgramContext(new WordInfo(word0)), word1); return binaryDictionary.isValidNgram(new NgramContext(new WordInfo(word0)), word1); Loading Loading @@ -175,10 +180,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { setCurrentTimeForTestMode(mCurrentTime); setCurrentTimeForTestMode(mCurrentTime); final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addUnigramWord(binaryDictionary, "ab", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "ab", true /* isValidWord */); addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "aaa", true /* isValidWord */, "a"); addBigramWords(binaryDictionary, "a", "aaa", DUMMY_PROBABILITY); binaryDictionary.flushWithGC(); binaryDictionary.flushWithGC(); binaryDictionary.close(); binaryDictionary.close(); Loading Loading @@ -229,28 +233,27 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "a", false /* isValidWord */); assertFalse(binaryDictionary.isValidWord("a")); assertFalse(binaryDictionary.isValidWord("a")); addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "a", false /* isValidWord */); addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "a", false /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); assertTrue(binaryDictionary.isValidWord("a")); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "b", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("b")); assertTrue(binaryDictionary.isValidWord("b")); addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a"); assertFalse(isValidBigram(binaryDictionary, "a", "b")); assertFalse(isValidBigram(binaryDictionary, "a", "b")); addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "b")); assertTrue(isValidBigram(binaryDictionary, "a", "b")); addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "c", true /* isValidWord */, "a"); addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY); assertTrue(isValidBigram(binaryDictionary, "a", "c")); assertTrue(isValidBigram(binaryDictionary, "a", "c")); // Add bigrams of not valid unigrams. // Add bigrams of not valid unigrams. addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "y", false /* isValidWord */, "x"); assertFalse(isValidBigram(binaryDictionary, "x", "y")); assertFalse(isValidBigram(binaryDictionary, "x", "y")); addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "y", true /* isValidWord */, "x"); assertFalse(isValidBigram(binaryDictionary, "x", "y")); assertFalse(isValidBigram(binaryDictionary, "x", "y")); binaryDictionary.close(); binaryDictionary.close(); Loading @@ -266,36 +269,32 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); forcePassingShortTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); assertFalse(binaryDictionary.isValidWord("a")); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); forcePassingShortTime(binaryDictionary); assertTrue(binaryDictionary.isValidWord("a")); assertTrue(binaryDictionary.isValidWord("a")); forcePassingLongTime(binaryDictionary); forcePassingLongTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); assertFalse(binaryDictionary.isValidWord("a")); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); assertTrue(isValidBigram(binaryDictionary, "a", "b")); assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); forcePassingShortTime(binaryDictionary); assertFalse(isValidBigram(binaryDictionary, "a", "b")); assertFalse(isValidBigram(binaryDictionary, "a", "b")); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); assertTrue(isValidBigram(binaryDictionary, "a", "b")); assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); forcePassingShortTime(binaryDictionary); assertTrue(isValidBigram(binaryDictionary, "a", "b")); assertTrue(isValidBigram(binaryDictionary, "a", "b")); Loading Loading @@ -333,7 +332,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY)); binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY)); for (int i = 0; i < unigramTypedCount; i++) { for (int i = 0; i < unigramTypedCount; i++) { final String word = words.get(random.nextInt(words.size())); final String word = words.get(random.nextInt(words.size())); addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); onInputWord(binaryDictionary, word, true /* isValidWord */); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int unigramCountBeforeGC = final int unigramCountBeforeGC = Loading Loading @@ -380,10 +379,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String strong = "strong"; final String strong = "strong"; final String weak = "weak"; final String weak = "weak"; for (int j = 0; j < strongUnigramTypedCount; j++) { for (int j = 0; j < strongUnigramTypedCount; j++) { addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY); onInputWord(binaryDictionary, strong, true /* isValidWord */); } } for (int j = 0; j < weakUnigramTypedCount; j++) { for (int j = 0; j < weakUnigramTypedCount; j++) { addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY); onInputWord(binaryDictionary, weak, true /* isValidWord */); } } assertTrue(binaryDictionary.isValidWord(strong)); assertTrue(binaryDictionary.isValidWord(strong)); assertTrue(binaryDictionary.isValidWord(weak)); assertTrue(binaryDictionary.isValidWord(weak)); Loading @@ -391,7 +390,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { for (int i = 0; i < unigramCount; i++) { for (int i = 0; i < unigramCount; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); final String word = CodePointUtils.generateWord(random, codePointSet); for (int j = 0; j < eachUnigramTypedCount; j++) { for (int j = 0; j < eachUnigramTypedCount; j++) { addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); onInputWord(binaryDictionary, word, true /* isValidWord */); } } if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int unigramCountBeforeGC = final int unigramCountBeforeGC = Loading Loading @@ -450,12 +449,13 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } } final int maxBigramCount = Integer.parseInt( final int maxBigramCount = Integer.parseInt( binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); binaryDictionary.getPropertyForGettingStats( BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); for (int i = 0; i < bigramTypedCount; ++i) { for (int i = 0; i < bigramTypedCount; ++i) { final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size())); final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size())); addUnigramWord(binaryDictionary, bigram.first, DUMMY_PROBABILITY); onInputWord(binaryDictionary, bigram.first, true /* isValidWord */); addUnigramWord(binaryDictionary, bigram.second, DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, bigram.second, true /* isValidWord */, addBigramWords(binaryDictionary, bigram.first, bigram.second, DUMMY_PROBABILITY); bigram.first); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int bigramCountBeforeGC = final int bigramCountBeforeGC = Loading Loading @@ -506,23 +506,22 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String word = CodePointUtils.generateWord(random, codePointSet); final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); words.add(word); for (int j = 0; j < unigramTypedCount; j++) { for (int j = 0; j < unigramTypedCount; j++) { addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); onInputWord(binaryDictionary, word, true /* isValidWord */); } } } } final String strong = "strong"; final String strong = "strong"; final String weak = "weak"; final String weak = "weak"; final String target = "target"; final String target = "target"; for (int j = 0; j < unigramTypedCount; j++) { for (int j = 0; j < unigramTypedCount; j++) { addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY); onInputWord(binaryDictionary, weak, true /* isValidWord */); addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY); onInputWord(binaryDictionary, strong, true /* isValidWord */); addUnigramWord(binaryDictionary, target, DUMMY_PROBABILITY); } } binaryDictionary.flushWithGC(); binaryDictionary.flushWithGC(); for (int j = 0; j < strongBigramTypedCount; j++) { for (int j = 0; j < strongBigramTypedCount; j++) { addBigramWords(binaryDictionary, strong, target, DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, strong); } } for (int j = 0; j < weakBigramTypedCount; j++) { for (int j = 0; j < weakBigramTypedCount; j++) { addBigramWords(binaryDictionary, weak, target, DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, weak); } } assertTrue(isValidBigram(binaryDictionary, strong, target)); assertTrue(isValidBigram(binaryDictionary, strong, target)); assertTrue(isValidBigram(binaryDictionary, weak, target)); assertTrue(isValidBigram(binaryDictionary, weak, target)); Loading @@ -535,7 +534,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String word1 = words.get(word1Index); final String word1 = words.get(word1Index); for (int j = 0; j < eachBigramTypedCount; j++) { for (int j = 0; j < eachBigramTypedCount; j++) { addBigramWords(binaryDictionary, word0, word1, DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, word1, true /* isValidWord */, word0); } } if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int bigramCountBeforeGC = final int bigramCountBeforeGC = Loading Loading @@ -563,19 +562,18 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { setCurrentTimeForTestMode(mCurrentTime); setCurrentTimeForTestMode(mCurrentTime); final File dictFile = createEmptyDictionaryAndGetFile(fromFormatVersion); final File dictFile = createEmptyDictionaryAndGetFile(fromFormatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "aaa", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("aaa")); assertTrue(binaryDictionary.isValidWord("aaa")); addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "ccc", true /* isValidWord */); assertFalse(binaryDictionary.isValidWord("bbb")); onInputWord(binaryDictionary, "ccc", true /* isValidWord */); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "ccc", true /* isValidWord */); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "ccc", true /* isValidWord */); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "ccc", true /* isValidWord */); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "abc", true /* isValidWord */, "aaa"); addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY); addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa"); assertFalse(binaryDictionary.isValidWord("bbb")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); Loading @@ -585,11 +583,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { assertTrue(binaryDictionary.isValidWord("aaa")); assertTrue(binaryDictionary.isValidWord("aaa")); assertFalse(binaryDictionary.isValidWord("bbb")); assertFalse(binaryDictionary.isValidWord("bbb")); assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc")); assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc")); addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "bbb", false /* isValidWord */); assertTrue(binaryDictionary.isValidWord("bbb")); assertTrue(binaryDictionary.isValidWord("bbb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa"); assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); binaryDictionary.close(); binaryDictionary.close(); } } Loading @@ -612,28 +610,19 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isBlacklisted */, true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isBlacklisted */, mCurrentTime); mCurrentTime); final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE; final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE; addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY, mCurrentTime); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY, onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); mCurrentTime); onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); binaryDictionary.addNgramEntry(beginningOfSentenceContext, "bbb", DUMMY_PROBABILITY, mCurrentTime); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); forcePassingLongTime(binaryDictionary); forcePassingLongTime(binaryDictionary); assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY, onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); mCurrentTime); addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); binaryDictionary.addNgramEntry(beginningOfSentenceContext, "bbb", DUMMY_PROBABILITY, mCurrentTime); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); binaryDictionary.close(); binaryDictionary.close(); Loading @@ -651,10 +640,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); addUnigramWord(binaryDictionary, "aaa", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "aaa", false /* isValidWord */); assertFalse(binaryDictionary.isValidWord("aaa")); assertFalse(binaryDictionary.isValidWord("aaa")); for (int i = 0; i < unigramInputCount; i++) { for (int i = 0; i < unigramInputCount; i++) { addUnigramWord(binaryDictionary, "aaa", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "aaa", false /* isValidWord */); } } assertTrue(binaryDictionary.isValidWord("aaa")); assertTrue(binaryDictionary.isValidWord("aaa")); assertTrue(binaryDictionary.removeUnigramEntry("aaa")); assertTrue(binaryDictionary.removeUnigramEntry("aaa")); Loading Loading
java/src/com/android/inputmethod/latin/BinaryDictionary.java +20 −0 Original line number Original line Diff line number Diff line Loading @@ -199,6 +199,7 @@ public final class BinaryDictionary extends Dictionary { int[] word, int probability, int timestamp); int[] word, int probability, int timestamp); private static native boolean removeNgramEntryNative(long dict, private static native boolean removeNgramEntryNative(long dict, int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word); int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word); // TODO: Rename to updateEntriesForWordWithNgramContextNative. private static native boolean updateCounterNative(long dict, private static native boolean updateCounterNative(long dict, int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[][] prevWordCodePointArrays, boolean[] isBeginningOfSentenceArray, int[] word, boolean isValidWord, int count, int timestamp); int[] word, boolean isValidWord, int count, int timestamp); Loading Loading @@ -493,6 +494,25 @@ public final class BinaryDictionary extends Dictionary { return true; return true; } } // Update entries for the word occurrence with the ngramContext. @UsedForTesting public boolean updateEntriesForWordWithNgramContext(final NgramContext ngramContext, final String word, final boolean isValidWord, final int count, final int timestamp) { if (TextUtils.isEmpty(word)) { return false; } final int[][] prevWordCodePointArrays = new int[ngramContext.getPrevWordCount()][]; final boolean[] isBeginningOfSentenceArray = new boolean[ngramContext.getPrevWordCount()]; ngramContext.outputToArray(prevWordCodePointArrays, isBeginningOfSentenceArray); final int[] wordCodePoints = StringUtils.toCodePointArray(word); if (!updateCounterNative(mNativeDict, prevWordCodePointArrays, isBeginningOfSentenceArray, wordCodePoints, isValidWord, count, timestamp)) { return false; } mHasUpdated = true; return true; } @UsedForTesting @UsedForTesting public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) { public void addMultipleDictionaryEntries(final LanguageModelParam[] languageModelParams) { if (!isValidDictionary()) return; if (!isValidDictionary()) return; Loading
tests/src/com/android/inputmethod/latin/BinaryDictionaryDecayingTests.java +73 −84 Original line number Original line Diff line number Diff line Loading @@ -75,20 +75,25 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { return formatVersion > FormatSpec.VERSION401; return formatVersion > FormatSpec.VERSION401; } } private void addUnigramWord(final BinaryDictionary binaryDictionary, final String word, private void onInputWord(final BinaryDictionary binaryDictionary, final String word, final int probability) { final boolean isValidWord) { binaryDictionary.addUnigramEntry(word, probability, "" /* shortcutTarget */, binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.EMPTY_PREV_WORDS_INFO, BinaryDictionary.NOT_A_PROBABILITY /* shortcutProbability */, word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); false /* isBeginningOfSentence */, false /* isNotAWord */, false /* isBlacklisted */, mCurrentTime /* timestamp */); } } private void addBigramWords(final BinaryDictionary binaryDictionary, final String word0, private void onInputWordWithPrevWord(final BinaryDictionary binaryDictionary, final String word, final String word1, final int probability) { final boolean isValidWord, final String prevWord) { binaryDictionary.addNgramEntry(new NgramContext(new WordInfo(word0)), word1, probability, binaryDictionary.updateEntriesForWordWithNgramContext( new NgramContext(new WordInfo(prevWord)), word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); mCurrentTime /* timestamp */); } } private void onInputWordWithBeginningOfSentenceContext( final BinaryDictionary binaryDictionary, final String word, final boolean isValidWord) { binaryDictionary.updateEntriesForWordWithNgramContext(NgramContext.BEGINNING_OF_SENTENCE, word, isValidWord, 1 /* count */, mCurrentTime /* timestamp */); } private static boolean isValidBigram(final BinaryDictionary binaryDictionary, private static boolean isValidBigram(final BinaryDictionary binaryDictionary, final String word0, final String word1) { final String word0, final String word1) { return binaryDictionary.isValidNgram(new NgramContext(new WordInfo(word0)), word1); return binaryDictionary.isValidNgram(new NgramContext(new WordInfo(word0)), word1); Loading Loading @@ -175,10 +180,9 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { setCurrentTimeForTestMode(mCurrentTime); setCurrentTimeForTestMode(mCurrentTime); final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addUnigramWord(binaryDictionary, "ab", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "ab", true /* isValidWord */); addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "aaa", true /* isValidWord */, "a"); addBigramWords(binaryDictionary, "a", "aaa", DUMMY_PROBABILITY); binaryDictionary.flushWithGC(); binaryDictionary.flushWithGC(); binaryDictionary.close(); binaryDictionary.close(); Loading Loading @@ -229,28 +233,27 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "a", false /* isValidWord */); assertFalse(binaryDictionary.isValidWord("a")); assertFalse(binaryDictionary.isValidWord("a")); addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "a", false /* isValidWord */); addUnigramWord(binaryDictionary, "a", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "a", false /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); assertTrue(binaryDictionary.isValidWord("a")); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "b", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("b")); assertTrue(binaryDictionary.isValidWord("b")); addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a"); assertFalse(isValidBigram(binaryDictionary, "a", "b")); assertFalse(isValidBigram(binaryDictionary, "a", "b")); addBigramWords(binaryDictionary, "a", "b", Dictionary.NOT_A_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "b", false /* isValidWord */, "a"); assertTrue(isValidBigram(binaryDictionary, "a", "b")); assertTrue(isValidBigram(binaryDictionary, "a", "b")); addUnigramWord(binaryDictionary, "c", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "c", true /* isValidWord */, "a"); addBigramWords(binaryDictionary, "a", "c", DUMMY_PROBABILITY); assertTrue(isValidBigram(binaryDictionary, "a", "c")); assertTrue(isValidBigram(binaryDictionary, "a", "c")); // Add bigrams of not valid unigrams. // Add bigrams of not valid unigrams. addBigramWords(binaryDictionary, "x", "y", Dictionary.NOT_A_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "y", false /* isValidWord */, "x"); assertFalse(isValidBigram(binaryDictionary, "x", "y")); assertFalse(isValidBigram(binaryDictionary, "x", "y")); addBigramWords(binaryDictionary, "x", "y", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "y", true /* isValidWord */, "x"); assertFalse(isValidBigram(binaryDictionary, "x", "y")); assertFalse(isValidBigram(binaryDictionary, "x", "y")); binaryDictionary.close(); binaryDictionary.close(); Loading @@ -266,36 +269,32 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); forcePassingShortTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); assertFalse(binaryDictionary.isValidWord("a")); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("a")); assertTrue(binaryDictionary.isValidWord("a")); forcePassingShortTime(binaryDictionary); forcePassingShortTime(binaryDictionary); assertTrue(binaryDictionary.isValidWord("a")); assertTrue(binaryDictionary.isValidWord("a")); forcePassingLongTime(binaryDictionary); forcePassingLongTime(binaryDictionary); assertFalse(binaryDictionary.isValidWord("a")); assertFalse(binaryDictionary.isValidWord("a")); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); assertTrue(isValidBigram(binaryDictionary, "a", "b")); assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); forcePassingShortTime(binaryDictionary); assertFalse(isValidBigram(binaryDictionary, "a", "b")); assertFalse(isValidBigram(binaryDictionary, "a", "b")); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "a", true /* isValidWord */); addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "b", true /* isValidWord */, "a"); addUnigramWord(binaryDictionary, "a", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "b", DUMMY_PROBABILITY); addBigramWords(binaryDictionary, "a", "b", DUMMY_PROBABILITY); assertTrue(isValidBigram(binaryDictionary, "a", "b")); assertTrue(isValidBigram(binaryDictionary, "a", "b")); forcePassingShortTime(binaryDictionary); forcePassingShortTime(binaryDictionary); assertTrue(isValidBigram(binaryDictionary, "a", "b")); assertTrue(isValidBigram(binaryDictionary, "a", "b")); Loading Loading @@ -333,7 +332,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY)); binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_UNIGRAM_COUNT_QUERY)); for (int i = 0; i < unigramTypedCount; i++) { for (int i = 0; i < unigramTypedCount; i++) { final String word = words.get(random.nextInt(words.size())); final String word = words.get(random.nextInt(words.size())); addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); onInputWord(binaryDictionary, word, true /* isValidWord */); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int unigramCountBeforeGC = final int unigramCountBeforeGC = Loading Loading @@ -380,10 +379,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String strong = "strong"; final String strong = "strong"; final String weak = "weak"; final String weak = "weak"; for (int j = 0; j < strongUnigramTypedCount; j++) { for (int j = 0; j < strongUnigramTypedCount; j++) { addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY); onInputWord(binaryDictionary, strong, true /* isValidWord */); } } for (int j = 0; j < weakUnigramTypedCount; j++) { for (int j = 0; j < weakUnigramTypedCount; j++) { addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY); onInputWord(binaryDictionary, weak, true /* isValidWord */); } } assertTrue(binaryDictionary.isValidWord(strong)); assertTrue(binaryDictionary.isValidWord(strong)); assertTrue(binaryDictionary.isValidWord(weak)); assertTrue(binaryDictionary.isValidWord(weak)); Loading @@ -391,7 +390,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { for (int i = 0; i < unigramCount; i++) { for (int i = 0; i < unigramCount; i++) { final String word = CodePointUtils.generateWord(random, codePointSet); final String word = CodePointUtils.generateWord(random, codePointSet); for (int j = 0; j < eachUnigramTypedCount; j++) { for (int j = 0; j < eachUnigramTypedCount; j++) { addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); onInputWord(binaryDictionary, word, true /* isValidWord */); } } if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int unigramCountBeforeGC = final int unigramCountBeforeGC = Loading Loading @@ -450,12 +449,13 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { } } final int maxBigramCount = Integer.parseInt( final int maxBigramCount = Integer.parseInt( binaryDictionary.getPropertyForGettingStats(BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); binaryDictionary.getPropertyForGettingStats( BinaryDictionary.MAX_BIGRAM_COUNT_QUERY)); for (int i = 0; i < bigramTypedCount; ++i) { for (int i = 0; i < bigramTypedCount; ++i) { final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size())); final Pair<String, String> bigram = bigrams.get(random.nextInt(bigrams.size())); addUnigramWord(binaryDictionary, bigram.first, DUMMY_PROBABILITY); onInputWord(binaryDictionary, bigram.first, true /* isValidWord */); addUnigramWord(binaryDictionary, bigram.second, DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, bigram.second, true /* isValidWord */, addBigramWords(binaryDictionary, bigram.first, bigram.second, DUMMY_PROBABILITY); bigram.first); if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int bigramCountBeforeGC = final int bigramCountBeforeGC = Loading Loading @@ -506,23 +506,22 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String word = CodePointUtils.generateWord(random, codePointSet); final String word = CodePointUtils.generateWord(random, codePointSet); words.add(word); words.add(word); for (int j = 0; j < unigramTypedCount; j++) { for (int j = 0; j < unigramTypedCount; j++) { addUnigramWord(binaryDictionary, word, DUMMY_PROBABILITY); onInputWord(binaryDictionary, word, true /* isValidWord */); } } } } final String strong = "strong"; final String strong = "strong"; final String weak = "weak"; final String weak = "weak"; final String target = "target"; final String target = "target"; for (int j = 0; j < unigramTypedCount; j++) { for (int j = 0; j < unigramTypedCount; j++) { addUnigramWord(binaryDictionary, strong, DUMMY_PROBABILITY); onInputWord(binaryDictionary, weak, true /* isValidWord */); addUnigramWord(binaryDictionary, weak, DUMMY_PROBABILITY); onInputWord(binaryDictionary, strong, true /* isValidWord */); addUnigramWord(binaryDictionary, target, DUMMY_PROBABILITY); } } binaryDictionary.flushWithGC(); binaryDictionary.flushWithGC(); for (int j = 0; j < strongBigramTypedCount; j++) { for (int j = 0; j < strongBigramTypedCount; j++) { addBigramWords(binaryDictionary, strong, target, DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, strong); } } for (int j = 0; j < weakBigramTypedCount; j++) { for (int j = 0; j < weakBigramTypedCount; j++) { addBigramWords(binaryDictionary, weak, target, DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, target, true /* isValidWord */, weak); } } assertTrue(isValidBigram(binaryDictionary, strong, target)); assertTrue(isValidBigram(binaryDictionary, strong, target)); assertTrue(isValidBigram(binaryDictionary, weak, target)); assertTrue(isValidBigram(binaryDictionary, weak, target)); Loading @@ -535,7 +534,7 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final String word1 = words.get(word1Index); final String word1 = words.get(word1Index); for (int j = 0; j < eachBigramTypedCount; j++) { for (int j = 0; j < eachBigramTypedCount; j++) { addBigramWords(binaryDictionary, word0, word1, DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, word1, true /* isValidWord */, word0); } } if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { if (binaryDictionary.needsToRunGC(true /* mindsBlockByGC */)) { final int bigramCountBeforeGC = final int bigramCountBeforeGC = Loading Loading @@ -563,19 +562,18 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { setCurrentTimeForTestMode(mCurrentTime); setCurrentTimeForTestMode(mCurrentTime); final File dictFile = createEmptyDictionaryAndGetFile(fromFormatVersion); final File dictFile = createEmptyDictionaryAndGetFile(fromFormatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "aaa", true /* isValidWord */); assertTrue(binaryDictionary.isValidWord("aaa")); assertTrue(binaryDictionary.isValidWord("aaa")); addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "ccc", true /* isValidWord */); assertFalse(binaryDictionary.isValidWord("bbb")); onInputWord(binaryDictionary, "ccc", true /* isValidWord */); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "ccc", true /* isValidWord */); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "ccc", true /* isValidWord */); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); onInputWord(binaryDictionary, "ccc", true /* isValidWord */); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); addUnigramWord(binaryDictionary, "ccc", DUMMY_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "abc", true /* isValidWord */, "aaa"); addUnigramWord(binaryDictionary, "abc", DUMMY_PROBABILITY); addBigramWords(binaryDictionary, "aaa", "abc", DUMMY_PROBABILITY); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa"); assertFalse(binaryDictionary.isValidWord("bbb")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); assertEquals(fromFormatVersion, binaryDictionary.getFormatVersion()); Loading @@ -585,11 +583,11 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { assertTrue(binaryDictionary.isValidWord("aaa")); assertTrue(binaryDictionary.isValidWord("aaa")); assertFalse(binaryDictionary.isValidWord("bbb")); assertFalse(binaryDictionary.isValidWord("bbb")); assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc")); assertTrue(binaryDictionary.getFrequency("aaa") < binaryDictionary.getFrequency("ccc")); addUnigramWord(binaryDictionary, "bbb", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "bbb", false /* isValidWord */); assertTrue(binaryDictionary.isValidWord("bbb")); assertTrue(binaryDictionary.isValidWord("bbb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); assertTrue(isValidBigram(binaryDictionary, "aaa", "abc")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); assertFalse(isValidBigram(binaryDictionary, "aaa", "bbb")); addBigramWords(binaryDictionary, "aaa", "bbb", Dictionary.NOT_A_PROBABILITY); onInputWordWithPrevWord(binaryDictionary, "bbb", false /* isValidWord */, "aaa"); assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); assertTrue(isValidBigram(binaryDictionary, "aaa", "bbb")); binaryDictionary.close(); binaryDictionary.close(); } } Loading @@ -612,28 +610,19 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isBlacklisted */, true /* isBeginningOfSentence */, true /* isNotAWord */, false /* isBlacklisted */, mCurrentTime); mCurrentTime); final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE; final NgramContext beginningOfSentenceContext = NgramContext.BEGINNING_OF_SENTENCE; addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY, mCurrentTime); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY, onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); mCurrentTime); onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); binaryDictionary.addNgramEntry(beginningOfSentenceContext, "bbb", DUMMY_PROBABILITY, mCurrentTime); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); forcePassingLongTime(binaryDictionary); forcePassingLongTime(binaryDictionary); assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); assertFalse(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); addUnigramWord(binaryDictionary, "aaa", DUMMY_PROBABILITY); onInputWordWithBeginningOfSentenceContext(binaryDictionary, "aaa", true /* isValidWord */); binaryDictionary.addNgramEntry(beginningOfSentenceContext, "aaa", DUMMY_PROBABILITY, onInputWordWithBeginningOfSentenceContext(binaryDictionary, "bbb", true /* isValidWord */); mCurrentTime); addUnigramWord(binaryDictionary, "bbb", DUMMY_PROBABILITY); binaryDictionary.addNgramEntry(beginningOfSentenceContext, "bbb", DUMMY_PROBABILITY, mCurrentTime); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "aaa")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); assertTrue(binaryDictionary.isValidNgram(beginningOfSentenceContext, "bbb")); binaryDictionary.close(); binaryDictionary.close(); Loading @@ -651,10 +640,10 @@ public class BinaryDictionaryDecayingTests extends AndroidTestCase { final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final File dictFile = createEmptyDictionaryAndGetFile(formatVersion); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); final BinaryDictionary binaryDictionary = getBinaryDictionary(dictFile); addUnigramWord(binaryDictionary, "aaa", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "aaa", false /* isValidWord */); assertFalse(binaryDictionary.isValidWord("aaa")); assertFalse(binaryDictionary.isValidWord("aaa")); for (int i = 0; i < unigramInputCount; i++) { for (int i = 0; i < unigramInputCount; i++) { addUnigramWord(binaryDictionary, "aaa", Dictionary.NOT_A_PROBABILITY); onInputWord(binaryDictionary, "aaa", false /* isValidWord */); } } assertTrue(binaryDictionary.isValidWord("aaa")); assertTrue(binaryDictionary.isValidWord("aaa")); assertTrue(binaryDictionary.removeUnigramEntry("aaa")); assertTrue(binaryDictionary.removeUnigramEntry("aaa")); Loading