Make Distracter filter use getMaxFrequencyOfExactMatches(). (166d8c22) · Commits · e / os / android_packages_inputmethods_LatinIME

java/src/com/android/inputmethod/latin/DictionaryCollection.java

+11 −3

Original line number	Diff line number	Diff line
		@@ -89,9 +89,17 @@ public final class DictionaryCollection extends Dictionary {
		int maxFreq = -1;
		for (int i = mDictionaries.size() - 1; i >= 0; --i) {
		final int tempFreq = mDictionaries.get(i).getFrequency(word);
		if (tempFreq >= maxFreq) {
		maxFreq = tempFreq;
		maxFreq = Math.max(tempFreq, maxFreq);
		}
		return maxFreq;
		}

		@Override
		public int getMaxFrequencyOfExactMatches(final String word) {
		int maxFreq = -1;
		for (int i = mDictionaries.size() - 1; i >= 0; --i) {
		final int tempFreq = mDictionaries.get(i).getMaxFrequencyOfExactMatches(word);
		maxFreq = Math.max(tempFreq, maxFreq);
		}
		return maxFreq;
		}

java/src/com/android/inputmethod/latin/utils/DistracterFilterUsingSuggestion.java

+12 −143

Original line number	Diff line number	Diff line
		@@ -16,33 +16,22 @@

		package com.android.inputmethod.latin.utils;

		import java.util.HashMap;
		import java.util.List;
		import java.util.Locale;
		import java.util.Map;
		import java.util.concurrent.TimeUnit;

		import android.content.Context;
		import android.content.res.Resources;
		import android.text.InputType;
		import android.util.Log;
		import android.view.inputmethod.EditorInfo;
		import android.view.inputmethod.InputMethodSubtype;

		import com.android.inputmethod.keyboard.Keyboard;
		import com.android.inputmethod.keyboard.KeyboardId;
		import com.android.inputmethod.keyboard.KeyboardLayoutSet;
		import com.android.inputmethod.latin.Constants;
		import com.android.inputmethod.latin.Dictionary;
		import com.android.inputmethod.latin.DictionaryFacilitator;
		import com.android.inputmethod.latin.PrevWordsInfo;
		import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
		import com.android.inputmethod.latin.WordComposer;

		/**
		* This class is used to prevent distracters being added to personalization
		* or user history dictionaries
		*/
		// TODO: Rename.
		public class DistracterFilterUsingSuggestion implements DistracterFilter {
		private static final String TAG = DistracterFilterUsingSuggestion.class.getSimpleName();
		private static final boolean DEBUG = false;
		@@ -50,10 +39,7 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
		private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120;

		private final Context mContext;
		private final Map<Locale, InputMethodSubtype> mLocaleToSubtypeMap;
		private final Map<Locale, Keyboard> mLocaleToKeyboardMap;
		private final DictionaryFacilitator mDictionaryFacilitator;
		private Keyboard mKeyboard;
		private final Object mLock = new Object();

		/**
		@@ -63,10 +49,7 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
		*/
		public DistracterFilterUsingSuggestion(final Context context) {
		mContext = context;
		mLocaleToSubtypeMap = new HashMap<>();
		mLocaleToKeyboardMap = new HashMap<>();
		mDictionaryFacilitator = new DictionaryFacilitator();
		mKeyboard = null;
		}

		@Override
		@@ -76,94 +59,6 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {

		@Override
		public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
		final Map<Locale, InputMethodSubtype> newLocaleToSubtypeMap = new HashMap<>();
		if (enabledSubtypes != null) {
		for (final InputMethodSubtype subtype : enabledSubtypes) {
		final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype);
		if (newLocaleToSubtypeMap.containsKey(locale)) {
		// Multiple subtypes are enabled for one locale.
		// TODO: Investigate what we should do for this case.
		continue;
		}
		newLocaleToSubtypeMap.put(locale, subtype);
		}
		}
		if (mLocaleToSubtypeMap.equals(newLocaleToSubtypeMap)) {
		// Enabled subtypes have not been changed.
		return;
		}
		synchronized (mLock) {
		mLocaleToSubtypeMap.clear();
		mLocaleToSubtypeMap.putAll(newLocaleToSubtypeMap);
		mLocaleToKeyboardMap.clear();
		}
		}

		private boolean isDistracter(
		final SuggestionResults suggestionResults, final String consideredWord) {
		int perfectMatchProbability = Dictionary.NOT_A_PROBABILITY;
		for (final SuggestedWordInfo suggestedWordInfo : suggestionResults) {
		if (suggestedWordInfo.mWord.equals(consideredWord)) {
		perfectMatchProbability = mDictionaryFacilitator.getFrequency(consideredWord);
		continue;
		}
		// Exact match can include case errors, accent errors, digraph conversions.
		final boolean isExactMatch = suggestedWordInfo.isExactMatch();
		final boolean isExactMatchWithIntentionalOmission =
		suggestedWordInfo.isExactMatchWithIntentionalOmission();

		if (DEBUG) {
		final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore(
		consideredWord, suggestedWordInfo.mWord, suggestedWordInfo.mScore);
		Log.d(TAG, "consideredWord: " + consideredWord);
		Log.d(TAG, "top suggestion: " + suggestedWordInfo.mWord);
		Log.d(TAG, "suggestionScore: " + suggestedWordInfo.mScore);
		Log.d(TAG, "normalizedScore: " + normalizedScore);
		Log.d(TAG, "isExactMatch: " + isExactMatch);
		Log.d(TAG, "isExactMatchWithIntentionalOmission: "
		+ isExactMatchWithIntentionalOmission);
		}
		if (perfectMatchProbability != Dictionary.NOT_A_PROBABILITY) {
		final int topNonPerfectProbability = mDictionaryFacilitator.getFrequency(
		suggestedWordInfo.mWord);
		if (DEBUG) {
		Log.d(TAG, "perfectMatchProbability: " + perfectMatchProbability);
		Log.d(TAG, "topNonPerfectProbability: " + topNonPerfectProbability);
		}
		if (perfectMatchProbability > topNonPerfectProbability) {
		return false;
		}
		}
		return isExactMatch \|\| isExactMatchWithIntentionalOmission;
		}
		return false;
		}

		private void loadKeyboardForLocale(final Locale newLocale) {
		final Keyboard cachedKeyboard = mLocaleToKeyboardMap.get(newLocale);
		if (cachedKeyboard != null) {
		mKeyboard = cachedKeyboard;
		return;
		}
		final InputMethodSubtype subtype;
		synchronized (mLock) {
		subtype = mLocaleToSubtypeMap.get(newLocale);
		}
		if (subtype == null) {
		return;
		}
		final EditorInfo editorInfo = new EditorInfo();
		editorInfo.inputType = InputType.TYPE_CLASS_TEXT;
		final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder(
		mContext, editorInfo);
		final Resources res = mContext.getResources();
		final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res);
		final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res);
		builder.setKeyboardGeometry(keyboardWidth, keyboardHeight);
		builder.setSubtype(subtype);
		builder.setIsSpellChecker(false /* isSpellChecker */);
		final KeyboardLayoutSet layoutSet = builder.build();
		mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET);
		}

		private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException {
		@@ -191,12 +86,6 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
		}
		if (!locale.equals(mDictionaryFacilitator.getLocale())) {
		synchronized (mLock) {
		if (!mLocaleToSubtypeMap.containsKey(locale)) {
		Log.e(TAG, "Locale " + locale + " is not enabled.");
		// TODO: Investigate what we should do for disabled locales.
		return false;
		}
		loadKeyboardForLocale(locale);
		// Reset dictionaries for the locale.
		try {
		loadDictionariesForLocale(locale);
		@@ -207,37 +96,17 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
		}
		}
		}
		if (mKeyboard == null) {
		return false;
		}
		final WordComposer composer = new WordComposer();
		final int[] codePoints = StringUtils.toCodePointArray(testedWord);
		final int[] coordinates = mKeyboard.getCoordinates(codePoints);
		composer.setComposingWord(codePoints, coordinates, PrevWordsInfo.EMPTY_PREV_WORDS_INFO);

		final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord);
		final String consideredWord = trailingSingleQuotesCount > 0 ?
		testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) :
		testedWord;
		final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<>();
		ExecutorUtils.getExecutor("check distracters").execute(new Runnable() {
		@Override
		public void run() {
		final SuggestionResults suggestionResults =
		mDictionaryFacilitator.getSuggestionResults(
		composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO,
		mKeyboard.getProximityInfo(), true /* blockOffensiveWords */,
		null /* additionalFeaturesOptions /, 0 / sessionId */,
		null /* rawSuggestions */);
		if (suggestionResults.isEmpty()) {
		holder.set(false);
		return;
		}
		holder.set(isDistracter(suggestionResults, consideredWord));
		// The tested word is a distracter when there is a word that is exact matched to the tested
		// word and its probability is higher than the tested word's probability.
		final int perfectMatchFreq = mDictionaryFacilitator.getFrequency(testedWord);
		final int exactMatchFreq = mDictionaryFacilitator.getMaxFrequencyOfExactMatches(testedWord);
		final boolean isDistracter = perfectMatchFreq < exactMatchFreq;
		if (DEBUG) {
		Log.d(TAG, "testedWord: " + testedWord);
		Log.d(TAG, "perfectMatchFreq: " + perfectMatchFreq);
		Log.d(TAG, "exactMatchFreq: " + exactMatchFreq);
		Log.d(TAG, "isDistracter: " + isDistracter);
		}
		});
		// It's OK to block the distracter filtering, but the dictionary lookup should be done
		// sequentially using ExecutorUtils.
		return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT);
		return isDistracter;
		}
		}

tests/src/com/android/inputmethod/latin/DistracterFilterTest.java

+54 −11

Original line number	Diff line number	Diff line
		@@ -42,11 +42,6 @@ public class DistracterFilterTest extends InputTestsBase {
		final Locale localeEnUs = new Locale("en", "US");
		String typedWord;

		typedWord = "google";
		// For this test case, we consider "google" is a distracter to "Google".
		assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));

		typedWord = "Bill";
		// For this test case, we consider "Bill" is a distracter to "bill".
		assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
		@@ -83,15 +78,20 @@ public class DistracterFilterTest extends InputTestsBase {
		EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));

		typedWord = "cafe";
		// For this test case, we consider "café" is not a distracter to any word in dictionaries.
		// For this test case, we consider "cafe" is not a distracter to any word in dictionaries.
		assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));

		typedWord = "ill";
		// For this test case, we consider "ill" is not a distracter to any word in dictionaries.
		typedWord = "I'll";
		// For this test case, we consider "I'll" is not a distracter to any word in dictionaries.
		assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));

		typedWord = "ill";
		// For this test case, we consider "ill" is a distracter to "I'll"
		assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));

		typedWord = "asdfd";
		// For this test case, we consider "asdfd" is not a distracter to any word in dictionaries.
		assertFalse(
		@@ -101,8 +101,51 @@ public class DistracterFilterTest extends InputTestsBase {
		typedWord = "thank";
		// For this test case, we consider "thank" is not a distracter to any other word
		// in dictionaries.
		assertFalse(
		mDistracterFilter.isDistracterToWordsInDictionaries(
		assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));

		final Locale localeDeDe = new Locale("de", "DE");

		typedWord = "fuer";
		// For this test case, we consider "fuer" is a distracter to "für".
		assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));

		typedWord = "fUEr";
		// For this test case, we consider "fUEr" is a distracter to "für".
		assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));

		typedWord = "fur";
		// For this test case, we consider "fur" is a distracter to "für".
		assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));

		final Locale localeFrFr = new Locale("fr", "FR");

		typedWord = "a";
		// For this test case, we consider "a" is a distracter to "à".
		assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));

		typedWord = "à";
		// For this test case, we consider "à" is not a distracter to any word in dictionaries.
		assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));

		typedWord = "etre";
		// For this test case, we consider "etre" is a distracter to "être".
		assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));

		typedWord = "États-unis";
		// For this test case, we consider "États-unis" is a distracter to "États-Unis".
		assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));

		typedWord = "ÉtatsUnis";
		// For this test case, we consider "ÉtatsUnis" is a distracter to "États-Unis".
		assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
		EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
		}
		}