Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 166d8c22 authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi
Browse files

Make Distracter filter use getMaxFrequencyOfExactMatches().

Bug: 13142176
Bug: 15428247

Change-Id: I5c23fbea2851f891f76f19d9da2cb70ae964569b
parent d9b8602f
Loading
Loading
Loading
Loading
+11 −3
Original line number Diff line number Diff line
@@ -89,9 +89,17 @@ public final class DictionaryCollection extends Dictionary {
        int maxFreq = -1;
        for (int i = mDictionaries.size() - 1; i >= 0; --i) {
            final int tempFreq = mDictionaries.get(i).getFrequency(word);
            if (tempFreq >= maxFreq) {
                maxFreq = tempFreq;
            maxFreq = Math.max(tempFreq, maxFreq);
        }
        return maxFreq;
    }

    @Override
    public int getMaxFrequencyOfExactMatches(final String word) {
        int maxFreq = -1;
        for (int i = mDictionaries.size() - 1; i >= 0; --i) {
            final int tempFreq = mDictionaries.get(i).getMaxFrequencyOfExactMatches(word);
            maxFreq = Math.max(tempFreq, maxFreq);
        }
        return maxFreq;
    }
+12 −143
Original line number Diff line number Diff line
@@ -16,33 +16,22 @@

package com.android.inputmethod.latin.utils;

import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import android.content.Context;
import android.content.res.Resources;
import android.text.InputType;
import android.util.Log;
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InputMethodSubtype;

import com.android.inputmethod.keyboard.Keyboard;
import com.android.inputmethod.keyboard.KeyboardId;
import com.android.inputmethod.keyboard.KeyboardLayoutSet;
import com.android.inputmethod.latin.Constants;
import com.android.inputmethod.latin.Dictionary;
import com.android.inputmethod.latin.DictionaryFacilitator;
import com.android.inputmethod.latin.PrevWordsInfo;
import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
import com.android.inputmethod.latin.WordComposer;

/**
 * This class is used to prevent distracters being added to personalization
 * or user history dictionaries
 */
// TODO: Rename.
public class DistracterFilterUsingSuggestion implements DistracterFilter {
    private static final String TAG = DistracterFilterUsingSuggestion.class.getSimpleName();
    private static final boolean DEBUG = false;
@@ -50,10 +39,7 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
    private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120;

    private final Context mContext;
    private final Map<Locale, InputMethodSubtype> mLocaleToSubtypeMap;
    private final Map<Locale, Keyboard> mLocaleToKeyboardMap;
    private final DictionaryFacilitator mDictionaryFacilitator;
    private Keyboard mKeyboard;
    private final Object mLock = new Object();

    /**
@@ -63,10 +49,7 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
     */
    public DistracterFilterUsingSuggestion(final Context context) {
        mContext = context;
        mLocaleToSubtypeMap = new HashMap<>();
        mLocaleToKeyboardMap = new HashMap<>();
        mDictionaryFacilitator = new DictionaryFacilitator();
        mKeyboard = null;
    }

    @Override
@@ -76,94 +59,6 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {

    @Override
    public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
        final Map<Locale, InputMethodSubtype> newLocaleToSubtypeMap = new HashMap<>();
        if (enabledSubtypes != null) {
            for (final InputMethodSubtype subtype : enabledSubtypes) {
                final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype);
                if (newLocaleToSubtypeMap.containsKey(locale)) {
                    // Multiple subtypes are enabled for one locale.
                    // TODO: Investigate what we should do for this case.
                    continue;
                }
                newLocaleToSubtypeMap.put(locale, subtype);
            }
        }
        if (mLocaleToSubtypeMap.equals(newLocaleToSubtypeMap)) {
            // Enabled subtypes have not been changed.
            return;
        }
        synchronized (mLock) {
            mLocaleToSubtypeMap.clear();
            mLocaleToSubtypeMap.putAll(newLocaleToSubtypeMap);
            mLocaleToKeyboardMap.clear();
        }
    }

    private boolean isDistracter(
            final SuggestionResults suggestionResults, final String consideredWord) {
        int perfectMatchProbability = Dictionary.NOT_A_PROBABILITY;
        for (final SuggestedWordInfo suggestedWordInfo : suggestionResults) {
            if (suggestedWordInfo.mWord.equals(consideredWord)) {
                perfectMatchProbability = mDictionaryFacilitator.getFrequency(consideredWord);
                continue;
            }
            // Exact match can include case errors, accent errors, digraph conversions.
            final boolean isExactMatch = suggestedWordInfo.isExactMatch();
            final boolean isExactMatchWithIntentionalOmission =
                    suggestedWordInfo.isExactMatchWithIntentionalOmission();

            if (DEBUG) {
                final float normalizedScore = BinaryDictionaryUtils.calcNormalizedScore(
                        consideredWord, suggestedWordInfo.mWord, suggestedWordInfo.mScore);
                Log.d(TAG, "consideredWord: " +  consideredWord);
                Log.d(TAG, "top suggestion: " +  suggestedWordInfo.mWord);
                Log.d(TAG, "suggestionScore: " +  suggestedWordInfo.mScore);
                Log.d(TAG, "normalizedScore: " +  normalizedScore);
                Log.d(TAG, "isExactMatch: " + isExactMatch);
                Log.d(TAG, "isExactMatchWithIntentionalOmission: "
                            + isExactMatchWithIntentionalOmission);
            }
            if (perfectMatchProbability != Dictionary.NOT_A_PROBABILITY) {
                final int topNonPerfectProbability = mDictionaryFacilitator.getFrequency(
                        suggestedWordInfo.mWord);
                if (DEBUG) {
                    Log.d(TAG, "perfectMatchProbability: " + perfectMatchProbability);
                    Log.d(TAG, "topNonPerfectProbability: " + topNonPerfectProbability);
                }
                if (perfectMatchProbability > topNonPerfectProbability) {
                    return false;
                }
            }
            return isExactMatch || isExactMatchWithIntentionalOmission;
        }
        return false;
    }

    private void loadKeyboardForLocale(final Locale newLocale) {
        final Keyboard cachedKeyboard = mLocaleToKeyboardMap.get(newLocale);
        if (cachedKeyboard != null) {
            mKeyboard = cachedKeyboard;
            return;
        }
        final InputMethodSubtype subtype;
        synchronized (mLock) {
            subtype = mLocaleToSubtypeMap.get(newLocale);
        }
        if (subtype == null) {
            return;
        }
        final EditorInfo editorInfo = new EditorInfo();
        editorInfo.inputType = InputType.TYPE_CLASS_TEXT;
        final KeyboardLayoutSet.Builder builder = new KeyboardLayoutSet.Builder(
                mContext, editorInfo);
        final Resources res = mContext.getResources();
        final int keyboardWidth = ResourceUtils.getDefaultKeyboardWidth(res);
        final int keyboardHeight = ResourceUtils.getDefaultKeyboardHeight(res);
        builder.setKeyboardGeometry(keyboardWidth, keyboardHeight);
        builder.setSubtype(subtype);
        builder.setIsSpellChecker(false /* isSpellChecker */);
        final KeyboardLayoutSet layoutSet = builder.build();
        mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET);
    }

    private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException {
@@ -191,12 +86,6 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
        }
        if (!locale.equals(mDictionaryFacilitator.getLocale())) {
            synchronized (mLock) {
                if (!mLocaleToSubtypeMap.containsKey(locale)) {
                    Log.e(TAG, "Locale " + locale + " is not enabled.");
                    // TODO: Investigate what we should do for disabled locales.
                    return false;
                }
                loadKeyboardForLocale(locale);
                // Reset dictionaries for the locale.
                try {
                    loadDictionariesForLocale(locale);
@@ -207,37 +96,17 @@ public class DistracterFilterUsingSuggestion implements DistracterFilter {
                }
            }
        }
        if (mKeyboard == null) {
            return false;
        }
        final WordComposer composer = new WordComposer();
        final int[] codePoints = StringUtils.toCodePointArray(testedWord);
        final int[] coordinates = mKeyboard.getCoordinates(codePoints);
        composer.setComposingWord(codePoints, coordinates, PrevWordsInfo.EMPTY_PREV_WORDS_INFO);

        final int trailingSingleQuotesCount = StringUtils.getTrailingSingleQuotesCount(testedWord);
        final String consideredWord = trailingSingleQuotesCount > 0 ?
                testedWord.substring(0, testedWord.length() - trailingSingleQuotesCount) :
                testedWord;
        final AsyncResultHolder<Boolean> holder = new AsyncResultHolder<>();
        ExecutorUtils.getExecutor("check distracters").execute(new Runnable() {
            @Override
            public void run() {
                final SuggestionResults suggestionResults =
                        mDictionaryFacilitator.getSuggestionResults(
                                composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO,
                                mKeyboard.getProximityInfo(), true /* blockOffensiveWords */,
                                null /* additionalFeaturesOptions */, 0 /* sessionId */,
                                null /* rawSuggestions */);
                if (suggestionResults.isEmpty()) {
                    holder.set(false);
                    return;
                }
                holder.set(isDistracter(suggestionResults, consideredWord));
        // The tested word is a distracter when there is a word that is exact matched to the tested
        // word and its probability is higher than the tested word's probability.
        final int perfectMatchFreq = mDictionaryFacilitator.getFrequency(testedWord);
        final int exactMatchFreq = mDictionaryFacilitator.getMaxFrequencyOfExactMatches(testedWord);
        final boolean isDistracter = perfectMatchFreq < exactMatchFreq;
        if (DEBUG) {
            Log.d(TAG, "testedWord: " + testedWord);
            Log.d(TAG, "perfectMatchFreq: " + perfectMatchFreq);
            Log.d(TAG, "exactMatchFreq: " + exactMatchFreq);
            Log.d(TAG, "isDistracter: " + isDistracter);
        }
        });
        // It's OK to block the distracter filtering, but the dictionary lookup should be done
        // sequentially using ExecutorUtils.
        return holder.get(false /* defaultValue */, Constants.GET_SUGGESTED_WORDS_TIMEOUT);
        return isDistracter;
    }
}
+54 −11
Original line number Diff line number Diff line
@@ -42,11 +42,6 @@ public class DistracterFilterTest extends InputTestsBase {
        final Locale localeEnUs = new Locale("en", "US");
        String typedWord;

        typedWord = "google";
        // For this test case, we consider "google" is a distracter to "Google".
        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));

        typedWord = "Bill";
        // For this test case, we consider "Bill" is a distracter to "bill".
        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
@@ -83,15 +78,20 @@ public class DistracterFilterTest extends InputTestsBase {
                EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));

        typedWord = "cafe";
        // For this test case, we consider "café" is not a distracter to any word in dictionaries.
        // For this test case, we consider "cafe" is not a distracter to any word in dictionaries.
        assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));

        typedWord = "ill";
        // For this test case, we consider "ill" is not a distracter to any word in dictionaries.
        typedWord = "I'll";
        // For this test case, we consider "I'll" is not a distracter to any word in dictionaries.
        assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));

        typedWord = "ill";
        // For this test case, we consider "ill" is a distracter to "I'll"
        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));

        typedWord = "asdfd";
        // For this test case, we consider "asdfd" is not a distracter to any word in dictionaries.
        assertFalse(
@@ -101,8 +101,51 @@ public class DistracterFilterTest extends InputTestsBase {
        typedWord = "thank";
        // For this test case, we consider "thank" is not a distracter to any other word
        // in dictionaries.
        assertFalse(
                mDistracterFilter.isDistracterToWordsInDictionaries(
        assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeEnUs));

        final Locale localeDeDe = new Locale("de", "DE");

        typedWord = "fuer";
        // For this test case, we consider "fuer" is a distracter to "für".
        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));

        typedWord = "fUEr";
        // For this test case, we consider "fUEr" is a distracter to "für".
        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));

        typedWord = "fur";
        // For this test case, we consider "fur" is a distracter to "für".
        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeDeDe));

        final Locale localeFrFr = new Locale("fr", "FR");

        typedWord = "a";
        // For this test case, we consider "a" is a distracter to "à".
        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));

        typedWord = "à";
        // For this test case, we consider "à" is not a distracter to any word in dictionaries.
        assertFalse(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));

        typedWord = "etre";
        // For this test case, we consider "etre" is a distracter to "être".
        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));

        typedWord = "États-unis";
        // For this test case, we consider "États-unis" is a distracter to "États-Unis".
        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));

        typedWord = "ÉtatsUnis";
        // For this test case, we consider "ÉtatsUnis" is a distracter to "États-Unis".
        assertTrue(mDistracterFilter.isDistracterToWordsInDictionaries(
                EMPTY_PREV_WORDS_INFO, typedWord, localeFrFr));
    }
}