Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e59f3e4f authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi
Browse files

Use DictionaryFacilitatorLruCache for personalization. 

Bug: 16547557

Change-Id: I5faba5e26d072b49c0fffcaeaf5062f9e0c2dcc0
parent f9577035

java/src/com/android/inputmethod/latin/DictionaryFacilitator.java

+17 −3
Original line number Diff line number Diff line
@@ -33,6 +33,7 @@ import com.android.inputmethod.latin.personalization.UserHistoryDictionary; 
import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion;

import com.android.inputmethod.latin.settings.SpacingAndPunctuations;

import com.android.inputmethod.latin.utils.DistracterFilter;

import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatchesAndSuggestions;

import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary;

import com.android.inputmethod.latin.utils.ExecutorUtils;

import com.android.inputmethod.latin.utils.LanguageModelParam;
@@ -59,6 +60,7 @@ public class DictionaryFacilitator { 
    // HACK: This threshold is being used when adding a capitalized entry in the User History

    // dictionary.

    private static final int CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT = 140;

    private static final int MAX_DICTIONARY_FACILITATOR_CACHE_SIZE = 3;



    private Dictionaries mDictionaries = new Dictionaries();

    private boolean mIsUserDictEnabled = false;
@@ -66,6 +68,7 @@ public class DictionaryFacilitator { 
    // To synchronize assigning mDictionaries to ensure closing dictionaries.

    private final Object mLock = new Object();

    private final DistracterFilter mDistracterFilter;

    private final DictionaryFacilitatorLruCache mFacilitatorCacheForPersonalization;



    private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS =

            new String[] {
@@ -173,10 +176,14 @@ public class DictionaryFacilitator { 


    public DictionaryFacilitator() {

        mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER;

        mFacilitatorCacheForPersonalization = null;

    }



    public DictionaryFacilitator(final DistracterFilter distracterFilter) {

        mDistracterFilter = distracterFilter;

    public DictionaryFacilitator(final Context context) {

        mFacilitatorCacheForPersonalization = new DictionaryFacilitatorLruCache(context,

                MAX_DICTIONARY_FACILITATOR_CACHE_SIZE, "" /* dictionaryNamePrefix */);

        mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context,

                mFacilitatorCacheForPersonalization);

    }



    public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
@@ -351,6 +358,9 @@ public class DictionaryFacilitator { 
        for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) {

            dictionaries.closeDict(dictType);

        }

        if (mFacilitatorCacheForPersonalization != null) {

            mFacilitatorCacheForPersonalization.evictAll();

        }

        mDistracterFilter.close();

    }
@@ -597,11 +607,15 @@ public class DictionaryFacilitator { 
            }

            return;

        }

        // TODO: Get locale from personalizationDataChunk.mDetectedLanguage.

        final Locale dataChunkLocale = getLocale();

        final DictionaryFacilitator dictionaryFacilitatorForLocale =

                mFacilitatorCacheForPersonalization.get(dataChunkLocale);

        final ArrayList<LanguageModelParam> languageModelParams =

                LanguageModelParam.createLanguageModelParamsFrom(

                        personalizationDataChunk.mTokens,

                        personalizationDataChunk.mTimestampInSeconds,

                        this /* dictionaryFacilitator */, spacingAndPunctuations,

                        dictionaryFacilitatorForLocale, spacingAndPunctuations,

                        new DistracterFilterCheckingIsInDictionary(

                                mDistracterFilter, personalizationDict));

        if (languageModelParams == null || languageModelParams.isEmpty()) {

java/src/com/android/inputmethod/latin/LatinIME.java

+1 −2
Original line number Diff line number Diff line
@@ -130,8 +130,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen 


    private final Settings mSettings;

    private final DictionaryFacilitator mDictionaryFacilitator =

            new DictionaryFacilitator(

                    new DistracterFilterCheckingExactMatchesAndSuggestions(this /* context */));

            new DictionaryFacilitator(this /* context */);

    // TODO: Move from LatinIME.

    private final PersonalizationDictionaryUpdater mPersonalizationDictionaryUpdater =

            new PersonalizationDictionaryUpdater(this /* context */, mDictionaryFacilitator);

java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingExactMatchesAndSuggestions.java

+29 −33
Original line number Diff line number Diff line
@@ -20,7 +20,6 @@ import java.util.HashMap; 
import java.util.List;

import java.util.Locale;

import java.util.Map;

import java.util.concurrent.TimeUnit;



import android.content.Context;

import android.content.res.Resources;
@@ -34,6 +33,7 @@ import com.android.inputmethod.keyboard.Keyboard; 
import com.android.inputmethod.keyboard.KeyboardId;

import com.android.inputmethod.keyboard.KeyboardLayoutSet;

import com.android.inputmethod.latin.DictionaryFacilitator;

import com.android.inputmethod.latin.DictionaryFacilitatorLruCache;

import com.android.inputmethod.latin.PrevWordsInfo;

import com.android.inputmethod.latin.RichInputMethodSubtype;

import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
@@ -49,14 +49,15 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
            DistracterFilterCheckingExactMatchesAndSuggestions.class.getSimpleName();

    private static final boolean DEBUG = false;



    private static final long TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS = 120;

    private static final int MAX_DISTRACTERS_CACHE_SIZE = 512;



    private final Context mContext;

    private final Map<Locale, InputMethodSubtype> mLocaleToSubtypeMap;

    private final Map<Locale, Keyboard> mLocaleToKeyboardMap;

    private final DictionaryFacilitator mDictionaryFacilitator;

    private final DictionaryFacilitatorLruCache mDictionaryFacilitatorLruCache;

    private final LruCache<String, Boolean> mDistractersCache;

    // TODO: Remove and support multiple locales at the same time.

    private Locale mCurrentLocale;

    private Keyboard mKeyboard;

    private final Object mLock = new Object();
@@ -71,19 +72,26 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
     * Create a DistracterFilter instance.

     *

     * @param context the context.

     * @param dictionaryFacilitatorLruCache the cache of dictionaryFacilitators that are used for

     * checking distracters.

     */

    public DistracterFilterCheckingExactMatchesAndSuggestions(final Context context) {

    public DistracterFilterCheckingExactMatchesAndSuggestions(final Context context,

            final DictionaryFacilitatorLruCache dictionaryFacilitatorLruCache) {

        mContext = context;

        mLocaleToSubtypeMap = new HashMap<>();

        mLocaleToKeyboardMap = new HashMap<>();

        mDictionaryFacilitator = new DictionaryFacilitator();

        mDictionaryFacilitatorLruCache = dictionaryFacilitatorLruCache;

        mDistractersCache = new LruCache<>(MAX_DISTRACTERS_CACHE_SIZE);

        mCurrentLocale = null;

        mKeyboard = null;

    }



    @Override

    public void close() {

        mDictionaryFacilitator.closeDictionaries();

        mLocaleToKeyboardMap.clear();

        mDistractersCache.evictAll();

        mCurrentLocale = null;

        mKeyboard = null;

    }



    @Override
@@ -138,14 +146,6 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
        mKeyboard = layoutSet.getKeyboard(KeyboardId.ELEMENT_ALPHABET);

    }



    private void loadDictionariesForLocale(final Locale newlocale) throws InterruptedException {

        mDictionaryFacilitator.resetDictionaries(mContext, newlocale,

                false /* useContactsDict */, false /* usePersonalizedDicts */,

                false /* forceReloadMainDictionary */, null /* listener */);

        mDictionaryFacilitator.waitForLoadingMainDictionary(

                TIMEOUT_TO_WAIT_LOADING_DICTIONARIES_IN_SECONDS, TimeUnit.SECONDS);

    }



    /**

     * Determine whether a word is a distracter to words in dictionaries.

     *
@@ -161,26 +161,20 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
        if (locale == null) {

            return false;

        }

        if (!locale.equals(mDictionaryFacilitator.getLocale())) {

        if (!locale.equals(mCurrentLocale)) {

            synchronized (mLock) {

                if (!mLocaleToSubtypeMap.containsKey(locale)) {

                    Log.e(TAG, "Locale " + locale + " is not enabled.");

                    // TODO: Investigate what we should do for disabled locales.

                    return false;

                }

                mCurrentLocale = locale;

                loadKeyboardForLocale(locale);

                // Reset dictionaries for the locale.

                try {

                mDistractersCache.evictAll();

                    loadDictionariesForLocale(locale);

                } catch (final InterruptedException e) {

                    Log.e(TAG, "Interrupted while waiting for loading dicts in DistracterFilter",

                            e);

                    return false;

            }

        }

        }



        final DictionaryFacilitator dictionaryFacilitator =

                mDictionaryFacilitatorLruCache.get(locale);

        if (DEBUG) {

            Log.d(TAG, "testedWord: " + testedWord);

        }
@@ -193,13 +187,13 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
        }



        final boolean isDistracterCheckedByGetMaxFreqencyOfExactMatches =

                checkDistracterUsingMaxFreqencyOfExactMatches(testedWord);

                checkDistracterUsingMaxFreqencyOfExactMatches(dictionaryFacilitator, testedWord);

        if (isDistracterCheckedByGetMaxFreqencyOfExactMatches) {

            // Add the word to the cache.

            mDistractersCache.put(testedWord, Boolean.TRUE);

            return true;

        }

        final boolean isValidWord = mDictionaryFacilitator.isValidWord(testedWord,

        final boolean isValidWord = dictionaryFacilitator.isValidWord(testedWord,

                false /* ignoreCase */);

        if (isValidWord) {

            // Valid word is not a distractor.
@@ -210,7 +204,7 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
        }



        final boolean isDistracterCheckedByGetSuggestion =

                checkDistracterUsingGetSuggestions(testedWord);

                checkDistracterUsingGetSuggestions(dictionaryFacilitator, testedWord);

        if (isDistracterCheckedByGetSuggestion) {

            // Add the word to the cache.

            mDistractersCache.put(testedWord, Boolean.TRUE);
@@ -219,11 +213,12 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
        return false;

    }



    private boolean checkDistracterUsingMaxFreqencyOfExactMatches(final String testedWord) {

    private static boolean checkDistracterUsingMaxFreqencyOfExactMatches(

            final DictionaryFacilitator dictionaryFacilitator, final String testedWord) {

        // The tested word is a distracter when there is a word that is exact matched to the tested

        // word and its probability is higher than the tested word's probability.

        final int perfectMatchFreq = mDictionaryFacilitator.getFrequency(testedWord);

        final int exactMatchFreq = mDictionaryFacilitator.getMaxFrequencyOfExactMatches(testedWord);

        final int perfectMatchFreq = dictionaryFacilitator.getFrequency(testedWord);

        final int exactMatchFreq = dictionaryFacilitator.getMaxFrequencyOfExactMatches(testedWord);

        final boolean isDistracter = perfectMatchFreq < exactMatchFreq;

        if (DEBUG) {

            Log.d(TAG, "perfectMatchFreq: " + perfectMatchFreq);
@@ -233,7 +228,8 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
        return isDistracter;

    }



    private boolean checkDistracterUsingGetSuggestions(final String testedWord) {

    private boolean checkDistracterUsingGetSuggestions(

            final DictionaryFacilitator dictionaryFacilitator, final String testedWord) {

        if (mKeyboard == null) {

            return false;

        }
@@ -251,7 +247,7 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
        synchronized (mLock) {

            final int[] coordinates = mKeyboard.getCoordinates(codePoints);

            composer.setComposingWord(codePoints, coordinates);

            final SuggestionResults suggestionResults = mDictionaryFacilitator.getSuggestionResults(

            final SuggestionResults suggestionResults = dictionaryFacilitator.getSuggestionResults(

                    composer, PrevWordsInfo.EMPTY_PREV_WORDS_INFO, mKeyboard.getProximityInfo(),

                    settingsValuesForSuggestion, 0 /* sessionId */);

            if (suggestionResults.isEmpty()) {

tests/src/com/android/inputmethod/latin/DistracterFilterTest.java

+10 −1
Original line number Diff line number Diff line
@@ -31,13 +31,17 @@ import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatchesA 
 */

@LargeTest

public class DistracterFilterTest extends AndroidTestCase {

    private DictionaryFacilitatorLruCache mDictionaryFacilitatorLruCache;

    private DistracterFilterCheckingExactMatchesAndSuggestions mDistracterFilter;



    @Override

    protected void setUp() throws Exception {

        super.setUp();

        final Context context = getContext();

        mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context);

        mDictionaryFacilitatorLruCache = new DictionaryFacilitatorLruCache(context,

                2 /* maxSize */, "" /* dictionaryNamePrefix */);

        mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context,

                mDictionaryFacilitatorLruCache);

        RichInputMethodManager.init(context);

        final RichInputMethodManager richImm = RichInputMethodManager.getInstance();

        final ArrayList<InputMethodSubtype> subtypes = new ArrayList<>();
@@ -50,6 +54,11 @@ public class DistracterFilterTest extends AndroidTestCase { 
        mDistracterFilter.updateEnabledSubtypes(subtypes);

    }



    @Override

    protected void tearDown() {

        mDictionaryFacilitatorLruCache.evictAll();

    }



    public void testIsDistractorToWordsInDictionaries() {

        final PrevWordsInfo EMPTY_PREV_WORDS_INFO = PrevWordsInfo.EMPTY_PREV_WORDS_INFO;