Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d8430811 authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi
Browse files

Move case and OOV detection logic into distracter filter. 

Bug: 16547557

Change-Id: I8502585976deb5e93fff3b1e0266654b8a927bda
parent 13864787

java/src/com/android/inputmethod/latin/DictionaryFacilitator.java

+3 −15
Original line number Diff line number Diff line
@@ -60,7 +60,6 @@ public class DictionaryFacilitator { 
    // HACK: This threshold is being used when adding a capitalized entry in the User History

    // dictionary.

    private static final int CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT = 140;

    private static final int MAX_DICTIONARY_FACILITATOR_CACHE_SIZE = 3;



    private DictionaryGroup mDictionaryGroup = new DictionaryGroup();

    private boolean mIsUserDictEnabled = false;
@@ -68,7 +67,6 @@ public class DictionaryFacilitator { 
    // To synchronize assigning mDictionaryGroup to ensure closing dictionaries.

    private final Object mLock = new Object();

    private final DistracterFilter mDistracterFilter;

    private final DictionaryFacilitatorLruCache mFacilitatorCacheForPersonalization;



    private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS =

            new String[] {
@@ -176,14 +174,10 @@ public class DictionaryFacilitator { 


    public DictionaryFacilitator() {

        mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER;

        mFacilitatorCacheForPersonalization = null;

    }



    public DictionaryFacilitator(final Context context) {

        mFacilitatorCacheForPersonalization = new DictionaryFacilitatorLruCache(context,

                MAX_DICTIONARY_FACILITATOR_CACHE_SIZE, "" /* dictionaryNamePrefix */);

        mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context,

                mFacilitatorCacheForPersonalization);

        mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context);

    }



    public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
@@ -358,9 +352,6 @@ public class DictionaryFacilitator { 
        for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) {

            dictionaryGroup.closeDict(dictType);

        }

        if (mFacilitatorCacheForPersonalization != null) {

            mFacilitatorCacheForPersonalization.evictAll();

        }

        mDistracterFilter.close();

    }
@@ -609,14 +600,11 @@ public class DictionaryFacilitator { 
        }

        // TODO: Get locale from personalizationDataChunk.mDetectedLanguage.

        final Locale dataChunkLocale = getLocale();

        final DictionaryFacilitator dictionaryFacilitatorForLocale =

                mFacilitatorCacheForPersonalization.get(dataChunkLocale);

        final ArrayList<LanguageModelParam> languageModelParams =

                LanguageModelParam.createLanguageModelParamsFrom(

                        personalizationDataChunk.mTokens,

                        personalizationDataChunk.mTimestampInSeconds,

                        dictionaryFacilitatorForLocale, spacingAndPunctuations,

                        new DistracterFilterCheckingIsInDictionary(

                        personalizationDataChunk.mTimestampInSeconds, spacingAndPunctuations,

                        dataChunkLocale, new DistracterFilterCheckingIsInDictionary(

                                mDistracterFilter, personalizationDict));

        if (languageModelParams == null || languageModelParams.isEmpty()) {

            if (callback != null) {

java/src/com/android/inputmethod/latin/utils/DistracterFilter.java

+34 −0
Original line number Diff line number Diff line
@@ -36,10 +36,38 @@ public interface DistracterFilter { 
    public boolean isDistracterToWordsInDictionaries(final PrevWordsInfo prevWordsInfo,

            final String testedWord, final Locale locale);



    public int getWordHandlingType(final PrevWordsInfo prevWordsInfo, final String testedWord,

            final Locale locale);



    public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes);



    public void close();



    public static final class HandlingType {

        private final static int REQUIRE_NO_SPECIAL_HANDLINGS = 0x0;

        private final static int SHOULD_BE_LOWER_CASED = 0x1;

        private final static int SHOULD_BE_HANDLED_AS_OOV = 0x2;



        public static int getHandlingType(final boolean shouldBeLowerCased, final boolean isOov) {

            int wordHandlingType = HandlingType.REQUIRE_NO_SPECIAL_HANDLINGS;

            if (shouldBeLowerCased) {

                wordHandlingType |= HandlingType.SHOULD_BE_LOWER_CASED;

            }

            if (isOov) {

                wordHandlingType |= HandlingType.SHOULD_BE_HANDLED_AS_OOV;

            }

            return wordHandlingType;

        }



        public static boolean shouldBeLowerCased(final int handlingType) {

            return (handlingType & SHOULD_BE_LOWER_CASED) != 0;

        }



        public static boolean shouldBeHandledAsOov(final int handlingType) {

            return (handlingType & SHOULD_BE_HANDLED_AS_OOV) != 0;

        }

    };



    public static final DistracterFilter EMPTY_DISTRACTER_FILTER = new DistracterFilter() {

        @Override

        public boolean isDistracterToWordsInDictionaries(PrevWordsInfo prevWordsInfo,
@@ -47,6 +75,12 @@ public interface DistracterFilter { 
            return false;

        }



        @Override

        public int getWordHandlingType(final PrevWordsInfo prevWordsInfo,

                final String testedWord, final Locale locale) {

            return HandlingType.REQUIRE_NO_SPECIAL_HANDLINGS;

        }



        @Override

        public void close() {

        }

java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingExactMatchesAndSuggestions.java

+45 −9
Original line number Diff line number Diff line
@@ -51,6 +51,7 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
            DistracterFilterCheckingExactMatchesAndSuggestions.class.getSimpleName();

    private static final boolean DEBUG = false;



    private static final int MAX_DICTIONARY_FACILITATOR_CACHE_SIZE = 3;

    private static final int MAX_DISTRACTERS_CACHE_SIZE = 1024;



    private final Context mContext;
@@ -73,15 +74,13 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
     * Create a DistracterFilter instance.

     *

     * @param context the context.

     * @param dictionaryFacilitatorLruCache the cache of dictionaryFacilitators that are used for

     * checking distracters.

     */

    public DistracterFilterCheckingExactMatchesAndSuggestions(final Context context,

            final DictionaryFacilitatorLruCache dictionaryFacilitatorLruCache) {

    public DistracterFilterCheckingExactMatchesAndSuggestions(final Context context) {

        mContext = context;

        mLocaleToSubtypeCache = new ConcurrentHashMap<>();

        mLocaleToKeyboardCache = new ConcurrentHashMap<>();

        mDictionaryFacilitatorLruCache = dictionaryFacilitatorLruCache;

        mDictionaryFacilitatorLruCache = new DictionaryFacilitatorLruCache(context,

                MAX_DICTIONARY_FACILITATOR_CACHE_SIZE, "" /* dictionaryNamePrefix */);

        mDistractersCache = new LruCache<>(MAX_DISTRACTERS_CACHE_SIZE);

    }
@@ -89,7 +88,8 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
    public void close() {

        mLocaleToSubtypeCache.clear();

        mLocaleToKeyboardCache.clear();

        mDistractersCache.evictAll();

        mDictionaryFacilitatorLruCache.evictAll();

        // Don't clear mDistractersCache.

    }



    @Override
@@ -194,9 +194,8 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
            mDistractersCache.put(cacheKey, Boolean.TRUE);

            return true;

        }

        final boolean isValidWord = dictionaryFacilitator.isValidWord(testedWord,

                false /* ignoreCase */);

        if (isValidWord) {

        final boolean Word = dictionaryFacilitator.isValidWord(testedWord, false /* ignoreCase */);

        if (Word) {

            // Valid word is not a distractor.

            if (DEBUG) {

                Log.d(TAG, "isDistracter: false (valid word)");
@@ -283,4 +282,41 @@ public class DistracterFilterCheckingExactMatchesAndSuggestions implements Distr 
        }

        return false;

    }



    private boolean shouldBeLowerCased(final PrevWordsInfo prevWordsInfo, final String testedWord,

            final Locale locale) {

        final DictionaryFacilitator dictionaryFacilitator =

                mDictionaryFacilitatorLruCache.get(locale);

        if (dictionaryFacilitator.isValidWord(testedWord, false /* ignoreCase */)) {

            return false;

        }

        final String lowerCaseTargetWord = testedWord.toLowerCase(locale);

        if (testedWord.equals(lowerCaseTargetWord)) {

            return false;

        }

        if (dictionaryFacilitator.isValidWord(lowerCaseTargetWord, false /* ignoreCase */)) {

            return true;

        }

        if (StringUtils.getCapitalizationType(testedWord) == StringUtils.CAPITALIZE_FIRST

                && !prevWordsInfo.isValid()) {

            // TODO: Check beginning-of-sentence.

            return true;

        }

        return false;

    }



    @Override

    public int getWordHandlingType(final PrevWordsInfo prevWordsInfo, final String testedWord,

            final Locale locale) {

        // TODO: Use this method for user history dictionary.

        if (testedWord == null|| locale == null) {

            return HandlingType.getHandlingType(false /* shouldBeLowerCased */, false /* isOov */);

        }

        final boolean shouldBeLowerCased = shouldBeLowerCased(prevWordsInfo, testedWord, locale);

        final String caseModifiedWord =

                shouldBeLowerCased ? testedWord.toLowerCase(locale) : testedWord;

        final boolean isOov = !mDictionaryFacilitatorLruCache.get(locale).isValidWord(

                caseModifiedWord, false /* ignoreCase */);

        return HandlingType.getHandlingType(shouldBeLowerCased, isOov);

    }

}

java/src/com/android/inputmethod/latin/utils/DistracterFilterCheckingIsInDictionary.java

+6 −0
Original line number Diff line number Diff line
@@ -47,6 +47,12 @@ public class DistracterFilterCheckingIsInDictionary implements DistracterFilter 
        }

    }



    @Override

    public int getWordHandlingType(final PrevWordsInfo prevWordsInfo, final String testedWord,

            final Locale locale) {

        return mDistracterFilter.getWordHandlingType(prevWordsInfo, testedWord, locale);

    }



    @Override

    public void updateEnabledSubtypes(List<InputMethodSubtype> enabledSubtypes) {

        // Do nothing.

java/src/com/android/inputmethod/latin/utils/LanguageModelParam.java

+15 −38
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ import com.android.inputmethod.latin.Dictionary; 
import com.android.inputmethod.latin.DictionaryFacilitator;

import com.android.inputmethod.latin.PrevWordsInfo;

import com.android.inputmethod.latin.settings.SpacingAndPunctuations;

import com.android.inputmethod.latin.utils.DistracterFilter.HandlingType;



import java.util.ArrayList;

import java.util.List;
@@ -81,8 +82,7 @@ public final class LanguageModelParam { 
    // Process a list of words and return a list of {@link LanguageModelParam} objects.

    public static ArrayList<LanguageModelParam> createLanguageModelParamsFrom(

            final List<String> tokens, final int timestamp,

            final DictionaryFacilitator dictionaryFacilitator,

            final SpacingAndPunctuations spacingAndPunctuations,

            final SpacingAndPunctuations spacingAndPunctuations, final Locale locale,

            final DistracterFilter distracterFilter) {

        final ArrayList<LanguageModelParam> languageModelParams = new ArrayList<>();

        final int N = tokens.size();
@@ -111,8 +111,7 @@ public final class LanguageModelParam { 
            }

            final LanguageModelParam languageModelParam =

                    detectWhetherVaildWordOrNotAndGetLanguageModelParam(

                            prevWordsInfo, tempWord, timestamp, dictionaryFacilitator,

                            distracterFilter);

                            prevWordsInfo, tempWord, timestamp, locale, distracterFilter);

            if (languageModelParam == null) {

                continue;

            }
@@ -125,47 +124,25 @@ public final class LanguageModelParam { 


    private static LanguageModelParam detectWhetherVaildWordOrNotAndGetLanguageModelParam(

            final PrevWordsInfo prevWordsInfo, final String targetWord, final int timestamp,

            final DictionaryFacilitator dictionaryFacilitator,

            final DistracterFilter distracterFilter) {

        final Locale locale = dictionaryFacilitator.getLocale();

            final Locale locale, final DistracterFilter distracterFilter) {

        if (locale == null) {

            return null;

        }

        if (dictionaryFacilitator.isValidWord(targetWord, false /* ignoreCase */)) {

            return createAndGetLanguageModelParamOfWord(prevWordsInfo, targetWord, timestamp,

                    true /* isValidWord */, locale, distracterFilter);

        }



        final String lowerCaseTargetWord = targetWord.toLowerCase(locale);

        if (dictionaryFacilitator.isValidWord(lowerCaseTargetWord, false /* ignoreCase */)) {

            // Add the lower-cased word.

            return createAndGetLanguageModelParamOfWord(prevWordsInfo, lowerCaseTargetWord,

                    timestamp, true /* isValidWord */, locale, distracterFilter);

        final int wordHandlingType = distracterFilter.getWordHandlingType(prevWordsInfo,

                targetWord, locale);

        final String word = HandlingType.shouldBeLowerCased(wordHandlingType) ?

                targetWord.toLowerCase(locale) : targetWord;

        if (distracterFilter.isDistracterToWordsInDictionaries(prevWordsInfo, targetWord, locale)) {

            // The word is a distracter.

            return null;

        }



        // Treat the word as an OOV word.

        return createAndGetLanguageModelParamOfWord(prevWordsInfo, targetWord, timestamp,

                false /* isValidWord */, locale, distracterFilter);

        return createAndGetLanguageModelParamOfWord(prevWordsInfo, word, timestamp,

                !HandlingType.shouldBeHandledAsOov(wordHandlingType));

    }



    private static LanguageModelParam createAndGetLanguageModelParamOfWord(

            final PrevWordsInfo prevWordsInfo, final String targetWord, final int timestamp,

            final boolean isValidWord, final Locale locale,

            final DistracterFilter distracterFilter) {

        final String word;

        if (StringUtils.getCapitalizationType(targetWord) == StringUtils.CAPITALIZE_FIRST

                && !prevWordsInfo.isValid() && !isValidWord) {

            word = targetWord.toLowerCase(locale);

        } else {

            word = targetWord;

        }

        // Check whether the word is a distracter to words in the dictionaries.

        if (distracterFilter.isDistracterToWordsInDictionaries(prevWordsInfo, word, locale)) {

            if (DEBUG) {

                Log.d(TAG, "The word (" + word + ") is a distracter. Skip this word.");

            }

            return null;

        }

            final PrevWordsInfo prevWordsInfo, final String word, final int timestamp,

            final boolean isValidWord) {

        final int unigramProbability = isValidWord ?

                UNIGRAM_PROBABILITY_FOR_VALID_WORD : UNIGRAM_PROBABILITY_FOR_OOV_WORD;

        if (!prevWordsInfo.isValid()) {