Get locale using detected language for personalization. (89e34169) · Commits · e / os / android_packages_inputmethods_LatinIME

java/src/com/android/inputmethod/latin/DictionaryFacilitator.java

+17 −29

Original line number	Diff line number	Diff line
		@@ -24,6 +24,7 @@ import android.view.inputmethod.InputMethodSubtype;

		import com.android.inputmethod.annotations.UsedForTesting;
		import com.android.inputmethod.keyboard.ProximityInfo;
		import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback;
		import com.android.inputmethod.latin.PrevWordsInfo.WordInfo;
		import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
		import com.android.inputmethod.latin.personalization.ContextualDictionary;
		@@ -36,7 +37,6 @@ import com.android.inputmethod.latin.utils.DistracterFilter;
		import com.android.inputmethod.latin.utils.DistracterFilterCheckingExactMatchesAndSuggestions;
		import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary;
		import com.android.inputmethod.latin.utils.ExecutorUtils;
		import com.android.inputmethod.latin.utils.LanguageModelParam;
		import com.android.inputmethod.latin.utils.SuggestionResults;

		import java.io.File;
		@@ -67,6 +67,7 @@ public class DictionaryFacilitator {
		// To synchronize assigning mDictionaryGroup to ensure closing dictionaries.
		private final Object mLock = new Object();
		private final DistracterFilter mDistracterFilter;
		private final PersonalizationDictionaryFacilitator mPersonalizationDictionaryFacilitator;

		private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS =
		new String[] {
		@@ -174,14 +175,18 @@ public class DictionaryFacilitator {

		public DictionaryFacilitator() {
		mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER;
		mPersonalizationDictionaryFacilitator = null;
		}

		public DictionaryFacilitator(final Context context) {
		mDistracterFilter = new DistracterFilterCheckingExactMatchesAndSuggestions(context);
		mPersonalizationDictionaryFacilitator =
		new PersonalizationDictionaryFacilitator(context, mDistracterFilter);
		}

		public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
		mDistracterFilter.updateEnabledSubtypes(enabledSubtypes);
		mPersonalizationDictionaryFacilitator.updateEnabledSubtypes(enabledSubtypes);
		}

		public Locale getLocale() {
		@@ -353,6 +358,9 @@ public class DictionaryFacilitator {
		dictionaryGroup.closeDict(dictType);
		}
		mDistracterFilter.close();
		if (mPersonalizationDictionaryFacilitator != null) {
		mPersonalizationDictionaryFacilitator.close();
		}
		}

		@UsedForTesting
		@@ -372,11 +380,11 @@ public class DictionaryFacilitator {
		}

		public void flushPersonalizationDictionary() {
		final ExpandableBinaryDictionary personalizationDict =
		final ExpandableBinaryDictionary personalizationDictUsedForSuggestion =
		mDictionaryGroup.getSubDict(Dictionary.TYPE_PERSONALIZATION);
		if (personalizationDict != null) {
		personalizationDict.asyncFlushBinaryDictionary();
		}
		mPersonalizationDictionaryFacilitator.flushPersonalizationDictionariesToUpdate(
		personalizationDictUsedForSuggestion);
		mDistracterFilter.close();
		}

		public void waitForLoadingMainDictionary(final long timeout, final TimeUnit unit)
		@@ -580,6 +588,7 @@ public class DictionaryFacilitator {
		// personalization dictionary.
		public void clearPersonalizationDictionary() {
		clearSubDictionary(Dictionary.TYPE_PERSONALIZATION);
		mPersonalizationDictionaryFacilitator.clearDictionariesToUpdate();
		}

		public void clearContextualDictionary() {
		@@ -589,30 +598,9 @@ public class DictionaryFacilitator {
		public void addEntriesToPersonalizationDictionary(
		final PersonalizationDataChunk personalizationDataChunk,
		final SpacingAndPunctuations spacingAndPunctuations,
		final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) {
		final ExpandableBinaryDictionary personalizationDict =
		mDictionaryGroup.getSubDict(Dictionary.TYPE_PERSONALIZATION);
		if (personalizationDict == null) {
		if (callback != null) {
		callback.onFinished();
		}
		return;
		}
		// TODO: Get locale from personalizationDataChunk.mDetectedLanguage.
		final Locale dataChunkLocale = getLocale();
		final ArrayList<LanguageModelParam> languageModelParams =
		LanguageModelParam.createLanguageModelParamsFrom(
		personalizationDataChunk.mTokens,
		personalizationDataChunk.mTimestampInSeconds, spacingAndPunctuations,
		dataChunkLocale, new DistracterFilterCheckingIsInDictionary(
		mDistracterFilter, personalizationDict));
		if (languageModelParams == null \|\| languageModelParams.isEmpty()) {
		if (callback != null) {
		callback.onFinished();
		}
		return;
		}
		personalizationDict.addMultipleDictionaryEntriesDynamically(languageModelParams, callback);
		final AddMultipleDictionaryEntriesCallback callback) {
		mPersonalizationDictionaryFacilitator.addEntriesToPersonalizationDictionariesToUpdate(
		personalizationDataChunk, spacingAndPunctuations, callback);
		}

		public void addPhraseToContextualDictionary(final String[] phrase, final int probability,

java/src/com/android/inputmethod/latin/PersonalizationDictionaryFacilitator.java

0 → 100644

+174 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2014 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.inputmethod.latin;

		import java.util.ArrayList;
		import java.util.HashMap;
		import java.util.HashSet;
		import java.util.List;
		import java.util.Locale;
		import java.util.concurrent.atomic.AtomicInteger;

		import android.content.Context;
		import android.view.inputmethod.InputMethodSubtype;

		import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback;
		import com.android.inputmethod.latin.personalization.PersonalizationDataChunk;
		import com.android.inputmethod.latin.personalization.PersonalizationDictionary;
		import com.android.inputmethod.latin.settings.SpacingAndPunctuations;
		import com.android.inputmethod.latin.utils.DistracterFilter;
		import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary;
		import com.android.inputmethod.latin.utils.LanguageModelParam;
		import com.android.inputmethod.latin.utils.SubtypeLocaleUtils;

		/**
		* Class for managing and updating personalization dictionaries.
		*/
		public class PersonalizationDictionaryFacilitator {
		private final Context mContext;
		private final DistracterFilter mDistracterFilter;
		private final HashMap<String, HashSet<Locale>> mLangToLocalesMap = new HashMap<>();
		private final HashMap<Locale, ExpandableBinaryDictionary> mPersonalizationDictsToUpdate =
		new HashMap<>();

		PersonalizationDictionaryFacilitator(final Context context,
		final DistracterFilter distracterFilter) {
		mContext = context;
		mDistracterFilter = distracterFilter;
		}

		public void close() {
		mLangToLocalesMap.clear();
		for (final ExpandableBinaryDictionary dict : mPersonalizationDictsToUpdate.values()) {
		dict.close();
		}
		mPersonalizationDictsToUpdate.clear();
		}

		public void clearDictionariesToUpdate() {
		for (final ExpandableBinaryDictionary dict : mPersonalizationDictsToUpdate.values()) {
		dict.clear();
		}
		mPersonalizationDictsToUpdate.clear();
		}

		public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) {
		for (final InputMethodSubtype subtype : enabledSubtypes) {
		final Locale locale = SubtypeLocaleUtils.getSubtypeLocale(subtype);
		final String language = locale.getLanguage();
		final HashSet<Locale> locales = mLangToLocalesMap.get(language);
		if (locales != null) {
		locales.add(locale);
		} else {
		final HashSet<Locale> localeSet = new HashSet<>();
		localeSet.add(locale);
		mLangToLocalesMap.put(language, localeSet);
		}
		}
		}

		/**
		* Flush personalization dictionaries to dictionary files. Close dictionaries after writing
		* files except the dictionary that is used for generating suggestions.
		*
		* @param personalizationDictUsedForSuggestion the personalization dictionary used for
		* generating suggestions that won't be closed.
		*/
		public void flushPersonalizationDictionariesToUpdate(
		final ExpandableBinaryDictionary personalizationDictUsedForSuggestion) {
		for (final ExpandableBinaryDictionary personalizationDict :
		mPersonalizationDictsToUpdate.values()) {
		personalizationDict.asyncFlushBinaryDictionary();
		if (personalizationDict != personalizationDictUsedForSuggestion) {
		// Close if the dictionary is not being used for suggestion.
		personalizationDict.close();
		}
		}
		mDistracterFilter.close();
		mPersonalizationDictsToUpdate.clear();
		}

		private ExpandableBinaryDictionary getPersonalizationDictToUpdate(final Context context,
		final Locale locale) {
		ExpandableBinaryDictionary personalizationDict = mPersonalizationDictsToUpdate.get(locale);
		if (personalizationDict != null) {
		return personalizationDict;
		}
		personalizationDict = PersonalizationDictionary.getDictionary(context, locale,
		null /* dictFile /, "" / dictNamePrefix */);
		mPersonalizationDictsToUpdate.put(locale, personalizationDict);
		return personalizationDict;
		}

		private void addEntriesToPersonalizationDictionariesForLocale(final Locale locale,
		final PersonalizationDataChunk personalizationDataChunk,
		final SpacingAndPunctuations spacingAndPunctuations,
		final AddMultipleDictionaryEntriesCallback callback) {
		final ExpandableBinaryDictionary personalizationDict =
		getPersonalizationDictToUpdate(mContext, locale);
		if (personalizationDict == null) {
		if (callback != null) {
		callback.onFinished();
		}
		return;
		}
		final ArrayList<LanguageModelParam> languageModelParams =
		LanguageModelParam.createLanguageModelParamsFrom(
		personalizationDataChunk.mTokens,
		personalizationDataChunk.mTimestampInSeconds, spacingAndPunctuations,
		locale, new DistracterFilterCheckingIsInDictionary(
		mDistracterFilter, personalizationDict));
		if (languageModelParams == null \|\| languageModelParams.isEmpty()) {
		if (callback != null) {
		callback.onFinished();
		}
		return;
		}
		personalizationDict.addMultipleDictionaryEntriesDynamically(languageModelParams, callback);
		}

		public void addEntriesToPersonalizationDictionariesToUpdate(
		final PersonalizationDataChunk personalizationDataChunk,
		final SpacingAndPunctuations spacingAndPunctuations,
		final AddMultipleDictionaryEntriesCallback callback) {
		final HashSet<Locale> locales =
		mLangToLocalesMap.get(personalizationDataChunk.mDetectedLanguage);
		if (locales == null \|\| locales.isEmpty()) {
		if (callback != null) {
		callback.onFinished();
		}
		return;
		}
		final AtomicInteger remainingTaskCount = new AtomicInteger(locales.size());
		final AddMultipleDictionaryEntriesCallback callbackForLocales =
		new AddMultipleDictionaryEntriesCallback() {
		@Override
		public void onFinished() {
		if (remainingTaskCount.decrementAndGet() == 0) {
		// Update tasks for all locales have been finished.
		if (callback != null) {
		callback.onFinished();
		}
		}
		}
		};
		for (final Locale locale : locales) {
		addEntriesToPersonalizationDictionariesForLocale(locale, personalizationDataChunk,
		spacingAndPunctuations, callbackForLocales);
		}
		}
		}

tests/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryTests.java

+17 −3

Original line number	Diff line number	Diff line
		@@ -29,6 +29,7 @@ import com.android.inputmethod.latin.BinaryDictionary;
		import com.android.inputmethod.latin.Dictionary;
		import com.android.inputmethod.latin.DictionaryFacilitator;
		import com.android.inputmethod.latin.ExpandableBinaryDictionary;
		import com.android.inputmethod.latin.RichInputMethodManager;
		import com.android.inputmethod.latin.ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback;
		import com.android.inputmethod.latin.makedict.CodePointUtils;
		import com.android.inputmethod.latin.settings.SpacingAndPunctuations;
		@@ -36,6 +37,7 @@ import com.android.inputmethod.latin.settings.SpacingAndPunctuations;
		import android.test.AndroidTestCase;
		import android.test.suitebuilder.annotation.LargeTest;
		import android.util.Log;
		import android.view.inputmethod.InputMethodSubtype;

		/**
		* Unit tests for personalization dictionary
		@@ -55,16 +57,28 @@ public class PersonalizationDictionaryTests extends AndroidTestCase {
		final DictionaryFacilitator dictionaryFacilitator = new DictionaryFacilitator(getContext());
		dictionaryFacilitator.resetDictionariesForTesting(getContext(), LOCALE_EN_US, dictTypes,
		new HashMap<String, File>(), new HashMap<String, Map<String, String>>());
		// Set subtypes.
		RichInputMethodManager.init(getContext());
		final RichInputMethodManager richImm = RichInputMethodManager.getInstance();
		final ArrayList<InputMethodSubtype> subtypes = new ArrayList<>();
		subtypes.add(richImm.findSubtypeByLocaleAndKeyboardLayoutSet(
		LOCALE_EN_US.toString(), "qwerty"));
		dictionaryFacilitator.updateEnabledSubtypes(subtypes);
		return dictionaryFacilitator;
		}

		public void testAddManyTokens() {
		final DictionaryFacilitator dictionaryFacilitator = getDictionaryFacilitator();
		dictionaryFacilitator.clearPersonalizationDictionary();
		final int dataChunkCount = 20;
		final int wordCountInOneChunk = 2000;
		final int dataChunkCount = 100;
		final int wordCountInOneChunk = 200;
		final int uniqueWordCount = 100;
		final Random random = new Random(System.currentTimeMillis());
		final int[] codePointSet = CodePointUtils.LATIN_ALPHABETS_LOWER;
		final ArrayList<String> words = new ArrayList<>();
		for (int i = 0; i < uniqueWordCount; i++) {
		words.add(CodePointUtils.generateWord(random, codePointSet));
		}

		final SpacingAndPunctuations spacingAndPunctuations =
		new SpacingAndPunctuations(getContext().getResources());
		@@ -75,7 +89,7 @@ public class PersonalizationDictionaryTests extends AndroidTestCase {
		for (int i = 0; i < dataChunkCount; i++) {
		final ArrayList<String> tokens = new ArrayList<>();
		for (int j = 0; j < wordCountInOneChunk; j++) {
		tokens.add(CodePointUtils.generateWord(random, codePointSet));
		tokens.add(words.get(random.nextInt(words.size())));
		}
		final PersonalizationDataChunk personalizationDataChunk = new PersonalizationDataChunk(
		true /* inputByUser */, tokens, timeStampInSeconds, DUMMY_PACKAGE_NAME,