Merge "Refactor on the user history dictionary" (acb3cc74) · Commits · e / os / android_packages_inputmethods_LatinIME

java/src/com/android/inputmethod/latin/LatinIME.java

+8 −7

Original line number	Diff line number	Diff line
		@@ -76,7 +76,7 @@ import com.android.inputmethod.keyboard.MainKeyboardView;
		import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
		import com.android.inputmethod.latin.define.ProductionFlag;
		import com.android.inputmethod.latin.personalization.PersonalizationDictionaryHelper;
		import com.android.inputmethod.latin.personalization.UserHistoryDictionary;
		import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary;
		import com.android.inputmethod.latin.settings.Settings;
		import com.android.inputmethod.latin.settings.SettingsActivity;
		import com.android.inputmethod.latin.settings.SettingsValues;
		@@ -169,7 +169,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen

		private boolean mIsMainDictionaryAvailable;
		private UserBinaryDictionary mUserDictionary;
		private UserHistoryDictionary mUserHistoryDictionary;
		private UserHistoryPredictionDictionary mUserHistoryPredictionDictionary;
		private boolean mIsUserDictionaryAvailable;

		private LastComposedWord mLastComposedWord = LastComposedWord.NOT_A_COMPOSED_WORD;
		@@ -565,9 +565,9 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
		resetContactsDictionary(oldContactsDictionary);

		final SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this);
		mUserHistoryDictionary =
		PersonalizationDictionaryHelper.getUserHistoryDictionary(this, localeStr, prefs);
		mSuggest.setUserHistoryDictionary(mUserHistoryDictionary);
		mUserHistoryPredictionDictionary = PersonalizationDictionaryHelper
		.getUserHistoryPredictionDictionary(this, localeStr, prefs);
		mSuggest.setUserHistoryPredictionDictionary(mUserHistoryPredictionDictionary);
		}

		/**
		@@ -2507,7 +2507,8 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
		if (!currentSettings.mCorrectionEnabled) return null;

		final Suggest suggest = mSuggest;
		final UserHistoryDictionary userHistoryDictionary = mUserHistoryDictionary;
		final UserHistoryPredictionDictionary userHistoryDictionary =
		mUserHistoryPredictionDictionary;
		if (suggest == null \|\| userHistoryDictionary == null) {
		// Avoid concurrent issue
		return null;
		@@ -2657,7 +2658,7 @@ public class LatinIME extends InputMethodService implements KeyboardActionListen
		}
		mConnection.deleteSurroundingText(deleteLength, 0);
		if (!TextUtils.isEmpty(previousWord) && !TextUtils.isEmpty(committedWord)) {
		mUserHistoryDictionary.cancelAddingUserHistory(previousWord, committedWord);
		mUserHistoryPredictionDictionary.cancelAddingUserHistory(previousWord, committedWord);
		}
		mConnection.commitText(originallyTypedWord + mLastComposedWord.mSeparatorString, 1);
		if (mSettings.isInternal()) {

java/src/com/android/inputmethod/latin/Suggest.java

+5 −3

Original line number	Diff line number	Diff line
		@@ -22,7 +22,7 @@ import android.text.TextUtils;
		import com.android.inputmethod.annotations.UsedForTesting;
		import com.android.inputmethod.keyboard.ProximityInfo;
		import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
		import com.android.inputmethod.latin.personalization.UserHistoryDictionary;
		import com.android.inputmethod.latin.personalization.UserHistoryPredictionDictionary;
		import com.android.inputmethod.latin.utils.AutoCorrectionUtils;
		import com.android.inputmethod.latin.utils.BoundedTreeSet;
		import com.android.inputmethod.latin.utils.CollectionUtils;
		@@ -168,8 +168,10 @@ public final class Suggest {
		addOrReplaceDictionary(mDictionaries, Dictionary.TYPE_CONTACTS, contactsDictionary);
		}

		public void setUserHistoryDictionary(final UserHistoryDictionary userHistoryDictionary) {
		addOrReplaceDictionary(mDictionaries, Dictionary.TYPE_USER_HISTORY, userHistoryDictionary);
		public void setUserHistoryPredictionDictionary(
		final UserHistoryPredictionDictionary userHistoryPredictionDictionary) {
		addOrReplaceDictionary(mDictionaries, Dictionary.TYPE_USER_HISTORY,
		userHistoryPredictionDictionary);
		}

		public void setAutoCorrectionThreshold(float threshold) {

java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryHelper.java

+8 −6

Original line number	Diff line number	Diff line
		@@ -29,15 +29,16 @@ public class PersonalizationDictionaryHelper {
		private static final String TAG = PersonalizationDictionaryHelper.class.getSimpleName();
		private static final boolean DEBUG = false;

		private static final ConcurrentHashMap<String, SoftReference<UserHistoryDictionary>>
		private static final ConcurrentHashMap<String, SoftReference<UserHistoryPredictionDictionary>>
		sLangDictCache = CollectionUtils.newConcurrentHashMap();

		public static UserHistoryDictionary getUserHistoryDictionary(
		public static UserHistoryPredictionDictionary getUserHistoryPredictionDictionary(
		final Context context, final String locale, final SharedPreferences sp) {
		synchronized (sLangDictCache) {
		if (sLangDictCache.containsKey(locale)) {
		final SoftReference<UserHistoryDictionary> ref = sLangDictCache.get(locale);
		final UserHistoryDictionary dict = ref == null ? null : ref.get();
		final SoftReference<UserHistoryPredictionDictionary> ref =
		sLangDictCache.get(locale);
		final UserHistoryPredictionDictionary dict = ref == null ? null : ref.get();
		if (dict != null) {
		if (DEBUG) {
		Log.w(TAG, "Use cached UserHistoryDictionary for " + locale);
		@@ -45,8 +46,9 @@ public class PersonalizationDictionaryHelper {
		return dict;
		}
		}
		final UserHistoryDictionary dict = new UserHistoryDictionary(context, locale, sp);
		sLangDictCache.put(locale, new SoftReference<UserHistoryDictionary>(dict));
		final UserHistoryPredictionDictionary dict =
		new UserHistoryPredictionDictionary(context, locale, sp);
		sLangDictCache.put(locale, new SoftReference<UserHistoryPredictionDictionary>(dict));
		return dict;
		}
		}

java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateListener.java

+1 −1

Original line number	Diff line number	Diff line
		@@ -16,6 +16,6 @@

		package com.android.inputmethod.latin.personalization;

		public class PersonalizationDictionaryUpdateListener {
		public interface PersonalizationDictionaryUpdateListener {
		// TODO: Implement
		}

java/src/com/android/inputmethod/latin/personalization/PersonalizationPredictionDictionary.java

+362 −8

Original line number	Diff line number	Diff line
		@@ -16,11 +16,34 @@

		package com.android.inputmethod.latin.personalization;

		import com.android.inputmethod.latin.Dictionary;
		import com.android.inputmethod.latin.ExpandableDictionary;

		import android.content.Context;
		import android.content.SharedPreferences;
		import android.os.AsyncTask;
		import android.util.Log;

		import com.android.inputmethod.annotations.UsedForTesting;
		import com.android.inputmethod.keyboard.ProximityInfo;
		import com.android.inputmethod.latin.Constants;
		import com.android.inputmethod.latin.ExpandableDictionary;
		import com.android.inputmethod.latin.LatinImeLogger;
		import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
		import com.android.inputmethod.latin.WordComposer;
		import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
		import com.android.inputmethod.latin.settings.Settings;
		import com.android.inputmethod.latin.utils.ByteArrayWrapper;
		import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
		import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
		import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
		import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils;
		import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams;

		import java.io.File;
		import java.io.FileInputStream;
		import java.io.FileNotFoundException;
		import java.io.FileOutputStream;
		import java.io.IOException;
		import java.util.ArrayList;
		import java.util.concurrent.locks.ReentrantLock;

		/**
		* This class is a dictionary for the personalized prediction language model implemented in Java.
		@@ -30,17 +53,348 @@ public class PersonalizationPredictionDictionary extends ExpandableDictionary {
		// TODO: Implement
		}

		private static final String TAG = PersonalizationPredictionDictionary.class.getSimpleName();
		private static final String NAME = PersonalizationPredictionDictionary.class.getSimpleName();
		public static final boolean DBG_SAVE_RESTORE = false;
		public static final boolean DBG_STRESS_TEST = false;
		public static final boolean DBG_ALWAYS_WRITE = false;
		public static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;

		private static final FormatOptions VERSION3 = new FormatOptions(3,
		true /* supportsDynamicUpdate */);

		/** Any pair being typed or picked */
		private static final int FREQUENCY_FOR_TYPED = 2;

		/** Maximum number of pairs. Pruning will start when databases goes above this number. */
		public static final int MAX_HISTORY_BIGRAMS = 10000;

		/**
		* When it hits maximum bigram pair, it will delete until you are left with
		* only (sMaxHistoryBigrams - sDeleteHistoryBigrams) pairs.
		* Do not keep this number small to avoid deleting too often.
		*/
		public static final int DELETE_HISTORY_BIGRAMS = 1000;

		/** Locale for which this user history dictionary is storing words */
		private final String mLocale;

		private final UserHistoryDictionaryBigramList mBigramList =
		new UserHistoryDictionaryBigramList();
		private final ReentrantLock mBigramListLock = new ReentrantLock();
		private final SharedPreferences mPrefs;

		// Singleton
		private PersonalizationPredictionDictionary(final Context context, final String locale,
		final SharedPreferences sp) {
		super(context, Dictionary.TYPE_PERSONALIZATION_PREDICTION_IN_JAVA);
		// Should always be false except when we use this class for test
		@UsedForTesting boolean isTest = false;

		/* package */ PersonalizationPredictionDictionary(final Context context, final String locale,
		final SharedPreferences sp, final String dictionaryType) {
		super(context, dictionaryType);
		mLocale = locale;
		mPrefs = sp;
		if (mLocale != null && mLocale.length() > 1) {
		loadDictionary();
		}
		}

		// TODO: Implement
		@Override
		public void close() {
		flushPendingWrites();
		// Don't close the database as locale changes will require it to be reopened anyway
		// Also, the database is written to somewhat frequently, so it needs to be kept alive
		// throughout the life of the process.
		// mOpenHelper.close();
		// Ignore close because we cache PersonalizationPredictionDictionary for each language.
		// See getInstance() above.
		// super.close();
		}

		@Override
		protected ArrayList<SuggestedWordInfo> getWordsInner(final WordComposer composer,
		final String prevWord, final ProximityInfo proximityInfo) {
		// Inhibit suggestions (not predictions) for user history for now. Removing this method
		// is enough to use it through the standard ExpandableDictionary way.
		return null;
		}

		/**
		* Return whether the passed charsequence is in the dictionary.
		*/
		@Override
		public synchronized boolean isValidWord(final String word) {
		// TODO: figure out what is the correct thing to do here.
		return false;
		}

		/**
		* Pair will be added to the user history dictionary.
		*
		* The first word may be null. That means we don't know the context, in other words,
		* it's only a unigram. The first word may also be an empty string : this means start
		* context, as in beginning of a sentence for example.
		* The second word may not be null (a NullPointerException would be thrown).
		*/
		public int addToUserHistory(final String word1, final String word2, final boolean isValid) {
		if (word2.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH \|\|
		(word1 != null && word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
		return -1;
		}
		if (mBigramListLock.tryLock()) {
		try {
		super.addWord(
		word2, null /* the "shortcut" parameter is null */, FREQUENCY_FOR_TYPED);
		mBigramList.addBigram(null, word2, (byte)FREQUENCY_FOR_TYPED);
		// Do not insert a word as a bigram of itself
		if (word2.equals(word1)) {
		return 0;
		}
		final int freq;
		if (null == word1) {
		freq = FREQUENCY_FOR_TYPED;
		} else {
		freq = super.setBigramAndGetFrequency(
		word1, word2, new ForgettingCurveParams(isValid));
		}
		mBigramList.addBigram(word1, word2);
		return freq;
		} finally {
		mBigramListLock.unlock();
		}
		}
		return -1;
		}

		public boolean cancelAddingUserHistory(final String word1, final String word2) {
		if (mBigramListLock.tryLock()) {
		try {
		if (mBigramList.removeBigram(word1, word2)) {
		return super.removeBigram(word1, word2);
		}
		} finally {
		mBigramListLock.unlock();
		}
		}
		return false;
		}

		/**
		* Schedules a background thread to write any pending words to the database.
		*/
		private void flushPendingWrites() {
		// Create a background thread to write the pending entries
		new UpdateBinaryTask(mBigramList, mLocale, this, mPrefs, getContext()).execute();
		}

		@Override
		public void loadDictionaryAsync() {
		// This must be run on non-main thread
		mBigramListLock.lock();
		try {
		loadDictionaryAsyncLocked();
		} finally {
		mBigramListLock.unlock();
		}
		}

		private int profTotal;

		private void loadDictionaryAsyncLocked() {
		if (DBG_STRESS_TEST) {
		try {
		Log.w(TAG, "Start stress in loading: " + mLocale);
		Thread.sleep(15000);
		Log.w(TAG, "End stress in loading");
		} catch (InterruptedException e) {
		}
		}
		final long last = Settings.readLastUserHistoryWriteTime(mPrefs, mLocale);
		final boolean initializing = last == 0;
		final long now = System.currentTimeMillis();
		profTotal = 0;
		final String fileName = NAME + "." + mLocale + ".dict";
		final ExpandableDictionary dictionary = this;
		final OnAddWordListener listener = new OnAddWordListener() {
		@Override
		public void setUnigram(final String word, final String shortcutTarget,
		final int frequency) {
		profTotal++;
		if (DBG_SAVE_RESTORE) {
		Log.d(TAG, "load unigram: " + word + "," + frequency);
		}
		dictionary.addWord(word, shortcutTarget, frequency);
		mBigramList.addBigram(null, word, (byte)frequency);
		}

		@Override
		public void setBigram(final String word1, final String word2, final int frequency) {
		if (word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH
		&& word2.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) {
		profTotal++;
		if (DBG_SAVE_RESTORE) {
		Log.d(TAG, "load bigram: " + word1 + "," + word2 + "," + frequency);
		}
		dictionary.setBigramAndGetFrequency(
		word1, word2, initializing ? new ForgettingCurveParams(true)
		: new ForgettingCurveParams(frequency, now, last));
		}
		mBigramList.addBigram(word1, word2, (byte)frequency);
		}
		};

		// Load the dictionary from binary file
		FileInputStream inStream = null;
		try {
		final File file = new File(getContext().getFilesDir(), fileName);
		final byte[] buffer = new byte[(int)file.length()];
		inStream = new FileInputStream(file);
		inStream.read(buffer);
		UserHistoryDictIOUtils.readDictionaryBinary(
		new ByteArrayWrapper(buffer), listener);
		} catch (FileNotFoundException e) {
		// This is an expected condition: we don't have a user history dictionary for this
		// language yet. It will be created sometime later.
		} catch (IOException e) {
		Log.e(TAG, "IOException on opening a bytebuffer", e);
		} finally {
		if (inStream != null) {
		try {
		inStream.close();
		} catch (IOException e) {
		// do nothing
		}
		}
		if (PROFILE_SAVE_RESTORE) {
		final long diff = System.currentTimeMillis() - now;
		Log.d(TAG, "PROF: Load UserHistoryDictionary: "
		+ mLocale + ", " + diff + "ms. load " + profTotal + "entries.");
		}
		}
		}

		/**
		* Async task to write pending words to the binarydicts.
		*/
		private static final class UpdateBinaryTask extends AsyncTask<Void, Void, Void>
		implements BigramDictionaryInterface {
		private final UserHistoryDictionaryBigramList mBigramList;
		private final boolean mAddLevel0Bigrams;
		private final String mLocale;
		private final PersonalizationPredictionDictionary mPersonalizationPredictionDictionary;
		private final SharedPreferences mPrefs;
		private final Context mContext;

		public UpdateBinaryTask(final UserHistoryDictionaryBigramList pendingWrites,
		final String locale, final PersonalizationPredictionDictionary dict,
		final SharedPreferences prefs, final Context context) {
		mBigramList = pendingWrites;
		mLocale = locale;
		mPersonalizationPredictionDictionary = dict;
		mPrefs = prefs;
		mContext = context;
		mAddLevel0Bigrams = mBigramList.size() <= MAX_HISTORY_BIGRAMS;
		}

		@Override
		protected Void doInBackground(final Void... v) {
		if (mPersonalizationPredictionDictionary.isTest) {
		// If isTest == true, wait until the lock is released.
		mPersonalizationPredictionDictionary.mBigramListLock.lock();
		try {
		doWriteTaskLocked();
		} finally {
		mPersonalizationPredictionDictionary.mBigramListLock.unlock();
		}
		} else if (mPersonalizationPredictionDictionary.mBigramListLock.tryLock()) {
		try {
		doWriteTaskLocked();
		} finally {
		mPersonalizationPredictionDictionary.mBigramListLock.unlock();
		}
		}
		return null;
		}

		private void doWriteTaskLocked() {
		if (DBG_STRESS_TEST) {
		try {
		Log.w(TAG, "Start stress in closing: " + mLocale);
		Thread.sleep(15000);
		Log.w(TAG, "End stress in closing");
		} catch (InterruptedException e) {
		Log.e(TAG, "In stress test", e);
		}
		}

		final long now = PROFILE_SAVE_RESTORE ? System.currentTimeMillis() : 0;
		final String fileName = NAME + "." + mLocale + ".dict";
		final File file = new File(mContext.getFilesDir(), fileName);
		FileOutputStream out = null;

		try {
		out = new FileOutputStream(file);
		UserHistoryDictIOUtils.writeDictionaryBinary(out, this, mBigramList, VERSION3);
		out.flush();
		out.close();
		} catch (IOException e) {
		Log.e(TAG, "IO Exception while writing file", e);
		} finally {
		if (out != null) {
		try {
		out.close();
		} catch (IOException e) {
		// ignore
		}
		}
		}

		// Save the timestamp after we finish writing the binary dictionary.
		Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
		if (PROFILE_SAVE_RESTORE) {
		final long diff = System.currentTimeMillis() - now;
		Log.w(TAG, "PROF: Write User HistoryDictionary: " + mLocale + ", " + diff + "ms.");
		}
		}

		@Override
		public int getFrequency(final String word1, final String word2) {
		final int freq;
		if (word1 == null) { // unigram
		freq = FREQUENCY_FOR_TYPED;
		final byte prevFc = mBigramList.getBigrams(word1).get(word2);
		} else { // bigram
		final NextWord nw =
		mPersonalizationPredictionDictionary.getBigramWord(word1, word2);
		if (nw != null) {
		final ForgettingCurveParams fcp = nw.getFcParams();
		final byte prevFc = mBigramList.getBigrams(word1).get(word2);
		final byte fc = fcp.getFc();
		final boolean isValid = fcp.isValid();
		if (prevFc > 0 && prevFc == fc) {
		freq = fc & 0xFF;
		} else if (UserHistoryForgettingCurveUtils.
		needsToSave(fc, isValid, mAddLevel0Bigrams)) {
		freq = fc & 0xFF;
		} else {
		// Delete this entry
		freq = -1;
		}
		} else {
		// Delete this entry
		freq = -1;
		}
		}
		return freq;
		}
		}

		@UsedForTesting
		/* package for test */ void forceAddWordForTest(
		final String word1, final String word2, final boolean isValid) {
		mBigramListLock.lock();
		try {
		addToUserHistory(word1, word2, isValid);
		} finally {
		mBigramListLock.unlock();
		}
		}
		}