Quit using ExpandableDictionary. (2e58670d) · Commits · e / os / android_packages_inputmethods_LatinIME

java/src/com/android/inputmethod/latin/BinaryDictionary.java

+18 −3

Original line number	Diff line number	Diff line
		@@ -27,6 +27,7 @@ import com.android.inputmethod.latin.utils.CollectionUtils;
		import com.android.inputmethod.latin.utils.JniUtils;
		import com.android.inputmethod.latin.utils.StringUtils;

		import java.io.File;
		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.Locale;
		@@ -244,11 +245,18 @@ public final class BinaryDictionary extends Dictionary {
		return getBigramProbabilityNative(mNativeDict, codePoints0, codePoints1);
		}

		private void runGCIfRequired() {
		if (needsToRunGCNative(mNativeDict)) {
		flushWithGC();
		}
		}

		// Add a unigram entry to binary dictionary in native code.
		public void addUnigramWord(final String word, final int probability) {
		if (TextUtils.isEmpty(word)) {
		return;
		}
		runGCIfRequired();
		final int[] codePoints = StringUtils.toCodePointArray(word);
		addUnigramWordNative(mNativeDict, codePoints, probability);
		}
		@@ -258,6 +266,7 @@ public final class BinaryDictionary extends Dictionary {
		if (TextUtils.isEmpty(word0) \|\| TextUtils.isEmpty(word1)) {
		return;
		}
		runGCIfRequired();
		final int[] codePoints0 = StringUtils.toCodePointArray(word0);
		final int[] codePoints1 = StringUtils.toCodePointArray(word1);
		addBigramWordsNative(mNativeDict, codePoints0, codePoints1, probability);
		@@ -268,24 +277,30 @@ public final class BinaryDictionary extends Dictionary {
		if (TextUtils.isEmpty(word0) \|\| TextUtils.isEmpty(word1)) {
		return;
		}
		runGCIfRequired();
		final int[] codePoints0 = StringUtils.toCodePointArray(word0);
		final int[] codePoints1 = StringUtils.toCodePointArray(word1);
		removeBigramWordsNative(mNativeDict, codePoints0, codePoints1);
		}

		@UsedForTesting
		public void flush() {
		if (!isValidDictionary()) return;
		flushNative(mNativeDict, mDictFilePath);
		closeNative(mNativeDict);
		final File dictFile = new File(mDictFilePath);
		mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
		dictFile.length(), true /* isUpdatable */);
		}

		@UsedForTesting
		public void flushWithGC() {
		if (!isValidDictionary()) return;
		flushWithGCNative(mNativeDict, mDictFilePath);
		closeNative(mNativeDict);
		final File dictFile = new File(mDictFilePath);
		mNativeDict = openNative(dictFile.getAbsolutePath(), 0 /* startOffset */,
		dictFile.length(), true /* isUpdatable */);
		}

		@UsedForTesting
		public boolean needsToRunGC() {
		if (!isValidDictionary()) return false;
		return needsToRunGCNative(mNativeDict);

java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java

+115 −34

Original line number	Diff line number	Diff line
		@@ -22,14 +22,22 @@ import android.util.Log;

		import com.android.inputmethod.annotations.UsedForTesting;
		import com.android.inputmethod.keyboard.ProximityInfo;
		import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
		import com.android.inputmethod.latin.makedict.DictEncoder;
		import com.android.inputmethod.latin.makedict.FormatSpec;
		import com.android.inputmethod.latin.makedict.FusionDictionary;
		import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
		import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
		import com.android.inputmethod.latin.makedict.Ver3DictEncoder;
		import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;
		import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
		import com.android.inputmethod.latin.utils.AsyncResultHolder;
		import com.android.inputmethod.latin.utils.CollectionUtils;
		import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;

		import java.io.File;
		import java.io.IOException;
		import java.util.ArrayList;
		import java.util.HashMap;
		import java.util.concurrent.ConcurrentHashMap;
		import java.util.concurrent.atomic.AtomicReference;

		@@ -49,9 +57,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		/** Whether to print debug output to log */
		private static boolean DEBUG = false;

		// TODO: Remove and enable dynamic update in native code.
		// TODO: Remove.
		/** Whether to call binary dictionary dynamically updating methods. */
		private static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = false;
		public static boolean ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE = true;

		private static final int TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS = 100;

		@@ -60,6 +68,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		*/
		protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;

		private static final FormatSpec.FormatOptions FORMAT_OPTIONS =
		new FormatSpec.FormatOptions(3 /* version /, true / supportsDynamicUpdate */);

		/**
		* A static map of time recorders, each of which records the time of accesses to a single binary
		* dictionary file. The key for this map is the filename and the value is the shared dictionary
		@@ -154,7 +165,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		private static AbstractDictionaryWriter getDictionaryWriter(final Context context,
		final String dictType, final boolean isDynamicPersonalizationDictionary) {
		if (isDynamicPersonalizationDictionary) {
		if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
		return null;
		} else {
		return new DynamicPersonalizationDictionaryWriter(context, dictType);
		}
		} else {
		return new DictionaryWriter(context, dictType);
		}
		@@ -198,8 +213,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		mBinaryDictionary.close();
		mBinaryDictionary = null;
		}
		if (mDictionaryWriter != null) {
		mDictionaryWriter.close();
		}
		}
		});
		}

		@@ -220,8 +237,24 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		getExecutor(mFilename).execute(new Runnable() {
		@Override
		public void run() {
		if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) {
		mBinaryDictionary.close();
		final File file = new File(mContext.getFilesDir(), mFilename);
		final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
		new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
		false, false));
		final DictEncoder dictEncoder = new Ver3DictEncoder(file);
		try {
		dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
		} catch (IOException e) {
		Log.e(TAG, "Exception in creating new dictionary file.", e);
		} catch (UnsupportedFormatException e) {
		Log.e(TAG, "Exception in creating new dictionary file.", e);
		}
		} else {
		mDictionaryWriter.clear();
		}
		}
		});
		}

		@@ -257,10 +290,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		public void run() {
		if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
		mBinaryDictionary.addUnigramWord(word, frequency);
		}
		} else {
		// TODO: Remove.
		mDictionaryWriter.addUnigramWord(word, shortcutTarget, frequency, isNotAWord);
		}
		}
		});
		}

		@@ -280,11 +314,12 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		public void run() {
		if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
		mBinaryDictionary.addBigramWords(word0, word1, frequency);
		}
		} else {
		// TODO: Remove.
		mDictionaryWriter.addBigramWords(word0, word1, frequency, isValid,
		0 /* lastTouchedTime */);
		}
		}
		});
		}

		@@ -303,10 +338,11 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		public void run() {
		if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
		mBinaryDictionary.removeBigramWords(word0, word1);
		}
		} else {
		// TODO: Remove.
		mDictionaryWriter.removeBigramWords(word0, word1);
		}
		}
		});
		}

		@@ -322,16 +358,28 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		getExecutor(mFilename).executePrioritized(new Runnable() {
		@Override
		public void run() {
		final ArrayList<SuggestedWordInfo> inMemDictSuggestion = composer.isBatchMode() ?
		null : mDictionaryWriter.getSuggestionsWithSessionId(composer, prevWord,
		if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
		if (mBinaryDictionary == null) {
		holder.set(null);
		return;
		}
		final ArrayList<SuggestedWordInfo> binarySuggestion =
		mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
		proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
		sessionId);
		holder.set(binarySuggestion);
		} else {
		final ArrayList<SuggestedWordInfo> inMemDictSuggestion =
		composer.isBatchMode() ? null :
		mDictionaryWriter.getSuggestionsWithSessionId(composer,
		prevWord, proximityInfo, blockOffensiveWords,
		additionalFeaturesOptions, sessionId);
		// TODO: Remove checking mIsUpdatable and use native suggestion.
		if (mBinaryDictionary != null && !mIsUpdatable) {
		final ArrayList<SuggestedWordInfo> binarySuggestion =
		mBinaryDictionary.getSuggestionsWithSessionId(composer, prevWord,
		proximityInfo, blockOffensiveWords, additionalFeaturesOptions,
		sessionId);
		proximityInfo, blockOffensiveWords,
		additionalFeaturesOptions, sessionId);
		if (inMemDictSuggestion == null) {
		holder.set(binarySuggestion);
		} else if (binarySuggestion == null) {
		@@ -344,6 +392,7 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		holder.set(inMemDictSuggestion);
		}
		}
		}
		});
		return holder.get(null, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS);
		}
		@@ -411,8 +460,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		final BinaryDictionary newBinaryDictionary = new BinaryDictionary(filename, 0, length,
		true /* useFullEditDistance */, null, mDictType, mIsUpdatable);

		// Ensure all threads accessing the current dictionary have finished before swapping in
		// the new one.
		// Ensure all threads accessing the current dictionary have finished before
		// swapping in the new one.
		// TODO: Ensure multi-thread assignment of mBinaryDictionary.
		final BinaryDictionary oldBinaryDictionary = mBinaryDictionary;
		getExecutor(mFilename).executePrioritized(new Runnable() {
		@Override
		@@ -443,9 +493,34 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		if (needsToReloadBeforeWriting()) {
		mDictionaryWriter.clear();
		loadDictionaryAsync();
		mDictionaryWriter.write(mFilename);
		} else {
		if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
		if (mBinaryDictionary == null \|\| !mBinaryDictionary.isValidDictionary()) {
		final File file = new File(mContext.getFilesDir(), mFilename);
		final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),
		new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),
		false, false));
		final DictEncoder dictEncoder = new Ver3DictEncoder(file);
		try {
		dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);
		} catch (IOException e) {
		Log.e(TAG, "Exception in creating new dictionary file.", e);
		} catch (UnsupportedFormatException e) {
		Log.e(TAG, "Exception in creating new dictionary file.", e);
		}
		} else {
		if (mBinaryDictionary.needsToRunGC()) {
		mBinaryDictionary.flushWithGC();
		} else {
		mBinaryDictionary.flush();
		}
		}
		} else {
		mDictionaryWriter.write(mFilename);
		}
		}
		}

		/**
		* Marks that the dictionary is out of date and requires a reload.
		@@ -539,15 +614,17 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		getExecutor(mFilename).executePrioritized(new Runnable() {
		@Override
		public void run() {
		if (!ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
		loadDictionaryAsync();
		}
		}
		});
		}

		/**
		* Generate binary dictionary using DictionaryWriter.
		*/
		protected void asyncWriteBinaryDictionary() {
		protected void asyncFlashAllBinaryDictionary() {
		final Runnable newTask = new Runnable() {
		@Override
		public void run() {
		@@ -620,10 +697,14 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		@Override
		public void run() {
		if (mDictType == Dictionary.TYPE_USER_HISTORY) {
		if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
		holder.set(mBinaryDictionary.isValidWord(word));
		} else {
		holder.set(((DynamicPersonalizationDictionaryWriter) mDictionaryWriter)
		.isInDictionaryForTests(word));
		}
		}
		}
		});
		return holder.get(false, TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS);
		}

java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java

+5 −5

Original line number	Diff line number	Diff line
		@@ -74,12 +74,12 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi

		@Override
		public void close() {
		// Close only binary dictionary to reuse this dictionary.
		// super.close();
		if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
		closeBinaryDictionary();
		}
		// Flush pending writes.
		// TODO: Remove after this class become to use a dynamic binary dictionary.
		asyncWriteBinaryDictionary();
		asyncFlashAllBinaryDictionary();
		Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
		}

		@@ -212,6 +212,6 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi
		// Clear the node structure on memory
		clear();
		// Then flush the cleared state of the dictionary on disk.
		asyncWriteBinaryDictionary();
		asyncFlashAllBinaryDictionary();
		}
		}

tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java

+42 −1

Original line number	Diff line number	Diff line
		@@ -100,7 +100,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
		Thread.sleep(TimeUnit.MILLISECONDS.convert(5L, TimeUnit.SECONDS));
		} catch (InterruptedException e) {
		}
		for (int i = 0; i < 10 && i < numberOfWords; ++i) {
		// Limit word count to check when using a Java on memory dictionary.
		final int wordCountToCheck =
		ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
		numberOfWords : 10;
		for (int i = 0; i < wordCountToCheck; ++i) {
		final String word = words.get(i);
		// This may fail as long as we use tryLock on inserting the bigram words
		assertTrue(dict.isInDictionaryForTests(word));
		@@ -202,4 +206,41 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
		}
		}
		}

		public void testAddManyWords() {
		File dictFile = null;
		final String testFilenameSuffix = "testRandomWords" + System.currentTimeMillis();
		final int numberOfWords =
		ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
		10000 : 1000;
		final Random random = new Random(123456);

		UserHistoryPredictionDictionary dict =
		PersonalizationHelper.getUserHistoryPredictionDictionary(getContext(),
		testFilenameSuffix, mPrefs);
		try {
		addAndWriteRandomWords(testFilenameSuffix, numberOfWords, random,
		true /* checksContents */);
		dict.close();
		} finally {
		try {
		Log.d(TAG, "waiting for writing ...");
		dict.shutdownExecutorForTests();
		while (!dict.isTerminatedForTests()) {
		Thread.sleep(WAIT_TERMINATING_IN_MILLISECONDS);
		}
		} catch (InterruptedException e) {
		Log.d(TAG, "InterruptedException: ", e);
		}
		final String fileName = UserHistoryPredictionDictionary.NAME + "." + testFilenameSuffix
		+ ExpandableBinaryDictionary.DICT_FILE_EXTENSION;
		dictFile = new File(getContext().getFilesDir(), fileName);
		if (dictFile != null) {
		assertTrue(dictFile.exists());
		assertTrue(dictFile.length() >= MIN_USER_HISTORY_DICTIONARY_FILE_SIZE);
		dictFile.delete();
		}
		}
		}

		}