Merge "Hook for fetching sync content from UserHistoryDict" (2bdd5290) · Commits · e / os / android_packages_inputmethods_LatinIME

java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java

+37 −0

Original line number	Diff line number	Diff line
		@@ -77,6 +77,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {

		private static final int DICTIONARY_FORMAT_VERSION = FormatSpec.VERSION4;

		private static final WordProperty[] DEFAULT_WORD_PROPERTIES_FOR_SYNC =
		new WordProperty[0] /* default */;

		/** The application context. */
		protected final Context mContext;

		@@ -802,4 +805,38 @@ abstract public class ExpandableBinaryDictionary extends Dictionary {
		}
		});
		}

		/**
		* Returns dictionary content required for syncing.
		*/
		public WordProperty[] getWordPropertiesForSyncing() {
		reloadDictionaryIfRequired();
		final AsyncResultHolder<WordProperty[]> result = new AsyncResultHolder<>();
		asyncExecuteTaskWithLock(mLock.readLock(), "sync-read", new Runnable() {
		@Override
		public void run() {
		final ArrayList<WordProperty> wordPropertyList = new ArrayList<>();
		final BinaryDictionary binaryDictionary = getBinaryDictionary();
		if (binaryDictionary == null) {
		return;
		}
		int token = 0;
		do {
		// TODO: We need a new API that returns new un-synced data.
		final BinaryDictionary.GetNextWordPropertyResult result =
		binaryDictionary.getNextWordProperty(token);
		final WordProperty wordProperty = result.mWordProperty;
		if (wordProperty == null) {
		break;
		}
		wordPropertyList.add(wordProperty);
		token = result.mNextToken;
		} while (token != 0);
		result.set(wordPropertyList.toArray(new WordProperty[wordPropertyList.size()]));
		}
		});
		// TODO: Figure out the best timeout duration for this API.
		return result.get(DEFAULT_WORD_PROPERTIES_FOR_SYNC,
		TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS);
		}
		}

java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java

+1 −2

Original line number	Diff line number	Diff line
		@@ -58,8 +58,7 @@ public class PersonalizationHelper {
		final UserHistoryDictionary dict = ref == null ? null : ref.get();
		if (dict != null) {
		if (DEBUG) {
		Log.d(TAG, "Use cached UserHistoryDictionary for " + locale +
		" & account" + accountName);
		Log.d(TAG, "Use cached UserHistoryDictionary with lookup: " + lookupStr);
		}
		dict.reloadDictionaryIfRequired();
		return dict;

tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java

+31 −105

Original line number	Diff line number	Diff line
		@@ -25,15 +25,11 @@ import android.util.Log;
		import com.android.inputmethod.latin.ExpandableBinaryDictionary;
		import com.android.inputmethod.latin.NgramContext;
		import com.android.inputmethod.latin.NgramContext.WordInfo;
		import com.android.inputmethod.latin.common.FileUtils;
		import com.android.inputmethod.latin.settings.LocalSettingsConstants;
		import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;
		import com.android.inputmethod.latin.utils.DistracterFilter;

		import java.io.File;
		import java.io.FilenameFilter;
		import java.util.ArrayList;
		import java.util.HashSet;
		import java.util.List;
		import java.util.Locale;
		import java.util.Random;
		@@ -48,34 +44,13 @@ import javax.annotation.Nullable;
		public class UserHistoryDictionaryTests extends AndroidTestCase {
		private static final String TAG = UserHistoryDictionaryTests.class.getSimpleName();
		private static final int WAIT_FOR_WRITING_FILE_IN_MILLISECONDS = 3000;
		private static final String TEST_LOCALE_PREFIX = "test_";
		private static final String TEST_ACCOUNT = "account@example.com";

		private static final String[] CHARACTERS = {
		"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
		"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
		};

		private int mCurrentTime = 0;

		private SharedPreferences mPrefs;
		private String mLastKnownAccount = null;

		private void removeAllTestDictFiles() {
		final Locale dummyLocale = new Locale(TEST_LOCALE_PREFIX);
		final String dictName = UserHistoryDictionary.getUserHistoryDictName(
		UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
		final File dictFile = ExpandableBinaryDictionary.getDictFile(
		mContext, dictName, null /* dictFile */);
		final FilenameFilter filenameFilter = new FilenameFilter() {
		@Override
		public boolean accept(final File dir, final String filename) {
		return filename.startsWith(UserHistoryDictionary.NAME + "." + TEST_LOCALE_PREFIX);
		}
		};
		FileUtils.deleteFilteredFiles(dictFile.getParentFile(), filenameFilter);
		}

		private static void printAllFiles(final File dir) {
		Log.d(TAG, dir.getAbsolutePath());
		for (final File file : dir.listFiles()) {
		@@ -83,7 +58,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
		}
		}

		private static void checkExistenceAndRemoveDictFile(final UserHistoryDictionary dict,
		private static void assertDictionaryExists(final UserHistoryDictionary dict,
		final File dictFile) {
		Log.d(TAG, "waiting for writing ...");
		dict.waitAllTasksForTests();
		@@ -97,12 +72,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
		Log.e(TAG, "Interrupted during waiting for writing the dict file.");
		}
		}
		assertTrue("check exisiting of " + dictFile, dictFile.exists());
		FileUtils.deleteRecursively(dictFile);
		}

		private static Locale getDummyLocale(final String name) {
		return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());
		assertTrue("Following dictionary file doesn't exist: " + dictFile, dictFile.exists());
		}

		@Override
		@@ -115,12 +85,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
		updateAccountName(TEST_ACCOUNT);

		resetCurrentTimeForTestMode();
		removeAllTestDictFiles();
		UserHistoryDictionaryTestsHelper.removeAllTestDictFiles(
		UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext);
		}

		@Override
		protected void tearDown() throws Exception {
		removeAllTestDictFiles();
		UserHistoryDictionaryTestsHelper.removeAllTestDictFiles(
		UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext);
		stopTestModeInNativeCode();

		// Restore the account that was present before running the test.
		@@ -164,58 +136,6 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
		return BinaryDictionaryUtils.setCurrentTimeForTest(-1);
		}

		/**
		* Generates a random word.
		*/
		private static String generateWord(final int value) {
		final int lengthOfChars = CHARACTERS.length;
		final StringBuilder builder = new StringBuilder();
		long lvalue = Math.abs((long)value);
		while (lvalue > 0) {
		builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);
		lvalue /= lengthOfChars;
		}
		return builder.toString();
		}

		private static List<String> generateWords(final int number, final Random random) {
		final HashSet<String> wordSet = new HashSet<>();
		while (wordSet.size() < number) {
		wordSet.add(generateWord(random.nextInt()));
		}
		return new ArrayList<>(wordSet);
		}

		private static void addToDict(final UserHistoryDictionary dict, final List<String> words,
		final int timestamp) {
		NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
		for (final String word : words) {
		UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,
		DistracterFilter.EMPTY_DISTRACTER_FILTER);
		ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
		}
		}

		/**
		* @param checkContents if true, checks whether written words are actually in the dictionary
		* or not.
		*/
		private void addAndWriteRandomWords(final UserHistoryDictionary dict,
		final int numberOfWords, final Random random, final boolean checkContents) {
		final List<String> words = generateWords(numberOfWords, random);
		// Add random words to the user history dictionary.
		addToDict(dict, words, mCurrentTime);
		if (checkContents) {
		dict.waitAllTasksForTests();
		for (int i = 0; i < numberOfWords; ++i) {
		final String word = words.get(i);
		assertTrue(dict.isInDictionary(word));
		}
		}
		// write to file.
		dict.close();
		}

		/**
		* Clear all entries in the user history dictionary.
		* @param dict the user history dictionary.
		@@ -230,19 +150,19 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
		public void testRandomWords() {
		Log.d(TAG, "This test can be used for profiling.");
		Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true.");
		final Locale dummyLocale = getDummyLocale("random_words");
		final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("random_words");
		final String dictName = UserHistoryDictionary.getUserHistoryDictName(
		UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
		final File dictFile = ExpandableBinaryDictionary.getDictFile(
		mContext, dictName, null /* dictFile */);
		final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
		getContext(), dummyLocale, TEST_ACCOUNT);

		clearHistory(dict);
		final int numberOfWords = 1000;
		final Random random = new Random(123456);
		clearHistory(dict);
		addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);
		checkExistenceAndRemoveDictFile(dict, dictFile);
		assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(
		dict, numberOfWords, random, true /* checksContents */, mCurrentTime));
		assertDictionaryExists(dict, dictFile);
		}

		public void testStressTestForSwitchingLanguagesAndAddingWords() {
		@@ -258,7 +178,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {

		// Create filename suffixes for this test.
		for (int i = 0; i < numberOfLanguages; i++) {
		final Locale dummyLocale = getDummyLocale("switching_languages" + i);
		final Locale dummyLocale =
		UserHistoryDictionaryTestsHelper.getDummyLocale("switching_languages" + i);
		final String dictName = UserHistoryDictionary.getUserHistoryDictName(
		UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
		dictFiles[i] = ExpandableBinaryDictionary.getDictFile(
		@@ -273,8 +194,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
		for (int i = 0; i < numberOfLanguageSwitching; i++) {
		final int index = i % numberOfLanguages;
		// Switch to dicts[index].
		addAndWriteRandomWords(dicts[index], numberOfWordsInsertedForEachLanguageSwitch,
		random, false /* checksContents */);
		assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dicts[index],
		numberOfWordsInsertedForEachLanguageSwitch,
		random,
		false /* checksContents */,
		mCurrentTime));
		}

		final long end = System.currentTimeMillis();
		@@ -282,13 +206,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
		+ (end - start) + " ms");
		} finally {
		for (int i = 0; i < numberOfLanguages; i++) {
		checkExistenceAndRemoveDictFile(dicts[i], dictFiles[i]);
		assertDictionaryExists(dicts[i], dictFiles[i]);
		}
		}
		}

		public void testAddManyWords() {
		final Locale dummyLocale = getDummyLocale("many_random_words");
		final Locale dummyLocale =
		UserHistoryDictionaryTestsHelper.getDummyLocale("many_random_words");
		final String dictName = UserHistoryDictionary.getUserHistoryDictName(
		UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());
		final File dictFile = ExpandableBinaryDictionary.getDictFile(
		@@ -298,23 +223,23 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
		final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
		getContext(), dummyLocale, TEST_ACCOUNT);
		clearHistory(dict);
		try {
		addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);
		} finally {
		checkExistenceAndRemoveDictFile(dict, dictFile);
		}
		assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dict,
		numberOfWords, random, true /* checksContents */, mCurrentTime));
		assertDictionaryExists(dict, dictFile);
		}

		public void testDecaying() {
		final Locale dummyLocale = getDummyLocale("decaying");
		final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("decaying");
		final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(
		getContext(), dummyLocale, TEST_ACCOUNT);
		final int numberOfWords = 5000;
		final Random random = new Random(123456);
		resetCurrentTimeForTestMode();
		clearHistory(dict);
		final List<String> words = generateWords(numberOfWords, random);
		dict.waitAllTasksForTests();

		final int numberOfWords = 5000;
		final Random random = new Random(123456);
		final List<String> words = UserHistoryDictionaryTestsHelper.generateWords(numberOfWords,
		random);
		NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
		for (final String word : words) {
		UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime,
		@@ -329,6 +254,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase {
		for (final String word : words) {
		assertTrue(dict.isInDictionary(word));
		}
		// Long term decay results in words removed from the dictionary.
		forcePassingLongTime();
		dict.runGCIfRequired();
		dict.waitAllTasksForTests();

tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java

0 → 100644

+144 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2014 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.inputmethod.latin.personalization;

		import android.content.Context;

		import com.android.inputmethod.latin.NgramContext;
		import com.android.inputmethod.latin.NgramContext.WordInfo;
		import com.android.inputmethod.latin.common.FileUtils;
		import com.android.inputmethod.latin.utils.DistracterFilter;

		import java.io.File;
		import java.io.FilenameFilter;
		import java.util.ArrayList;
		import java.util.HashSet;
		import java.util.List;
		import java.util.Locale;
		import java.util.Random;

		/**
		* Utility class for helping while running tests involving {@link UserHistoryDictionary}.
		*/
		public class UserHistoryDictionaryTestsHelper {

		/**
		* Locale prefix for generating dummy locales for tests.
		*/
		public static final String TEST_LOCALE_PREFIX = "test-";

		/**
		* Characters for generating random words.
		*/
		private static final String[] CHARACTERS = {
		"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
		"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
		};

		/**
		* Remove all the test dictionary files created for the given locale.
		*/
		public static void removeAllTestDictFiles(final String filter, final Context context) {
		final FilenameFilter filenameFilter = new FilenameFilter() {
		@Override
		public boolean accept(final File dir, final String filename) {
		return filename.startsWith(UserHistoryDictionary.NAME + "." + filter);
		}
		};
		FileUtils.deleteFilteredFiles(context.getFilesDir(), filenameFilter);
		}

		/**
		* Generates and writes random words to dictionary. Caller can be assured
		* that the write tasks would be finished; and its success would be reflected
		* in the returned boolean.
		*
		* @param dict {@link UserHistoryDictionary} to which words should be added.
		* @param numberOfWords number of words to be added.
		* @param random helps generate random words.
		* @param checkContents if true, checks whether written words are actually in the dictionary.
		* @param currentTime timestamp that would be used for adding the words.
		* @returns true if all words have been written to dictionary successfully.
		*/
		public static boolean addAndWriteRandomWords(final UserHistoryDictionary dict,
		final int numberOfWords, final Random random, final boolean checkContents,
		final int currentTime) {
		final List<String> words = generateWords(numberOfWords, random);
		// Add random words to the user history dictionary.
		addWordsToDictionary(dict, words, currentTime);
		boolean success = true;
		if (checkContents) {
		dict.waitAllTasksForTests();
		for (int i = 0; i < numberOfWords; ++i) {
		final String word = words.get(i);
		if (!dict.isInDictionary(word)) {
		success = false;
		break;
		}
		}
		}
		// write to file.
		dict.close();
		dict.waitAllTasksForTests();
		return success;
		}

		private static void addWordsToDictionary(final UserHistoryDictionary dict,
		final List<String> words, final int timestamp) {
		NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;
		for (final String word : words) {
		UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,
		DistracterFilter.EMPTY_DISTRACTER_FILTER);
		ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));
		}
		}

		/**
		* Creates unique test locale for using within tests.
		*/
		public static Locale getDummyLocale(final String name) {
		return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());
		}

		/**
		* Generates random words.
		*
		* @param numberOfWords number of words to generate.
		* @param random salt used for generating random words.
		*/
		public static List<String> generateWords(final int numberOfWords, final Random random) {
		final HashSet<String> wordSet = new HashSet<>();
		while (wordSet.size() < numberOfWords) {
		wordSet.add(generateWord(random.nextInt()));
		}
		return new ArrayList<>(wordSet);
		}

		/**
		* Generates a random word.
		*/
		private static String generateWord(final int value) {
		final int lengthOfChars = CHARACTERS.length;
		final StringBuilder builder = new StringBuilder();
		long lvalue = Math.abs((long)value);
		while (lvalue > 0) {
		builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);
		lvalue /= lengthOfChars;
		}
		return builder.toString();
		}
		}