Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 12d63820 authored by Jatin Matani's avatar Jatin Matani
Browse files

Hook for fetching sync content from UserHistoryDict 

Add API to ExpandableBinaryDictionary to dump content
from a given dictionary. We use this for dumping data
for sync process.

Refactored UserHistoryDictionaryTests to scrap out the util
methods for testing. These utility methods would be used
for testing sync + user dictionary code in LatinIMEGoogleTests

Bug:18106539
Change-Id: I357f9192ea1bd69a526d0b620c25616a2e8e9d5b
parent 4195567d

java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java

+37 −0
Original line number Diff line number Diff line
@@ -77,6 +77,9 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { 


    private static final int DICTIONARY_FORMAT_VERSION = FormatSpec.VERSION4;



    private static final WordProperty[] DEFAULT_WORD_PROPERTIES_FOR_SYNC =

            new WordProperty[0] /* default */;



    /** The application context. */

    protected final Context mContext;
@@ -802,4 +805,38 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { 
            }

        });

    }



    /**

     * Returns dictionary content required for syncing.

     */

    public WordProperty[] getWordPropertiesForSyncing() {

        reloadDictionaryIfRequired();

        final AsyncResultHolder<WordProperty[]> result = new AsyncResultHolder<>();

        asyncExecuteTaskWithLock(mLock.readLock(), "sync-read", new Runnable() {

            @Override

            public void run() {

                final ArrayList<WordProperty> wordPropertyList = new ArrayList<>();

                final BinaryDictionary binaryDictionary = getBinaryDictionary();

                if (binaryDictionary == null) {

                    return;

                }

                int token = 0;

                do {

                    // TODO: We need a new API that returns *new* un-synced data.

                    final BinaryDictionary.GetNextWordPropertyResult result =

                            binaryDictionary.getNextWordProperty(token);

                    final WordProperty wordProperty = result.mWordProperty;

                    if (wordProperty == null) {

                        break;

                    }

                    wordPropertyList.add(wordProperty);

                    token = result.mNextToken;

                } while (token != 0);

                result.set(wordPropertyList.toArray(new WordProperty[wordPropertyList.size()]));

            }

        });

        // TODO: Figure out the best timeout duration for this API.

        return result.get(DEFAULT_WORD_PROPERTIES_FOR_SYNC,

                TIMEOUT_FOR_READ_OPS_IN_MILLISECONDS);

    }

}

java/src/com/android/inputmethod/latin/personalization/PersonalizationHelper.java

+1 −2
Original line number Diff line number Diff line
@@ -58,8 +58,7 @@ public class PersonalizationHelper { 
                final UserHistoryDictionary dict = ref == null ? null : ref.get();

                if (dict != null) {

                    if (DEBUG) {

                        Log.d(TAG, "Use cached UserHistoryDictionary for " + locale +

                                " & account" + accountName);

                        Log.d(TAG, "Use cached UserHistoryDictionary with lookup: " + lookupStr);

                    }

                    dict.reloadDictionaryIfRequired();

                    return dict;

tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTests.java

+31 −105
Original line number Diff line number Diff line
@@ -25,15 +25,11 @@ import android.util.Log; 
import com.android.inputmethod.latin.ExpandableBinaryDictionary;

import com.android.inputmethod.latin.NgramContext;

import com.android.inputmethod.latin.NgramContext.WordInfo;

import com.android.inputmethod.latin.common.FileUtils;

import com.android.inputmethod.latin.settings.LocalSettingsConstants;

import com.android.inputmethod.latin.utils.BinaryDictionaryUtils;

import com.android.inputmethod.latin.utils.DistracterFilter;



import java.io.File;

import java.io.FilenameFilter;

import java.util.ArrayList;

import java.util.HashSet;

import java.util.List;

import java.util.Locale;

import java.util.Random;
@@ -48,34 +44,13 @@ import javax.annotation.Nullable; 
public class UserHistoryDictionaryTests extends AndroidTestCase {

    private static final String TAG = UserHistoryDictionaryTests.class.getSimpleName();

    private static final int WAIT_FOR_WRITING_FILE_IN_MILLISECONDS = 3000;

    private static final String TEST_LOCALE_PREFIX = "test_";

    private static final String TEST_ACCOUNT = "account@example.com";



    private static final String[] CHARACTERS = {

        "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",

        "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"

    };



    private int mCurrentTime = 0;



    private SharedPreferences mPrefs;

    private String mLastKnownAccount = null;



    private void removeAllTestDictFiles() {

        final Locale dummyLocale = new Locale(TEST_LOCALE_PREFIX);

        final String dictName = UserHistoryDictionary.getUserHistoryDictName(

                UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());

        final File dictFile = ExpandableBinaryDictionary.getDictFile(

                mContext, dictName, null /* dictFile */);

        final FilenameFilter filenameFilter = new FilenameFilter() {

            @Override

            public boolean accept(final File dir, final String filename) {

                return filename.startsWith(UserHistoryDictionary.NAME + "." + TEST_LOCALE_PREFIX);

            }

        };

        FileUtils.deleteFilteredFiles(dictFile.getParentFile(), filenameFilter);

    }



    private static void printAllFiles(final File dir) {

        Log.d(TAG, dir.getAbsolutePath());

        for (final File file : dir.listFiles()) {
@@ -83,7 +58,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { 
        }

    }



    private static void checkExistenceAndRemoveDictFile(final UserHistoryDictionary dict,

    private static void assertDictionaryExists(final UserHistoryDictionary dict,

            final File dictFile) {

        Log.d(TAG, "waiting for writing ...");

        dict.waitAllTasksForTests();
@@ -97,12 +72,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { 
                Log.e(TAG, "Interrupted during waiting for writing the dict file.");

            }

        }

        assertTrue("check exisiting of " + dictFile, dictFile.exists());

        FileUtils.deleteRecursively(dictFile);

    }



    private static Locale getDummyLocale(final String name) {

        return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());

        assertTrue("Following dictionary file doesn't exist: " + dictFile, dictFile.exists());

    }



    @Override
@@ -115,12 +85,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { 
        updateAccountName(TEST_ACCOUNT);



        resetCurrentTimeForTestMode();

        removeAllTestDictFiles();

        UserHistoryDictionaryTestsHelper.removeAllTestDictFiles(

                UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext);

    }



    @Override

    protected void tearDown() throws Exception {

        removeAllTestDictFiles();

        UserHistoryDictionaryTestsHelper.removeAllTestDictFiles(

                UserHistoryDictionaryTestsHelper.TEST_LOCALE_PREFIX, mContext);

        stopTestModeInNativeCode();



        // Restore the account that was present before running the test.
@@ -164,58 +136,6 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { 
        return BinaryDictionaryUtils.setCurrentTimeForTest(-1);

    }



    /**

     * Generates a random word.

     */

    private static String generateWord(final int value) {

        final int lengthOfChars = CHARACTERS.length;

        final StringBuilder builder = new StringBuilder();

        long lvalue = Math.abs((long)value);

        while (lvalue > 0) {

            builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);

            lvalue /= lengthOfChars;

        }

        return builder.toString();

    }



    private static List<String> generateWords(final int number, final Random random) {

        final HashSet<String> wordSet = new HashSet<>();

        while (wordSet.size() < number) {

            wordSet.add(generateWord(random.nextInt()));

        }

        return new ArrayList<>(wordSet);

    }



    private static void addToDict(final UserHistoryDictionary dict, final List<String> words,

            final int timestamp) {

        NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;

        for (final String word : words) {

            UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,

                    DistracterFilter.EMPTY_DISTRACTER_FILTER);

            ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));

        }

    }



    /**

     * @param checkContents if true, checks whether written words are actually in the dictionary

     * or not.

     */

    private void addAndWriteRandomWords(final UserHistoryDictionary dict,

            final int numberOfWords, final Random random, final boolean checkContents) {

        final List<String> words = generateWords(numberOfWords, random);

        // Add random words to the user history dictionary.

        addToDict(dict, words, mCurrentTime);

        if (checkContents) {

            dict.waitAllTasksForTests();

            for (int i = 0; i < numberOfWords; ++i) {

                final String word = words.get(i);

                assertTrue(dict.isInDictionary(word));

            }

        }

        // write to file.

        dict.close();

    }



    /**

     * Clear all entries in the user history dictionary.

     * @param dict the user history dictionary.
@@ -230,19 +150,19 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { 
    public void testRandomWords() {

        Log.d(TAG, "This test can be used for profiling.");

        Log.d(TAG, "Usage: please set UserHistoryDictionary.PROFILE_SAVE_RESTORE to true.");

        final Locale dummyLocale = getDummyLocale("random_words");

        final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("random_words");

        final String dictName = UserHistoryDictionary.getUserHistoryDictName(

                UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());

        final File dictFile = ExpandableBinaryDictionary.getDictFile(

                mContext, dictName, null /* dictFile */);

        final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(

                getContext(), dummyLocale, TEST_ACCOUNT);



        clearHistory(dict);

        final int numberOfWords = 1000;

        final Random random = new Random(123456);

        clearHistory(dict);

        addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);

        checkExistenceAndRemoveDictFile(dict, dictFile);

        assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(

                dict, numberOfWords, random, true /* checksContents */, mCurrentTime));

        assertDictionaryExists(dict, dictFile);

    }



    public void testStressTestForSwitchingLanguagesAndAddingWords() {
@@ -258,7 +178,8 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { 


            // Create filename suffixes for this test.

            for (int i = 0; i < numberOfLanguages; i++) {

                final Locale dummyLocale = getDummyLocale("switching_languages" + i);

                final Locale dummyLocale =

                        UserHistoryDictionaryTestsHelper.getDummyLocale("switching_languages" + i);

                final String dictName = UserHistoryDictionary.getUserHistoryDictName(

                        UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());

                dictFiles[i] = ExpandableBinaryDictionary.getDictFile(
@@ -273,8 +194,11 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { 
            for (int i = 0; i < numberOfLanguageSwitching; i++) {

                final int index = i % numberOfLanguages;

                // Switch to dicts[index].

                addAndWriteRandomWords(dicts[index], numberOfWordsInsertedForEachLanguageSwitch,

                        random, false /* checksContents */);

                assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dicts[index],

                        numberOfWordsInsertedForEachLanguageSwitch,

                        random,

                        false /* checksContents */,

                        mCurrentTime));

            }



            final long end = System.currentTimeMillis();
@@ -282,13 +206,14 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { 
                    + (end - start) + " ms");

        } finally {

            for (int i = 0; i < numberOfLanguages; i++) {

                checkExistenceAndRemoveDictFile(dicts[i], dictFiles[i]);

                assertDictionaryExists(dicts[i], dictFiles[i]);

            }

        }

    }



    public void testAddManyWords() {

        final Locale dummyLocale = getDummyLocale("many_random_words");

        final Locale dummyLocale =

                UserHistoryDictionaryTestsHelper.getDummyLocale("many_random_words");

        final String dictName = UserHistoryDictionary.getUserHistoryDictName(

                UserHistoryDictionary.NAME, dummyLocale, null /* dictFile */, getContext());

        final File dictFile = ExpandableBinaryDictionary.getDictFile(
@@ -298,23 +223,23 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { 
        final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(

                getContext(), dummyLocale, TEST_ACCOUNT);

        clearHistory(dict);

        try {

            addAndWriteRandomWords(dict, numberOfWords, random, true /* checksContents */);

        } finally {

            checkExistenceAndRemoveDictFile(dict, dictFile);

        }

        assertTrue(UserHistoryDictionaryTestsHelper.addAndWriteRandomWords(dict,

                numberOfWords, random, true /* checksContents */, mCurrentTime));

        assertDictionaryExists(dict, dictFile);

    }



    public void testDecaying() {

        final Locale dummyLocale = getDummyLocale("decaying");

        final Locale dummyLocale = UserHistoryDictionaryTestsHelper.getDummyLocale("decaying");

        final UserHistoryDictionary dict = PersonalizationHelper.getUserHistoryDictionary(

                getContext(), dummyLocale, TEST_ACCOUNT);

        final int numberOfWords = 5000;

        final Random random = new Random(123456);

        resetCurrentTimeForTestMode();

        clearHistory(dict);

        final List<String> words = generateWords(numberOfWords, random);

        dict.waitAllTasksForTests();



        final int numberOfWords = 5000;

        final Random random = new Random(123456);

        final List<String> words = UserHistoryDictionaryTestsHelper.generateWords(numberOfWords,

                random);

        NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;

        for (final String word : words) {

            UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, mCurrentTime,
@@ -329,6 +254,7 @@ public class UserHistoryDictionaryTests extends AndroidTestCase { 
        for (final String word : words) {

            assertTrue(dict.isInDictionary(word));

        }

        // Long term decay results in words removed from the dictionary.

        forcePassingLongTime();

        dict.runGCIfRequired();

        dict.waitAllTasksForTests();

tests/src/com/android/inputmethod/latin/personalization/UserHistoryDictionaryTestsHelper.java

0 → 100644
+144 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2014 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.inputmethod.latin.personalization;



import android.content.Context;



import com.android.inputmethod.latin.NgramContext;

import com.android.inputmethod.latin.NgramContext.WordInfo;

import com.android.inputmethod.latin.common.FileUtils;

import com.android.inputmethod.latin.utils.DistracterFilter;



import java.io.File;

import java.io.FilenameFilter;

import java.util.ArrayList;

import java.util.HashSet;

import java.util.List;

import java.util.Locale;

import java.util.Random;



/**

 * Utility class for helping while running tests involving {@link UserHistoryDictionary}.

 */

public class UserHistoryDictionaryTestsHelper {



    /**

     * Locale prefix for generating dummy locales for tests.

     */

    public static final String TEST_LOCALE_PREFIX = "test-";



    /**

     * Characters for generating random words.

     */

    private static final String[] CHARACTERS = {

        "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",

        "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"

    };



    /**

     * Remove all the test dictionary files created for the given locale.

     */

    public static void removeAllTestDictFiles(final String filter, final Context context) {

        final FilenameFilter filenameFilter = new FilenameFilter() {

            @Override

            public boolean accept(final File dir, final String filename) {

                return filename.startsWith(UserHistoryDictionary.NAME + "." + filter);

            }

        };

        FileUtils.deleteFilteredFiles(context.getFilesDir(), filenameFilter);

    }



    /**

     * Generates and writes random words to dictionary. Caller can be assured

     * that the write tasks would be finished; and its success would be reflected

     * in the returned boolean.

     *

     * @param dict {@link UserHistoryDictionary} to which words should be added.

     * @param numberOfWords number of words to be added.

     * @param random helps generate random words.

     * @param checkContents if true, checks whether written words are actually in the dictionary.

     * @param currentTime timestamp that would be used for adding the words.

     * @returns true if all words have been written to dictionary successfully.

     */

    public static boolean addAndWriteRandomWords(final UserHistoryDictionary dict,

            final int numberOfWords, final Random random, final boolean checkContents,

            final int currentTime) {

        final List<String> words = generateWords(numberOfWords, random);

        // Add random words to the user history dictionary.

        addWordsToDictionary(dict, words, currentTime);

        boolean success = true;

        if (checkContents) {

            dict.waitAllTasksForTests();

            for (int i = 0; i < numberOfWords; ++i) {

                final String word = words.get(i);

                if (!dict.isInDictionary(word)) {

                    success = false;

                    break;

                }

            }

        }

        // write to file.

        dict.close();

        dict.waitAllTasksForTests();

        return success;

    }



    private static void addWordsToDictionary(final UserHistoryDictionary dict,

            final List<String> words, final int timestamp) {

        NgramContext ngramContext = NgramContext.EMPTY_PREV_WORDS_INFO;

        for (final String word : words) {

            UserHistoryDictionary.addToDictionary(dict, ngramContext, word, true, timestamp,

                    DistracterFilter.EMPTY_DISTRACTER_FILTER);

            ngramContext = ngramContext.getNextNgramContext(new WordInfo(word));

        }

    }



    /**

     * Creates unique test locale for using within tests.

     */

    public static Locale getDummyLocale(final String name) {

        return new Locale(TEST_LOCALE_PREFIX + name + System.currentTimeMillis());

    }



    /**

     * Generates random words.

     *

     * @param numberOfWords number of words to generate.

     * @param random salt used for generating random words.

     */

    public static List<String> generateWords(final int numberOfWords, final Random random) {

        final HashSet<String> wordSet = new HashSet<>();

        while (wordSet.size() < numberOfWords) {

            wordSet.add(generateWord(random.nextInt()));

        }

        return new ArrayList<>(wordSet);

    }



    /**

     * Generates a random word.

     */

    private static String generateWord(final int value) {

        final int lengthOfChars = CHARACTERS.length;

        final StringBuilder builder = new StringBuilder();

        long lvalue = Math.abs((long)value);

        while (lvalue > 0) {

            builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);

            lvalue /= lengthOfChars;

        }

        return builder.toString();

    }

}