Use BinaryDictInputOutput to save UserHistoryDictionary. (84d858ed) · Commits · e / os / android_packages_inputmethods_LatinIME

java/src/com/android/inputmethod/latin/UserHistoryDictIOUtils.java

+15 −13

Original line number	Diff line number	Diff line
		@@ -115,11 +115,10 @@ public class UserHistoryDictIOUtils {
		public static void writeDictionaryBinary(final OutputStream destination,
		final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams,
		final FormatOptions formatOptions) {

		final FusionDictionary fusionDict = constructFusionDictionary(dict, bigrams);

		try {
		BinaryDictInputOutput.writeDictionaryBinary(destination, fusionDict, formatOptions);
		Log.d(TAG, "end writing");
		} catch (IOException e) {
		Log.e(TAG, "IO exception while writing file: " + e);
		} catch (UnsupportedFormatException e) {
		@@ -132,16 +131,18 @@ public class UserHistoryDictIOUtils {
		*/
		/* packages for test */ static FusionDictionary constructFusionDictionary(
		final BigramDictionaryInterface dict, final UserHistoryDictionaryBigramList bigrams) {

		final FusionDictionary fusionDict = new FusionDictionary(new Node(),
		new FusionDictionary.DictionaryOptions(
		new HashMap<String,String>(), false, false));

		new FusionDictionary.DictionaryOptions(new HashMap<String, String>(), false,
		false));
		int profTotal = 0;
		for (final String word1 : bigrams.keySet()) {
		final HashMap<String, Byte> word1Bigrams = bigrams.getBigrams(word1);
		for (final String word2 : word1Bigrams.keySet()) {
		final int freq = dict.getFrequency(word1, word2);

		if (freq == -1) {
		// don't add this bigram.
		continue;
		}
		if (DEBUG) {
		if (word1 == null) {
		Log.d(TAG, "add unigram: " + word2 + "," + Integer.toString(freq));
		@@ -149,17 +150,22 @@ public class UserHistoryDictIOUtils {
		Log.d(TAG, "add bigram: " + word1
		+ "," + word2 + "," + Integer.toString(freq));
		}
		profTotal++;
		}

		if (word1 == null) { // unigram
		fusionDict.add(word2, freq, null, false /* isNotAWord */);
		} else { // bigram
		if (FusionDictionary.findWordInTree(fusionDict.mRoot, word1) == null) {
		fusionDict.add(word1, 2, null, false /* isNotAWord */);
		}
		fusionDict.setBigram(word1, word2, freq);
		}
		bigrams.updateBigram(word1, word2, (byte)freq);
		}
		}

		if (DEBUG) {
		Log.d(TAG, "add " + profTotal + "words");
		}
		return fusionDict;
		}

		@@ -171,7 +177,6 @@ public class UserHistoryDictIOUtils {
		final Map<Integer, String> unigrams = CollectionUtils.newTreeMap();
		final Map<Integer, Integer> frequencies = CollectionUtils.newTreeMap();
		final Map<Integer, ArrayList<PendingAttribute>> bigrams = CollectionUtils.newTreeMap();

		try {
		BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, unigrams, frequencies,
		bigrams);
		@@ -189,14 +194,11 @@ public class UserHistoryDictIOUtils {
		/* package for test */ static void addWordsFromWordMap(final Map<Integer, String> unigrams,
		final Map<Integer, Integer> frequencies,
		final Map<Integer, ArrayList<PendingAttribute>> bigrams, final OnAddWordListener to) {

		for (Map.Entry<Integer, String> entry : unigrams.entrySet()) {
		final String word1 = entry.getValue();
		final int unigramFrequency = frequencies.get(entry.getKey());
		to.setUnigram(word1, null, unigramFrequency);

		final ArrayList<PendingAttribute> attrList = bigrams.get(entry.getKey());

		if (attrList != null) {
		for (final PendingAttribute attr : attrList) {
		to.setBigram(word1, unigrams.get(attr.mAddress),

java/src/com/android/inputmethod/latin/UserHistoryDictionary.java

+142 −340

File changed.

Preview size limit exceeded, changes collapsed.

java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java

+4 −1

Original line number	Diff line number	Diff line
		@@ -77,7 +77,10 @@ public class BinaryDictIOUtils {
		p.mAddress += BinaryDictInputOutput.getGroupCountSize(p.mNumOfCharGroup);
		p.mPosition = 0;
		}

		if (p.mNumOfCharGroup == 0) {
		stack.pop();
		continue;
		}
		CharGroupInfo info = BinaryDictInputOutput.readCharGroup(buffer,
		p.mAddress - headerSize, formatOptions);
		for (int i = 0; i < info.mCharacters.length; ++i) {

tests/src/com/android/inputmethod/latin/UserHistoryDictionaryTests.java

0 → 100644

+109 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2012 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.inputmethod.latin;

		import com.android.inputmethod.latin.UserHistoryDictionary;

		import android.content.SharedPreferences;
		import android.preference.PreferenceManager;
		import android.test.AndroidTestCase;
		import android.util.Log;

		import java.util.ArrayList;
		import java.util.List;
		import java.util.Random;
		import java.util.Set;

		/**
		* Unit tests for UserHistoryDictionary
		*/
		public class UserHistoryDictionaryTests extends AndroidTestCase {
		private static final String TAG = UserHistoryDictionaryTests.class.getSimpleName();
		private SharedPreferences mPrefs;

		private static final String[] CHARACTERS = {
		"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
		"n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
		};

		@Override
		public void setUp() {
		mPrefs = PreferenceManager.getDefaultSharedPreferences(getContext());
		}

		/**
		* Generates a random word.
		*/
		private String generateWord(final int value) {
		final int lengthOfChars = CHARACTERS.length;
		StringBuilder builder = new StringBuilder();
		long lvalue = Math.abs((long)value);
		while (lvalue > 0) {
		builder.append(CHARACTERS[(int)(lvalue % lengthOfChars)]);
		lvalue /= lengthOfChars;
		}
		return builder.toString();
		}

		private List<String> generateWords(final int number, final Random random) {
		final Set<String> wordSet = CollectionUtils.newHashSet();
		while (wordSet.size() < number) {
		wordSet.add(generateWord(random.nextInt()));
		}
		return new ArrayList<String>(wordSet);
		}

		private void addToDict(final UserHistoryDictionary dict, final List<String> words) {
		String prevWord = null;
		for (String word : words) {
		dict.forceAddWordForTest(prevWord, word, true);
		prevWord = word;
		}
		}

		public void testRandomWords() {
		Log.d(TAG, "This test can be used for profiling.");
		Log.d(TAG, "Usage: please set UserHisotoryDictionary.PROFILE_SAVE_RESTORE to true.");
		final int numberOfWords = 1000;
		final Random random = new Random(123456);
		List<String> words = generateWords(numberOfWords, random);

		final String locale = "testRandomWords";
		final UserHistoryDictionary dict = UserHistoryDictionary.getInstance(getContext(),
		locale, mPrefs);
		dict.isTest = true;

		addToDict(dict, words);

		try {
		Log.d(TAG, "waiting for adding the word ...");
		Thread.sleep(2000);
		} catch (InterruptedException e) {
		Log.d(TAG, "InterruptedException: " + e);
		}

		// write to file
		dict.close();

		try {
		Log.d(TAG, "waiting for writing ...");
		Thread.sleep(5000);
		} catch (InterruptedException e) {
		Log.d(TAG, "InterruptedException: " + e);
		}
		}
		}