Add BinaryDictReader. (3feacba1) · Commits · e / os / android_packages_inputmethods_LatinIME

java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java

+5 −5

Original line number	Diff line number	Diff line
		@@ -145,21 +145,21 @@ public final class BinaryDictIOUtils {
		* Reads unigrams and bigrams from the binary file.
		* Doesn't make the memory representation of the dictionary.
		*
		* @param buffer the buffer to read.
		* @param reader the reader.
		* @param words the map to store the address as a key and the word as a value.
		* @param frequencies the map to store the address as a key and the frequency as a value.
		* @param bigrams the map to store the address as a key and the list of address as a value.
		* @throws IOException
		* @throws UnsupportedFormatException
		*/
		public static void readUnigramsAndBigramsBinary(final FusionDictionaryBufferInterface buffer,
		public static void readUnigramsAndBigramsBinary(final BinaryDictReader reader,
		final Map<Integer, String> words, final Map<Integer, Integer> frequencies,
		final Map<Integer, ArrayList<PendingAttribute>> bigrams) throws IOException,
		UnsupportedFormatException {
		// Read header
		final FileHeader header = BinaryDictInputOutput.readHeader(buffer);
		readUnigramsAndBigramsBinaryInner(buffer, header.mHeaderSize, words, frequencies, bigrams,
		header.mFormatOptions);
		final FileHeader header = BinaryDictInputOutput.readHeader(reader.getBuffer());
		readUnigramsAndBigramsBinaryInner(reader.getBuffer(), header.mHeaderSize, words,
		frequencies, bigrams, header.mFormatOptions);
		}

		/**

java/src/com/android/inputmethod/latin/makedict/BinaryDictInputOutput.java

+14 −6

Original line number	Diff line number	Diff line
		@@ -66,6 +66,7 @@ public final class BinaryDictInputOutput {
		public void position(int newPosition);
		public void put(final byte b);
		public int limit();
		@UsedForTesting
		public int capacity();
		}

		@@ -1722,23 +1723,30 @@ public final class BinaryDictInputOutput {
		* FusionDictionary structure. The optional dict argument is an existing dictionary to
		* which words from the buffer should be added. If it is null, a new dictionary is created.
		*
		* @param buffer the buffer to read.
		* @param reader the reader.
		* @param dict an optional dictionary to add words to, or null.
		* @return the created (or merged) dictionary.
		*/
		@UsedForTesting
		public static FusionDictionary readDictionaryBinary(
		final FusionDictionaryBufferInterface buffer, final FusionDictionary dict)
		throws IOException, UnsupportedFormatException {
		public static FusionDictionary readDictionaryBinary(final BinaryDictReader reader,
		final FusionDictionary dict) throws FileNotFoundException, IOException,
		UnsupportedFormatException {
		// clear cache
		wordCache.clear();

		// if the buffer has not been opened, open the buffer with bytebuffer.
		if (reader.getBuffer() == null) reader.openBuffer(
		new BinaryDictReader.FusionDictionaryBufferFromByteBufferFactory());
		if (reader.getBuffer() == null) {
		MakedictLog.e("Cannot open the buffer");
		}

		// Read header
		final FileHeader header = readHeader(buffer);
		final FileHeader header = readHeader(reader.getBuffer());

		Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>();
		Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>();
		final Node root = readNode(buffer, header.mHeaderSize, reverseNodeMapping,
		final Node root = readNode(reader.getBuffer(), header.mHeaderSize, reverseNodeMapping,
		reverseGroupMapping, header.mFormatOptions);

		FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions);

java/src/com/android/inputmethod/latin/makedict/BinaryDictReader.java

0 → 100644

+109 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2013 The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		package com.android.inputmethod.latin.makedict;

		import com.android.inputmethod.annotations.UsedForTesting;
		import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
		import com.android.inputmethod.latin.utils.ByteArrayWrapper;

		import java.io.File;
		import java.io.FileInputStream;
		import java.io.FileNotFoundException;
		import java.io.IOException;
		import java.nio.ByteBuffer;
		import java.nio.channels.FileChannel;

		public class BinaryDictReader {

		public interface FusionDictionaryBufferFactory {
		public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
		throws FileNotFoundException, IOException;
		}

		/**
		* Creates FusionDictionaryBuffer from a ByteBuffer
		*/
		public static final class FusionDictionaryBufferFromByteBufferFactory
		implements FusionDictionaryBufferFactory {
		@Override
		public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
		throws FileNotFoundException, IOException {
		FileInputStream inStream = null;
		ByteBuffer buffer = null;
		try {
		inStream = new FileInputStream(file);
		buffer = inStream.getChannel().map(FileChannel.MapMode.READ_ONLY,
		0, file.length());
		} finally {
		if (inStream != null) {
		inStream.close();
		}
		}
		if (buffer != null) {
		return new BinaryDictInputOutput.ByteBufferWrapper(buffer);
		}
		return null;
		}
		}

		/**
		* Creates FusionDictionaryBuffer from a byte array
		*/
		public static final class FusionDictionaryBufferFromByteArrayFactory
		implements FusionDictionaryBufferFactory {
		@Override
		public FusionDictionaryBufferInterface getFusionDictionaryBuffer(final File file)
		throws FileNotFoundException, IOException {
		FileInputStream inStream = null;
		try {
		inStream = new FileInputStream(file);
		final byte[] array = new byte[(int) file.length()];
		inStream.read(array);
		return new ByteArrayWrapper(array);
		} finally {
		if (inStream != null) {
		inStream.close();
		}
		}
		}
		}

		private final File mDictionaryBinaryFile;
		private FusionDictionaryBufferInterface mFusionDictionaryBuffer;

		public BinaryDictReader(final File file) {
		mDictionaryBinaryFile = file;
		mFusionDictionaryBuffer = null;
		}

		public void openBuffer(final FusionDictionaryBufferFactory factory)
		throws FileNotFoundException, IOException {
		mFusionDictionaryBuffer = factory.getFusionDictionaryBuffer(mDictionaryBinaryFile);
		}

		public FusionDictionaryBufferInterface getBuffer() {
		return mFusionDictionaryBuffer;
		}

		@UsedForTesting
		public FusionDictionaryBufferInterface openAndGetBuffer(
		final FusionDictionaryBufferFactory factory)
		throws FileNotFoundException, IOException {
		openBuffer(factory);
		return getBuffer();
		}
		}

java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java

+5 −16

Original line number	Diff line number	Diff line
		@@ -28,9 +28,9 @@ import com.android.inputmethod.latin.ExpandableDictionary;
		import com.android.inputmethod.latin.LatinImeLogger;
		import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;
		import com.android.inputmethod.latin.WordComposer;
		import com.android.inputmethod.latin.makedict.BinaryDictReader;
		import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
		import com.android.inputmethod.latin.settings.Settings;
		import com.android.inputmethod.latin.utils.ByteArrayWrapper;
		import com.android.inputmethod.latin.utils.CollectionUtils;
		import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
		import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.BigramDictionaryInterface;
		@@ -39,7 +39,6 @@ import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils;
		import com.android.inputmethod.latin.utils.UserHistoryForgettingCurveUtils.ForgettingCurveParams;

		import java.io.File;
		import java.io.FileInputStream;
		import java.io.FileNotFoundException;
		import java.io.FileOutputStream;
		import java.io.IOException;
		@@ -234,27 +233,17 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableDictiona
		};

		// Load the dictionary from binary file
		FileInputStream inStream = null;
		final BinaryDictReader reader = new BinaryDictReader(
		new File(getContext().getFilesDir(), fileName));
		try {
		final File file = new File(getContext().getFilesDir(), fileName);
		final byte[] buffer = new byte[(int)file.length()];
		inStream = new FileInputStream(file);
		inStream.read(buffer);
		UserHistoryDictIOUtils.readDictionaryBinary(
		new ByteArrayWrapper(buffer), listener);
		reader.openBuffer(new BinaryDictReader.FusionDictionaryBufferFromByteArrayFactory());
		UserHistoryDictIOUtils.readDictionaryBinary(reader, listener);
		} catch (FileNotFoundException e) {
		// This is an expected condition: we don't have a user history dictionary for this
		// language yet. It will be created sometime later.
		} catch (IOException e) {
		Log.e(TAG, "IOException on opening a bytebuffer", e);
		} finally {
		if (inStream != null) {
		try {
		inStream.close();
		} catch (IOException e) {
		// do nothing
		}
		}
		if (PROFILE_SAVE_RESTORE) {
		final long diff = System.currentTimeMillis() - now;
		Log.d(TAG, "PROF: Load UserHistoryDictionary: "

java/src/com/android/inputmethod/latin/utils/UserHistoryDictIOUtils.java

+3 −3

Original line number	Diff line number	Diff line
		@@ -21,7 +21,7 @@ import android.util.Log;
		import com.android.inputmethod.annotations.UsedForTesting;
		import com.android.inputmethod.latin.makedict.BinaryDictIOUtils;
		import com.android.inputmethod.latin.makedict.BinaryDictInputOutput;
		import com.android.inputmethod.latin.makedict.BinaryDictInputOutput.FusionDictionaryBufferInterface;
		import com.android.inputmethod.latin.makedict.BinaryDictReader;
		import com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions;
		import com.android.inputmethod.latin.makedict.FusionDictionary;
		import com.android.inputmethod.latin.makedict.FusionDictionary.Node;
		@@ -118,13 +118,13 @@ public final class UserHistoryDictIOUtils {
		/**
		* Reads dictionary from file.
		*/
		public static void readDictionaryBinary(final FusionDictionaryBufferInterface buffer,
		public static void readDictionaryBinary(final BinaryDictReader reader,
		final OnAddWordListener dict) {
		final Map<Integer, String> unigrams = CollectionUtils.newTreeMap();
		final Map<Integer, Integer> frequencies = CollectionUtils.newTreeMap();
		final Map<Integer, ArrayList<PendingAttribute>> bigrams = CollectionUtils.newTreeMap();
		try {
		BinaryDictIOUtils.readUnigramsAndBigramsBinary(buffer, unigrams, frequencies,
		BinaryDictIOUtils.readUnigramsAndBigramsBinary(reader, unigrams, frequencies,
		bigrams);
		} catch (IOException e) {
		Log.e(TAG, "IO exception while reading file", e);