Merge "Make PtNode have ProbabilityInfo instead of raw value." (2ef87aee) · Commits · e / os / android_packages_inputmethods_LatinIME

java/src/com/android/inputmethod/latin/DictionaryWriter.java

+9 −7

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@ import com.android.inputmethod.latin.makedict.FormatSpec;
		import com.android.inputmethod.latin.makedict.FusionDictionary;
		import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;
		import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;
		import com.android.inputmethod.latin.makedict.ProbabilityInfo;
		import com.android.inputmethod.latin.makedict.UnsupportedFormatException;
		import com.android.inputmethod.latin.utils.CollectionUtils;

		@@ -56,22 +57,23 @@ public class DictionaryWriter extends AbstractDictionaryWriter {
		// TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries,
		// considering performance regression.
		@Override
		public void addUnigramWord(final String word, final String shortcutTarget, final int frequency,
		final int shortcutFreq, final boolean isNotAWord) {
		public void addUnigramWord(final String word, final String shortcutTarget,
		final int probability, final int shortcutProbability, final boolean isNotAWord) {
		if (shortcutTarget == null) {
		mFusionDictionary.add(word, frequency, null, isNotAWord);
		mFusionDictionary.add(word, new ProbabilityInfo(probability), null, isNotAWord);
		} else {
		// TODO: Do this in the subclass, with this class taking an arraylist.
		final ArrayList<WeightedString> shortcutTargets = CollectionUtils.newArrayList();
		shortcutTargets.add(new WeightedString(shortcutTarget, shortcutFreq));
		mFusionDictionary.add(word, frequency, shortcutTargets, isNotAWord);
		shortcutTargets.add(new WeightedString(shortcutTarget, shortcutProbability));
		mFusionDictionary.add(word, new ProbabilityInfo(probability), shortcutTargets,
		isNotAWord);
		}
		}

		@Override
		public void addBigramWords(final String word0, final String word1, final int frequency,
		public void addBigramWords(final String word0, final String word1, final int probability,
		final boolean isValid, final long lastModifiedTime) {
		mFusionDictionary.setBigram(word0, word1, frequency);
		mFusionDictionary.setBigram(word0, word1, new ProbabilityInfo(probability));
		}

		@Override

java/src/com/android/inputmethod/latin/makedict/BinaryDictDecoderUtils.java

+10 −8

Original line number	Diff line number	Diff line
		@@ -408,7 +408,7 @@ public final class BinaryDictDecoderUtils {
		private static WeightedString getWordAtPositionWithParentAddress(final DictDecoder dictDecoder,
		final int pos, final FormatOptions options) {
		int currentPos = pos;
		int frequency = Integer.MIN_VALUE;
		ProbabilityInfo probabilityInfo = null;
		final StringBuilder builder = new StringBuilder();
		// the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH
		for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) {
		@@ -424,13 +424,15 @@ public final class BinaryDictDecoderUtils {
		MakedictLog.d("Too many jumps - probably a bug");
		}
		} while (BinaryDictIOUtils.isMovedPtNode(currentInfo.mFlags, options));
		if (Integer.MIN_VALUE == frequency) frequency = currentInfo.mFrequency;
		if (probabilityInfo == null) {
		probabilityInfo = currentInfo.mProbabilityInfo;
		}
		builder.insert(0,
		new String(currentInfo.mCharacters, 0, currentInfo.mCharacters.length));
		if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break;
		currentPos = currentInfo.mParentAddress + currentInfo.mOriginalAddress;
		}
		return new WeightedString(builder.toString(), frequency);
		return new WeightedString(builder.toString(), probabilityInfo);
		}

		private static WeightedString getWordAtPositionWithoutParentAddress(
		@@ -448,7 +450,7 @@ public final class BinaryDictDecoderUtils {
		groupPos = info.mEndAddress;
		if (info.mOriginalAddress == pos) {
		builder.append(new String(info.mCharacters, 0, info.mCharacters.length));
		result = new WeightedString(builder.toString(), info.mFrequency);
		result = new WeightedString(builder.toString(), info.mProbabilityInfo);
		break; // and return
		}
		if (BinaryDictIOUtils.hasChildrenAddress(info.mChildrenAddress)) {
		@@ -527,13 +529,13 @@ public final class BinaryDictDecoderUtils {
		}
		nodeArrayContents.add(
		new PtNode(info.mCharacters, shortcutTargets, bigrams,
		info.mFrequency,
		info.mProbabilityInfo,
		0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
		0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children));
		} else {
		nodeArrayContents.add(
		new PtNode(info.mCharacters, shortcutTargets, bigrams,
		info.mFrequency,
		info.mProbabilityInfo,
		0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD),
		0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED)));
		}
		@@ -611,7 +613,7 @@ public final class BinaryDictDecoderUtils {
		newDict.addBlacklistEntry(wordProperty.mWord, wordProperty.mShortcutTargets,
		wordProperty.mIsNotAWord);
		} else {
		newDict.add(wordProperty.mWord, wordProperty.getProbability(),
		newDict.add(wordProperty.mWord, wordProperty.mProbabilityInfo,
		wordProperty.mShortcutTargets, wordProperty.mIsNotAWord);
		}
		}
		@@ -620,7 +622,7 @@ public final class BinaryDictDecoderUtils {
		// words that are not also registered as unigrams so we don't have to avoid
		// them explicitly here.
		for (final WeightedString bigram : wordProperty.mBigrams) {
		newDict.setBigram(wordProperty.mWord, bigram.mWord, bigram.getProbability());
		newDict.setBigram(wordProperty.mWord, bigram.mWord, bigram.mProbabilityInfo);
		}
		}
		}

java/src/com/android/inputmethod/latin/makedict/BinaryDictEncoderUtils.java

+4 −4

Original line number	Diff line number	Diff line
		@@ -673,7 +673,7 @@ public class BinaryDictEncoderUtils {

		/* package */ static byte makePtNodeFlags(final PtNode node, final int childrenOffset,
		final FormatOptions formatOptions) {
		return (byte) makePtNodeFlags(node.mChars.length > 1, node.mFrequency >= 0,
		return (byte) makePtNodeFlags(node.mChars.length > 1, node.isTerminal(),
		getByteSize(childrenOffset),
		node.mShortcutTargets != null && !node.mShortcutTargets.isEmpty(),
		node.mBigrams != null, node.mIsNotAWord, node.mIsBlacklistEntry, formatOptions);
		@@ -833,10 +833,10 @@ public class BinaryDictEncoderUtils {
		+ ptNode.mCachedAddressAfterUpdate);
		}
		// Sanity checks.
		if (DBG && ptNode.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) {
		if (DBG && ptNode.getProbability() > FormatSpec.MAX_TERMINAL_FREQUENCY) {
		throw new RuntimeException("A node has a frequency > "
		+ FormatSpec.MAX_TERMINAL_FREQUENCY
		+ " : " + ptNode.mFrequency);
		+ " : " + ptNode.mProbabilityInfo.toString());
		}
		dictEncoder.writePtNode(ptNode, parentPosition, formatOptions, dict);
		}
		@@ -871,7 +871,7 @@ public class BinaryDictEncoderUtils {
		for (final PtNode ptNode : ptNodeArray.mData) {
		++ptNodes;
		if (ptNode.mChars.length > maxRuns) maxRuns = ptNode.mChars.length;
		if (ptNode.mFrequency >= 0) {
		if (ptNode.isTerminal()) {
		if (ptNodeArray.mCachedAddressAfterUpdate < firstTerminalAddress)
		firstTerminalAddress = ptNodeArray.mCachedAddressAfterUpdate;
		if (ptNodeArray.mCachedAddressAfterUpdate > lastTerminalAddress)

java/src/com/android/inputmethod/latin/makedict/BinaryDictIOUtils.java

+15 −14

Original line number	Diff line number	Diff line
		@@ -91,21 +91,23 @@ public final class BinaryDictIOUtils {
		stack.pop();
		continue;
		}
		PtNodeInfo info = dictDecoder.readPtNode(p.mAddress, formatOptions);
		for (int i = 0; i < info.mCharacters.length; ++i) {
		pushedChars[index++] = info.mCharacters[i];
		final PtNodeInfo ptNodeInfo = dictDecoder.readPtNode(p.mAddress, formatOptions);
		for (int i = 0; i < ptNodeInfo.mCharacters.length; ++i) {
		pushedChars[index++] = ptNodeInfo.mCharacters[i];
		}
		p.mPosition++;

		final boolean isMovedPtNode = isMovedPtNode(info.mFlags,
		final boolean isMovedPtNode = isMovedPtNode(ptNodeInfo.mFlags,
		formatOptions);
		final boolean isDeletedPtNode = isDeletedPtNode(info.mFlags,
		final boolean isDeletedPtNode = isDeletedPtNode(ptNodeInfo.mFlags,
		formatOptions);
		if (!isMovedPtNode && !isDeletedPtNode
		&& info.mFrequency != FusionDictionary.PtNode.NOT_A_TERMINAL) {// found word
		words.put(info.mOriginalAddress, new String(pushedChars, 0, index));
		frequencies.put(info.mOriginalAddress, info.mFrequency);
		if (info.mBigrams != null) bigrams.put(info.mOriginalAddress, info.mBigrams);
		if (!isMovedPtNode && !isDeletedPtNode && ptNodeInfo.isTerminal()) {// found word
		words.put(ptNodeInfo.mOriginalAddress, new String(pushedChars, 0, index));
		frequencies.put(
		ptNodeInfo.mOriginalAddress, ptNodeInfo.mProbabilityInfo.mProbability);
		if (ptNodeInfo.mBigrams != null) {
		bigrams.put(ptNodeInfo.mOriginalAddress, ptNodeInfo.mBigrams);
		}
		}

		if (p.mPosition == p.mNumOfPtNode) {
		@@ -127,8 +129,8 @@ public final class BinaryDictIOUtils {
		p.mAddress = dictDecoder.getPosition();
		}

		if (!isMovedPtNode && hasChildrenAddress(info.mChildrenAddress)) {
		final Position childrenPos = new Position(info.mChildrenAddress, index);
		if (!isMovedPtNode && hasChildrenAddress(ptNodeInfo.mChildrenAddress)) {
		final Position childrenPos = new Position(ptNodeInfo.mChildrenAddress, index);
		stack.push(childrenPos);
		}
		}
		@@ -203,8 +205,7 @@ public final class BinaryDictIOUtils {
		if (same) {
		// found the PtNode matches the word.
		if (wordPos + currentInfo.mCharacters.length == wordLen) {
		if (currentInfo.mFrequency == PtNode.NOT_A_TERMINAL
		\|\| isDeletedNode) {
		if (!currentInfo.isTerminal() \|\| isDeletedNode) {
		return FormatSpec.NOT_VALID_WORD;
		} else {
		return ptNodePos;

java/src/com/android/inputmethod/latin/makedict/FusionDictionary.java

+64 −52

File changed.

Preview size limit exceeded, changes collapsed.