Consolidating dict buffers into BufferWithExtendableBuffer. (3e76487c) · Commits · e / os / android_packages_inputmethods_LatinIME

native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h

+16 −18

Original line number	Diff line number	Diff line
		@@ -31,50 +31,48 @@ namespace latinime {
		*/
		class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy {
		public:
		DynamicBigramListPolicy(const uint8_t *const bigramsBuf, const int bufSize,
		const ExtendableBuffer *const additionalBuffer)
		: mDictRoot(bigramsBuf), mBufSize(bufSize), mAdditionalBuffer(additionalBuffer) {}
		DynamicBigramListPolicy(const BufferWithExtendableBuffer *const buffer)
		: mBuffer(buffer) {}

		~DynamicBigramListPolicy() {}

		void getNextBigram(int const outBigramPos, int const outProbability, bool *const outHasNext,
		int *const pos) const {
		const bool usesAdditionalBuffer = *pos >= mBufSize;
		const uint8_t *const buffer = (usesAdditionalBuffer) ?
		mAdditionalBuffer->getBuffer() : mDictRoot;
		const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos);
		const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer);
		if (usesAdditionalBuffer) {
		*pos -= mBufSize;
		*pos -= mBuffer->getOriginalBufferSize();
		}
		const BigramListReadingUtils::BigramFlags flags =
		BigramListReadingUtils::getFlagsAndForwardPointer(buffer, pos);
		*outBigramPos = BigramListReadingUtils::getBigramAddressAndForwardPointer(
		buffer, flags, pos);
		if (usesAdditionalBuffer) {
		*outBigramPos += mBufSize;
		*outBigramPos += mBuffer->getOriginalBufferSize();
		}
		*outProbability = BigramListReadingUtils::getProbabilityFromFlags(flags);
		*outHasNext = BigramListReadingUtils::hasNext(flags);
		if (usesAdditionalBuffer) {
		*pos += mBufSize;
		*pos += mBuffer->getOriginalBufferSize();
		}
		}

		void skipAllBigrams(int *const pos) const {
		if (*pos >= mBufSize) {
		*pos -= mBufSize;
		BigramListReadingUtils::skipExistingBigrams(mAdditionalBuffer->getBuffer(), pos);
		*pos += mBufSize;
		} else {
		BigramListReadingUtils::skipExistingBigrams(mDictRoot, pos);
		const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos);
		const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer);
		if (usesAdditionalBuffer) {
		*pos -= mBuffer->getOriginalBufferSize();
		}
		BigramListReadingUtils::skipExistingBigrams(buffer, pos);
		if (usesAdditionalBuffer) {
		*pos += mBuffer->getOriginalBufferSize();
		}
		}

		private:
		DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicBigramListPolicy);

		const uint8_t *const mDictRoot;
		const int mBufSize;
		const ExtendableBuffer *const mAdditionalBuffer;
		const BufferWithExtendableBuffer *const mBuffer;
		};
		} // namespace latinime
		#endif // LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp

+8 −6

Original line number	Diff line number	Diff line
		@@ -25,10 +25,12 @@ namespace latinime {

		void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(const int nodePos,
		const int maxCodePointCount, int *const outCodePoints) {
		const bool usesAdditionalBuffer = nodePos >= mOriginalDictSize;
		const uint8_t *const dictBuf =
		usesAdditionalBuffer ? mExtendableBuffer->getBuffer() : mDictRoot;
		int pos = (usesAdditionalBuffer) ? nodePos - mOriginalDictSize : nodePos;
		const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(nodePos);
		const uint8_t *const dictBuf = mBuffer->getBuffer(usesAdditionalBuffer);
		int pos = nodePos;
		if (usesAdditionalBuffer) {
		pos -= mBuffer->getOriginalBufferSize();
		}
		mFlags = PatriciaTrieReadingUtils::getFlagsAndAdvancePosition(dictBuf, &pos);
		const int parentPos =
		DynamicPatriciaTrieReadingUtils::getParentPosAndAdvancePosition(dictBuf, &pos);
		@@ -48,10 +50,10 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c
		mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition(
		dictBuf, mFlags, &pos);
		if (usesAdditionalBuffer && mChildrenPos != NOT_A_DICT_POS) {
		mChildrenPos += mOriginalDictSize;
		mChildrenPos += mBuffer->getOriginalBufferSize();
		}
		if (usesAdditionalBuffer) {
		pos += mOriginalDictSize;
		pos += mBuffer->getOriginalBufferSize();
		}
		if (PatriciaTrieReadingUtils::hasShortcutTargets(mFlags)) {
		mShortcutPos = pos;

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h

+4 −9

Original line number	Diff line number	Diff line
		@@ -25,9 +25,9 @@

		namespace latinime {

		class BufferWithExtendableBuffer;
		class DictionaryBigramsStructurePolicy;
		class DictionaryShortcutsStructurePolicy;
		class ExtendableBuffer;

		/*
		* This class is used for helping to read nodes of dynamic patricia trie. This class handles moved
		@@ -35,12 +35,10 @@ class ExtendableBuffer;
		*/
		class DynamicPatriciaTrieNodeReader {
		public:
		DynamicPatriciaTrieNodeReader(const uint8_t *const dictRoot, const int originalDictSize,
		const ExtendableBuffer *const extendableBuffer,
		DynamicPatriciaTrieNodeReader(const BufferWithExtendableBuffer *const buffer,
		const DictionaryBigramsStructurePolicy *const bigramsPolicy,
		const DictionaryShortcutsStructurePolicy *const shortcutsPolicy)
		: mDictRoot(dictRoot), mOriginalDictSize(originalDictSize),
		mExtendableBuffer(extendableBuffer), mBigramsPolicy(bigramsPolicy),
		: mBuffer(buffer), mBigramsPolicy(bigramsPolicy),
		mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS), mFlags(0),
		mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mProbability(NOT_A_PROBABILITY),
		mChildrenPos(NOT_A_DICT_POS), mShortcutPos(NOT_A_DICT_POS),
		@@ -124,10 +122,7 @@ class DynamicPatriciaTrieNodeReader {
		private:
		DISALLOW_COPY_AND_ASSIGN(DynamicPatriciaTrieNodeReader);

		// TODO: Consolidate mDictRoot.
		const uint8_t *const mDictRoot;
		const int mOriginalDictSize;
		const ExtendableBuffer *const mExtendableBuffer;
		const BufferWithExtendableBuffer *const mBuffer;
		const DictionaryBigramsStructurePolicy *const mBigramsPolicy;
		const DictionaryShortcutsStructurePolicy *const mShortcutsPolicy;
		int mNodePos;

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp

+9 −9

Original line number	Diff line number	Diff line
		@@ -31,8 +31,8 @@ void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const d
		if (!dicNode->hasChildren()) {
		return;
		}
		DynamicPatriciaTrieReadingHelper readingHelper(mDictRoot, mOriginalDictSize,
		&mExtendableBuffer, getBigramsStructurePolicy(), getShortcutsStructurePolicy());
		DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer,
		getBigramsStructurePolicy(), getShortcutsStructurePolicy());
		readingHelper.initWithNodeArrayPos(dicNode->getChildrenPos());
		const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader();
		while (!readingHelper.isEnd()) {
		@@ -51,8 +51,8 @@ int DynamicPatriciaTriePolicy::getCodePointsAndProbabilityAndReturnCodePointCoun
		// This method traverses parent nodes from the terminal by following parent pointers; thus,
		// node code points are stored in the buffer in the reverse order.
		int reverseCodePoints[maxCodePointCount];
		DynamicPatriciaTrieReadingHelper readingHelper(mDictRoot, mOriginalDictSize,
		&mExtendableBuffer, getBigramsStructurePolicy(), getShortcutsStructurePolicy());
		DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer,
		getBigramsStructurePolicy(), getShortcutsStructurePolicy());
		// First, read the terminal node and get its probability.
		readingHelper.initWithNodePos(nodePos);
		if (!readingHelper.isValidTerminalNode()) {
		@@ -94,8 +94,8 @@ int DynamicPatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const in
		for (int i = 0; i < length; ++i) {
		searchCodePoints[i] = forceLowerCaseSearch ? CharUtils::toLowerCase(inWord[i]) : inWord[i];
		}
		DynamicPatriciaTrieReadingHelper readingHelper(mDictRoot, mOriginalDictSize,
		&mExtendableBuffer, getBigramsStructurePolicy(), getShortcutsStructurePolicy());
		DynamicPatriciaTrieReadingHelper readingHelper(&mBufferWithExtendableBuffer,
		getBigramsStructurePolicy(), getShortcutsStructurePolicy());
		readingHelper.initWithNodeArrayPos(getRootPosition());
		const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader();
		while (!readingHelper.isEnd()) {
		@@ -137,7 +137,7 @@ int DynamicPatriciaTriePolicy::getUnigramProbability(const int nodePos) const {
		if (nodePos == NOT_A_VALID_WORD_POS) {
		return NOT_A_PROBABILITY;
		}
		DynamicPatriciaTrieNodeReader nodeReader(mDictRoot, mOriginalDictSize, &mExtendableBuffer,
		DynamicPatriciaTrieNodeReader nodeReader(&mBufferWithExtendableBuffer,
		getBigramsStructurePolicy(), getShortcutsStructurePolicy());
		nodeReader.fetchNodeInfoFromBuffer(nodePos);
		if (nodeReader.isDeleted() \|\| nodeReader.isBlacklisted() \|\| nodeReader.isNotAWord()) {
		@@ -150,7 +150,7 @@ int DynamicPatriciaTriePolicy::getShortcutPositionOfNode(const int nodePos) cons
		if (nodePos == NOT_A_VALID_WORD_POS) {
		return NOT_A_DICT_POS;
		}
		DynamicPatriciaTrieNodeReader nodeReader(mDictRoot, mOriginalDictSize, &mExtendableBuffer,
		DynamicPatriciaTrieNodeReader nodeReader(&mBufferWithExtendableBuffer,
		getBigramsStructurePolicy(), getShortcutsStructurePolicy());
		nodeReader.fetchNodeInfoFromBuffer(nodePos);
		if (nodeReader.isDeleted()) {
		@@ -163,7 +163,7 @@ int DynamicPatriciaTriePolicy::getBigramsPositionOfNode(const int nodePos) const
		if (nodePos == NOT_A_VALID_WORD_POS) {
		return NOT_A_DICT_POS;
		}
		DynamicPatriciaTrieNodeReader nodeReader(mDictRoot, mOriginalDictSize, &mExtendableBuffer,
		DynamicPatriciaTrieNodeReader nodeReader(&mBufferWithExtendableBuffer,
		getBigramsStructurePolicy(), getShortcutsStructurePolicy());
		nodeReader.fetchNodeInfoFromBuffer(nodePos);
		if (nodeReader.isDeleted()) {

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h

+6 −12

Original line number	Diff line number	Diff line
		@@ -17,8 +17,6 @@
		#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H
		#define LATINIME_DYNAMIC_PATRICIA_TRIE_POLICY_H

		#include <stdint.h>

		#include "defines.h"
		#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
		#include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h"
		@@ -35,11 +33,11 @@ class DicNodeVector;
		class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
		public:
		DynamicPatriciaTriePolicy(const MmappedBuffer *const buffer)
		: mBuffer(buffer), mExtendableBuffer(), mHeaderPolicy(mBuffer->getBuffer()),
		mDictRoot(mBuffer->getBuffer() + mHeaderPolicy.getSize()),
		mOriginalDictSize(mBuffer->getBufferSize() - mHeaderPolicy.getSize()),
		mBigramListPolicy(mDictRoot, mOriginalDictSize, &mExtendableBuffer),
		mShortcutListPolicy(mDictRoot, mOriginalDictSize, &mExtendableBuffer) {}
		: mBuffer(buffer), mHeaderPolicy(mBuffer->getBuffer()),
		mBufferWithExtendableBuffer(mBuffer->getBuffer() + mHeaderPolicy.getSize(),
		mBuffer->getBufferSize() - mHeaderPolicy.getSize()),
		mBigramListPolicy(&mBufferWithExtendableBuffer),
		mShortcutListPolicy(&mBufferWithExtendableBuffer) {}

		~DynamicPatriciaTriePolicy() {
		delete mBuffer;
		@@ -89,12 +87,8 @@ class DynamicPatriciaTriePolicy : public DictionaryStructureWithBufferPolicy {
		DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTriePolicy);

		const MmappedBuffer *const mBuffer;
		const ExtendableBuffer mExtendableBuffer;
		const HeaderPolicy mHeaderPolicy;
		// TODO: Consolidate mDictRoot.
		// CAVEAT!: Be careful about array out of bound access with mDictRoot
		const uint8_t *const mDictRoot;
		const int mOriginalDictSize;
		const BufferWithExtendableBuffer mBufferWithExtendableBuffer;
		const DynamicBigramListPolicy mBigramListPolicy;
		const DynamicShortcutListPolicy mShortcutListPolicy;
		};