Loading native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp +4 −0 Original line number Original line Diff line number Diff line Loading @@ -52,6 +52,10 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c mProbabilityFieldPos = NOT_A_DICT_POS; mProbabilityFieldPos = NOT_A_DICT_POS; mProbability = NOT_A_PROBABILITY; mProbability = NOT_A_PROBABILITY; } } mChildrenPosFieldPos = pos; if (usesAdditionalBuffer) { mChildrenPosFieldPos += mBuffer->getOriginalBufferSize(); } mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( dictBuf, mFlags, &pos); dictBuf, mFlags, &pos); if (usesAdditionalBuffer && mChildrenPos != NOT_A_DICT_POS) { if (usesAdditionalBuffer && mChildrenPos != NOT_A_DICT_POS) { Loading native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h +9 −3 Original line number Original line Diff line number Diff line Loading @@ -42,8 +42,9 @@ class DynamicPatriciaTrieNodeReader { mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS), mFlags(0), mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS), mFlags(0), mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mProbabilityFieldPos(NOT_A_DICT_POS), mProbability(NOT_A_PROBABILITY), mProbabilityFieldPos(NOT_A_DICT_POS), mProbability(NOT_A_PROBABILITY), mChildrenPos(NOT_A_DICT_POS), mShortcutPos(NOT_A_DICT_POS), mChildrenPosFieldPos(NOT_A_DICT_POS), mChildrenPos(NOT_A_DICT_POS), mBigramPos(NOT_A_DICT_POS), mSiblingPos(NOT_A_VALID_WORD_POS) {} mShortcutPos(NOT_A_DICT_POS), mBigramPos(NOT_A_DICT_POS), mSiblingPos(NOT_A_VALID_WORD_POS) {} ~DynamicPatriciaTrieNodeReader() {} ~DynamicPatriciaTrieNodeReader() {} Loading Loading @@ -104,7 +105,11 @@ class DynamicPatriciaTrieNodeReader { return mProbability; return mProbability; } } // Children node group position // Children PtNode array position AK_FORCE_INLINE int getChildrenPosFieldPos() const { return mChildrenPosFieldPos; } AK_FORCE_INLINE int getChildrenPos() const { AK_FORCE_INLINE int getChildrenPos() const { return mChildrenPos; return mChildrenPos; } } Loading Loading @@ -136,6 +141,7 @@ class DynamicPatriciaTrieNodeReader { uint8_t mCodePointCount; uint8_t mCodePointCount; int mProbabilityFieldPos; int mProbabilityFieldPos; int mProbability; int mProbability; int mChildrenPosFieldPos; int mChildrenPos; int mChildrenPos; int mShortcutPos; int mShortcutPos; int mBigramPos; int mBigramPos; Loading native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp +44 −19 Original line number Original line Diff line number Diff line Loading @@ -60,16 +60,21 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord( // All characters are matched. // All characters are matched. if (codePointCount == readingHelper->getTotalCodePointCount()) { if (codePointCount == readingHelper->getTotalCodePointCount()) { if (ENABLE_DYNAMIC_UPDATE) { if (ENABLE_DYNAMIC_UPDATE) { setPtNodeProbability(nodeReader, probability, return setPtNodeProbability(nodeReader, probability, readingHelper->getMergedNodeCodePoints()); readingHelper->getMergedNodeCodePoints()); } else { } else { return false; return false; } } } } if (!nodeReader->hasChildren()) { if (!nodeReader->hasChildren()) { // TODO: Create children node array and add new node as a child. if (ENABLE_DYNAMIC_UPDATE) { return createChildrenPtNodeArrayAndAChildPtNode(nodeReader, probability, wordCodePoints + readingHelper->getTotalCodePointCount(), codePointCount - readingHelper->getTotalCodePointCount()); } else { return false; return false; } } } // Advance to the children nodes. // Advance to the children nodes. parentPos = nodeReader->getNodePos(); parentPos = nodeReader->getNodePos(); readingHelper->readChildNode(); readingHelper->readChildNode(); Loading Loading @@ -201,22 +206,8 @@ bool DynamicPatriciaTrieWritingHelper::createAndInsertNodeIntoPtNodeArray(const newPtNodeArrayPos, forwardLinkFieldPos)) { newPtNodeArrayPos, forwardLinkFieldPos)) { return false; return false; } } int writingPos = newPtNodeArrayPos; return createNewPtNodeArrayWithAChildPtNode(parentPos, nodeCodePoints, nodeCodePointCount, if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, probability); 1 /* arraySize */, &writingPos)) { return false; } if (!writeNodeToBuffer(false /* isBlacklisted */, false /* isNotAWord */, parentPos, nodeCodePoints, nodeCodePointCount, probability, NOT_A_DICT_POS /* childrenPos */, NOT_A_DICT_POS /* originalBigramsPos */, NOT_A_DICT_POS /* originalShortcutPos */, &writingPos)) { return false; } if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } return true; } } bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( Loading Loading @@ -245,4 +236,38 @@ bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( return true; return true; } } bool DynamicPatriciaTrieWritingHelper::createChildrenPtNodeArrayAndAChildPtNode( const DynamicPatriciaTrieNodeReader *const parentNode, const int probability, const int *const codePoints, const int codePointCount) { const int newPtNodeArrayPos = mBuffer->getTailPosition(); int childrenPosFieldPos = parentNode->getChildrenPosFieldPos(); if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBuffer, newPtNodeArrayPos, &childrenPosFieldPos)) { return false; } return createNewPtNodeArrayWithAChildPtNode(parentNode->getNodePos(), codePoints, codePointCount, probability); } bool DynamicPatriciaTrieWritingHelper::createNewPtNodeArrayWithAChildPtNode( const int parentPtNodePos, const int *const nodeCodePoints, const int nodeCodePointCount, const int probability) { int writingPos = mBuffer->getTailPosition(); if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, 1 /* arraySize */, &writingPos)) { return false; } if (!writeNodeToBuffer(false /* isBlacklisted */, false /* isNotAWord */, parentPtNodePos, nodeCodePoints, nodeCodePointCount, probability, NOT_A_DICT_POS /* childrenPos */, NOT_A_DICT_POS /* originalBigramsPos */, NOT_A_DICT_POS /* originalShortcutPos */, &writingPos)) { return false; } if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } return true; } } // namespace latinime } // namespace latinime native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h +7 −0 Original line number Original line Diff line number Diff line Loading @@ -67,6 +67,13 @@ class DynamicPatriciaTrieWritingHelper { bool setPtNodeProbability(const DynamicPatriciaTrieNodeReader *const originalNode, bool setPtNodeProbability(const DynamicPatriciaTrieNodeReader *const originalNode, const int probability, const int *const codePoints); const int probability, const int *const codePoints); bool createChildrenPtNodeArrayAndAChildPtNode( const DynamicPatriciaTrieNodeReader *const parentNode, const int probability, const int *const codePoints, const int codePointCount); bool createNewPtNodeArrayWithAChildPtNode(const int parentPos, const int *const nodeCodePoints, const int nodeCodePointCount, const int probability); }; }; } // namespace latinime } // namespace latinime #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */ #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */ Loading
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp +4 −0 Original line number Original line Diff line number Diff line Loading @@ -52,6 +52,10 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c mProbabilityFieldPos = NOT_A_DICT_POS; mProbabilityFieldPos = NOT_A_DICT_POS; mProbability = NOT_A_PROBABILITY; mProbability = NOT_A_PROBABILITY; } } mChildrenPosFieldPos = pos; if (usesAdditionalBuffer) { mChildrenPosFieldPos += mBuffer->getOriginalBufferSize(); } mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( mChildrenPos = DynamicPatriciaTrieReadingUtils::readChildrenPositionAndAdvancePosition( dictBuf, mFlags, &pos); dictBuf, mFlags, &pos); if (usesAdditionalBuffer && mChildrenPos != NOT_A_DICT_POS) { if (usesAdditionalBuffer && mChildrenPos != NOT_A_DICT_POS) { Loading
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h +9 −3 Original line number Original line Diff line number Diff line Loading @@ -42,8 +42,9 @@ class DynamicPatriciaTrieNodeReader { mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS), mFlags(0), mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS), mFlags(0), mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mProbabilityFieldPos(NOT_A_DICT_POS), mProbability(NOT_A_PROBABILITY), mProbabilityFieldPos(NOT_A_DICT_POS), mProbability(NOT_A_PROBABILITY), mChildrenPos(NOT_A_DICT_POS), mShortcutPos(NOT_A_DICT_POS), mChildrenPosFieldPos(NOT_A_DICT_POS), mChildrenPos(NOT_A_DICT_POS), mBigramPos(NOT_A_DICT_POS), mSiblingPos(NOT_A_VALID_WORD_POS) {} mShortcutPos(NOT_A_DICT_POS), mBigramPos(NOT_A_DICT_POS), mSiblingPos(NOT_A_VALID_WORD_POS) {} ~DynamicPatriciaTrieNodeReader() {} ~DynamicPatriciaTrieNodeReader() {} Loading Loading @@ -104,7 +105,11 @@ class DynamicPatriciaTrieNodeReader { return mProbability; return mProbability; } } // Children node group position // Children PtNode array position AK_FORCE_INLINE int getChildrenPosFieldPos() const { return mChildrenPosFieldPos; } AK_FORCE_INLINE int getChildrenPos() const { AK_FORCE_INLINE int getChildrenPos() const { return mChildrenPos; return mChildrenPos; } } Loading Loading @@ -136,6 +141,7 @@ class DynamicPatriciaTrieNodeReader { uint8_t mCodePointCount; uint8_t mCodePointCount; int mProbabilityFieldPos; int mProbabilityFieldPos; int mProbability; int mProbability; int mChildrenPosFieldPos; int mChildrenPos; int mChildrenPos; int mShortcutPos; int mShortcutPos; int mBigramPos; int mBigramPos; Loading
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp +44 −19 Original line number Original line Diff line number Diff line Loading @@ -60,16 +60,21 @@ bool DynamicPatriciaTrieWritingHelper::addUnigramWord( // All characters are matched. // All characters are matched. if (codePointCount == readingHelper->getTotalCodePointCount()) { if (codePointCount == readingHelper->getTotalCodePointCount()) { if (ENABLE_DYNAMIC_UPDATE) { if (ENABLE_DYNAMIC_UPDATE) { setPtNodeProbability(nodeReader, probability, return setPtNodeProbability(nodeReader, probability, readingHelper->getMergedNodeCodePoints()); readingHelper->getMergedNodeCodePoints()); } else { } else { return false; return false; } } } } if (!nodeReader->hasChildren()) { if (!nodeReader->hasChildren()) { // TODO: Create children node array and add new node as a child. if (ENABLE_DYNAMIC_UPDATE) { return createChildrenPtNodeArrayAndAChildPtNode(nodeReader, probability, wordCodePoints + readingHelper->getTotalCodePointCount(), codePointCount - readingHelper->getTotalCodePointCount()); } else { return false; return false; } } } // Advance to the children nodes. // Advance to the children nodes. parentPos = nodeReader->getNodePos(); parentPos = nodeReader->getNodePos(); readingHelper->readChildNode(); readingHelper->readChildNode(); Loading Loading @@ -201,22 +206,8 @@ bool DynamicPatriciaTrieWritingHelper::createAndInsertNodeIntoPtNodeArray(const newPtNodeArrayPos, forwardLinkFieldPos)) { newPtNodeArrayPos, forwardLinkFieldPos)) { return false; return false; } } int writingPos = newPtNodeArrayPos; return createNewPtNodeArrayWithAChildPtNode(parentPos, nodeCodePoints, nodeCodePointCount, if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, probability); 1 /* arraySize */, &writingPos)) { return false; } if (!writeNodeToBuffer(false /* isBlacklisted */, false /* isNotAWord */, parentPos, nodeCodePoints, nodeCodePointCount, probability, NOT_A_DICT_POS /* childrenPos */, NOT_A_DICT_POS /* originalBigramsPos */, NOT_A_DICT_POS /* originalShortcutPos */, &writingPos)) { return false; } if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } return true; } } bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( Loading Loading @@ -245,4 +236,38 @@ bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( return true; return true; } } bool DynamicPatriciaTrieWritingHelper::createChildrenPtNodeArrayAndAChildPtNode( const DynamicPatriciaTrieNodeReader *const parentNode, const int probability, const int *const codePoints, const int codePointCount) { const int newPtNodeArrayPos = mBuffer->getTailPosition(); int childrenPosFieldPos = parentNode->getChildrenPosFieldPos(); if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBuffer, newPtNodeArrayPos, &childrenPosFieldPos)) { return false; } return createNewPtNodeArrayWithAChildPtNode(parentNode->getNodePos(), codePoints, codePointCount, probability); } bool DynamicPatriciaTrieWritingHelper::createNewPtNodeArrayWithAChildPtNode( const int parentPtNodePos, const int *const nodeCodePoints, const int nodeCodePointCount, const int probability) { int writingPos = mBuffer->getTailPosition(); if (!DynamicPatriciaTrieWritingUtils::writePtNodeArraySizeAndAdvancePosition(mBuffer, 1 /* arraySize */, &writingPos)) { return false; } if (!writeNodeToBuffer(false /* isBlacklisted */, false /* isNotAWord */, parentPtNodePos, nodeCodePoints, nodeCodePointCount, probability, NOT_A_DICT_POS /* childrenPos */, NOT_A_DICT_POS /* originalBigramsPos */, NOT_A_DICT_POS /* originalShortcutPos */, &writingPos)) { return false; } if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, NOT_A_DICT_POS /* forwardLinkPos */, &writingPos)) { return false; } return true; } } // namespace latinime } // namespace latinime
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h +7 −0 Original line number Original line Diff line number Diff line Loading @@ -67,6 +67,13 @@ class DynamicPatriciaTrieWritingHelper { bool setPtNodeProbability(const DynamicPatriciaTrieNodeReader *const originalNode, bool setPtNodeProbability(const DynamicPatriciaTrieNodeReader *const originalNode, const int probability, const int *const codePoints); const int probability, const int *const codePoints); bool createChildrenPtNodeArrayAndAChildPtNode( const DynamicPatriciaTrieNodeReader *const parentNode, const int probability, const int *const codePoints, const int codePointCount); bool createNewPtNodeArrayWithAChildPtNode(const int parentPos, const int *const nodeCodePoints, const int nodeCodePointCount, const int probability); }; }; } // namespace latinime } // namespace latinime #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */ #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */