Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 04bf3cd4 authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi Committed by Android (Google) Code Review
Browse files

Merge "Employ "bigram link" for handling moved bigram target."

parents fbc7e61a 8c69ddb5

native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp

+66 −6
Original line number Diff line number Diff line
@@ -18,6 +18,42 @@ 


namespace latinime {



const int DynamicBigramListPolicy::BIGRAM_LINK_COUNT_LIMIT = 10000;



void DynamicBigramListPolicy::getNextBigram(int *const outBigramPos, int *const outProbability,

        bool *const outHasNext, int *const pos) const {

    const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos);

    const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer);

    if (usesAdditionalBuffer) {

        *pos -= mBuffer->getOriginalBufferSize();

    }

    const BigramListReadWriteUtils::BigramFlags flags =

            BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, pos);

    int originalBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer(

            buffer, flags, pos);

    if (usesAdditionalBuffer && originalBigramPos != NOT_A_VALID_WORD_POS) {

        originalBigramPos += mBuffer->getOriginalBufferSize();

    }

    *outBigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos);

    *outProbability = BigramListReadWriteUtils::getProbabilityFromFlags(flags);

    *outHasNext = BigramListReadWriteUtils::hasNext(flags);

    if (usesAdditionalBuffer) {

        *pos += mBuffer->getOriginalBufferSize();

    }

}



void DynamicBigramListPolicy::skipAllBigrams(int *const pos) const {

    const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos);

    const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer);

    if (usesAdditionalBuffer) {

        *pos -= mBuffer->getOriginalBufferSize();

    }

    BigramListReadWriteUtils::skipExistingBigrams(buffer, pos);

    if (usesAdditionalBuffer) {

        *pos += mBuffer->getOriginalBufferSize();

    }

}



bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos) {

    const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos);

    if (usesAdditionalBuffer) {
@@ -28,15 +64,16 @@ bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPo 
        // The buffer address can be changed after calling buffer writing methods.

        const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer);

        flags = BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, fromPos);

        int bigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer(

        int originalBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer(

                buffer, flags, fromPos);

        if (bigramPos == NOT_A_VALID_WORD_POS) {

        if (originalBigramPos == NOT_A_VALID_WORD_POS) {

            // skip invalid bigram entry.

            continue;

        }

        if (usesAdditionalBuffer) {

            bigramPos += mBuffer->getOriginalBufferSize();

            originalBigramPos += mBuffer->getOriginalBufferSize();

        }

        const int bigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos);

        BigramListReadWriteUtils::BigramFlags newBigramFlags;

        uint32_t newBigramOffset;

        int newBigramOffsetFieldSize;
@@ -133,11 +170,12 @@ bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int ta 
        if (usesAdditionalBuffer) {

            bigramOffsetFieldPos += mBuffer->getOriginalBufferSize();

        }

        int bigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer(

        int originalBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer(

                buffer, flags, &pos);

        if (usesAdditionalBuffer && bigramPos != NOT_A_VALID_WORD_POS) {

            bigramPos += mBuffer->getOriginalBufferSize();

        if (usesAdditionalBuffer && originalBigramPos != NOT_A_VALID_WORD_POS) {

            originalBigramPos += mBuffer->getOriginalBufferSize();

        }

        const int bigramPos = followBigramLinkAndGetCurrentBigramPtNodePos(originalBigramPos);

        if (bigramPos != targetBigramPos) {

            continue;

        }
@@ -152,4 +190,26 @@ bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int ta 
    return false;

}



int DynamicBigramListPolicy::followBigramLinkAndGetCurrentBigramPtNodePos(

        const int originalBigramPos) const {

    if (originalBigramPos == NOT_A_VALID_WORD_POS) {

        return NOT_A_VALID_WORD_POS;

    }

    int currentPos = originalBigramPos;

    DynamicPatriciaTrieNodeReader nodeReader(mBuffer, this /* bigramsPolicy */, mShortcutPolicy);

    nodeReader.fetchNodeInfoFromBuffer(currentPos);

    int bigramLinkCount = 0;

    while (nodeReader.getBigramLinkedNodePos() != NOT_A_DICT_POS) {

        currentPos = nodeReader.getBigramLinkedNodePos();

        nodeReader.fetchNodeInfoFromBuffer(currentPos);

        bigramLinkCount++;

        if (bigramLinkCount > BIGRAM_LINK_COUNT_LIMIT) {

            AKLOGI("Bigram link is invalid. start position: %d", bigramPos);

            ASSERT(false);

            return NOT_A_VALID_WORD_POS;

        }

    }

    return currentPos;

}



} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h

+13 −32
Original line number Diff line number Diff line
@@ -21,7 +21,9 @@ 


#include "defines.h"

#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"

#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"

#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h"

#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_helper.h"

#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"



namespace latinime {
@@ -31,43 +33,16 @@ namespace latinime { 
 */

class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy {

 public:

    DynamicBigramListPolicy(BufferWithExtendableBuffer *const buffer)

            : mBuffer(buffer) {}

    DynamicBigramListPolicy(BufferWithExtendableBuffer *const buffer,

            const DictionaryShortcutsStructurePolicy *const shortcutPolicy)

            : mBuffer(buffer), mShortcutPolicy(shortcutPolicy) {}



    ~DynamicBigramListPolicy() {}



    void getNextBigram(int *const outBigramPos, int *const outProbability, bool *const outHasNext,

            int *const pos) const {

        const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos);

        const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer);

        if (usesAdditionalBuffer) {

            *pos -= mBuffer->getOriginalBufferSize();

        }

        const BigramListReadWriteUtils::BigramFlags flags =

                BigramListReadWriteUtils::getFlagsAndForwardPointer(buffer, pos);

        *outBigramPos = BigramListReadWriteUtils::getBigramAddressAndForwardPointer(

                buffer, flags, pos);

        if (usesAdditionalBuffer && *outBigramPos != NOT_A_VALID_WORD_POS) {

            *outBigramPos += mBuffer->getOriginalBufferSize();

        }

        *outProbability = BigramListReadWriteUtils::getProbabilityFromFlags(flags);

        *outHasNext = BigramListReadWriteUtils::hasNext(flags);

        if (usesAdditionalBuffer) {

            *pos += mBuffer->getOriginalBufferSize();

        }

    }

            int *const pos) const;



    void skipAllBigrams(int *const pos) const {

        const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*pos);

        const uint8_t *const buffer = mBuffer->getBuffer(usesAdditionalBuffer);

        if (usesAdditionalBuffer) {

            *pos -= mBuffer->getOriginalBufferSize();

        }

        BigramListReadWriteUtils::skipExistingBigrams(buffer, pos);

        if (usesAdditionalBuffer) {

            *pos += mBuffer->getOriginalBufferSize();

        }

    }

    void skipAllBigrams(int *const pos) const;



    // Copy bigrams from the bigram list that starts at fromPos to toPos and advance these

    // positions after bigram lists. This method skips invalid bigram entries.
@@ -81,7 +56,13 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { 
 private:

    DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicBigramListPolicy);



    static const int BIGRAM_LINK_COUNT_LIMIT;



    BufferWithExtendableBuffer *const mBuffer;

    const DictionaryShortcutsStructurePolicy *const mShortcutPolicy;



    // Follow bigram link and return the position of bigram target PtNode that is currently valid.

    int followBigramLinkAndGetCurrentBigramPtNodePos(const int originalBigramPos) const;

};

} // namespace latinime

#endif // LATINIME_DYNAMIC_BIGRAM_LIST_POLICY_H

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.cpp

+5 −0
Original line number Diff line number Diff line
@@ -62,6 +62,11 @@ void DynamicPatriciaTrieNodeReader::fetchNodeInfoFromBufferAndProcessMovedNode(c 
    if (usesAdditionalBuffer && mChildrenPos != NOT_A_DICT_POS) {

        mChildrenPos += mBuffer->getOriginalBufferSize();

    }

    if (mSiblingPos == NOT_A_VALID_WORD_POS && DynamicPatriciaTrieReadingUtils::isMoved(mFlags)) {

        mBigramLinkedNodePos = mChildrenPos;

    } else {

        mBigramLinkedNodePos = NOT_A_DICT_POS;

    }

    if (usesAdditionalBuffer) {

        pos += mBuffer->getOriginalBufferSize();

    }

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h

+13 −12
Original line number Diff line number Diff line
@@ -39,10 +39,10 @@ class DynamicPatriciaTrieNodeReader { 
            const DictionaryBigramsStructurePolicy *const bigramsPolicy,

            const DictionaryShortcutsStructurePolicy *const shortcutsPolicy)

            : mBuffer(buffer), mBigramsPolicy(bigramsPolicy),

              mShortcutsPolicy(shortcutsPolicy), mNodePos(NOT_A_VALID_WORD_POS),

              mHeadPos(NOT_A_DICT_POS), mFlags(0), mParentPos(NOT_A_DICT_POS), mCodePointCount(0),

              mProbabilityFieldPos(NOT_A_DICT_POS), mProbability(NOT_A_PROBABILITY),

              mChildrenPosFieldPos(NOT_A_DICT_POS), mChildrenPos(NOT_A_DICT_POS),

              mShortcutsPolicy(shortcutsPolicy), mHeadPos(NOT_A_VALID_WORD_POS), mFlags(0),

              mParentPos(NOT_A_DICT_POS), mCodePointCount(0), mProbabilityFieldPos(NOT_A_DICT_POS),

              mProbability(NOT_A_PROBABILITY), mChildrenPosFieldPos(NOT_A_DICT_POS),

              mChildrenPos(NOT_A_DICT_POS), mBigramLinkedNodePos(NOT_A_DICT_POS),

              mShortcutPos(NOT_A_DICT_POS),  mBigramPos(NOT_A_DICT_POS),

              mSiblingPos(NOT_A_VALID_WORD_POS) {}
@@ -56,13 +56,9 @@ class DynamicPatriciaTrieNodeReader { 


    AK_FORCE_INLINE void fetchNodeInfoFromBufferAndGetNodeCodePoints(const int nodePos,

            const int maxCodePointCount, int *const outCodePoints) {

        mNodePos = nodePos;

        mSiblingPos = NOT_A_VALID_WORD_POS;

        fetchNodeInfoFromBufferAndProcessMovedNode(mNodePos, maxCodePointCount, outCodePoints);

    }



    AK_FORCE_INLINE int getNodePos() const {

        return mNodePos;

        mBigramLinkedNodePos = NOT_A_DICT_POS;

        fetchNodeInfoFromBufferAndProcessMovedNode(nodePos, maxCodePointCount, outCodePoints);

    }



    // HeadPos is different from NodePos when the current PtNode is a moved PtNode.
@@ -119,6 +115,11 @@ class DynamicPatriciaTrieNodeReader { 
        return mChildrenPos;

    }



    // Bigram linked node position.

    AK_FORCE_INLINE int getBigramLinkedNodePos() const {

        return mBigramLinkedNodePos;

    }



    // Shortcutlist position

    AK_FORCE_INLINE int getShortcutPos() const {

        return mShortcutPos;
@@ -140,7 +141,6 @@ class DynamicPatriciaTrieNodeReader { 
    const BufferWithExtendableBuffer *const mBuffer;

    const DictionaryBigramsStructurePolicy *const mBigramsPolicy;

    const DictionaryShortcutsStructurePolicy *const mShortcutsPolicy;

    int mNodePos;

    int mHeadPos;

    DynamicPatriciaTrieReadingUtils::NodeFlags mFlags;

    int mParentPos;
@@ -149,6 +149,7 @@ class DynamicPatriciaTrieNodeReader { 
    int mProbability;

    int mChildrenPosFieldPos;

    int mChildrenPos;

    int mBigramLinkedNodePos;

    int mShortcutPos;

    int mBigramPos;

    int mSiblingPos;

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.cpp

+2 −2
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ void DynamicPatriciaTriePolicy::createAndGetAllChildNodes(const DicNode *const d 
    readingHelper.initWithNodeArrayPos(dicNode->getChildrenPos());

    const DynamicPatriciaTrieNodeReader *const nodeReader = readingHelper.getNodeReader();

    while (!readingHelper.isEnd()) {

        childDicNodes->pushLeavingChild(dicNode, nodeReader->getNodePos(),

        childDicNodes->pushLeavingChild(dicNode, nodeReader->getHeadPos(),

                nodeReader->getChildrenPos(), nodeReader->getProbability(),

                nodeReader->isTerminal() && !nodeReader->isDeleted(),

                nodeReader->hasChildren(), nodeReader->isBlacklisted() || nodeReader->isNotAWord(),
@@ -122,7 +122,7 @@ int DynamicPatriciaTriePolicy::getTerminalNodePositionOfWord(const int *const in 
        // All characters are matched.

        if (length == readingHelper.getTotalCodePointCount()) {

            // Terminal position is found.

            return nodeReader->getNodePos();

            return nodeReader->getHeadPos();

        }

        if (!nodeReader->hasChildren()) {

            return NOT_A_VALID_WORD_POS;