Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit cc6081c5 authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi Committed by Android (Google) Code Review
Browse files

Merge "Use word id for methods related to n-grams."

parents 95de7ad1 89a003b1

native/jni/src/suggest/core/dicnode/dic_node.h

+17 −12
Original line number Diff line number Diff line
@@ -103,10 +103,10 @@ class DicNode { 
        PROF_NODE_COPY(&dicNode->mProfiler, mProfiler);

    }



    // Init for root with prevWordsPtNodePos which is used for n-gram

    void initAsRoot(const int rootPtNodeArrayPos, const int *const prevWordsPtNodePos) {

    // Init for root with prevWordIds which is used for n-gram

    void initAsRoot(const int rootPtNodeArrayPos, const int *const prevWordIds) {

        mIsCachedForNextSuggestion = false;

        mDicNodeProperties.init(rootPtNodeArrayPos, prevWordsPtNodePos);

        mDicNodeProperties.init(rootPtNodeArrayPos, prevWordIds);

        mDicNodeState.init();

        PROF_NODE_RESET(mProfiler);

    }
@@ -114,12 +114,12 @@ class DicNode { 
    // Init for root with previous word

    void initAsRootWithPreviousWord(const DicNode *const dicNode, const int rootPtNodeArrayPos) {

        mIsCachedForNextSuggestion = dicNode->mIsCachedForNextSuggestion;

        int newPrevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];

        newPrevWordsPtNodePos[0] = dicNode->mDicNodeProperties.getPtNodePos();

        for (size_t i = 1; i < NELEMS(newPrevWordsPtNodePos); ++i) {

            newPrevWordsPtNodePos[i] = dicNode->getPrevWordsTerminalPtNodePos()[i - 1];

        int newPrevWordIds[MAX_PREV_WORD_COUNT_FOR_N_GRAM];

        newPrevWordIds[0] = dicNode->mDicNodeProperties.getWordId();

        for (size_t i = 1; i < NELEMS(newPrevWordIds); ++i) {

            newPrevWordIds[i] = dicNode->getPrevWordIds()[i - 1];

        }

        mDicNodeProperties.init(rootPtNodeArrayPos, newPrevWordsPtNodePos);

        mDicNodeProperties.init(rootPtNodeArrayPos, newPrevWordIds);

        mDicNodeState.initAsRootWithPreviousWord(&dicNode->mDicNodeState,

                dicNode->mDicNodeProperties.getDepth());

        PROF_NODE_COPY(&dicNode->mProfiler, mProfiler);
@@ -145,7 +145,7 @@ class DicNode { 
                dicNode->mDicNodeProperties.getLeavingDepth() + mergedNodeCodePointCount);

        mDicNodeProperties.init(ptNodePos, childrenPtNodeArrayPos, mergedNodeCodePoints[0],

                probability, wordId, hasChildren, isBlacklistedOrNotAWord, newDepth,

                newLeavingDepth, dicNode->mDicNodeProperties.getPrevWordsTerminalPtNodePos());

                newLeavingDepth, dicNode->mDicNodeProperties.getPrevWordIds());

        mDicNodeState.init(&dicNode->mDicNodeState, mergedNodeCodePointCount,

                mergedNodeCodePoints);

        PROF_NODE_COPY(&dicNode->mProfiler, mProfiler);
@@ -204,13 +204,18 @@ class DicNode { 
    }



    // Used to get n-gram probability in DicNodeUtils.

    int getWordId() const {

        return mDicNodeProperties.getWordId();

    }



    // TODO: Remove

    int getPtNodePos() const {

        return mDicNodeProperties.getPtNodePos();

    }



    // TODO: Use view class to return PtNodePos array.

    const int *getPrevWordsTerminalPtNodePos() const {

        return mDicNodeProperties.getPrevWordsTerminalPtNodePos();

    // TODO: Use view class to return word id array.

    const int *getPrevWordIds() const {

        return mDicNodeProperties.getPrevWordIds();

    }



    // Used in DicNodeUtils

native/jni/src/suggest/core/dicnode/dic_node_utils.cpp

+4 −4
Original line number Diff line number Diff line
@@ -29,8 +29,8 @@ namespace latinime { 


/* static */ void DicNodeUtils::initAsRoot(

        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,

        const int *const prevWordsPtNodePos, DicNode *const newRootDicNode) {

    newRootDicNode->initAsRoot(dictionaryStructurePolicy->getRootPosition(), prevWordsPtNodePos);

        const int *const prevWordIds, DicNode *const newRootDicNode) {

    newRootDicNode->initAsRoot(dictionaryStructurePolicy->getRootPosition(), prevWordIds);

}



/*static */ void DicNodeUtils::initAsRootWithPreviousWord(
@@ -86,9 +86,9 @@ namespace latinime { 
        const DicNode *const dicNode, MultiBigramMap *const multiBigramMap) {

    const int unigramProbability = dicNode->getProbability();

    if (multiBigramMap) {

        const int *const prevWordsPtNodePos = dicNode->getPrevWordsTerminalPtNodePos();

        const int *const prevWordIds = dicNode->getPrevWordIds();

        return multiBigramMap->getBigramProbability(dictionaryStructurePolicy,

                prevWordsPtNodePos, dicNode->getPtNodePos(), unigramProbability);

                prevWordIds, dicNode->getWordId(), unigramProbability);

    }

    return dictionaryStructurePolicy->getProbability(unigramProbability,

            NOT_A_PROBABILITY);

native/jni/src/suggest/core/dicnode/dic_node_utils.h

+1 −1
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ class DicNodeUtils { 
 public:

    static void initAsRoot(

            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,

            const int *const prevWordPtNodePos, DicNode *const newRootDicNode);

            const int *const prevWordIds, DicNode *const newRootDicNode);

    static void initAsRootWithPreviousWord(

            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,

            const DicNode *const prevWordLastDicNode, DicNode *const newRootDicNode);

native/jni/src/suggest/core/dicnode/internal/dic_node_properties.h

+13 −11
Original line number Diff line number Diff line
@@ -39,7 +39,7 @@ class DicNodeProperties { 
    // Should be called only once per DicNode is initialized.

    void init(const int pos, const int childrenPos, const int nodeCodePoint, const int probability,

            const int wordId, const bool hasChildren, const bool isBlacklistedOrNotAWord,

            const uint16_t depth, const uint16_t leavingDepth, const int *const prevWordsNodePos) {

            const uint16_t depth, const uint16_t leavingDepth, const int *const prevWordIds) {

        mPtNodePos = pos;

        mChildrenPtNodeArrayPos = childrenPos;

        mDicNodeCodePoint = nodeCodePoint;
@@ -49,11 +49,11 @@ class DicNodeProperties { 
        mIsBlacklistedOrNotAWord = isBlacklistedOrNotAWord;

        mDepth = depth;

        mLeavingDepth = leavingDepth;

        memmove(mPrevWordsTerminalPtNodePos, prevWordsNodePos, sizeof(mPrevWordsTerminalPtNodePos));

        memmove(mPrevWordIds, prevWordIds, sizeof(mPrevWordIds));

    }



    // Init for root with prevWordsPtNodePos which is used for n-gram

    void init(const int rootPtNodeArrayPos, const int *const prevWordsNodePos) {

    void init(const int rootPtNodeArrayPos, const int *const prevWordIds) {

        mPtNodePos = NOT_A_DICT_POS;

        mChildrenPtNodeArrayPos = rootPtNodeArrayPos;

        mDicNodeCodePoint = NOT_A_CODE_POINT;
@@ -63,7 +63,7 @@ class DicNodeProperties { 
        mIsBlacklistedOrNotAWord = false;

        mDepth = 0;

        mLeavingDepth = 0;

        memmove(mPrevWordsTerminalPtNodePos, prevWordsNodePos, sizeof(mPrevWordsTerminalPtNodePos));

        memmove(mPrevWordIds, prevWordIds, sizeof(mPrevWordIds));

    }



    void initByCopy(const DicNodeProperties *const dicNodeProp) {
@@ -76,8 +76,7 @@ class DicNodeProperties { 
        mIsBlacklistedOrNotAWord = dicNodeProp->mIsBlacklistedOrNotAWord;

        mDepth = dicNodeProp->mDepth;

        mLeavingDepth = dicNodeProp->mLeavingDepth;

        memmove(mPrevWordsTerminalPtNodePos, dicNodeProp->mPrevWordsTerminalPtNodePos,

                sizeof(mPrevWordsTerminalPtNodePos));

        memmove(mPrevWordIds, dicNodeProp->mPrevWordIds, sizeof(mPrevWordIds));

    }



    // Init as passing child
@@ -91,8 +90,7 @@ class DicNodeProperties { 
        mIsBlacklistedOrNotAWord = dicNodeProp->mIsBlacklistedOrNotAWord;

        mDepth = dicNodeProp->mDepth + 1; // Increment the depth of a passing child

        mLeavingDepth = dicNodeProp->mLeavingDepth;

        memmove(mPrevWordsTerminalPtNodePos, dicNodeProp->mPrevWordsTerminalPtNodePos,

                sizeof(mPrevWordsTerminalPtNodePos));

        memmove(mPrevWordIds, dicNodeProp->mPrevWordIds, sizeof(mPrevWordIds));

    }



    int getPtNodePos() const {
@@ -132,8 +130,12 @@ class DicNodeProperties { 
        return mIsBlacklistedOrNotAWord;

    }



    const int *getPrevWordsTerminalPtNodePos() const {

        return mPrevWordsTerminalPtNodePos;

    const int *getPrevWordIds() const {

        return mPrevWordIds;

    }



    int getWordId() const {

        return mWordId;

    }



 private:
@@ -149,7 +151,7 @@ class DicNodeProperties { 
    bool mIsBlacklistedOrNotAWord;

    uint16_t mDepth;

    uint16_t mLeavingDepth;

    int mPrevWordsTerminalPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];

    int mPrevWordIds[MAX_PREV_WORD_COUNT_FOR_N_GRAM];

};

} // namespace latinime

#endif // LATINIME_DIC_NODE_PROPERTIES_H

native/jni/src/suggest/core/dictionary/dictionary.cpp

+10 −12
Original line number Diff line number Diff line
@@ -93,11 +93,10 @@ void Dictionary::getPredictions(const PrevWordsInfo *const prevWordsInfo, 
    TimeKeeper::setCurrentTime();

    NgramListenerForPrediction listener(prevWordsInfo, outSuggestionResults,

            mDictionaryStructureWithBufferPolicy.get());

    int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];

    prevWordsInfo->getPrevWordsTerminalPtNodePos(

            mDictionaryStructureWithBufferPolicy.get(), prevWordsPtNodePos,

    int prevWordIds[MAX_PREV_WORD_COUNT_FOR_N_GRAM];

    prevWordsInfo->getPrevWordIds(mDictionaryStructureWithBufferPolicy.get(), prevWordIds,

            true /* tryLowerCaseSearch */);

    mDictionaryStructureWithBufferPolicy->iterateNgramEntries(prevWordsPtNodePos, &listener);

    mDictionaryStructureWithBufferPolicy->iterateNgramEntries(prevWordIds, &listener);

}



int Dictionary::getProbability(const int *word, int length) const {
@@ -113,18 +112,17 @@ int Dictionary::getMaxProbabilityOfExactMatches(const int *word, int length) con 
int Dictionary::getNgramProbability(const PrevWordsInfo *const prevWordsInfo, const int *word,

        int length) const {

    TimeKeeper::setCurrentTime();

    int nextWordPos = mDictionaryStructureWithBufferPolicy->getTerminalPtNodePositionOfWord(

    int wordId = mDictionaryStructureWithBufferPolicy->getWordId(

            CodePointArrayView(word, length), false /* forceLowerCaseSearch */);

    if (NOT_A_DICT_POS == nextWordPos) return NOT_A_PROBABILITY;

    if (wordId == NOT_A_WORD_ID) return NOT_A_PROBABILITY;

    if (!prevWordsInfo) {

        return getDictionaryStructurePolicy()->getProbabilityOfPtNode(

                nullptr /* prevWordsPtNodePos */, nextWordPos);

        return getDictionaryStructurePolicy()->getProbabilityOfWord(

                nullptr /* prevWordsPtNodePos */, wordId);

    }

    int prevWordsPtNodePos[MAX_PREV_WORD_COUNT_FOR_N_GRAM];

    prevWordsInfo->getPrevWordsTerminalPtNodePos(

            mDictionaryStructureWithBufferPolicy.get(), prevWordsPtNodePos,

    int prevWordIds[MAX_PREV_WORD_COUNT_FOR_N_GRAM];

    prevWordsInfo->getPrevWordIds(mDictionaryStructureWithBufferPolicy.get(), prevWordIds,

            true /* tryLowerCaseSearch */);

    return getDictionaryStructurePolicy()->getProbabilityOfPtNode(prevWordsPtNodePos, nextWordPos);

    return getDictionaryStructurePolicy()->getProbabilityOfWord(prevWordIds, wordId);

}



bool Dictionary::addUnigramEntry(const int *const word, const int length,