Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c7f1de82 authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi Committed by Android (Google) Code Review
Browse files

Merge "Use LanguageModelDictContent in getWordProperty()."

parents 8b5788d7 8b4409f4

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.cpp

+25 −46
Original line number Diff line number Diff line
@@ -188,18 +188,6 @@ int Ver4PatriciaTriePolicy::getShortcutPositionOfPtNode(const int ptNodePos) con 
            ptNodeParams.getTerminalId());

}



int Ver4PatriciaTriePolicy::getBigramsPositionOfPtNode(const int ptNodePos) const {

    if (ptNodePos == NOT_A_DICT_POS) {

        return NOT_A_DICT_POS;

    }

    const PtNodeParams ptNodeParams(mNodeReader.fetchPtNodeParamsInBufferFromPtNodePos(ptNodePos));

    if (ptNodeParams.isDeleted()) {

        return NOT_A_DICT_POS;

    }

    return mBuffers->getBigramDictContent()->getBigramListHeadPos(

            ptNodeParams.getTerminalId());

}



bool Ver4PatriciaTriePolicy::addUnigramEntry(const int *const word, const int length,

        const UnigramProperty *const unigramProperty) {

    if (!mBuffers->isUpdatable()) {
@@ -480,25 +468,17 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code 
                    ptNodeParams.getTerminalId());

    const HistoricalInfo *const historicalInfo = probabilityEntry.getHistoricalInfo();

    // Fetch bigram information.

    // TODO: Support n-gram.

    std::vector<BigramProperty> bigrams;

    const int bigramListPos = getBigramsPositionOfPtNode(ptNodePos);

    if (bigramListPos != NOT_A_DICT_POS) {

        int bigramWord1CodePoints[MAX_WORD_LENGTH];

        const BigramDictContent *const bigramDictContent = mBuffers->getBigramDictContent();

    const int wordId = ptNodeParams.getTerminalId();

    const WordIdArrayView prevWordIds = WordIdArrayView::fromObject(&wordId);

    const TerminalPositionLookupTable *const terminalPositionLookupTable =

            mBuffers->getTerminalPositionLookupTable();

        bool hasNext = true;

        int readingPos = bigramListPos;

        while (hasNext) {

            const BigramEntry bigramEntry =

                    bigramDictContent->getBigramEntryAndAdvancePosition(&readingPos);

            hasNext = bigramEntry.hasNext();

            const int word1TerminalId = bigramEntry.getTargetTerminalId();

    int bigramWord1CodePoints[MAX_WORD_LENGTH];

    for (const auto entry : mBuffers->getLanguageModelDictContent()->getProbabilityEntries(

            prevWordIds)) {

        const int word1TerminalPtNodePos =

                    terminalPositionLookupTable->getTerminalPtNodePosition(word1TerminalId);

            if (word1TerminalPtNodePos == NOT_A_DICT_POS) {

                continue;

            }

                terminalPositionLookupTable->getTerminalPtNodePosition(entry.getWordId());

        // Word (unigram) probability

        int word1Probability = NOT_A_PROBABILITY;

        const int codePointCount = getCodePointsAndProbabilityAndReturnCodePointCount(
@@ -506,16 +486,15 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code 
                &word1Probability);

        const std::vector<int> word1(bigramWord1CodePoints,

                bigramWord1CodePoints + codePointCount);

            const HistoricalInfo *const historicalInfo = bigramEntry.getHistoricalInfo();

            const int probability = bigramEntry.hasHistoricalInfo() ?

                    ForgettingCurveUtils::decodeProbability(

                            bigramEntry.getHistoricalInfo(), mHeaderPolicy) :

                    bigramEntry.getProbability();

        const ProbabilityEntry probabilityEntry = entry.getProbabilityEntry();

        const HistoricalInfo *const historicalInfo = probabilityEntry.getHistoricalInfo();

        const int probability = probabilityEntry.hasHistoricalInfo() ?

                ForgettingCurveUtils::decodeProbability(historicalInfo, mHeaderPolicy) :

                probabilityEntry.getProbability();

        bigrams.emplace_back(&word1, probability,

                historicalInfo->getTimeStamp(), historicalInfo->getLevel(),

                historicalInfo->getCount());

    }

    }

    // Fetch shortcut information.

    std::vector<UnigramProperty::ShortcutProperty> shortcuts;

    int shortcutPos = getShortcutPositionOfPtNode(ptNodePos);

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_patricia_trie_policy.h

+0 −2
Original line number Diff line number Diff line
@@ -143,8 +143,6 @@ class Ver4PatriciaTriePolicy : public DictionaryStructureWithBufferPolicy { 
    int mBigramCount;

    std::vector<int> mTerminalPtNodePositionsForIteratingWords;

    mutable bool mIsCorrupted;



    int getBigramsPositionOfPtNode(const int ptNodePos) const;

};

} // namespace latinime

#endif // LATINIME_VER4_PATRICIA_TRIE_POLICY_H