Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit dd7d95bd authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi Committed by Android (Google) Code Review
Browse files

Merge "GC step 4. Update all positions in new dict and add a test."

parents 50704b7d 2cfe7f9e

native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp

+5 −2
Original line number Diff line number Diff line
@@ -101,10 +101,13 @@ const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; 
/* static */ bool BigramListReadWriteUtils::writeBigramEntry(

        BufferWithExtendableBuffer *const bufferToWrite, const BigramFlags flags,

        const int targetPtNodePos, int *const writingPos) {

    if (!bufferToWrite->writeUintAndAdvancePosition(flags, 1 /* size */, writingPos)) {

    const int offset = (targetPtNodePos != NOT_A_DICT_POS) ?

            targetPtNodePos - (*writingPos + 1) : 0;

    const BigramFlags flagsToWrite = (offset < 0) ?

            (flags | FLAG_ATTRIBUTE_OFFSET_NEGATIVE) : flags;

    if (!bufferToWrite->writeUintAndAdvancePosition(flagsToWrite, 1 /* size */, writingPos)) {

        return false;

    }

    const int offset = (targetPtNodePos != NOT_A_DICT_POS) ? targetPtNodePos - *writingPos : 0;

    const uint32_t absOffest = abs(offset);

    const int bigramTargetFieldSize = attributeAddressSize(flags);

    return bufferToWrite->writeUintAndAdvancePosition(absOffest, bigramTargetFieldSize,

native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp

+57 −4
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"

#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h"

#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"

#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"

#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"



namespace latinime {
@@ -71,7 +72,8 @@ bool DynamicBigramListPolicy::copyAllBigrams(BufferWithExtendableBuffer *const b 
    int bigramEntryCount = 0;

    do {

        if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {

            AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",

                    bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            ASSERT(false);

            return false;

        }
@@ -114,7 +116,8 @@ bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries( 
    int bigramEntryCount = 0;

    do {

        if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {

            AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",

                    bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            ASSERT(false);

            return false;

        }
@@ -150,6 +153,54 @@ bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries( 
    return true;

}



// Updates bigram target PtNode positions in the list after the placing step in GC.

bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bigramListPos,

        const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const

                ptNodePositionRelocationMap) {

    const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);

    if (usesAdditionalBuffer) {

        *bigramListPos -= mBuffer->getOriginalBufferSize();

    }

    BigramListReadWriteUtils::BigramFlags bigramFlags;

    int bigramEntryCount = 0;

    do {

        if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {

            AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",

                    bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            ASSERT(false);

            return false;

        }

        int bigramEntryPos = *bigramListPos;

        if (usesAdditionalBuffer) {

            bigramEntryPos += mBuffer->getOriginalBufferSize();

        }

        int bigramTargetPtNodePos;

        // The buffer address can be changed after calling buffer writing methods.

        BigramListReadWriteUtils::getBigramEntryPropertiesAndAdvancePosition(

                mBuffer->getBuffer(usesAdditionalBuffer), &bigramFlags, &bigramTargetPtNodePos,

                bigramListPos);

        if (bigramTargetPtNodePos == NOT_A_DICT_POS) {

            continue;

        }

        if (usesAdditionalBuffer) {

            bigramTargetPtNodePos += mBuffer->getOriginalBufferSize();

        }



        DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it =

                ptNodePositionRelocationMap->find(bigramTargetPtNodePos);

        if (it != ptNodePositionRelocationMap->end()) {

            bigramTargetPtNodePos = it->second;

        } else {

            bigramTargetPtNodePos = NOT_A_DICT_POS;

        }

        if (!BigramListReadWriteUtils::writeBigramEntry(mBuffer, bigramFlags,

                bigramTargetPtNodePos, &bigramEntryPos)) {

            return false;

        }

    } while(BigramListReadWriteUtils::hasNext(bigramFlags));

    return true;

}



bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTargetPos,

        const int probability, int *const bigramListPos) {

    const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);
@@ -160,7 +211,8 @@ bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTarg 
    int bigramEntryCount = 0;

    do {

        if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {

            AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",

                    bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            ASSERT(false);

            return false;

        }
@@ -222,7 +274,8 @@ bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int bi 
    int bigramEntryCount = 0;

    do {

        if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {

            AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",

                    bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            ASSERT(false);

            return false;

        }

native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h

+5 −0
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ 


#include "defines.h"

#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"

#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"



namespace latinime {
@@ -51,6 +52,10 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { 


    bool updateAllBigramEntriesAndDeleteUselessEntries(int *const bigramListPos);



    bool updateAllBigramTargetPtNodePositions(int *const bigramListPos,

            const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const

                    ptNodePositionRelocationMap);



    bool addNewBigramEntryToBigramList(const int bigramTargetPos, const int probability,

            int *const bigramListPos);

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp

+59 −8
Original line number Diff line number Diff line
@@ -19,7 +19,7 @@ 
namespace latinime {



bool DynamicPatriciaTrieGcEventListeners

        ::ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted

        ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted

                ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,

                        const int *const nodeCodePoints) {

    // PtNode is useless when the PtNode is not a terminal and doesn't have any not useless
@@ -47,11 +47,13 @@ bool DynamicPatriciaTrieGcEventListeners 
}



// Writes dummy PtNode array size when the head of PtNode array is read.

bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer

bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer

        ::onDescend(const int ptNodeArrayPos) {

    mValidPtNodeCount = 0;

    int writingPos = mBufferToWrite->getTailPosition();

    mPositionMap->insert(hash_map_compat<int, int>::value_type(ptNodeArrayPos,  writingPos));

    mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.insert(

            DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::value_type(

                    ptNodeArrayPos, writingPos));

    // Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes.

    // This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count.

    mPtNodeArraySizeFieldPos = writingPos;
@@ -60,7 +62,7 @@ bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNod 
}



// Write PtNode array terminal and actual PtNode array size.

bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer

bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer

        ::onReadingPtNodeArrayTail() {

    int writingPos = mBufferToWrite->getTailPosition();

    // Write PtNode array terminal.
@@ -77,17 +79,20 @@ bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNod 
}



// Write valid PtNode to buffer and memorize mapping from the old position to the new position.

bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer

bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer

        ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,

                const int *const nodeCodePoints) {

    if (node->isDeleted()) {

        // Current PtNode is not written in new buffer because it has been deleted.

        mPositionMap->insert(hash_map_compat<int, int>::value_type(node->getHeadPos(),

                NOT_A_DICT_POS));

        mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert(

                DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type(

                        node->getHeadPos(), NOT_A_DICT_POS));

        return true;

    }

    int writingPos = mBufferToWrite->getTailPosition();

    mPositionMap->insert(hash_map_compat<int, int>::value_type(node->getHeadPos(), writingPos));

    mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert(

            DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type(

                    node->getHeadPos(), writingPos));

    mValidPtNodeCount++;

    // Writes current PtNode.

    return mWritingHelper->writePtNodeToBufferByCopyingPtNodeInfo(mBufferToWrite, node,
@@ -95,4 +100,50 @@ bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNod 
            node->getProbability(), &writingPos);

}



bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields

        ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,

                const int *const nodeCodePoints) {

    // Updates parent position.

    int parentPos = node->getParentPos();

    if (parentPos != NOT_A_DICT_POS) {

        DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it =

                mDictPositionRelocationMap->mPtNodePositionRelocationMap.find(parentPos);

        if (it != mDictPositionRelocationMap->mPtNodePositionRelocationMap.end()) {

            parentPos = it->second;

        }

    }

    int writingPos = node->getHeadPos() + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE;

    const int parentPosOffset = (parentPos != NOT_A_DICT_POS) ?

            parentPos - node->getHeadPos() : NOT_A_DICT_POS;

    // Write updated parent offset.

    if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(mBufferToWrite,

            parentPosOffset, &writingPos)) {

        return false;

    }



    // Updates children position.

    int childrenPos = node->getChildrenPos();

    if (childrenPos != NOT_A_DICT_POS) {

        DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::const_iterator it =

                mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.find(childrenPos);

        if (it != mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.end()) {

            childrenPos = it->second;

        }

    }

    writingPos = node->getChildrenPosFieldPos();

    if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBufferToWrite,

            childrenPos, &writingPos)) {

        return false;

    }



    // Updates bigram target PtNode positions in the bigram list.

    int bigramsPos = node->getBigramsPos();

    if (bigramsPos != NOT_A_DICT_POS) {

        mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos,

                &mDictPositionRelocationMap->mPtNodePositionRelocationMap);

    }



    return true;

}



} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h

+47 −17
Original line number Diff line number Diff line
@@ -34,16 +34,16 @@ class DynamicPatriciaTrieGcEventListeners { 
    // Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or

    // not and marks useless PtNodes as deleted. Such deleted PtNodes will be discarded in the GC.

    // TODO: Concatenate non-terminal PtNodes.

    class ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted

    class TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted

        : public DynamicPatriciaTrieReadingHelper::TraversingEventListener {

     public:

        ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted(

        TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(

                DynamicPatriciaTrieWritingHelper *const writingHelper,

                BufferWithExtendableBuffer *const buffer)

                : mWritingHelper(writingHelper), mBuffer(buffer), valueStack(),

                  mChildrenValue(0) {}



        ~ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted() {};

        ~TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted() {};



        bool onAscend() {

            if (valueStack.empty()) {
@@ -66,7 +66,7 @@ class DynamicPatriciaTrieGcEventListeners { 


     private:

        DISALLOW_IMPLICIT_CONSTRUCTORS(

                ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted);

                TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted);



        DynamicPatriciaTrieWritingHelper *const mWritingHelper;

        BufferWithExtendableBuffer *const mBuffer;
@@ -76,10 +76,10 @@ class DynamicPatriciaTrieGcEventListeners { 


    // Updates all bigram entries that are held by valid PtNodes. This removes useless bigram

    // entries.

    class ListenerForUpdatingBigramProbability

    class TraversePolicyToUpdateBigramProbability

            : public DynamicPatriciaTrieReadingHelper::TraversingEventListener {

     public:

        ListenerForUpdatingBigramProbability(DynamicBigramListPolicy *const bigramPolicy)

        TraversePolicyToUpdateBigramProbability(DynamicBigramListPolicy *const bigramPolicy)

                : mBigramPolicy(bigramPolicy) {}



        bool onAscend() { return true; }
@@ -102,20 +102,21 @@ class DynamicPatriciaTrieGcEventListeners { 
        }



     private:

        DISALLOW_IMPLICIT_CONSTRUCTORS(ListenerForUpdatingBigramProbability);

        DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateBigramProbability);



        DynamicBigramListPolicy *const mBigramPolicy;

    };



    class ListenerForPlacingAndWritingValidPtNodesToBuffer

    class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer

            : public DynamicPatriciaTrieReadingHelper::TraversingEventListener {

     public:

        ListenerForPlacingAndWritingValidPtNodesToBuffer(

        TraversePolicyToPlaceAndWriteValidPtNodesToBuffer(

                DynamicPatriciaTrieWritingHelper *const writingHelper,

                BufferWithExtendableBuffer *const bufferToWrite,

                hash_map_compat<int, int> *const positionMap)

                DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const

                        dictPositionRelocationMap)

                : mWritingHelper(writingHelper), mBufferToWrite(bufferToWrite),

                  mPositionMap(positionMap), mValidPtNodeCount(0),

                  mDictPositionRelocationMap(dictPositionRelocationMap), mValidPtNodeCount(0),

                  mPtNodeArraySizeFieldPos(NOT_A_DICT_POS) {};



        bool onAscend() { return true; }
@@ -127,20 +128,49 @@ class DynamicPatriciaTrieGcEventListeners { 
        bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,

                const int *const nodeCodePoints);



        hash_map_compat<int, int> *getPositionMap() const {

            return mPositionMap;

        }



     private:

        DISALLOW_IMPLICIT_CONSTRUCTORS(ListenerForPlacingAndWritingValidPtNodesToBuffer);

        DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToPlaceAndWriteValidPtNodesToBuffer);



        DynamicPatriciaTrieWritingHelper *const mWritingHelper;

        BufferWithExtendableBuffer *const mBufferToWrite;

        hash_map_compat<int, int> *const mPositionMap;

        DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const

                mDictPositionRelocationMap;

        int mValidPtNodeCount;

        int mPtNodeArraySizeFieldPos;

    };



    class TraversePolicyToUpdateAllPositionFields

            : public DynamicPatriciaTrieReadingHelper::TraversingEventListener {

     public:

        TraversePolicyToUpdateAllPositionFields(

                DynamicPatriciaTrieWritingHelper *const writingHelper,

                DynamicBigramListPolicy *const bigramPolicy,

                BufferWithExtendableBuffer *const bufferToWrite,

                const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const

                        dictPositionRelocationMap)

                : mWritingHelper(writingHelper), mBigramPolicy(bigramPolicy),

                  mBufferToWrite(bufferToWrite),

                  mDictPositionRelocationMap(dictPositionRelocationMap) {};



        bool onAscend() { return true; }



        bool onDescend(const int ptNodeArrayPos) { return true; }



        bool onReadingPtNodeArrayTail() { return true; }



        bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,

                const int *const nodeCodePoints);



     private:

        DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateAllPositionFields);



        DynamicPatriciaTrieWritingHelper *const mWritingHelper;

        DynamicBigramListPolicy *const mBigramPolicy;

        BufferWithExtendableBuffer *const mBufferToWrite;

        const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const

                mDictPositionRelocationMap;

    };



 private:

    DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieGcEventListeners);

};