Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 2cfe7f9e authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi
Browse files

GC step 4. Update all positions in new dict and add a test. 

Bug: 6669677
Change-Id: I035d848a210653dcd3c1a1f21781968adced9225
parent 0355bf2d

native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp

+5 −2
Original line number Diff line number Diff line
@@ -101,10 +101,13 @@ const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; 
/* static */ bool BigramListReadWriteUtils::writeBigramEntry(

        BufferWithExtendableBuffer *const bufferToWrite, const BigramFlags flags,

        const int targetPtNodePos, int *const writingPos) {

    if (!bufferToWrite->writeUintAndAdvancePosition(flags, 1 /* size */, writingPos)) {

    const int offset = (targetPtNodePos != NOT_A_DICT_POS) ?

            targetPtNodePos - (*writingPos + 1) : 0;

    const BigramFlags flagsToWrite = (offset < 0) ?

            (flags | FLAG_ATTRIBUTE_OFFSET_NEGATIVE) : flags;

    if (!bufferToWrite->writeUintAndAdvancePosition(flagsToWrite, 1 /* size */, writingPos)) {

        return false;

    }

    const int offset = (targetPtNodePos != NOT_A_DICT_POS) ? targetPtNodePos - *writingPos : 0;

    const uint32_t absOffest = abs(offset);

    const int bigramTargetFieldSize = attributeAddressSize(flags);

    return bufferToWrite->writeUintAndAdvancePosition(absOffest, bigramTargetFieldSize,

native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp

+57 −4
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ 
#include "suggest/core/policy/dictionary_shortcuts_structure_policy.h"

#include "suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h"

#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"

#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"

#include "suggest/policyimpl/dictionary/utils/buffer_with_extendable_buffer.h"



namespace latinime {
@@ -71,7 +72,8 @@ bool DynamicBigramListPolicy::copyAllBigrams(BufferWithExtendableBuffer *const b 
    int bigramEntryCount = 0;

    do {

        if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {

            AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",

                    bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            ASSERT(false);

            return false;

        }
@@ -114,7 +116,8 @@ bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries( 
    int bigramEntryCount = 0;

    do {

        if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {

            AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",

                    bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            ASSERT(false);

            return false;

        }
@@ -150,6 +153,54 @@ bool DynamicBigramListPolicy::updateAllBigramEntriesAndDeleteUselessEntries( 
    return true;

}



// Updates bigram target PtNode positions in the list after the placing step in GC.

bool DynamicBigramListPolicy::updateAllBigramTargetPtNodePositions(int *const bigramListPos,

        const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const

                ptNodePositionRelocationMap) {

    const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);

    if (usesAdditionalBuffer) {

        *bigramListPos -= mBuffer->getOriginalBufferSize();

    }

    BigramListReadWriteUtils::BigramFlags bigramFlags;

    int bigramEntryCount = 0;

    do {

        if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {

            AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",

                    bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            ASSERT(false);

            return false;

        }

        int bigramEntryPos = *bigramListPos;

        if (usesAdditionalBuffer) {

            bigramEntryPos += mBuffer->getOriginalBufferSize();

        }

        int bigramTargetPtNodePos;

        // The buffer address can be changed after calling buffer writing methods.

        BigramListReadWriteUtils::getBigramEntryPropertiesAndAdvancePosition(

                mBuffer->getBuffer(usesAdditionalBuffer), &bigramFlags, &bigramTargetPtNodePos,

                bigramListPos);

        if (bigramTargetPtNodePos == NOT_A_DICT_POS) {

            continue;

        }

        if (usesAdditionalBuffer) {

            bigramTargetPtNodePos += mBuffer->getOriginalBufferSize();

        }



        DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it =

                ptNodePositionRelocationMap->find(bigramTargetPtNodePos);

        if (it != ptNodePositionRelocationMap->end()) {

            bigramTargetPtNodePos = it->second;

        } else {

            bigramTargetPtNodePos = NOT_A_DICT_POS;

        }

        if (!BigramListReadWriteUtils::writeBigramEntry(mBuffer, bigramFlags,

                bigramTargetPtNodePos, &bigramEntryPos)) {

            return false;

        }

    } while(BigramListReadWriteUtils::hasNext(bigramFlags));

    return true;

}



bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTargetPos,

        const int probability, int *const bigramListPos) {

    const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*bigramListPos);
@@ -160,7 +211,8 @@ bool DynamicBigramListPolicy::addNewBigramEntryToBigramList(const int bigramTarg 
    int bigramEntryCount = 0;

    do {

        if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {

            AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",

                    bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            ASSERT(false);

            return false;

        }
@@ -222,7 +274,8 @@ bool DynamicBigramListPolicy::removeBigram(const int bigramListPos, const int bi 
    int bigramEntryCount = 0;

    do {

        if (++bigramEntryCount > BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT) {

            AKLOGE("Too many bigram entries. %d", BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            AKLOGE("Too many bigram entries. Entry count: %d, Limit: %d",

                    bigramEntryCount, BIGRAM_ENTRY_COUNT_IN_A_BIGRAM_LIST_LIMIT);

            ASSERT(false);

            return false;

        }

native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h

+5 −0
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ 


#include "defines.h"

#include "suggest/core/policy/dictionary_bigrams_structure_policy.h"

#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"



namespace latinime {
@@ -51,6 +52,10 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { 


    bool updateAllBigramEntriesAndDeleteUselessEntries(int *const bigramListPos);



    bool updateAllBigramTargetPtNodePositions(int *const bigramListPos,

            const DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap *const

                    ptNodePositionRelocationMap);



    bool addNewBigramEntryToBigramList(const int bigramTargetPos, const int probability,

            int *const bigramListPos);

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.cpp

+59 −8
Original line number Diff line number Diff line
@@ -19,7 +19,7 @@ 
namespace latinime {



bool DynamicPatriciaTrieGcEventListeners

        ::ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted

        ::TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted

                ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,

                        const int *const nodeCodePoints) {

    // PtNode is useless when the PtNode is not a terminal and doesn't have any not useless
@@ -47,11 +47,13 @@ bool DynamicPatriciaTrieGcEventListeners 
}



// Writes dummy PtNode array size when the head of PtNode array is read.

bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer

bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer

        ::onDescend(const int ptNodeArrayPos) {

    mValidPtNodeCount = 0;

    int writingPos = mBufferToWrite->getTailPosition();

    mPositionMap->insert(hash_map_compat<int, int>::value_type(ptNodeArrayPos,  writingPos));

    mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.insert(

            DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::value_type(

                    ptNodeArrayPos, writingPos));

    // Writes dummy PtNode array size because arrays can have a forward link or needles PtNodes.

    // This field will be updated later in onReadingPtNodeArrayTail() with actual PtNode count.

    mPtNodeArraySizeFieldPos = writingPos;
@@ -60,7 +62,7 @@ bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNod 
}



// Write PtNode array terminal and actual PtNode array size.

bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer

bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer

        ::onReadingPtNodeArrayTail() {

    int writingPos = mBufferToWrite->getTailPosition();

    // Write PtNode array terminal.
@@ -77,17 +79,20 @@ bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNod 
}



// Write valid PtNode to buffer and memorize mapping from the old position to the new position.

bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNodesToBuffer

bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToPlaceAndWriteValidPtNodesToBuffer

        ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,

                const int *const nodeCodePoints) {

    if (node->isDeleted()) {

        // Current PtNode is not written in new buffer because it has been deleted.

        mPositionMap->insert(hash_map_compat<int, int>::value_type(node->getHeadPos(),

                NOT_A_DICT_POS));

        mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert(

                DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type(

                        node->getHeadPos(), NOT_A_DICT_POS));

        return true;

    }

    int writingPos = mBufferToWrite->getTailPosition();

    mPositionMap->insert(hash_map_compat<int, int>::value_type(node->getHeadPos(), writingPos));

    mDictPositionRelocationMap->mPtNodePositionRelocationMap.insert(

            DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::value_type(

                    node->getHeadPos(), writingPos));

    mValidPtNodeCount++;

    // Writes current PtNode.

    return mWritingHelper->writePtNodeToBufferByCopyingPtNodeInfo(mBufferToWrite, node,
@@ -95,4 +100,50 @@ bool DynamicPatriciaTrieGcEventListeners::ListenerForPlacingAndWritingValidPtNod 
            node->getProbability(), &writingPos);

}



bool DynamicPatriciaTrieGcEventListeners::TraversePolicyToUpdateAllPositionFields

        ::onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,

                const int *const nodeCodePoints) {

    // Updates parent position.

    int parentPos = node->getParentPos();

    if (parentPos != NOT_A_DICT_POS) {

        DynamicPatriciaTrieWritingHelper::PtNodePositionRelocationMap::const_iterator it =

                mDictPositionRelocationMap->mPtNodePositionRelocationMap.find(parentPos);

        if (it != mDictPositionRelocationMap->mPtNodePositionRelocationMap.end()) {

            parentPos = it->second;

        }

    }

    int writingPos = node->getHeadPos() + DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE;

    const int parentPosOffset = (parentPos != NOT_A_DICT_POS) ?

            parentPos - node->getHeadPos() : NOT_A_DICT_POS;

    // Write updated parent offset.

    if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(mBufferToWrite,

            parentPosOffset, &writingPos)) {

        return false;

    }



    // Updates children position.

    int childrenPos = node->getChildrenPos();

    if (childrenPos != NOT_A_DICT_POS) {

        DynamicPatriciaTrieWritingHelper::PtNodeArrayPositionRelocationMap::const_iterator it =

                mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.find(childrenPos);

        if (it != mDictPositionRelocationMap->mPtNodeArrayPositionRelocationMap.end()) {

            childrenPos = it->second;

        }

    }

    writingPos = node->getChildrenPosFieldPos();

    if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBufferToWrite,

            childrenPos, &writingPos)) {

        return false;

    }



    // Updates bigram target PtNode positions in the bigram list.

    int bigramsPos = node->getBigramsPos();

    if (bigramsPos != NOT_A_DICT_POS) {

        mBigramPolicy->updateAllBigramTargetPtNodePositions(&bigramsPos,

                &mDictPositionRelocationMap->mPtNodePositionRelocationMap);

    }



    return true;

}



} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h

+47 −17
Original line number Diff line number Diff line
@@ -34,16 +34,16 @@ class DynamicPatriciaTrieGcEventListeners { 
    // Updates all PtNodes that can be reached from the root. Checks if each PtNode is useless or

    // not and marks useless PtNodes as deleted. Such deleted PtNodes will be discarded in the GC.

    // TODO: Concatenate non-terminal PtNodes.

    class ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted

    class TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted

        : public DynamicPatriciaTrieReadingHelper::TraversingEventListener {

     public:

        ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted(

        TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted(

                DynamicPatriciaTrieWritingHelper *const writingHelper,

                BufferWithExtendableBuffer *const buffer)

                : mWritingHelper(writingHelper), mBuffer(buffer), valueStack(),

                  mChildrenValue(0) {}



        ~ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted() {};

        ~TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted() {};



        bool onAscend() {

            if (valueStack.empty()) {
@@ -66,7 +66,7 @@ class DynamicPatriciaTrieGcEventListeners { 


     private:

        DISALLOW_IMPLICIT_CONSTRUCTORS(

                ListenerForUpdatingUnigramProbabilityAndMarkingUselessPtNodesAsDeleted);

                TraversePolicyToUpdateUnigramProbabilityAndMarkUselessPtNodesAsDeleted);



        DynamicPatriciaTrieWritingHelper *const mWritingHelper;

        BufferWithExtendableBuffer *const mBuffer;
@@ -76,10 +76,10 @@ class DynamicPatriciaTrieGcEventListeners { 


    // Updates all bigram entries that are held by valid PtNodes. This removes useless bigram

    // entries.

    class ListenerForUpdatingBigramProbability

    class TraversePolicyToUpdateBigramProbability

            : public DynamicPatriciaTrieReadingHelper::TraversingEventListener {

     public:

        ListenerForUpdatingBigramProbability(DynamicBigramListPolicy *const bigramPolicy)

        TraversePolicyToUpdateBigramProbability(DynamicBigramListPolicy *const bigramPolicy)

                : mBigramPolicy(bigramPolicy) {}



        bool onAscend() { return true; }
@@ -102,20 +102,21 @@ class DynamicPatriciaTrieGcEventListeners { 
        }



     private:

        DISALLOW_IMPLICIT_CONSTRUCTORS(ListenerForUpdatingBigramProbability);

        DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateBigramProbability);



        DynamicBigramListPolicy *const mBigramPolicy;

    };



    class ListenerForPlacingAndWritingValidPtNodesToBuffer

    class TraversePolicyToPlaceAndWriteValidPtNodesToBuffer

            : public DynamicPatriciaTrieReadingHelper::TraversingEventListener {

     public:

        ListenerForPlacingAndWritingValidPtNodesToBuffer(

        TraversePolicyToPlaceAndWriteValidPtNodesToBuffer(

                DynamicPatriciaTrieWritingHelper *const writingHelper,

                BufferWithExtendableBuffer *const bufferToWrite,

                hash_map_compat<int, int> *const positionMap)

                DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const

                        dictPositionRelocationMap)

                : mWritingHelper(writingHelper), mBufferToWrite(bufferToWrite),

                  mPositionMap(positionMap), mValidPtNodeCount(0),

                  mDictPositionRelocationMap(dictPositionRelocationMap), mValidPtNodeCount(0),

                  mPtNodeArraySizeFieldPos(NOT_A_DICT_POS) {};



        bool onAscend() { return true; }
@@ -127,20 +128,49 @@ class DynamicPatriciaTrieGcEventListeners { 
        bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,

                const int *const nodeCodePoints);



        hash_map_compat<int, int> *getPositionMap() const {

            return mPositionMap;

        }



     private:

        DISALLOW_IMPLICIT_CONSTRUCTORS(ListenerForPlacingAndWritingValidPtNodesToBuffer);

        DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToPlaceAndWriteValidPtNodesToBuffer);



        DynamicPatriciaTrieWritingHelper *const mWritingHelper;

        BufferWithExtendableBuffer *const mBufferToWrite;

        hash_map_compat<int, int> *const mPositionMap;

        DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const

                mDictPositionRelocationMap;

        int mValidPtNodeCount;

        int mPtNodeArraySizeFieldPos;

    };



    class TraversePolicyToUpdateAllPositionFields

            : public DynamicPatriciaTrieReadingHelper::TraversingEventListener {

     public:

        TraversePolicyToUpdateAllPositionFields(

                DynamicPatriciaTrieWritingHelper *const writingHelper,

                DynamicBigramListPolicy *const bigramPolicy,

                BufferWithExtendableBuffer *const bufferToWrite,

                const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const

                        dictPositionRelocationMap)

                : mWritingHelper(writingHelper), mBigramPolicy(bigramPolicy),

                  mBufferToWrite(bufferToWrite),

                  mDictPositionRelocationMap(dictPositionRelocationMap) {};



        bool onAscend() { return true; }



        bool onDescend(const int ptNodeArrayPos) { return true; }



        bool onReadingPtNodeArrayTail() { return true; }



        bool onVisitingPtNode(const DynamicPatriciaTrieNodeReader *const node,

                const int *const nodeCodePoints);



     private:

        DISALLOW_IMPLICIT_CONSTRUCTORS(TraversePolicyToUpdateAllPositionFields);



        DynamicPatriciaTrieWritingHelper *const mWritingHelper;

        DynamicBigramListPolicy *const mBigramPolicy;

        BufferWithExtendableBuffer *const mBufferToWrite;

        const DynamicPatriciaTrieWritingHelper::DictPositionRelocationMap *const

                mDictPositionRelocationMap;

    };



 private:

    DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieGcEventListeners);

};