Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit e1ebef61 authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi
Browse files

Make DictionaryStructurePolicy have buffer info. 

To support various format dictionary, extendable
buffers or multiple files.

Bug: 6669677
Change-Id: I203a5a4adc049a6322cfba4eacb4cb6715f8dfc2
parent 5c057b32

native/jni/Android.mk

+1 −0
Original line number Diff line number Diff line
@@ -71,6 +71,7 @@ LATIN_IME_CORE_SRC_FILES := \ 
    suggest/core/policy/weighting.cpp \

    suggest/core/session/dic_traverse_session.cpp \

    $(addprefix suggest/policyimpl/dictionary/, \

        dictionary_structure_with_buffer_policy_factory.cpp \

        dynamic_patricia_trie_node_reader.cpp \

        dynamic_patricia_trie_policy.cpp \

        dynamic_patricia_trie_reading_utils.cpp \

native/jni/src/suggest/core/dicnode/dic_node_proximity_filter.h

+2 −2
Original line number Diff line number Diff line
@@ -20,11 +20,11 @@ 
#include "defines.h"

#include "suggest/core/layout/proximity_info_state.h"

#include "suggest/core/layout/proximity_info_utils.h"

#include "suggest/core/policy/dictionary_structure_policy.h"

#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"



namespace latinime {



class DicNodeProximityFilter : public DictionaryStructurePolicy::NodeFilter {

class DicNodeProximityFilter : public DictionaryStructureWithBufferPolicy::NodeFilter {

 public:

    DicNodeProximityFilter(const ProximityInfoState *const pInfoState,

            const int pointIndex, const bool exactOnly)

native/jni/src/suggest/core/dicnode/dic_node_utils.cpp

+2 −2
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ 
#include "suggest/core/dictionary/binary_dictionary_info.h"

#include "suggest/core/dictionary/multi_bigram_map.h"

#include "suggest/core/dictionary/probability_utils.h"

#include "suggest/core/policy/dictionary_structure_policy.h"

#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"

#include "utils/char_utils.h"



namespace latinime {
@@ -83,7 +83,7 @@ namespace latinime { 
        DicNodeUtils::createAndGetPassingChildNode(dicNode, &childrenFilter, childDicNodes);

    } else {

        binaryDictionaryInfo->getStructurePolicy()->createAndGetAllChildNodes(dicNode,

                binaryDictionaryInfo, &childrenFilter, childDicNodes);

                &childrenFilter, childDicNodes);

    }

}

native/jni/src/suggest/core/dictionary/bigram_dictionary.cpp

+5 −7
Original line number Diff line number Diff line
@@ -116,9 +116,8 @@ int BigramDictionary::getPredictions(const int *prevWord, const int prevWordLeng 
    while (bigramsIt.hasNext()) {

        bigramsIt.next();

        const int length = mBinaryDictionaryInfo->getStructurePolicy()->

                getCodePointsAndProbabilityAndReturnCodePointCount(

                        mBinaryDictionaryInfo, bigramsIt.getBigramPos(), MAX_WORD_LENGTH,

                        bigramBuffer, &unigramProbability);

                getCodePointsAndProbabilityAndReturnCodePointCount(bigramsIt.getBigramPos(),

                        MAX_WORD_LENGTH, bigramBuffer, &unigramProbability);

        // Due to space constraints, the probability for bigrams is approximate - the lower the

        // unigram probability, the worse the precision. The theoritical maximum error in

        // resulting probability is 8 - although in the practice it's never bigger than 3 or 4
@@ -139,10 +138,9 @@ int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const in 
        const bool forceLowerCaseSearch) const {

    if (0 >= prevWordLength) return NOT_A_DICT_POS;

    int pos = mBinaryDictionaryInfo->getStructurePolicy()->getTerminalNodePositionOfWord(

            mBinaryDictionaryInfo, prevWord, prevWordLength, forceLowerCaseSearch);

            prevWord, prevWordLength, forceLowerCaseSearch);

    if (NOT_A_VALID_WORD_POS == pos) return NOT_A_DICT_POS;

    return mBinaryDictionaryInfo->getStructurePolicy()->getBigramsPositionOfNode(

            mBinaryDictionaryInfo, pos);

    return mBinaryDictionaryInfo->getStructurePolicy()->getBigramsPositionOfNode(pos);

}



bool BigramDictionary::isValidBigram(const int *word0, int length0, const int *word1,
@@ -151,7 +149,7 @@ bool BigramDictionary::isValidBigram(const int *word0, int length0, const int *w 
    // getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams

    if (NOT_A_DICT_POS == pos) return false;

    int nextWordPos = mBinaryDictionaryInfo->getStructurePolicy()->getTerminalNodePositionOfWord(

            mBinaryDictionaryInfo, word1, length1, false /* forceLowerCaseSearch */);

            word1, length1, false /* forceLowerCaseSearch */);

    if (NOT_A_VALID_WORD_POS == nextWordPos) return false;



    BinaryDictionaryBigramsIterator bigramsIt(mBinaryDictionaryInfo, pos);

native/jni/src/suggest/core/dictionary/binary_dictionary_info.h

+15 −5
Original line number Diff line number Diff line
@@ -23,7 +23,7 @@ 
#include "jni.h"

#include "suggest/core/dictionary/binary_dictionary_format_utils.h"

#include "suggest/core/dictionary/binary_dictionary_header.h"

#include "suggest/policyimpl/dictionary/dictionary_structure_policy_factory.h"

#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"

#include "utils/log_utils.h"



namespace latinime {
@@ -37,11 +37,16 @@ class BinaryDictionaryInfo { 
              mDictionaryFormat(BinaryDictionaryFormatUtils::detectFormatVersion(

                      mDictBuf, mDictSize)),

              mDictionaryHeader(this), mDictRoot(mDictBuf + mDictionaryHeader.getSize()),

              mStructurePolicy(DictionaryStructurePolicyFactory::getDictionaryStructurePolicy(

                      mDictionaryFormat)) {

              // TODO: Remove.

              mStructurePolicy(DictionaryStructureWithBufferPolicyFactory

                      ::newDictionaryStructurePolicy(this)) {

        logDictionaryInfo(env);

    }



    ~BinaryDictionaryInfo() {

        delete mStructurePolicy;

    }



    AK_FORCE_INLINE const uint8_t *getDictBuf() const {

        return mDictBuf;

    }
@@ -66,6 +71,7 @@ class BinaryDictionaryInfo { 
        return mDictionaryFormat;

    }



    // TODO: Move to DictionaryStructurePolicy.

    AK_FORCE_INLINE const BinaryDictionaryHeader *getHeader() const {

        return &mDictionaryHeader;

    }
@@ -76,7 +82,8 @@ class BinaryDictionaryInfo { 
        return mIsUpdatable && isUpdatableDictionaryFormat;

    }



    AK_FORCE_INLINE const DictionaryStructurePolicy *getStructurePolicy() const {

    // TODO: remove

    AK_FORCE_INLINE const DictionaryStructureWithBufferPolicy *getStructurePolicy() const {

        return mStructurePolicy;

    }
@@ -89,9 +96,12 @@ class BinaryDictionaryInfo { 
    const int mDictBufOffset;

    const bool mIsUpdatable;

    const BinaryDictionaryFormatUtils::FORMAT_VERSION mDictionaryFormat;

    // TODO: Move BinaryDictionaryHeader to policyimpl and introduce dedicated API to the

    // DictionaryStructurePolicy.

    const BinaryDictionaryHeader mDictionaryHeader;

    const uint8_t *const mDictRoot;

    const DictionaryStructurePolicy *const mStructurePolicy;

    // TODO: remove

    const DictionaryStructureWithBufferPolicy *const mStructurePolicy;



    AK_FORCE_INLINE void logDictionaryInfo(JNIEnv *const env) const {

        const int BUFFER_SIZE = 16;