Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d81654cd authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi
Browse files

Use DictionaryStructureWithBufferPolicy for traversing.

Bug: 6669677
Change-Id: If7bc038278b7217ed5ba4cb77b1238b2dad0001b
parent e607e072
Loading
Loading
Loading
Loading
+16 −15
Original line number Diff line number Diff line
@@ -21,7 +21,6 @@
#include "suggest/core/dicnode/dic_node.h"
#include "suggest/core/dicnode/dic_node_proximity_filter.h"
#include "suggest/core/dicnode/dic_node_vector.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/multi_bigram_map.h"
#include "suggest/core/dictionary/probability_utils.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
@@ -33,17 +32,17 @@ namespace latinime {
// Node initialization utils //
///////////////////////////////

/* static */ void DicNodeUtils::initAsRoot(const BinaryDictionaryInfo *const binaryDictionaryInfo,
/* static */ void DicNodeUtils::initAsRoot(
        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
        const int prevWordNodePos, DicNode *const newRootNode) {
    newRootNode->initAsRoot(binaryDictionaryInfo->getStructurePolicy()->getRootPosition(),
            prevWordNodePos);
    newRootNode->initAsRoot(dictionaryStructurePolicy->getRootPosition(), prevWordNodePos);
}

/*static */ void DicNodeUtils::initAsRootWithPreviousWord(
        const BinaryDictionaryInfo *const binaryDictionaryInfo,
        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
        DicNode *const prevWordLastNode, DicNode *const newRootNode) {
    newRootNode->initAsRootWithPreviousWord(
            prevWordLastNode, binaryDictionaryInfo->getStructurePolicy()->getRootPosition());
            prevWordLastNode, dictionaryStructurePolicy->getRootPosition());
}

/* static */ void DicNodeUtils::initByCopy(DicNode *srcNode, DicNode *destNode) {
@@ -67,12 +66,13 @@ namespace latinime {
}

/* static */ void DicNodeUtils::getAllChildDicNodes(DicNode *dicNode,
        const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNodeVector *childDicNodes) {
    getProximityChildDicNodes(dicNode, binaryDictionaryInfo, 0, 0, false, childDicNodes);
        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
        DicNodeVector *childDicNodes) {
    getProximityChildDicNodes(dicNode, dictionaryStructurePolicy, 0, 0, false, childDicNodes);
}

/* static */ void DicNodeUtils::getProximityChildDicNodes(DicNode *dicNode,
        const BinaryDictionaryInfo *const binaryDictionaryInfo,
        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
        const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly,
        DicNodeVector *childDicNodes) {
    if (dicNode->isTotalInputSizeExceedingLimit()) {
@@ -82,7 +82,7 @@ namespace latinime {
    if (!dicNode->isLeavingNode()) {
        DicNodeUtils::createAndGetPassingChildNode(dicNode, &childrenFilter, childDicNodes);
    } else {
        binaryDictionaryInfo->getStructurePolicy()->createAndGetAllChildNodes(dicNode,
        dictionaryStructurePolicy->createAndGetAllChildNodes(dicNode,
                &childrenFilter, childDicNodes);
    }
}
@@ -94,12 +94,13 @@ namespace latinime {
 * Computes the combined bigram / unigram cost for the given dicNode.
 */
/* static */ float DicNodeUtils::getBigramNodeImprobability(
        const BinaryDictionaryInfo *const binaryDictionaryInfo,
        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
        const DicNode *const node, MultiBigramMap *multiBigramMap) {
    if (node->hasMultipleWords() && !node->isValidMultipleWordSuggestion()) {
        return static_cast<float>(MAX_VALUE_FOR_WEIGHTING);
    }
    const int probability = getBigramNodeProbability(binaryDictionaryInfo, node, multiBigramMap);
    const int probability = getBigramNodeProbability(dictionaryStructurePolicy, node,
            multiBigramMap);
    // TODO: This equation to calculate the improbability looks unreasonable.  Investigate this.
    const float cost = static_cast<float>(MAX_PROBABILITY - probability)
            / static_cast<float>(MAX_PROBABILITY);
@@ -107,7 +108,7 @@ namespace latinime {
}

/* static */ int DicNodeUtils::getBigramNodeProbability(
        const BinaryDictionaryInfo *const binaryDictionaryInfo,
        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
        const DicNode *const node, MultiBigramMap *multiBigramMap) {
    const int unigramProbability = node->getProbability();
    const int wordPos = node->getPos();
@@ -118,8 +119,8 @@ namespace latinime {
        return ProbabilityUtils::backoff(unigramProbability);
    }
    if (multiBigramMap) {
        return multiBigramMap->getBigramProbability(binaryDictionaryInfo->getStructurePolicy(),
                prevWordPos, wordPos, unigramProbability);
        return multiBigramMap->getBigramProbability(dictionaryStructurePolicy, prevWordPos,
                wordPos, unigramProbability);
    }
    return ProbabilityUtils::backoff(unigramProbability);
}
+12 −7
Original line number Diff line number Diff line
@@ -23,10 +23,10 @@

namespace latinime {

class BinaryDictionaryInfo;
class DicNode;
class DicNodeProximityFilter;
class DicNodeVector;
class DictionaryStructureWithBufferPolicy;
class ProximityInfoState;
class MultiBigramMap;

@@ -34,18 +34,22 @@ class DicNodeUtils {
 public:
    static int appendTwoWords(const int *src0, const int16_t length0, const int *src1,
            const int16_t length1, int *dest);
    static void initAsRoot(const BinaryDictionaryInfo *const binaryDictionaryInfo,
    static void initAsRoot(
            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
            const int prevWordNodePos, DicNode *newRootNode);
    static void initAsRootWithPreviousWord(const BinaryDictionaryInfo *const binaryDictionaryInfo,
    static void initAsRootWithPreviousWord(
            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
            DicNode *prevWordLastNode, DicNode *newRootNode);
    static void initByCopy(DicNode *srcNode, DicNode *destNode);
    static void getAllChildDicNodes(DicNode *dicNode,
            const BinaryDictionaryInfo *const binaryDictionaryInfo, DicNodeVector *childDicNodes);
    static float getBigramNodeImprobability(const BinaryDictionaryInfo *const binaryDictionaryInfo,
            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
            DicNodeVector *childDicNodes);
    static float getBigramNodeImprobability(
            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
            const DicNode *const node, MultiBigramMap *const multiBigramMap);
    // TODO: Move to private
    static void getProximityChildDicNodes(DicNode *dicNode,
            const BinaryDictionaryInfo *const binaryDictionaryInfo,
            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
            const ProximityInfoState *pInfoState, const int pointIndex, bool exactOnly,
            DicNodeVector *childDicNodes);

@@ -54,7 +58,8 @@ class DicNodeUtils {
    // Max number of bigrams to look up
    static const int MAX_BIGRAMS_CONSIDERED_PER_CONTEXT = 500;

    static int getBigramNodeProbability(const BinaryDictionaryInfo *const binaryDictionaryInfo,
    static int getBigramNodeProbability(
            const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy,
            const DicNode *const node, MultiBigramMap *multiBigramMap);
    static void createAndGetPassingChildNode(DicNode *dicNode,
            const DicNodeProximityFilter *const childrenFilter, DicNodeVector *childDicNodes);
+12 −15
Original line number Diff line number Diff line
@@ -22,15 +22,16 @@

#include "defines.h"
#include "suggest/core/dictionary/binary_dictionary_bigrams_iterator.h"
#include "suggest/core/dictionary/binary_dictionary_info.h"
#include "suggest/core/dictionary/dictionary.h"
#include "suggest/core/dictionary/probability_utils.h"
#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
#include "utils/char_utils.h"

namespace latinime {

BigramDictionary::BigramDictionary(const BinaryDictionaryInfo *const binaryDictionaryInfo)
        : mBinaryDictionaryInfo(binaryDictionaryInfo) {
BigramDictionary::BigramDictionary(
        const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy)
        : mDictionaryStructurePolicy(dictionaryStructurePolicy) {
    if (DEBUG_DICT) {
        AKLOGI("BigramDictionary - constructor");
    }
@@ -112,12 +113,11 @@ int BigramDictionary::getPredictions(const int *prevWord, const int prevWordLeng
    int bigramCount = 0;
    int unigramProbability = 0;
    int bigramBuffer[MAX_WORD_LENGTH];
    const DictionaryStructureWithBufferPolicy *const structurePolicy =
            mBinaryDictionaryInfo->getStructurePolicy();
    BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(), pos);
    BinaryDictionaryBigramsIterator bigramsIt(
            mDictionaryStructurePolicy->getBigramsStructurePolicy(), pos);
    while (bigramsIt.hasNext()) {
        bigramsIt.next();
        const int length = structurePolicy->
        const int length = mDictionaryStructurePolicy->
                getCodePointsAndProbabilityAndReturnCodePointCount(bigramsIt.getBigramPos(),
                        MAX_WORD_LENGTH, bigramBuffer, &unigramProbability);
        // Due to space constraints, the probability for bigrams is approximate - the lower the
@@ -139,12 +139,10 @@ int BigramDictionary::getPredictions(const int *prevWord, const int prevWordLeng
int BigramDictionary::getBigramListPositionForWord(const int *prevWord, const int prevWordLength,
        const bool forceLowerCaseSearch) const {
    if (0 >= prevWordLength) return NOT_A_DICT_POS;
    const DictionaryStructureWithBufferPolicy *const structurePolicy =
            mBinaryDictionaryInfo->getStructurePolicy();
    int pos = structurePolicy->getTerminalNodePositionOfWord(prevWord, prevWordLength,
    int pos = mDictionaryStructurePolicy->getTerminalNodePositionOfWord(prevWord, prevWordLength,
            forceLowerCaseSearch);
    if (NOT_A_VALID_WORD_POS == pos) return NOT_A_DICT_POS;
    return structurePolicy->getBigramsPositionOfNode(pos);
    return mDictionaryStructurePolicy->getBigramsPositionOfNode(pos);
}

bool BigramDictionary::isValidBigram(const int *word0, int length0, const int *word1,
@@ -152,13 +150,12 @@ bool BigramDictionary::isValidBigram(const int *word0, int length0, const int *w
    int pos = getBigramListPositionForWord(word0, length0, false /* forceLowerCaseSearch */);
    // getBigramListPositionForWord returns 0 if this word isn't in the dictionary or has no bigrams
    if (NOT_A_DICT_POS == pos) return false;
    const DictionaryStructureWithBufferPolicy *const structurePolicy =
            mBinaryDictionaryInfo->getStructurePolicy();
    int nextWordPos = structurePolicy->getTerminalNodePositionOfWord(word1, length1,
    int nextWordPos = mDictionaryStructurePolicy->getTerminalNodePositionOfWord(word1, length1,
            false /* forceLowerCaseSearch */);
    if (NOT_A_VALID_WORD_POS == nextWordPos) return false;

    BinaryDictionaryBigramsIterator bigramsIt(structurePolicy->getBigramsStructurePolicy(), pos);
    BinaryDictionaryBigramsIterator bigramsIt(
            mDictionaryStructurePolicy->getBigramsStructurePolicy(), pos);
    while (bigramsIt.hasNext()) {
        bigramsIt.next();
        if (bigramsIt.getBigramPos() == nextWordPos) {
+3 −3
Original line number Diff line number Diff line
@@ -21,11 +21,11 @@

namespace latinime {

class BinaryDictionaryInfo;
class DictionaryStructureWithBufferPolicy;

class BigramDictionary {
 public:
    BigramDictionary(const BinaryDictionaryInfo *const binaryDictionaryInfo);
    BigramDictionary(const DictionaryStructureWithBufferPolicy *const dictionaryStructurePolicy);

    int getPredictions(const int *word, int length, int *outBigramCodePoints,
            int *outBigramProbability, int *outputTypes) const;
@@ -40,7 +40,7 @@ class BigramDictionary {
    int getBigramListPositionForWord(const int *prevWord, const int prevWordLength,
            const bool forceLowerCaseSearch) const;

    const BinaryDictionaryInfo *const mBinaryDictionaryInfo;
    const DictionaryStructureWithBufferPolicy *const mDictionaryStructurePolicy;
};
} // namespace latinime
#endif // LATINIME_BIGRAM_DICTIONARY_H
+0 −2
Original line number Diff line number Diff line
@@ -24,8 +24,6 @@

namespace latinime {

class BinaryDictionaryInfo;

/**
 * This class abstracts dictionary header structures and provide interface to access dictionary
 * header information.
Loading