Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit b4531d86 authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi
Browse files

Add method to remove entry from language model dict content. 

Bug: 14425059
Change-Id: Id21af0110e770caa3e95cb5d7ba8b3d1af8e0b12
parent 90d51d94

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp

+10 −0
Original line number Diff line number Diff line
@@ -53,6 +53,16 @@ bool LanguageModelDictContent::setNgramProbabilityEntry(const WordIdArrayView pr 
    return mTrieMap.put(terminalId, probabilityEntry->encode(mHasHistoricalInfo), bitmapEntryIndex);

}



bool LanguageModelDictContent::removeNgramProbabilityEntry(const WordIdArrayView prevWordIds,

        const int wordId) {

    const int bitmapEntryIndex = getBitmapEntryIndex(prevWordIds);

    if (bitmapEntryIndex == TrieMap::INVALID_INDEX) {

        // Cannot find bitmap entry for the probability entry. The entry doesn't exist.

        return false;

    }

    return mTrieMap.remove(wordId, bitmapEntryIndex);

}



bool LanguageModelDictContent::runGCInner(

        const TerminalPositionLookupTable::TerminalIdMap *const terminalIdMap,

        const TrieMap::TrieMapRange trieMapRange,

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h

+6 −0
Original line number Diff line number Diff line
@@ -61,12 +61,18 @@ class LanguageModelDictContent { 
        return setNgramProbabilityEntry(WordIdArrayView(), wordId, probabilityEntry);

    }



    bool removeProbabilityEntry(const int wordId) {

        return removeNgramProbabilityEntry(WordIdArrayView(), wordId);

    }



    ProbabilityEntry getNgramProbabilityEntry(const WordIdArrayView prevWordIds,

            const int wordId) const;



    bool setNgramProbabilityEntry(const WordIdArrayView prevWordIds, const int wordId,

            const ProbabilityEntry *const probabilityEntry);



    bool removeNgramProbabilityEntry(const WordIdArrayView prevWordIds, const int wordId);



 private:

    DISALLOW_COPY_AND_ASSIGN(LanguageModelDictContent);

native/jni/src/suggest/policyimpl/dictionary/utils/trie_map.cpp

+5 −1
Original line number Diff line number Diff line
@@ -420,6 +420,10 @@ bool TrieMap::addNewEntryByExpandingTable(const uint32_t key, const uint64_t val 


bool TrieMap::removeInner(const Entry &bitmapEntry) {

    const int tableSize = popCount(bitmapEntry.getBitmap());

    if (tableSize <= 0) {

        // The table is empty. No need to remove any entries.

        return true;

    }

    for (int i = 0; i < tableSize; ++i) {

        const int entryIndex = bitmapEntry.getTableIndex() + i;

        const Entry entry = readEntry(entryIndex);
@@ -444,7 +448,7 @@ bool TrieMap::removeInner(const Entry &bitmapEntry) { 
            }

        }

    }

    return freeTable(bitmapEntry.getTableIndex(), tableSize);

    return true;

}



}  // namespace latinime

native/jni/tests/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content_test.cpp

+14 −0
Original line number Diff line number Diff line
@@ -35,6 +35,13 @@ TEST(LanguageModelDictContentTest, TestUnigramProbability) { 
            LanguageModelDictContent.getProbabilityEntry(wordId);

    EXPECT_EQ(flag, entry.getFlags());

    EXPECT_EQ(probability, entry.getProbability());



    // Remove

    EXPECT_TRUE(LanguageModelDictContent.removeProbabilityEntry(wordId));

    EXPECT_FALSE(LanguageModelDictContent.getProbabilityEntry(wordId).isValid());

    EXPECT_FALSE(LanguageModelDictContent.removeProbabilityEntry(wordId));

    EXPECT_TRUE(LanguageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry));

    EXPECT_TRUE(LanguageModelDictContent.getProbabilityEntry(wordId).isValid());

}



TEST(LanguageModelDictContentTest, TestUnigramProbabilityWithHistoricalInfo) {
@@ -53,6 +60,13 @@ TEST(LanguageModelDictContentTest, TestUnigramProbabilityWithHistoricalInfo) { 
    EXPECT_EQ(timestamp, entry.getHistoricalInfo()->getTimeStamp());

    EXPECT_EQ(level, entry.getHistoricalInfo()->getLevel());

    EXPECT_EQ(count, entry.getHistoricalInfo()->getCount());



    // Remove

    EXPECT_TRUE(LanguageModelDictContent.removeProbabilityEntry(wordId));

    EXPECT_FALSE(LanguageModelDictContent.getProbabilityEntry(wordId).isValid());

    EXPECT_FALSE(LanguageModelDictContent.removeProbabilityEntry(wordId));

    EXPECT_TRUE(LanguageModelDictContent.setProbabilityEntry(wordId, &probabilityEntry));

    EXPECT_TRUE(LanguageModelDictContent.removeProbabilityEntry(wordId));

}



}  // namespace

native/jni/tests/suggest/policyimpl/dictionary/utils/trie_map_test.cpp

+2 −0
Original line number Diff line number Diff line
@@ -70,6 +70,8 @@ TEST(TrieMapTest, TestRemove) { 
    EXPECT_FALSE(result.mIsValid);

    EXPECT_EQ(TrieMap::INVALID_INDEX, result.mNextLevelBitmapEntryIndex);

    EXPECT_EQ(11ull, trieMap.getRoot(12).mValue);

    EXPECT_TRUE(trieMap.putRoot(S_INT_MAX, 0xFFFFFFFFFull));

    EXPECT_TRUE(trieMap.remove(S_INT_MAX, trieMap.getRootBitmapEntryIndex()));

}



TEST(TrieMapTest, TestSetAndGetLarge) {