Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 64341927 authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi
Browse files

Quit use bigram probability diff for ver4 dict.

Change-Id: I2cfcfbcf351877d1dff466a24974dbb05908f14e
parent 04536f14
Loading
Loading
Loading
Loading
+0 −8
Original line number Diff line number Diff line
@@ -214,8 +214,6 @@ public final class BinaryDictionary extends Dictionary {
    private static native void removeBigramWordsNative(long dict, int[] word0, int[] word1);
    private static native int addMultipleDictionaryEntriesNative(long dict,
            LanguageModelParam[] languageModelParams, int startIndex);
    private static native int calculateProbabilityNative(long dict, int unigramProbability,
            int bigramProbability);
    private static native String getPropertyNative(long dict, String query);
    private static native boolean isCorruptedNative(long dict);
    private static native boolean migrateNative(long dict, String dictFilePath,
@@ -551,12 +549,6 @@ public final class BinaryDictionary extends Dictionary {
        return true;
    }

    @UsedForTesting
    public int calculateProbability(final int unigramProbability, final int bigramProbability) {
        if (!isValidDictionary()) return NOT_A_PROBABILITY;
        return calculateProbabilityNative(mNativeDict, unigramProbability, bigramProbability);
    }

    @UsedForTesting
    public String getPropertyForTest(final String query) {
        if (!isValidDictionary()) return "";
+0 −15
Original line number Diff line number Diff line
@@ -467,16 +467,6 @@ static int latinime_BinaryDictionary_addMultipleDictionaryEntries(JNIEnv *env, j
    return languageModelParamCount;
}

static int latinime_BinaryDictionary_calculateProbabilityNative(JNIEnv *env, jclass clazz,
        jlong dict, jint unigramProbability, jint bigramProbability) {
    Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
    if (!dictionary) {
        return NOT_A_PROBABILITY;
    }
    return dictionary->getDictionaryStructurePolicy()->getProbability(unigramProbability,
            bigramProbability);
}

static jstring latinime_BinaryDictionary_getProperty(JNIEnv *env, jclass clazz, jlong dict,
        jstring query) {
    Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);
@@ -669,11 +659,6 @@ static const JNINativeMethod sMethods[] = {
                "(J[Lcom/android/inputmethod/latin/utils/LanguageModelParam;I)I"),
        reinterpret_cast<void *>(latinime_BinaryDictionary_addMultipleDictionaryEntries)
    },
    {
        const_cast<char *>("calculateProbabilityNative"),
        const_cast<char *>("(JII)I"),
        reinterpret_cast<void *>(latinime_BinaryDictionary_calculateProbabilityNative)
    },
    {
        const_cast<char *>("getPropertyNative"),
        const_cast<char *>("(JLjava/lang/String;)Ljava/lang/String;"),
+10 −12
Original line number Diff line number Diff line
@@ -38,8 +38,6 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
    int level = 0;
    int count = 0;
    if (mHasHistoricalInfo) {
        probability = bigramListBuffer->readUintAndAdvancePosition(
                Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
        timestamp = bigramListBuffer->readUintAndAdvancePosition(
                Ver4DictConstants::TIME_STAMP_FIELD_SIZE, bigramEntryPos);
        level = bigramListBuffer->readUintAndAdvancePosition(
@@ -47,7 +45,8 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
        count = bigramListBuffer->readUintAndAdvancePosition(
                Ver4DictConstants::WORD_COUNT_FIELD_SIZE, bigramEntryPos);
    } else {
        probability = bigramFlags & Ver4DictConstants::BIGRAM_PROBABILITY_MASK;
        probability = bigramListBuffer->readUintAndAdvancePosition(
                Ver4DictConstants::PROBABILITY_SIZE, bigramEntryPos);
    }
    const int encodedTargetTerminalId = bigramListBuffer->readUintAndAdvancePosition(
            Ver4DictConstants::BIGRAM_TARGET_TERMINAL_ID_FIELD_SIZE, bigramEntryPos);
@@ -65,21 +64,13 @@ const BigramEntry BigramDictContent::getBigramEntryAndAdvancePosition(
bool BigramDictContent::writeBigramEntryAndAdvancePosition(
        const BigramEntry *const bigramEntryToWrite, int *const entryWritingPos) {
    BufferWithExtendableBuffer *const bigramListBuffer = getWritableContentBuffer();
    const int bigramFlags = createAndGetBigramFlags(
            mHasHistoricalInfo ? 0 : bigramEntryToWrite->getProbability(),
            bigramEntryToWrite->hasNext());
    const int bigramFlags = createAndGetBigramFlags(bigramEntryToWrite->hasNext());
    if (!bigramListBuffer->writeUintAndAdvancePosition(bigramFlags,
            Ver4DictConstants::BIGRAM_FLAGS_FIELD_SIZE, entryWritingPos)) {
        AKLOGE("Cannot write bigram flags. pos: %d, flags: %x", *entryWritingPos, bigramFlags);
        return false;
    }
    if (mHasHistoricalInfo) {
        if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
                Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
            AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
                    bigramEntryToWrite->getProbability());
            return false;
        }
        const HistoricalInfo *const historicalInfo = bigramEntryToWrite->getHistoricalInfo();
        if (!bigramListBuffer->writeUintAndAdvancePosition(historicalInfo->getTimeStamp(),
                Ver4DictConstants::TIME_STAMP_FIELD_SIZE, entryWritingPos)) {
@@ -99,6 +90,13 @@ bool BigramDictContent::writeBigramEntryAndAdvancePosition(
                    historicalInfo->getCount());
            return false;
        }
    } else {
        if (!bigramListBuffer->writeUintAndAdvancePosition(bigramEntryToWrite->getProbability(),
                Ver4DictConstants::PROBABILITY_SIZE, entryWritingPos)) {
            AKLOGE("Cannot write bigram probability. pos: %d, probability: %d", *entryWritingPos,
                    bigramEntryToWrite->getProbability());
            return false;
        }
    }
    const int targetTerminalIdToWrite =
            (bigramEntryToWrite->getTargetTerminalId() == Ver4DictConstants::NOT_A_TERMINAL_ID) ?
+2 −3
Original line number Diff line number Diff line
@@ -95,9 +95,8 @@ class BigramDictContent : public SparseTableDictContent {
 private:
    DISALLOW_COPY_AND_ASSIGN(BigramDictContent);

    int createAndGetBigramFlags(const int probability, const bool hasNext) const {
        return (probability & Ver4DictConstants::BIGRAM_PROBABILITY_MASK)
                | (hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0);
    int createAndGetBigramFlags(const bool hasNext) const {
        return hasNext ? Ver4DictConstants::BIGRAM_HAS_NEXT_MASK : 0;
    }

    bool runGCBigramList(const int bigramListPos,
+2 −4
Original line number Diff line number Diff line
@@ -115,9 +115,7 @@ int Ver4PatriciaTriePolicy::getProbability(const int unigramProbability,
        } else if (bigramProbability == NOT_A_PROBABILITY) {
            return ProbabilityUtils::backoff(unigramProbability);
        } else {
            // bigramProbability is a bigram probability delta.
            return ProbabilityUtils::computeProbabilityForBigram(unigramProbability,
                    bigramProbability);
            return bigramProbability;
        }
    }
}
@@ -398,7 +396,7 @@ const WordProperty Ver4PatriciaTriePolicy::getWordProperty(const int *const code
            const int probability = bigramEntry.hasHistoricalInfo() ?
                    ForgettingCurveUtils::decodeProbability(
                            bigramEntry.getHistoricalInfo(), mHeaderPolicy) :
                    getProbability(word1Probability, bigramEntry.getProbability());
                    bigramEntry.getProbability();
            bigrams.emplace_back(&word1, probability,
                    historicalInfo->getTimeStamp(), historicalInfo->getLevel(),
                    historicalInfo->getCount());
Loading