Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit be0cf722 authored by satok's avatar satok
Browse files

Move auto correction thresthold to the native code

bug: 5858137

Change-Id: Ic4b6270c6e51ef4ed25a6a1d8ddd7fdfa70fd78d
parent 53f56dde
Loading
Loading
Loading
Loading
+3 −2
Original line number Diff line number Diff line
@@ -118,8 +118,9 @@ public class AutoCorrection {
            final int autoCorrectionSuggestionScore = sortedScores[0];
            // TODO: when the normalized score of the first suggestion is nearly equals to
            //       the normalized score of the second suggestion, behave less aggressive.
            mNormalizedScore = Utils.calcNormalizedScore(
                    typedWord,autoCorrectionSuggestion, autoCorrectionSuggestionScore);
            mNormalizedScore = BinaryDictionary.calcNormalizedScore(
                    typedWord.toString(), autoCorrectionSuggestion.toString(),
                    autoCorrectionSuggestionScore);
            if (DBG) {
                Log.d(TAG, "Normalized " + typedWord + "," + autoCorrectionSuggestion + ","
                        + autoCorrectionSuggestionScore + ", " + mNormalizedScore
+14 −0
Original line number Diff line number Diff line
@@ -118,6 +118,10 @@ public class BinaryDictionary extends Dictionary {
    private native int getBigramsNative(long dict, char[] prevWord, int prevWordLength,
            int[] inputCodes, int inputCodesLength, char[] outputChars, int[] scores,
            int maxWordLength, int maxBigrams, int maxAlternatives);
    private static native double calcNormalizedScoreNative(
            char[] before, int beforeLength, char[] after, int afterLength, int score);
    private static native int editDistanceNative(
            char[] before, int beforeLength, char[] after, int afterLength);

    private final void loadDictionary(String path, long startOffset, long length) {
        mNativeDict = openNative(path, startOffset, length,
@@ -211,6 +215,16 @@ public class BinaryDictionary extends Dictionary {
                mFlags, outputChars, scores);
    }

    public static double calcNormalizedScore(String before, String after, int score) {
        return calcNormalizedScoreNative(before.toCharArray(), before.length(),
                after.toCharArray(), after.length(), score);
    }

    public static int editDistance(String before, String after) {
        return editDistanceNative(
                before.toCharArray(), before.length(), after.toCharArray(), after.length());
    }

    @Override
    public boolean isValidWord(CharSequence word) {
        if (word == null) return false;
+2 −93
Original line number Diff line number Diff line
@@ -191,7 +191,8 @@ public class Utils {
        final int typedWordLength = typedWord.length();
        final int maxEditDistanceOfNativeDictionary =
                (typedWordLength < 5 ? 2 : typedWordLength / 2) + 1;
        final int distance = Utils.editDistance(typedWord, suggestionWord);
        final int distance = BinaryDictionary.editDistance(
                typedWord.toString(), suggestionWord.toString());
        if (DBG) {
            Log.d(TAG, "Autocorrected edit distance = " + distance
                    + ", " + maxEditDistanceOfNativeDictionary);
@@ -323,49 +324,6 @@ public class Utils {
        }
    }


    /* Damerau-Levenshtein distance */
    public static int editDistance(CharSequence s, CharSequence t) {
        if (s == null || t == null) {
            throw new IllegalArgumentException("editDistance: Arguments should not be null.");
        }
        final int sl = s.length();
        final int tl = t.length();
        int[][] dp = new int [sl + 1][tl + 1];
        for (int i = 0; i <= sl; i++) {
            dp[i][0] = i;
        }
        for (int j = 0; j <= tl; j++) {
            dp[0][j] = j;
        }
        for (int i = 0; i < sl; ++i) {
            for (int j = 0; j < tl; ++j) {
                final char sc = Character.toLowerCase(s.charAt(i));
                final char tc = Character.toLowerCase(t.charAt(j));
                final int cost = sc == tc ? 0 : 1;
                dp[i + 1][j + 1] = Math.min(
                        dp[i][j + 1] + 1, Math.min(dp[i + 1][j] + 1, dp[i][j] + cost));
                // Overwrite for transposition cases
                if (i > 0 && j > 0
                        && sc == Character.toLowerCase(t.charAt(j - 1))
                        && tc == Character.toLowerCase(s.charAt(i - 1))) {
                    dp[i + 1][j + 1] = Math.min(dp[i + 1][j + 1], dp[i - 1][j - 1] + cost);
                }
            }
        }
        if (DBG_EDIT_DISTANCE) {
            Log.d(TAG, "editDistance:" + s + "," + t);
            for (int i = 0; i < dp.length; ++i) {
                StringBuffer sb = new StringBuffer();
                for (int j = 0; j < dp[i].length; ++j) {
                    sb.append(dp[i][j]).append(',');
                }
                Log.d(TAG, i + ":" + sb.toString());
            }
        }
        return dp[sl][tl];
    }

    // Get the current stack trace
    public static String getStackTrace() {
        StringBuilder sb = new StringBuilder();
@@ -379,55 +337,6 @@ public class Utils {
        return sb.toString();
    }

    // In dictionary.cpp, getSuggestion() method,
    // suggestion scores are computed using the below formula.
    // original score
    //  := pow(mTypedLetterMultiplier (this is defined 2),
    //         (the number of matched characters between typed word and suggested word))
    //     * (individual word's score which defined in the unigram dictionary,
    //         and this score is defined in range [0, 255].)
    // Then, the following processing is applied.
    //     - If the dictionary word is matched up to the point of the user entry
    //       (full match up to min(before.length(), after.length())
    //       => Then multiply by FULL_MATCHED_WORDS_PROMOTION_RATE (this is defined 1.2)
    //     - If the word is a true full match except for differences in accents or
    //       capitalization, then treat it as if the score was 255.
    //     - If before.length() == after.length()
    //       => multiply by mFullWordMultiplier (this is defined 2))
    // So, maximum original score is pow(2, min(before.length(), after.length())) * 255 * 2 * 1.2
    // For historical reasons we ignore the 1.2 modifier (because the measure for a good
    // autocorrection threshold was done at a time when it didn't exist). This doesn't change
    // the result.
    // So, we can normalize original score by dividing pow(2, min(b.l(),a.l())) * 255 * 2.
    private static final int MAX_INITIAL_SCORE = 255;
    private static final int TYPED_LETTER_MULTIPLIER = 2;
    private static final int FULL_WORD_MULTIPLIER = 2;
    private static final int S_INT_MAX = 2147483647;
    public static double calcNormalizedScore(CharSequence before, CharSequence after, int score) {
        final int beforeLength = before.length();
        final int afterLength = after.length();
        if (beforeLength == 0 || afterLength == 0) return 0;
        final int distance = editDistance(before, after);
        // If afterLength < beforeLength, the algorithm is suggesting a word by excessive character
        // correction.
        int spaceCount = 0;
        for (int i = 0; i < afterLength; ++i) {
            if (after.charAt(i) == Keyboard.CODE_SPACE) {
                ++spaceCount;
            }
        }
        if (spaceCount == afterLength) return 0;
        final double maximumScore = score == S_INT_MAX ? S_INT_MAX : MAX_INITIAL_SCORE
                * Math.pow(
                        TYPED_LETTER_MULTIPLIER, Math.min(beforeLength, afterLength - spaceCount))
                * FULL_WORD_MULTIPLIER;
        // add a weight based on edit distance.
        // distance <= max(afterLength, beforeLength) == afterLength,
        // so, 0 <= distance / afterLength <= 1
        final double weight = 1.0 - (double) distance / afterLength;
        return (score / maximumScore) * weight;
    }

    public static class UsabilityStudyLogUtils {
        private static final String USABILITY_TAG = UsabilityStudyLogUtils.class.getSimpleName();
        private static final String FILENAME = "log.txt";
+5 −4
Original line number Diff line number Diff line
@@ -270,7 +270,7 @@ public class AndroidSpellCheckerService extends SpellCheckerService
            // make the threshold.
            final String wordString = new String(word, wordOffset, wordLength);
            final double normalizedScore =
                    Utils.calcNormalizedScore(mOriginalText, wordString, score);
                    BinaryDictionary.calcNormalizedScore(mOriginalText, wordString, score);
            if (normalizedScore < mSuggestionThreshold) {
                if (DBG) Log.i(TAG, wordString + " does not make the score threshold");
                return true;
@@ -303,8 +303,8 @@ public class AndroidSpellCheckerService extends SpellCheckerService
                    hasRecommendedSuggestions = false;
                } else {
                    gatheredSuggestions = EMPTY_STRING_ARRAY;
                    final double normalizedScore =
                            Utils.calcNormalizedScore(mOriginalText, mBestSuggestion, mBestScore);
                    final double normalizedScore = BinaryDictionary.calcNormalizedScore(
                            mOriginalText, mBestSuggestion, mBestScore);
                    hasRecommendedSuggestions = (normalizedScore > mRecommendedThreshold);
                }
            } else {
@@ -338,7 +338,8 @@ public class AndroidSpellCheckerService extends SpellCheckerService
                final int bestScore = mScores[mLength - 1];
                final CharSequence bestSuggestion = mSuggestions.get(0);
                final double normalizedScore =
                        Utils.calcNormalizedScore(mOriginalText, bestSuggestion, bestScore);
                        BinaryDictionary.calcNormalizedScore(
                                mOriginalText, bestSuggestion.toString(), bestScore);
                hasRecommendedSuggestions = (normalizedScore > mRecommendedThreshold);
                if (DBG) {
                    Log.i(TAG, "Best suggestion : " + bestSuggestion + ", score " + bestScore);
+28 −1
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@
#define LOG_TAG "LatinIME: jni: BinaryDictionary"

#include "binary_format.h"
#include "correction.h"
#include "com_android_inputmethod_latin_BinaryDictionary.h"
#include "dictionary.h"
#include "jni.h"
@@ -188,6 +189,29 @@ static jboolean latinime_BinaryDictionary_isValidWord(JNIEnv *env, jobject objec
    return result;
}

static jdouble latinime_BinaryDictionary_calcNormalizedScore(JNIEnv *env, jobject object,
        jcharArray before, jint beforeLength, jcharArray after, jint afterLength, jint score) {
    jchar *beforeChars = env->GetCharArrayElements(before, 0);
    jchar *afterChars = env->GetCharArrayElements(after, 0);
    jdouble result = Correction::RankingAlgorithm::calcNormalizedScore(
            (unsigned short*)beforeChars, beforeLength, (unsigned short*)afterChars, afterLength,
                    score);
    env->ReleaseCharArrayElements(before, beforeChars, JNI_ABORT);
    env->ReleaseCharArrayElements(after, afterChars, JNI_ABORT);
    return result;
}

static jint latinime_BinaryDictionary_editDistance(JNIEnv *env, jobject object,
        jcharArray before, jint beforeLength, jcharArray after, jint afterLength) {
    jchar *beforeChars = env->GetCharArrayElements(before, 0);
    jchar *afterChars = env->GetCharArrayElements(after, 0);
    jint result = Correction::RankingAlgorithm::editDistance(
            (unsigned short*)beforeChars, beforeLength, (unsigned short*)afterChars, afterLength);
    env->ReleaseCharArrayElements(before, beforeChars, JNI_ABORT);
    env->ReleaseCharArrayElements(after, afterChars, JNI_ABORT);
    return result;
}

static void latinime_BinaryDictionary_close(JNIEnv *env, jobject object, jlong dict) {
    Dictionary *dictionary = (Dictionary*)dict;
    if (!dictionary) return;
@@ -222,7 +246,10 @@ static JNINativeMethod sMethods[] = {
    {"closeNative", "(J)V", (void*)latinime_BinaryDictionary_close},
    {"getSuggestionsNative", "(JJ[I[I[III[C[I)I", (void*)latinime_BinaryDictionary_getSuggestions},
    {"isValidWordNative", "(J[CI)Z", (void*)latinime_BinaryDictionary_isValidWord},
    {"getBigramsNative", "(J[CI[II[C[IIII)I", (void*)latinime_BinaryDictionary_getBigrams}
    {"getBigramsNative", "(J[CI[II[C[IIII)I", (void*)latinime_BinaryDictionary_getBigrams},
    {"calcNormalizedScoreNative", "([CI[CII)D",
            (void*)latinime_BinaryDictionary_calcNormalizedScore},
    {"editDistanceNative", "([CI[CI)I", (void*)latinime_BinaryDictionary_editDistance}
};

int register_BinaryDictionary(JNIEnv *env) {
Loading