Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 5ef62096 authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi
Browse files

Create empty dictionary file in native code. 

Bug: 6669677
Change-Id: I46d23deee1fd091678f4593561cb8687eb815212
parent 10b56df4

java/src/com/android/inputmethod/latin/BinaryDictionary.java

+17 −0
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ import java.io.File; 
import java.util.ArrayList;

import java.util.Arrays;

import java.util.Locale;

import java.util.Map;



/**

 * Implements a static, compacted, binary dictionary of standard words.
@@ -104,6 +105,8 @@ public final class BinaryDictionary extends Dictionary { 
        JniUtils.loadNativeLibrary();

    }



    private static native boolean createEmptyDictFileNative(String filePath, long dictVersion,

            String[] attributeKeyStringArray, String[] attributeValueStringArray);

    private static native long openNative(String sourceDir, long dictOffset, long dictSize,

            boolean isUpdatable);

    private static native void flushNative(long dict, String filePath);
@@ -127,6 +130,20 @@ public final class BinaryDictionary extends Dictionary { 
    private static native int calculateProbabilityNative(long dict, int unigramProbability,

            int bigramProbability);



    @UsedForTesting

    public static boolean createEmptyDictFile(final String filePath, final long dictVersion,

            final Map<String, String> attributeMap) {

        final String[] keyArray = new String[attributeMap.size()];

        final String[] valueArray = new String[attributeMap.size()];

        int index = 0;

        for (final String key : attributeMap.keySet()) {

            keyArray[index] = key;

            valueArray[index] = attributeMap.get(key);

            index++;

        }

        return createEmptyDictFileNative(filePath, dictVersion, keyArray, valueArray);

    }



    // TODO: Move native dict into session

    private final void loadDictionary(final String path, final long startOffset,

            final long length, final boolean isUpdatable) {

java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java

+16 −30
Original line number Diff line number Diff line
@@ -22,12 +22,7 @@ import android.util.Log; 


import com.android.inputmethod.annotations.UsedForTesting;

import com.android.inputmethod.keyboard.ProximityInfo;

import com.android.inputmethod.latin.makedict.DictEncoder;

import com.android.inputmethod.latin.makedict.FormatSpec;

import com.android.inputmethod.latin.makedict.FusionDictionary;

import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;

import com.android.inputmethod.latin.makedict.UnsupportedFormatException;

import com.android.inputmethod.latin.makedict.Ver3DictEncoder;

import com.android.inputmethod.latin.personalization.DynamicPersonalizationDictionaryWriter;

import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo;

import com.android.inputmethod.latin.utils.AsyncResultHolder;
@@ -35,9 +30,9 @@ import com.android.inputmethod.latin.utils.CollectionUtils; 
import com.android.inputmethod.latin.utils.PrioritizedSerialExecutor;



import java.io.File;

import java.io.IOException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.Map;

import java.util.concurrent.ConcurrentHashMap;

import java.util.concurrent.atomic.AtomicReference;
@@ -68,8 +63,10 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { 
     */

    protected static final int MAX_WORD_LENGTH = Constants.DICTIONARY_MAX_WORD_LENGTH;



    private static final FormatSpec.FormatOptions FORMAT_OPTIONS =

            new FormatSpec.FormatOptions(3 /* version */, true /* supportsDynamicUpdate */);

    private static final int DICTIONARY_FORMAT_VERSION = 3;



    private static final String SUPPORTS_DYNAMIC_UPDATE =

            FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE;



    /**

     * A static map of time recorders, each of which records the time of accesses to a single binary
@@ -233,6 +230,13 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { 
        });

    }



    protected Map<String, String> getHeaderAttributeMap() {

        HashMap<String, String> attributeMap = new HashMap<String, String>();

        attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,

                SUPPORTS_DYNAMIC_UPDATE);

        return attributeMap;

    }



    protected void clear() {

        getExecutor(mFilename).execute(new Runnable() {

            @Override
@@ -240,17 +244,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { 
                if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE && mDictionaryWriter == null) {

                    mBinaryDictionary.close();

                    final File file = new File(mContext.getFilesDir(), mFilename);

                    final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),

                            new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),

                                    false, false));

                    final DictEncoder dictEncoder = new Ver3DictEncoder(file);

                    try {

                        dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);

                    } catch (IOException e) {

                        Log.e(TAG, "Exception in creating new dictionary file.", e);

                    } catch (UnsupportedFormatException e) {

                        Log.e(TAG, "Exception in creating new dictionary file.", e);

                    }

                    BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),

                            DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());

                } else {

                    mDictionaryWriter.clear();

                }
@@ -498,17 +493,8 @@ abstract public class ExpandableBinaryDictionary extends Dictionary { 
            if (ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {

                if (mBinaryDictionary == null || !mBinaryDictionary.isValidDictionary()) {

                    final File file = new File(mContext.getFilesDir(), mFilename);

                    final FusionDictionary dict = new FusionDictionary(new PtNodeArray(),

                            new FusionDictionary.DictionaryOptions(new HashMap<String,String>(),

                                    false, false));

                    final DictEncoder dictEncoder = new Ver3DictEncoder(file);

                    try {

                        dictEncoder.writeDictionary(dict, FORMAT_OPTIONS);

                    } catch (IOException e) {

                        Log.e(TAG, "Exception in creating new dictionary file.", e);

                    } catch (UnsupportedFormatException e) {

                        Log.e(TAG, "Exception in creating new dictionary file.", e);

                    }

                    BinaryDictionary.createEmptyDictFile(file.getAbsolutePath(),

                            DICTIONARY_FORMAT_VERSION, getHeaderAttributeMap());

                } else {

                    if (mBinaryDictionary.needsToRunGC()) {

                        mBinaryDictionary.flushWithGC();

java/src/com/android/inputmethod/latin/makedict/FormatSpec.java

+6 −0
Original line number Diff line number Diff line
@@ -325,6 +325,12 @@ public final class FormatSpec { 
        public final int mHeaderSize;

        public final DictionaryOptions mDictionaryOptions;

        public final FormatOptions mFormatOptions;

        // Note that these are corresponding definitions in native code in latinime::HeaderPolicy

        // and latinime::HeaderReadWriteUtils.

        public static final String SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE = "SUPPORTS_DYNAMIC_UPDATE";

        public static final String USES_FORGETTING_CURVE_ATTRIBUTE = "USES_FORGETTING_CURVE";

        public static final String ATTRIBUTE_VALUE_TRUE = "1";



        private static final String DICTIONARY_VERSION_ATTRIBUTE = "version";

        private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale";

        private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary";

java/src/com/android/inputmethod/latin/personalization/DynamicPredictionDictionaryBase.java→java/src/com/android/inputmethod/latin/personalization/DecayingExpandableBinaryDictionaryBase.java

+19 −5
Original line number Diff line number Diff line
@@ -34,12 +34,15 @@ import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListe 
import java.io.File;

import java.io.IOException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.Map;



/**

 * This class is a base class of a dictionary for the personalized prediction language model.

 * This class is a base class of a dictionary that supports decaying for the personalized language

 * model.

 */

public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDictionary {

    private static final String TAG = DynamicPredictionDictionaryBase.class.getSimpleName();

public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary {

    private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName();

    public static final boolean DBG_SAVE_RESTORE = false;

    private static final boolean DBG_STRESS_TEST = false;

    private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
@@ -60,8 +63,9 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi 
    // Should always be false except when we use this class for test

    @UsedForTesting boolean mIsTest = false;



    /* package */ DynamicPredictionDictionaryBase(final Context context, final String locale,

            final SharedPreferences sp, final String dictionaryType, final String fileName) {

    /* package */ DecayingExpandableBinaryDictionaryBase(final Context context,

            final String locale, final SharedPreferences sp, final String dictionaryType,

            final String fileName) {

        super(context, fileName, dictionaryType, true);

        mLocale = locale;

        mFileName = fileName;
@@ -83,6 +87,16 @@ public abstract class DynamicPredictionDictionaryBase extends ExpandableBinaryDi 
        Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);

    }



    @Override

    protected Map<String, String> getHeaderAttributeMap() {

        HashMap<String, String> attributeMap = new HashMap<String, String>();

        attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,

                FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);

        attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,

                FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);

        return attributeMap;

    }



    @Override

    protected boolean hasContentChanged() {

        return false;

java/src/com/android/inputmethod/latin/personalization/PersonalizationDictionaryUpdateSession.java

+9 −8
Original line number Diff line number Diff line
@@ -46,7 +46,7 @@ public abstract class PersonalizationDictionaryUpdateSession { 


    // TODO: Use a dynamic binary dictionary instead

    public WeakReference<PersonalizationDictionary> mDictionary;

    public WeakReference<DynamicPredictionDictionaryBase> mPredictionDictionary;

    public WeakReference<DecayingExpandableBinaryDictionaryBase> mPredictionDictionary;

    public final String mSystemLocale;

    public PersonalizationDictionaryUpdateSession(String locale) {

        mSystemLocale = locale;
@@ -60,15 +60,16 @@ public abstract class PersonalizationDictionaryUpdateSession { 
        mDictionary = new WeakReference<PersonalizationDictionary>(dictionary);

    }



    public void setPredictionDictionary(DynamicPredictionDictionaryBase dictionary) {

        mPredictionDictionary = new WeakReference<DynamicPredictionDictionaryBase>(dictionary);

    public void setPredictionDictionary(DecayingExpandableBinaryDictionaryBase dictionary) {

        mPredictionDictionary =

                new WeakReference<DecayingExpandableBinaryDictionaryBase>(dictionary);

    }



    protected PersonalizationDictionary getDictionary() {

        return mDictionary == null ? null : mDictionary.get();

    }



    protected DynamicPredictionDictionaryBase getPredictionDictionary() {

    protected DecayingExpandableBinaryDictionaryBase getPredictionDictionary() {

        return mPredictionDictionary == null ? null : mPredictionDictionary.get();

    }
@@ -81,7 +82,7 @@ public abstract class PersonalizationDictionaryUpdateSession { 
    }



    private void unsetPredictionDictionary() {

        final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();

        final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();

        if (dictionary == null) {

            return;

        }
@@ -89,7 +90,7 @@ public abstract class PersonalizationDictionaryUpdateSession { 
    }



    public void clearAndFlushPredictionDictionary(Context context) {

        final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();

        final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();

        if (dictionary == null) {

            return;

        }
@@ -105,7 +106,7 @@ public abstract class PersonalizationDictionaryUpdateSession { 
    // TODO: Support multi locale to add bigram

    public void addBigramToPersonalizationDictionary(String word0, String word1, boolean isValid,

            int frequency) {

        final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();

        final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();

        if (dictionary == null) {

            return;

        }
@@ -116,7 +117,7 @@ public abstract class PersonalizationDictionaryUpdateSession { 
    // TODO: Support multi locale to add bigram

    public void addBigramsToPersonalizationDictionary(

            final ArrayList<PersonalizationLanguageModelParam> lmParams) {

        final DynamicPredictionDictionaryBase dictionary = getPredictionDictionary();

        final DecayingExpandableBinaryDictionaryBase dictionary = getPredictionDictionary();

        if (dictionary == null) {

            return;

        }