Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit ea89cb40 authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi
Browse files

Use dynamic operations to construct all ver4 dicts. 

Bug: 8187060
Bug: 13127350
Change-Id: I081ee904c41898128efa8ba7a1bf3fa0a46c6231
parent d1394b69

java/src/com/android/inputmethod/latin/AbstractDictionaryWriter.java

deleted100644 → 0
+0 −72
Original line number Original line Diff line number Diff line 
/*

 * Copyright (C) 2013 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.inputmethod.latin;



import android.util.Log;



import com.android.inputmethod.latin.makedict.DictEncoder;

import com.android.inputmethod.latin.makedict.UnsupportedFormatException;

import com.android.inputmethod.latin.makedict.Ver4DictEncoder;

import com.android.inputmethod.latin.utils.FileUtils;



import java.io.File;

import java.io.IOException;

import java.util.Map;



abstract public class AbstractDictionaryWriter {

    /** Used for Log actions from this class */

    private static final String TAG = AbstractDictionaryWriter.class.getSimpleName();



    public AbstractDictionaryWriter() {

    }



    abstract public void clear();



    /**

     * Add a unigram with an optional shortcut to the dictionary.

     * @param word The word to add.

     * @param shortcutTarget A shortcut target for this word, or null if none.

     * @param frequency The frequency for this unigram.

     * @param shortcutFreq The frequency of the shortcut (0~15, with 15 = whitelist). Ignored

     *   if shortcutTarget is null.

     * @param isNotAWord true if this is not a word, i.e. shortcut only.

     */

    abstract public void addUnigramWord(final String word, final String shortcutTarget,

            final int frequency, final int shortcutFreq, final boolean isNotAWord);



    // TODO: Remove lastModifiedTime after making binary dictionary support forgetting curve.

    abstract public void addBigramWords(final String word0, final String word1,

            final int frequency, final boolean isValid, final long lastModifiedTime);



    abstract public void removeBigramWords(final String word0, final String word1);



    abstract protected void writeDictionary(final DictEncoder dictEncoder,

            final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException;



    public void write(final File file, final Map<String, String> attributeMap) {

        try {

            FileUtils.deleteRecursively(file);

            file.mkdir();

            final DictEncoder dictEncoder = new Ver4DictEncoder(file);

            writeDictionary(dictEncoder, attributeMap);

        } catch (IOException e) {

            Log.e(TAG, "IO exception while writing file", e);

        } catch (UnsupportedFormatException e) {

            Log.e(TAG, "Unsupported format", e);

        }

    }

}

java/src/com/android/inputmethod/latin/BinaryDictionary.java

+0 −1
Original line number Original line Diff line number Diff line
@@ -30,7 +30,6 @@ import com.android.inputmethod.latin.makedict.UnsupportedFormatException; 
import com.android.inputmethod.latin.makedict.WordProperty;

import com.android.inputmethod.latin.makedict.WordProperty;

import com.android.inputmethod.latin.settings.NativeSuggestOptions;

import com.android.inputmethod.latin.settings.NativeSuggestOptions;

import com.android.inputmethod.latin.utils.CollectionUtils;

import com.android.inputmethod.latin.utils.CollectionUtils;

import com.android.inputmethod.latin.utils.FileUtils;

import com.android.inputmethod.latin.utils.JniUtils;

import com.android.inputmethod.latin.utils.JniUtils;

import com.android.inputmethod.latin.utils.LanguageModelParam;

import com.android.inputmethod.latin.utils.LanguageModelParam;

import com.android.inputmethod.latin.utils.StringUtils;

import com.android.inputmethod.latin.utils.StringUtils;

java/src/com/android/inputmethod/latin/ContactsBinaryDictionary.java

+30 −26
Original line number Original line Diff line number Diff line
@@ -78,7 +78,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { 
    public ContactsBinaryDictionary(final Context context, final Locale locale,

    public ContactsBinaryDictionary(final Context context, final Locale locale,

            final File dictFile) {

            final File dictFile) {

        super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_CONTACTS,

        super(context, getDictName(NAME, locale, dictFile), locale, Dictionary.TYPE_CONTACTS,

                false /* isUpdatable */, dictFile);

                dictFile);

        mLocale = locale;

        mLocale = locale;

        mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale);

        mUseFirstLastBigrams = useFirstLastBigramsForLocale(locale);

        registerObserver(context);

        registerObserver(context);
@@ -114,14 +114,14 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { 
    }

    }





    @Override

    @Override

    public void loadDictionaryAsync() {

    public void loadInitialContentsLocked() {

        loadDeviceAccountsEmailAddresses();

        loadDeviceAccountsEmailAddressesLocked();

        loadDictionaryAsyncForUri(ContactsContract.Profile.CONTENT_URI);

        loadDictionaryForUriLocked(ContactsContract.Profile.CONTENT_URI);

        // TODO: Switch this URL to the newer ContactsContract too

        // TODO: Switch this URL to the newer ContactsContract too

        loadDictionaryAsyncForUri(Contacts.CONTENT_URI);

        loadDictionaryForUriLocked(Contacts.CONTENT_URI);

    }

    }





    private void loadDeviceAccountsEmailAddresses() {

    private void loadDeviceAccountsEmailAddressesLocked() {

        final List<String> accountVocabulary =

        final List<String> accountVocabulary =

                AccountUtils.getDeviceAccountsEmailAddresses(mContext);

                AccountUtils.getDeviceAccountsEmailAddresses(mContext);

        if (accountVocabulary == null || accountVocabulary.isEmpty()) {

        if (accountVocabulary == null || accountVocabulary.isEmpty()) {
@@ -131,12 +131,14 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { 
            if (DEBUG) {

            if (DEBUG) {

                Log.d(TAG, "loadAccountVocabulary: " + word);

                Log.d(TAG, "loadAccountVocabulary: " + word);

            }

            }

            super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS, 0 /* shortcutFreq */,

            runGCIfRequiredLocked(true /* mindsBlockByGC */);

                    false /* isNotAWord */);

            addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS, null /* shortcut */,

                    0 /* shortcutFreq */, false /* isNotAWord */, false /* isBlacklisted */,

                    BinaryDictionary.NOT_A_VALID_TIMESTAMP);

        }

        }

    }

    }





    private void loadDictionaryAsyncForUri(final Uri uri) {

    private void loadDictionaryForUriLocked(final Uri uri) {

        Cursor cursor = null;

        Cursor cursor = null;

        try {

        try {

            cursor = mContext.getContentResolver().query(uri, PROJECTION, null, null, null);

            cursor = mContext.getContentResolver().query(uri, PROJECTION, null, null, null);
@@ -145,7 +147,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { 
            }

            }

            if (cursor.moveToFirst()) {

            if (cursor.moveToFirst()) {

                sContactCountAtLastRebuild = getContactCount();

                sContactCountAtLastRebuild = getContactCount();

                addWords(cursor);

                addWordsLocked(cursor);

            }

            }

        } catch (final SQLiteException e) {

        } catch (final SQLiteException e) {

            Log.e(TAG, "SQLiteException in the remote Contacts process.", e);

            Log.e(TAG, "SQLiteException in the remote Contacts process.", e);
@@ -166,12 +168,12 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { 
        return false;

        return false;

    }

    }





    private void addWords(final Cursor cursor) {

    private void addWordsLocked(final Cursor cursor) {

        int count = 0;

        int count = 0;

        while (!cursor.isAfterLast() && count < MAX_CONTACT_COUNT) {

        while (!cursor.isAfterLast() && count < MAX_CONTACT_COUNT) {

            String name = cursor.getString(INDEX_NAME);

            String name = cursor.getString(INDEX_NAME);

            if (isValidName(name)) {

            if (isValidName(name)) {

                addName(name);

                addNameLocked(name);

                ++count;

                ++count;

            } else {

            } else {

                if (DEBUG_DUMP) {

                if (DEBUG_DUMP) {
@@ -207,7 +209,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { 
     * Adds the words in a name (e.g., firstname/lastname) to the binary dictionary along with their

     * Adds the words in a name (e.g., firstname/lastname) to the binary dictionary along with their

     * bigrams depending on locale.

     * bigrams depending on locale.

     */

     */

    private void addName(final String name) {

    private void addNameLocked(final String name) {

        int len = StringUtils.codePointCount(name);

        int len = StringUtils.codePointCount(name);

        String prevWord = null;

        String prevWord = null;

        // TODO: Better tokenization for non-Latin writing systems

        // TODO: Better tokenization for non-Latin writing systems
@@ -226,13 +228,15 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { 
                    if (DEBUG) {

                    if (DEBUG) {

                        Log.d(TAG, "addName " + name + ", " + word + ", " + prevWord);

                        Log.d(TAG, "addName " + name + ", " + word + ", " + prevWord);

                    }

                    }

                    super.addWord(word, null /* shortcut */, FREQUENCY_FOR_CONTACTS,

                    runGCIfRequiredLocked(true /* mindsBlockByGC */);

                            0 /* shortcutFreq */, false /* isNotAWord */);

                    addWordDynamicallyLocked(word, FREQUENCY_FOR_CONTACTS,

                    if (!TextUtils.isEmpty(prevWord)) {

                            null /* shortcut */, 0 /* shortcutFreq */, false /* isNotAWord */,

                        if (mUseFirstLastBigrams) {

                            false /* isBlacklisted */, BinaryDictionary.NOT_A_VALID_TIMESTAMP);

                            super.addBigram(prevWord, word, FREQUENCY_FOR_CONTACTS_BIGRAM,

                    if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) {

                                    0 /* lastModifiedTime */);

                        runGCIfRequiredLocked(true /* mindsBlockByGC */);

                        }

                        addBigramDynamicallyLocked(prevWord, word,

                                FREQUENCY_FOR_CONTACTS_BIGRAM,

                                BinaryDictionary.NOT_A_VALID_TIMESTAMP);

                    }

                    }

                    prevWord = word;

                    prevWord = word;

                }

                }
@@ -258,12 +262,12 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { 
    }

    }





    @Override

    @Override

    protected boolean needsToReloadBeforeWriting() {

    protected boolean needsToReloadAfterCreation() {

        return true;

        return true;

    }

    }





    @Override

    @Override

    protected boolean hasContentChanged() {

    protected boolean haveContentsChanged() {

        final long startTime = SystemClock.uptimeMillis();

        final long startTime = SystemClock.uptimeMillis();

        final int contactCount = getContactCount();

        final int contactCount = getContactCount();

        if (contactCount > MAX_CONTACT_COUNT) {

        if (contactCount > MAX_CONTACT_COUNT) {
@@ -291,7 +295,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { 
            if (cursor.moveToFirst()) {

            if (cursor.moveToFirst()) {

                while (!cursor.isAfterLast()) {

                while (!cursor.isAfterLast()) {

                    String name = cursor.getString(INDEX_NAME);

                    String name = cursor.getString(INDEX_NAME);

                    if (isValidName(name) && !isNameInDictionary(name)) {

                    if (isValidName(name) && !isNameInDictionaryLocked(name)) {

                        if (DEBUG) {

                        if (DEBUG) {

                            Log.d(TAG, "Contact name missing: " + name + " (runtime = "

                            Log.d(TAG, "Contact name missing: " + name + " (runtime = "

                                    + (SystemClock.uptimeMillis() - startTime) + " ms)");

                                    + (SystemClock.uptimeMillis() - startTime) + " ms)");
@@ -321,7 +325,7 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { 
    /**

    /**

     * Checks if the words in a name are in the current binary dictionary.

     * Checks if the words in a name are in the current binary dictionary.

     */

     */

    private boolean isNameInDictionary(final String name) {

    private boolean isNameInDictionaryLocked(final String name) {

        int len = StringUtils.codePointCount(name);

        int len = StringUtils.codePointCount(name);

        String prevWord = null;

        String prevWord = null;

        for (int i = 0; i < len; i++) {

        for (int i = 0; i < len; i++) {
@@ -332,11 +336,11 @@ public class ContactsBinaryDictionary extends ExpandableBinaryDictionary { 
                final int wordLen = StringUtils.codePointCount(word);

                final int wordLen = StringUtils.codePointCount(word);

                if (wordLen < MAX_WORD_LENGTH && wordLen > 1) {

                if (wordLen < MAX_WORD_LENGTH && wordLen > 1) {

                    if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) {

                    if (!TextUtils.isEmpty(prevWord) && mUseFirstLastBigrams) {

                        if (!super.isValidBigramLocked(prevWord, word)) {

                        if (!isValidBigramLocked(prevWord, word)) {

                            return false;

                            return false;

                        }

                        }

                    } else {

                    } else {

                        if (!super.isValidWordLocked(word)) {

                        if (!isValidWordLocked(word)) {

                            return false;

                            return false;

                        }

                        }

                    }

                    }

java/src/com/android/inputmethod/latin/DictionaryWriter.java

deleted100644 → 0
+0 −92
Original line number Original line Diff line number Diff line 
/*

 * Copyright (C) 2013 The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *      http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



package com.android.inputmethod.latin;



import com.android.inputmethod.latin.makedict.DictEncoder;

import com.android.inputmethod.latin.makedict.FormatSpec;

import com.android.inputmethod.latin.makedict.FusionDictionary;

import com.android.inputmethod.latin.makedict.FusionDictionary.PtNodeArray;

import com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString;

import com.android.inputmethod.latin.makedict.ProbabilityInfo;

import com.android.inputmethod.latin.makedict.UnsupportedFormatException;

import com.android.inputmethod.latin.utils.CollectionUtils;



import java.io.IOException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.Map;



/**

 * An in memory dictionary for memorizing entries and writing a binary dictionary.

 */

public class DictionaryWriter extends AbstractDictionaryWriter {

    private static final int BINARY_DICT_VERSION = FormatSpec.VERSION4;

    private static final FormatSpec.FormatOptions FORMAT_OPTIONS =

            new FormatSpec.FormatOptions(BINARY_DICT_VERSION, false /* hasTimestamp */);



    private FusionDictionary mFusionDictionary;



    public DictionaryWriter() {

        clear();

    }



    @Override

    public void clear() {

        final HashMap<String, String> attributes = CollectionUtils.newHashMap();

        mFusionDictionary = new FusionDictionary(new PtNodeArray(),

                new FusionDictionary.DictionaryOptions(attributes));

    }



    /**

     * Adds a word unigram to the fusion dictionary.

     */

    // TODO: Create "cache dictionary" to cache fresh words for frequently updated dictionaries,

    // considering performance regression.

    @Override

    public void addUnigramWord(final String word, final String shortcutTarget,

            final int probability, final int shortcutProbability, final boolean isNotAWord) {

        if (shortcutTarget == null) {

            mFusionDictionary.add(word, new ProbabilityInfo(probability), null, isNotAWord);

        } else {

            // TODO: Do this in the subclass, with this class taking an arraylist.

            final ArrayList<WeightedString> shortcutTargets = CollectionUtils.newArrayList();

            shortcutTargets.add(new WeightedString(shortcutTarget, shortcutProbability));

            mFusionDictionary.add(word, new ProbabilityInfo(probability), shortcutTargets,

                    isNotAWord);

        }

    }



    @Override

    public void addBigramWords(final String word0, final String word1, final int probability,

            final boolean isValid, final long lastModifiedTime) {

        mFusionDictionary.setBigram(word0, word1, new ProbabilityInfo(probability));

    }



    @Override

    public void removeBigramWords(final String word0, final String word1) {

        // This class don't support removing bigram words.

    }



    @Override

    protected void writeDictionary(final DictEncoder dictEncoder,

            final Map<String, String> attributeMap) throws IOException, UnsupportedFormatException {

        for (final Map.Entry<String, String> entry : attributeMap.entrySet()) {

            mFusionDictionary.addOptionAttribute(entry.getKey(), entry.getValue());

        }

        dictEncoder.writeDictionary(mFusionDictionary, FORMAT_OPTIONS);

    }

}

java/src/com/android/inputmethod/latin/ExpandableBinaryDictionary.java

+95 −195

File changed.

Preview size limit exceeded, changes collapsed.