Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit d14ffcfc authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi
Browse files

Remove BinaryDictionaryInfo and introduce MmapedBuffer. 

Bug: 6669677

Change-Id: I1459bb53a9761d920427e4b5d925a2844955c88c
parent 08ce337d

native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp

+9 −59
Original line number Diff line number Diff line
@@ -18,42 +18,24 @@ 


#include "com_android_inputmethod_latin_BinaryDictionary.h"



#include <cerrno>

#include <cstring> // for memset()

#include <fcntl.h>

#include <sys/mman.h>

#include <unistd.h>



#include "defines.h"

#include "jni.h"

#include "jni_common.h"

#include "suggest/core/dictionary/binary_dictionary_info.h"

#include "suggest/core/dictionary/dictionary.h"

#include "suggest/core/suggest_options.h"

#include "suggest/policyimpl/dictionary/utils/format_utils.h"

#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"

#include "utils/autocorrection_threshold_utils.h"



namespace latinime {



class ProximityInfo;



// Helper method

static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd) {

    int ret = munmap(const_cast<void *>(dictBuf), length);

    if (ret != 0) {

        AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno);

    }

    ret = close(fd);

    if (ret != 0) {

        AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno);

    }

}



static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,

        jlong dictOffset, jlong dictSize, jboolean isUpdatable) {

    PROF_OPEN;

    PROF_START(66);

    // TODO: Move dictionary buffer handling to policyimpl.

    const jsize sourceDirUtf8Length = env->GetStringUTFLength(sourceDir);

    if (sourceDirUtf8Length <= 0) {

        AKLOGE("DICT: Can't get sourceDir string");
@@ -62,41 +44,16 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s 
    char sourceDirChars[sourceDirUtf8Length + 1];

    env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars);

    sourceDirChars[sourceDirUtf8Length] = '\0';

    int fd = 0;

    void *dictBuf = 0;

    int offset = 0;

    const bool updatableMmap = (isUpdatable == JNI_TRUE);

    const int openMode = updatableMmap ? O_RDWR : O_RDONLY;

    fd = open(sourceDirChars, openMode);

    if (fd < 0) {

        AKLOGE("DICT: Can't open sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno);

        return 0;

    }

    int pagesize = getpagesize();

    offset = static_cast<int>(dictOffset) % pagesize;

    int adjDictOffset = static_cast<int>(dictOffset) - offset;

    int adjDictSize = static_cast<int>(dictSize) + offset;

    const int protMode = updatableMmap ? PROT_READ | PROT_WRITE : PROT_READ;

    dictBuf = mmap(0, adjDictSize, protMode, MAP_PRIVATE, fd, adjDictOffset);

    if (dictBuf == MAP_FAILED) {

        AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno);

    DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy =

            DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy(

                    sourceDirChars, static_cast<int>(sourceDirUtf8Length),

                    static_cast<int>(dictOffset), static_cast<int>(dictSize),

                    isUpdatable == JNI_TRUE);

    if (!dictionaryStructureWithBufferPolicy) {

        return 0;

    }

    dictBuf = static_cast<char *>(dictBuf) + offset;

    if (!dictBuf) {

        AKLOGE("DICT: dictBuf is null");

        return 0;

    }

    Dictionary *dictionary = 0;

    if (FormatUtils::UNKNOWN_VERSION

            == FormatUtils::detectFormatVersion(static_cast<uint8_t *>(dictBuf),

                    static_cast<int>(dictSize))) {

        AKLOGE("DICT: dictionary format is unknown, bad magic number");

        releaseDictBuf(static_cast<const char *>(dictBuf) - offset, adjDictSize, fd);

    } else {

        dictionary = new Dictionary(env, dictBuf, static_cast<int>(dictSize), fd, offset,

                updatableMmap);

    }



    Dictionary *const dictionary = new Dictionary(env, dictionaryStructureWithBufferPolicy);

    PROF_END(66);

    PROF_CLOSE;

    return reinterpret_cast<jlong>(dictionary);
@@ -105,13 +62,6 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s 
static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) {

    Dictionary *dictionary = reinterpret_cast<Dictionary *>(dict);

    if (!dictionary) return;

    const BinaryDictionaryInfo *const binaryDictionaryInfo = dictionary->getBinaryDictionaryInfo();

    const int dictBufOffset = binaryDictionaryInfo->getDictBufOffset();

    const void *dictBuf = binaryDictionaryInfo->getDictBuf();

    if (!dictBuf) return;

    releaseDictBuf(static_cast<const char *>(dictBuf) - dictBufOffset,

            binaryDictionaryInfo->getDictSize() + dictBufOffset,

            binaryDictionaryInfo->getMmapFd());

    delete dictionary;

}

native/jni/src/suggest/core/dictionary/binary_dictionary_info.h

deleted100644 → 0
+0 −65
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2013, The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *     http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#ifndef LATINIME_BINARY_DICTIONARY_INFO_H

#define LATINIME_BINARY_DICTIONARY_INFO_H



#include <stdint.h>



#include "defines.h"



namespace latinime {



class BinaryDictionaryInfo {

 public:

     AK_FORCE_INLINE BinaryDictionaryInfo(const uint8_t *const dictBuf,

            const int dictSize, const int mmapFd, const int dictBufOffset, const bool isUpdatable)

            : mDictBuf(dictBuf), mDictSize(dictSize), mMmapFd(mmapFd),

              mDictBufOffset(dictBufOffset), mIsUpdatable(isUpdatable) {}



    ~BinaryDictionaryInfo() {}



    AK_FORCE_INLINE const uint8_t *getDictBuf() const {

        return mDictBuf;

    }



    AK_FORCE_INLINE int getDictSize() const {

        return mDictSize;

    }



    AK_FORCE_INLINE int getMmapFd() const {

        return mMmapFd;

    }



    AK_FORCE_INLINE int getDictBufOffset() const {

        return mDictBufOffset;

    }



    AK_FORCE_INLINE bool isDynamicallyUpdatable() const {

        return mIsUpdatable;

    }



 private:

    DISALLOW_COPY_AND_ASSIGN(BinaryDictionaryInfo);



    const uint8_t *const mDictBuf;

    const int mDictSize;

    const int mMmapFd;

    const int mDictBufOffset;

    const bool mIsUpdatable;

};

}

#endif /* LATINIME_BINARY_DICTIONARY_INFO_H */

native/jni/src/suggest/core/dictionary/dictionary.cpp

+6 −26
Original line number Diff line number Diff line
@@ -23,23 +23,19 @@ 
#include "defines.h"

#include "suggest/core/dictionary/bigram_dictionary.h"

#include "suggest/core/policy/dictionary_header_structure_policy.h"

#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"

#include "suggest/core/session/dic_traverse_session.h"

#include "suggest/core/suggest.h"

#include "suggest/core/suggest_options.h"

#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"

#include "suggest/policyimpl/gesture/gesture_suggest_policy_factory.h"

#include "suggest/policyimpl/typing/typing_suggest_policy_factory.h"

#include "utils/log_utils.h"



namespace latinime {



Dictionary::Dictionary(JNIEnv *env, void *dict, int dictSize, int mmapFd,

        int dictBufOffset, bool isUpdatable)

        : mBinaryDictionaryInfo(static_cast<const uint8_t *>(dict), dictSize, mmapFd,

                dictBufOffset, isUpdatable),

          mDictionaryStructureWithBufferPolicy(DictionaryStructureWithBufferPolicyFactory

                  ::newDictionaryStructureWithBufferPolicy(

                          static_cast<const uint8_t *>(dict), dictSize)),

Dictionary::Dictionary(JNIEnv *env,

        DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy)

        : mDictionaryStructureWithBufferPolicy(dictionaryStructureWithBufferPolicy),

          mBigramDictionary(new BigramDictionary(mDictionaryStructureWithBufferPolicy)),

          mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())),

          mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) {
@@ -102,32 +98,17 @@ bool Dictionary::isValidBigram(const int *word0, int length0, const int *word1, 
}



void Dictionary::addUnigramWord(const int *const word, const int length, const int probability) {

    if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) {

        // This method should not be called for non-updatable dictionary.

        AKLOGI("Warning: Dictionary::addUnigramWord() is called for non-updatable dictionary.");

        return;

    }

    mDictionaryStructureWithBufferPolicy->addUnigramWord(word, length, probability);

}



void Dictionary::addBigramWords(const int *const word0, const int length0, const int *const word1,

        const int length1, const int probability) {

    if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) {

        // This method should not be called for non-updatable dictionary.

        AKLOGI("Warning: Dictionary::addBigramWords() is called for non-updatable dictionary.");

        return;

    }

    mDictionaryStructureWithBufferPolicy->addBigramWords(word0, length0, word1, length1,

            probability);

}



void Dictionary::removeBigramWords(const int *const word0, const int length0,

        const int *const word1, const int length1) {

    if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) {

        // This method should not be called for non-updatable dictionary.

        AKLOGI("Warning: Dictionary::removeBigramWords() is called for non-updatable dictionary.");

        return;

    }

    mDictionaryStructureWithBufferPolicy->removeBigramWords(word0, length0, word1, length1);

}
@@ -155,9 +136,8 @@ void Dictionary::logDictionaryInfo(JNIEnv *const env) const { 
            dateStringCharBuffer, BUFFER_SIZE);



    LogUtils::logToJava(env,

            "Dictionary info: dictionary = %s ; version = %s ; date = %s ; filesize = %i",

            dictionaryIdCharBuffer, versionStringCharBuffer, dateStringCharBuffer,

            mBinaryDictionaryInfo.getDictSize());

            "Dictionary info: dictionary = %s ; version = %s ; date = %s",

            dictionaryIdCharBuffer, versionStringCharBuffer, dateStringCharBuffer);

}



} // namespace latinime

native/jni/src/suggest/core/dictionary/dictionary.h

+2 −9
Original line number Diff line number Diff line
@@ -21,7 +21,6 @@ 


#include "defines.h"

#include "jni.h"

#include "suggest/core/dictionary/binary_dictionary_info.h"



namespace latinime {
@@ -54,8 +53,8 @@ class Dictionary { 
    static const int KIND_FLAG_POSSIBLY_OFFENSIVE = 0x80000000;

    static const int KIND_FLAG_EXACT_MATCH = 0x40000000;



    Dictionary(JNIEnv *env, void *dict, int dictSize, int mmapFd, int dictBufOffset,

            bool isUpdatable);

    Dictionary(JNIEnv *env,

            DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPoilcy);



    int getSuggestions(ProximityInfo *proximityInfo, DicTraverseSession *traverseSession,

            int *xcoordinates, int *ycoordinates, int *times, int *pointerIds, int *inputCodePoints,
@@ -78,11 +77,6 @@ class Dictionary { 
    void removeBigramWords(const int *const word0, const int length0, const int *const word1,

            const int length1);



    // TODO: Remove.

    const BinaryDictionaryInfo *getBinaryDictionaryInfo() const {

        return &mBinaryDictionaryInfo;

    }



    const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const {

        return mDictionaryStructureWithBufferPolicy;

    }
@@ -92,7 +86,6 @@ class Dictionary { 
 private:

    DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);



    const BinaryDictionaryInfo mBinaryDictionaryInfo;

    DictionaryStructureWithBufferPolicy *const mDictionaryStructureWithBufferPolicy;

    const BigramDictionary *const mBigramDictionary;

    const SuggestInterface *const mGestureSuggest;

native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp

+16 −5
Original line number Diff line number Diff line
@@ -22,18 +22,29 @@ 
#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h"

#include "suggest/policyimpl/dictionary/patricia_trie_policy.h"

#include "suggest/policyimpl/dictionary/utils/format_utils.h"

#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h"



namespace latinime {



/* static */ DictionaryStructureWithBufferPolicy *DictionaryStructureWithBufferPolicyFactory

        ::newDictionaryStructureWithBufferPolicy(const uint8_t *const dictBuf,

                const int dictSize) {

    switch (FormatUtils::detectFormatVersion(dictBuf, dictSize)) {

        ::newDictionaryStructureWithBufferPolicy(const char *const path, const int pathLength,

                const int bufOffset, const int size, const bool isUpdatable) {

    // Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of

    // impl classes of DictionaryStructureWithBufferPolicy.

    const MmapedBuffer *const mmapedBuffer = MmapedBuffer::openBuffer(path, pathLength, bufOffset,

            size, isUpdatable);

    if (!mmapedBuffer) {

        return 0;

    }

    switch (FormatUtils::detectFormatVersion(mmapedBuffer->getBuffer(),

            mmapedBuffer->getBufferSize())) {

        case FormatUtils::VERSION_2:

            return new PatriciaTriePolicy(dictBuf);

            return new PatriciaTriePolicy(mmapedBuffer);

        case FormatUtils::VERSION_3:

            return new DynamicPatriciaTriePolicy(dictBuf);

            return new DynamicPatriciaTriePolicy(mmapedBuffer);

        default:

            AKLOGE("DICT: dictionary format is unknown, bad magic number");

            delete mmapedBuffer;

            ASSERT(false);

            return 0;

    }