Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit c4696b2e authored by Keisuke Kuroyanagi's avatar Keisuke Kuroyanagi
Browse files

Save language model in the body buffer. 

Bug: 14425059
Change-Id: Iaec277f7bed03d6c6780c6ce90fbe5fe799e175e
parent c0c674cd

native/jni/NativeFileList.mk

+1 −0
Original line number Diff line number Diff line
@@ -72,6 +72,7 @@ LATIN_IME_CORE_SRC_FILES := \ 
        ver4_pt_node_array_reader.cpp) \

    $(addprefix suggest/policyimpl/dictionary/structure/v4/content/, \

        bigram_dict_content.cpp \

        language_model_dict_content.cpp \

        probability_dict_content.cpp \

        shortcut_dict_content.cpp \

        sparse_table_dict_content.cpp \

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.cpp

0 → 100644
+25 −0
Original line number Diff line number Diff line 
/*

 * Copyright (C) 2014, The Android Open Source Project

 *

 * Licensed under the Apache License, Version 2.0 (the "License");

 * you may not use this file except in compliance with the License.

 * You may obtain a copy of the License at

 *

 *     http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */



#include "suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h"



namespace latinime {



bool LanguageModelDictContent::save(FILE *const file) const {

    return mTrieMap.save(file);

}



} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/structure/v4/content/language_model_dict_content.h

+13 −1
Original line number Diff line number Diff line
@@ -17,16 +17,28 @@ 
#ifndef LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H

#define LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H



#include <cstdio>



#include "defines.h"

#include "suggest/policyimpl/dictionary/utils/trie_map.h"

#include "utils/byte_array_view.h"



namespace latinime {



class LanguageModelDictContent {

 public:

    explicit LanguageModelDictContent(const bool hasHistoricalInfo) {}

    LanguageModelDictContent(const ReadWriteByteArrayView trieMapBuffer,

            const bool hasHistoricalInfo)

            : mTrieMap(trieMapBuffer) {}



    explicit LanguageModelDictContent(const bool hasHistoricalInfo) : mTrieMap() {}



    bool save(FILE *const file) const;



 private:

    DISALLOW_COPY_AND_ASSIGN(LanguageModelDictContent);



    TrieMap mTrieMap;

};

} // namespace latinime

#endif /* LATINIME_LANGUAGE_MODEL_DICT_CONTENT_H */

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_buffers.cpp

+10 −1
Original line number Diff line number Diff line
@@ -162,6 +162,11 @@ bool Ver4DictBuffers::flushDictBuffers(FILE *const file) const { 
        AKLOGE("Probability dict content cannot be written.");

        return false;

    }

    // Write language model content.

    if (!mLanguageModelDictContent.save(file)) {

        AKLOGE("Language model dict content cannot be written.");

        return false;

    }

    // Write bigram dict content.

    if (!mBigramDictContent.flushToFile(file)) {

        AKLOGE("Bigram dict content cannot be written.");
@@ -195,7 +200,11 @@ Ver4DictBuffers::Ver4DictBuffers(MmappedBuffer::MmappedBufferPtr &&headerBuffer, 
                  contentBuffers[Ver4DictConstants::PROBABILITY_BUFFER_INDEX],

                  contentBufferSizes[Ver4DictConstants::PROBABILITY_BUFFER_INDEX],

                  mHeaderPolicy.hasHistoricalInfoOfWords()),

          mLanguageModelDictContent(mHeaderPolicy.hasHistoricalInfoOfWords()),

          mLanguageModelDictContent(

                  ReadWriteByteArrayView(

                          contentBuffers[Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX],

                          contentBufferSizes[Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX]),

                  mHeaderPolicy.hasHistoricalInfoOfWords()),

          mBigramDictContent(&contentBuffers[Ver4DictConstants::BIGRAM_BUFFERS_INDEX],

                  &contentBufferSizes[Ver4DictConstants::BIGRAM_BUFFERS_INDEX],

                  mHeaderPolicy.hasHistoricalInfoOfWords()),

native/jni/src/suggest/policyimpl/dictionary/structure/v4/ver4_dict_constants.cpp

+5 −1
Original line number Diff line number Diff line
@@ -31,14 +31,17 @@ const int Ver4DictConstants::MAX_DICT_EXTENDED_REGION_SIZE = 1 * 1024 * 1024; 
// NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT for bigram and shortcut.

const size_t Ver4DictConstants::NUM_OF_CONTENT_BUFFERS_IN_BODY_FILE =

        NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT * 3

                + NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT

                + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT * 2;

const int Ver4DictConstants::TRIE_BUFFER_INDEX = 0;

const int Ver4DictConstants::TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX =

        TRIE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;

const int Ver4DictConstants::PROBABILITY_BUFFER_INDEX =

        TERMINAL_ADDRESS_LOOKUP_TABLE_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;

const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX =

const int Ver4DictConstants::LANGUAGE_MODEL_BUFFER_INDEX =

        PROBABILITY_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT;

const int Ver4DictConstants::BIGRAM_BUFFERS_INDEX =

        LANGUAGE_MODEL_BUFFER_INDEX + NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT;

const int Ver4DictConstants::SHORTCUT_BUFFERS_INDEX =

        BIGRAM_BUFFERS_INDEX + NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT;
@@ -73,5 +76,6 @@ const int Ver4DictConstants::SHORTCUT_HAS_NEXT_MASK = 0x80; 


const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SINGLE_DICT_CONTENT = 1;

const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_SPARSE_TABLE_DICT_CONTENT = 3;

const size_t Ver4DictConstants::NUM_OF_BUFFERS_FOR_LANGUAGE_MODEL_DICT_CONTENT = 1;



} // namespace latinime