Merge "Revert "Remove BinaryDictionaryInfo and introduce MmapedBuffer."" (0238e1eb) · Commits · e / os / android_packages_inputmethods_LatinIME

native/jni/com_android_inputmethod_latin_BinaryDictionary.cpp

+59 −9

Original line number	Diff line number	Diff line
		@@ -18,24 +18,42 @@

		#include "com_android_inputmethod_latin_BinaryDictionary.h"

		#include <cerrno>
		#include <cstring> // for memset()
		#include <fcntl.h>
		#include <sys/mman.h>
		#include <unistd.h>

		#include "defines.h"
		#include "jni.h"
		#include "jni_common.h"
		#include "suggest/core/dictionary/binary_dictionary_info.h"
		#include "suggest/core/dictionary/dictionary.h"
		#include "suggest/core/suggest_options.h"
		#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
		#include "suggest/policyimpl/dictionary/utils/format_utils.h"
		#include "utils/autocorrection_threshold_utils.h"

		namespace latinime {

		class ProximityInfo;

		// Helper method
		static void releaseDictBuf(const void *dictBuf, const size_t length, const int fd) {
		int ret = munmap(const_cast<void *>(dictBuf), length);
		if (ret != 0) {
		AKLOGE("DICT: Failure in munmap. ret=%d errno=%d", ret, errno);
		}
		ret = close(fd);
		if (ret != 0) {
		AKLOGE("DICT: Failure in close. ret=%d errno=%d", ret, errno);
		}
		}

		static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring sourceDir,
		jlong dictOffset, jlong dictSize, jboolean isUpdatable) {
		PROF_OPEN;
		PROF_START(66);
		// TODO: Move dictionary buffer handling to policyimpl.
		const jsize sourceDirUtf8Length = env->GetStringUTFLength(sourceDir);
		if (sourceDirUtf8Length <= 0) {
		AKLOGE("DICT: Can't get sourceDir string");
		@@ -44,16 +62,41 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s
		char sourceDirChars[sourceDirUtf8Length + 1];
		env->GetStringUTFRegion(sourceDir, 0, env->GetStringLength(sourceDir), sourceDirChars);
		sourceDirChars[sourceDirUtf8Length] = '\0';
		DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy =
		DictionaryStructureWithBufferPolicyFactory::newDictionaryStructureWithBufferPolicy(
		sourceDirChars, static_cast<int>(sourceDirUtf8Length),
		static_cast<int>(dictOffset), static_cast<int>(dictSize),
		isUpdatable == JNI_TRUE);
		if (!dictionaryStructureWithBufferPolicy) {
		int fd = 0;
		void *dictBuf = 0;
		int offset = 0;
		const bool updatableMmap = (isUpdatable == JNI_TRUE);
		const int openMode = updatableMmap ? O_RDWR : O_RDONLY;
		fd = open(sourceDirChars, openMode);
		if (fd < 0) {
		AKLOGE("DICT: Can't open sourceDir. sourceDirChars=%s errno=%d", sourceDirChars, errno);
		return 0;
		}

		Dictionary *const dictionary = new Dictionary(env, dictionaryStructureWithBufferPolicy);
		int pagesize = getpagesize();
		offset = static_cast<int>(dictOffset) % pagesize;
		int adjDictOffset = static_cast<int>(dictOffset) - offset;
		int adjDictSize = static_cast<int>(dictSize) + offset;
		const int protMode = updatableMmap ? PROT_READ \| PROT_WRITE : PROT_READ;
		dictBuf = mmap(0, adjDictSize, protMode, MAP_PRIVATE, fd, adjDictOffset);
		if (dictBuf == MAP_FAILED) {
		AKLOGE("DICT: Can't mmap dictionary. errno=%d", errno);
		return 0;
		}
		dictBuf = static_cast<char *>(dictBuf) + offset;
		if (!dictBuf) {
		AKLOGE("DICT: dictBuf is null");
		return 0;
		}
		Dictionary *dictionary = 0;
		if (FormatUtils::UNKNOWN_VERSION
		== FormatUtils::detectFormatVersion(static_cast<uint8_t *>(dictBuf),
		static_cast<int>(dictSize))) {
		AKLOGE("DICT: dictionary format is unknown, bad magic number");
		releaseDictBuf(static_cast<const char *>(dictBuf) - offset, adjDictSize, fd);
		} else {
		dictionary = new Dictionary(env, dictBuf, static_cast<int>(dictSize), fd, offset,
		updatableMmap);
		}
		PROF_END(66);
		PROF_CLOSE;
		return reinterpret_cast<jlong>(dictionary);
		@@ -62,6 +105,13 @@ static jlong latinime_BinaryDictionary_open(JNIEnv *env, jclass clazz, jstring s
		static void latinime_BinaryDictionary_close(JNIEnv *env, jclass clazz, jlong dict) {
		Dictionary dictionary = reinterpret_cast<Dictionary >(dict);
		if (!dictionary) return;
		const BinaryDictionaryInfo *const binaryDictionaryInfo = dictionary->getBinaryDictionaryInfo();
		const int dictBufOffset = binaryDictionaryInfo->getDictBufOffset();
		const void *dictBuf = binaryDictionaryInfo->getDictBuf();
		if (!dictBuf) return;
		releaseDictBuf(static_cast<const char *>(dictBuf) - dictBufOffset,
		binaryDictionaryInfo->getDictSize() + dictBufOffset,
		binaryDictionaryInfo->getMmapFd());
		delete dictionary;
		}

native/jni/src/suggest/core/dictionary/binary_dictionary_info.h

0 → 100644

+65 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2013, The Android Open Source Project
		*
		* Licensed under the Apache License, Version 2.0 (the "License");
		* you may not use this file except in compliance with the License.
		* You may obtain a copy of the License at
		*
		* http://www.apache.org/licenses/LICENSE-2.0
		*
		* Unless required by applicable law or agreed to in writing, software
		* distributed under the License is distributed on an "AS IS" BASIS,
		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		* See the License for the specific language governing permissions and
		* limitations under the License.
		*/

		#ifndef LATINIME_BINARY_DICTIONARY_INFO_H
		#define LATINIME_BINARY_DICTIONARY_INFO_H

		#include <stdint.h>

		#include "defines.h"

		namespace latinime {

		class BinaryDictionaryInfo {
		public:
		AK_FORCE_INLINE BinaryDictionaryInfo(const uint8_t *const dictBuf,
		const int dictSize, const int mmapFd, const int dictBufOffset, const bool isUpdatable)
		: mDictBuf(dictBuf), mDictSize(dictSize), mMmapFd(mmapFd),
		mDictBufOffset(dictBufOffset), mIsUpdatable(isUpdatable) {}

		~BinaryDictionaryInfo() {}

		AK_FORCE_INLINE const uint8_t *getDictBuf() const {
		return mDictBuf;
		}

		AK_FORCE_INLINE int getDictSize() const {
		return mDictSize;
		}

		AK_FORCE_INLINE int getMmapFd() const {
		return mMmapFd;
		}

		AK_FORCE_INLINE int getDictBufOffset() const {
		return mDictBufOffset;
		}

		AK_FORCE_INLINE bool isDynamicallyUpdatable() const {
		return mIsUpdatable;
		}

		private:
		DISALLOW_COPY_AND_ASSIGN(BinaryDictionaryInfo);

		const uint8_t *const mDictBuf;
		const int mDictSize;
		const int mMmapFd;
		const int mDictBufOffset;
		const bool mIsUpdatable;
		};
		}
		#endif /* LATINIME_BINARY_DICTIONARY_INFO_H */

native/jni/src/suggest/core/dictionary/dictionary.cpp

+26 −6

Original line number	Diff line number	Diff line
		@@ -23,19 +23,23 @@
		#include "defines.h"
		#include "suggest/core/dictionary/bigram_dictionary.h"
		#include "suggest/core/policy/dictionary_header_structure_policy.h"
		#include "suggest/core/policy/dictionary_structure_with_buffer_policy.h"
		#include "suggest/core/session/dic_traverse_session.h"
		#include "suggest/core/suggest.h"
		#include "suggest/core/suggest_options.h"
		#include "suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.h"
		#include "suggest/policyimpl/gesture/gesture_suggest_policy_factory.h"
		#include "suggest/policyimpl/typing/typing_suggest_policy_factory.h"
		#include "utils/log_utils.h"

		namespace latinime {

		Dictionary::Dictionary(JNIEnv *env,
		DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPolicy)
		: mDictionaryStructureWithBufferPolicy(dictionaryStructureWithBufferPolicy),
		Dictionary::Dictionary(JNIEnv env, void dict, int dictSize, int mmapFd,
		int dictBufOffset, bool isUpdatable)
		: mBinaryDictionaryInfo(static_cast<const uint8_t *>(dict), dictSize, mmapFd,
		dictBufOffset, isUpdatable),
		mDictionaryStructureWithBufferPolicy(DictionaryStructureWithBufferPolicyFactory
		::newDictionaryStructureWithBufferPolicy(
		static_cast<const uint8_t *>(dict), dictSize)),
		mBigramDictionary(new BigramDictionary(mDictionaryStructureWithBufferPolicy)),
		mGestureSuggest(new Suggest(GestureSuggestPolicyFactory::getGestureSuggestPolicy())),
		mTypingSuggest(new Suggest(TypingSuggestPolicyFactory::getTypingSuggestPolicy())) {
		@@ -98,17 +102,32 @@ bool Dictionary::isValidBigram(const int word0, int length0, const int word1,
		}

		void Dictionary::addUnigramWord(const int *const word, const int length, const int probability) {
		if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) {
		// This method should not be called for non-updatable dictionary.
		AKLOGI("Warning: Dictionary::addUnigramWord() is called for non-updatable dictionary.");
		return;
		}
		mDictionaryStructureWithBufferPolicy->addUnigramWord(word, length, probability);
		}

		void Dictionary::addBigramWords(const int const word0, const int length0, const int const word1,
		const int length1, const int probability) {
		if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) {
		// This method should not be called for non-updatable dictionary.
		AKLOGI("Warning: Dictionary::addBigramWords() is called for non-updatable dictionary.");
		return;
		}
		mDictionaryStructureWithBufferPolicy->addBigramWords(word0, length0, word1, length1,
		probability);
		}

		void Dictionary::removeBigramWords(const int *const word0, const int length0,
		const int *const word1, const int length1) {
		if (!mBinaryDictionaryInfo.isDynamicallyUpdatable()) {
		// This method should not be called for non-updatable dictionary.
		AKLOGI("Warning: Dictionary::removeBigramWords() is called for non-updatable dictionary.");
		return;
		}
		mDictionaryStructureWithBufferPolicy->removeBigramWords(word0, length0, word1, length1);
		}

		@@ -136,8 +155,9 @@ void Dictionary::logDictionaryInfo(JNIEnv *const env) const {
		dateStringCharBuffer, BUFFER_SIZE);

		LogUtils::logToJava(env,
		"Dictionary info: dictionary = %s ; version = %s ; date = %s",
		dictionaryIdCharBuffer, versionStringCharBuffer, dateStringCharBuffer);
		"Dictionary info: dictionary = %s ; version = %s ; date = %s ; filesize = %i",
		dictionaryIdCharBuffer, versionStringCharBuffer, dateStringCharBuffer,
		mBinaryDictionaryInfo.getDictSize());
		}

		} // namespace latinime

native/jni/src/suggest/core/dictionary/dictionary.h

+9 −2

Original line number	Diff line number	Diff line
		@@ -21,6 +21,7 @@

		#include "defines.h"
		#include "jni.h"
		#include "suggest/core/dictionary/binary_dictionary_info.h"

		namespace latinime {

		@@ -53,8 +54,8 @@ class Dictionary {
		static const int KIND_FLAG_POSSIBLY_OFFENSIVE = 0x80000000;
		static const int KIND_FLAG_EXACT_MATCH = 0x40000000;

		Dictionary(JNIEnv *env,
		DictionaryStructureWithBufferPolicy *const dictionaryStructureWithBufferPoilcy);
		Dictionary(JNIEnv env, void dict, int dictSize, int mmapFd, int dictBufOffset,
		bool isUpdatable);

		int getSuggestions(ProximityInfo proximityInfo, DicTraverseSession traverseSession,
		int xcoordinates, int ycoordinates, int times, int pointerIds, int *inputCodePoints,
		@@ -77,6 +78,11 @@ class Dictionary {
		void removeBigramWords(const int const word0, const int length0, const int const word1,
		const int length1);

		// TODO: Remove.
		const BinaryDictionaryInfo *getBinaryDictionaryInfo() const {
		return &mBinaryDictionaryInfo;
		}

		const DictionaryStructureWithBufferPolicy *getDictionaryStructurePolicy() const {
		return mDictionaryStructureWithBufferPolicy;
		}
		@@ -86,6 +92,7 @@ class Dictionary {
		private:
		DISALLOW_IMPLICIT_CONSTRUCTORS(Dictionary);

		const BinaryDictionaryInfo mBinaryDictionaryInfo;
		DictionaryStructureWithBufferPolicy *const mDictionaryStructureWithBufferPolicy;
		const BigramDictionary *const mBigramDictionary;
		const SuggestInterface *const mGestureSuggest;

native/jni/src/suggest/policyimpl/dictionary/dictionary_structure_with_buffer_policy_factory.cpp

+5 −16

Original line number	Diff line number	Diff line
		@@ -22,29 +22,18 @@
		#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_policy.h"
		#include "suggest/policyimpl/dictionary/patricia_trie_policy.h"
		#include "suggest/policyimpl/dictionary/utils/format_utils.h"
		#include "suggest/policyimpl/dictionary/utils/mmaped_buffer.h"

		namespace latinime {

		/* static / DictionaryStructureWithBufferPolicy DictionaryStructureWithBufferPolicyFactory
		::newDictionaryStructureWithBufferPolicy(const char *const path, const int pathLength,
		const int bufOffset, const int size, const bool isUpdatable) {
		// Allocated buffer in MmapedBuffer::openBuffer() will be freed in the destructor of
		// impl classes of DictionaryStructureWithBufferPolicy.
		const MmapedBuffer *const mmapedBuffer = MmapedBuffer::openBuffer(path, pathLength, bufOffset,
		size, isUpdatable);
		if (!mmapedBuffer) {
		return 0;
		}
		switch (FormatUtils::detectFormatVersion(mmapedBuffer->getBuffer(),
		mmapedBuffer->getBufferSize())) {
		::newDictionaryStructureWithBufferPolicy(const uint8_t *const dictBuf,
		const int dictSize) {
		switch (FormatUtils::detectFormatVersion(dictBuf, dictSize)) {
		case FormatUtils::VERSION_2:
		return new PatriciaTriePolicy(mmapedBuffer);
		return new PatriciaTriePolicy(dictBuf);
		case FormatUtils::VERSION_3:
		return new DynamicPatriciaTriePolicy(mmapedBuffer);
		return new DynamicPatriciaTriePolicy(dictBuf);
		default:
		AKLOGE("DICT: dictionary format is unknown, bad magic number");
		delete mmapedBuffer;
		ASSERT(false);
		return 0;
		}