Prepare dictionary file creating in native code. (1592eb8a) · Commits · e / os / android_packages_inputmethods_LatinIME

native/jni/Android.mk

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -85,6 +85,7 @@ LATIN_IME_CORE_SRC_FILES := \
	$(addprefix suggest/policyimpl/dictionary/utils/, \		$(addprefix suggest/policyimpl/dictionary/utils/, \
	buffer_with_extendable_buffer.cpp \		buffer_with_extendable_buffer.cpp \
	byte_array_utils.cpp \		byte_array_utils.cpp \
			dict_file_writing_utils.cpp \
	format_utils.cpp) \		format_utils.cpp) \
	suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \		suggest/policyimpl/gesture/gesture_suggest_policy_factory.cpp \
	$(addprefix suggest/policyimpl/typing/, \		$(addprefix suggest/policyimpl/typing/, \

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp

+3 −61

Original line number	Original line	Diff line number	Diff line
	@@ -16,9 +16,6 @@

	#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"		#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h"

	#include <cstdio>
	#include <cstring>

	#include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h"		#include "suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h"
	#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h"		#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_gc_event_listeners.h"
	#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"		#include "suggest/policyimpl/dictionary/dynamic_patricia_trie_node_reader.h"
	@@ -28,13 +25,12 @@
	#include "suggest/policyimpl/dictionary/header/header_policy.h"		#include "suggest/policyimpl/dictionary/header/header_policy.h"
	#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"		#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h"
	#include "suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h"		#include "suggest/policyimpl/dictionary/shortcut/dynamic_shortcut_list_policy.h"
			#include "suggest/policyimpl/dictionary/utils/dict_file_writing_utils.h"
	#include "utils/hash_map_compat.h"		#include "utils/hash_map_compat.h"

	namespace latinime {		namespace latinime {

	const int DynamicPatriciaTrieWritingHelper::CHILDREN_POSITION_FIELD_SIZE = 3;		const int DynamicPatriciaTrieWritingHelper::CHILDREN_POSITION_FIELD_SIZE = 3;
	const char *const DynamicPatriciaTrieWritingHelper::TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE =
	".tmp";
	// TODO: Make MAX_DICTIONARY_SIZE 8MB.		// TODO: Make MAX_DICTIONARY_SIZE 8MB.
	const size_t DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024;		const size_t DynamicPatriciaTrieWritingHelper::MAX_DICTIONARY_SIZE = 2 * 1024 * 1024;

	@@ -147,7 +143,7 @@ void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileNam
	if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) {		if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) {
	return;		return;
	}		}
	flushAllToFile(fileName, &headerBuffer, mBuffer);		DictFileWritingUtils::flushAllHeaderAndBodyToFile(fileName, &headerBuffer, mBuffer);
	}		}

	void DynamicPatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,		void DynamicPatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos,
	@@ -161,7 +157,7 @@ void DynamicPatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNod
	if (!runGC(rootPtNodeArrayPos, &newDictBuffer)) {		if (!runGC(rootPtNodeArrayPos, &newDictBuffer)) {
	return;		return;
	}		}
	flushAllToFile(fileName, &headerBuffer, &newDictBuffer);		DictFileWritingUtils::flushAllHeaderAndBodyToFile(fileName, &headerBuffer, &newDictBuffer);
	}		}

	bool DynamicPatriciaTrieWritingHelper::markNodeAsDeleted(		bool DynamicPatriciaTrieWritingHelper::markNodeAsDeleted(
	@@ -463,60 +459,6 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes(
	return true;		return true;
	}		}

	// TODO: Create a struct which contains header, body and etc... and use here as an argument.
	void DynamicPatriciaTrieWritingHelper::flushAllToFile(const char *const fileName,
	BufferWithExtendableBuffer *const dictHeader,
	BufferWithExtendableBuffer *const dictBody) const {
	const int tmpFileNameBufSize = strlen(fileName)
	+ strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1 /* terminator */;
	// Name of a temporary file used for writing that is a connected string of original name and
	// TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE.
	char tmpFileName[tmpFileNameBufSize];
	snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", fileName,
	TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE);
	FILE *const file = fopen(tmpFileName, "wb");
	if (!file) {
	AKLOGI("Dictionary file %s cannnot be opened.", tmpFileName);
	ASSERT(false);
	return;
	}
	// Write the dictionary header.
	if (!writeBufferToFilePointer(file, dictHeader)) {
	remove(tmpFileName);
	AKLOGI("Dictionary header cannnot be written. size: %d", dictHeader->getTailPosition());
	ASSERT(false);
	return;
	}
	// Write the dictionary body.
	if (!writeBufferToFilePointer(file, dictBody)) {
	remove(tmpFileName);
	AKLOGI("Dictionary body cannnot be written. size: %d", dictBody->getTailPosition());
	ASSERT(false);
	return;
	}
	fclose(file);
	rename(tmpFileName, fileName);
	}

	// This closes file pointer when an error is caused and returns whether the writing was succeeded
	// or not.
	bool DynamicPatriciaTrieWritingHelper::writeBufferToFilePointer(FILE *const file,
	const BufferWithExtendableBuffer *const buffer) const {
	const int originalBufSize = buffer->getOriginalBufferSize();
	if (originalBufSize > 0 && fwrite(buffer->getBuffer(false /* usesAdditionalBuffer */),
	originalBufSize, 1, file) < 1) {
	fclose(file);
	return false;
	}
	const int additionalBufSize = buffer->getTailPosition() - buffer->getOriginalBufferSize();
	if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */),
	additionalBufSize, 1, file) < 1) {
	fclose(file);
	return false;
	}
	return true;
	}

	bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,		bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos,
	BufferWithExtendableBuffer *const bufferToWrite) {		BufferWithExtendableBuffer *const bufferToWrite) {
	DynamicPatriciaTrieReadingHelper readingHelper(mBuffer, mBigramPolicy, mShortcutPolicy);		DynamicPatriciaTrieReadingHelper readingHelper(mBuffer, mBigramPolicy, mShortcutPolicy);

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h

+0 −9

Original line number	Original line	Diff line number	Diff line
	@@ -17,7 +17,6 @@
	#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H		#ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H
	#define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H		#define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H

	#include <cstdio>
	#include <stdint.h>		#include <stdint.h>

	#include "defines.h"		#include "defines.h"
	@@ -85,7 +84,6 @@ class DynamicPatriciaTrieWritingHelper {
	DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper);		DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper);

	static const int CHILDREN_POSITION_FIELD_SIZE;		static const int CHILDREN_POSITION_FIELD_SIZE;
	static const char *const TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE;
	static const size_t MAX_DICTIONARY_SIZE;		static const size_t MAX_DICTIONARY_SIZE;

	BufferWithExtendableBuffer *const mBuffer;		BufferWithExtendableBuffer *const mBuffer;
	@@ -124,13 +122,6 @@ class DynamicPatriciaTrieWritingHelper {
	const int probabilityOfNewPtNode, const int *const newNodeCodePoints,		const int probabilityOfNewPtNode, const int *const newNodeCodePoints,
	const int newNodeCodePointCount);		const int newNodeCodePointCount);

	void flushAllToFile(const char *const fileName,
	BufferWithExtendableBuffer *const dictHeader,
	BufferWithExtendableBuffer *const dictBody) const;

	bool writeBufferToFilePointer(FILE *const file,
	const BufferWithExtendableBuffer *const buffer) const;

	bool runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite);		bool runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite);
	};		};
	} // namespace latinime		} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.cpp

+10 −0

Original line number	Original line	Diff line number	Diff line
	@@ -36,6 +36,16 @@ const int DynamicPatriciaTrieWritingUtils::DICT_OFFSET_NEGATIVE_FLAG = 0x800000;
	const int DynamicPatriciaTrieWritingUtils::PROBABILITY_FIELD_SIZE = 1;		const int DynamicPatriciaTrieWritingUtils::PROBABILITY_FIELD_SIZE = 1;
	const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1;		const int DynamicPatriciaTrieWritingUtils::NODE_FLAG_FIELD_SIZE = 1;

			/* static */ bool DynamicPatriciaTrieWritingUtils::writeEmptyDictionary(
			BufferWithExtendableBuffer *const buffer, const int rootPos) {
			int writingPos = rootPos;
			if (!writePtNodeArraySizeAndAdvancePosition(buffer, 0 /* arraySize */, &writingPos)) {
			return false;
			}
			return writeForwardLinkPositionAndAdvancePosition(buffer, NOT_A_DICT_POS /* forwardLinkPos */,
			&writingPos);
			}

	/* static */ bool DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(		/* static */ bool DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(
	BufferWithExtendableBuffer *const buffer, const int forwardLinkPos,		BufferWithExtendableBuffer *const buffer, const int forwardLinkPos,
	int *const forwardLinkFieldPos) {		int *const forwardLinkFieldPos) {

native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_utils.h

+2 −0

Original line number	Original line	Diff line number	Diff line
	@@ -30,6 +30,8 @@ class DynamicPatriciaTrieWritingUtils {
	public:		public:
	static const int NODE_FLAG_FIELD_SIZE;		static const int NODE_FLAG_FIELD_SIZE;

			static bool writeEmptyDictionary(BufferWithExtendableBuffer *const buffer, const int rootPos);

	static bool writeForwardLinkPositionAndAdvancePosition(		static bool writeForwardLinkPositionAndAdvancePosition(
	BufferWithExtendableBuffer *const buffer, const int forwardLinkPos,		BufferWithExtendableBuffer *const buffer, const int forwardLinkPos,
	int *const forwardLinkFieldPos);		int *const forwardLinkFieldPos);