Employ a header attribute map in headerPolicy. (5754817a) · Commits · e / os / android_packages_inputmethods_LatinIME

native/jni/src/suggest/policyimpl/dictionary/header/header_policy.cpp

+76 −10

Original line number	Diff line number	Diff line
		@@ -16,24 +16,90 @@

		#include "suggest/policyimpl/dictionary/header/header_policy.h"

		#include <cstddef>

		namespace latinime {

		const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY =
		"MULTIPLE_WORDS_DEMOTION_RATE";
		const float HeaderPolicy::DEFAULT_MULTI_WORD_COST_MULTIPLIER = 1.0f;
		const float HeaderPolicy::MULTI_WORD_COST_MULTIPLIER_SCALE = 100.0f;
		const char *const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY = "MULTIPLE_WORDS_DEMOTION_RATE";
		const float HeaderPolicy::DEFAULT_MULTIPLE_WORD_COST_MULTIPLIER = 1.0f;
		const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f;

		float HeaderPolicy::readMultiWordCostMultiplier() const {
		const int headerValue = HeaderReadingUtils::readHeaderValueInt(
		mDictBuf, MULTIPLE_WORDS_DEMOTION_RATE_KEY);
		// Used for logging. Question mark is used to indicate that the key is not found.
		void HeaderPolicy::readHeaderValueOrQuestionMark(const char const key, int outValue,
		int outValueSize) const {
		if (outValueSize <= 0) return;
		if (outValueSize == 1) {
		outValue[0] = '\0';
		return;
		}
		std::vector<int> keyCodePointVector;
		insertCharactersIntoVector(key, &keyCodePointVector);
		HeaderReadingUtils::AttributeMap::const_iterator it = mAttributeMap.find(keyCodePointVector);
		if (it == mAttributeMap.end()) {
		// The key was not found.
		outValue[0] = '?';
		outValue[1] = '\0';
		return;
		}
		const int terminalIndex = min(static_cast<int>(it->second.size()), outValueSize - 1);
		for (int i = 0; i < terminalIndex; ++i) {
		outValue[i] = it->second[i];
		}
		outValue[terminalIndex] = '\0';
		}

		float HeaderPolicy::readMultipleWordCostMultiplier() const {
		std::vector<int> multipleWordsDemotionRateKeyVector;
		insertCharactersIntoVector(MULTIPLE_WORDS_DEMOTION_RATE_KEY,
		&multipleWordsDemotionRateKeyVector);
		HeaderReadingUtils::AttributeMap::const_iterator it =
		mAttributeMap.find(multipleWordsDemotionRateKeyVector);
		if (it == mAttributeMap.end()) {
		// The key was not found.
		return DEFAULT_MULTIPLE_WORD_COST_MULTIPLIER;
		}
		const int headerValue = parseIntAttributeValue(&(it->second));
		if (headerValue == S_INT_MIN) {
		// not found
		return DEFAULT_MULTI_WORD_COST_MULTIPLIER;
		// Invalid value
		return DEFAULT_MULTIPLE_WORD_COST_MULTIPLIER;
		}
		if (headerValue <= 0) {
		return static_cast<float>(MAX_VALUE_FOR_WEIGHTING);
		}
		return MULTI_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(headerValue);
		return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(headerValue);
		}

		/* static */ HeaderReadingUtils::AttributeMap HeaderPolicy::createAttributeMapAndReadAllAttributes(
		const uint8_t *const dictBuf) {
		HeaderReadingUtils::AttributeMap attributeMap;
		HeaderReadingUtils::fetchAllHeaderAttributes(dictBuf, &attributeMap);
		return attributeMap;
		}

		/* static */ int HeaderPolicy::parseIntAttributeValue(
		const std::vector<int> *const attributeValue) {
		int value = 0;
		bool isNegative = false;
		for (size_t i = 0; i < attributeValue->size(); ++i) {
		if (i == 0 && attributeValue->at(i) == '-') {
		isNegative = true;
		} else {
		if (!isdigit(attributeValue->at(i))) {
		// If not a number, return S_INT_MIN
		return S_INT_MIN;
		}
		value *= 10;
		value += attributeValue->at(i) - '0';
		}
		}
		return isNegative ? -value : value;
		}

		/* static / void HeaderPolicy::insertCharactersIntoVector(const char const characters,
		std::vector<int> *const vector) {
		for (int i = 0; characters[i]; ++i) {
		vector->push_back(characters[i]);
		}
		}

		} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/header/header_policy.h

+17 −18

Original line number	Diff line number	Diff line
		@@ -17,6 +17,7 @@
		#ifndef LATINIME_HEADER_POLICY_H
		#define LATINIME_HEADER_POLICY_H

		#include <cctype>
		#include <stdint.h>

		#include "defines.h"
		@@ -30,7 +31,8 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
		explicit HeaderPolicy(const uint8_t *const dictBuf)
		: mDictBuf(dictBuf), mDictionaryFlags(HeaderReadingUtils::getFlags(dictBuf)),
		mSize(HeaderReadingUtils::getHeaderSize(dictBuf)),
		mMultiWordCostMultiplier(readMultiWordCostMultiplier()) {}
		mAttributeMap(createAttributeMapAndReadAllAttributes(mDictBuf)),
		mMultiWordCostMultiplier(readMultipleWordCostMultiplier()) {}

		~HeaderPolicy() {}

		@@ -55,34 +57,31 @@ class HeaderPolicy : public DictionaryHeaderStructurePolicy {
		return mMultiWordCostMultiplier;
		}

		AK_FORCE_INLINE void readHeaderValueOrQuestionMark(const char *const key,
		int *outValue, int outValueSize) const {
		if (outValueSize <= 0) return;
		if (outValueSize == 1) {
		outValue[0] = '\0';
		return;
		}
		if (!HeaderReadingUtils::readHeaderValue(mDictBuf,
		key, outValue, outValueSize)) {
		outValue[0] = '?';
		outValue[1] = '\0';
		}
		}
		void readHeaderValueOrQuestionMark(const char *const key,
		int *outValue, int outValueSize) const;

		private:
		DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderPolicy);

		static const char *const MULTIPLE_WORDS_DEMOTION_RATE_KEY;
		static const float DEFAULT_MULTI_WORD_COST_MULTIPLIER;
		static const float MULTI_WORD_COST_MULTIPLIER_SCALE;
		static const float DEFAULT_MULTIPLE_WORD_COST_MULTIPLIER;
		static const float MULTIPLE_WORD_COST_MULTIPLIER_SCALE;

		const uint8_t *const mDictBuf;
		const HeaderReadingUtils::DictionaryFlags mDictionaryFlags;
		const int mSize;
		HeaderReadingUtils::AttributeMap mAttributeMap;
		const float mMultiWordCostMultiplier;

		float readMultiWordCostMultiplier() const;
		};
		float readMultipleWordCostMultiplier() const;

		static HeaderReadingUtils::AttributeMap createAttributeMapAndReadAllAttributes(
		const uint8_t *const dictBuf);

		static int parseIntAttributeValue(const std::vector<int> *const attributeValue);

		static void insertCharactersIntoVector(
		const char const characters, std::vector<int> const vector);
		};
		} // namespace latinime
		#endif /* LATINIME_HEADER_POLICY_H */

native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.cpp

+18 −45

Original line number	Diff line number	Diff line
		@@ -16,23 +16,22 @@

		#include "suggest/policyimpl/dictionary/header/header_reading_utils.h"

		#include <cctype>
		#include <cstdlib>
		#include <vector>

		#include "defines.h"
		#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h"

		namespace latinime {

		const int HeaderReadingUtils::MAX_OPTION_KEY_LENGTH = 256;
		const int HeaderReadingUtils::MAX_ATTRIBUTE_KEY_LENGTH = 256;
		const int HeaderReadingUtils::MAX_ATTRIBUTE_VALUE_LENGTH = 256;

		const int HeaderReadingUtils::HEADER_MAGIC_NUMBER_SIZE = 4;
		const int HeaderReadingUtils::HEADER_DICTIONARY_VERSION_SIZE = 2;
		const int HeaderReadingUtils::HEADER_FLAG_SIZE = 2;
		const int HeaderReadingUtils::HEADER_SIZE_FIELD_SIZE = 4;

		const HeaderReadingUtils::DictionaryFlags
		HeaderReadingUtils::NO_FLAGS = 0;
		const HeaderReadingUtils::DictionaryFlags HeaderReadingUtils::NO_FLAGS = 0;
		// Flags for special processing
		// Those must match the flags in makedict (FormatSpec#*_PROCESSING_FLAG) or
		// something very bad (like, the apocalypse) will happen. Please update both at the same time.
		@@ -56,53 +55,27 @@ const HeaderReadingUtils::DictionaryFlags
		HEADER_MAGIC_NUMBER_SIZE + HEADER_DICTIONARY_VERSION_SIZE);
		}

		// Returns if the key is found or not and reads the found value into outValue.
		/* static / bool HeaderReadingUtils::readHeaderValue(const uint8_t const dictBuf,
		const char const key, int outValue, const int outValueSize) {
		if (outValueSize <= 0) {
		return false;
		}
		/* static / void HeaderReadingUtils::fetchAllHeaderAttributes(const uint8_t const dictBuf,
		AttributeMap *const headerAttributes) {
		const int headerSize = getHeaderSize(dictBuf);
		int pos = getHeaderOptionsPosition();
		if (pos == NOT_A_DICT_POS) {
		// The header doesn't have header options.
		return false;
		return;
		}
		int keyBuffer[MAX_ATTRIBUTE_KEY_LENGTH];
		int valueBuffer[MAX_ATTRIBUTE_VALUE_LENGTH];
		while (pos < headerSize) {
		if(ByteArrayUtils::compareStringInBufferWithCharArray(
		dictBuf, key, headerSize - pos, &pos) == 0) {
		// The key was found.
		const int length = ByteArrayUtils::readStringAndAdvancePosition(dictBuf, outValueSize,
		outValue, &pos);
		// Add a 0 terminator to the string.
		outValue[length < outValueSize ? length : outValueSize - 1] = '\0';
		return true;
		}
		ByteArrayUtils::advancePositionToBehindString(dictBuf, headerSize - pos, &pos);
		}
		// The key was not found.
		return false;
		}

		/* static */ int HeaderReadingUtils::readHeaderValueInt(
		const uint8_t const dictBuf, const char const key) {
		const int bufferSize = LARGEST_INT_DIGIT_COUNT;
		int intBuffer[bufferSize];
		char charBuffer[bufferSize];
		if (!readHeaderValue(dictBuf, key, intBuffer, bufferSize)) {
		return S_INT_MIN;
		}
		for (int i = 0; i < bufferSize; ++i) {
		charBuffer[i] = intBuffer[i];
		if (charBuffer[i] == '0') {
		break;
		}
		if (!isdigit(charBuffer[i])) {
		// If not a number, return S_INT_MIN
		return S_INT_MIN;
		}
		const int keyLength = ByteArrayUtils::readStringAndAdvancePosition(dictBuf,
		MAX_ATTRIBUTE_KEY_LENGTH, keyBuffer, &pos);
		std::vector<int> key;
		key.insert(key.end(), keyBuffer, keyBuffer + keyLength);
		const int valueLength = ByteArrayUtils::readStringAndAdvancePosition(dictBuf,
		MAX_ATTRIBUTE_VALUE_LENGTH, valueBuffer, &pos);
		std::vector<int> value;
		value.insert(value.end(), valueBuffer, valueBuffer + valueLength);
		headerAttributes->insert(AttributeMap::value_type(key, value));
		}
		return atoi(charBuffer);
		}

		} // namespace latinime

native/jni/src/suggest/policyimpl/dictionary/header/header_reading_utils.h

+8 −6

Original line number	Diff line number	Diff line
		@@ -17,7 +17,9 @@
		#ifndef LATINIME_HEADER_READING_UTILS_H
		#define LATINIME_HEADER_READING_UTILS_H

		#include <map>
		#include <stdint.h>
		#include <vector>

		#include "defines.h"

		@@ -26,8 +28,7 @@ namespace latinime {
		class HeaderReadingUtils {
		public:
		typedef uint16_t DictionaryFlags;

		static const int MAX_OPTION_KEY_LENGTH;
		typedef std::map<std::vector<int>, std::vector<int> > AttributeMap;

		static int getHeaderSize(const uint8_t *const dictBuf);

		@@ -50,14 +51,15 @@ class HeaderReadingUtils {
		+ HEADER_SIZE_FIELD_SIZE;
		}

		static bool readHeaderValue(const uint8_t *const dictBuf,
		const char const key, int outValue, const int outValueSize);

		static int readHeaderValueInt(const uint8_t const dictBuf, const char const key);
		static void fetchAllHeaderAttributes(const uint8_t *const dictBuf,
		AttributeMap *const headerAttributes);

		private:
		DISALLOW_IMPLICIT_CONSTRUCTORS(HeaderReadingUtils);

		static const int MAX_ATTRIBUTE_KEY_LENGTH;
		static const int MAX_ATTRIBUTE_VALUE_LENGTH;

		static const int HEADER_MAGIC_NUMBER_SIZE;
		static const int HEADER_DICTIONARY_VERSION_SIZE;
		static const int HEADER_FLAG_SIZE;

native/jni/src/suggest/policyimpl/dictionary/utils/byte_array_utils.h

+0 −33

Original line number	Diff line number	Diff line
		@@ -176,39 +176,6 @@ class ByteArrayUtils {
		return length;
		}

		// Returns an integer less than, equal to, or greater than zero when string starting from pos
		// in buffer is less than, match, or is greater than charArray.
		static AK_FORCE_INLINE int compareStringInBufferWithCharArray(const uint8_t *const buffer,
		const char const charArray, const int maxLength, int const pos) {
		int index = 0;
		int codePoint = readCodePointAndAdvancePosition(buffer, pos);
		const uint8_t *const uint8CharArrayForComparison =
		reinterpret_cast<const uint8_t *>(charArray);
		while (NOT_A_CODE_POINT != codePoint
		&& '\0' != uint8CharArrayForComparison[index] && index < maxLength) {
		if (codePoint != uint8CharArrayForComparison[index]) {
		// Different character is found.
		// Skip the rest of the string in the buffer.
		advancePositionToBehindString(buffer, maxLength - index, pos);
		return codePoint - uint8CharArrayForComparison[index];
		}
		// Advance
		codePoint = readCodePointAndAdvancePosition(buffer, pos);
		++index;
		}
		if (NOT_A_CODE_POINT != codePoint && index < maxLength) {
		// Skip the rest of the string in the buffer.
		advancePositionToBehindString(buffer, maxLength - index, pos);
		}
		if (NOT_A_CODE_POINT == codePoint && '\0' == uint8CharArrayForComparison[index]) {
		// When both of the last characters are terminals, we consider the string in the buffer
		// matches the given char array
		return 0;
		} else {
		return codePoint - uint8CharArrayForComparison[index];
		}
		}

		private:
		DISALLOW_IMPLICIT_CONSTRUCTORS(ByteArrayUtils);