Loading native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp +16 −0 Original line number Diff line number Diff line Loading @@ -38,6 +38,22 @@ const BigramListReadWriteUtils::BigramFlags BigramListReadWriteUtils::MASK_ATTRIBUTE_PROBABILITY = 0x0F; const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; /* static */ BigramListReadWriteUtils::BigramFlags BigramListReadWriteUtils::getFlagsAndForwardPointer(const uint8_t *const bigramsBuf, int *const pos) { return ByteArrayUtils::readUint8AndAdvancePosition(bigramsBuf, pos); } /* static */ void BigramListReadWriteUtils::skipExistingBigrams(const uint8_t *const bigramsBuf, int *const pos) { BigramFlags flags = getFlagsAndForwardPointer(bigramsBuf, pos); while (hasNext(flags)) { *pos += attributeAddressSize(flags); flags = getFlagsAndForwardPointer(bigramsBuf, pos); } *pos += attributeAddressSize(flags); } /* static */ int BigramListReadWriteUtils::getBigramAddressAndForwardPointer( const uint8_t *const bigramsBuf, const BigramFlags flags, int *const pos) { int offset = 0; Loading native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h +2 −14 Original line number Diff line number Diff line Loading @@ -21,7 +21,6 @@ #include <stdint.h> #include "defines.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { Loading @@ -29,10 +28,7 @@ class BigramListReadWriteUtils { public: typedef uint8_t BigramFlags; static AK_FORCE_INLINE BigramFlags getFlagsAndForwardPointer( const uint8_t *const bigramsBuf, int *const pos) { return ByteArrayUtils::readUint8AndAdvancePosition(bigramsBuf, pos); } static BigramFlags getFlagsAndForwardPointer(const uint8_t *const bigramsBuf, int *const pos); static AK_FORCE_INLINE int getProbabilityFromFlags(const BigramFlags flags) { return flags & MASK_ATTRIBUTE_PROBABILITY; Loading @@ -43,15 +39,7 @@ public: } // Bigrams reading methods static AK_FORCE_INLINE void skipExistingBigrams(const uint8_t *const bigramsBuf, int *const pos) { BigramFlags flags = getFlagsAndForwardPointer(bigramsBuf, pos); while (hasNext(flags)) { *pos += attributeAddressSize(flags); flags = getFlagsAndForwardPointer(bigramsBuf, pos); } *pos += attributeAddressSize(flags); } static void skipExistingBigrams(const uint8_t *const bigramsBuf, int *const pos); static int getBigramAddressAndForwardPointer(const uint8_t *const bigramsBuf, const BigramFlags flags, int *const pos); Loading native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp +11 −0 Original line number Diff line number Diff line Loading @@ -28,6 +28,17 @@ const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_NOT_MOVED = 0xC0; const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40; const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80; /* static */ int DptReadingUtils::getForwardLinkPosition(const uint8_t *const buffer, const int pos) { int linkAddressPos = pos; return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos); } /* static */ int DptReadingUtils::getParentPosAndAdvancePosition(const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); } /* static */ int DptReadingUtils::readChildrenPositionAndAdvancePosition( const uint8_t *const buffer, int *const pos) { const int base = *pos; Loading native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h +2 −9 Original line number Diff line number Diff line Loading @@ -20,7 +20,6 @@ #include <stdint.h> #include "defines.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { Loading @@ -28,19 +27,13 @@ class DynamicPatriciaTrieReadingUtils { public: typedef uint8_t NodeFlags; static AK_FORCE_INLINE int getForwardLinkPosition(const uint8_t *const buffer, const int pos) { int linkAddressPos = pos; return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos); } static int getForwardLinkPosition(const uint8_t *const buffer, const int pos); static AK_FORCE_INLINE bool isValidForwardLinkPosition(const int forwardLinkAddress) { return forwardLinkAddress != 0; } static AK_FORCE_INLINE int getParentPosAndAdvancePosition(const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); } static int getParentPosAndAdvancePosition(const uint8_t *const buffer, int *const pos); static int readChildrenPositionAndAdvancePosition(const uint8_t *const buffer, int *const pos); Loading native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp +57 −0 Original line number Diff line number Diff line Loading @@ -42,6 +42,63 @@ const PtReadingUtils::NodeFlags PtReadingUtils::FLAG_IS_NOT_A_WORD = 0x02; // Flag for blacklist const PtReadingUtils::NodeFlags PtReadingUtils::FLAG_IS_BLACKLISTED = 0x01; /* static */ int PtReadingUtils::getPtNodeArraySizeAndAdvancePosition( const uint8_t *const buffer, int *const pos) { const uint8_t firstByte = ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); if (firstByte < 0x80) { return firstByte; } else { return ((firstByte & 0x7F) << 8) ^ ByteArrayUtils::readUint8AndAdvancePosition( buffer, pos); } } /* static */ PtReadingUtils::NodeFlags PtReadingUtils::getFlagsAndAdvancePosition( const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); } /* static */ int PtReadingUtils::getCodePointAndAdvancePosition(const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readCodePointAndAdvancePosition(buffer, pos); } // Returns the number of read characters. /* static */ int PtReadingUtils::getCharsAndAdvancePosition(const uint8_t *const buffer, const NodeFlags flags, const int maxLength, int *const outBuffer, int *const pos) { int length = 0; if (hasMultipleChars(flags)) { length = ByteArrayUtils::readStringAndAdvancePosition(buffer, maxLength, outBuffer, pos); } else { if (maxLength > 0) { outBuffer[0] = getCodePointAndAdvancePosition(buffer, pos); length = 1; } } return length; } // Returns the number of skipped characters. /* static */ int PtReadingUtils::skipCharacters(const uint8_t *const buffer, const NodeFlags flags, const int maxLength, int *const pos) { if (hasMultipleChars(flags)) { return ByteArrayUtils::advancePositionToBehindString(buffer, maxLength, pos); } else { if (maxLength > 0) { getCodePointAndAdvancePosition(buffer, pos); return 1; } else { return 0; } } } /* static */ int PtReadingUtils::readProbabilityAndAdvancePosition(const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); } /* static */ int PtReadingUtils::readChildrenPositionAndAdvancePosition( const uint8_t *const buffer, const NodeFlags flags, int *const pos) { const int base = *pos; Loading Loading
native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.cpp +16 −0 Original line number Diff line number Diff line Loading @@ -38,6 +38,22 @@ const BigramListReadWriteUtils::BigramFlags BigramListReadWriteUtils::MASK_ATTRIBUTE_PROBABILITY = 0x0F; const int BigramListReadWriteUtils::ATTRIBUTE_ADDRESS_SHIFT = 4; /* static */ BigramListReadWriteUtils::BigramFlags BigramListReadWriteUtils::getFlagsAndForwardPointer(const uint8_t *const bigramsBuf, int *const pos) { return ByteArrayUtils::readUint8AndAdvancePosition(bigramsBuf, pos); } /* static */ void BigramListReadWriteUtils::skipExistingBigrams(const uint8_t *const bigramsBuf, int *const pos) { BigramFlags flags = getFlagsAndForwardPointer(bigramsBuf, pos); while (hasNext(flags)) { *pos += attributeAddressSize(flags); flags = getFlagsAndForwardPointer(bigramsBuf, pos); } *pos += attributeAddressSize(flags); } /* static */ int BigramListReadWriteUtils::getBigramAddressAndForwardPointer( const uint8_t *const bigramsBuf, const BigramFlags flags, int *const pos) { int offset = 0; Loading
native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h +2 −14 Original line number Diff line number Diff line Loading @@ -21,7 +21,6 @@ #include <stdint.h> #include "defines.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { Loading @@ -29,10 +28,7 @@ class BigramListReadWriteUtils { public: typedef uint8_t BigramFlags; static AK_FORCE_INLINE BigramFlags getFlagsAndForwardPointer( const uint8_t *const bigramsBuf, int *const pos) { return ByteArrayUtils::readUint8AndAdvancePosition(bigramsBuf, pos); } static BigramFlags getFlagsAndForwardPointer(const uint8_t *const bigramsBuf, int *const pos); static AK_FORCE_INLINE int getProbabilityFromFlags(const BigramFlags flags) { return flags & MASK_ATTRIBUTE_PROBABILITY; Loading @@ -43,15 +39,7 @@ public: } // Bigrams reading methods static AK_FORCE_INLINE void skipExistingBigrams(const uint8_t *const bigramsBuf, int *const pos) { BigramFlags flags = getFlagsAndForwardPointer(bigramsBuf, pos); while (hasNext(flags)) { *pos += attributeAddressSize(flags); flags = getFlagsAndForwardPointer(bigramsBuf, pos); } *pos += attributeAddressSize(flags); } static void skipExistingBigrams(const uint8_t *const bigramsBuf, int *const pos); static int getBigramAddressAndForwardPointer(const uint8_t *const bigramsBuf, const BigramFlags flags, int *const pos); Loading
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.cpp +11 −0 Original line number Diff line number Diff line Loading @@ -28,6 +28,17 @@ const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_NOT_MOVED = 0xC0; const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_MOVED = 0x40; const DptReadingUtils::NodeFlags DptReadingUtils::FLAG_IS_DELETED = 0x80; /* static */ int DptReadingUtils::getForwardLinkPosition(const uint8_t *const buffer, const int pos) { int linkAddressPos = pos; return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos); } /* static */ int DptReadingUtils::getParentPosAndAdvancePosition(const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); } /* static */ int DptReadingUtils::readChildrenPositionAndAdvancePosition( const uint8_t *const buffer, int *const pos) { const int base = *pos; Loading
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_reading_utils.h +2 −9 Original line number Diff line number Diff line Loading @@ -20,7 +20,6 @@ #include <stdint.h> #include "defines.h" #include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" namespace latinime { Loading @@ -28,19 +27,13 @@ class DynamicPatriciaTrieReadingUtils { public: typedef uint8_t NodeFlags; static AK_FORCE_INLINE int getForwardLinkPosition(const uint8_t *const buffer, const int pos) { int linkAddressPos = pos; return ByteArrayUtils::readSint24AndAdvancePosition(buffer, &linkAddressPos); } static int getForwardLinkPosition(const uint8_t *const buffer, const int pos); static AK_FORCE_INLINE bool isValidForwardLinkPosition(const int forwardLinkAddress) { return forwardLinkAddress != 0; } static AK_FORCE_INLINE int getParentPosAndAdvancePosition(const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readSint24AndAdvancePosition(buffer, pos); } static int getParentPosAndAdvancePosition(const uint8_t *const buffer, int *const pos); static int readChildrenPositionAndAdvancePosition(const uint8_t *const buffer, int *const pos); Loading
native/jni/src/suggest/policyimpl/dictionary/patricia_trie_reading_utils.cpp +57 −0 Original line number Diff line number Diff line Loading @@ -42,6 +42,63 @@ const PtReadingUtils::NodeFlags PtReadingUtils::FLAG_IS_NOT_A_WORD = 0x02; // Flag for blacklist const PtReadingUtils::NodeFlags PtReadingUtils::FLAG_IS_BLACKLISTED = 0x01; /* static */ int PtReadingUtils::getPtNodeArraySizeAndAdvancePosition( const uint8_t *const buffer, int *const pos) { const uint8_t firstByte = ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); if (firstByte < 0x80) { return firstByte; } else { return ((firstByte & 0x7F) << 8) ^ ByteArrayUtils::readUint8AndAdvancePosition( buffer, pos); } } /* static */ PtReadingUtils::NodeFlags PtReadingUtils::getFlagsAndAdvancePosition( const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); } /* static */ int PtReadingUtils::getCodePointAndAdvancePosition(const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readCodePointAndAdvancePosition(buffer, pos); } // Returns the number of read characters. /* static */ int PtReadingUtils::getCharsAndAdvancePosition(const uint8_t *const buffer, const NodeFlags flags, const int maxLength, int *const outBuffer, int *const pos) { int length = 0; if (hasMultipleChars(flags)) { length = ByteArrayUtils::readStringAndAdvancePosition(buffer, maxLength, outBuffer, pos); } else { if (maxLength > 0) { outBuffer[0] = getCodePointAndAdvancePosition(buffer, pos); length = 1; } } return length; } // Returns the number of skipped characters. /* static */ int PtReadingUtils::skipCharacters(const uint8_t *const buffer, const NodeFlags flags, const int maxLength, int *const pos) { if (hasMultipleChars(flags)) { return ByteArrayUtils::advancePositionToBehindString(buffer, maxLength, pos); } else { if (maxLength > 0) { getCodePointAndAdvancePosition(buffer, pos); return 1; } else { return 0; } } } /* static */ int PtReadingUtils::readProbabilityAndAdvancePosition(const uint8_t *const buffer, int *const pos) { return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); } /* static */ int PtReadingUtils::readChildrenPositionAndAdvancePosition( const uint8_t *const buffer, const NodeFlags flags, int *const pos) { const int base = *pos; Loading