Loading native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h +7 −0 Original line number Original line Diff line number Diff line Loading @@ -98,6 +98,13 @@ public: flags |= FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; flags |= FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; *outOffsetFieldSize = 1; *outOffsetFieldSize = 1; } } // Currently, all newly written bigram position fields are 3 bytes to simplify dictionary // writing. // TODO: Remove following 2 lines and optimize memory space. flags = (flags & (~MASK_ATTRIBUTE_ADDRESS_TYPE)) | FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; *outOffsetFieldSize = 3; *outBigramFlags = flags; *outBigramFlags = flags; *outOffset = absOffest; *outOffset = absOffest; return true; return true; Loading native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp +4 −4 Original line number Original line Diff line number Diff line Loading @@ -54,8 +54,8 @@ void DynamicBigramListPolicy::skipAllBigrams(int *const pos) const { } } } } bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos, bool DynamicBigramListPolicy::copyAllBigrams(BufferWithExtendableBuffer *const bufferToWrite, int *outBigramsCount) { int *const fromPos, int *const toPos, int *const outBigramsCount) const { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); if (usesAdditionalBuffer) { if (usesAdditionalBuffer) { *fromPos -= mBuffer->getOriginalBufferSize(); *fromPos -= mBuffer->getOriginalBufferSize(); Loading Loading @@ -86,10 +86,10 @@ bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPo continue; continue; } } // Write bigram entry. Target buffer is always the additional buffer. // Write bigram entry. Target buffer is always the additional buffer. if (!mBuffer->writeUintAndAdvancePosition(newBigramFlags, 1 /* size */,toPos)) { if (!bufferToWrite->writeUintAndAdvancePosition(newBigramFlags, 1 /* size */,toPos)) { return false; return false; } } if (!mBuffer->writeUintAndAdvancePosition(newBigramOffset, newBigramOffsetFieldSize, if (!bufferToWrite->writeUintAndAdvancePosition(newBigramOffset, newBigramOffsetFieldSize, toPos)) { toPos)) { return false; return false; } } Loading native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h +5 −4 Original line number Original line Diff line number Diff line Loading @@ -44,10 +44,11 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { void skipAllBigrams(int *const pos) const; void skipAllBigrams(int *const pos) const; // Copy bigrams from the bigram list that starts at fromPos to toPos and advance these // Copy bigrams from the bigram list that starts at fromPos in mBuffer to toPos in // positions after bigram lists. This method skips invalid bigram entries and write the valid // bufferToWrite and advance these positions after bigram lists. This method skips invalid // bigram entry count to outBigramsCount. // bigram entries and write the valid bigram entry count to outBigramsCount. bool copyAllBigrams(int *const fromPos, int *const toPos, int *outBigramsCount); bool copyAllBigrams(BufferWithExtendableBuffer *const bufferToWrite, int *const fromPos, int *const toPos, int *const outBigramsCount) const; bool addNewBigramEntryToBigramList(const int bigramPos, const int probability, int *const pos); bool addNewBigramEntryToBigramList(const int bigramPos, const int probability, int *const pos); Loading native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp +102 −53 Original line number Original line Diff line number Diff line Loading @@ -97,8 +97,8 @@ bool DynamicPatriciaTrieWritingHelper::addBigramWords(const int word0Pos, const return false; return false; } } int writingPos = newNodePos; int writingPos = newNodePos; // Write a new PtNode using original PtNode's info to the tail of the dictionary. // Write a new PtNode using original PtNode's info to the tail of the dictionary in mBuffer. if (!writePtNodeToBufferByCopyingPtNodeInfo(&nodeReader, nodeReader.getParentPos(), if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, &nodeReader, nodeReader.getParentPos(), mMergedNodeCodePoints, nodeReader.getCodePointCount(), nodeReader.getProbability(), mMergedNodeCodePoints, nodeReader.getCodePointCount(), nodeReader.getProbability(), &writingPos)) { &writingPos)) { return false; return false; Loading Loading @@ -143,38 +143,20 @@ void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileNam if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) { if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) { return; return; } } const int tmpFileNameBufSize = strlen(fileName) flushAllToFile(fileName, &headerBuffer, mBuffer); + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1; char tmpFileName[tmpFileNameBufSize]; snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", fileName, TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE); FILE *const file = fopen(tmpFileName, "wb"); if (!file) { return; } } // Write header. if (fwrite(headerBuffer.getBuffer(true /* usesAdditionalBuffer */), void DynamicPatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos, headerBuffer.getTailPosition(), 1, file) < 1) { const char *const fileName, const HeaderPolicy *const headerPolicy) { fclose(file); BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); remove(tmpFileName); if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */)) { return; } // Write data in original buffer. if (fwrite(mBuffer->getBuffer(false /* usesAdditionalBuffer */), mBuffer->getOriginalBufferSize(), 1, file) < 1) { fclose(file); remove(tmpFileName); return; return; } } // Write data in additional buffer. BufferWithExtendableBuffer newDictBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); if (fwrite(mBuffer->getBuffer(true /* usesAdditionalBuffer */), if (!runGC(rootPtNodeArrayPos, &newDictBuffer)) { mBuffer->getTailPosition() - mBuffer->getOriginalBufferSize(), 1, file) < 1) { fclose(file); remove(tmpFileName); return; return; } } fclose(file); flushAllToFile(fileName, &headerBuffer, &newDictBuffer); rename(tmpFileName, fileName); } } bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( Loading Loading @@ -232,7 +214,8 @@ bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( } } // Write new PtNode at writingPos. // Write new PtNode at writingPos. bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer( BufferWithExtendableBuffer *const bufferToWrite, const bool isBlacklisted, const bool isNotAWord, const int parentPos, const int *const codePoints, const bool isNotAWord, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int childrenPos, const int codePointCount, const int probability, const int childrenPos, const int originalBigramListPos, const int originalShortcutListPos, const int originalBigramListPos, const int originalShortcutListPos, Loading @@ -240,38 +223,39 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo const int nodePos = *writingPos; const int nodePos = *writingPos; // Write dummy flags. The Node flags are updated with appropriate flags at the last step of the // Write dummy flags. The Node flags are updated with appropriate flags at the last step of the // PtNode writing. // PtNode writing. if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, 0 /* nodeFlags */, if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(bufferToWrite, writingPos)) { 0 /* nodeFlags */, writingPos)) { return false; return false; } } // Calculate a parent offset and write the offset. // Calculate a parent offset and write the offset. const int parentOffset = (parentPos != NOT_A_DICT_POS) ? parentPos - nodePos : NOT_A_DICT_POS; const int parentOffset = (parentPos != NOT_A_DICT_POS) ? parentPos - nodePos : NOT_A_DICT_POS; if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(mBuffer, if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(bufferToWrite, parentOffset, writingPos)) { parentOffset, writingPos)) { return false; return false; } } // Write code points // Write code points if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(mBuffer, codePoints, if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(bufferToWrite, codePointCount, writingPos)) { codePoints, codePointCount, writingPos)) { return false; return false; } } // Write probability when the probability is a valid probability, which means this node is // Write probability when the probability is a valid probability, which means this node is // terminal. // terminal. if (probability != NOT_A_PROBABILITY) { if (probability != NOT_A_PROBABILITY) { if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(mBuffer, if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(bufferToWrite, probability, writingPos)) { probability, writingPos)) { return false; return false; } } } } // Write children position // Write children position if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBuffer, if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(bufferToWrite, childrenPos, writingPos)) { childrenPos, writingPos)) { return false; return false; } } // Copy shortcut list when the originalShortcutListPos is valid dictionary position. // Copy shortcut list when the originalShortcutListPos is valid dictionary position. if (originalShortcutListPos != NOT_A_DICT_POS) { if (originalShortcutListPos != NOT_A_DICT_POS) { int fromPos = originalShortcutListPos; int fromPos = originalShortcutListPos; if (!mShortcutPolicy->copyAllShortcutsAndReturnIfSucceededOrNot(&fromPos, writingPos)) { if (!mShortcutPolicy->copyAllShortcutsAndReturnIfSucceededOrNot(bufferToWrite, &fromPos, writingPos)) { return false; return false; } } } } Loading @@ -279,7 +263,7 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo int bigramCount = 0; int bigramCount = 0; if (originalBigramListPos != NOT_A_DICT_POS) { if (originalBigramListPos != NOT_A_DICT_POS) { int fromPos = originalBigramListPos; int fromPos = originalBigramListPos; if (!mBigramPolicy->copyAllBigrams(&fromPos, writingPos, &bigramCount)) { if (!mBigramPolicy->copyAllBigrams(bufferToWrite, &fromPos, writingPos, &bigramCount)) { return false; return false; } } } } Loading @@ -291,27 +275,29 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo bigramCount > 0 /* hasBigrams */, codePointCount > 1 /* hasMultipleChars */, bigramCount > 0 /* hasBigrams */, codePointCount > 1 /* hasMultipleChars */, CHILDREN_POSITION_FIELD_SIZE); CHILDREN_POSITION_FIELD_SIZE); int flagsFieldPos = nodePos; int flagsFieldPos = nodePos; if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, nodeFlags, if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(bufferToWrite, nodeFlags, &flagsFieldPos)) { &flagsFieldPos)) { return false; return false; } } return true; return true; } } bool DynamicPatriciaTrieWritingHelper::writePtNodeToBuffer(const int parentPos, bool DynamicPatriciaTrieWritingHelper::writePtNodeToBuffer( BufferWithExtendableBuffer *const bufferToWrite, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int *const codePoints, const int codePointCount, const int probability, int *const writingPos) { int *const writingPos) { return writePtNodeWithFullInfoToBuffer(false /* isBlacklisted */, false /* isNotAWord */, return writePtNodeWithFullInfoToBuffer(bufferToWrite, false /* isBlacklisted */, parentPos, codePoints, codePointCount, probability, false /* isNotAWord */, parentPos, codePoints, codePointCount, probability, NOT_A_DICT_POS /* childrenPos */, NOT_A_DICT_POS /* originalBigramsPos */, NOT_A_DICT_POS /* childrenPos */, NOT_A_DICT_POS /* originalBigramsPos */, NOT_A_DICT_POS /* originalShortcutPos */, writingPos); NOT_A_DICT_POS /* originalShortcutPos */, writingPos); } } bool DynamicPatriciaTrieWritingHelper::writePtNodeToBufferByCopyingPtNodeInfo( bool DynamicPatriciaTrieWritingHelper::writePtNodeToBufferByCopyingPtNodeInfo( BufferWithExtendableBuffer *const bufferToWrite, const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int *const codePoints, const int codePointCount, const int probability, int *const writingPos) { int *const writingPos) { return writePtNodeWithFullInfoToBuffer(originalNode->isBlacklisted(), return writePtNodeWithFullInfoToBuffer(bufferToWrite, originalNode->isBlacklisted(), originalNode->isNotAWord(), parentPos, codePoints, codePointCount, probability, originalNode->isNotAWord(), parentPos, codePoints, codePointCount, probability, originalNode->getChildrenPos(), originalNode->getBigramsPos(), originalNode->getChildrenPos(), originalNode->getBigramsPos(), originalNode->getShortcutPos(), writingPos); originalNode->getShortcutPos(), writingPos); Loading Loading @@ -345,8 +331,9 @@ bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos, movedPos)) { if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos, movedPos)) { return false; return false; } } if (!writePtNodeToBufferByCopyingPtNodeInfo(originalPtNode, originalPtNode->getParentPos(), if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, originalPtNode, codePoints, originalPtNode->getCodePointCount(), probability, &movedPos)) { originalPtNode->getParentPos(), codePoints, originalPtNode->getCodePointCount(), probability, &movedPos)) { return false; return false; } } } } Loading Loading @@ -374,8 +361,8 @@ bool DynamicPatriciaTrieWritingHelper::createNewPtNodeArrayWithAChildPtNode( 1 /* arraySize */, &writingPos)) { 1 /* arraySize */, &writingPos)) { return false; return false; } } if (!writePtNodeToBuffer(parentPtNodePos, nodeCodePoints, nodeCodePointCount, probability, if (!writePtNodeToBuffer(mBuffer, parentPtNodePos, nodeCodePoints, nodeCodePointCount, &writingPos)) { probability, &writingPos)) { return false; return false; } } if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, Loading Loading @@ -404,8 +391,9 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( // Write the 1st part of the reallocating node. The children position will be updated later // Write the 1st part of the reallocating node. The children position will be updated later // with actual children position. // with actual children position. const int newProbability = addsExtraChild ? NOT_A_PROBABILITY : probabilityOfNewPtNode; const int newProbability = addsExtraChild ? NOT_A_PROBABILITY : probabilityOfNewPtNode; if (!writePtNodeToBuffer(reallocatingPtNode->getParentPos(), reallocatingPtNodeCodePoints, if (!writePtNodeToBuffer(mBuffer, reallocatingPtNode->getParentPos(), overlappingCodePointCount, newProbability, &writingPos)) { reallocatingPtNodeCodePoints, overlappingCodePointCount, newProbability, &writingPos)) { return false; return false; } } const int actualChildrenPos = writingPos; const int actualChildrenPos = writingPos; Loading @@ -417,14 +405,15 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( } } // Write the 2nd part of the reallocating node. // Write the 2nd part of the reallocating node. const int secondPartOfReallocatedPtNodePos = writingPos; const int secondPartOfReallocatedPtNodePos = writingPos; if (!writePtNodeToBufferByCopyingPtNodeInfo(reallocatingPtNode, firstPartOfReallocatedPtNodePos, if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, reallocatingPtNode, firstPartOfReallocatedPtNodePos, reallocatingPtNodeCodePoints + overlappingCodePointCount, reallocatingPtNodeCodePoints + overlappingCodePointCount, reallocatingPtNode->getCodePointCount() - overlappingCodePointCount, reallocatingPtNode->getCodePointCount() - overlappingCodePointCount, reallocatingPtNode->getProbability(), &writingPos)) { reallocatingPtNode->getProbability(), &writingPos)) { return false; return false; } } if (addsExtraChild) { if (addsExtraChild) { if (!writePtNodeToBuffer(firstPartOfReallocatedPtNodePos, if (!writePtNodeToBuffer(mBuffer, firstPartOfReallocatedPtNodePos, newNodeCodePoints + overlappingCodePointCount, newNodeCodePoints + overlappingCodePointCount, newNodeCodePointCount - overlappingCodePointCount, probabilityOfNewPtNode, newNodeCodePointCount - overlappingCodePointCount, probabilityOfNewPtNode, &writingPos)) { &writingPos)) { Loading Loading @@ -452,4 +441,64 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( return true; return true; } } // TODO: Create a struct which contains header, body and etc... and use here as an argument. void DynamicPatriciaTrieWritingHelper::flushAllToFile(const char *const fileName, BufferWithExtendableBuffer *const dictHeader, BufferWithExtendableBuffer *const dictBody) const { const int tmpFileNameBufSize = strlen(fileName) + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1 /* terminator */; // Name of a temporary file used for writing that is a connected string of original name and // TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE. char tmpFileName[tmpFileNameBufSize]; snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", fileName, TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE); FILE *const file = fopen(tmpFileName, "wb"); if (!file) { AKLOGI("Dictionary file %s cannnot be opened.", tmpFileName); ASSERT(false); return; } // Write the dictionary header. if (!writeBufferToFilePointer(file, dictHeader)) { remove(tmpFileName); AKLOGI("Dictionary header cannnot be written. size: %d", dictHeader->getTailPosition()); ASSERT(false); return; } // Write the dictionary body. if (!writeBufferToFilePointer(file, dictBody)) { remove(tmpFileName); AKLOGI("Dictionary body cannnot be written. size: %d", dictBody->getTailPosition()); ASSERT(false); return; } fclose(file); rename(tmpFileName, fileName); } // This closes file pointer when an error is caused and returns whether the writing was succeeded // or not. bool DynamicPatriciaTrieWritingHelper::writeBufferToFilePointer(FILE *const file, const BufferWithExtendableBuffer *const buffer) const { const int originalBufSize = buffer->getOriginalBufferSize(); if (originalBufSize > 0 && fwrite(buffer->getBuffer(false /* usesAdditionalBuffer */), originalBufSize, 1, file) < 1) { fclose(file); return false; } const int additionalBufSize = buffer->getTailPosition() - buffer->getOriginalBufferSize(); if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */), additionalBufSize, 1, file) < 1) { fclose(file); return false; } return true; } bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite) { // TODO: Implement. return false; } } // namespace latinime } // namespace latinime native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h +18 −5 Original line number Original line Diff line number Diff line Loading @@ -17,6 +17,7 @@ #ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H #ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H #define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H #define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H #include <cstdio> #include <stdint.h> #include <stdint.h> #include "defines.h" #include "defines.h" Loading Loading @@ -51,7 +52,8 @@ class DynamicPatriciaTrieWritingHelper { void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy); void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy); void writeToDictFileWithGC(const char *const fileName, const HeaderPolicy *const headerPolicy); void writeToDictFileWithGC(const int rootPtNodeArrayPos, const char *const fileName, const HeaderPolicy *const headerPolicy); private: private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper); DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper); Loading @@ -66,15 +68,17 @@ class DynamicPatriciaTrieWritingHelper { bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate, bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate, const int movedPos, const int bigramLinkedNodePos); const int movedPos, const int bigramLinkedNodePos); bool writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, const bool isNotAWord, bool writePtNodeWithFullInfoToBuffer(BufferWithExtendableBuffer *const bufferToWrite, const bool isBlacklisted, const bool isNotAWord, const int parentPos, const int *const codePoints, const int codePointCount, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int childrenPos, const int originalBigramListPos, const int probability, const int childrenPos, const int originalBigramListPos, const int originalShortcutListPos, int *const writingPos); const int originalShortcutListPos, int *const writingPos); bool writePtNodeToBuffer(const int parentPos, const int *const codePoints, bool writePtNodeToBuffer(BufferWithExtendableBuffer *const bufferToWrite, const int codePointCount, const int probability, int *const writingPos); const int parentPos, const int *const codePoints, const int codePointCount, const int probability, int *const writingPos); bool writePtNodeToBufferByCopyingPtNodeInfo( bool writePtNodeToBufferByCopyingPtNodeInfo(BufferWithExtendableBuffer *const bufferToWrite, const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int *const codePoints, const int codePointCount, const int probability, int *const writingPos); int *const writingPos); Loading @@ -97,6 +101,15 @@ class DynamicPatriciaTrieWritingHelper { const int *const reallocatingPtNodeCodePoints, const int overlappingCodePointCount, const int *const reallocatingPtNodeCodePoints, const int overlappingCodePointCount, const int probabilityOfNewPtNode, const int *const newNodeCodePoints, const int probabilityOfNewPtNode, const int *const newNodeCodePoints, const int newNodeCodePointCount); const int newNodeCodePointCount); void flushAllToFile(const char *const fileName, BufferWithExtendableBuffer *const dictHeader, BufferWithExtendableBuffer *const dictBody) const; bool writeBufferToFilePointer(FILE *const file, const BufferWithExtendableBuffer *const buffer) const; bool runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite); }; }; } // namespace latinime } // namespace latinime #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */ #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */ Loading
native/jni/src/suggest/policyimpl/dictionary/bigram/bigram_list_read_write_utils.h +7 −0 Original line number Original line Diff line number Diff line Loading @@ -98,6 +98,13 @@ public: flags |= FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; flags |= FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; *outOffsetFieldSize = 1; *outOffsetFieldSize = 1; } } // Currently, all newly written bigram position fields are 3 bytes to simplify dictionary // writing. // TODO: Remove following 2 lines and optimize memory space. flags = (flags & (~MASK_ATTRIBUTE_ADDRESS_TYPE)) | FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; *outOffsetFieldSize = 3; *outBigramFlags = flags; *outBigramFlags = flags; *outOffset = absOffest; *outOffset = absOffest; return true; return true; Loading
native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.cpp +4 −4 Original line number Original line Diff line number Diff line Loading @@ -54,8 +54,8 @@ void DynamicBigramListPolicy::skipAllBigrams(int *const pos) const { } } } } bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPos, bool DynamicBigramListPolicy::copyAllBigrams(BufferWithExtendableBuffer *const bufferToWrite, int *outBigramsCount) { int *const fromPos, int *const toPos, int *const outBigramsCount) const { const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); const bool usesAdditionalBuffer = mBuffer->isInAdditionalBuffer(*fromPos); if (usesAdditionalBuffer) { if (usesAdditionalBuffer) { *fromPos -= mBuffer->getOriginalBufferSize(); *fromPos -= mBuffer->getOriginalBufferSize(); Loading Loading @@ -86,10 +86,10 @@ bool DynamicBigramListPolicy::copyAllBigrams(int *const fromPos, int *const toPo continue; continue; } } // Write bigram entry. Target buffer is always the additional buffer. // Write bigram entry. Target buffer is always the additional buffer. if (!mBuffer->writeUintAndAdvancePosition(newBigramFlags, 1 /* size */,toPos)) { if (!bufferToWrite->writeUintAndAdvancePosition(newBigramFlags, 1 /* size */,toPos)) { return false; return false; } } if (!mBuffer->writeUintAndAdvancePosition(newBigramOffset, newBigramOffsetFieldSize, if (!bufferToWrite->writeUintAndAdvancePosition(newBigramOffset, newBigramOffsetFieldSize, toPos)) { toPos)) { return false; return false; } } Loading
native/jni/src/suggest/policyimpl/dictionary/bigram/dynamic_bigram_list_policy.h +5 −4 Original line number Original line Diff line number Diff line Loading @@ -44,10 +44,11 @@ class DynamicBigramListPolicy : public DictionaryBigramsStructurePolicy { void skipAllBigrams(int *const pos) const; void skipAllBigrams(int *const pos) const; // Copy bigrams from the bigram list that starts at fromPos to toPos and advance these // Copy bigrams from the bigram list that starts at fromPos in mBuffer to toPos in // positions after bigram lists. This method skips invalid bigram entries and write the valid // bufferToWrite and advance these positions after bigram lists. This method skips invalid // bigram entry count to outBigramsCount. // bigram entries and write the valid bigram entry count to outBigramsCount. bool copyAllBigrams(int *const fromPos, int *const toPos, int *outBigramsCount); bool copyAllBigrams(BufferWithExtendableBuffer *const bufferToWrite, int *const fromPos, int *const toPos, int *const outBigramsCount) const; bool addNewBigramEntryToBigramList(const int bigramPos, const int probability, int *const pos); bool addNewBigramEntryToBigramList(const int bigramPos, const int probability, int *const pos); Loading
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.cpp +102 −53 Original line number Original line Diff line number Diff line Loading @@ -97,8 +97,8 @@ bool DynamicPatriciaTrieWritingHelper::addBigramWords(const int word0Pos, const return false; return false; } } int writingPos = newNodePos; int writingPos = newNodePos; // Write a new PtNode using original PtNode's info to the tail of the dictionary. // Write a new PtNode using original PtNode's info to the tail of the dictionary in mBuffer. if (!writePtNodeToBufferByCopyingPtNodeInfo(&nodeReader, nodeReader.getParentPos(), if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, &nodeReader, nodeReader.getParentPos(), mMergedNodeCodePoints, nodeReader.getCodePointCount(), nodeReader.getProbability(), mMergedNodeCodePoints, nodeReader.getCodePointCount(), nodeReader.getProbability(), &writingPos)) { &writingPos)) { return false; return false; Loading Loading @@ -143,38 +143,20 @@ void DynamicPatriciaTrieWritingHelper::writeToDictFile(const char *const fileNam if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) { if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, false /* updatesLastUpdatedTime */)) { return; return; } } const int tmpFileNameBufSize = strlen(fileName) flushAllToFile(fileName, &headerBuffer, mBuffer); + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1; char tmpFileName[tmpFileNameBufSize]; snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", fileName, TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE); FILE *const file = fopen(tmpFileName, "wb"); if (!file) { return; } } // Write header. if (fwrite(headerBuffer.getBuffer(true /* usesAdditionalBuffer */), void DynamicPatriciaTrieWritingHelper::writeToDictFileWithGC(const int rootPtNodeArrayPos, headerBuffer.getTailPosition(), 1, file) < 1) { const char *const fileName, const HeaderPolicy *const headerPolicy) { fclose(file); BufferWithExtendableBuffer headerBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); remove(tmpFileName); if (!headerPolicy->writeHeaderToBuffer(&headerBuffer, true /* updatesLastUpdatedTime */)) { return; } // Write data in original buffer. if (fwrite(mBuffer->getBuffer(false /* usesAdditionalBuffer */), mBuffer->getOriginalBufferSize(), 1, file) < 1) { fclose(file); remove(tmpFileName); return; return; } } // Write data in additional buffer. BufferWithExtendableBuffer newDictBuffer(0 /* originalBuffer */, 0 /* originalBufferSize */); if (fwrite(mBuffer->getBuffer(true /* usesAdditionalBuffer */), if (!runGC(rootPtNodeArrayPos, &newDictBuffer)) { mBuffer->getTailPosition() - mBuffer->getOriginalBufferSize(), 1, file) < 1) { fclose(file); remove(tmpFileName); return; return; } } fclose(file); flushAllToFile(fileName, &headerBuffer, &newDictBuffer); rename(tmpFileName, fileName); } } bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( Loading Loading @@ -232,7 +214,8 @@ bool DynamicPatriciaTrieWritingHelper::markNodeAsMovedAndSetPosition( } } // Write new PtNode at writingPos. // Write new PtNode at writingPos. bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer( BufferWithExtendableBuffer *const bufferToWrite, const bool isBlacklisted, const bool isNotAWord, const int parentPos, const int *const codePoints, const bool isNotAWord, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int childrenPos, const int codePointCount, const int probability, const int childrenPos, const int originalBigramListPos, const int originalShortcutListPos, const int originalBigramListPos, const int originalShortcutListPos, Loading @@ -240,38 +223,39 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo const int nodePos = *writingPos; const int nodePos = *writingPos; // Write dummy flags. The Node flags are updated with appropriate flags at the last step of the // Write dummy flags. The Node flags are updated with appropriate flags at the last step of the // PtNode writing. // PtNode writing. if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, 0 /* nodeFlags */, if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(bufferToWrite, writingPos)) { 0 /* nodeFlags */, writingPos)) { return false; return false; } } // Calculate a parent offset and write the offset. // Calculate a parent offset and write the offset. const int parentOffset = (parentPos != NOT_A_DICT_POS) ? parentPos - nodePos : NOT_A_DICT_POS; const int parentOffset = (parentPos != NOT_A_DICT_POS) ? parentPos - nodePos : NOT_A_DICT_POS; if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(mBuffer, if (!DynamicPatriciaTrieWritingUtils::writeParentOffsetAndAdvancePosition(bufferToWrite, parentOffset, writingPos)) { parentOffset, writingPos)) { return false; return false; } } // Write code points // Write code points if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(mBuffer, codePoints, if (!DynamicPatriciaTrieWritingUtils::writeCodePointsAndAdvancePosition(bufferToWrite, codePointCount, writingPos)) { codePoints, codePointCount, writingPos)) { return false; return false; } } // Write probability when the probability is a valid probability, which means this node is // Write probability when the probability is a valid probability, which means this node is // terminal. // terminal. if (probability != NOT_A_PROBABILITY) { if (probability != NOT_A_PROBABILITY) { if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(mBuffer, if (!DynamicPatriciaTrieWritingUtils::writeProbabilityAndAdvancePosition(bufferToWrite, probability, writingPos)) { probability, writingPos)) { return false; return false; } } } } // Write children position // Write children position if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(mBuffer, if (!DynamicPatriciaTrieWritingUtils::writeChildrenPositionAndAdvancePosition(bufferToWrite, childrenPos, writingPos)) { childrenPos, writingPos)) { return false; return false; } } // Copy shortcut list when the originalShortcutListPos is valid dictionary position. // Copy shortcut list when the originalShortcutListPos is valid dictionary position. if (originalShortcutListPos != NOT_A_DICT_POS) { if (originalShortcutListPos != NOT_A_DICT_POS) { int fromPos = originalShortcutListPos; int fromPos = originalShortcutListPos; if (!mShortcutPolicy->copyAllShortcutsAndReturnIfSucceededOrNot(&fromPos, writingPos)) { if (!mShortcutPolicy->copyAllShortcutsAndReturnIfSucceededOrNot(bufferToWrite, &fromPos, writingPos)) { return false; return false; } } } } Loading @@ -279,7 +263,7 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo int bigramCount = 0; int bigramCount = 0; if (originalBigramListPos != NOT_A_DICT_POS) { if (originalBigramListPos != NOT_A_DICT_POS) { int fromPos = originalBigramListPos; int fromPos = originalBigramListPos; if (!mBigramPolicy->copyAllBigrams(&fromPos, writingPos, &bigramCount)) { if (!mBigramPolicy->copyAllBigrams(bufferToWrite, &fromPos, writingPos, &bigramCount)) { return false; return false; } } } } Loading @@ -291,27 +275,29 @@ bool DynamicPatriciaTrieWritingHelper::writePtNodeWithFullInfoToBuffer(const boo bigramCount > 0 /* hasBigrams */, codePointCount > 1 /* hasMultipleChars */, bigramCount > 0 /* hasBigrams */, codePointCount > 1 /* hasMultipleChars */, CHILDREN_POSITION_FIELD_SIZE); CHILDREN_POSITION_FIELD_SIZE); int flagsFieldPos = nodePos; int flagsFieldPos = nodePos; if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(mBuffer, nodeFlags, if (!DynamicPatriciaTrieWritingUtils::writeFlagsAndAdvancePosition(bufferToWrite, nodeFlags, &flagsFieldPos)) { &flagsFieldPos)) { return false; return false; } } return true; return true; } } bool DynamicPatriciaTrieWritingHelper::writePtNodeToBuffer(const int parentPos, bool DynamicPatriciaTrieWritingHelper::writePtNodeToBuffer( BufferWithExtendableBuffer *const bufferToWrite, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int *const codePoints, const int codePointCount, const int probability, int *const writingPos) { int *const writingPos) { return writePtNodeWithFullInfoToBuffer(false /* isBlacklisted */, false /* isNotAWord */, return writePtNodeWithFullInfoToBuffer(bufferToWrite, false /* isBlacklisted */, parentPos, codePoints, codePointCount, probability, false /* isNotAWord */, parentPos, codePoints, codePointCount, probability, NOT_A_DICT_POS /* childrenPos */, NOT_A_DICT_POS /* originalBigramsPos */, NOT_A_DICT_POS /* childrenPos */, NOT_A_DICT_POS /* originalBigramsPos */, NOT_A_DICT_POS /* originalShortcutPos */, writingPos); NOT_A_DICT_POS /* originalShortcutPos */, writingPos); } } bool DynamicPatriciaTrieWritingHelper::writePtNodeToBufferByCopyingPtNodeInfo( bool DynamicPatriciaTrieWritingHelper::writePtNodeToBufferByCopyingPtNodeInfo( BufferWithExtendableBuffer *const bufferToWrite, const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int *const codePoints, const int codePointCount, const int probability, int *const writingPos) { int *const writingPos) { return writePtNodeWithFullInfoToBuffer(originalNode->isBlacklisted(), return writePtNodeWithFullInfoToBuffer(bufferToWrite, originalNode->isBlacklisted(), originalNode->isNotAWord(), parentPos, codePoints, codePointCount, probability, originalNode->isNotAWord(), parentPos, codePoints, codePointCount, probability, originalNode->getChildrenPos(), originalNode->getBigramsPos(), originalNode->getChildrenPos(), originalNode->getBigramsPos(), originalNode->getShortcutPos(), writingPos); originalNode->getShortcutPos(), writingPos); Loading Loading @@ -345,8 +331,9 @@ bool DynamicPatriciaTrieWritingHelper::setPtNodeProbability( if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos, movedPos)) { if (!markNodeAsMovedAndSetPosition(originalPtNode, movedPos, movedPos)) { return false; return false; } } if (!writePtNodeToBufferByCopyingPtNodeInfo(originalPtNode, originalPtNode->getParentPos(), if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, originalPtNode, codePoints, originalPtNode->getCodePointCount(), probability, &movedPos)) { originalPtNode->getParentPos(), codePoints, originalPtNode->getCodePointCount(), probability, &movedPos)) { return false; return false; } } } } Loading Loading @@ -374,8 +361,8 @@ bool DynamicPatriciaTrieWritingHelper::createNewPtNodeArrayWithAChildPtNode( 1 /* arraySize */, &writingPos)) { 1 /* arraySize */, &writingPos)) { return false; return false; } } if (!writePtNodeToBuffer(parentPtNodePos, nodeCodePoints, nodeCodePointCount, probability, if (!writePtNodeToBuffer(mBuffer, parentPtNodePos, nodeCodePoints, nodeCodePointCount, &writingPos)) { probability, &writingPos)) { return false; return false; } } if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, if (!DynamicPatriciaTrieWritingUtils::writeForwardLinkPositionAndAdvancePosition(mBuffer, Loading Loading @@ -404,8 +391,9 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( // Write the 1st part of the reallocating node. The children position will be updated later // Write the 1st part of the reallocating node. The children position will be updated later // with actual children position. // with actual children position. const int newProbability = addsExtraChild ? NOT_A_PROBABILITY : probabilityOfNewPtNode; const int newProbability = addsExtraChild ? NOT_A_PROBABILITY : probabilityOfNewPtNode; if (!writePtNodeToBuffer(reallocatingPtNode->getParentPos(), reallocatingPtNodeCodePoints, if (!writePtNodeToBuffer(mBuffer, reallocatingPtNode->getParentPos(), overlappingCodePointCount, newProbability, &writingPos)) { reallocatingPtNodeCodePoints, overlappingCodePointCount, newProbability, &writingPos)) { return false; return false; } } const int actualChildrenPos = writingPos; const int actualChildrenPos = writingPos; Loading @@ -417,14 +405,15 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( } } // Write the 2nd part of the reallocating node. // Write the 2nd part of the reallocating node. const int secondPartOfReallocatedPtNodePos = writingPos; const int secondPartOfReallocatedPtNodePos = writingPos; if (!writePtNodeToBufferByCopyingPtNodeInfo(reallocatingPtNode, firstPartOfReallocatedPtNodePos, if (!writePtNodeToBufferByCopyingPtNodeInfo(mBuffer, reallocatingPtNode, firstPartOfReallocatedPtNodePos, reallocatingPtNodeCodePoints + overlappingCodePointCount, reallocatingPtNodeCodePoints + overlappingCodePointCount, reallocatingPtNode->getCodePointCount() - overlappingCodePointCount, reallocatingPtNode->getCodePointCount() - overlappingCodePointCount, reallocatingPtNode->getProbability(), &writingPos)) { reallocatingPtNode->getProbability(), &writingPos)) { return false; return false; } } if (addsExtraChild) { if (addsExtraChild) { if (!writePtNodeToBuffer(firstPartOfReallocatedPtNodePos, if (!writePtNodeToBuffer(mBuffer, firstPartOfReallocatedPtNodePos, newNodeCodePoints + overlappingCodePointCount, newNodeCodePoints + overlappingCodePointCount, newNodeCodePointCount - overlappingCodePointCount, probabilityOfNewPtNode, newNodeCodePointCount - overlappingCodePointCount, probabilityOfNewPtNode, &writingPos)) { &writingPos)) { Loading Loading @@ -452,4 +441,64 @@ bool DynamicPatriciaTrieWritingHelper::reallocatePtNodeAndAddNewPtNodes( return true; return true; } } // TODO: Create a struct which contains header, body and etc... and use here as an argument. void DynamicPatriciaTrieWritingHelper::flushAllToFile(const char *const fileName, BufferWithExtendableBuffer *const dictHeader, BufferWithExtendableBuffer *const dictBody) const { const int tmpFileNameBufSize = strlen(fileName) + strlen(TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE) + 1 /* terminator */; // Name of a temporary file used for writing that is a connected string of original name and // TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE. char tmpFileName[tmpFileNameBufSize]; snprintf(tmpFileName, tmpFileNameBufSize, "%s%s", fileName, TEMP_FILE_SUFFIX_FOR_WRITING_DICT_FILE); FILE *const file = fopen(tmpFileName, "wb"); if (!file) { AKLOGI("Dictionary file %s cannnot be opened.", tmpFileName); ASSERT(false); return; } // Write the dictionary header. if (!writeBufferToFilePointer(file, dictHeader)) { remove(tmpFileName); AKLOGI("Dictionary header cannnot be written. size: %d", dictHeader->getTailPosition()); ASSERT(false); return; } // Write the dictionary body. if (!writeBufferToFilePointer(file, dictBody)) { remove(tmpFileName); AKLOGI("Dictionary body cannnot be written. size: %d", dictBody->getTailPosition()); ASSERT(false); return; } fclose(file); rename(tmpFileName, fileName); } // This closes file pointer when an error is caused and returns whether the writing was succeeded // or not. bool DynamicPatriciaTrieWritingHelper::writeBufferToFilePointer(FILE *const file, const BufferWithExtendableBuffer *const buffer) const { const int originalBufSize = buffer->getOriginalBufferSize(); if (originalBufSize > 0 && fwrite(buffer->getBuffer(false /* usesAdditionalBuffer */), originalBufSize, 1, file) < 1) { fclose(file); return false; } const int additionalBufSize = buffer->getTailPosition() - buffer->getOriginalBufferSize(); if (additionalBufSize > 0 && fwrite(buffer->getBuffer(true /* usesAdditionalBuffer */), additionalBufSize, 1, file) < 1) { fclose(file); return false; } return true; } bool DynamicPatriciaTrieWritingHelper::runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite) { // TODO: Implement. return false; } } // namespace latinime } // namespace latinime
native/jni/src/suggest/policyimpl/dictionary/dynamic_patricia_trie_writing_helper.h +18 −5 Original line number Original line Diff line number Diff line Loading @@ -17,6 +17,7 @@ #ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H #ifndef LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H #define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H #define LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H #include <cstdio> #include <stdint.h> #include <stdint.h> #include "defines.h" #include "defines.h" Loading Loading @@ -51,7 +52,8 @@ class DynamicPatriciaTrieWritingHelper { void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy); void writeToDictFile(const char *const fileName, const HeaderPolicy *const headerPolicy); void writeToDictFileWithGC(const char *const fileName, const HeaderPolicy *const headerPolicy); void writeToDictFileWithGC(const int rootPtNodeArrayPos, const char *const fileName, const HeaderPolicy *const headerPolicy); private: private: DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper); DISALLOW_IMPLICIT_CONSTRUCTORS(DynamicPatriciaTrieWritingHelper); Loading @@ -66,15 +68,17 @@ class DynamicPatriciaTrieWritingHelper { bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate, bool markNodeAsMovedAndSetPosition(const DynamicPatriciaTrieNodeReader *const nodeToUpdate, const int movedPos, const int bigramLinkedNodePos); const int movedPos, const int bigramLinkedNodePos); bool writePtNodeWithFullInfoToBuffer(const bool isBlacklisted, const bool isNotAWord, bool writePtNodeWithFullInfoToBuffer(BufferWithExtendableBuffer *const bufferToWrite, const bool isBlacklisted, const bool isNotAWord, const int parentPos, const int *const codePoints, const int codePointCount, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int childrenPos, const int originalBigramListPos, const int probability, const int childrenPos, const int originalBigramListPos, const int originalShortcutListPos, int *const writingPos); const int originalShortcutListPos, int *const writingPos); bool writePtNodeToBuffer(const int parentPos, const int *const codePoints, bool writePtNodeToBuffer(BufferWithExtendableBuffer *const bufferToWrite, const int codePointCount, const int probability, int *const writingPos); const int parentPos, const int *const codePoints, const int codePointCount, const int probability, int *const writingPos); bool writePtNodeToBufferByCopyingPtNodeInfo( bool writePtNodeToBufferByCopyingPtNodeInfo(BufferWithExtendableBuffer *const bufferToWrite, const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, const DynamicPatriciaTrieNodeReader *const originalNode, const int parentPos, const int *const codePoints, const int codePointCount, const int probability, const int *const codePoints, const int codePointCount, const int probability, int *const writingPos); int *const writingPos); Loading @@ -97,6 +101,15 @@ class DynamicPatriciaTrieWritingHelper { const int *const reallocatingPtNodeCodePoints, const int overlappingCodePointCount, const int *const reallocatingPtNodeCodePoints, const int overlappingCodePointCount, const int probabilityOfNewPtNode, const int *const newNodeCodePoints, const int probabilityOfNewPtNode, const int *const newNodeCodePoints, const int newNodeCodePointCount); const int newNodeCodePointCount); void flushAllToFile(const char *const fileName, BufferWithExtendableBuffer *const dictHeader, BufferWithExtendableBuffer *const dictBody) const; bool writeBufferToFilePointer(FILE *const file, const BufferWithExtendableBuffer *const buffer) const; bool runGC(const int rootPtNodeArrayPos, BufferWithExtendableBuffer *const bufferToWrite); }; }; } // namespace latinime } // namespace latinime #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */ #endif /* LATINIME_DYNAMIC_PATRICIA_TRIE_WRITING_HELPER_H */