diff --git a/src/KTrie.h b/src/KTrie.h index a2e5f705..d535d34e 100644 --- a/src/KTrie.h +++ b/src/KTrie.h @@ -64,9 +64,9 @@ namespace kiwi uint32_t typoFormId = 0; uint32_t spaceErrors = 0; - KGraphNode(uint16_t _startPos = 0, uint16_t _endPos = 0, const Form* _form = nullptr, float _typoCost = 0) + KGraphNode(uint32_t _startPos = 0, uint32_t _endPos = 0, const Form* _form = nullptr, float _typoCost = 0) : form(_form), startPos(_startPos), endPos(_endPos), typoCost(_typoCost) {} - KGraphNode(uint16_t _startPos, uint16_t _endPos, U16StringView _uform, float _typoCost = 0) + KGraphNode(uint32_t _startPos, uint32_t _endPos, U16StringView _uform, float _typoCost = 0) : uform(_uform), startPos(_startPos), endPos(_endPos), typoCost(_typoCost) {} KGraphNode* getPrev() { return prev ? this - prev : nullptr; } diff --git a/src/KiwiBuilder.cpp b/src/KiwiBuilder.cpp index b9aca146..b9f69c29 100644 --- a/src/KiwiBuilder.cpp +++ b/src/KiwiBuilder.cpp @@ -986,23 +986,32 @@ void KiwiBuilder::saveMorphBin(std::ostream& os) const ModelType KiwiBuilder::getModelType(const KiwiBuilder::StreamProvider& provider, bool largest) { // Check for different model files by trying to create streams - try { - if (auto stream = provider("cong.mdl")) { + try + { + if (auto stream = provider("cong.mdl")) + { return largest ? ModelType::congGlobal : ModelType::cong; } - } catch (...) {} + } + catch (...) {} - try { - if (auto stream = provider("skipbigram.mdl")) { + try + { + if (auto stream = provider("skipbigram.mdl")) + { return largest ? ModelType::sbg : ModelType::knlm; } - } catch (...) {} + } + catch (...) {} - try { - if (auto stream = provider("sj.knlm")) { + try + { + if (auto stream = provider("sj.knlm")) + { return ModelType::knlm; } - } catch (...) {} + } + catch (...) {} return ModelType::none; } @@ -2431,6 +2440,11 @@ namespace kiwi Kiwi KiwiBuilder::build(const TypoTransformer& typos, float typoCostThreshold) const { + if (!ready()) + { + return Kiwi{}; + } + Kiwi ret{ archType, langMdl, !typos.empty(), typos.isContinualTypoEnabled(), typos.isLengtheningTypoEnabled() }; ret.enabledDialects = enabledDialects; ret.nounChrMdl = nounChrMdl; diff --git a/test/test_cpp.cpp b/test/test_cpp.cpp index d25881cd..c5c91569 100644 --- a/test/test_cpp.cpp +++ b/test/test_cpp.cpp @@ -2183,6 +2183,24 @@ TEST(KiwiCpp, IssueP172_LengthError) EXPECT_GT(res.size(), 0); } +TEST(KiwiCpp, LongChunkOver65536) +{ + // 공백 없이 라틴 문자와 기호가 이어진 덩어리가 65536자를 넘으면 KGraphNode의 위치가 16비트로 잘려 크래시가 났다. + Kiwi& kiwi = reuseKiwiInstance(); + for (auto p : { std::make_pair(127, 512), std::make_pair(100, 700) }) + { + std::u16string text; + for (int i = 0; i < p.second; ++i) + { + text += std::u16string(p.first, u'A'); + text += u"#"; + } + auto res = kiwi.analyze(text, Match::allWithNormalizing).first; + ASSERT_GT(res.size(), 0); + EXPECT_EQ(res.back().position + res.back().length, text.size()); + } +} + TEST(KiwiCpp, IssueP189) { Kiwi& kiwi = reuseKiwiInstance();