Skip to content

Fix long chunk bug #285

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bab2min merged 3 commits into from
Oct 18, 2026
Merged

Fix long chunk bug #285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
19cf3eb
Add test cases for analyzing long chunk
bab2min Oct 18, 2026
a80c70f
Fix `KGraphNode` constructor to use uint32_t for start and end positions
bab2min Oct 18, 2026
cacb28a
Fix brace style
bab2min Oct 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/KTrie.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,9 @@ namespace kiwi
uint32_t typoFormId = 0;
uint32_t spaceErrors = 0;

KGraphNode(uint16_t _startPos = 0, uint16_t _endPos = 0, const Form* _form = nullptr, float _typoCost = 0)
KGraphNode(uint32_t _startPos = 0, uint32_t _endPos = 0, const Form* _form = nullptr, float _typoCost = 0)
: form(_form), startPos(_startPos), endPos(_endPos), typoCost(_typoCost) {}
KGraphNode(uint16_t _startPos, uint16_t _endPos, U16StringView _uform, float _typoCost = 0)
KGraphNode(uint32_t _startPos, uint32_t _endPos, U16StringView _uform, float _typoCost = 0)
: uform(_uform), startPos(_startPos), endPos(_endPos), typoCost(_typoCost) {}

KGraphNode* getPrev() { return prev ? this - prev : nullptr; }
Expand Down
32 changes: 23 additions & 9 deletions src/KiwiBuilder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -986,23 +986,32 @@ void KiwiBuilder::saveMorphBin(std::ostream& os) const
ModelType KiwiBuilder::getModelType(const KiwiBuilder::StreamProvider& provider, bool largest)
{
// Check for different model files by trying to create streams
try {
if (auto stream = provider("cong.mdl")) {
try
{
if (auto stream = provider("cong.mdl"))
{
return largest ? ModelType::congGlobal : ModelType::cong;
}
} catch (...) {}
}
catch (...) {}

try {
if (auto stream = provider("skipbigram.mdl")) {
try
{
if (auto stream = provider("skipbigram.mdl"))
{
return largest ? ModelType::sbg : ModelType::knlm;
}
} catch (...) {}
}
catch (...) {}

try {
if (auto stream = provider("sj.knlm")) {
try
{
if (auto stream = provider("sj.knlm"))
{
return ModelType::knlm;
}
} catch (...) {}
}
catch (...) {}

return ModelType::none;
}
Expand Down Expand Up @@ -2431,6 +2440,11 @@ namespace kiwi

Kiwi KiwiBuilder::build(const TypoTransformer& typos, float typoCostThreshold) const
{
if (!ready())
{
return Kiwi{};
}

Kiwi ret{ archType, langMdl, !typos.empty(), typos.isContinualTypoEnabled(), typos.isLengtheningTypoEnabled() };
ret.enabledDialects = enabledDialects;
ret.nounChrMdl = nounChrMdl;
Expand Down
18 changes: 18 additions & 0 deletions test/test_cpp.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2183,6 +2183,24 @@ TEST(KiwiCpp, IssueP172_LengthError)
EXPECT_GT(res.size(), 0);
}

TEST(KiwiCpp, LongChunkOver65536)
{
// 공백 없이 라틴 문자와 기호가 이어진 덩어리가 65536자를 넘으면 KGraphNode의 위치가 16비트로 잘려 크래시가 났다.
Kiwi& kiwi = reuseKiwiInstance();
for (auto p : { std::make_pair(127, 512), std::make_pair(100, 700) })
{
std::u16string text;
for (int i = 0; i < p.second; ++i)
{
text += std::u16string(p.first, u'A');
text += u"#";
}
auto res = kiwi.analyze(text, Match::allWithNormalizing).first;
ASSERT_GT(res.size(), 0);
EXPECT_EQ(res.back().position + res.back().length, text.size());
}
}

TEST(KiwiCpp, IssueP189)
{
Kiwi& kiwi = reuseKiwiInstance();
Expand Down
Loading