From a1d3bc5e2d6c0ab9f1ae3320d481f623937d305a Mon Sep 17 00:00:00 2001 From: mansiverma897993 Date: Fri, 14 Aug 2026 10:17:21 +0530 Subject: [PATCH 1/2] fix(server): enforce completion stop sequences host-side for qairt The qairt plugin rejects any generate call carrying stop sequences (PARAM_NOT_SUPPORTED, -100016), and FIM autocompletion clients always send a stop list, so /v1/completions failed on every QAIRT model. For non-llama_cpp runtimes the stop list is no longer forwarded to the plugin. A stopScanner matches the stop strings against the streamed tokens instead, holding back the longest tail that could still start a match so a stop spanning token boundaries never leaks, cancels generation through the token callback on a match, and truncates the returned text at the match. llama_cpp keeps its native stop handling. Fixes #1341 Signed-off-by: mansiverma897993 --- cli/server/handler/completion.go | 95 +++++++++++++++++++++++++-- cli/server/handler/completion_test.go | 40 +++++++++++ 2 files changed, 131 insertions(+), 4 deletions(-) diff --git a/cli/server/handler/completion.go b/cli/server/handler/completion.go index f3a03e3ad..808112432 100644 --- a/cli/server/handler/completion.go +++ b/cli/server/handler/completion.go @@ -8,6 +8,7 @@ import ( "io" "log/slog" "net/http" + "strings" "sync" "sync/atomic" @@ -80,6 +81,60 @@ func completionStop(u openai.CompletionNewParamsStopUnion) []string { return u.OfStringArray } +// stopScanner enforces stop sequences host-side for plugins that reject them +// (qairt): it scans the streamed tokens for a match, holding back the longest +// tail that could still start one so a match spanning tokens never leaks out. +type stopScanner struct { + stops []string + hold int + pending string + matched bool +} + +func newStopScanner(stops []string) *stopScanner { + s := &stopScanner{} + for _, stop := range stops { + if stop == "" { + continue + } + s.stops = append(s.stops, stop) + if len(stop)-1 > s.hold { + s.hold = len(stop) - 1 + } + } + return s +} + +// feed returns the text that is safe to emit. Once matched is true the text +// up to the match has been returned and generation should be cancelled. +func (s *stopScanner) feed(token string) (emit string, matched bool) { + s.pending += token + cut := -1 + for _, stop := range s.stops { + if i := strings.Index(s.pending, stop); i >= 0 && (cut < 0 || i < cut) { + cut = i + } + } + if cut >= 0 { + s.matched = true + return s.pending[:cut], true + } + if safe := len(s.pending) - s.hold; safe > 0 { + emit, s.pending = s.pending[:safe], s.pending[safe:] + } + return emit, false +} + +// flush returns any held-back text once generation ends without a match. +func (s *stopScanner) flush() string { + if s.matched { + return "" + } + pending := s.pending + s.pending = "" + return pending +} + func completionUnsupported(p CompletionNewParams) error { switch { case p.Suffix.Valid() && p.Suffix.Value != "": @@ -174,6 +229,13 @@ func Completions(c *gin.Context) { Seed: int32(req.Seed.Value), }, } + // The qairt plugin rejects stop sequences (PARAM_NOT_SUPPORTED), so for + // non-llama_cpp runtimes enforce them host-side instead of forwarding them. + var scanner *stopScanner + if len(genConfig.Stop) > 0 && paths.RuntimeID != geniex_sdk.RuntimeLlamaCpp { + scanner = newStopScanner(genConfig.Stop) + genConfig.Stop = nil + } echo := "" if req.Echo.Valid() && req.Echo.Value { echo = prompt @@ -200,8 +262,15 @@ func Completions(c *gin.Context) { if stopGen.Load() { return false } - dataCh <- token - return true + if scanner == nil { + dataCh <- token + return true + } + emit, matched := scanner.feed(token) + if emit != "" { + dataCh <- emit + } + return !matched }, Config: genConfig, }) @@ -209,6 +278,11 @@ func Completions(c *gin.Context) { if out != nil { profile = out.ProfileData } + if scanner != nil && genErr == nil { + if tail := scanner.flush(); tail != "" { + dataCh <- tail + } + } close(dataCh) }() @@ -220,10 +294,23 @@ func Completions(c *gin.Context) { } } else { // blocking - out, err := p.Generate(geniex_sdk.LlmGenerateInput{ + input := geniex_sdk.LlmGenerateInput{ PromptUTF8: prompt, Config: genConfig, - }) + } + var text strings.Builder + if scanner != nil { + input.OnToken = func(token string) bool { + emit, matched := scanner.feed(token) + text.WriteString(emit) + return !matched + } + } + out, err := p.Generate(input) + if scanner != nil && out != nil { + text.WriteString(scanner.flush()) + out.FullText = text.String() + } if errors.Is(err, geniex_sdk.ErrLlmTokenizationContextLength) { writeCompletionContextLengthExceeded(c, echo+out.FullText, out.ProfileData) return diff --git a/cli/server/handler/completion_test.go b/cli/server/handler/completion_test.go index e90d97476..a2f3e636c 100644 --- a/cli/server/handler/completion_test.go +++ b/cli/server/handler/completion_test.go @@ -5,6 +5,7 @@ package handler import ( "reflect" + "strings" "testing" "github.com/openai/openai-go/v3" @@ -59,6 +60,45 @@ func TestCompletionStop(t *testing.T) { } } +func TestStopScanner(t *testing.T) { + run := func(s *stopScanner, tokens []string) (string, bool) { + var out strings.Builder + for _, tok := range tokens { + emit, matched := s.feed(tok) + out.WriteString(emit) + if matched { + return out.String(), true + } + } + out.WriteString(s.flush()) + return out.String(), false + } + + tests := []struct { + name string + stops []string + tokens []string + want string + matched bool + }{ + {"no match", []string{"<|endoftext|>"}, []string{"hello", " world"}, "hello world", false}, + {"match inside one token", []string{""}, []string{"a)tail"}, "a)", true}, + {"match spanning tokens", []string{""}, []string{"donerest"}, "done", true}, + {"earliest stop wins", []string{"", ""}, "x", true}, + {"stop at start", []string{"\n\n"}, []string{"\n\n", "more"}, "", true}, + {"empty stops pass through", []string{""}, []string{"abc"}, "abc", false}, + {"held tail flushed at end", []string{"<|endoftext|>"}, []string{"end<|endo"}, "end<|endo", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, matched := run(newStopScanner(tt.stops), tt.tokens) + if got != tt.want || matched != tt.matched { + t.Errorf("feed() = %q, %v, want %q, %v", got, matched, tt.want, tt.matched) + } + }) + } +} + func TestCompletionUnsupported(t *testing.T) { tests := []struct { name string From 91f188d14b9dec6bfeb8334729471351213facb6 Mon Sep 17 00:00:00 2001 From: mansiverma897993 Date: Sat, 15 Aug 2026 23:23:03 +0530 Subject: [PATCH 2/2] feat(qairt): native stop sequences in the qairt plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements stop sequences natively in the geniex-qairt pipeline instead of enforcing them host-side, as requested in review: the pipeline's generateTokens now matches stop strings byte-wise against the streamed output (mirroring llama_cpp), so every consumer gets the same behavior. The qairt plugin adapter now forwards the FFI stop list into GenerationConfig::stop_sequences instead of rejecting it with PARAM_NOT_SUPPORTED (-100016), and maps the pipeline's new "stop_sequence" stop_reason through to profile_data. The temporary host-side stopScanner in the /v1/completions handler is removed — the handler forwards the stop list to every plugin as it already did before, and llama_cpp keeps its native handling unchanged. Bumps third-party/geniex-qairt to e90b66b (feat/code): native stop-sequence support in the LLM pipeline. Fixes #1341 Signed-off-by: mansiverma897993 --- cli/server/handler/completion.go | 95 ++------------------------- cli/server/handler/completion_test.go | 40 ----------- sdk/plugins/qairt/src/llm.cpp | 20 ++++-- third-party/geniex-qairt | 2 +- 4 files changed, 19 insertions(+), 138 deletions(-) diff --git a/cli/server/handler/completion.go b/cli/server/handler/completion.go index 808112432..f3a03e3ad 100644 --- a/cli/server/handler/completion.go +++ b/cli/server/handler/completion.go @@ -8,7 +8,6 @@ import ( "io" "log/slog" "net/http" - "strings" "sync" "sync/atomic" @@ -81,60 +80,6 @@ func completionStop(u openai.CompletionNewParamsStopUnion) []string { return u.OfStringArray } -// stopScanner enforces stop sequences host-side for plugins that reject them -// (qairt): it scans the streamed tokens for a match, holding back the longest -// tail that could still start one so a match spanning tokens never leaks out. -type stopScanner struct { - stops []string - hold int - pending string - matched bool -} - -func newStopScanner(stops []string) *stopScanner { - s := &stopScanner{} - for _, stop := range stops { - if stop == "" { - continue - } - s.stops = append(s.stops, stop) - if len(stop)-1 > s.hold { - s.hold = len(stop) - 1 - } - } - return s -} - -// feed returns the text that is safe to emit. Once matched is true the text -// up to the match has been returned and generation should be cancelled. -func (s *stopScanner) feed(token string) (emit string, matched bool) { - s.pending += token - cut := -1 - for _, stop := range s.stops { - if i := strings.Index(s.pending, stop); i >= 0 && (cut < 0 || i < cut) { - cut = i - } - } - if cut >= 0 { - s.matched = true - return s.pending[:cut], true - } - if safe := len(s.pending) - s.hold; safe > 0 { - emit, s.pending = s.pending[:safe], s.pending[safe:] - } - return emit, false -} - -// flush returns any held-back text once generation ends without a match. -func (s *stopScanner) flush() string { - if s.matched { - return "" - } - pending := s.pending - s.pending = "" - return pending -} - func completionUnsupported(p CompletionNewParams) error { switch { case p.Suffix.Valid() && p.Suffix.Value != "": @@ -229,13 +174,6 @@ func Completions(c *gin.Context) { Seed: int32(req.Seed.Value), }, } - // The qairt plugin rejects stop sequences (PARAM_NOT_SUPPORTED), so for - // non-llama_cpp runtimes enforce them host-side instead of forwarding them. - var scanner *stopScanner - if len(genConfig.Stop) > 0 && paths.RuntimeID != geniex_sdk.RuntimeLlamaCpp { - scanner = newStopScanner(genConfig.Stop) - genConfig.Stop = nil - } echo := "" if req.Echo.Valid() && req.Echo.Value { echo = prompt @@ -262,15 +200,8 @@ func Completions(c *gin.Context) { if stopGen.Load() { return false } - if scanner == nil { - dataCh <- token - return true - } - emit, matched := scanner.feed(token) - if emit != "" { - dataCh <- emit - } - return !matched + dataCh <- token + return true }, Config: genConfig, }) @@ -278,11 +209,6 @@ func Completions(c *gin.Context) { if out != nil { profile = out.ProfileData } - if scanner != nil && genErr == nil { - if tail := scanner.flush(); tail != "" { - dataCh <- tail - } - } close(dataCh) }() @@ -294,23 +220,10 @@ func Completions(c *gin.Context) { } } else { // blocking - input := geniex_sdk.LlmGenerateInput{ + out, err := p.Generate(geniex_sdk.LlmGenerateInput{ PromptUTF8: prompt, Config: genConfig, - } - var text strings.Builder - if scanner != nil { - input.OnToken = func(token string) bool { - emit, matched := scanner.feed(token) - text.WriteString(emit) - return !matched - } - } - out, err := p.Generate(input) - if scanner != nil && out != nil { - text.WriteString(scanner.flush()) - out.FullText = text.String() - } + }) if errors.Is(err, geniex_sdk.ErrLlmTokenizationContextLength) { writeCompletionContextLengthExceeded(c, echo+out.FullText, out.ProfileData) return diff --git a/cli/server/handler/completion_test.go b/cli/server/handler/completion_test.go index a2f3e636c..e90d97476 100644 --- a/cli/server/handler/completion_test.go +++ b/cli/server/handler/completion_test.go @@ -5,7 +5,6 @@ package handler import ( "reflect" - "strings" "testing" "github.com/openai/openai-go/v3" @@ -60,45 +59,6 @@ func TestCompletionStop(t *testing.T) { } } -func TestStopScanner(t *testing.T) { - run := func(s *stopScanner, tokens []string) (string, bool) { - var out strings.Builder - for _, tok := range tokens { - emit, matched := s.feed(tok) - out.WriteString(emit) - if matched { - return out.String(), true - } - } - out.WriteString(s.flush()) - return out.String(), false - } - - tests := []struct { - name string - stops []string - tokens []string - want string - matched bool - }{ - {"no match", []string{"<|endoftext|>"}, []string{"hello", " world"}, "hello world", false}, - {"match inside one token", []string{""}, []string{"a)tail"}, "a)", true}, - {"match spanning tokens", []string{""}, []string{"donerest"}, "done", true}, - {"earliest stop wins", []string{"", ""}, "x", true}, - {"stop at start", []string{"\n\n"}, []string{"\n\n", "more"}, "", true}, - {"empty stops pass through", []string{""}, []string{"abc"}, "abc", false}, - {"held tail flushed at end", []string{"<|endoftext|>"}, []string{"end<|endo"}, "end<|endo", false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, matched := run(newStopScanner(tt.stops), tt.tokens) - if got != tt.want || matched != tt.matched { - t.Errorf("feed() = %q, %v, want %q, %v", got, matched, tt.want, tt.matched) - } - }) - } -} - func TestCompletionUnsupported(t *testing.T) { tests := []struct { name string diff --git a/sdk/plugins/qairt/src/llm.cpp b/sdk/plugins/qairt/src/llm.cpp index aeb150b5b..8c7345e47 100644 --- a/sdk/plugins/qairt/src/llm.cpp +++ b/sdk/plugins/qairt/src/llm.cpp @@ -215,12 +215,6 @@ int32_t QairtLlm::generate(const geniex_LlmGenerateInput* input, geniex_LlmGener bool has_input_ids = input->input_ids != nullptr && input->input_ids_count > 0; - // Reject llama.cpp-only parameters that have no meaning in the QAIRT plugin - if (input->config && input->config->stop && input->config->stop_count > 0) { - GENIEX_LOG_ERROR("--stop / --stop-file (stop sequences) is not supported by the qairt plugin"); - return GENIEX_ERROR_COMMON_PARAM_NOT_SUPPORTED; - } - if (!has_input_ids && !input->prompt_utf8) return GENIEX_ERROR_COMMON_INVALID_INPUT; // Map geniex_GenerationConfig -> geniex::GenerationConfig @@ -229,6 +223,18 @@ int32_t QairtLlm::generate(const geniex_LlmGenerateInput* input, geniex_LlmGener gen_cfg.max_tokens = input->config->max_tokens > 0 ? input->config->max_tokens : 512; qairt::apply_sampler_config(input->config->sampler_config, gen_cfg, bundle_sampler_); + // Stop sequences are handled natively by the pipeline's generateTokens + // (byte-level matching across tokens, mirroring llama_cpp). + gen_cfg.stop_sequences.clear(); + if (input->config->stop && input->config->stop_count > 0) { + gen_cfg.stop_sequences.reserve(static_cast(input->config->stop_count)); + for (int32_t i = 0; i < input->config->stop_count; ++i) { + if (input->config->stop[i]) { + gen_cfg.stop_sequences.emplace_back(input->config->stop[i]); + } + } + } + // Opt-in ring-buffer context eviction. llama_cpp // ignores this field (it always context-shifts). gen_cfg.sliding_window = input->config->sliding_window; @@ -273,6 +279,8 @@ int32_t QairtLlm::generate(const geniex_LlmGenerateInput* input, geniex_LlmGener output->profile_data.stop_reason = "user"; } else if (result.stop_reason == "length") { output->profile_data.stop_reason = "length"; + } else if (result.stop_reason == "stop_sequence") { + output->profile_data.stop_reason = "stop_sequence"; } else if (result.stop_reason == "context_length") { output->profile_data.stop_reason = "length"; GENIEX_LOG_WARN("QAIRT generate: context length exceeded (partial result populated)"); diff --git a/third-party/geniex-qairt b/third-party/geniex-qairt index 7c592f1cc..e90b66b8a 160000 --- a/third-party/geniex-qairt +++ b/third-party/geniex-qairt @@ -1 +1 @@ -Subproject commit 7c592f1cc5e6b3f0f7e1795a1d6ab1f3ccd22a3e +Subproject commit e90b66b8ad3b3e71c2801225b1e394d7aeed55a9