diff --git a/packages/inference/package.json b/packages/inference/package.json index 9ebe6ab3c1..11723225e5 100644 --- a/packages/inference/package.json +++ b/packages/inference/package.json @@ -192,16 +192,16 @@ "@qvac/asr-ggml": "^0.3.0", "@qvac/audiogen-ggml": "^0.2.1", "@qvac/bci-whispercpp": "^0.7.1", - "@qvac/classification-ggml": "^0.20.0", + "@qvac/classification-ggml": "^0.22.0", "@qvac/decoder-audio": "^0.5.0", "@qvac/diffusion-cpp": "^0.20.0", - "@qvac/embed-llamacpp": "^0.34.0", + "@qvac/embed-llamacpp": "^0.36.0", "@qvac/langdetect-text": "^0.1.2", - "@qvac/llm-llamacpp": "^0.45.0", - "@qvac/ocr-ggml": "^0.18.0", - "@qvac/translation-nmtcpp": "^0.10.0", + "@qvac/llm-llamacpp": "^0.47.0", + "@qvac/ocr-ggml": "^0.20.0", + "@qvac/translation-nmtcpp": "^0.12.0", "@qvac/tts-ggml": "^0.7.4", - "@qvac/vla-ggml": "^0.21.1" + "@qvac/vla-ggml": "^0.23.0" }, "peerDependenciesMeta": { "@qvac/asr-ggml": { @@ -248,16 +248,16 @@ "@qvac/asr-ggml": "^0.3.0", "@qvac/audiogen-ggml": "^0.2.1", "@qvac/bci-whispercpp": "^0.7.1", - "@qvac/classification-ggml": "^0.20.0", + "@qvac/classification-ggml": "^0.22.0", "@qvac/decoder-audio": "^0.5.0", "@qvac/diffusion-cpp": "^0.20.0", - "@qvac/embed-llamacpp": "^0.34.0", + "@qvac/embed-llamacpp": "^0.36.0", "@qvac/langdetect-text": "^0.1.2", - "@qvac/llm-llamacpp": "^0.45.0", - "@qvac/ocr-ggml": "^0.18.0", - "@qvac/translation-nmtcpp": "^0.10.0", + "@qvac/llm-llamacpp": "^0.47.0", + "@qvac/ocr-ggml": "^0.20.0", + "@qvac/translation-nmtcpp": "^0.12.0", "@qvac/tts-ggml": "^0.7.4", - "@qvac/vla-ggml": "^0.21.1", + "@qvac/vla-ggml": "^0.23.0", "@types/brittle": "^3.5.0", "bare": "*", "bare-console": "*", diff --git a/packages/inference/src/schemas/llamacpp-config.ts b/packages/inference/src/schemas/llamacpp-config.ts index 6462916511..6eb72c8873 100644 --- a/packages/inference/src/schemas/llamacpp-config.ts +++ b/packages/inference/src/schemas/llamacpp-config.ts @@ -66,7 +66,12 @@ export const llmConfigBaseSchema = z.object({ .describe( "Seeds conversation history on the JS side only; never forwarded to the addon. Default `'You are a helpful assistant.'`" ), - no_mmap: z.boolean().optional().describe('Disable memory-mapped model loading. Default false.'), + load_mode: z + .enum(['none', 'mmap', 'mlock', 'mmap+mlock', 'dio']) + .optional() + .describe( + "Model loading mode: `'none'`, `'mmap'`, `'mlock'`, `'mmap+mlock'`, or `'dio'`. Unset uses the addon's default (`'mmap'`)." + ), verbosity: verbositySchema .optional() .describe('Native log verbosity: `0`=ERROR, `1`=WARN, `2`=INFO, `3`=DEBUG. Default 0.'), diff --git a/packages/inference/test/llm-config-schema.test.ts b/packages/inference/test/llm-config-schema.test.ts index 5d56697811..0e29d0d2d0 100644 --- a/packages/inference/test/llm-config-schema.test.ts +++ b/packages/inference/test/llm-config-schema.test.ts @@ -26,6 +26,28 @@ test('llmConfigBaseSchema: split-mode is optional', (t) => { t.is(llmConfigBaseSchema.safeParse({}).success, true) }) +test('llmConfigBaseSchema: accepts every load_mode value', (t) => { + for (const load_mode of ['none', 'mmap', 'mlock', 'mmap+mlock', 'dio'] as const) { + const result = llmConfigBaseSchema.safeParse({ load_mode }) + t.is(result.success, true, `${load_mode} must be accepted`) + if (result.success) t.is(result.data.load_mode, load_mode) + } +}) + +test('llmConfigBaseSchema: rejects invalid load_mode values', (t) => { + t.is(llmConfigBaseSchema.safeParse({ load_mode: 'buffered' }).success, false) +}) + +test('llmConfigBaseSchema: rejects legacy no_mmap under strict validation', (t) => { + t.is(llmConfigBaseSchema.strict().safeParse({ no_mmap: true }).success, false) +}) + +test('llmConfigSchema: leaves load_mode unset by default', (t) => { + const result = llmConfigSchema.safeParse({}) + t.is(result.success, true) + if (result.success) t.is(result.data.load_mode, undefined) +}) + test('llmConfigBaseSchema: accepts continuous-batching parallel slots', (t) => { const result = llmConfigBaseSchema.safeParse({ parallel: 4 }) t.is(result.success, true) diff --git a/packages/inference/test/llm-plugin-transform.test.ts b/packages/inference/test/llm-plugin-transform.test.ts index 66e8dfc8b8..1a22192a42 100644 --- a/packages/inference/test/llm-plugin-transform.test.ts +++ b/packages/inference/test/llm-plugin-transform.test.ts @@ -42,6 +42,13 @@ test('transformLlmConfig: positive reasoning_budget survives as string token cap ) }) +test('transformLlmConfig: load_mode survives as an underscore key', (t) => { + const config = makeConfig({ load_mode: 'mmap+mlock' }) + const result = transformLlmConfig(config) + t.is(result['load_mode'], 'mmap+mlock') + t.absent('load-mode' in result) +}) + test('transformLlmConfig: stop_sequences is renamed to reverse_prompt', (t) => { const config = makeConfig({ stop_sequences: ['', '<|im_end|>'] }) const result = transformLlmConfig(config) diff --git a/packages/sdk-python/examples/notebook.ipynb b/packages/sdk-python/examples/notebook.ipynb index d48981e94a..8674275b9e 100644 --- a/packages/sdk-python/examples/notebook.ipynb +++ b/packages/sdk-python/examples/notebook.ipynb @@ -98,7 +98,7 @@ "metadata": {}, "outputs": [], "source": [ - "llm = client.load_model(model_src=QWEN3_600M_INST_Q4, model_config={\"n_ctx\": 2048})\n", + "llm = client.load_model(model_src=QWEN3_600M_INST_Q4, model_config={\"ctx_size\": 2048})\n", "text = client.completion(\n", " llm,\n", " \"Explain what an embedding is in one sentence.\",\n", diff --git a/packages/sdk-python/examples/notebook.py b/packages/sdk-python/examples/notebook.py index c27236e984..d3dde7f9d2 100644 --- a/packages/sdk-python/examples/notebook.py +++ b/packages/sdk-python/examples/notebook.py @@ -41,7 +41,7 @@ def main() -> int: print("\n▸ Completion, streaming live into the cell/stdout") llm = client.load_model( - model_src=QWEN3_600M_INST_Q4, model_config={"n_ctx": 2048} + model_src=QWEN3_600M_INST_Q4, model_config={"ctx_size": 2048} ) text = client.completion( llm, diff --git a/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/__init__.py b/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/__init__.py index 3db4d5f390..3c2313496d 100644 --- a/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/__init__.py +++ b/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/__init__.py @@ -498,6 +498,7 @@ LoadModelSrcRequestLlamacppCompletionModelConfig, LoadModelSrcRequestLlamacppCompletionModelConfigImageNoUpscale, LoadModelSrcRequestLlamacppCompletionModelConfigImageTileMode, + LoadModelSrcRequestLlamacppCompletionModelConfigLoadMode, LoadModelSrcRequestLlamacppCompletionModelConfigMainGpu, LoadModelSrcRequestLlamacppCompletionModelConfigProjectionModelSrc, LoadModelSrcRequestLlamacppCompletionModelConfigProjectionModelSrcAddon, @@ -1290,6 +1291,7 @@ "LoadModelSrcRequestLlamacppCompletionModelConfig", "LoadModelSrcRequestLlamacppCompletionModelConfigImageNoUpscale", "LoadModelSrcRequestLlamacppCompletionModelConfigImageTileMode", + "LoadModelSrcRequestLlamacppCompletionModelConfigLoadMode", "LoadModelSrcRequestLlamacppCompletionModelConfigMainGpu", "LoadModelSrcRequestLlamacppCompletionModelConfigProjectionModelSrc", "LoadModelSrcRequestLlamacppCompletionModelConfigProjectionModelSrcAddon", diff --git a/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/_internal.py b/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/_internal.py index c531f32692..0027fedbbd 100644 --- a/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/_internal.py +++ b/packages/sdk-python/src/tetherto/qvac_sdk/_generated/models/_internal.py @@ -6712,6 +6712,14 @@ class Predict(RootModel[int]): ] +class LoadModelSrcRequestLlamacppCompletionModelConfigLoadMode(Enum): + none = "none" + mmap = "mmap" + mlock = "mlock" + mmap_mlock = "mmap+mlock" + dio = "dio" + + class LoadModelSrcRequestLlamacppCompletionModelConfigVerbosity(Enum): number_0 = 0 number_1 = 1 @@ -6921,9 +6929,12 @@ class LoadModelSrcRequestLlamacppCompletionModelConfig(GeneratedBaseModel): description="Seeds conversation history on the JS side only; never forwarded to the addon. Default `'You are a helpful assistant.'`" ), ] = None - no_mmap: Annotated[ - bool | None, - Field(description="Disable memory-mapped model loading. Default false."), + load_mode: Annotated[ + LoadModelSrcRequestLlamacppCompletionModelConfigLoadMode | None, + Field( + description="Model loading mode: `'none'`, `'mmap'`, `'mlock'`, `'mmap+mlock'`, or `'dio'`. Unset uses the addon's default (`'mmap'`).", + title="LoadModelSrcRequestLlamacppCompletionModelConfigLoadMode", + ), ] = None verbosity: Annotated[ LoadModelSrcRequestLlamacppCompletionModelConfigVerbosity | None, diff --git a/packages/sdk-python/src/tetherto/qvac_sdk/errors.py b/packages/sdk-python/src/tetherto/qvac_sdk/errors.py index 47560f3354..42cef67cdc 100644 --- a/packages/sdk-python/src/tetherto/qvac_sdk/errors.py +++ b/packages/sdk-python/src/tetherto/qvac_sdk/errors.py @@ -169,7 +169,7 @@ def __init__( class ContextOverflowError(QvacError): """Prompt exceeded the loaded model's context window. Distinct from a - generic failure so callers can drive UX (truncate, raise `n_ctx`, start a + generic failure so callers can drive UX (truncate, raise `ctx_size`, start a new thread). The token/ctx fields are present only when the worker's error message carried them.""" diff --git a/packages/sdk-python/tests/test_bare_rpc_transport.py b/packages/sdk-python/tests/test_bare_rpc_transport.py index d351d6002a..727d00c9c8 100644 --- a/packages/sdk-python/tests/test_bare_rpc_transport.py +++ b/packages/sdk-python/tests/test_bare_rpc_transport.py @@ -80,7 +80,7 @@ async def test_load_model_and_completion_stream(transport) -> None: # Qwen3 is a thinking model: the worker reserves context for the # reasoning trace, so the metadata-default budget overflows even a # tiny prompt. Give it an explicit window (matches the SDK e2e). - "modelConfig": {"n_ctx": 2048}, + "modelConfig": {"ctx_size": 2048}, } ) load_response = await load_model(transport, load_request) @@ -94,7 +94,7 @@ async def test_load_model_and_completion_stream(transport) -> None: "history": [{"role": "user", "content": "Say hello in five words."}], "stream": True, # Bound + seed the generation: Qwen3's thinking trace otherwise - # rambles nondeterministically and can outgrow n_ctx mid-stream, + # rambles nondeterministically and can outgrow ctx_size mid-stream, # surfacing as a flaky CONTEXT_OVERFLOW. "generationParams": {"predict": 512, "temp": 0, "seed": 42}, } @@ -349,7 +349,7 @@ async def test_completion_orchestrate_without_tools(transport) -> None: "type": "loadModel", "modelSrc": QWEN3_600M_INST_Q4.src, "modelType": "llamacpp-completion", - "modelConfig": {"n_ctx": 2048}, + "modelConfig": {"ctx_size": 2048}, } ) load_response = await load_model(transport, load_request) @@ -385,7 +385,7 @@ async def test_completion_orchestrate_runs_the_tool_loop(transport) -> None: "type": "loadModel", "modelSrc": QWEN3_600M_INST_Q4.src, "modelType": "llamacpp-completion", - "modelConfig": {"n_ctx": 4096, "tools": True}, + "modelConfig": {"ctx_size": 4096, "tools": True}, } ) load_response = await load_model(transport, load_request) @@ -442,7 +442,7 @@ async def test_completion_orchestrate_cancel_stops_generation(transport) -> None "type": "loadModel", "modelSrc": QWEN3_600M_INST_Q4.src, "modelType": "llamacpp-completion", - "modelConfig": {"n_ctx": 2048}, + "modelConfig": {"ctx_size": 2048}, } ) load_response = await load_model(transport, load_request) diff --git a/packages/sdk-python/tests/test_llm_load_mode.py b/packages/sdk-python/tests/test_llm_load_mode.py new file mode 100644 index 0000000000..050d1a6c6e --- /dev/null +++ b/packages/sdk-python/tests/test_llm_load_mode.py @@ -0,0 +1,35 @@ +"""Generated llamacpp load config accepts every `load_mode` the addon does.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from tetherto.qvac_sdk._generated.models import ( + LoadModelSrcRequestLlamacppCompletionModelConfig, +) + + +@pytest.mark.parametrize("load_mode", ["none", "mmap", "mlock", "mmap+mlock", "dio"]) +def test_load_mode_accepts_every_addon_value(load_mode: str) -> None: + config = LoadModelSrcRequestLlamacppCompletionModelConfig.model_validate( + {"ctx_size": 2048, "load_mode": load_mode} + ) + + assert config.load_mode is not None + assert config.load_mode.value == load_mode + + +def test_load_mode_rejects_unknown_value() -> None: + with pytest.raises(ValidationError): + LoadModelSrcRequestLlamacppCompletionModelConfig.model_validate( + {"load_mode": "buffered"} + ) + + +def test_load_mode_is_optional() -> None: + config = LoadModelSrcRequestLlamacppCompletionModelConfig.model_validate( + {"ctx_size": 2048} + ) + + assert config.load_mode is None diff --git a/packages/sdk-python/tests/test_notebook.py b/packages/sdk-python/tests/test_notebook.py index 62ebcf33f2..7d260c3dab 100644 --- a/packages/sdk-python/tests/test_notebook.py +++ b/packages/sdk-python/tests/test_notebook.py @@ -168,7 +168,7 @@ def test_sync_client_notebook_flow_against_real_worker(): with SyncClient(worker_path=WORKER_PATH, bare_path=BARE_BIN) as client: model_id = client.load_model( - model_src=QWEN3_600M_INST_Q4, model_config={"n_ctx": 2048} + model_src=QWEN3_600M_INST_Q4, model_config={"ctx_size": 2048} ) text = client.completion( model_id, diff --git a/packages/sdk/contract/schema.json b/packages/sdk/contract/schema.json index 0f0963836f..b874de4204 100644 --- a/packages/sdk/contract/schema.json +++ b/packages/sdk/contract/schema.json @@ -8282,9 +8282,11 @@ "description": "Seeds conversation history on the JS side only; never forwarded to the addon. Default `'You are a helpful assistant.'`", "type": "string" }, - "no_mmap": { - "description": "Disable memory-mapped model loading. Default false.", - "type": "boolean" + "load_mode": { + "description": "Model loading mode: `'none'`, `'mmap'`, `'mlock'`, `'mmap+mlock'`, or `'dio'`. Unset uses the addon's default (`'mmap'`).", + "type": "string", + "enum": ["none", "mmap", "mlock", "mmap+mlock", "dio"], + "title": "LoadModelSrcRequestLlamacppCompletionModelConfigLoadMode" }, "verbosity": { "description": "Native log verbosity: `0`=ERROR, `1`=WARN, `2`=INFO, `3`=DEBUG. Default 0.", diff --git a/packages/sdk/e2e/conformance/cases.json b/packages/sdk/e2e/conformance/cases.json index c3b376b8d0..79939d4b45 100644 --- a/packages/sdk/e2e/conformance/cases.json +++ b/packages/sdk/e2e/conformance/cases.json @@ -88,7 +88,7 @@ "category": "completionOrchestrate", "model": "QWEN3_600M_INST_Q4", "modelType": "llamacpp-completion", - "modelConfig": { "n_ctx": 4096, "tools": true }, + "modelConfig": { "ctx_size": 4096, "tools": true }, "params": { "history": [ { "role": "user", "content": "Call the get_secret_code tool, then tell me the code." } diff --git a/packages/sdk/e2e/tests/mobile/executors/config-reload-executor.ts b/packages/sdk/e2e/tests/mobile/executors/config-reload-executor.ts index f6d758146c..12496bb1d1 100644 --- a/packages/sdk/e2e/tests/mobile/executors/config-reload-executor.ts +++ b/packages/sdk/e2e/tests/mobile/executors/config-reload-executor.ts @@ -98,7 +98,7 @@ export class MobileConfigReloadExecutor extends ModelAssetExecutor { + const { loadMode } = params as { loadMode: 'none' | 'mmap' } + const dep = `llm-load-mode-${loadMode}` + try { + const modelId = await loadModel({ + modelSrc: LLAMA_3_2_1B_INST_Q4_0, + modelType: 'llamacpp-completion', + modelConfig: { verbosity: 0, ctx_size: 2048, load_mode: loadMode } + }) + this.resources.register(dep, modelId) + return ValidationHelpers.validate(modelId, expectation) + } finally { + await this.resources.evict(dep) + } + } + + async rejectLegacyNoMmap( + params: typeof modelLoadLlmLegacyNoMmapRejected.params, + expectation: typeof modelLoadLlmLegacyNoMmapRejected.expectation + ): Promise { + const { noMmap } = params as { noMmap: boolean } + const dep = 'llm-legacy-no-mmap' + try { + const modelId = await loadModel({ + modelSrc: LLAMA_3_2_1B_INST_Q4_0, + modelType: 'llamacpp-completion', + modelConfig: { no_mmap: noMmap } + } as unknown as Parameters[0]) + // Should not happen, but release it rather than leak on the failure path. + this.resources.register(dep, modelId) + await this.resources.evict(dep) + return { passed: false, output: 'Legacy no_mmap should have been rejected' } + } catch (error) { + const errorMsg = error instanceof Error ? error.message : JSON.stringify(error) + // A native --no-mmap error means the key reached the addon: the regression. + if (/invalid argument|--no-mmap/.test(errorMsg)) { + return { + passed: false, + output: `no_mmap reached the addon instead of failing SDK validation: ${errorMsg}` + } + } + // The code is on the error, not in its message. + const code = (error as { code?: number }).code + return ValidationHelpers.validate(`${code} ${errorMsg}`, expectation) + } + } + async lifecycleNmt( params: typeof modelLifecycleNmt.params, expectation: typeof modelLifecycleNmt.expectation diff --git a/packages/sdk/e2e/tests/shared/executors/node/config-reload-executor.ts b/packages/sdk/e2e/tests/shared/executors/node/config-reload-executor.ts index 190ddb8000..f6e1f2bbc3 100644 --- a/packages/sdk/e2e/tests/shared/executors/node/config-reload-executor.ts +++ b/packages/sdk/e2e/tests/shared/executors/node/config-reload-executor.ts @@ -82,7 +82,7 @@ export class ConfigReloadExecutor extends AbstractModelExecutor