diff --git a/managed-inference/images/llama-cpp/image.yaml b/managed-inference/images/llama-cpp/image.yaml index 858e2606911..25ca1a91fa6 100644 --- a/managed-inference/images/llama-cpp/image.yaml +++ b/managed-inference/images/llama-cpp/image.yaml @@ -43,6 +43,7 @@ spec: qualification: required: true execution: enabled + requestGuard: required profile: dgx-spark-gb10-single recipeRef: llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1 platform: linux/arm64 diff --git a/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml b/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml index 5527231d794..437ff556676 100644 --- a/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml +++ b/managed-inference/recipes/llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1.yaml @@ -86,7 +86,13 @@ spec: value: f16 speculativeDecoding: disabled limits: + maxRequestBodyBytes: 1048576 + maxRequestHeaderBytes: 32768 + maxOutputTokens: 4096 requestTimeoutSeconds: 900 + shutdownTimeoutSeconds: 25 + requestGuard: + upstreamPort: 8082 readiness: contractRef: llama-cpp.server-readiness/v1 diff --git a/managed-inference/schemas/recipe.schema.json b/managed-inference/schemas/recipe.schema.json index 02ff60352ae..275128accbf 100644 --- a/managed-inference/schemas/recipe.schema.json +++ b/managed-inference/schemas/recipe.schema.json @@ -482,13 +482,54 @@ "limits": { "type": "object", "required": [ - "requestTimeoutSeconds" + "maxRequestBodyBytes", + "maxRequestHeaderBytes", + "maxOutputTokens", + "requestTimeoutSeconds", + "shutdownTimeoutSeconds" ], "properties": { + "maxRequestBodyBytes": { + "type": "integer", + "minimum": 1, + "maximum": 67108864 + }, + "maxRequestHeaderBytes": { + "type": "integer", + "minimum": 1, + "maximum": 1048576 + }, + "maxOutputTokens": { + "type": "integer", + "minimum": 1, + "maximum": 1048576 + }, "requestTimeoutSeconds": { "type": "integer", "minimum": 1, "maximum": 86400 + }, + "shutdownTimeoutSeconds": { + "type": "integer", + "minimum": 1, + "maximum": 86400 + } + }, + "additionalProperties": false + }, + "requestGuard": { + "type": "object", + "required": [ + "upstreamPort" + ], + "properties": { + "upstreamPort": { + "type": "integer", + "minimum": 1, + "maximum": 65535, + "not": { + "const": 8081 + } } }, "additionalProperties": false @@ -1239,7 +1280,8 @@ "flashAttention", "kvCache", "speculativeDecoding", - "limits" + "limits", + "requestGuard" ], "properties": { "protocol": {}, @@ -1264,7 +1306,8 @@ "flashAttention": {}, "kvCache": {}, "speculativeDecoding": {}, - "limits": {} + "limits": {}, + "requestGuard": {} }, "propertyNames": { "not": { diff --git a/scripts/checks/export-llama-cpp-image-config.mts b/scripts/checks/export-llama-cpp-image-config.mts index bcbb7d3a85f..63bea90d52c 100644 --- a/scripts/checks/export-llama-cpp-image-config.mts +++ b/scripts/checks/export-llama-cpp-image-config.mts @@ -59,6 +59,7 @@ type ServerImageManifest = { probes?: unknown; profile?: unknown; recipeRef?: unknown; + requestGuard?: unknown; required?: unknown; runner?: unknown; }; @@ -136,8 +137,13 @@ type LlamaCppQualificationRecipe = { kvCache: { key: string; value: string }; speculativeDecoding: string; limits: { + maxRequestBodyBytes: number; + maxRequestHeaderBytes: number; + maxOutputTokens: number; requestTimeoutSeconds: number; + shutdownTimeoutSeconds: number; }; + requestGuard: { upstreamPort: number }; }; readiness: { contractRef: string; @@ -381,6 +387,7 @@ export function loadLlamaCppImageConfig( "probes", "profile", "recipeRef", + "requestGuard", "required", "runner", ]); @@ -426,6 +433,11 @@ export function loadLlamaCppImageConfig( "qualification recipe reference", /^llama-cpp\.nemotron-3-nano-30b-a3b\.spark-single\.v1$/u, ); + const qualificationRequestGuard = requiredString( + qualification?.requestGuard, + "qualification request guard", + /^required$/u, + ); const qualificationModel = qualification?.model as | { digest?: unknown; hostPath?: unknown; id?: unknown } | undefined; @@ -588,6 +600,7 @@ export function loadLlamaCppImageConfig( probes: qualification?.probes, profile: qualification?.profile, recipeRef: qualificationRecipeRef, + requestGuard: qualificationRequestGuard, required: qualification?.required, runner: qualificationRunner, }; @@ -725,6 +738,7 @@ export function loadLlamaCppImageConfig( agentQualification, probeBounds: qualification?.probeBounds, probes: qualification?.probes, + requestGuard: qualificationRequestGuard, }, recipe: { capabilities: recipe.spec.capabilities, diff --git a/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts b/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts index 3175f948e65..c425aaffa2b 100644 --- a/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts +++ b/scripts/checks/llama-cpp-dgx-spark-qualification-contract.mts @@ -151,6 +151,7 @@ export type LlamaCppDgxSparkQualificationPlan = { readonly probes: typeof LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROBES; readonly profile: typeof LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROFILE; readonly recipeRef: typeof LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE; + readonly requestGuard: "required"; readonly required: true; readonly runner: string | null; }; @@ -196,6 +197,7 @@ export type LlamaCppDgxSparkExecutionPlan = { readonly agentQualification: LlamaCppDgxSparkAgentQualificationPlan; readonly probeBounds: LlamaCppDgxSparkQualificationPlan["probeBounds"]; readonly probes: typeof LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROBES; + readonly requestGuard: "required"; }; readonly recipe: { readonly capabilities: { @@ -274,7 +276,14 @@ export type LlamaCppDgxSparkExecutionPlan = { readonly value: "f16" | "q8_0" | "q4_0"; }; readonly limits: { + readonly maxRequestBodyBytes: number; + readonly maxRequestHeaderBytes: number; + readonly maxOutputTokens: number; readonly requestTimeoutSeconds: number; + readonly shutdownTimeoutSeconds: number; + }; + readonly requestGuard: { + readonly upstreamPort: number; }; readonly microBatchSize: number; readonly port: 8081; @@ -915,6 +924,7 @@ export function parseLlamaCppDgxSparkQualificationPlan( "probes", "profile", "recipeRef", + "requestGuard", "required", "runner", ], @@ -933,6 +943,7 @@ export function parseLlamaCppDgxSparkQualificationPlan( plan.required !== true || plan.profile !== LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROFILE || plan.recipeRef !== LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE || + plan.requestGuard !== "required" || plan.platform !== LLAMA_CPP_DGX_SPARK_QUALIFICATION_PLATFORM || gpu.vendor !== "nvidia" || gpu.fullOffload !== true || @@ -957,6 +968,7 @@ export function parseLlamaCppDgxSparkQualificationPlan( probes: LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROBES, profile: LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROFILE, recipeRef: LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE, + requestGuard: "required", required: true, runner: infrastructure.runner, }; @@ -979,7 +991,7 @@ export function parseLlamaCppDgxSparkExecutionPlan( const qualification = record(plan.qualification, "compiled protocol qualification"); requireExactKeys( qualification, - ["agentQualification", "probeBounds", "probes"], + ["agentQualification", "probeBounds", "probes", "requestGuard"], "compiled protocol qualification", ); if ( @@ -988,6 +1000,9 @@ export function parseLlamaCppDgxSparkExecutionPlan( ) { throw new Error("compiled llama.cpp DGX Spark protocol probes are invalid"); } + if (qualification.requestGuard !== "required") { + throw new Error("compiled llama.cpp DGX Spark request-guard activation is invalid"); + } const protocolProbeBounds = parseProtocolProbeBounds(qualification.probeBounds); const agentQualification = parseAgentQualification(qualification.agentQualification); @@ -1245,6 +1260,7 @@ export function parseLlamaCppDgxSparkExecutionPlan( "microBatchSize", "port", "protocol", + "requestGuard", "slots", "speculativeDecoding", ], @@ -1267,13 +1283,55 @@ export function parseLlamaCppDgxSparkExecutionPlan( requireExactKeys(kvCache, ["key", "value"], "compiled qualification KV cache"); const allowedKvTypes = new Set(["f16", "q8_0", "q4_0"]); const limits = record(serve.limits, "compiled qualification request limits"); - requireExactKeys(limits, ["requestTimeoutSeconds"], "compiled qualification request limits"); + requireExactKeys( + limits, + [ + "maxOutputTokens", + "maxRequestBodyBytes", + "maxRequestHeaderBytes", + "requestTimeoutSeconds", + "shutdownTimeoutSeconds", + ], + "compiled qualification request limits", + ); + const maxRequestBodyBytes = boundedInteger( + limits.maxRequestBodyBytes, + "compiled qualification maximum request body bytes", + 1, + 64 * 1024 * 1024, + ); + const maxRequestHeaderBytes = boundedInteger( + limits.maxRequestHeaderBytes, + "compiled qualification maximum request header bytes", + 1, + 1024 * 1024, + ); + const maxOutputTokens = boundedInteger( + limits.maxOutputTokens, + "compiled qualification maximum output tokens", + 1, + contextSize, + ); const requestTimeoutSeconds = boundedInteger( limits.requestTimeoutSeconds, "compiled qualification request timeout", 1, 3600, ); + const shutdownTimeoutSeconds = boundedInteger( + limits.shutdownTimeoutSeconds, + "compiled qualification shutdown timeout", + 1, + 3600, + ); + const requestGuard = record(serve.requestGuard, "compiled qualification request guard"); + requireExactKeys(requestGuard, ["upstreamPort"], "compiled qualification request guard"); + const upstreamPort = boundedInteger( + requestGuard.upstreamPort, + "compiled qualification request-guard upstream port", + 1, + 65535, + ); if ( serve.protocol !== "openai-completions" || serve.authentication !== "bearer" || @@ -1283,6 +1341,7 @@ export function parseLlamaCppDgxSparkExecutionPlan( serve.idleSleepSeconds !== -1 || serve.flashAttention !== "enabled" || serve.speculativeDecoding !== "disabled" || + upstreamPort === serve.port || typeof kvCache.key !== "string" || !allowedKvTypes.has(kvCache.key) || typeof kvCache.value !== "string" || @@ -1347,6 +1406,7 @@ export function parseLlamaCppDgxSparkExecutionPlan( agentQualification, probeBounds: protocolProbeBounds, probes: LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROBES, + requestGuard: "required", }, recipe: { capabilities: { @@ -1419,8 +1479,13 @@ export function parseLlamaCppDgxSparkExecutionPlan( }, speculativeDecoding: "disabled", limits: { + maxRequestBodyBytes, + maxRequestHeaderBytes, + maxOutputTokens, requestTimeoutSeconds, + shutdownTimeoutSeconds, }, + requestGuard: { upstreamPort }, }, server: { technology: "llama.cpp", diff --git a/scripts/checks/run-llama-cpp-dgx-spark-qualification.mts b/scripts/checks/run-llama-cpp-dgx-spark-qualification.mts index ae02729de18..7e2e1be3ae5 100644 --- a/scripts/checks/run-llama-cpp-dgx-spark-qualification.mts +++ b/scripts/checks/run-llama-cpp-dgx-spark-qualification.mts @@ -9,7 +9,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { - buildLlamaCppHostLocalDockerArgv, + buildLlamaCppRequestGuardDockerArgv, type VerifiedLocalModelArtifact, } from "../../src/lib/inference/llama-cpp/host-local-runtime.ts"; import { @@ -332,7 +332,10 @@ export function buildServerContainerArgv( hostPort?: number; }, ): string[] { - return buildLlamaCppHostLocalDockerArgv(plan.recipe, { + if (plan.qualification.requestGuard !== "required") { + throw new Error("llama.cpp qualification requires the declarative request guard"); + } + return buildLlamaCppRequestGuardDockerArgv(plan.recipe, { apiKeyHostPath: options.apiKeyHostPath, containerName: options.containerName, imageReference: options.imageReference, diff --git a/src/lib/inference/llama-cpp/host-local-runtime.test.ts b/src/lib/inference/llama-cpp/host-local-runtime.test.ts index a8bd81d4da6..f8d18e22fed 100644 --- a/src/lib/inference/llama-cpp/host-local-runtime.test.ts +++ b/src/lib/inference/llama-cpp/host-local-runtime.test.ts @@ -18,6 +18,9 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { LLAMA_CPP_PORT } from "./contract"; import { buildLlamaCppHostLocalDockerArgv, + buildLlamaCppRequestGuardDockerArgv, + LLAMA_CPP_HOST_LOCAL_REQUEST_GUARD_PATH, + LLAMA_CPP_HOST_LOCAL_SERVER_PATH, type LlamaCppHostLocalLaunchContract, type LlamaCppHostLocalRuntimeBindings, } from "./host-local-runtime"; @@ -83,8 +86,13 @@ function contract(): LlamaCppHostLocalLaunchContract { idleSleepSeconds: -1, kvCache: { key: "f16", value: "f16" }, limits: { + maxRequestBodyBytes: 1_048_576, + maxRequestHeaderBytes: 32_768, + maxOutputTokens: 4_096, requestTimeoutSeconds: 900, + shutdownTimeoutSeconds: 25, }, + requestGuard: { upstreamPort: 8_082 }, microBatchSize: 512, port: 8_081, protocol: "openai-completions", @@ -144,6 +152,7 @@ describe("llama.cpp host-local runtime materializer", () => { expect(valuesAfter(argv, "--timeout")).toEqual([ String(input.serve.limits.requestTimeoutSeconds), ]); + expect(argv).not.toContain("--n-predict"); expect(argv).toEqual( expect.arrayContaining([ "--read-only", @@ -264,6 +273,59 @@ describe("llama.cpp host-local runtime materializer", () => { ]); }); + it("activates the owned-image request guard from the declared recipe values (#8144)", () => { + const input = contract(); + const runtime = bindings(); + const argv = buildLlamaCppRequestGuardDockerArgv(input, runtime); + const separator = argv.indexOf("--"); + + expect(valuesAfter(argv, "--entrypoint")).toEqual([LLAMA_CPP_HOST_LOCAL_REQUEST_GUARD_PATH]); + expect(valuesAfter(argv, "--listen-port")).toEqual([String(input.serve.port)]); + expect(valuesAfter(argv, "--upstream-port")).toEqual([ + String(input.serve.requestGuard.upstreamPort), + ]); + expect(valuesAfter(argv, "--max-request-body-bytes")).toEqual([ + String(input.serve.limits.maxRequestBodyBytes), + ]); + expect(valuesAfter(argv, "--max-request-header-bytes")).toEqual([ + String(input.serve.limits.maxRequestHeaderBytes), + ]); + expect(valuesAfter(argv, "--max-output-tokens")).toEqual([ + String(input.serve.limits.maxOutputTokens), + ]); + expect(valuesAfter(argv, "--request-timeout-seconds")).toEqual([ + String(input.serve.limits.requestTimeoutSeconds), + ]); + expect(valuesAfter(argv, "--shutdown-timeout-seconds")).toEqual([ + String(input.serve.limits.shutdownTimeoutSeconds), + ]); + expect(argv.slice(separator + 1, separator + 8)).toEqual([ + LLAMA_CPP_HOST_LOCAL_SERVER_PATH, + "--model", + `/models/${input.model.file.path}`, + "--alias", + input.model.servedName, + "--host", + "127.0.0.1", + ]); + expect(valuesAfter(argv.slice(separator), "--port")).toEqual([ + String(input.serve.requestGuard.upstreamPort), + ]); + expect(valuesAfter(argv.slice(separator), "--n-predict")).toEqual([ + String(input.serve.limits.maxOutputTokens), + ]); + }); + + it("keeps the upstream-image launch on the llama-server entrypoint (#8144)", () => { + const argv = buildLlamaCppHostLocalDockerArgv(contract(), bindings()); + + expect(argv).not.toContain("--entrypoint"); + expect(argv).not.toContain(LLAMA_CPP_HOST_LOCAL_REQUEST_GUARD_PATH); + expect(argv).not.toContain("--n-predict"); + expect(valuesAfter(argv, "--host")).toEqual(["0.0.0.0"]); + expect(valuesAfter(argv, "--port")).toEqual(["8081"]); + }); + it("rejects an artifact that does not match the declared GGUF identity (#8144)", () => { expect(() => buildLlamaCppHostLocalDockerArgv(contract(), { diff --git a/src/lib/inference/llama-cpp/host-local-runtime.ts b/src/lib/inference/llama-cpp/host-local-runtime.ts index af93acca8b3..be0c37cb1de 100644 --- a/src/lib/inference/llama-cpp/host-local-runtime.ts +++ b/src/lib/inference/llama-cpp/host-local-runtime.ts @@ -8,6 +8,9 @@ import { isSafeLlamaCppServedModelAlias } from "./contract"; export const LLAMA_CPP_HOST_LOCAL_CONTAINER_API_KEY_PATH = "/run/secrets/llama-cpp-api-key" as const; +export const LLAMA_CPP_HOST_LOCAL_REQUEST_GUARD_PATH = + "/usr/local/bin/nemoclaw-llama-cpp-request-guard" as const; +export const LLAMA_CPP_HOST_LOCAL_SERVER_PATH = "/usr/local/bin/llama-server" as const; const SAFE_HOST_PATH = /^\/(?:[A-Za-z0-9._-]+\/)*[A-Za-z0-9._-]+$/u; const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u; @@ -59,7 +62,14 @@ export interface LlamaCppHostLocalLaunchContract { readonly value: "f16" | "q8_0" | "q4_0"; }; readonly limits: { + readonly maxRequestBodyBytes: number; + readonly maxRequestHeaderBytes: number; + readonly maxOutputTokens: number; readonly requestTimeoutSeconds: number; + readonly shutdownTimeoutSeconds: number; + }; + readonly requestGuard: { + readonly upstreamPort: number; }; readonly microBatchSize: number; readonly port: number; @@ -160,7 +170,34 @@ function validateContract(contract: LlamaCppHostLocalLaunchContract): void { positiveInteger(contract.serve.contextSize, "llama.cpp context size"); positiveInteger(contract.serve.batchSize, "llama.cpp batch size"); positiveInteger(contract.serve.microBatchSize, "llama.cpp micro-batch size"); + positiveInteger( + contract.serve.limits.maxRequestBodyBytes, + "llama.cpp maximum request body bytes", + 64 * 1_024 * 1_024, + ); + positiveInteger( + contract.serve.limits.maxRequestHeaderBytes, + "llama.cpp maximum request header bytes", + 1_024 * 1_024, + ); + positiveInteger( + contract.serve.limits.maxOutputTokens, + "llama.cpp maximum output tokens", + 1_024 * 1_024, + ); positiveInteger(contract.serve.limits.requestTimeoutSeconds, "llama.cpp request timeout", 86_400); + positiveInteger( + contract.serve.limits.shutdownTimeoutSeconds, + "llama.cpp shutdown timeout", + 86_400, + ); + positiveInteger(contract.serve.requestGuard.upstreamPort, "llama.cpp upstream port", 65_535); + if ( + contract.serve.limits.maxOutputTokens > contract.serve.contextSize || + contract.serve.requestGuard.upstreamPort === contract.serve.port + ) { + throw new Error("llama.cpp request-guard limits or ports are invalid"); + } if (contract.serve.microBatchSize > contract.serve.batchSize) { throw new Error("llama.cpp micro-batch size exceeds the batch size"); } @@ -265,6 +302,58 @@ export function buildLlamaCppHostLocalDockerArgv( ): string[] { validateContract(contract); validateBindings(contract, bindings); + return [ + ...buildLlamaCppHostLocalDockerRunArgv(contract, bindings), + bindings.imageReference, + ...buildLlamaCppHostLocalServerArgv(contract), + ]; +} + +/** Activate the request guard only for an image that declares the owned guard artifact. */ +export function buildLlamaCppRequestGuardDockerArgv( + contract: LlamaCppHostLocalLaunchContract, + bindings: LlamaCppHostLocalRuntimeBindings, +): string[] { + validateContract(contract); + validateBindings(contract, bindings); + const { limits } = contract.serve; + return [ + ...buildLlamaCppHostLocalDockerRunArgv(contract, bindings), + "--entrypoint", + LLAMA_CPP_HOST_LOCAL_REQUEST_GUARD_PATH, + bindings.imageReference, + "--listen-host", + "0.0.0.0", + "--listen-port", + String(contract.serve.port), + "--upstream-host", + "127.0.0.1", + "--upstream-port", + String(contract.serve.requestGuard.upstreamPort), + "--max-request-body-bytes", + String(limits.maxRequestBodyBytes), + "--max-request-header-bytes", + String(limits.maxRequestHeaderBytes), + "--max-output-tokens", + String(limits.maxOutputTokens), + "--request-timeout-seconds", + String(limits.requestTimeoutSeconds), + "--shutdown-timeout-seconds", + String(limits.shutdownTimeoutSeconds), + "--", + LLAMA_CPP_HOST_LOCAL_SERVER_PATH, + ...buildLlamaCppHostLocalServerArgvForAddress(contract, { + host: "127.0.0.1", + maxOutputTokens: limits.maxOutputTokens, + port: contract.serve.requestGuard.upstreamPort, + }), + ]; +} + +function buildLlamaCppHostLocalDockerRunArgv( + contract: LlamaCppHostLocalLaunchContract, + bindings: LlamaCppHostLocalRuntimeBindings, +): string[] { const { resources } = contract.runtime; const { serve } = contract; const containerModelPath = `/models/${contract.model.file.path}`; @@ -305,16 +394,29 @@ export function buildLlamaCppHostLocalDockerArgv( `type=bind,source=${bindings.model.hostPath},target=${containerModelPath},readonly`, "--mount", `type=bind,source=${bindings.apiKeyHostPath},target=${LLAMA_CPP_HOST_LOCAL_CONTAINER_API_KEY_PATH},readonly`, - bindings.imageReference, - ...buildLlamaCppHostLocalServerArgv(contract), ]; } /** Reconstruct the immutable in-container server command without host filesystem state. */ export function buildLlamaCppHostLocalServerArgv( contract: LlamaCppHostLocalLaunchContract, +): readonly string[] { + return buildLlamaCppHostLocalServerArgvForAddress(contract, { + host: "0.0.0.0", + port: contract.serve.port, + }); +} + +function buildLlamaCppHostLocalServerArgvForAddress( + contract: LlamaCppHostLocalLaunchContract, + address: { + readonly host: "0.0.0.0" | "127.0.0.1"; + readonly maxOutputTokens?: number; + readonly port: number; + }, ): readonly string[] { validateContract(contract); + positiveInteger(address.port, "llama.cpp command port", 65_535); const { serve } = contract; const containerModelPath = `/models/${contract.model.file.path}`; return Object.freeze([ @@ -323,9 +425,9 @@ export function buildLlamaCppHostLocalServerArgv( "--alias", contract.model.servedName, "--host", - "0.0.0.0", + address.host, "--port", - String(serve.port), + String(address.port), "--gpu-layers", "all", "--ctx-size", @@ -346,6 +448,9 @@ export function buildLlamaCppHostLocalServerArgv( "on", "--timeout", String(serve.limits.requestTimeoutSeconds), + ...(address.maxOutputTokens === undefined + ? [] + : ["--n-predict", String(address.maxOutputTokens)]), "--api-key-file", LLAMA_CPP_HOST_LOCAL_CONTAINER_API_KEY_PATH, "--metrics", diff --git a/src/lib/inference/llama-cpp/managed-installer.ts b/src/lib/inference/llama-cpp/managed-installer.ts index de3e90cb2bc..eeb65ca8d07 100644 --- a/src/lib/inference/llama-cpp/managed-installer.ts +++ b/src/lib/inference/llama-cpp/managed-installer.ts @@ -287,7 +287,14 @@ function launchContract( flashAttention: recipe.spec.serve.flashAttention, idleSleepSeconds: recipe.spec.serve.idleSleepSeconds, kvCache: recipe.spec.serve.kvCache, - limits: { requestTimeoutSeconds: recipe.spec.serve.limits.requestTimeoutSeconds }, + limits: { + maxRequestBodyBytes: recipe.spec.serve.limits.maxRequestBodyBytes, + maxRequestHeaderBytes: recipe.spec.serve.limits.maxRequestHeaderBytes, + maxOutputTokens: recipe.spec.serve.limits.maxOutputTokens, + requestTimeoutSeconds: recipe.spec.serve.limits.requestTimeoutSeconds, + shutdownTimeoutSeconds: recipe.spec.serve.limits.shutdownTimeoutSeconds, + }, + requestGuard: { upstreamPort: recipe.spec.serve.requestGuard.upstreamPort }, microBatchSize: recipe.spec.serve.microBatchSize, port: recipe.spec.serve.port, protocol: recipe.spec.serve.protocol, diff --git a/src/lib/inference/serving/catalog.test.ts b/src/lib/inference/serving/catalog.test.ts index 18a3d5cacf5..00bb2d94be3 100644 --- a/src/lib/inference/serving/catalog.test.ts +++ b/src/lib/inference/serving/catalog.test.ts @@ -195,7 +195,13 @@ spec: value: f16 speculativeDecoding: disabled limits: + maxRequestBodyBytes: 1048576 + maxRequestHeaderBytes: 32768 + maxOutputTokens: 4096 requestTimeoutSeconds: 120 + shutdownTimeoutSeconds: 25 + requestGuard: + upstreamPort: 8082 readiness: contractRef: test.readiness/v1 timeoutSeconds: 120 @@ -362,6 +368,14 @@ describe("managed inference serving catalog compiler", () => { flashAttention: "enabled", kvCache: { key: "f16", value: "f16" }, speculativeDecoding: "disabled", + limits: { + maxRequestBodyBytes: 1048576, + maxRequestHeaderBytes: 32768, + maxOutputTokens: 4096, + requestTimeoutSeconds: 120, + shutdownTimeoutSeconds: 25, + }, + requestGuard: { upstreamPort: 8082 }, }, capabilities: { agents: [], protocols: ["openai-completions"] }, }); @@ -433,6 +447,8 @@ describe("managed inference serving catalog compiler", () => { ["CPU fallback", " cpuFallback: reject", " cpuFallback: allow"], ["external network exposure", " networkExposure: loopback", " networkExposure: lan"], ["missing authentication", " authentication: bearer\n", ""], + ["a missing guard limit", " shutdownTimeoutSeconds: 25\n", ""], + ["a colliding guard port", " upstreamPort: 8082", " upstreamPort: 8081"], ["an unsafe KV cache type", " key: f16", " key: q4_0"], ["disabled flash attention", " flashAttention: enabled", " flashAttention: disabled"], [ diff --git a/src/lib/inference/serving/types.ts b/src/lib/inference/serving/types.ts index 048a66cbacb..ff943fd6da4 100644 --- a/src/lib/inference/serving/types.ts +++ b/src/lib/inference/serving/types.ts @@ -298,7 +298,14 @@ export interface LlamaCppServingRecipe extends ServingRecipeEnvelope { }; readonly speculativeDecoding: "disabled"; readonly limits: { + readonly maxRequestBodyBytes: number; + readonly maxRequestHeaderBytes: number; + readonly maxOutputTokens: number; readonly requestTimeoutSeconds: number; + readonly shutdownTimeoutSeconds: number; + }; + readonly requestGuard: { + readonly upstreamPort: number; }; }; readonly readiness: { diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test-support.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test-support.ts index 14832a519ca..e247a9d785d 100644 --- a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test-support.ts +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.test-support.ts @@ -85,7 +85,14 @@ export function contract(): LlamaCppHostLocalLaunchContract { flashAttention: "enabled", idleSleepSeconds: -1, kvCache: { key: "f16", value: "f16" }, - limits: { requestTimeoutSeconds: 900 }, + limits: { + maxRequestBodyBytes: 1_048_576, + maxRequestHeaderBytes: 32_768, + maxOutputTokens: 4_096, + requestTimeoutSeconds: 900, + shutdownTimeoutSeconds: 25, + }, + requestGuard: { upstreamPort: 8_082 }, microBatchSize: 512, port: LLAMA_CPP_PORT, protocol: "openai-completions", diff --git a/test/llama-cpp-dgx-spark-qualification-contract.test.ts b/test/llama-cpp-dgx-spark-qualification-contract.test.ts index 3d3802e8251..85bb7fb2a36 100644 --- a/test/llama-cpp-dgx-spark-qualification-contract.test.ts +++ b/test/llama-cpp-dgx-spark-qualification-contract.test.ts @@ -130,6 +130,7 @@ function disabledPlan() { probes: LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROBES, profile: LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROFILE, recipeRef: LLAMA_CPP_DGX_SPARK_QUALIFICATION_RECIPE, + requestGuard: "required", required: true, runner: null, }; @@ -291,6 +292,7 @@ function executionPlan() { agentQualification: agentQualification(), probeBounds: probeBounds(), probes: LLAMA_CPP_DGX_SPARK_QUALIFICATION_PROBES, + requestGuard: "required", }, recipe: { capabilities: { @@ -359,8 +361,13 @@ function executionPlan() { kvCache: { key: "f16", value: "f16" }, speculativeDecoding: "disabled", limits: { + maxRequestBodyBytes: 1048576, + maxRequestHeaderBytes: 32768, + maxOutputTokens: 4096, requestTimeoutSeconds: 900, + shutdownTimeoutSeconds: 25, }, + requestGuard: { upstreamPort: 8082 }, }, server: { technology: "llama.cpp", @@ -579,6 +586,7 @@ describe("llama.cpp DGX Spark qualification contract", () => { microBatchSize: 256, kvCache: { key: "q8_0", value: "q8_0" }, limits: { + ...value.recipe.serve.limits, requestTimeoutSeconds: 600, }, }, @@ -624,6 +632,40 @@ describe("llama.cpp DGX Spark qualification contract", () => { }, }), ).toThrow("micro-batch size is invalid"); + expect(() => + parseLlamaCppDgxSparkExecutionPlan({ + ...value, + recipe: { + ...value.recipe, + serve: { + ...value.recipe.serve, + limits: { ...value.recipe.serve.limits, maxOutputTokens: 262145 }, + }, + }, + }), + ).toThrow("maximum output tokens is invalid"); + expect(() => + parseLlamaCppDgxSparkExecutionPlan({ + ...value, + recipe: { + ...value.recipe, + serve: { + ...value.recipe.serve, + requestGuard: { upstreamPort: value.recipe.serve.port }, + }, + }, + }), + ).toThrow("serve contract is invalid"); + const { shutdownTimeoutSeconds: _removed, ...incompleteLimits } = value.recipe.serve.limits; + expect(() => + parseLlamaCppDgxSparkExecutionPlan({ + ...value, + recipe: { + ...value.recipe, + serve: { ...value.recipe.serve, limits: incompleteLimits }, + }, + }), + ).toThrow("request limits has unexpected fields"); expect(() => parseLlamaCppDgxSparkExecutionPlan({ ...value, @@ -660,6 +702,12 @@ describe("llama.cpp DGX Spark qualification contract", () => { }, }), ).toThrow("protocol probes are invalid"); + expect(() => + parseLlamaCppDgxSparkExecutionPlan({ + ...value, + qualification: { ...value.qualification, requestGuard: "disabled" }, + }), + ).toThrow("request-guard activation is invalid"); }); it("accepts one bounded receipt with only allowlisted workflow, image, model, and Spark evidence (#8260)", () => { diff --git a/test/llama-cpp-dgx-spark-qualification-runner.test.ts b/test/llama-cpp-dgx-spark-qualification-runner.test.ts index f66f52c2f0c..4b52566c403 100644 --- a/test/llama-cpp-dgx-spark-qualification-runner.test.ts +++ b/test/llama-cpp-dgx-spark-qualification-runner.test.ts @@ -282,7 +282,7 @@ describe("trusted llama.cpp DGX Spark qualification runner", () => { } }); - it("constructs a read-only bounded one-GPU server without putting the key on argv (#8260)", () => { + it("activates the YAML-bound guard without putting the key on argv (#8260)", () => { const content = Buffer.from("qualification model fixture\n", "utf8"); const testPlan = qualificationPlanForModel(content); const modelRoot = fs.realpathSync( @@ -339,7 +339,38 @@ describe("trusted llama.cpp DGX Spark qualification runner", () => { "--no-agent", ]), ); - expect(valuesAfter(argv, "--publish")).toEqual([]); + expect(valuesAfter(argv, "--publish")).toEqual(["127.0.0.1::8081"]); + expect(valuesAfter(argv, "--entrypoint")).toEqual([ + "/usr/local/bin/nemoclaw-llama-cpp-request-guard", + ]); + expect(valuesAfter(argv, "--listen-port")).toEqual([String(testPlan.recipe.serve.port)]); + expect(valuesAfter(argv, "--upstream-port")).toEqual([ + String(testPlan.recipe.serve.requestGuard.upstreamPort), + ]); + expect(valuesAfter(argv, "--max-request-body-bytes")).toEqual([ + String(testPlan.recipe.serve.limits.maxRequestBodyBytes), + ]); + expect(valuesAfter(argv, "--max-request-header-bytes")).toEqual([ + String(testPlan.recipe.serve.limits.maxRequestHeaderBytes), + ]); + expect(valuesAfter(argv, "--max-output-tokens")).toEqual([ + String(testPlan.recipe.serve.limits.maxOutputTokens), + ]); + expect(valuesAfter(argv, "--request-timeout-seconds")).toEqual([ + String(testPlan.recipe.serve.limits.requestTimeoutSeconds), + ]); + expect(valuesAfter(argv, "--shutdown-timeout-seconds")).toEqual([ + String(testPlan.recipe.serve.limits.shutdownTimeoutSeconds), + ]); + const separator = argv.indexOf("--"); + expect(argv[separator + 1]).toBe("/usr/local/bin/llama-server"); + expect(valuesAfter(argv.slice(separator), "--host")).toEqual(["127.0.0.1"]); + expect(valuesAfter(argv.slice(separator), "--port")).toEqual([ + String(testPlan.recipe.serve.requestGuard.upstreamPort), + ]); + expect(valuesAfter(argv.slice(separator), "--n-predict")).toEqual([ + String(testPlan.recipe.serve.limits.maxOutputTokens), + ]); const agentQualificationArgv = buildServerContainerArgv(testPlan, { apiKeyHostPath: "/work/tmp/api-key", containerName: "qualified-server", diff --git a/test/llama-cpp-image.test.ts b/test/llama-cpp-image.test.ts index 2fdfbc4641d..14348b0eef1 100644 --- a/test/llama-cpp-image.test.ts +++ b/test/llama-cpp-image.test.ts @@ -266,6 +266,7 @@ describe("declarative llama.cpp server image", () => { "agent-multi-turn", ], }, + requestGuard: "required", }, recipe: { id: "llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1",