fix(server): pre-populate storyboard-declared test kits on hosted storyboard runs

docs/building/operating/storyboard-troubleshooting.mdx

@@ -110,6 +110,8 @@ The `mechanism_required` phase found no contribution to `auth_mechanism_verified
 
 **The `--auth TOKEN` distinction:** The `--auth TOKEN` flag you pass to the runner is the runner's own session credential — it authorizes the runner's own requests to your agent. It is entirely separate from `test_kit.auth.api_key` or `test_kit.auth.basic`, which are the specific credentials the static-credential phases send during positive and invalid-credential probes. These are not the same token and are not interchangeable.
 
+**`prerequisites.test_kit` is a loading directive, not decoration:** when a storyboard declares `prerequisites.test_kit` (for example `comply_controller_mode_gate` declares `test-kits/acme-outdoor-live.yaml`), the runner loads that kit from the active compliance cache and its credentials flow into the storyboard's `from_test_kit` and `$test_kit.*` references without any CLI flag. An explicitly supplied kit (`options.test_kit` / hosted-run configuration) still wins. A step whose `auth.from_test_kit` resolves no credential fails with an explicit configuration error rather than sending an unauthenticated probe — a probe with no credential cannot test a credential-keyed contract.
+
 **Fix for Bearer API key agents:** Every default AdCP brand test kit declares its probe API key under `auth.api_key` using the `demo-<kit>-v1` naming convention. The default test kit (`acme-outdoor`) uses `demo-acme-outdoor-v1`. Configure your agent to accept the kit's probe key as a valid compliance-testing credential alongside your production key. The `demo-<kit>-` prefix is the AdCP conformance handle — accept any Bearer token matching the prefix for the kits you run against (the suffix can rotate across spec versions, the prefix stays stable):
 
 ```typescript

server/src/addie/config-version.ts

@@ -30,7 +30,7 @@ import { loadRules, loadResponseStyle } from './rules/index.js';
  * Format: YYYY.MM.N where N is incremented for multiple changes in a month
  * Example: 2025.01.1, 2025.01.2, 2025.02.1
  */
-export const CODE_VERSION = '2026.08.12';
+export const CODE_VERSION = '2026.08.13';
 
 // Types
 export interface ConfigVersion {

server/src/addie/mcp/member-tools.ts

@@ -72,6 +72,7 @@ import {
 } from '@adcp/sdk/testing';
 import { AuthenticationRequiredError } from '@adcp/sdk';
 import { renderAllHintFixPlans } from '../services/storyboard-fix-plan.js';
+import { getTestKitForStoryboard } from '../../services/storyboards.js';
 import {
   hostedComplianceTarget,
   hostedComplianceOptions,
@@ -5308,10 +5309,16 @@ export function createMemberToolHandlers(
 
     try {
       const authProbeTask = await inferHostedAuthProbeTask(resolved.resolvedUrl, authOption, runTarget);
+      // adcp#6735 — pre-populate the storyboard-declared test kit so
+      // `from_test_kit` steps run with the credential the storyboard was
+      // authored against; the run-auth bearer substitution no-ops when the
+      // kit already carries auth.
+      const declaredTestKit = getTestKitForStoryboard(storyboardId, runOptions);
       const result = await runStoryboard(
         resolved.resolvedUrl,
         sb,
         withSdkSafeTransport(withHostedStoryboardRunOptions({
+          ...(declaredTestKit && { test_kit: declaredTestKit }),
           ...(authOption && { auth: authOption }),
         }, runTarget, authProbeTask)),
       );
@@ -5525,11 +5532,14 @@ export function createMemberToolHandlers(
 
     try {
       const authProbeTask = await inferHostedAuthProbeTask(resolved.resolvedUrl, authOption, runTarget);
+      // adcp#6735 — same declared-kit pre-population as run_storyboard.
+      const declaredStepTestKit = getTestKitForStoryboard(storyboardId, runOptions);
       const result: StoryboardStepResult = await runStoryboardStep(
         resolved.resolvedUrl,
         sb,
         resolvedStepId,
         withSdkSafeTransport(withHostedStoryboardRunOptions({
+          ...(declaredStepTestKit && { test_kit: declaredStepTestKit }),
           context,
           ...(authOption && { auth: authOption }),
         }, runTarget, authProbeTask)),

server/src/conformance/run-storyboard-via-ws.ts

@@ -17,7 +17,7 @@ import { AgentClient } from '@adcp/sdk';
 import { runStoryboard } from '@adcp/sdk/testing';
 import type { StoryboardResult, StoryboardRunOptions } from '@adcp/sdk/testing';
 import { conformanceSessions } from './session-store.js';
-import { getStoryboard } from '../services/storyboards.js';
+import { getStoryboard, getTestKitForStoryboard } from '../services/storyboards.js';
 import { createLogger } from '../logger.js';
 import { hostedComplianceTarget, withHostedStoryboardRunOptions } from '../services/hosted-compliance-version.js';
 
@@ -99,7 +99,13 @@ export async function runStoryboardViaConformanceSocket(
   // `getOrCreateClient` but isn't on the public type. Cast through the
   // narrow runOptions shape rather than `as any` so unrelated typos still
   // get caught.
+  // adcp#6735 — pre-populate the storyboard-declared test kit so
+  // `from_test_kit` steps run with the credential the storyboard was
+  // authored against (the bearer substitution in `withHostedAuthTestKit`
+  // no-ops when the kit already carries auth).
+  const declaredTestKit = getTestKitForStoryboard(storyboardId);
   const runOptions = withHostedStoryboardRunOptions({
+    ...(declaredTestKit && { test_kit: declaredTestKit }),
     _client: agentClient,
     test_session_id: testSessionId,
     timeout_ms: options.timeoutMs ?? 60_000,

server/src/routes/registry-api.ts

@@ -8097,14 +8097,17 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { router: R
   router.get("/storyboards/:id", async (req, res) => {
     try {
       const runTarget = targetFromRequestValue(req.query.compliance_target);
+      const storyboardOptions = runTarget === complianceTarget
+        ? complianceOptions
+        : hostedComplianceOptions(runTarget);
       const storyboard = runTarget === complianceTarget
         ? getStoryboard(req.params.id)
-        : getComplianceStoryboardById(req.params.id, hostedComplianceOptions(runTarget));
+        : getComplianceStoryboardById(req.params.id, storyboardOptions);
       if (!storyboard) {
         return res.status(404).json({ error: "Storyboard not found" });
       }
 
-      const testKit = getTestKitForStoryboard(req.params.id);
+      const testKit = getTestKitForStoryboard(req.params.id, storyboardOptions);
       res.json({
         requested_compliance_target: runTarget.requested,
         adcp_version: runTarget.version,
@@ -8346,11 +8349,17 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { router: R
           return res.status(400).json({ error: "context too large" });
         }
 
+        // adcp#6735 — pre-populate the storyboard-declared test kit so
+        // `from_test_kit` steps run with the credential the storyboard was
+        // authored against; `withHostedAuthTestKit`'s `!nextAuth.api_key`
+        // guard then no-ops the run-auth bearer substitution.
+        const declaredTestKit = getTestKitForStoryboard(storyboard.id, runOptions);
         const result = await runStoryboardStep(
           agentUrl,
           storyboard,
           req.params.stepId,
           withSdkSafeTransport(withHostedStoryboardRunOptions({
+            ...(declaredTestKit && { test_kit: declaredTestKit }),
             ...(sdkAuth && { auth: sdkAuth }),
             ...(context && { context }),
           }, runTarget, authProbeTask)),
@@ -8462,7 +8471,8 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { router: R
           });
         }
         const runTarget = runTargetSelection.target;
-        const storyboard = getComplianceStoryboardById(req.params.storyboardId, hostedComplianceOptions(runTarget));
+        const storyboardOptions = hostedComplianceOptions(runTarget);
+        const storyboard = getComplianceStoryboardById(req.params.storyboardId, storyboardOptions);
         if (!storyboard) {
           return res.status(404).json({ error: "Storyboard not found" });
         }
@@ -8584,7 +8594,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { router: R
           }),
         }));
 
-        const testKit = getTestKitForStoryboard(req.params.storyboardId);
+        const testKit = getTestKitForStoryboard(req.params.storyboardId, storyboardOptions);
 
         res.json({
           storyboard: {

server/src/services/storyboards.ts

@@ -74,26 +74,33 @@ export interface StoryboardSummary {
   step_count: number;
 }
 
-const testKits = new Map<string, TestKit>();
+type StoryboardComplianceOptions = ReturnType<typeof hostedComplianceOptions>;
 
-function findTestKitsDir(): string | null {
+const testKitsByDir = new Map<string, Map<string, TestKit>>();
+
+function findTestKitsDir(options: StoryboardComplianceOptions): string | null {
   try {
-    const dir = join(getComplianceCacheDir(complianceOptions), 'test-kits');
+    const dir = join(getComplianceCacheDir(options), 'test-kits');
     return existsSync(dir) ? dir : null;
   } catch (err) {
     logger.info({ err }, 'Compliance cache not resolvable; test-kit features disabled');
     return null;
   }
 }
 
-function loadTestKits(): void {
+function loadTestKits(options: StoryboardComplianceOptions): Map<string, TestKit> {
   try {
-    const dir = findTestKitsDir();
+    const dir = findTestKitsDir(options);
     if (!dir) {
       logger.info('Test kits directory not found; test kit features disabled');
-      return;
+      return new Map();
     }
 
+    const cached = testKitsByDir.get(dir);
+    if (cached) return cached;
+
+    const testKits = new Map<string, TestKit>();
+
     const files = readdirSync(dir).filter((f) => f.endsWith('.yaml'));
     for (const file of files) {
       try {
@@ -108,14 +115,17 @@ function loadTestKits(): void {
       }
     }
 
+    testKitsByDir.set(dir, testKits);
     logger.info({ testKits: testKits.size }, 'Test kits loaded');
+    return testKits;
   } catch (err) {
     logger.error({ err }, 'Failed to load test kits');
+    return new Map();
   }
 }
 
 // Load test kits on import
-loadTestKits();
+loadTestKits(complianceOptions);
 
 // ── Public API ──────────────────────────────────────────────────
 
@@ -198,17 +208,23 @@ export function getStoryboardIdsForVersion(adcpVersion: string): string[] {
   return getStoryboardsForVersion(adcpVersion).map((sb) => sb.id);
 }
 
-export function getTestKit(id: string): TestKit | undefined {
-  return testKits.get(id);
+export function getTestKit(
+  id: string,
+  options: StoryboardComplianceOptions = complianceOptions,
+): TestKit | undefined {
+  return loadTestKits(options).get(id);
 }
 
-export function getTestKitForStoryboard(storyboardId: string): TestKit | undefined {
-  const sb = getComplianceStoryboardById(storyboardId, complianceOptions);
+export function getTestKitForStoryboard(
+  storyboardId: string,
+  options: StoryboardComplianceOptions = complianceOptions,
+): TestKit | undefined {
+  const sb = getComplianceStoryboardById(storyboardId, options);
   if (!sb?.prerequisites?.test_kit) return undefined;
 
   // test_kit is like "test-kits/acme-outdoor.yaml" — extract the id
   const filename = sb.prerequisites.test_kit.replace(/^test-kits\//, '').replace(/\.yaml$/, '');
   // Convert filename to id format (acme-outdoor → acme_outdoor)
   const kitId = filename.replace(/-/g, '_');
-  return testKits.get(kitId);
+  return getTestKit(kitId, options);
 }

server/tests/unit/storyboards.test.ts

@@ -156,6 +156,27 @@ describe('getTestKitForStoryboard', () => {
     }
     throw new Error('Expected at least one storyboard to declare prerequisites.test_kit');
   });
+
+  // adcp#6735 — the mode-gate storyboard's declared kit must resolve WITH its
+  // auth block: the hosted call sites pre-populate options.test_kit from this
+  // resolver, and withHostedAuthTestKit's !nextAuth.api_key guard relies on
+  // the kit carrying the credential to no-op the run-auth bearer substitution.
+  it('resolves the mode-gate kit with its live credential', () => {
+    const kit = getTestKitForStoryboard('comply_controller_mode_gate');
+    expect(kit).toBeDefined();
+    const auth = kit!.auth as { api_key?: string } | undefined;
+    expect(auth?.api_key).toBeTruthy();
+  });
+
+  it('resolves a storyboard kit from the selected compliance target', () => {
+    const target = hostedComplianceTarget('3.1');
+    const options = hostedComplianceOptions(target);
+    const kit = getTestKitForStoryboard('media_buy_seller/canonical_formats', options);
+
+    expect(kit?.id).toBe('acme_outdoor');
+    const auth = kit!.auth as { api_key?: string } | undefined;
+    expect(auth?.api_key).toBeTruthy();
+  });
 });
 
 describe('wrapper contract', () => {
@@ -322,6 +343,23 @@ describe('wrapper contract', () => {
     expect(options.test_kit?.auth?.probe_task).toBe('list_creatives');
   });
 
+  // adcp#6735 — the load-bearing guard for declared-kit pre-population: when
+  // the call site placed the storyboard-declared kit (with its own api_key)
+  // into options.test_kit, the run-auth bearer must NOT substitute over it.
+  it('does not substitute the run bearer over a pre-populated declared kit credential', () => {
+    const declaredKit = getTestKitForStoryboard('comply_controller_mode_gate');
+    expect(declaredKit).toBeDefined();
+    const options = withHostedAuthTestKit({
+      test_kit: declaredKit,
+      auth: { type: 'bearer', token: 'seller-run-bearer' },
+    });
+
+    const declaredKey = (declaredKit!.auth as { api_key?: string }).api_key;
+    expect(declaredKey).toBeTruthy();
+    expect(options.test_kit?.auth?.api_key).toBe(declaredKey);
+    expect(options.test_kit?.auth?.api_key).not.toBe('seller-run-bearer');
+  });
+
   it('threads hosted static fixture auth into the runtime test kit when no operator auth is supplied', () => {
     const apiKey = hostedStaticApiKeyForProfile({
       tools: ['get_adcp_capabilities', 'get_products'],