diff --git a/docs/building/operating/storyboard-troubleshooting.mdx b/docs/building/operating/storyboard-troubleshooting.mdx index cd0b6e8fe6..a65230518d 100644 --- a/docs/building/operating/storyboard-troubleshooting.mdx +++ b/docs/building/operating/storyboard-troubleshooting.mdx @@ -110,6 +110,8 @@ The `mechanism_required` phase found no contribution to `auth_mechanism_verified **The `--auth TOKEN` distinction:** The `--auth TOKEN` flag you pass to the runner is the runner's own session credential — it authorizes the runner's own requests to your agent. It is entirely separate from `test_kit.auth.api_key` or `test_kit.auth.basic`, which are the specific credentials the static-credential phases send during positive and invalid-credential probes. These are not the same token and are not interchangeable. +**`prerequisites.test_kit` is a loading directive, not decoration:** when a storyboard declares `prerequisites.test_kit` (for example `comply_controller_mode_gate` declares `test-kits/acme-outdoor-live.yaml`), the runner loads that kit from the active compliance cache and its credentials flow into the storyboard's `from_test_kit` and `$test_kit.*` references without any CLI flag. An explicitly supplied kit (`options.test_kit` / hosted-run configuration) still wins. A step whose `auth.from_test_kit` resolves no credential fails with an explicit configuration error rather than sending an unauthenticated probe — a probe with no credential cannot test a credential-keyed contract. + **Fix for Bearer API key agents:** Every default AdCP brand test kit declares its probe API key under `auth.api_key` using the `demo--v1` naming convention. The default test kit (`acme-outdoor`) uses `demo-acme-outdoor-v1`. Configure your agent to accept the kit's probe key as a valid compliance-testing credential alongside your production key. The `demo--` prefix is the AdCP conformance handle — accept any Bearer token matching the prefix for the kits you run against (the suffix can rotate across spec versions, the prefix stays stable): ```typescript diff --git a/server/src/addie/config-version.ts b/server/src/addie/config-version.ts index 5dff406710..49d012e20b 100644 --- a/server/src/addie/config-version.ts +++ b/server/src/addie/config-version.ts @@ -30,7 +30,7 @@ import { loadRules, loadResponseStyle } from './rules/index.js'; * Format: YYYY.MM.N where N is incremented for multiple changes in a month * Example: 2025.01.1, 2025.01.2, 2025.02.1 */ -export const CODE_VERSION = '2026.08.12'; +export const CODE_VERSION = '2026.08.13'; // Types export interface ConfigVersion { diff --git a/server/src/addie/mcp/member-tools.ts b/server/src/addie/mcp/member-tools.ts index 1a19f5ffc8..85e2cb53c3 100644 --- a/server/src/addie/mcp/member-tools.ts +++ b/server/src/addie/mcp/member-tools.ts @@ -72,6 +72,7 @@ import { } from '@adcp/sdk/testing'; import { AuthenticationRequiredError } from '@adcp/sdk'; import { renderAllHintFixPlans } from '../services/storyboard-fix-plan.js'; +import { getTestKitForStoryboard } from '../../services/storyboards.js'; import { hostedComplianceTarget, hostedComplianceOptions, @@ -5308,10 +5309,16 @@ export function createMemberToolHandlers( try { const authProbeTask = await inferHostedAuthProbeTask(resolved.resolvedUrl, authOption, runTarget); + // adcp#6735 — pre-populate the storyboard-declared test kit so + // `from_test_kit` steps run with the credential the storyboard was + // authored against; the run-auth bearer substitution no-ops when the + // kit already carries auth. + const declaredTestKit = getTestKitForStoryboard(storyboardId, runOptions); const result = await runStoryboard( resolved.resolvedUrl, sb, withSdkSafeTransport(withHostedStoryboardRunOptions({ + ...(declaredTestKit && { test_kit: declaredTestKit }), ...(authOption && { auth: authOption }), }, runTarget, authProbeTask)), ); @@ -5525,11 +5532,14 @@ export function createMemberToolHandlers( try { const authProbeTask = await inferHostedAuthProbeTask(resolved.resolvedUrl, authOption, runTarget); + // adcp#6735 — same declared-kit pre-population as run_storyboard. + const declaredStepTestKit = getTestKitForStoryboard(storyboardId, runOptions); const result: StoryboardStepResult = await runStoryboardStep( resolved.resolvedUrl, sb, resolvedStepId, withSdkSafeTransport(withHostedStoryboardRunOptions({ + ...(declaredStepTestKit && { test_kit: declaredStepTestKit }), context, ...(authOption && { auth: authOption }), }, runTarget, authProbeTask)), diff --git a/server/src/conformance/run-storyboard-via-ws.ts b/server/src/conformance/run-storyboard-via-ws.ts index 01b27c1f80..d18304b425 100644 --- a/server/src/conformance/run-storyboard-via-ws.ts +++ b/server/src/conformance/run-storyboard-via-ws.ts @@ -17,7 +17,7 @@ import { AgentClient } from '@adcp/sdk'; import { runStoryboard } from '@adcp/sdk/testing'; import type { StoryboardResult, StoryboardRunOptions } from '@adcp/sdk/testing'; import { conformanceSessions } from './session-store.js'; -import { getStoryboard } from '../services/storyboards.js'; +import { getStoryboard, getTestKitForStoryboard } from '../services/storyboards.js'; import { createLogger } from '../logger.js'; import { hostedComplianceTarget, withHostedStoryboardRunOptions } from '../services/hosted-compliance-version.js'; @@ -99,7 +99,13 @@ export async function runStoryboardViaConformanceSocket( // `getOrCreateClient` but isn't on the public type. Cast through the // narrow runOptions shape rather than `as any` so unrelated typos still // get caught. + // adcp#6735 — pre-populate the storyboard-declared test kit so + // `from_test_kit` steps run with the credential the storyboard was + // authored against (the bearer substitution in `withHostedAuthTestKit` + // no-ops when the kit already carries auth). + const declaredTestKit = getTestKitForStoryboard(storyboardId); const runOptions = withHostedStoryboardRunOptions({ + ...(declaredTestKit && { test_kit: declaredTestKit }), _client: agentClient, test_session_id: testSessionId, timeout_ms: options.timeoutMs ?? 60_000, diff --git a/server/src/routes/registry-api.ts b/server/src/routes/registry-api.ts index 3b9eee8989..85ffdc7ca9 100644 --- a/server/src/routes/registry-api.ts +++ b/server/src/routes/registry-api.ts @@ -8097,14 +8097,17 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { router: R router.get("/storyboards/:id", async (req, res) => { try { const runTarget = targetFromRequestValue(req.query.compliance_target); + const storyboardOptions = runTarget === complianceTarget + ? complianceOptions + : hostedComplianceOptions(runTarget); const storyboard = runTarget === complianceTarget ? getStoryboard(req.params.id) - : getComplianceStoryboardById(req.params.id, hostedComplianceOptions(runTarget)); + : getComplianceStoryboardById(req.params.id, storyboardOptions); if (!storyboard) { return res.status(404).json({ error: "Storyboard not found" }); } - const testKit = getTestKitForStoryboard(req.params.id); + const testKit = getTestKitForStoryboard(req.params.id, storyboardOptions); res.json({ requested_compliance_target: runTarget.requested, adcp_version: runTarget.version, @@ -8346,11 +8349,17 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { router: R return res.status(400).json({ error: "context too large" }); } + // adcp#6735 — pre-populate the storyboard-declared test kit so + // `from_test_kit` steps run with the credential the storyboard was + // authored against; `withHostedAuthTestKit`'s `!nextAuth.api_key` + // guard then no-ops the run-auth bearer substitution. + const declaredTestKit = getTestKitForStoryboard(storyboard.id, runOptions); const result = await runStoryboardStep( agentUrl, storyboard, req.params.stepId, withSdkSafeTransport(withHostedStoryboardRunOptions({ + ...(declaredTestKit && { test_kit: declaredTestKit }), ...(sdkAuth && { auth: sdkAuth }), ...(context && { context }), }, runTarget, authProbeTask)), @@ -8462,7 +8471,8 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { router: R }); } const runTarget = runTargetSelection.target; - const storyboard = getComplianceStoryboardById(req.params.storyboardId, hostedComplianceOptions(runTarget)); + const storyboardOptions = hostedComplianceOptions(runTarget); + const storyboard = getComplianceStoryboardById(req.params.storyboardId, storyboardOptions); if (!storyboard) { return res.status(404).json({ error: "Storyboard not found" }); } @@ -8584,7 +8594,7 @@ export function createRegistryApiRouters(config: RegistryApiConfig): { router: R }), })); - const testKit = getTestKitForStoryboard(req.params.storyboardId); + const testKit = getTestKitForStoryboard(req.params.storyboardId, storyboardOptions); res.json({ storyboard: { diff --git a/server/src/services/storyboards.ts b/server/src/services/storyboards.ts index ed7684d511..51cb07e3e9 100644 --- a/server/src/services/storyboards.ts +++ b/server/src/services/storyboards.ts @@ -74,11 +74,13 @@ export interface StoryboardSummary { step_count: number; } -const testKits = new Map(); +type StoryboardComplianceOptions = ReturnType; -function findTestKitsDir(): string | null { +const testKitsByDir = new Map>(); + +function findTestKitsDir(options: StoryboardComplianceOptions): string | null { try { - const dir = join(getComplianceCacheDir(complianceOptions), 'test-kits'); + const dir = join(getComplianceCacheDir(options), 'test-kits'); return existsSync(dir) ? dir : null; } catch (err) { logger.info({ err }, 'Compliance cache not resolvable; test-kit features disabled'); @@ -86,14 +88,19 @@ function findTestKitsDir(): string | null { } } -function loadTestKits(): void { +function loadTestKits(options: StoryboardComplianceOptions): Map { try { - const dir = findTestKitsDir(); + const dir = findTestKitsDir(options); if (!dir) { logger.info('Test kits directory not found; test kit features disabled'); - return; + return new Map(); } + const cached = testKitsByDir.get(dir); + if (cached) return cached; + + const testKits = new Map(); + const files = readdirSync(dir).filter((f) => f.endsWith('.yaml')); for (const file of files) { try { @@ -108,14 +115,17 @@ function loadTestKits(): void { } } + testKitsByDir.set(dir, testKits); logger.info({ testKits: testKits.size }, 'Test kits loaded'); + return testKits; } catch (err) { logger.error({ err }, 'Failed to load test kits'); + return new Map(); } } // Load test kits on import -loadTestKits(); +loadTestKits(complianceOptions); // ── Public API ────────────────────────────────────────────────── @@ -198,17 +208,23 @@ export function getStoryboardIdsForVersion(adcpVersion: string): string[] { return getStoryboardsForVersion(adcpVersion).map((sb) => sb.id); } -export function getTestKit(id: string): TestKit | undefined { - return testKits.get(id); +export function getTestKit( + id: string, + options: StoryboardComplianceOptions = complianceOptions, +): TestKit | undefined { + return loadTestKits(options).get(id); } -export function getTestKitForStoryboard(storyboardId: string): TestKit | undefined { - const sb = getComplianceStoryboardById(storyboardId, complianceOptions); +export function getTestKitForStoryboard( + storyboardId: string, + options: StoryboardComplianceOptions = complianceOptions, +): TestKit | undefined { + const sb = getComplianceStoryboardById(storyboardId, options); if (!sb?.prerequisites?.test_kit) return undefined; // test_kit is like "test-kits/acme-outdoor.yaml" — extract the id const filename = sb.prerequisites.test_kit.replace(/^test-kits\//, '').replace(/\.yaml$/, ''); // Convert filename to id format (acme-outdoor → acme_outdoor) const kitId = filename.replace(/-/g, '_'); - return testKits.get(kitId); + return getTestKit(kitId, options); } diff --git a/server/tests/unit/storyboards.test.ts b/server/tests/unit/storyboards.test.ts index f69ce134e9..65e7ce70b0 100644 --- a/server/tests/unit/storyboards.test.ts +++ b/server/tests/unit/storyboards.test.ts @@ -156,6 +156,27 @@ describe('getTestKitForStoryboard', () => { } throw new Error('Expected at least one storyboard to declare prerequisites.test_kit'); }); + + // adcp#6735 — the mode-gate storyboard's declared kit must resolve WITH its + // auth block: the hosted call sites pre-populate options.test_kit from this + // resolver, and withHostedAuthTestKit's !nextAuth.api_key guard relies on + // the kit carrying the credential to no-op the run-auth bearer substitution. + it('resolves the mode-gate kit with its live credential', () => { + const kit = getTestKitForStoryboard('comply_controller_mode_gate'); + expect(kit).toBeDefined(); + const auth = kit!.auth as { api_key?: string } | undefined; + expect(auth?.api_key).toBeTruthy(); + }); + + it('resolves a storyboard kit from the selected compliance target', () => { + const target = hostedComplianceTarget('3.1'); + const options = hostedComplianceOptions(target); + const kit = getTestKitForStoryboard('media_buy_seller/canonical_formats', options); + + expect(kit?.id).toBe('acme_outdoor'); + const auth = kit!.auth as { api_key?: string } | undefined; + expect(auth?.api_key).toBeTruthy(); + }); }); describe('wrapper contract', () => { @@ -322,6 +343,23 @@ describe('wrapper contract', () => { expect(options.test_kit?.auth?.probe_task).toBe('list_creatives'); }); + // adcp#6735 — the load-bearing guard for declared-kit pre-population: when + // the call site placed the storyboard-declared kit (with its own api_key) + // into options.test_kit, the run-auth bearer must NOT substitute over it. + it('does not substitute the run bearer over a pre-populated declared kit credential', () => { + const declaredKit = getTestKitForStoryboard('comply_controller_mode_gate'); + expect(declaredKit).toBeDefined(); + const options = withHostedAuthTestKit({ + test_kit: declaredKit, + auth: { type: 'bearer', token: 'seller-run-bearer' }, + }); + + const declaredKey = (declaredKit!.auth as { api_key?: string }).api_key; + expect(declaredKey).toBeTruthy(); + expect(options.test_kit?.auth?.api_key).toBe(declaredKey); + expect(options.test_kit?.auth?.api_key).not.toBe('seller-run-bearer'); + }); + it('threads hosted static fixture auth into the runtime test kit when no operator auth is supplied', () => { const apiKey = hostedStaticApiKeyForProfile({ tools: ['get_adcp_capabilities', 'get_products'],