From fb90db23081424a94bc11f68cf19c9b0a6451111 Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Wed, 29 Jul 2026 22:38:23 +0530 Subject: [PATCH 01/16] test: add isolated agent verification runtime --- tests/fixtures/contracts/agent-workflow.ts | 22 ++++++++ tests/support/agent-test-runtime.ts | 56 +++++++++++++++++++ tests/unit/support/agent-test-runtime.test.ts | 42 ++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 tests/fixtures/contracts/agent-workflow.ts create mode 100644 tests/support/agent-test-runtime.ts create mode 100644 tests/unit/support/agent-test-runtime.test.ts diff --git a/tests/fixtures/contracts/agent-workflow.ts b/tests/fixtures/contracts/agent-workflow.ts new file mode 100644 index 0000000..05d7b7d --- /dev/null +++ b/tests/fixtures/contracts/agent-workflow.ts @@ -0,0 +1,22 @@ +export const AGENT_WORKFLOW_FIXTURES = { + sessions: { + alpha: 'session-alpha', + beta: 'session-beta', + }, + agents: { + codex: 'Codex', + claudeCode: 'Claude Code', + }, + workspace: 'relay-verification', + titles: { + alphaOpen: 'Alpha open capture', + alphaCompleted: 'Alpha completed capture', + alphaArchived: 'Alpha archived capture', + betaOpen: 'Beta open capture', + duplicate: 'Duplicate candidate capture', + }, + malformed: { + session: 'bad session id!', + taskId: 'not-a-valid-task-id', + }, +} as const; diff --git a/tests/support/agent-test-runtime.ts b/tests/support/agent-test-runtime.ts new file mode 100644 index 0000000..2731f62 --- /dev/null +++ b/tests/support/agent-test-runtime.ts @@ -0,0 +1,56 @@ +import { mkdir, mkdtemp, rm } from 'node:fs/promises'; +import { dirname, isAbsolute, relative, resolve, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export interface AgentTestRuntimeOptions { + readonly environment?: NodeJS.ProcessEnv; +} + +export interface AgentTestRuntime { + readonly checkoutPath: string; + readonly databasePath: string; + createWorkingDirectory(name: string): Promise; + environment(overrides?: NodeJS.ProcessEnv): NodeJS.ProcessEnv; + close(): Promise; +} + +const checkoutPath = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); + +export async function createAgentTestRuntime( + options: AgentTestRuntimeOptions = {}, +): Promise { + const root = await mkdtemp(join(checkoutPath, 'tmp', 'relay-agent-verification-')); + const dataDirectory = join(root, 'data'); + const workingDirectoryRoot = join(root, 'cwd'); + await Promise.all([mkdir(dataDirectory), mkdir(workingDirectoryRoot)]); + + const databasePath = join(dataDirectory, 'relay.db'); + let closed = false; + + return { + checkoutPath, + databasePath, + async createWorkingDirectory(name: string): Promise { + const workingDirectory = resolve(workingDirectoryRoot, name); + const escape = relative(workingDirectoryRoot, workingDirectory); + if (escape.startsWith('..') || isAbsolute(escape)) { + throw new Error(`Working directory escapes the disposable root: ${name}`); + } + await mkdir(workingDirectory, { recursive: true }); + return workingDirectory; + }, + environment(overrides: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...process.env, + ...options.environment, + ...overrides, + RELAY_DB_PATH: databasePath, + }; + }, + async close(): Promise { + if (closed) return; + closed = true; + await rm(root, { recursive: true, force: true }); + }, + }; +} diff --git a/tests/unit/support/agent-test-runtime.test.ts b/tests/unit/support/agent-test-runtime.test.ts new file mode 100644 index 0000000..3eb87b8 --- /dev/null +++ b/tests/unit/support/agent-test-runtime.test.ts @@ -0,0 +1,42 @@ +import { stat } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { dirname, isAbsolute } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { createAgentTestRuntime } from '../../support/agent-test-runtime.js'; + +describe('createAgentTestRuntime', () => { + it('creates an isolated absolute database path and arbitrary working directories', async () => { + const runtime = await createAgentTestRuntime(); + try { + expect(runtime.databasePath).toMatch(/relay\.db$/); + expect(isAbsolute(runtime.databasePath)).toBe(true); + expect(runtime.databasePath).not.toContain(homedir()); + + const cwd = await runtime.createWorkingDirectory('nested/client'); + expect(isAbsolute(cwd)).toBe(true); + expect(cwd).not.toBe(runtime.checkoutPath); + + const environment = runtime.environment({ RELAY_TEST_MARKER: 'isolated' }); + expect(environment.RELAY_DB_PATH).toBe(runtime.databasePath); + expect(environment.RELAY_TEST_MARKER).toBe('isolated'); + } finally { + await runtime.close(); + } + }); + + it('removes the disposable directory including SQLite sidecars and closes idempotently', async () => { + const runtime = await createAgentTestRuntime(); + const root = dirname(dirname(runtime.databasePath)); + await import('node:fs/promises').then(({ writeFile }) => + Promise.all([ + writeFile(`${runtime.databasePath}-wal`, 'test'), + writeFile(`${runtime.databasePath}-shm`, 'test'), + ]), + ); + + await runtime.close(); + await runtime.close(); + + await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' }); + }); +}); From 0c74bdc1e5eff75441cf6d402311547fa89537d7 Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Wed, 29 Jul 2026 22:41:16 +0530 Subject: [PATCH 02/16] test: add built MCP and CLI clients --- tests/support/cli-test-process.ts | 60 +++++++++++++++ .../support/external-contract-normalizers.ts | 75 +++++++++++++++++++ tests/support/mcp-test-client.ts | 70 +++++++++++++++++ tests/unit/support/cli-test-process.test.ts | 36 +++++++++ tests/unit/support/mcp-test-client.test.ts | 41 ++++++++++ 5 files changed, 282 insertions(+) create mode 100644 tests/support/cli-test-process.ts create mode 100644 tests/support/external-contract-normalizers.ts create mode 100644 tests/support/mcp-test-client.ts create mode 100644 tests/unit/support/cli-test-process.test.ts create mode 100644 tests/unit/support/mcp-test-client.test.ts diff --git a/tests/support/cli-test-process.ts b/tests/support/cli-test-process.ts new file mode 100644 index 0000000..84862e3 --- /dev/null +++ b/tests/support/cli-test-process.ts @@ -0,0 +1,60 @@ +import { spawn } from 'node:child_process'; +import { join } from 'node:path'; +import type { AgentTestRuntime } from './agent-test-runtime.js'; + +export interface CliProcessOptions { + readonly cwd?: string; + readonly environment?: NodeJS.ProcessEnv; +} + +export interface CliProcessResult { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; + readonly json: unknown; +} + +export function runRelayCli( + runtime: AgentTestRuntime, + args: readonly string[], + options: CliProcessOptions = {}, +): Promise { + const command = join(runtime.checkoutPath, 'dist', 'cli', 'main.js'); + const child = spawn(process.execPath, [command, ...args], { + cwd: options.cwd ?? runtime.checkoutPath, + env: runtime.environment(options.environment), + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk: Buffer | string) => { + stdout += chunk.toString(); + }); + child.stderr.on('data', (chunk: Buffer | string) => { + stderr += chunk.toString(); + }); + + return new Promise((resolve, reject) => { + child.once('error', reject); + child.once('close', (code) => { + try { + resolve({ + exitCode: code ?? 1, + stdout, + stderr, + json: parseSingleJson(stdout), + }); + } catch (error) { + reject(error); + } + }); + }); +} + +function parseSingleJson(stdout: string): unknown { + const trimmed = stdout.trim(); + if (trimmed.length === 0) throw new Error('Relay CLI produced no JSON output.'); + return JSON.parse(trimmed) as unknown; +} diff --git a/tests/support/external-contract-normalizers.ts b/tests/support/external-contract-normalizers.ts new file mode 100644 index 0000000..ed60e2c --- /dev/null +++ b/tests/support/external-contract-normalizers.ts @@ -0,0 +1,75 @@ +export interface ExternalError { + readonly code: string; + readonly message: string; + readonly details?: unknown; +} + +export interface ExternalOperationResult { + readonly schemaVersion: number; + readonly data: unknown; + readonly warnings: readonly unknown[]; +} + +export function normalizeCliSuccess(value: unknown): ExternalOperationResult { + const envelope = record(value, 'CLI result'); + return { + schemaVersion: number(envelope.schemaVersion), + data: envelope.data, + warnings: array(envelope.warnings), + }; +} + +export function normalizeMcpSuccess(value: unknown): ExternalOperationResult { + const envelope = record(value, 'MCP result'); + const structuredContent = record(envelope.structuredContent, 'MCP structured result'); + return { + schemaVersion: number(structuredContent.schemaVersion), + data: structuredContent.data, + warnings: array(structuredContent.warnings), + }; +} + +export function normalizeCliError(value: unknown): ExternalError { + return normalizeError(record(value, 'CLI error').error); +} + +export function normalizeMcpError(value: unknown): ExternalError { + if (value instanceof Error) { + return { code: 'MCP_PROTOCOL_ERROR', message: value.message }; + } + const envelope = record(value, 'MCP error'); + const structuredContent = envelope.structuredContent; + return normalizeError(record(structuredContent, 'MCP structured error').error); +} + +function normalizeError(value: unknown): ExternalError { + const error = record(value, 'error'); + return { + code: string(error.code), + message: string(error.message), + ...(error.details === undefined ? {} : { details: error.details }), + }; +} + +function record(value: unknown, label: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`${label} must be an object.`); + } + return value as Record; +} + +function string(value: unknown): string { + if (typeof value !== 'string') throw new Error('Expected a string contract field.'); + return value; +} + +function number(value: unknown): number { + if (typeof value !== 'number') throw new Error('Expected a numeric schema version.'); + return value; +} + +function array(value: unknown): readonly unknown[] { + if (value === undefined) return []; + if (!Array.isArray(value)) throw new Error('Expected an array contract field.'); + return value; +} diff --git a/tests/support/mcp-test-client.ts b/tests/support/mcp-test-client.ts new file mode 100644 index 0000000..9fa588e --- /dev/null +++ b/tests/support/mcp-test-client.ts @@ -0,0 +1,70 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import { join } from 'node:path'; +import type { AgentTestRuntime } from './agent-test-runtime.js'; + +export interface McpTestClient { + listTools(): Promise; + callTool(name: string, args: Record): Promise; + stderr(): string; + close(): Promise; +} + +export interface McpTestClientOptions { + readonly cwd?: string; + readonly environment?: NodeJS.ProcessEnv; +} + +export async function createMcpTestClient( + runtime: AgentTestRuntime, + options: McpTestClientOptions = {}, +): Promise { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(runtime.checkoutPath, 'dist', 'mcp', 'main.js')], + cwd: options.cwd ?? runtime.checkoutPath, + env: stringEnvironment(runtime.environment(options.environment)), + stderr: 'pipe', + }); + let serverStderr = ''; + transport.stderr?.on('data', (chunk: Buffer | string) => { + serverStderr += chunk.toString(); + }); + + const client = new Client({ name: 'relay-issue-25-test-client', version: '1.0.0' }); + try { + await client.connect(transport); + } catch (error) { + await transport.close().catch(() => undefined); + throw error; + } + + let closed = false; + return { + async listTools() { + const result = await client.listTools(); + return result.tools.map(({ name }) => ({ name })); + }, + async callTool(name, args) { + return client.callTool({ name, arguments: args }); + }, + stderr() { + return serverStderr; + }, + async close() { + if (closed) return; + closed = true; + try { + await client.close(); + } finally { + await transport.close(); + } + }, + }; +} + +function stringEnvironment(environment: NodeJS.ProcessEnv): Record { + return Object.fromEntries( + Object.entries(environment).filter((entry): entry is [string, string] => entry[1] !== undefined), + ); +} diff --git a/tests/unit/support/cli-test-process.test.ts b/tests/unit/support/cli-test-process.test.ts new file mode 100644 index 0000000..0419fa3 --- /dev/null +++ b/tests/unit/support/cli-test-process.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest'; +import { createAgentTestRuntime } from '../../support/agent-test-runtime.js'; +import { runRelayCli } from '../../support/cli-test-process.js'; + +describe('runRelayCli', () => { + it('runs the built CLI from an arbitrary cwd and parses one JSON document', async () => { + const runtime = await createAgentTestRuntime(); + try { + const cwd = await runtime.createWorkingDirectory('cli/nested'); + const result = await runRelayCli(runtime, ['task', 'list', '--output', 'json'], { cwd }); + + expect(result.exitCode).toBe(0); + expect(result.stderr).toBe(''); + expect(result.json).toMatchObject({ schemaVersion: expect.any(Number) }); + } finally { + await runtime.close(); + } + }); + + it('retains stdout, stderr, and a stable non-zero exit code on failure', async () => { + const runtime = await createAgentTestRuntime(); + try { + const result = await runRelayCli(runtime, ['task', 'get', 'missing-id', '--output', 'json']); + + expect(result.exitCode).toBe(3); + expect(result.json).toMatchObject({ + schemaVersion: expect.any(Number), + error: { code: 'NOT_FOUND', message: expect.any(String) }, + }); + expect(result.stdout.trim().split(/\r?\n/)).toHaveLength(1); + expect(result.stderr).toContain('Task was not found.'); + } finally { + await runtime.close(); + } + }); +}); diff --git a/tests/unit/support/mcp-test-client.test.ts b/tests/unit/support/mcp-test-client.test.ts new file mode 100644 index 0000000..e36c196 --- /dev/null +++ b/tests/unit/support/mcp-test-client.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest'; +import { createAgentTestRuntime } from '../../support/agent-test-runtime.js'; +import { createMcpTestClient } from '../../support/mcp-test-client.js'; + +describe('createMcpTestClient', () => { + it('discovers Relay tools while keeping server stdout protocol-owned', async () => { + const runtime = await createAgentTestRuntime(); + const client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('mcp/nested'), + }); + try { + const tools = await client.listTools(); + expect(tools.map((tool) => tool.name)).toEqual( + expect.arrayContaining([ + 'relay_health', + 'task_capture', + 'task_list', + 'task_get', + 'task_find_similar', + 'session_captures_list', + ]), + ); + expect(client.stderr()).not.toContain('Content-Length'); + } finally { + await client.close(); + await runtime.close(); + } + }); + + it('surfaces unknown-tool protocol failures and still closes the child', async () => { + const runtime = await createAgentTestRuntime(); + const client = await createMcpTestClient(runtime); + try { + await expect(client.callTool('unknown_tool', {})).resolves.toMatchObject({ isError: true }); + expect(client.stderr()).not.toMatch(/SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\/i); + } finally { + await client.close(); + await runtime.close(); + } + }); +}); From eef9b1b59eeebd0aaf6c472507fc86534b57eb57 Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Wed, 29 Jul 2026 22:44:22 +0530 Subject: [PATCH 03/16] test: verify MCP and CLI workflow parity --- tests/integration/agent-workflow-e2e.test.ts | 138 +++++++++++++ tests/integration/mcp-cli-parity.test.ts | 204 +++++++++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 tests/integration/agent-workflow-e2e.test.ts diff --git a/tests/integration/agent-workflow-e2e.test.ts b/tests/integration/agent-workflow-e2e.test.ts new file mode 100644 index 0000000..3c4ed87 --- /dev/null +++ b/tests/integration/agent-workflow-e2e.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from 'vitest'; +import { createAgentTestRuntime } from '../support/agent-test-runtime.js'; +import { runRelayCli } from '../support/cli-test-process.js'; +import { createMcpTestClient } from '../support/mcp-test-client.js'; +import { + normalizeCliSuccess, + normalizeMcpSuccess, +} from '../support/external-contract-normalizers.js'; + +describe('built agent workflow end to end', () => { + it('isolates session review and includes open, completed, and archived captures', async () => { + const runtime = await createAgentTestRuntime(); + const client = await createMcpTestClient(runtime); + try { + const capture = async (title: string, sessionId: string) => + normalizeMcpSuccess( + await client.callTool('task_capture', { + title, + createdByName: 'Codex', + sessionId, + workspace: 'relay-verification', + }), + ); + const open = await capture('Alpha open capture', 'session-alpha'); + const completed = await capture('Alpha completed capture', 'session-alpha'); + const archived = await capture('Alpha archived capture', 'session-alpha'); + const beta = await runRelayCli(runtime, [ + 'task', + 'capture', + '--title', + 'Beta open capture', + '--agent', + 'Claude Code', + '--session', + 'session-beta', + '--output', + 'json', + ]); + expect(beta.exitCode).toBe(0); + + const completedId = String((completed.data as { task: { id: string } }).task.id); + const archivedId = String((archived.data as { task: { id: string } }).task.id); + await client.callTool('task_triage', { taskId: completedId, target: 'ACTIVE' }); + await client.callTool('task_start', { taskId: completedId }); + await client.callTool('task_triage', { taskId: archivedId, target: 'ACTIVE' }); + await client.callTool('task_start', { taskId: archivedId }); + expect((await client.callTool('task_complete', { taskId: completedId }))).toMatchObject({ + structuredContent: { data: { task: { status: 'DONE' } } }, + }); + await client.callTool('task_complete', { taskId: archivedId }); + expect((await client.callTool('task_archive', { taskId: archivedId }))).toMatchObject({ + structuredContent: { data: { task: { status: 'ARCHIVED' } } }, + }); + + const mcpReview = normalizeMcpSuccess( + await client.callTool('session_captures_list', { + sessionId: 'session-alpha', + limit: 100, + }), + ); + const cliReview = await runRelayCli(runtime, [ + 'session', + 'captures', + '--session', + 'session-alpha', + '--output', + 'json', + ]); + expect(cliReview.exitCode).toBe(0); + expect(normalizeCliSuccess(cliReview.json)).toEqual(mcpReview); + expect(mcpReview.data).toMatchObject({ + sessionId: 'session-alpha', + count: 3, + tasks: [ + { title: 'Alpha open capture', status: 'INBOX' }, + { title: 'Alpha completed capture', status: 'DONE' }, + { title: 'Alpha archived capture', status: 'ARCHIVED' }, + ], + }); + expect(mcpReview.data).not.toMatchObject({ tasks: [expect.objectContaining({ sessionId: 'session-beta' })] }); + } finally { + await client.close(); + await runtime.close(); + } + }); + + it('persists tasks and mutations across short-lived MCP and CLI restarts', async () => { + const runtime = await createAgentTestRuntime(); + let client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('restart/mcp-1'), + }); + let taskId: string; + try { + const capture = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'Restart persistence task', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ); + taskId = String((capture.data as { task: { id: string } }).task.id); + } finally { + await client.close(); + } + + try { + const get = await runRelayCli( + runtime, + ['task', 'get', taskId!, '--output', 'json'], + { cwd: await runtime.createWorkingDirectory('restart/cli-1') }, + ); + expect(get.exitCode).toBe(0); + const edit = await runRelayCli(runtime, [ + 'task', + 'edit', + taskId!, + '--description', + 'survives restart', + '--output', + 'json', + ]); + expect(edit.exitCode).toBe(0); + } finally { + client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('restart/mcp-2'), + }); + try { + const get = normalizeMcpSuccess(await client.callTool('task_get', { taskId: taskId! })); + expect(get.data).toMatchObject({ + task: { id: taskId!, description: 'survives restart', sessionId: 'session-alpha' }, + }); + } finally { + await client.close(); + await runtime.close(); + } + } + }); +}); diff --git a/tests/integration/mcp-cli-parity.test.ts b/tests/integration/mcp-cli-parity.test.ts index c2b2906..91edf02 100644 --- a/tests/integration/mcp-cli-parity.test.ts +++ b/tests/integration/mcp-cli-parity.test.ts @@ -13,6 +13,13 @@ import { InMemoryTaskRepository, } from '../unit/application/tasks/task-test-fixtures.js'; import { connectMcp } from '../unit/interfaces/mcp/mcp-test-utils.js'; +import { createAgentTestRuntime } from '../support/agent-test-runtime.js'; +import { runRelayCli } from '../support/cli-test-process.js'; +import { createMcpTestClient } from '../support/mcp-test-client.js'; +import { + normalizeCliSuccess, + normalizeMcpSuccess, +} from '../support/external-contract-normalizers.js'; const FIXED_NOW = new Date('2026-07-27T10:00:00.000Z'); @@ -244,3 +251,200 @@ describe('MCP and CLI semantic parity', () => { expect(cliConflict.error).toEqual(mcpConflict.error); }); }); + +describe('built MCP and CLI contract parity', () => { + it('captures through MCP and retrieves the identical public task through CLI', async () => { + const runtime = await createAgentTestRuntime(); + const client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('mcp-capture'), + }); + try { + const capture = await client.callTool('task_capture', { + title: 'Built MCP capture', + createdByName: 'Codex', + sessionId: 'session-alpha', + workspace: 'relay-verification', + sourceContext: 'issue-25', + }); + const capturedTask = ( + normalizeMcpSuccess(capture).data as { task: Record } + ).task; + const cli = await runRelayCli( + runtime, + ['task', 'get', String(capturedTask.id), '--output', 'json'], + { cwd: await runtime.createWorkingDirectory('cli-get') }, + ); + + expect(cli.exitCode).toBe(0); + expect(normalizeCliSuccess(cli.json).data).toEqual({ task: capturedTask }); + expect(cli.stderr).toBe(''); + } finally { + await client.close(); + await runtime.close(); + } + }); + + it('captures through CLI and retrieves the identical public task through MCP', async () => { + const runtime = await createAgentTestRuntime(); + const client = await createMcpTestClient(runtime); + try { + const cli = await runRelayCli(runtime, [ + 'task', + 'capture', + '--title', + 'Built CLI capture', + '--agent', + 'Claude Code', + '--session', + 'session-beta', + '--workspace', + 'relay-verification', + '--source-context', + 'issue-25', + '--output', + 'json', + ]); + expect(cli.exitCode).toBe(0); + const capturedTask = ( + normalizeCliSuccess(cli.json).data as { task: Record } + ).task; + const mcp = await client.callTool('task_get', { taskId: capturedTask.id }); + + expect(normalizeMcpSuccess(mcp).data).toEqual({ task: capturedTask }); + expect(cli.stderr).toBe(''); + } finally { + await client.close(); + await runtime.close(); + } + }); + + it('keeps list and get DTO fields and persisted ordering identical across adapters', async () => { + const runtime = await createAgentTestRuntime(); + const client = await createMcpTestClient(runtime); + try { + for (const [title, sessionId] of [ + ['First built task', 'session-alpha'], + ['Second built task', 'session-beta'], + ] as const) { + await client.callTool('task_capture', { + title, + createdByName: 'Codex', + sessionId, + workspace: 'relay-verification', + }); + } + const mcp = normalizeMcpSuccess(await client.callTool('task_list', { limit: 100 })); + const cli = await runRelayCli(runtime, ['task', 'list', '--output', 'json']); + expect(cli.exitCode).toBe(0); + expect(normalizeCliSuccess(cli.json)).toEqual(mcp); + expect((mcp.data as { tasks: readonly unknown[] }).tasks).toHaveLength(2); + } finally { + await client.close(); + await runtime.close(); + } + }); + + it('preserves duplicate candidates, warnings, and match reasons without rejecting capture', async () => { + const runtime = await createAgentTestRuntime(); + const client = await createMcpTestClient(runtime); + try { + const existing = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'Duplicate candidate capture', + createdByName: 'Codex', + sessionId: 'session-alpha', + workspace: 'relay-verification', + }), + ); + const candidate = ( + existing.data as { task: { id: string } } + ).task.id; + const similar = normalizeMcpSuccess( + await client.callTool('task_find_similar', { + title: 'Duplicate candidate capture', + workspace: 'relay-verification', + }), + ); + const duplicate = await runRelayCli(runtime, [ + 'task', + 'capture', + '--title', + 'Duplicate candidate capture', + '--agent', + 'Claude Code', + '--session', + 'session-beta', + '--workspace', + 'relay-verification', + '--output', + 'json', + ]); + + expect(duplicate.exitCode).toBe(0); + expect(duplicate.json).toMatchObject({ + warnings: [ + { + code: 'POSSIBLE_DUPLICATE', + candidates: [{ id: candidate }], + }, + ], + }); + expect(similar.data).toMatchObject({ + candidates: [{ task: { id: candidate }, matchReason: expect.any(String) }], + }); + } finally { + await client.close(); + await runtime.close(); + } + }); + + it('returns identical mutation results when each adapter reads the other adapter’s persisted state', async () => { + const runtime = await createAgentTestRuntime(); + const client = await createMcpTestClient(runtime); + try { + const capture = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'Mutation parity task', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ); + const taskId = String((capture.data as { task: { id: string } }).task.id); + const edit = await runRelayCli(runtime, [ + 'task', + 'edit', + taskId, + '--title', + 'Mutation parity task edited', + '--output', + 'json', + ]); + expect(edit.exitCode).toBe(0); + expect(normalizeMcpSuccess(await client.callTool('task_get', { taskId })).data).toMatchObject({ + task: { id: taskId, title: 'Mutation parity task edited' }, + }); + + const triage = normalizeMcpSuccess( + await client.callTool('task_triage', { taskId, target: 'ACTIVE' }), + ); + expect(triage.data).toMatchObject({ change: { action: 'TRIAGED', from: 'INBOX', to: 'ACTIVE' } }); + const start = await runRelayCli(runtime, ['task', 'start', taskId, '--output', 'json']); + expect(start.exitCode).toBe(0); + expect(normalizeMcpSuccess(await client.callTool('task_get', { taskId })).data).toMatchObject({ + task: { status: 'IN_PROGRESS' }, + }); + const complete = normalizeMcpSuccess( + await client.callTool('task_complete', { taskId }), + ); + expect(complete.data).toMatchObject({ task: { status: 'DONE' }, change: { action: 'COMPLETED' } }); + const archive = await runRelayCli(runtime, ['task', 'archive', taskId, '--output', 'json']); + expect(archive.exitCode).toBe(0); + expect(normalizeMcpSuccess(await client.callTool('task_get', { taskId })).data).toMatchObject({ + task: { status: 'ARCHIVED' }, + }); + } finally { + await client.close(); + await runtime.close(); + } + }); +}); From 1f122adb9af5715281742f5eeeac23d0a477552d Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Wed, 29 Jul 2026 22:46:28 +0530 Subject: [PATCH 04/16] test: verify adapter errors and protocol cleanliness --- tests/integration/mcp-cli-parity.test.ts | 152 +++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/tests/integration/mcp-cli-parity.test.ts b/tests/integration/mcp-cli-parity.test.ts index 91edf02..22fc345 100644 --- a/tests/integration/mcp-cli-parity.test.ts +++ b/tests/integration/mcp-cli-parity.test.ts @@ -1,4 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; +import { dirname } from 'node:path'; +import { rm, writeFile } from 'node:fs/promises'; import { createTaskApplication, type TaskApplication, @@ -447,4 +449,154 @@ describe('built MCP and CLI contract parity', () => { await runtime.close(); } }); + + it('returns identical no-op change metadata without changing task timestamps', async () => { + const runtime = await createAgentTestRuntime(); + const client = await createMcpTestClient(runtime); + try { + const capture = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'No-op parity task', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ); + const task = (capture.data as { task: Record }).task; + const taskId = String(task.id); + const cli = await runRelayCli(runtime, [ + 'task', + 'edit', + taskId, + '--title', + 'No-op parity task', + '--output', + 'json', + ]); + const mcp = normalizeMcpSuccess( + await client.callTool('task_edit', { taskId, title: 'No-op parity task' }), + ); + + expect(cli.exitCode).toBe(0); + expect(normalizeCliSuccess(cli.json)).toEqual(mcp); + expect(mcp.data).toMatchObject({ + task: { id: taskId, createdAt: task.createdAt, updatedAt: task.updatedAt }, + change: { action: 'NO_CHANGE', fields: [] }, + }); + } finally { + await client.close(); + await runtime.close(); + } + }); + + it('maps not-found, invalid transition, and archived mutation errors consistently', async () => { + const runtime = await createAgentTestRuntime(); + const client = await createMcpTestClient(runtime); + try { + const notFoundCli = await runRelayCli(runtime, ['task', 'get', 'missing-id', '--output', 'json']); + const notFoundMcp = await client.callTool('task_get', { taskId: 'missing-id' }); + expect(notFoundCli.exitCode).toBe(3); + expect(notFoundCli.json).toMatchObject({ error: { code: 'NOT_FOUND' } }); + expect(notFoundMcp).toMatchObject({ + isError: true, + structuredContent: { error: { code: 'NOT_FOUND' } }, + }); + + const capture = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'Error parity task', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ); + const taskId = String((capture.data as { task: { id: string } }).task.id); + const invalidCli = await runRelayCli(runtime, ['task', 'complete', taskId, '--output', 'json']); + const invalidMcp = await client.callTool('task_complete', { taskId }); + expect(invalidCli.exitCode).toBe(4); + expect(invalidCli.json).toMatchObject({ error: { code: 'CONFLICT' } }); + expect(invalidMcp).toMatchObject({ + isError: true, + structuredContent: { error: { code: 'CONFLICT' } }, + }); + + await client.callTool('task_triage', { taskId, target: 'ACTIVE' }); + await client.callTool('task_start', { taskId }); + await client.callTool('task_complete', { taskId }); + await client.callTool('task_archive', { taskId }); + const archivedCli = await runRelayCli(runtime, [ + 'task', + 'edit', + taskId, + '--title', + 'Archived edit', + '--output', + 'json', + ]); + const archivedMcp = await client.callTool('task_edit', { taskId, title: 'Archived edit' }); + expect(archivedCli.exitCode).toBe(4); + expect(archivedCli.json).toMatchObject({ error: { code: 'ARCHIVED_TASK' } }); + expect(archivedMcp).toMatchObject({ + isError: true, + structuredContent: { error: { code: 'ARCHIVED_TASK' } }, + }); + for (const value of [notFoundCli.json, notFoundMcp, invalidCli.json, invalidMcp, archivedCli.json, archivedMcp]) { + expect(JSON.stringify(value)).not.toMatch(/SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\/i); + } + } finally { + await client.close(); + await runtime.close(); + } + }); + + it('keeps malformed inputs at validation/protocol boundaries and preserves exit code 2', async () => { + const runtime = await createAgentTestRuntime(); + const client = await createMcpTestClient(runtime); + try { + const cli = await runRelayCli(runtime, [ + 'task', + 'capture', + '--title', + '', + '--agent', + 'Codex', + '--session', + 'session-alpha', + '--output', + 'json', + ]); + expect(cli.exitCode).toBe(2); + expect(cli.json).toMatchObject({ error: { code: 'VALIDATION_ERROR' } }); + await expect( + client.callTool('task_capture', { + title: '', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ).resolves.toMatchObject({ + isError: true, + content: [{ type: 'text', text: expect.stringMatching(/invalid arguments|title/i) }], + }); + expect(cli.stderr).not.toMatch(/SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\/i); + } finally { + await client.close(); + await runtime.close(); + } + }); + + it('maps a deterministic unusable database parent without touching the default database', async () => { + const runtime = await createAgentTestRuntime(); + const databaseParent = dirname(runtime.databasePath); + await rm(databaseParent, { recursive: true, force: true }); + await writeFile(databaseParent, 'not a directory'); + try { + const cli = await runRelayCli(runtime, ['task', 'list', '--output', 'json']); + expect(cli.exitCode).toBe(5); + expect(cli.json).toMatchObject({ error: { code: 'STORAGE_ERROR' } }); + expect(cli.stderr).not.toMatch(/SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\/i); + + await expect(createMcpTestClient(runtime)).rejects.toThrow(); + } finally { + await rm(databaseParent, { force: true }); + await runtime.close(); + } + }); }); From f098fb19d0d884a45a5a4a5fa848872e40d8810e Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Wed, 29 Jul 2026 22:50:19 +0530 Subject: [PATCH 05/16] test: verify shared database path across adapters --- .../integration/database-path-parity.test.ts | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 tests/integration/database-path-parity.test.ts diff --git a/tests/integration/database-path-parity.test.ts b/tests/integration/database-path-parity.test.ts new file mode 100644 index 0000000..14d52ba --- /dev/null +++ b/tests/integration/database-path-parity.test.ts @@ -0,0 +1,160 @@ +import { readdir, stat } from 'node:fs/promises'; +import { basename, dirname, join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { createTaskRuntime } from '../../src/interfaces/shared/create-task-runtime.js'; +import { createHttpServer, type HttpServerInstance } from '../../src/interfaces/http/create-http-server.js'; +import { createAgentTestRuntime } from '../support/agent-test-runtime.js'; +import { runRelayCli } from '../support/cli-test-process.js'; +import { createMcpTestClient, type McpTestClient } from '../support/mcp-test-client.js'; +import { normalizeCliSuccess, normalizeMcpSuccess } from '../support/external-contract-normalizers.js'; + +describe('shared database path across HTTP, MCP, and CLI', () => { + it('uses one configured database from arbitrary CWDs and preserves data after all adapters restart', async () => { + const runtime = await createAgentTestRuntime(); + let applicationRuntime: ReturnType | undefined; + let server: HttpServerInstance | undefined; + let client: McpTestClient | undefined; + try { + applicationRuntime = createTaskRuntime({ databasePath: runtime.databasePath }); + server = await createHttpServer({ + host: '127.0.0.1', + port: 0, + taskApplication: applicationRuntime.taskApplication, + }); + client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('cwd/mcp'), + }); + + const capture = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'Shared MCP task', + createdByName: 'Codex', + sessionId: 'session-alpha', + workspace: 'relay-verification', + }), + ); + const capturedTask = (capture.data as { task: { id: string } }).task; + const cliGet = await runRelayCli( + runtime, + ['task', 'get', capturedTask.id, '--output', 'json'], + { cwd: await runtime.createWorkingDirectory('cwd/cli') }, + ); + expect(cliGet.exitCode).toBe(0); + expect(normalizeCliSuccess(cliGet.json).data).toEqual({ task: capturedTask }); + + const cliEdit = await runRelayCli(runtime, [ + 'task', + 'edit', + capturedTask.id, + '--title', + 'Shared edited task', + '--output', + 'json', + ]); + expect(cliEdit.exitCode).toBe(0); + const httpEdited = await fetch(`${server.url}/api/tasks/${capturedTask.id}`); + expect(httpEdited.status).toBe(200); + await expect(httpEdited.json()).resolves.toMatchObject({ + task: { id: capturedTask.id, title: 'Shared edited task' }, + }); + + const humanResponse = await fetch(`${server.url}/api/tasks`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ title: 'Shared human task' }), + }); + expect(humanResponse.status).toBe(201); + const humanTask = (await humanResponse.json()) as { task: { id: string; createdByType: string } }; + expect(humanTask.task.createdByType).toBe('HUMAN'); + + const mcpHuman = normalizeMcpSuccess( + await client.callTool('task_get', { taskId: humanTask.task.id }), + ); + expect(mcpHuman.data).toMatchObject({ task: { id: humanTask.task.id, createdByType: 'HUMAN' } }); + const cliList = await runRelayCli(runtime, ['task', 'list', '--output', 'json']); + expect(cliList.exitCode).toBe(0); + expect(normalizeCliSuccess(cliList.json).data).toMatchObject({ + tasks: expect.arrayContaining([ + expect.objectContaining({ id: capturedTask.id, title: 'Shared edited task' }), + expect.objectContaining({ id: humanTask.task.id, title: 'Shared human task' }), + ]), + count: 2, + }); + + await client.close(); + client = undefined; + await server.stop(); + server = undefined; + applicationRuntime.close(); + applicationRuntime = undefined; + + applicationRuntime = createTaskRuntime({ databasePath: runtime.databasePath }); + server = await createHttpServer({ + host: '127.0.0.1', + port: 0, + taskApplication: applicationRuntime.taskApplication, + }); + client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('cwd/mcp-restart'), + }); + const restartedHttp = await fetch(`${server.url}/api/tasks/${capturedTask.id}`); + expect(restartedHttp.status).toBe(200); + await expect(restartedHttp.json()).resolves.toMatchObject({ + task: { id: capturedTask.id, title: 'Shared edited task' }, + }); + const restartedMcp = normalizeMcpSuccess( + await client.callTool('task_get', { taskId: humanTask.task.id }), + ); + expect(restartedMcp.data).toMatchObject({ + task: { id: humanTask.task.id, title: 'Shared human task', createdByType: 'HUMAN' }, + }); + } finally { + await client?.close(); + await server?.stop(); + applicationRuntime?.close(); + await runtime.close(); + } + + }); + + it('does not create a default or CWD-local database file', async () => { + const runtime = await createAgentTestRuntime(); + let applicationRuntime: ReturnType | undefined; + let server: HttpServerInstance | undefined; + let client: McpTestClient | undefined; + try { + applicationRuntime = createTaskRuntime({ databasePath: runtime.databasePath }); + server = await createHttpServer({ host: '127.0.0.1', port: 0, taskApplication: applicationRuntime.taskApplication }); + const mcpCwd = await runtime.createWorkingDirectory('mcp'); + const cliCwd = await runtime.createWorkingDirectory('cli'); + client = await createMcpTestClient(runtime, { cwd: mcpCwd }); + await client.callTool('task_capture', { + title: 'Database location task', + createdByName: 'Codex', + sessionId: 'session-alpha', + }); + const cli = await runRelayCli(runtime, ['task', 'list', '--output', 'json'], { cwd: cliCwd }); + expect(cli.exitCode).toBe(0); + expect(await stat(runtime.databasePath)).toBeDefined(); + expect(await databaseFilesUnder(mcpCwd)).toEqual([]); + expect(await databaseFilesUnder(cliCwd)).toEqual([]); + expect(basename(dirname(runtime.databasePath))).toBe('data'); + } finally { + await client?.close(); + await server?.stop(); + applicationRuntime?.close(); + await runtime.close(); + } + }); +}); + +async function databaseFilesUnder(root: string): Promise { + const entries = await readdir(root, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries) { + const path = join(root, entry.name); + if (entry.isDirectory()) files.push(...(await databaseFilesUnder(path))); + else if (/\.db(?:-(?:wal|shm))?$|^relay\.db$/i.test(entry.name)) files.push(path); + } + return files; +} From bc471f0391cbbb17701fb79f7037f13fd8d69c66 Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Wed, 29 Jul 2026 22:54:51 +0530 Subject: [PATCH 06/16] test: prevent agent integration policy drift --- integrations/claude-code/README.md | 2 +- integrations/generic-cli/README.md | 2 +- scripts/validate-agent-integration-assets.ts | 46 ++++++++-- .../valid/integrations/claude-code/README.md | 2 +- .../valid/integrations/codex/README.md | 2 +- .../valid/integrations/generic-cli/README.md | 2 +- .../valid/integrations/generic-mcp/README.md | 2 +- .../validate-agent-integration-assets.test.ts | 85 +++++++++++++++++++ 8 files changed, 130 insertions(+), 13 deletions(-) diff --git a/integrations/claude-code/README.md b/integrations/claude-code/README.md index 4b87d0e..31f7e9c 100644 --- a/integrations/claude-code/README.md +++ b/integrations/claude-code/README.md @@ -4,6 +4,6 @@ Build Relay and substitute an absolute checkout path. Add the stdio server with Install the canonical [Relay Capture](../../skills/relay-capture/SKILL.md) and [Relay Session Review](../../skills/relay-session-review/SKILL.md) skill directories by copying or symlinking them unchanged to `.claude/skills/relay-capture/` and `.claude/skills/relay-session-review/`. For a personal installation across projects, use the client’s documented user-scoped skills directory. Do not copy the policy text into Claude-specific documentation or use instruction-file imports as skill discovery. -Use `claude mcp list`, `claude mcp get relay`, and `/mcp` to validate and authorize the server. Confirm `relay_health`, the task tools, a disposable capture, and the exact session lookup when live validation is available. Remove it with `claude mcp remove relay` or by deleting the Relay entry and skill directories; the SQLite database remains untouched. +Use `claude mcp list`, `claude mcp get relay`, and `/mcp` to validate and authorize the server. Confirm `relay_health`, the task tools, a disposable capture, and the exact session lookup when live validation is available. Remove only the client configuration with `claude mcp remove relay` or by deleting the Relay entry and skill directories; the SQLite database remains untouched. For current syntax and skill discovery behavior, see the [Claude Code MCP documentation](https://code.claude.com/docs/en/mcp) and [Claude Code skills documentation](https://code.claude.com/docs/en/skills). diff --git a/integrations/generic-cli/README.md b/integrations/generic-cli/README.md index 563af0d..1b060c3 100644 --- a/integrations/generic-cli/README.md +++ b/integrations/generic-cli/README.md @@ -12,4 +12,4 @@ node __RELAY_CHECKOUT__/dist/cli/main.js task complete TASK_ID --output json node __RELAY_CHECKOUT__/dist/cli/main.js task archive TASK_ID --output json ``` -Exit codes are documented in [the CLI reference](../../docs/cli-reference.md). Capture can be autonomous; edit, triage, start, complete, and archive require explicit user direction. See [Relay Capture](../../skills/relay-capture/SKILL.md) and [Relay Session Review](../../skills/relay-session-review/SKILL.md). Removing a client integration does not delete data; the SQLite database remains untouched. +Exit codes are documented in [the CLI reference](../../docs/cli-reference.md). See [Relay Capture](../../skills/relay-capture/SKILL.md) and [Relay Session Review](../../skills/relay-session-review/SKILL.md) for behavioural policy. Removing a client integration does not delete data; the SQLite database remains untouched. diff --git a/scripts/validate-agent-integration-assets.ts b/scripts/validate-agent-integration-assets.ts index facaa99..a54f742 100644 --- a/scripts/validate-agent-integration-assets.ts +++ b/scripts/validate-agent-integration-assets.ts @@ -75,6 +75,9 @@ function validateCompatibilityClaims(shared: string): void { function validateVendorClaims(rootDir: string, shared: string): void { for (const readme of vendorReadmes) { const text = readAsset(rootDir, `integrations/${readme}/README.md`); + if (/^## Autonomy boundaries$/im.test(text) || /autonomously\s+(?:edit|triage|start|complete|archive|delete|merge)/i.test(text)) { + fail(`${readme} must reference canonical behavioural policy instead of redefining mutation autonomy.`); + } const claimsNoLiveTest = /(?:live smoke test|live validation)[^\n]*not completed/i.test(text); const claimsLiveEvidence = /(?:live|manual)[^\n]*(?:tested|performed|verified|discovered|captured)/i.test(text); @@ -99,6 +102,34 @@ function validateVendorClaims(rootDir: string, shared: string): void { } } +function validateCanonicalSkills(rootDir: string): void { + const capture = readAsset(rootDir, canonicalSkills[0]); + if (!/autonomously create only a new Relay task in `?INBOX`?/i.test(capture)) + fail('Canonical capture skill must define autonomous creation as INBOX-only.'); + if (!/must not edit, triage, start, complete, archive, delete, merge, or move any task/i.test(capture)) + fail('Canonical capture skill must prohibit autonomous lifecycle mutation.'); + if (/autonomously\s+(?:edit|triage|start|complete|archive|delete|merge)/i.test(capture)) + fail('Canonical capture skill must not grant autonomous lifecycle mutation.'); + + const review = readAsset(rootDir, canonicalSkills[1]); + if (!/completed and archived captures/i.test(review)) + fail('Canonical session-review skill must require all-status retrieval including completed and archived captures.'); + if (!/explicit user direction/i.test(review)) + fail('Canonical session-review skill must require explicit user direction for mutations.'); +} + +function validateRemovalGuidance(rootDir: string): void { + for (const readme of vendorReadmes) { + const text = readAsset(rootDir, `integrations/${readme}/README.md`); + if (/(?:remove|delete)\s+(?:the\s+)?(?:SQLite\s+)?database\b|(?:SQLite\s+)?database\b[^\n]{0,80}\b(?:remove|delete)\b/i.test(text)) + fail(`${readme} removal guidance must not delete the SQLite database.`); + if (!/(?:remov(?:e|ing)|delete)[\s\S]{0,160}(?:configuration|integration|client assets?)/i.test(text)) + fail(`${readme} removal guidance must distinguish configuration removal from stored data.`); + if (!/SQLite database remains untouched/i.test(text)) + fail(`${readme} removal guidance must state that the SQLite database remains untouched.`); + } +} + function validateTemplateShape(rootDir: string): void { const expectedMcpPath = '/tmp/relay-checkout/dist/mcp/main.js'; const jsonTemplates = [ @@ -121,7 +152,7 @@ function validateTemplateShape(rootDir: string): void { fail(`${path} must separate command and arguments for a stdio server.`); } if (server.args[0] !== expectedMcpPath) { - fail(`${path} must use the exact dist/mcp/main.js entry path.`); + fail(`${path} must use the canonical dist/mcp/main.js entry path.`); } if (typeof server.command !== 'string' || /[\\/\s]/.test(server.command)) { fail(`${path} must not embed a shell command in command.`); @@ -149,7 +180,7 @@ function validateTemplateShape(rootDir: string): void { codexServer.args[0] !== expectedMcpPath ) { fail( - 'integrations/codex/config.toml.example must use node plus dist/mcp/main.js as separate fields.', + 'integrations/codex/config.toml.example must use node plus the canonical dist/mcp/main.js entry as separate fields.', ); } if (codexServer.env?.RELAY_DB_PATH !== '/tmp/relay-checkout/.relay-validation/relay.db') { @@ -164,6 +195,9 @@ export function validateAgentIntegrationAssets( for (const path of requiredPaths) { if (!existsSync(join(rootDir, path))) fail(`Required path missing: ${path}`); } + for (const path of canonicalSkills) { + if (!existsSync(join(rootDir, path))) fail(`Canonical skill path missing: ${path}`); + } const integrationRoot = join(rootDir, 'integrations'); const contents = filesUnder(integrationRoot) @@ -173,6 +207,8 @@ export function validateAgentIntegrationAssets( const troubleshooting = readAsset(rootDir, 'docs/troubleshooting-agent-integration.md'); const all = `${contents}\n${shared}\n${troubleshooting}`; + validateCanonicalSkills(rootDir); + if (/(?:[A-Z]:[\\/]Users[\\/]|\/Users\/|\/home\/|~\/)[^\s"'`]+/i.test(all)) fail('Machine-specific absolute path found.'); for (const skill of canonicalSkills) { @@ -220,11 +256,7 @@ export function validateAgentIntegrationAssets( fail('Claude README must preserve complete canonical skill directories unchanged.'); if (/^## Autonomy boundaries$/m.test(contents)) fail('Vendor assets must not copy behavioural policy.'); - for (const readme of vendorReadmes) { - const text = readAsset(rootDir, `integrations/${readme}/README.md`); - if (!/SQLite database remains untouched/i.test(text)) - fail(`${readme} removal guidance must state that the SQLite database remains untouched.`); - } + validateRemovalGuidance(rootDir); for (const match of all.matchAll(/relay mcp/gi)) { const context = all.slice(Math.max(0, match.index! - 80), match.index! + 100); if (!/(future|not available|Epic #18)/i.test(context)) diff --git a/tests/fixtures/agent-integrations/valid/integrations/claude-code/README.md b/tests/fixtures/agent-integrations/valid/integrations/claude-code/README.md index 1dfedfa..65afaaf 100644 --- a/tests/fixtures/agent-integrations/valid/integrations/claude-code/README.md +++ b/tests/fixtures/agent-integrations/valid/integrations/claude-code/README.md @@ -1 +1 @@ -skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md .claude/skills/relay-capture/ .claude/skills/relay-session-review/ Copy or symlink the complete canonical skill directories and preserve each SKILL.md unchanged. SQLite database remains untouched. +skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md .claude/skills/relay-capture/ .claude/skills/relay-session-review/ Copy or symlink the complete canonical skill directories and preserve each SKILL.md unchanged. Remove only the client configuration and skill references; the SQLite database remains untouched. diff --git a/tests/fixtures/agent-integrations/valid/integrations/codex/README.md b/tests/fixtures/agent-integrations/valid/integrations/codex/README.md index ee032d2..4ea651f 100644 --- a/tests/fixtures/agent-integrations/valid/integrations/codex/README.md +++ b/tests/fixtures/agent-integrations/valid/integrations/codex/README.md @@ -1 +1 @@ -skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md .agents/skills/relay-capture/ .agents/skills/relay-session-review/ SQLite database remains untouched. +skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md .agents/skills/relay-capture/ .agents/skills/relay-session-review/ Remove only the client configuration and skill references; the SQLite database remains untouched. diff --git a/tests/fixtures/agent-integrations/valid/integrations/generic-cli/README.md b/tests/fixtures/agent-integrations/valid/integrations/generic-cli/README.md index 9d777a8..cebde18 100644 --- a/tests/fixtures/agent-integrations/valid/integrations/generic-cli/README.md +++ b/tests/fixtures/agent-integrations/valid/integrations/generic-cli/README.md @@ -1 +1 @@ -skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md `export RELAY_DB_PATH="__RELAY_CHECKOUT__/.relay-validation/relay.db"` SQLite database remains untouched. +skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md `export RELAY_DB_PATH="__RELAY_CHECKOUT__/.relay-validation/relay.db"` Remove only the client configuration; the SQLite database remains untouched. diff --git a/tests/fixtures/agent-integrations/valid/integrations/generic-mcp/README.md b/tests/fixtures/agent-integrations/valid/integrations/generic-mcp/README.md index 571f6ef..9f7a815 100644 --- a/tests/fixtures/agent-integrations/valid/integrations/generic-mcp/README.md +++ b/tests/fixtures/agent-integrations/valid/integrations/generic-mcp/README.md @@ -1 +1 @@ -skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md Validation requires explicit isolated RELAY_DB_PATH; omission is permitted only for non-validation use. relay_health task_capture task_list task_get task_find_similar session_captures_list SQLite database remains untouched. +skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md Validation requires explicit isolated RELAY_DB_PATH; omission is permitted only for non-validation use. Remove only the client configuration; the SQLite database remains untouched. relay_health task_capture task_list task_get task_find_similar session_captures_list SQLite database remains untouched. diff --git a/tests/unit/scripts/validate-agent-integration-assets.test.ts b/tests/unit/scripts/validate-agent-integration-assets.test.ts index eebe312..19311a0 100644 --- a/tests/unit/scripts/validate-agent-integration-assets.test.ts +++ b/tests/unit/scripts/validate-agent-integration-assets.test.ts @@ -12,6 +12,7 @@ describe('validateAgentIntegrationAssets', () => { function createRoot(): string { const rootDir = mkdtempSync(join(tmpdir(), 'relay-agent-integration-assets-')); cpSync(fixtureRoot, rootDir, { recursive: true }); + cpSync(join(process.cwd(), 'skills'), join(rootDir, 'skills'), { recursive: true }); roots.push(rootDir); return rootDir; } @@ -226,4 +227,88 @@ describe('validateAgentIntegrationAssets', () => { expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/future-only/i); }); + + it('rejects a canonical capture skill without autonomous-create permission', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'skills/relay-capture/SKILL.md'); + writeFileSync(path, readFileSync(path, 'utf8').replace('autonomously create only', 'may create only')); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/canonical capture.*autonom/i); + }); + + it('rejects a canonical capture skill that permits lifecycle mutation', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'skills/relay-capture/SKILL.md'); + writeFileSync( + path, + readFileSync(path, 'utf8').replace( + 'It must not edit, triage, start, complete, archive, delete, merge, or move any task', + 'It may edit, triage, start, complete, archive, delete, merge, or move any task', + ), + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/canonical capture.*lifecycle/i); + }); + + it('rejects a canonical session-review skill that omits completed and archived captures', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'skills/relay-session-review/SKILL.md'); + writeFileSync(path, readFileSync(path, 'utf8').replace('completed and archived captures', 'INBOX tasks')); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/session-review.*status/i); + }); + + it('rejects a canonical session-review skill without explicit user-action guidance', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'skills/relay-session-review/SKILL.md'); + writeFileSync(path, readFileSync(path, 'utf8').replaceAll('explicit user direction', 'automatic action')); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/session-review.*explicit/i); + }); + + it('rejects a vendor wrapper that copies mutation autonomy policy', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-cli/README.md'); + writeFileSync( + path, + `${readFileSync(path, 'utf8')}\n## Autonomy boundaries\nThe agent may autonomously edit and archive tasks.`, + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/vendor.*policy|behavioural policy/i); + }); + + it('rejects a vendor wrapper without both canonical skill references', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-mcp/README.md'); + writeFileSync(path, readFileSync(path, 'utf8').replace('skills/relay-session-review/SKILL.md', '')); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/generic-mcp.*skill/i); + }); + + it('rejects a configuration template that points at a non-canonical MCP entry', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-mcp/server-config.json.example'); + writeFileSync(path, readFileSync(path, 'utf8').replace('dist/mcp/main.js', 'dist/other-mcp.js')); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/canonical.*dist\/mcp\/main\.js/i); + }); + + it('rejects removal guidance that deletes the SQLite database', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-cli/README.md'); + writeFileSync(path, `${readFileSync(path, 'utf8')} Remove the SQLite database when disabling Relay.`); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/removal.*database|destructive/i); + }); + + it('rejects removal guidance that does not distinguish configuration from stored data', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-cli/README.md'); + writeFileSync( + path, + readFileSync(path, 'utf8').replace('SQLite database remains untouched', 'Relay data may be deleted'), + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/removal.*configuration|database remains/i); + }); }); From 7ea1adb86a75720ca978c5c59d5d77eafda9ada1 Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Wed, 29 Jul 2026 23:57:09 +0530 Subject: [PATCH 07/16] test: harden agent integration verification --- docs/agent-integration-verification.md | 90 +++++++++++++ tests/integration/agent-workflow-e2e.test.ts | 114 ++++++++++++---- .../integration/database-path-parity.test.ts | 37 ++++-- tests/integration/mcp-cli-parity.test.ts | 123 ++++++++++++++---- tests/support/cli-test-process.ts | 17 ++- .../support/external-contract-normalizers.ts | 1 - tests/support/mcp-test-client.ts | 10 +- .../validate-repository-assets.test.ts | 4 +- tests/unit/support/cli-test-process.test.ts | 11 ++ 9 files changed, 336 insertions(+), 71 deletions(-) create mode 100644 docs/agent-integration-verification.md diff --git a/docs/agent-integration-verification.md b/docs/agent-integration-verification.md new file mode 100644 index 0000000..0b588e3 --- /dev/null +++ b/docs/agent-integration-verification.md @@ -0,0 +1,90 @@ +# Agent Integration Verification + +## Scope and safety statement + +This evidence covers the source-checkout Relay MCP and CLI adapters, their shared task contract, the HTTP database path, and the canonical integration assets for issue #25. All automated scenarios use a fresh disposable database under the repository's `tmp/` directory and arbitrary disposable working directories. No test invokes an external LLM, reads or writes the default Relay database, or changes real Codex, Claude Code, or other client configuration. MCP diagnostics are captured from stderr; protocol data remains on stdout. + +## Automated scenario matrix + +| Scenario | Automated test or manual step | Result | Evidence | +| ------------------------------------------------------------------- | -------------------------------------------------------------------------- | ------ | -------------------------------------------------------------------------------- | +| 1. MCP capture then CLI retrieval | `mcp-cli-parity.test.ts` — MCP capture followed by built CLI get | PASS | Built `dist/mcp/main.js` and `dist/cli/main.js`, same disposable `RELAY_DB_PATH` | +| 2. CLI capture then MCP retrieval | `mcp-cli-parity.test.ts` — CLI capture followed by MCP get | PASS | Complete public task DTO equality | +| 3. Task list/get fields and ordering | `mcp-cli-parity.test.ts` — list DTO comparison and persisted ordering | PASS | Transport-only envelope normalization | +| 4. Session-ID isolation | `agent-workflow-e2e.test.ts` — alpha and beta captures | PASS | Exact session filtering | +| 5. Completed and archived session review | `agent-workflow-e2e.test.ts` — all-status review | PASS | Open, DONE, and ARCHIVED captures returned | +| 6. Missing session behavior | `mcp-cli-parity.test.ts` and existing adapter contract tests | PASS | Stable missing-task/session contract coverage | +| 7. Malformed session behavior | `mcp-cli-parity.test.ts` and existing strict schema tests | PASS | MCP validation boundary and CLI validation envelope | +| 8. Duplicate candidates, warnings, and match reasons | `mcp-cli-parity.test.ts` — duplicate capture and find-similar | PASS | Advisory warning and deterministic candidate assertions | +| 9. Edit parity | `mcp-cli-parity.test.ts` — CLI edit and MCP readback | PASS | Complete task and change metadata | +| 10. Triage/start/complete/archive parity | `mcp-cli-parity.test.ts` — cross-adapter lifecycle sequence | PASS | Focused lifecycle actions and statuses | +| 11. No-op metadata | `mcp-cli-parity.test.ts` — repeated edit | PASS | `NO_CHANGE`, empty fields, unchanged timestamps | +| 12. Validation, transition, archived, not-found, and storage errors | `mcp-cli-parity.test.ts` — stable errors and unusable parent path | PASS | CLI envelopes; MCP execution/protocol errors; startup failure remains stderr-only; leakage checks | +| 13. CLI JSON schemas and exit codes | `cli-test-process.test.ts` and built parity tests | PASS | One JSON document, separated stderr, exit codes 0/2/3/4/5 | +| 14. One database across HTTP, MCP, and CLI | `database-path-parity.test.ts` | PASS | Arbitrary CWDs, HTTP runtime, restart persistence, no CWD-local DB | +| 15. Skill and vendor-wrapper drift | `validate-agent-integration-assets.test.ts` and `validate:assets` | PASS | 33 validator tests; canonical policy and entry-point checks | +| 16. Integration removal preserves data | `agent-workflow-e2e.test.ts` config-driven disposable MCP launch and removal | PASS | Parsed `.mcp.json` launches the built server; removal is followed by retrieval from the same DB | + +## Clean-checkout environment + +- OS: Microsoft Windows NT 10.0.26200.0 +- Node: v24.18.0 +- pnpm: 10.2.0 through Corepack (`corepack pnpm --version`); direct global pnpm was 11.9.0 and was not used for authoritative final commands. +- Branch: `feature/issue-25-mcp-cli-compatibility` +- Verification base SHA: `e51307d066e70b95f8072ac52d2679b8e63c5244` +- Database strategy: each test calls `createAgentTestRuntime()` and uses a unique disposable `/data/relay.db`; client CWDs are created below the same disposable root. + +Exact setup and validation commands: + +```text +corepack pnpm install --frozen-lockfile +corepack pnpm build +corepack pnpm exec vitest run tests/integration/mcp-cli-parity.test.ts +corepack pnpm exec vitest run tests/integration/agent-workflow-e2e.test.ts +corepack pnpm exec vitest run tests/integration/database-path-parity.test.ts +corepack pnpm validate:assets +corepack pnpm verify +git status --short +``` + +The disposable strategy and assertions confirm that the default database and real client configuration were not touched. + +## Codex validation + +Live validation is unverified. `Get-Command codex` resolved the installed desktop executable, but `codex --version` failed with Windows `Access is denied`. No Codex process was started, no MCP discovery result is claimed, and no Codex configuration or profile was changed. + +The required isolated workflow remains: + +1. Use a clean checkout and `corepack pnpm install --frozen-lockfile`. +2. Run `corepack pnpm build`. +3. Create an isolated profile/configuration and disposable `RELAY_DB_PATH`. +4. Add the canonical `node /dist/mcp/main.js` server using the documented Codex configuration. +5. Install the canonical skills unchanged under `.agents/skills/`. +6. Restart Codex, discover Relay tools, capture two follow-ups with one session ID, review that session, perform one explicitly directed lifecycle action, use the CLI JSON fallback, remove only config/skill references, and confirm data remains. + +No step above is represented as executed in this environment. + +## Claude Code validation + +Live validation is unverified. `Get-Command claude` and `Get-Command claude-code` returned no executable. No Claude configuration was edited and no client result is claimed. + +The equivalent isolated workflow is documented in `integrations/claude-code/README.md`: project-scoped stdio configuration, canonical `.claude/skills/` directories, tool discovery, two same-session captures, exact session review, one explicitly directed mutation, CLI fallback, configuration-only removal, and post-removal data retrieval. + +## Cross-client differences and limitations + +The automated contract is client-neutral: built MCP uses protocol-owned stdout and CLI uses one JSON envelope plus stable exit codes. Codex and Claude Code syntax, skill-discovery locations, and availability could not be exercised live here. The issue #24 documentation records the official-source verification date and the current unavailable-client limitations. + +## Data and configuration preservation + +Automated tests only create disposable files under the test runtime root. They remove temporary configuration fixtures and restart against the same database, then retrieve persisted tasks. They never remove a database to disable an integration. The validator rejects removal guidance that deletes SQLite data and requires explicit configuration-only wording. + +## Epic #2 closure checklist + +- [x] Issues #19, #20, #21, #22, #23, #24, and #26 are closed on GitHub and their required artifacts exist locally. +- [x] Built MCP and CLI entry points are exercised from arbitrary working directories. +- [x] MCP stdout remains protocol-clean; diagnostics are stderr-only. +- [x] Automated contract, lifecycle, session, error, storage, restart, shared-path, and asset checks pass. +- [x] Default database, real client configuration, and external LLMs were not touched by automation. +- [ ] Live Codex workflow — blocked by executable access denied; human review required. +- [ ] Live Claude Code workflow — blocked because the client is unavailable; human review required. +- [ ] Human reviewer must inspect cleanup, normalizers, source-context safety, and one independent client workflow before merging. diff --git a/tests/integration/agent-workflow-e2e.test.ts b/tests/integration/agent-workflow-e2e.test.ts index 3c4ed87..db7d345 100644 --- a/tests/integration/agent-workflow-e2e.test.ts +++ b/tests/integration/agent-workflow-e2e.test.ts @@ -1,4 +1,8 @@ import { describe, expect, it } from 'vitest'; +import { readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import type { McpTestClient } from '../support/mcp-test-client.js'; +import type { AgentTestRuntime } from '../support/agent-test-runtime.js'; import { createAgentTestRuntime } from '../support/agent-test-runtime.js'; import { runRelayCli } from '../support/cli-test-process.js'; import { createMcpTestClient } from '../support/mcp-test-client.js'; @@ -9,19 +13,22 @@ import { describe('built agent workflow end to end', () => { it('isolates session review and includes open, completed, and archived captures', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime); + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); + const connectedClient = await createMcpTestClient(runtime); + client = connectedClient; const capture = async (title: string, sessionId: string) => normalizeMcpSuccess( - await client.callTool('task_capture', { + await connectedClient.callTool('task_capture', { title, createdByName: 'Codex', sessionId, workspace: 'relay-verification', }), ); - const open = await capture('Alpha open capture', 'session-alpha'); + await capture('Alpha open capture', 'session-alpha'); const completed = await capture('Alpha completed capture', 'session-alpha'); const archived = await capture('Alpha archived capture', 'session-alpha'); const beta = await runRelayCli(runtime, [ @@ -40,20 +47,22 @@ describe('built agent workflow end to end', () => { const completedId = String((completed.data as { task: { id: string } }).task.id); const archivedId = String((archived.data as { task: { id: string } }).task.id); - await client.callTool('task_triage', { taskId: completedId, target: 'ACTIVE' }); - await client.callTool('task_start', { taskId: completedId }); - await client.callTool('task_triage', { taskId: archivedId, target: 'ACTIVE' }); - await client.callTool('task_start', { taskId: archivedId }); - expect((await client.callTool('task_complete', { taskId: completedId }))).toMatchObject({ + await connectedClient.callTool('task_triage', { taskId: completedId, target: 'ACTIVE' }); + await connectedClient.callTool('task_start', { taskId: completedId }); + await connectedClient.callTool('task_triage', { taskId: archivedId, target: 'ACTIVE' }); + await connectedClient.callTool('task_start', { taskId: archivedId }); + expect( + await connectedClient.callTool('task_complete', { taskId: completedId }), + ).toMatchObject({ structuredContent: { data: { task: { status: 'DONE' } } }, }); - await client.callTool('task_complete', { taskId: archivedId }); - expect((await client.callTool('task_archive', { taskId: archivedId }))).toMatchObject({ + await connectedClient.callTool('task_complete', { taskId: archivedId }); + expect(await connectedClient.callTool('task_archive', { taskId: archivedId })).toMatchObject({ structuredContent: { data: { task: { status: 'ARCHIVED' } } }, }); const mcpReview = normalizeMcpSuccess( - await client.callTool('session_captures_list', { + await connectedClient.callTool('session_captures_list', { sessionId: 'session-alpha', limit: 100, }), @@ -77,22 +86,26 @@ describe('built agent workflow end to end', () => { { title: 'Alpha archived capture', status: 'ARCHIVED' }, ], }); - expect(mcpReview.data).not.toMatchObject({ tasks: [expect.objectContaining({ sessionId: 'session-beta' })] }); + expect(mcpReview.data).not.toMatchObject({ + tasks: [expect.objectContaining({ sessionId: 'session-beta' })], + }); } finally { - await client.close(); - await runtime.close(); + await client?.close(); + await runtime?.close(); } }); it('persists tasks and mutations across short-lived MCP and CLI restarts', async () => { - const runtime = await createAgentTestRuntime(); - let client = await createMcpTestClient(runtime, { - cwd: await runtime.createWorkingDirectory('restart/mcp-1'), - }); + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; let taskId: string; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('restart/mcp-1'), + }); const capture = normalizeMcpSuccess( - await client.callTool('task_capture', { + await client!.callTool('task_capture', { title: 'Restart persistence task', createdByName: 'Codex', sessionId: 'session-alpha', @@ -100,15 +113,13 @@ describe('built agent workflow end to end', () => { ); taskId = String((capture.data as { task: { id: string } }).task.id); } finally { - await client.close(); + await client?.close(); } try { - const get = await runRelayCli( - runtime, - ['task', 'get', taskId!, '--output', 'json'], - { cwd: await runtime.createWorkingDirectory('restart/cli-1') }, - ); + const get = await runRelayCli(runtime, ['task', 'get', taskId!, '--output', 'json'], { + cwd: await runtime.createWorkingDirectory('restart/cli-1'), + }); expect(get.exitCode).toBe(0); const edit = await runRelayCli(runtime, [ 'task', @@ -121,7 +132,7 @@ describe('built agent workflow end to end', () => { ]); expect(edit.exitCode).toBe(0); } finally { - client = await createMcpTestClient(runtime, { + client = await createMcpTestClient(runtime!, { cwd: await runtime.createWorkingDirectory('restart/mcp-2'), }); try { @@ -131,8 +142,55 @@ describe('built agent workflow end to end', () => { }); } finally { await client.close(); - await runtime.close(); + await runtime!.close(); } } }); + + it('preserves stored data when a disposable integration configuration is removed', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + const configDirectory = await runtime.createWorkingDirectory('integration-config'); + const configPath = join(configDirectory, '.mcp.json'); + const config = { + mcpServers: { + relay: { + command: process.execPath, + args: [join(runtime.checkoutPath, 'dist', 'mcp', 'main.js')], + }, + }, + }; + await writeFile(configPath, JSON.stringify(config)); + const loadedConfig = JSON.parse(await readFile(configPath, 'utf8')) as typeof config; + const server = loadedConfig.mcpServers.relay; + client = await createMcpTestClient(runtime, { + cwd: configDirectory, + command: server.command, + args: server.args, + }); + const capture = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'Configuration removal preserves data', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ); + const taskId = String((capture.data as { task: { id: string } }).task.id); + await client.close(); + client = undefined; + await rm(configPath); + await expect(stat(configPath)).rejects.toMatchObject({ code: 'ENOENT' }); + + client = await createMcpTestClient(runtime, { cwd: configDirectory }); + const retrieved = normalizeMcpSuccess(await client.callTool('task_get', { taskId })); + expect(retrieved.data).toMatchObject({ + task: { id: taskId, title: 'Configuration removal preserves data' }, + }); + } finally { + await client?.close(); + await runtime?.close(); + } + }); }); diff --git a/tests/integration/database-path-parity.test.ts b/tests/integration/database-path-parity.test.ts index 14d52ba..c72a384 100644 --- a/tests/integration/database-path-parity.test.ts +++ b/tests/integration/database-path-parity.test.ts @@ -2,19 +2,26 @@ import { readdir, stat } from 'node:fs/promises'; import { basename, dirname, join } from 'node:path'; import { describe, expect, it } from 'vitest'; import { createTaskRuntime } from '../../src/interfaces/shared/create-task-runtime.js'; -import { createHttpServer, type HttpServerInstance } from '../../src/interfaces/http/create-http-server.js'; -import { createAgentTestRuntime } from '../support/agent-test-runtime.js'; +import { + createHttpServer, + type HttpServerInstance, +} from '../../src/interfaces/http/create-http-server.js'; +import { createAgentTestRuntime, type AgentTestRuntime } from '../support/agent-test-runtime.js'; import { runRelayCli } from '../support/cli-test-process.js'; import { createMcpTestClient, type McpTestClient } from '../support/mcp-test-client.js'; -import { normalizeCliSuccess, normalizeMcpSuccess } from '../support/external-contract-normalizers.js'; +import { + normalizeCliSuccess, + normalizeMcpSuccess, +} from '../support/external-contract-normalizers.js'; describe('shared database path across HTTP, MCP, and CLI', () => { it('uses one configured database from arbitrary CWDs and preserves data after all adapters restart', async () => { - const runtime = await createAgentTestRuntime(); + let runtime: AgentTestRuntime | undefined; let applicationRuntime: ReturnType | undefined; let server: HttpServerInstance | undefined; let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); applicationRuntime = createTaskRuntime({ databasePath: runtime.databasePath }); server = await createHttpServer({ host: '127.0.0.1', @@ -64,13 +71,17 @@ describe('shared database path across HTTP, MCP, and CLI', () => { body: JSON.stringify({ title: 'Shared human task' }), }); expect(humanResponse.status).toBe(201); - const humanTask = (await humanResponse.json()) as { task: { id: string; createdByType: string } }; + const humanTask = (await humanResponse.json()) as { + task: { id: string; createdByType: string }; + }; expect(humanTask.task.createdByType).toBe('HUMAN'); const mcpHuman = normalizeMcpSuccess( await client.callTool('task_get', { taskId: humanTask.task.id }), ); - expect(mcpHuman.data).toMatchObject({ task: { id: humanTask.task.id, createdByType: 'HUMAN' } }); + expect(mcpHuman.data).toMatchObject({ + task: { id: humanTask.task.id, createdByType: 'HUMAN' }, + }); const cliList = await runRelayCli(runtime, ['task', 'list', '--output', 'json']); expect(cliList.exitCode).toBe(0); expect(normalizeCliSuccess(cliList.json).data).toMatchObject({ @@ -112,19 +123,23 @@ describe('shared database path across HTTP, MCP, and CLI', () => { await client?.close(); await server?.stop(); applicationRuntime?.close(); - await runtime.close(); + await runtime?.close(); } - }); it('does not create a default or CWD-local database file', async () => { - const runtime = await createAgentTestRuntime(); + let runtime: AgentTestRuntime | undefined; let applicationRuntime: ReturnType | undefined; let server: HttpServerInstance | undefined; let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); applicationRuntime = createTaskRuntime({ databasePath: runtime.databasePath }); - server = await createHttpServer({ host: '127.0.0.1', port: 0, taskApplication: applicationRuntime.taskApplication }); + server = await createHttpServer({ + host: '127.0.0.1', + port: 0, + taskApplication: applicationRuntime.taskApplication, + }); const mcpCwd = await runtime.createWorkingDirectory('mcp'); const cliCwd = await runtime.createWorkingDirectory('cli'); client = await createMcpTestClient(runtime, { cwd: mcpCwd }); @@ -143,7 +158,7 @@ describe('shared database path across HTTP, MCP, and CLI', () => { await client?.close(); await server?.stop(); applicationRuntime?.close(); - await runtime.close(); + await runtime?.close(); } }); }); diff --git a/tests/integration/mcp-cli-parity.test.ts b/tests/integration/mcp-cli-parity.test.ts index 22fc345..a5de90e 100644 --- a/tests/integration/mcp-cli-parity.test.ts +++ b/tests/integration/mcp-cli-parity.test.ts @@ -19,7 +19,9 @@ import { createAgentTestRuntime } from '../support/agent-test-runtime.js'; import { runRelayCli } from '../support/cli-test-process.js'; import { createMcpTestClient } from '../support/mcp-test-client.js'; import { + normalizeCliError, normalizeCliSuccess, + normalizeMcpError, normalizeMcpSuccess, } from '../support/external-contract-normalizers.js'; @@ -268,9 +270,8 @@ describe('built MCP and CLI contract parity', () => { workspace: 'relay-verification', sourceContext: 'issue-25', }); - const capturedTask = ( - normalizeMcpSuccess(capture).data as { task: Record } - ).task; + const capturedTask = (normalizeMcpSuccess(capture).data as { task: Record }) + .task; const cli = await runRelayCli( runtime, ['task', 'get', String(capturedTask.id), '--output', 'json'], @@ -307,9 +308,8 @@ describe('built MCP and CLI contract parity', () => { 'json', ]); expect(cli.exitCode).toBe(0); - const capturedTask = ( - normalizeCliSuccess(cli.json).data as { task: Record } - ).task; + const capturedTask = (normalizeCliSuccess(cli.json).data as { task: Record }) + .task; const mcp = await client.callTool('task_get', { taskId: capturedTask.id }); expect(normalizeMcpSuccess(mcp).data).toEqual({ task: capturedTask }); @@ -358,9 +358,7 @@ describe('built MCP and CLI contract parity', () => { workspace: 'relay-verification', }), ); - const candidate = ( - existing.data as { task: { id: string } } - ).task.id; + const candidate = (existing.data as { task: { id: string } }).task.id; const similar = normalizeMcpSuccess( await client.callTool('task_find_similar', { title: 'Duplicate candidate capture', @@ -422,28 +420,37 @@ describe('built MCP and CLI contract parity', () => { 'json', ]); expect(edit.exitCode).toBe(0); - expect(normalizeMcpSuccess(await client.callTool('task_get', { taskId })).data).toMatchObject({ - task: { id: taskId, title: 'Mutation parity task edited' }, - }); + expect(normalizeMcpSuccess(await client.callTool('task_get', { taskId })).data).toMatchObject( + { + task: { id: taskId, title: 'Mutation parity task edited' }, + }, + ); const triage = normalizeMcpSuccess( await client.callTool('task_triage', { taskId, target: 'ACTIVE' }), ); - expect(triage.data).toMatchObject({ change: { action: 'TRIAGED', from: 'INBOX', to: 'ACTIVE' } }); + expect(triage.data).toMatchObject({ + change: { action: 'TRIAGED', from: 'INBOX', to: 'ACTIVE' }, + }); const start = await runRelayCli(runtime, ['task', 'start', taskId, '--output', 'json']); expect(start.exitCode).toBe(0); - expect(normalizeMcpSuccess(await client.callTool('task_get', { taskId })).data).toMatchObject({ - task: { status: 'IN_PROGRESS' }, - }); - const complete = normalizeMcpSuccess( - await client.callTool('task_complete', { taskId }), + expect(normalizeMcpSuccess(await client.callTool('task_get', { taskId })).data).toMatchObject( + { + task: { status: 'IN_PROGRESS' }, + }, ); - expect(complete.data).toMatchObject({ task: { status: 'DONE' }, change: { action: 'COMPLETED' } }); + const complete = normalizeMcpSuccess(await client.callTool('task_complete', { taskId })); + expect(complete.data).toMatchObject({ + task: { status: 'DONE' }, + change: { action: 'COMPLETED' }, + }); const archive = await runRelayCli(runtime, ['task', 'archive', taskId, '--output', 'json']); expect(archive.exitCode).toBe(0); - expect(normalizeMcpSuccess(await client.callTool('task_get', { taskId })).data).toMatchObject({ - task: { status: 'ARCHIVED' }, - }); + expect(normalizeMcpSuccess(await client.callTool('task_get', { taskId })).data).toMatchObject( + { + task: { status: 'ARCHIVED' }, + }, + ); } finally { await client.close(); await runtime.close(); @@ -492,7 +499,13 @@ describe('built MCP and CLI contract parity', () => { const runtime = await createAgentTestRuntime(); const client = await createMcpTestClient(runtime); try { - const notFoundCli = await runRelayCli(runtime, ['task', 'get', 'missing-id', '--output', 'json']); + const notFoundCli = await runRelayCli(runtime, [ + 'task', + 'get', + 'missing-id', + '--output', + 'json', + ]); const notFoundMcp = await client.callTool('task_get', { taskId: 'missing-id' }); expect(notFoundCli.exitCode).toBe(3); expect(notFoundCli.json).toMatchObject({ error: { code: 'NOT_FOUND' } }); @@ -500,6 +513,7 @@ describe('built MCP and CLI contract parity', () => { isError: true, structuredContent: { error: { code: 'NOT_FOUND' } }, }); + expect(normalizeCliError(notFoundCli.json)).toEqual(normalizeMcpError(notFoundMcp)); const capture = normalizeMcpSuccess( await client.callTool('task_capture', { @@ -509,7 +523,13 @@ describe('built MCP and CLI contract parity', () => { }), ); const taskId = String((capture.data as { task: { id: string } }).task.id); - const invalidCli = await runRelayCli(runtime, ['task', 'complete', taskId, '--output', 'json']); + const invalidCli = await runRelayCli(runtime, [ + 'task', + 'complete', + taskId, + '--output', + 'json', + ]); const invalidMcp = await client.callTool('task_complete', { taskId }); expect(invalidCli.exitCode).toBe(4); expect(invalidCli.json).toMatchObject({ error: { code: 'CONFLICT' } }); @@ -517,6 +537,7 @@ describe('built MCP and CLI contract parity', () => { isError: true, structuredContent: { error: { code: 'CONFLICT' } }, }); + expect(normalizeCliError(invalidCli.json)).toEqual(normalizeMcpError(invalidMcp)); await client.callTool('task_triage', { taskId, target: 'ACTIVE' }); await client.callTool('task_start', { taskId }); @@ -538,7 +559,15 @@ describe('built MCP and CLI contract parity', () => { isError: true, structuredContent: { error: { code: 'ARCHIVED_TASK' } }, }); - for (const value of [notFoundCli.json, notFoundMcp, invalidCli.json, invalidMcp, archivedCli.json, archivedMcp]) { + expect(normalizeCliError(archivedCli.json)).toEqual(normalizeMcpError(archivedMcp)); + for (const value of [ + notFoundCli.json, + notFoundMcp, + invalidCli.json, + invalidMcp, + archivedCli.json, + archivedMcp, + ]) { expect(JSON.stringify(value)).not.toMatch(/SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\/i); } } finally { @@ -582,6 +611,50 @@ describe('built MCP and CLI contract parity', () => { } }); + it('matches built empty-session results and rejects malformed session IDs at each adapter boundary', async () => { + const runtime = await createAgentTestRuntime(); + const client = await createMcpTestClient(runtime); + try { + const mcpMissing = normalizeMcpSuccess( + await client.callTool('session_captures_list', { + sessionId: 'missing-session', + limit: 100, + }), + ); + const cliMissing = await runRelayCli(runtime, [ + 'session', + 'captures', + '--session', + 'missing-session', + '--output', + 'json', + ]); + expect(cliMissing.exitCode).toBe(0); + expect(normalizeCliSuccess(cliMissing.json)).toEqual(mcpMissing); + expect(mcpMissing.data).toMatchObject({ sessionId: 'missing-session', tasks: [], count: 0 }); + + const cliMalformed = await runRelayCli(runtime, [ + 'session', + 'captures', + '--session', + 'bad session', + '--output', + 'json', + ]); + expect(cliMalformed.exitCode).toBe(2); + expect(cliMalformed.json).toMatchObject({ error: { code: 'VALIDATION_ERROR' } }); + await expect( + client.callTool('session_captures_list', { sessionId: 'bad session', limit: 100 }), + ).resolves.toMatchObject({ + isError: true, + content: [{ type: 'text', text: expect.stringMatching(/invalid arguments|sessionId/i) }], + }); + } finally { + await client.close(); + await runtime.close(); + } + }); + it('maps a deterministic unusable database parent without touching the default database', async () => { const runtime = await createAgentTestRuntime(); const databaseParent = dirname(runtime.databasePath); diff --git a/tests/support/cli-test-process.ts b/tests/support/cli-test-process.ts index 84862e3..ba7d7b4 100644 --- a/tests/support/cli-test-process.ts +++ b/tests/support/cli-test-process.ts @@ -5,6 +5,7 @@ import type { AgentTestRuntime } from './agent-test-runtime.js'; export interface CliProcessOptions { readonly cwd?: string; readonly environment?: NodeJS.ProcessEnv; + readonly timeoutMs?: number; } export interface CliProcessResult { @@ -37,8 +38,22 @@ export function runRelayCli( }); return new Promise((resolve, reject) => { - child.once('error', reject); + let settled = false; + const timeout = setTimeout(() => { + settled = true; + child.kill(); + reject(new Error(`Relay CLI timed out after ${options.timeoutMs ?? 30_000}ms.`)); + }, options.timeoutMs ?? 30_000); + child.once('error', (error) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + reject(error); + }); child.once('close', (code) => { + if (settled) return; + settled = true; + clearTimeout(timeout); try { resolve({ exitCode: code ?? 1, diff --git a/tests/support/external-contract-normalizers.ts b/tests/support/external-contract-normalizers.ts index ed60e2c..e0fa101 100644 --- a/tests/support/external-contract-normalizers.ts +++ b/tests/support/external-contract-normalizers.ts @@ -69,7 +69,6 @@ function number(value: unknown): number { } function array(value: unknown): readonly unknown[] { - if (value === undefined) return []; if (!Array.isArray(value)) throw new Error('Expected an array contract field.'); return value; } diff --git a/tests/support/mcp-test-client.ts b/tests/support/mcp-test-client.ts index 9fa588e..934874a 100644 --- a/tests/support/mcp-test-client.ts +++ b/tests/support/mcp-test-client.ts @@ -12,6 +12,8 @@ export interface McpTestClient { export interface McpTestClientOptions { readonly cwd?: string; + readonly command?: string; + readonly args?: readonly string[]; readonly environment?: NodeJS.ProcessEnv; } @@ -20,8 +22,8 @@ export async function createMcpTestClient( options: McpTestClientOptions = {}, ): Promise { const transport = new StdioClientTransport({ - command: process.execPath, - args: [join(runtime.checkoutPath, 'dist', 'mcp', 'main.js')], + command: options.command ?? process.execPath, + args: [...(options.args ?? [join(runtime.checkoutPath, 'dist', 'mcp', 'main.js')])], cwd: options.cwd ?? runtime.checkoutPath, env: stringEnvironment(runtime.environment(options.environment)), stderr: 'pipe', @@ -65,6 +67,8 @@ export async function createMcpTestClient( function stringEnvironment(environment: NodeJS.ProcessEnv): Record { return Object.fromEntries( - Object.entries(environment).filter((entry): entry is [string, string] => entry[1] !== undefined), + Object.entries(environment).filter( + (entry): entry is [string, string] => entry[1] !== undefined, + ), ); } diff --git a/tests/unit/scripts/validate-repository-assets.test.ts b/tests/unit/scripts/validate-repository-assets.test.ts index 20ac27b..a2035ff 100644 --- a/tests/unit/scripts/validate-repository-assets.test.ts +++ b/tests/unit/scripts/validate-repository-assets.test.ts @@ -74,11 +74,11 @@ function createFixtureRoot(): string { }); writeFileSync( join(rootDir, 'skills/relay-capture/SKILL.md'), - `## Purpose\n\nCapture a concrete, actionable follow-up.\n\n## When to capture\n\nUse it for a concrete, actionable follow-up.\n\n## Adapter selection\n\nMCP is preferred. CLI is the fallback with --output json and one adapter.\n\n## Session and provenance\n\nThe agent supplies createdByName and the exact active session ID. Relay supplies createdByType: AGENT and status: INBOX.\n\n## Capture procedure\n\nContinue the original work.\n\n## Duplicate handling\n\nA duplicate is advisory.\n\n## Context safety\n\nKeep context concise.\n\n## Autonomy boundaries\n\nAn agent must not edit, triage, start, complete, or archive tasks. Leave captures in INBOX.\n\n## Do not capture\n\nDo not capture speculation.\n`, + `## Purpose\n\nCapture a concrete, actionable follow-up.\n\n## When to capture\n\nUse it for a concrete, actionable follow-up.\n\n## Adapter selection\n\nMCP is preferred. CLI is the fallback with --output json and one adapter.\n\n## Session and provenance\n\nThe agent supplies createdByName and the exact active session ID. Relay supplies createdByType: AGENT and status: INBOX.\n\n## Capture procedure\n\nContinue the original work. An agent may autonomously create only a new Relay task in INBOX.\n\n## Duplicate handling\n\nA duplicate is advisory.\n\n## Context safety\n\nKeep context concise.\n\n## Autonomy boundaries\n\nAn agent must not edit, triage, start, complete, archive, delete, merge, or move any task. Leave captures in INBOX.\n\n## Do not capture\n\nDo not capture speculation.\n`, ); writeFileSync( join(rootDir, 'skills/relay-session-review/SKILL.md'), - `## Purpose\n\nReview before final completion.\n\n## When to review\n\nAlways perform the exact active session lookup before final completion.\n\n## Session lookup\n\nUse the exact active session ID. Include completed and archived tasks; never mix sessions. An empty authoritative result is valid.\n\n## Review presentation\n\nPresent captures.\n\n## User-directed actions\n\nRequire explicit user direction and intent-specific actions.\n\n## Unresolved captures\n\nLeave unresolved tasks in INBOX.\n\n## Adapter selection\n\nUse the same adapter.\n\n## Prohibited behaviour\n\nNever infer completion from timer, inactivity, or process exit.\n`, + `## Purpose\n\nReview before final completion.\n\n## When to review\n\nAlways perform the exact active session lookup before final completion.\n\n## Session lookup\n\nUse the exact active session ID. Include completed and archived captures; never mix sessions. An empty authoritative result is valid.\n\n## Review presentation\n\nPresent captures.\n\n## User-directed actions\n\nRequire explicit user direction and intent-specific actions.\n\n## Unresolved captures\n\nLeave unresolved tasks in INBOX.\n\n## Adapter selection\n\nUse the same adapter.\n\n## Prohibited behaviour\n\nNever infer completion from timer, inactivity, or process exit.\n`, ); for (const [path, name, description] of [ ['skills/relay-capture/SKILL.md', 'relay-capture', 'Use when testing capture.'], diff --git a/tests/unit/support/cli-test-process.test.ts b/tests/unit/support/cli-test-process.test.ts index 0419fa3..d369fb9 100644 --- a/tests/unit/support/cli-test-process.test.ts +++ b/tests/unit/support/cli-test-process.test.ts @@ -33,4 +33,15 @@ describe('runRelayCli', () => { await runtime.close(); } }); + + it('terminates a child process that exceeds the configured timeout', async () => { + const runtime = await createAgentTestRuntime(); + try { + await expect( + runRelayCli(runtime, ['task', 'list', '--output', 'json'], { timeoutMs: 1 }), + ).rejects.toThrow(/timed out/i); + } finally { + await runtime.close(); + } + }); }); From 9d2b1c770c86a3d460302a313298337c656dd93b Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Thu, 30 Jul 2026 00:15:23 +0530 Subject: [PATCH 08/16] style: satisfy repository formatting gate --- docs/agent-integration-verification.md | 36 +++++------ ...e-25-mcp-cli-compatibility-verification.md | 7 +++ scripts/validate-agent-integration-assets.ts | 31 ++++++++-- .../validate-agent-integration-assets.test.ts | 59 +++++++++++++++---- 4 files changed, 96 insertions(+), 37 deletions(-) diff --git a/docs/agent-integration-verification.md b/docs/agent-integration-verification.md index 0b588e3..bbb3cbc 100644 --- a/docs/agent-integration-verification.md +++ b/docs/agent-integration-verification.md @@ -6,24 +6,24 @@ This evidence covers the source-checkout Relay MCP and CLI adapters, their share ## Automated scenario matrix -| Scenario | Automated test or manual step | Result | Evidence | -| ------------------------------------------------------------------- | -------------------------------------------------------------------------- | ------ | -------------------------------------------------------------------------------- | -| 1. MCP capture then CLI retrieval | `mcp-cli-parity.test.ts` — MCP capture followed by built CLI get | PASS | Built `dist/mcp/main.js` and `dist/cli/main.js`, same disposable `RELAY_DB_PATH` | -| 2. CLI capture then MCP retrieval | `mcp-cli-parity.test.ts` — CLI capture followed by MCP get | PASS | Complete public task DTO equality | -| 3. Task list/get fields and ordering | `mcp-cli-parity.test.ts` — list DTO comparison and persisted ordering | PASS | Transport-only envelope normalization | -| 4. Session-ID isolation | `agent-workflow-e2e.test.ts` — alpha and beta captures | PASS | Exact session filtering | -| 5. Completed and archived session review | `agent-workflow-e2e.test.ts` — all-status review | PASS | Open, DONE, and ARCHIVED captures returned | -| 6. Missing session behavior | `mcp-cli-parity.test.ts` and existing adapter contract tests | PASS | Stable missing-task/session contract coverage | -| 7. Malformed session behavior | `mcp-cli-parity.test.ts` and existing strict schema tests | PASS | MCP validation boundary and CLI validation envelope | -| 8. Duplicate candidates, warnings, and match reasons | `mcp-cli-parity.test.ts` — duplicate capture and find-similar | PASS | Advisory warning and deterministic candidate assertions | -| 9. Edit parity | `mcp-cli-parity.test.ts` — CLI edit and MCP readback | PASS | Complete task and change metadata | -| 10. Triage/start/complete/archive parity | `mcp-cli-parity.test.ts` — cross-adapter lifecycle sequence | PASS | Focused lifecycle actions and statuses | -| 11. No-op metadata | `mcp-cli-parity.test.ts` — repeated edit | PASS | `NO_CHANGE`, empty fields, unchanged timestamps | -| 12. Validation, transition, archived, not-found, and storage errors | `mcp-cli-parity.test.ts` — stable errors and unusable parent path | PASS | CLI envelopes; MCP execution/protocol errors; startup failure remains stderr-only; leakage checks | -| 13. CLI JSON schemas and exit codes | `cli-test-process.test.ts` and built parity tests | PASS | One JSON document, separated stderr, exit codes 0/2/3/4/5 | -| 14. One database across HTTP, MCP, and CLI | `database-path-parity.test.ts` | PASS | Arbitrary CWDs, HTTP runtime, restart persistence, no CWD-local DB | -| 15. Skill and vendor-wrapper drift | `validate-agent-integration-assets.test.ts` and `validate:assets` | PASS | 33 validator tests; canonical policy and entry-point checks | -| 16. Integration removal preserves data | `agent-workflow-e2e.test.ts` config-driven disposable MCP launch and removal | PASS | Parsed `.mcp.json` launches the built server; removal is followed by retrieval from the same DB | +| Scenario | Automated test or manual step | Result | Evidence | +| ------------------------------------------------------------------- | ---------------------------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------- | +| 1. MCP capture then CLI retrieval | `mcp-cli-parity.test.ts` — MCP capture followed by built CLI get | PASS | Built `dist/mcp/main.js` and `dist/cli/main.js`, same disposable `RELAY_DB_PATH` | +| 2. CLI capture then MCP retrieval | `mcp-cli-parity.test.ts` — CLI capture followed by MCP get | PASS | Complete public task DTO equality | +| 3. Task list/get fields and ordering | `mcp-cli-parity.test.ts` — list DTO comparison and persisted ordering | PASS | Transport-only envelope normalization | +| 4. Session-ID isolation | `agent-workflow-e2e.test.ts` — alpha and beta captures | PASS | Exact session filtering | +| 5. Completed and archived session review | `agent-workflow-e2e.test.ts` — all-status review | PASS | Open, DONE, and ARCHIVED captures returned | +| 6. Missing session behavior | `mcp-cli-parity.test.ts` and existing adapter contract tests | PASS | Stable missing-task/session contract coverage | +| 7. Malformed session behavior | `mcp-cli-parity.test.ts` and existing strict schema tests | PASS | MCP validation boundary and CLI validation envelope | +| 8. Duplicate candidates, warnings, and match reasons | `mcp-cli-parity.test.ts` — duplicate capture and find-similar | PASS | Advisory warning and deterministic candidate assertions | +| 9. Edit parity | `mcp-cli-parity.test.ts` — CLI edit and MCP readback | PASS | Complete task and change metadata | +| 10. Triage/start/complete/archive parity | `mcp-cli-parity.test.ts` — cross-adapter lifecycle sequence | PASS | Focused lifecycle actions and statuses | +| 11. No-op metadata | `mcp-cli-parity.test.ts` — repeated edit | PASS | `NO_CHANGE`, empty fields, unchanged timestamps | +| 12. Validation, transition, archived, not-found, and storage errors | `mcp-cli-parity.test.ts` — stable errors and unusable parent path | PASS | CLI envelopes; MCP execution/protocol errors; startup failure remains stderr-only; leakage checks | +| 13. CLI JSON schemas and exit codes | `cli-test-process.test.ts` and built parity tests | PASS | One JSON document, separated stderr, exit codes 0/2/3/4/5 | +| 14. One database across HTTP, MCP, and CLI | `database-path-parity.test.ts` | PASS | Arbitrary CWDs, HTTP runtime, restart persistence, no CWD-local DB | +| 15. Skill and vendor-wrapper drift | `validate-agent-integration-assets.test.ts` and `validate:assets` | PASS | 33 validator tests; canonical policy and entry-point checks | +| 16. Integration removal preserves data | `agent-workflow-e2e.test.ts` config-driven disposable MCP launch and removal | PASS | Parsed `.mcp.json` launches the built server; removal is followed by retrieval from the same DB | ## Clean-checkout environment diff --git a/docs/superpowers/plans/2026-07-29-issue-25-mcp-cli-compatibility-verification.md b/docs/superpowers/plans/2026-07-29-issue-25-mcp-cli-compatibility-verification.md index 89ba085..46fbbb7 100644 --- a/docs/superpowers/plans/2026-07-29-issue-25-mcp-cli-compatibility-verification.md +++ b/docs/superpowers/plans/2026-07-29-issue-25-mcp-cli-compatibility-verification.md @@ -668,12 +668,19 @@ Use these exact top-level sections: # Agent Integration Verification ## Scope and safety statement + ## Automated scenario matrix + ## Clean-checkout environment + ## Codex validation + ## Claude Code validation + ## Cross-client differences and limitations + ## Data and configuration preservation + ## Epic #2 closure checklist ``` diff --git a/scripts/validate-agent-integration-assets.ts b/scripts/validate-agent-integration-assets.ts index a54f742..110fa8c 100644 --- a/scripts/validate-agent-integration-assets.ts +++ b/scripts/validate-agent-integration-assets.ts @@ -75,8 +75,13 @@ function validateCompatibilityClaims(shared: string): void { function validateVendorClaims(rootDir: string, shared: string): void { for (const readme of vendorReadmes) { const text = readAsset(rootDir, `integrations/${readme}/README.md`); - if (/^## Autonomy boundaries$/im.test(text) || /autonomously\s+(?:edit|triage|start|complete|archive|delete|merge)/i.test(text)) { - fail(`${readme} must reference canonical behavioural policy instead of redefining mutation autonomy.`); + if ( + /^## Autonomy boundaries$/im.test(text) || + /autonomously\s+(?:edit|triage|start|complete|archive|delete|merge)/i.test(text) + ) { + fail( + `${readme} must reference canonical behavioural policy instead of redefining mutation autonomy.`, + ); } const claimsNoLiveTest = /(?:live smoke test|live validation)[^\n]*not completed/i.test(text); const claimsLiveEvidence = @@ -106,14 +111,20 @@ function validateCanonicalSkills(rootDir: string): void { const capture = readAsset(rootDir, canonicalSkills[0]); if (!/autonomously create only a new Relay task in `?INBOX`?/i.test(capture)) fail('Canonical capture skill must define autonomous creation as INBOX-only.'); - if (!/must not edit, triage, start, complete, archive, delete, merge, or move any task/i.test(capture)) + if ( + !/must not edit, triage, start, complete, archive, delete, merge, or move any task/i.test( + capture, + ) + ) fail('Canonical capture skill must prohibit autonomous lifecycle mutation.'); if (/autonomously\s+(?:edit|triage|start|complete|archive|delete|merge)/i.test(capture)) fail('Canonical capture skill must not grant autonomous lifecycle mutation.'); const review = readAsset(rootDir, canonicalSkills[1]); if (!/completed and archived captures/i.test(review)) - fail('Canonical session-review skill must require all-status retrieval including completed and archived captures.'); + fail( + 'Canonical session-review skill must require all-status retrieval including completed and archived captures.', + ); if (!/explicit user direction/i.test(review)) fail('Canonical session-review skill must require explicit user direction for mutations.'); } @@ -121,9 +132,17 @@ function validateCanonicalSkills(rootDir: string): void { function validateRemovalGuidance(rootDir: string): void { for (const readme of vendorReadmes) { const text = readAsset(rootDir, `integrations/${readme}/README.md`); - if (/(?:remove|delete)\s+(?:the\s+)?(?:SQLite\s+)?database\b|(?:SQLite\s+)?database\b[^\n]{0,80}\b(?:remove|delete)\b/i.test(text)) + if ( + /(?:remove|delete)\s+(?:the\s+)?(?:SQLite\s+)?database\b|(?:SQLite\s+)?database\b[^\n]{0,80}\b(?:remove|delete)\b/i.test( + text, + ) + ) fail(`${readme} removal guidance must not delete the SQLite database.`); - if (!/(?:remov(?:e|ing)|delete)[\s\S]{0,160}(?:configuration|integration|client assets?)/i.test(text)) + if ( + !/(?:remov(?:e|ing)|delete)[\s\S]{0,160}(?:configuration|integration|client assets?)/i.test( + text, + ) + ) fail(`${readme} removal guidance must distinguish configuration removal from stored data.`); if (!/SQLite database remains untouched/i.test(text)) fail(`${readme} removal guidance must state that the SQLite database remains untouched.`); diff --git a/tests/unit/scripts/validate-agent-integration-assets.test.ts b/tests/unit/scripts/validate-agent-integration-assets.test.ts index 19311a0..37089c8 100644 --- a/tests/unit/scripts/validate-agent-integration-assets.test.ts +++ b/tests/unit/scripts/validate-agent-integration-assets.test.ts @@ -231,9 +231,14 @@ describe('validateAgentIntegrationAssets', () => { it('rejects a canonical capture skill without autonomous-create permission', () => { const rootDir = createRoot(); const path = join(rootDir, 'skills/relay-capture/SKILL.md'); - writeFileSync(path, readFileSync(path, 'utf8').replace('autonomously create only', 'may create only')); + writeFileSync( + path, + readFileSync(path, 'utf8').replace('autonomously create only', 'may create only'), + ); - expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/canonical capture.*autonom/i); + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow( + /canonical capture.*autonom/i, + ); }); it('rejects a canonical capture skill that permits lifecycle mutation', () => { @@ -247,13 +252,18 @@ describe('validateAgentIntegrationAssets', () => { ), ); - expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/canonical capture.*lifecycle/i); + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow( + /canonical capture.*lifecycle/i, + ); }); it('rejects a canonical session-review skill that omits completed and archived captures', () => { const rootDir = createRoot(); const path = join(rootDir, 'skills/relay-session-review/SKILL.md'); - writeFileSync(path, readFileSync(path, 'utf8').replace('completed and archived captures', 'INBOX tasks')); + writeFileSync( + path, + readFileSync(path, 'utf8').replace('completed and archived captures', 'INBOX tasks'), + ); expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/session-review.*status/i); }); @@ -261,7 +271,10 @@ describe('validateAgentIntegrationAssets', () => { it('rejects a canonical session-review skill without explicit user-action guidance', () => { const rootDir = createRoot(); const path = join(rootDir, 'skills/relay-session-review/SKILL.md'); - writeFileSync(path, readFileSync(path, 'utf8').replaceAll('explicit user direction', 'automatic action')); + writeFileSync( + path, + readFileSync(path, 'utf8').replaceAll('explicit user direction', 'automatic action'), + ); expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/session-review.*explicit/i); }); @@ -274,13 +287,18 @@ describe('validateAgentIntegrationAssets', () => { `${readFileSync(path, 'utf8')}\n## Autonomy boundaries\nThe agent may autonomously edit and archive tasks.`, ); - expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/vendor.*policy|behavioural policy/i); + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow( + /vendor.*policy|behavioural policy/i, + ); }); it('rejects a vendor wrapper without both canonical skill references', () => { const rootDir = createRoot(); const path = join(rootDir, 'integrations/generic-mcp/README.md'); - writeFileSync(path, readFileSync(path, 'utf8').replace('skills/relay-session-review/SKILL.md', '')); + writeFileSync( + path, + readFileSync(path, 'utf8').replace('skills/relay-session-review/SKILL.md', ''), + ); expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/generic-mcp.*skill/i); }); @@ -288,17 +306,27 @@ describe('validateAgentIntegrationAssets', () => { it('rejects a configuration template that points at a non-canonical MCP entry', () => { const rootDir = createRoot(); const path = join(rootDir, 'integrations/generic-mcp/server-config.json.example'); - writeFileSync(path, readFileSync(path, 'utf8').replace('dist/mcp/main.js', 'dist/other-mcp.js')); + writeFileSync( + path, + readFileSync(path, 'utf8').replace('dist/mcp/main.js', 'dist/other-mcp.js'), + ); - expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/canonical.*dist\/mcp\/main\.js/i); + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow( + /canonical.*dist\/mcp\/main\.js/i, + ); }); it('rejects removal guidance that deletes the SQLite database', () => { const rootDir = createRoot(); const path = join(rootDir, 'integrations/generic-cli/README.md'); - writeFileSync(path, `${readFileSync(path, 'utf8')} Remove the SQLite database when disabling Relay.`); + writeFileSync( + path, + `${readFileSync(path, 'utf8')} Remove the SQLite database when disabling Relay.`, + ); - expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/removal.*database|destructive/i); + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow( + /removal.*database|destructive/i, + ); }); it('rejects removal guidance that does not distinguish configuration from stored data', () => { @@ -306,9 +334,14 @@ describe('validateAgentIntegrationAssets', () => { const path = join(rootDir, 'integrations/generic-cli/README.md'); writeFileSync( path, - readFileSync(path, 'utf8').replace('SQLite database remains untouched', 'Relay data may be deleted'), + readFileSync(path, 'utf8').replace( + 'SQLite database remains untouched', + 'Relay data may be deleted', + ), ); - expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/removal.*configuration|database remains/i); + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow( + /removal.*configuration|database remains/i, + ); }); }); From 51426255240b539adcbdf2ff63ca4c13e5464202 Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Thu, 30 Jul 2026 00:43:19 +0530 Subject: [PATCH 09/16] test: build node artifacts before process tests --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 181cb07..73361d6 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,8 @@ "relay-mcp": "./dist/mcp/main.js" }, "scripts": { - "test": "vitest run", - "test:coverage": "vitest run --coverage", + "test": "pnpm build:node && vitest run", + "test:coverage": "pnpm build:node && vitest run --coverage", "lint": "eslint . --max-warnings=0", "typecheck": "tsc --build --noEmit", "format": "prettier --write .", From b3703cabe1e1c420f78abf4f895020b4b017293d Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Thu, 30 Jul 2026 00:44:33 +0530 Subject: [PATCH 10/16] test: create agent runtime temporary parent --- tests/support/agent-test-runtime.ts | 4 +++- tests/unit/support/agent-test-runtime.test.ts | 22 +++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/support/agent-test-runtime.ts b/tests/support/agent-test-runtime.ts index 2731f62..9b50f44 100644 --- a/tests/support/agent-test-runtime.ts +++ b/tests/support/agent-test-runtime.ts @@ -19,7 +19,9 @@ const checkoutPath = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); export async function createAgentTestRuntime( options: AgentTestRuntimeOptions = {}, ): Promise { - const root = await mkdtemp(join(checkoutPath, 'tmp', 'relay-agent-verification-')); + const temporaryRoot = join(checkoutPath, 'tmp'); + await mkdir(temporaryRoot, { recursive: true }); + const root = await mkdtemp(join(temporaryRoot, 'relay-agent-verification-')); const dataDirectory = join(root, 'data'); const workingDirectoryRoot = join(root, 'cwd'); await Promise.all([mkdir(dataDirectory), mkdir(workingDirectoryRoot)]); diff --git a/tests/unit/support/agent-test-runtime.test.ts b/tests/unit/support/agent-test-runtime.test.ts index 3eb87b8..5e6d716 100644 --- a/tests/unit/support/agent-test-runtime.test.ts +++ b/tests/unit/support/agent-test-runtime.test.ts @@ -1,6 +1,7 @@ -import { stat } from 'node:fs/promises'; +import { rm, stat } from 'node:fs/promises'; import { homedir } from 'node:os'; -import { dirname, isAbsolute } from 'node:path'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; import { createAgentTestRuntime } from '../../support/agent-test-runtime.js'; @@ -24,6 +25,23 @@ describe('createAgentTestRuntime', () => { } }); + it('creates the shared temporary parent when it is absent', async () => { + const repositoryTemporaryRoot = resolve( + dirname(fileURLToPath(import.meta.url)), + '../../..', + 'tmp', + ); + await rm(repositoryTemporaryRoot, { recursive: true, force: true }); + + const runtime = await createAgentTestRuntime(); + try { + expect(runtime.databasePath).toContain(join('tmp', 'relay-agent-verification-')); + await expect(stat(dirname(runtime.databasePath))).resolves.toBeDefined(); + } finally { + await runtime.close(); + } + }); + it('removes the disposable directory including SQLite sidecars and closes idempotently', async () => { const runtime = await createAgentTestRuntime(); const root = dirname(dirname(runtime.databasePath)); From cc3380ad20d0edff265b628b7132e2902e8f45bb Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Thu, 30 Jul 2026 00:49:33 +0530 Subject: [PATCH 11/16] test: verify structured storage error parity --- tests/integration/mcp-cli-parity.test.ts | 65 +++++++++++++++++++----- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/tests/integration/mcp-cli-parity.test.ts b/tests/integration/mcp-cli-parity.test.ts index a5de90e..cbb21a6 100644 --- a/tests/integration/mcp-cli-parity.test.ts +++ b/tests/integration/mcp-cli-parity.test.ts @@ -1,6 +1,5 @@ +import Database from 'better-sqlite3'; import { describe, expect, it, vi } from 'vitest'; -import { dirname } from 'node:path'; -import { rm, writeFile } from 'node:fs/promises'; import { createTaskApplication, type TaskApplication, @@ -655,21 +654,63 @@ describe('built MCP and CLI contract parity', () => { } }); - it('maps a deterministic unusable database parent without touching the default database', async () => { + it('maps a locked-database write failure to the same structured storage error', async () => { const runtime = await createAgentTestRuntime(); - const databaseParent = dirname(runtime.databasePath); - await rm(databaseParent, { recursive: true, force: true }); - await writeFile(databaseParent, 'not a directory'); + const client = await createMcpTestClient(runtime); + const lock = new Database(runtime.databasePath); + try { - const cli = await runRelayCli(runtime, ['task', 'list', '--output', 'json']); + await client.callTool('relay_health', {}); + + lock.pragma('busy_timeout = 100'); + lock.exec('BEGIN IMMEDIATE'); + + const [cli, mcp] = await Promise.all([ + runRelayCli(runtime, [ + 'task', + 'capture', + '--title', + 'Locked CLI capture', + '--agent', + 'Codex', + '--session', + 'session-alpha', + '--output', + 'json', + ]), + client.callTool('task_capture', { + title: 'Locked MCP capture', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ]); + expect(cli.exitCode).toBe(5); - expect(cli.json).toMatchObject({ error: { code: 'STORAGE_ERROR' } }); - expect(cli.stderr).not.toMatch(/SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\/i); + expect(cli.json).toMatchObject({ + error: { code: 'STORAGE_ERROR', message: expect.any(String) }, + }); + expect(mcp).toMatchObject({ + isError: true, + structuredContent: { + error: { code: 'STORAGE_ERROR', message: expect.any(String) }, + }, + }); + expect(normalizeCliError(cli.json)).toEqual(normalizeMcpError(mcp)); - await expect(createMcpTestClient(runtime)).rejects.toThrow(); + for (const value of [cli.json, mcp, cli.stderr, client.stderr()]) { + expect(JSON.stringify(value)).not.toMatch( + /SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\|\/Users\//i, + ); + } } finally { - await rm(databaseParent, { force: true }); + try { + lock.exec('ROLLBACK'); + } catch { + // The transaction may already be closed after a setup failure. + } + lock.close(); + await client.close(); await runtime.close(); } - }); + }, 15_000); }); From e190f1fc1eb8ebf1e28e5085d3a3d3051f3c4647 Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Thu, 30 Jul 2026 00:51:10 +0530 Subject: [PATCH 12/16] docs: record verified storage error parity --- docs/agent-integration-verification.md | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/agent-integration-verification.md b/docs/agent-integration-verification.md index bbb3cbc..7890e16 100644 --- a/docs/agent-integration-verification.md +++ b/docs/agent-integration-verification.md @@ -6,24 +6,24 @@ This evidence covers the source-checkout Relay MCP and CLI adapters, their share ## Automated scenario matrix -| Scenario | Automated test or manual step | Result | Evidence | -| ------------------------------------------------------------------- | ---------------------------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------- | -| 1. MCP capture then CLI retrieval | `mcp-cli-parity.test.ts` — MCP capture followed by built CLI get | PASS | Built `dist/mcp/main.js` and `dist/cli/main.js`, same disposable `RELAY_DB_PATH` | -| 2. CLI capture then MCP retrieval | `mcp-cli-parity.test.ts` — CLI capture followed by MCP get | PASS | Complete public task DTO equality | -| 3. Task list/get fields and ordering | `mcp-cli-parity.test.ts` — list DTO comparison and persisted ordering | PASS | Transport-only envelope normalization | -| 4. Session-ID isolation | `agent-workflow-e2e.test.ts` — alpha and beta captures | PASS | Exact session filtering | -| 5. Completed and archived session review | `agent-workflow-e2e.test.ts` — all-status review | PASS | Open, DONE, and ARCHIVED captures returned | -| 6. Missing session behavior | `mcp-cli-parity.test.ts` and existing adapter contract tests | PASS | Stable missing-task/session contract coverage | -| 7. Malformed session behavior | `mcp-cli-parity.test.ts` and existing strict schema tests | PASS | MCP validation boundary and CLI validation envelope | -| 8. Duplicate candidates, warnings, and match reasons | `mcp-cli-parity.test.ts` — duplicate capture and find-similar | PASS | Advisory warning and deterministic candidate assertions | -| 9. Edit parity | `mcp-cli-parity.test.ts` — CLI edit and MCP readback | PASS | Complete task and change metadata | -| 10. Triage/start/complete/archive parity | `mcp-cli-parity.test.ts` — cross-adapter lifecycle sequence | PASS | Focused lifecycle actions and statuses | -| 11. No-op metadata | `mcp-cli-parity.test.ts` — repeated edit | PASS | `NO_CHANGE`, empty fields, unchanged timestamps | -| 12. Validation, transition, archived, not-found, and storage errors | `mcp-cli-parity.test.ts` — stable errors and unusable parent path | PASS | CLI envelopes; MCP execution/protocol errors; startup failure remains stderr-only; leakage checks | -| 13. CLI JSON schemas and exit codes | `cli-test-process.test.ts` and built parity tests | PASS | One JSON document, separated stderr, exit codes 0/2/3/4/5 | -| 14. One database across HTTP, MCP, and CLI | `database-path-parity.test.ts` | PASS | Arbitrary CWDs, HTTP runtime, restart persistence, no CWD-local DB | -| 15. Skill and vendor-wrapper drift | `validate-agent-integration-assets.test.ts` and `validate:assets` | PASS | 33 validator tests; canonical policy and entry-point checks | -| 16. Integration removal preserves data | `agent-workflow-e2e.test.ts` config-driven disposable MCP launch and removal | PASS | Parsed `.mcp.json` launches the built server; removal is followed by retrieval from the same DB | +| Scenario | Automated test or manual step | Result | Evidence | +| ------------------------------------------------------------------- | ---------------------------------------------------------------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1. MCP capture then CLI retrieval | `mcp-cli-parity.test.ts` — MCP capture followed by built CLI get | PASS | Built `dist/mcp/main.js` and `dist/cli/main.js`, same disposable `RELAY_DB_PATH` | +| 2. CLI capture then MCP retrieval | `mcp-cli-parity.test.ts` — CLI capture followed by MCP get | PASS | Complete public task DTO equality | +| 3. Task list/get fields and ordering | `mcp-cli-parity.test.ts` — list DTO comparison and persisted ordering | PASS | Transport-only envelope normalization | +| 4. Session-ID isolation | `agent-workflow-e2e.test.ts` — alpha and beta captures | PASS | Exact session filtering | +| 5. Completed and archived session review | `agent-workflow-e2e.test.ts` — all-status review | PASS | Open, DONE, and ARCHIVED captures returned | +| 6. Missing session behavior | `mcp-cli-parity.test.ts` and existing adapter contract tests | PASS | Stable missing-task/session contract coverage | +| 7. Malformed session behavior | `mcp-cli-parity.test.ts` and existing strict schema tests | PASS | MCP validation boundary and CLI validation envelope | +| 8. Duplicate candidates, warnings, and match reasons | `mcp-cli-parity.test.ts` — duplicate capture and find-similar | PASS | Advisory warning and deterministic candidate assertions | +| 9. Edit parity | `mcp-cli-parity.test.ts` — CLI edit and MCP readback | PASS | Complete task and change metadata | +| 10. Triage/start/complete/archive parity | `mcp-cli-parity.test.ts` — cross-adapter lifecycle sequence | PASS | Focused lifecycle actions and statuses | +| 11. No-op metadata | `mcp-cli-parity.test.ts` — repeated edit | PASS | `NO_CHANGE`, empty fields, unchanged timestamps | +| 12. Validation, transition, archived, not-found, and storage errors | `mcp-cli-parity.test.ts` — stable errors and locked-database write failure | PASS | Built MCP and CLI writes ran concurrently against one initialized disposable SQLite database while a separate connection held `BEGIN IMMEDIATE`; both returned equal normalized `STORAGE_ERROR` contracts, CLI exit code 5, and sanitized external output | +| 13. CLI JSON schemas and exit codes | `cli-test-process.test.ts` and built parity tests | PASS | One JSON document, separated stderr, exit codes 0/2/3/4/5 | +| 14. One database across HTTP, MCP, and CLI | `database-path-parity.test.ts` | PASS | Arbitrary CWDs, HTTP runtime, restart persistence, no CWD-local DB | +| 15. Skill and vendor-wrapper drift | `validate-agent-integration-assets.test.ts` and `validate:assets` | PASS | 33 validator tests; canonical policy and entry-point checks | +| 16. Integration removal preserves data | `agent-workflow-e2e.test.ts` config-driven disposable MCP launch and removal | PASS | Parsed `.mcp.json` launches the built server; removal is followed by retrieval from the same DB | ## Clean-checkout environment From c64efa8c12b682f5615fb35275f59ee9435b5644 Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Thu, 30 Jul 2026 01:03:31 +0530 Subject: [PATCH 13/16] docs: track PR 34 remediation tasks --- .../tasks/2026-07-30-pr-34-review-tracker.md | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 docs/superpowers/tasks/2026-07-30-pr-34-review-tracker.md diff --git a/docs/superpowers/tasks/2026-07-30-pr-34-review-tracker.md b/docs/superpowers/tasks/2026-07-30-pr-34-review-tracker.md new file mode 100644 index 0000000..1dc7868 --- /dev/null +++ b/docs/superpowers/tasks/2026-07-30-pr-34-review-tracker.md @@ -0,0 +1,87 @@ +# PR #34 Review Task Tracker + +Source: [Luna remediation implementation plan](https://github.com/krishna916/relay/pull/34#issuecomment-5122267820) + +Scope: complete every actionable task in the linked plan while preserving issue #25's acceptance claims, application contracts, adapter schemas, database defaults, canonical skills, and vendor policies. + +## Working rules + +- [x] Verify each task against the current checkout before editing. +- [x] Keep implementation changes limited to the files named by the plan, plus this requested local tracker. +- [x] Use test-first changes and record focused evidence for each task. +- [x] Do not commit `dist/`, coverage output, temporary runtimes, or default Relay data. +- [x] Do not modify client configuration or claim live Codex/Claude validation without evidence. +- [x] Do not reply to or resolve the GitHub review thread unless explicitly requested after local work is complete. + +## Task 1 - Build Node artifacts before process tests + +- [x] Record the clean-state baseline failure: with `dist/` absent, the pre-fix command ran Vitest without built entries and reported 19 failed tests across 4 failed suites. +- [x] Update only `test` and `test:coverage` scripts to build Node artifacts first. +- [x] Run the focused CLI process test with `dist/` initially absent. +- [x] Run the focused MCP coverage test with `dist/` initially absent. +- [x] Commit the focused Task 1 change as `5142625`. + +## Task 2 - Create the repository temporary parent + +- [x] Add the missing-parent regression test before implementation. +- [x] Confirm the regression test fails with the expected `ENOENT` baseline. +- [x] Create `/tmp` recursively before `mkdtemp()`. +- [x] Preserve generated-root-only cleanup and path-escape protections. +- [x] Run focused runtime tests and typecheck. +- [x] Commit the focused Task 2 change as `b3703ca`. + +## Task 3 - Verify structured MCP/CLI storage-error parity + +- [x] Replace startup-rejection coverage with a post-initialization locked-database test. +- [x] Run CLI and MCP writes concurrently against the same `BEGIN IMMEDIATE` lock. +- [x] Assert CLI exit code `5`, structured `STORAGE_ERROR`, MCP structured `STORAGE_ERROR`, normalized equality, and sanitized output. +- [x] Run the complete MCP/CLI parity file. +- [x] Commit the focused Task 3 change as `cc3380a`. + +## Task 4 - Correct evidence claims + +- [x] Update scenario 12 to describe the verified locked-database parity evidence. +- [x] Remove the old unusable-parent/startup-rejection parity claim. +- [x] Run formatting and repository asset validation. +- [x] Commit the focused Task 4 change as `e190f1f`. + +## Task 5 - Clean-state verification and reconciliation + +- [x] Run `corepack pnpm install --frozen-lockfile` successfully with the pinned pnpm 10.2.0 toolchain after the initial sandbox relink permission failure was retried with filesystem escalation. +- [x] Verify `corepack pnpm test` works after generated output is removed: 39 files/512 tests passed and `dist/` was recreated by the script. +- [x] Verify `corepack pnpm test:coverage` works after generated output is removed: `verify` started from deleted `dist/`, rebuilt Node artifacts first, and completed coverage successfully. +- [x] Run `corepack pnpm verify` from a clean generated-output state: passed formatting, lint, typecheck, 512 tests, coverage, builds, asset validation, and the high-severity audit gate. +- [x] Run the three focused issue #25 integration suites: MCP/CLI parity 25 passed; agent workflow 3 passed; database-path parity 2 passed. +- [x] Confirm the worktree contains no tracked generated output or disposable runtime changes; only pre-existing `.codegraph/` remains outside the task and the tracker is the requested local artifact. +- [x] Reconcile this tracker against the final acceptance checklist below. +- [x] Keep PR claims/documentation honest; do not publish GitHub replies or resolve the thread in this pass. + +## Verification log + +| Check | Result | Evidence | +| --------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Baseline | Pass | PR #34 comment and review context fetched; branch is `feature/issue-25-mcp-cli-compatibility`; only pre-existing untracked `.codegraph/` is outside this task. | +| Task 1 focused checks | Pass | Clean `dist/` then `corepack pnpm test -- tests/unit/support/cli-test-process.test.ts`: 39 files/511 tests passed; clean `dist/` then `corepack pnpm test:coverage -- tests/unit/support/mcp-test-client.test.ts`: 39 files/511 tests passed; coverage 88.74/81.34/88.94/90.90. | +| Task 2 focused checks | Pass | Red regression produced `ENOENT`; `vitest run tests/unit/support/agent-test-runtime.test.ts`: 3 tests passed; `corepack pnpm typecheck`: passed. | +| Task 3 focused checks | Pass | Focused lock test: 1 passed/24 skipped; complete `mcp-cli-parity.test.ts`: 25 passed. | +| Task 4 checks | Pass | `corepack pnpm format:check` passed; `corepack pnpm validate:assets` passed. | +| Full clean-state gate | Pass | `corepack pnpm verify` from deleted `dist/`: 39 files, 512 tests, 88.74% statements / 81.34% branches / 88.94% functions / 90.90% lines; build, assets, and audit gate passed. | + +## Final acceptance checklist + +- [x] `pnpm test` works when `dist/` is initially absent. +- [x] `pnpm test:coverage` works when `dist/` is initially absent. +- [x] `corepack pnpm verify` passes from a clean generated-output state. +- [x] `createAgentTestRuntime()` succeeds when `/tmp` is initially absent. +- [x] Runtime cleanup removes the generated unique directory and SQLite sidecars. +- [x] Storage failure occurs after MCP initialization, not during transport startup. +- [x] CLI returns exit code `5` and structured `STORAGE_ERROR`. +- [x] MCP returns structured `STORAGE_ERROR`. +- [x] Normalized CLI and MCP storage errors are equal. +- [x] External output contains no SQL, stack, database path, or user-directory leakage. +- [x] Documentation claims match the verified behavior; live Codex/Claude validation remains explicitly unverified. +- [x] Existing 80% coverage thresholds and all quality gates remain unchanged. + +## Publication status + +Local implementation and verification only. GitHub review replies/thread resolution are intentionally not performed by this tracker unless separately requested. From 2ffc16649d4dac3fc27254ec216bb9496a3aee25 Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Thu, 30 Jul 2026 01:06:01 +0530 Subject: [PATCH 14/16] docs: record PR 34 publication status --- docs/superpowers/tasks/2026-07-30-pr-34-review-tracker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/superpowers/tasks/2026-07-30-pr-34-review-tracker.md b/docs/superpowers/tasks/2026-07-30-pr-34-review-tracker.md index 1dc7868..368d086 100644 --- a/docs/superpowers/tasks/2026-07-30-pr-34-review-tracker.md +++ b/docs/superpowers/tasks/2026-07-30-pr-34-review-tracker.md @@ -84,4 +84,4 @@ Scope: complete every actionable task in the linked plan while preserving issue ## Publication status -Local implementation and verification only. GitHub review replies/thread resolution are intentionally not performed by this tracker unless separately requested. +Verified commits `5142625`, `b3703ca`, `cc3380a`, `e190f1f`, and `c64efa8` are pushed to `origin/feature/issue-25-mcp-cli-compatibility`. PR #34 remains draft and its description records the clean-checkout verification evidence. The GitHub review thread remains unresolved; no follow-up comment was posted because that remote write requires an explicit user request. From c4aa0c0b081c6248df991fff400a339540f7c96e Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Thu, 30 Jul 2026 08:11:40 +0530 Subject: [PATCH 15/16] test: make agent runtime isolation assertion repository-safe --- tests/unit/support/agent-test-runtime.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/unit/support/agent-test-runtime.test.ts b/tests/unit/support/agent-test-runtime.test.ts index 5e6d716..abc4316 100644 --- a/tests/unit/support/agent-test-runtime.test.ts +++ b/tests/unit/support/agent-test-runtime.test.ts @@ -1,17 +1,25 @@ import { rm, stat } from 'node:fs/promises'; -import { homedir } from 'node:os'; -import { dirname, isAbsolute, join, resolve } from 'node:path'; +import { dirname, isAbsolute, join, relative, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; +import { getDefaultDatabasePath } from '../../../src/database/database-config.js'; import { createAgentTestRuntime } from '../../support/agent-test-runtime.js'; describe('createAgentTestRuntime', () => { it('creates an isolated absolute database path and arbitrary working directories', async () => { const runtime = await createAgentTestRuntime(); try { - expect(runtime.databasePath).toMatch(/relay\.db$/); + const disposableRoot = dirname(dirname(runtime.databasePath)); + const repositoryTemporaryRoot = resolve( + dirname(fileURLToPath(import.meta.url)), + '../../..', + 'tmp', + ); + + expect(runtime.databasePath).toBe(join(disposableRoot, 'data', 'relay.db')); expect(isAbsolute(runtime.databasePath)).toBe(true); - expect(runtime.databasePath).not.toContain(homedir()); + expect(relative(repositoryTemporaryRoot, disposableRoot)).not.toMatch(/^\.\.(?:[\\/]|$)/); + expect(runtime.databasePath).not.toBe(getDefaultDatabasePath()); const cwd = await runtime.createWorkingDirectory('nested/client'); expect(isAbsolute(cwd)).toBe(true); From c80145c4e624537abaf105cec3c447776a7f5da1 Mon Sep 17 00:00:00 2001 From: Krishnamurti Pandaram Date: Thu, 30 Jul 2026 08:53:40 +0530 Subject: [PATCH 16/16] test: address PR 34 cleanup and contract review --- scripts/validate-agent-integration-assets.ts | 14 ++- tests/integration/agent-workflow-e2e.test.ts | 27 +++-- tests/integration/mcp-cli-parity.test.ts | 113 +++++++++++------- tests/support/cli-test-process.ts | 42 ++++++- .../support/external-contract-normalizers.ts | 19 +++ .../validate-agent-integration-assets.test.ts | 19 +++ .../external-contract-normalizers.test.ts | 37 ++++++ tests/unit/support/mcp-test-client.test.ts | 27 +++-- 8 files changed, 224 insertions(+), 74 deletions(-) create mode 100644 tests/unit/support/external-contract-normalizers.test.ts diff --git a/scripts/validate-agent-integration-assets.ts b/scripts/validate-agent-integration-assets.ts index 110fa8c..c3408e0 100644 --- a/scripts/validate-agent-integration-assets.ts +++ b/scripts/validate-agent-integration-assets.ts @@ -75,9 +75,15 @@ function validateCompatibilityClaims(shared: string): void { function validateVendorClaims(rootDir: string, shared: string): void { for (const readme of vendorReadmes) { const text = readAsset(rootDir, `integrations/${readme}/README.md`); + const affirmativeAutonomyText = text.replace( + /\b(?:(?:(?:must|should|shall|may|can|will|do(?:es)?|did)\s+)?not|never)\s+autonomously\s+(?:edit|triage|start|complete|archive|delete|merge)\b/gi, + '', + ); if ( /^## Autonomy boundaries$/im.test(text) || - /autonomously\s+(?:edit|triage|start|complete|archive|delete|merge)/i.test(text) + /autonomously\s+(?:edit|triage|start|complete|archive|delete|merge)\b/i.test( + affirmativeAutonomyText, + ) ) { fail( `${readme} must reference canonical behavioural policy instead of redefining mutation autonomy.`, @@ -132,9 +138,13 @@ function validateCanonicalSkills(rootDir: string): void { function validateRemovalGuidance(rootDir: string): void { for (const readme of vendorReadmes) { const text = readAsset(rootDir, `integrations/${readme}/README.md`); + const affirmativeRemovalText = text.replace( + /\b(?:(?:(?:must|should|shall|may|can|will|do(?:es)?|did)\s+)?not|never)\s+(?:remove|delete)\s+(?:the\s+)?(?:SQLite\s+)?database\b/gi, + '', + ); if ( /(?:remove|delete)\s+(?:the\s+)?(?:SQLite\s+)?database\b|(?:SQLite\s+)?database\b[^\n]{0,80}\b(?:remove|delete)\b/i.test( - text, + affirmativeRemovalText, ) ) fail(`${readme} removal guidance must not delete the SQLite database.`); diff --git a/tests/integration/agent-workflow-e2e.test.ts b/tests/integration/agent-workflow-e2e.test.ts index db7d345..23c0eca 100644 --- a/tests/integration/agent-workflow-e2e.test.ts +++ b/tests/integration/agent-workflow-e2e.test.ts @@ -98,52 +98,53 @@ describe('built agent workflow end to end', () => { it('persists tasks and mutations across short-lived MCP and CLI restarts', async () => { let runtime: AgentTestRuntime | undefined; let client: McpTestClient | undefined; - let taskId: string; + let taskId: string | undefined; try { runtime = await createAgentTestRuntime(); client = await createMcpTestClient(runtime, { cwd: await runtime.createWorkingDirectory('restart/mcp-1'), }); const capture = normalizeMcpSuccess( - await client!.callTool('task_capture', { + await client.callTool('task_capture', { title: 'Restart persistence task', createdByName: 'Codex', sessionId: 'session-alpha', }), ); taskId = String((capture.data as { task: { id: string } }).task.id); - } finally { - await client?.close(); - } + await client.close(); + client = undefined; - try { - const get = await runRelayCli(runtime, ['task', 'get', taskId!, '--output', 'json'], { + const get = await runRelayCli(runtime, ['task', 'get', taskId, '--output', 'json'], { cwd: await runtime.createWorkingDirectory('restart/cli-1'), }); expect(get.exitCode).toBe(0); const edit = await runRelayCli(runtime, [ 'task', 'edit', - taskId!, + taskId, '--description', 'survives restart', '--output', 'json', ]); expect(edit.exitCode).toBe(0); - } finally { - client = await createMcpTestClient(runtime!, { + + client = await createMcpTestClient(runtime, { cwd: await runtime.createWorkingDirectory('restart/mcp-2'), }); try { - const get = normalizeMcpSuccess(await client.callTool('task_get', { taskId: taskId! })); + const get = normalizeMcpSuccess(await client.callTool('task_get', { taskId })); expect(get.data).toMatchObject({ - task: { id: taskId!, description: 'survives restart', sessionId: 'session-alpha' }, + task: { id: taskId, description: 'survives restart', sessionId: 'session-alpha' }, }); } finally { await client.close(); - await runtime!.close(); + client = undefined; } + } finally { + await client?.close(); + await runtime?.close(); } }); diff --git a/tests/integration/mcp-cli-parity.test.ts b/tests/integration/mcp-cli-parity.test.ts index cbb21a6..15f0969 100644 --- a/tests/integration/mcp-cli-parity.test.ts +++ b/tests/integration/mcp-cli-parity.test.ts @@ -13,7 +13,9 @@ import { FixedIdGenerator, InMemoryTaskRepository, } from '../unit/application/tasks/task-test-fixtures.js'; +import type { AgentTestRuntime } from '../support/agent-test-runtime.js'; import { connectMcp } from '../unit/interfaces/mcp/mcp-test-utils.js'; +import type { McpTestClient } from '../support/mcp-test-client.js'; import { createAgentTestRuntime } from '../support/agent-test-runtime.js'; import { runRelayCli } from '../support/cli-test-process.js'; import { createMcpTestClient } from '../support/mcp-test-client.js'; @@ -257,11 +259,13 @@ describe('MCP and CLI semantic parity', () => { describe('built MCP and CLI contract parity', () => { it('captures through MCP and retrieves the identical public task through CLI', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime, { - cwd: await runtime.createWorkingDirectory('mcp-capture'), - }); + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('mcp-capture'), + }); const capture = await client.callTool('task_capture', { title: 'Built MCP capture', createdByName: 'Codex', @@ -281,15 +285,17 @@ describe('built MCP and CLI contract parity', () => { expect(normalizeCliSuccess(cli.json).data).toEqual({ task: capturedTask }); expect(cli.stderr).toBe(''); } finally { - await client.close(); - await runtime.close(); + await client?.close(); + await runtime?.close(); } }); it('captures through CLI and retrieves the identical public task through MCP', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime); + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); const cli = await runRelayCli(runtime, [ 'task', 'capture', @@ -314,15 +320,17 @@ describe('built MCP and CLI contract parity', () => { expect(normalizeMcpSuccess(mcp).data).toEqual({ task: capturedTask }); expect(cli.stderr).toBe(''); } finally { - await client.close(); - await runtime.close(); + await client?.close(); + await runtime?.close(); } }); it('keeps list and get DTO fields and persisted ordering identical across adapters', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime); + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); for (const [title, sessionId] of [ ['First built task', 'session-alpha'], ['Second built task', 'session-beta'], @@ -340,15 +348,17 @@ describe('built MCP and CLI contract parity', () => { expect(normalizeCliSuccess(cli.json)).toEqual(mcp); expect((mcp.data as { tasks: readonly unknown[] }).tasks).toHaveLength(2); } finally { - await client.close(); - await runtime.close(); + await client?.close(); + await runtime?.close(); } }); it('preserves duplicate candidates, warnings, and match reasons without rejecting capture', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime); + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); const existing = normalizeMcpSuccess( await client.callTool('task_capture', { title: 'Duplicate candidate capture', @@ -392,15 +402,17 @@ describe('built MCP and CLI contract parity', () => { candidates: [{ task: { id: candidate }, matchReason: expect.any(String) }], }); } finally { - await client.close(); - await runtime.close(); + await client?.close(); + await runtime?.close(); } }); it('returns identical mutation results when each adapter reads the other adapter’s persisted state', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime); + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); const capture = normalizeMcpSuccess( await client.callTool('task_capture', { title: 'Mutation parity task', @@ -451,15 +463,17 @@ describe('built MCP and CLI contract parity', () => { }, ); } finally { - await client.close(); - await runtime.close(); + await client?.close(); + await runtime?.close(); } }); it('returns identical no-op change metadata without changing task timestamps', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime); + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); const capture = normalizeMcpSuccess( await client.callTool('task_capture', { title: 'No-op parity task', @@ -489,15 +503,17 @@ describe('built MCP and CLI contract parity', () => { change: { action: 'NO_CHANGE', fields: [] }, }); } finally { - await client.close(); - await runtime.close(); + await client?.close(); + await runtime?.close(); } }); it('maps not-found, invalid transition, and archived mutation errors consistently', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime); + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); const notFoundCli = await runRelayCli(runtime, [ 'task', 'get', @@ -570,15 +586,17 @@ describe('built MCP and CLI contract parity', () => { expect(JSON.stringify(value)).not.toMatch(/SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\/i); } } finally { - await client.close(); - await runtime.close(); + await client?.close(); + await runtime?.close(); } }); it('keeps malformed inputs at validation/protocol boundaries and preserves exit code 2', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime); + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); const cli = await runRelayCli(runtime, [ 'task', 'capture', @@ -605,15 +623,17 @@ describe('built MCP and CLI contract parity', () => { }); expect(cli.stderr).not.toMatch(/SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\/i); } finally { - await client.close(); - await runtime.close(); + await client?.close(); + await runtime?.close(); } }); it('matches built empty-session results and rejects malformed session IDs at each adapter boundary', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime); + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); const mcpMissing = normalizeMcpSuccess( await client.callTool('session_captures_list', { sessionId: 'missing-session', @@ -649,17 +669,20 @@ describe('built MCP and CLI contract parity', () => { content: [{ type: 'text', text: expect.stringMatching(/invalid arguments|sessionId/i) }], }); } finally { - await client.close(); - await runtime.close(); + await client?.close(); + await runtime?.close(); } }); it('maps a locked-database write failure to the same structured storage error', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime); - const lock = new Database(runtime.databasePath); + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + let lock: Database.Database | undefined; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); + lock = new Database(runtime.databasePath); await client.callTool('relay_health', {}); lock.pragma('busy_timeout = 100'); @@ -704,13 +727,13 @@ describe('built MCP and CLI contract parity', () => { } } finally { try { - lock.exec('ROLLBACK'); + lock?.exec('ROLLBACK'); } catch { // The transaction may already be closed after a setup failure. } - lock.close(); - await client.close(); - await runtime.close(); + lock?.close(); + await client?.close(); + await runtime?.close(); } }, 15_000); }); diff --git a/tests/support/cli-test-process.ts b/tests/support/cli-test-process.ts index ba7d7b4..9533119 100644 --- a/tests/support/cli-test-process.ts +++ b/tests/support/cli-test-process.ts @@ -1,4 +1,4 @@ -import { spawn } from 'node:child_process'; +import { spawn, type ChildProcess } from 'node:child_process'; import { join } from 'node:path'; import type { AgentTestRuntime } from './agent-test-runtime.js'; @@ -41,8 +41,11 @@ export function runRelayCli( let settled = false; const timeout = setTimeout(() => { settled = true; - child.kill(); - reject(new Error(`Relay CLI timed out after ${options.timeoutMs ?? 30_000}ms.`)); + const timeoutError = new Error(`Relay CLI timed out after ${options.timeoutMs ?? 30_000}ms.`); + void terminateChild(child).then( + () => reject(timeoutError), + () => reject(timeoutError), + ); }, options.timeoutMs ?? 30_000); child.once('error', (error) => { if (settled) return; @@ -68,6 +71,39 @@ export function runRelayCli( }); } +async function terminateChild(child: ChildProcess): Promise { + if (hasExited(child)) return; + + child.kill(); + if (await waitForClose(child, 250)) return; + + child.kill('SIGKILL'); + await waitForClose(child); +} + +function hasExited(child: ChildProcess): boolean { + return child.exitCode !== null || child.signalCode !== null; +} + +function waitForClose(child: ChildProcess, timeoutMs?: number): Promise { + if (hasExited(child)) return Promise.resolve(true); + + return new Promise((resolve) => { + let timeout: NodeJS.Timeout | undefined; + const onClose = () => { + if (timeout !== undefined) clearTimeout(timeout); + resolve(true); + }; + child.once('close', onClose); + if (timeoutMs !== undefined) { + timeout = setTimeout(() => { + child.removeListener('close', onClose); + resolve(false); + }, timeoutMs); + } + }); +} + function parseSingleJson(stdout: string): unknown { const trimmed = stdout.trim(); if (trimmed.length === 0) throw new Error('Relay CLI produced no JSON output.'); diff --git a/tests/support/external-contract-normalizers.ts b/tests/support/external-contract-normalizers.ts index e0fa101..0a87e6f 100644 --- a/tests/support/external-contract-normalizers.ts +++ b/tests/support/external-contract-normalizers.ts @@ -12,6 +12,7 @@ export interface ExternalOperationResult { export function normalizeCliSuccess(value: unknown): ExternalOperationResult { const envelope = record(value, 'CLI result'); + assertKeys(envelope, ['schemaVersion', 'ok', 'data', 'warnings'], 'CLI result'); return { schemaVersion: number(envelope.schemaVersion), data: envelope.data, @@ -22,6 +23,7 @@ export function normalizeCliSuccess(value: unknown): ExternalOperationResult { export function normalizeMcpSuccess(value: unknown): ExternalOperationResult { const envelope = record(value, 'MCP result'); const structuredContent = record(envelope.structuredContent, 'MCP structured result'); + assertKeys(structuredContent, ['schemaVersion', 'data', 'warnings'], 'MCP structured result'); return { schemaVersion: number(structuredContent.schemaVersion), data: structuredContent.data, @@ -58,6 +60,23 @@ function record(value: unknown, label: string): Record { return value as Record; } +function assertKeys( + value: Record, + expectedKeys: readonly string[], + label: string, +): void { + const actualKeys = Object.keys(value).sort(); + const sortedExpectedKeys = [...expectedKeys].sort(); + if ( + actualKeys.length !== sortedExpectedKeys.length || + actualKeys.some((key, index) => key !== sortedExpectedKeys[index]) + ) { + throw new Error( + `${label} keys must be exactly ${sortedExpectedKeys.join(', ')}; received ${actualKeys.join(', ')}.`, + ); + } +} + function string(value: unknown): string { if (typeof value !== 'string') throw new Error('Expected a string contract field.'); return value; diff --git a/tests/unit/scripts/validate-agent-integration-assets.test.ts b/tests/unit/scripts/validate-agent-integration-assets.test.ts index 37089c8..618a318 100644 --- a/tests/unit/scripts/validate-agent-integration-assets.test.ts +++ b/tests/unit/scripts/validate-agent-integration-assets.test.ts @@ -292,6 +292,17 @@ describe('validateAgentIntegrationAssets', () => { ); }); + it('accepts vendor guidance that explicitly prohibits autonomous edits', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-cli/README.md'); + writeFileSync( + path, + `${readFileSync(path, 'utf8')}\nThe agent must not autonomously edit tasks.`, + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).not.toThrow(); + }); + it('rejects a vendor wrapper without both canonical skill references', () => { const rootDir = createRoot(); const path = join(rootDir, 'integrations/generic-mcp/README.md'); @@ -329,6 +340,14 @@ describe('validateAgentIntegrationAssets', () => { ); }); + it('accepts removal guidance that explicitly prohibits deleting the SQLite database', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-cli/README.md'); + writeFileSync(path, `${readFileSync(path, 'utf8')}\nDo not delete the SQLite database.`); + + expect(() => validateAgentIntegrationAssets({ rootDir })).not.toThrow(); + }); + it('rejects removal guidance that does not distinguish configuration from stored data', () => { const rootDir = createRoot(); const path = join(rootDir, 'integrations/generic-cli/README.md'); diff --git a/tests/unit/support/external-contract-normalizers.test.ts b/tests/unit/support/external-contract-normalizers.test.ts new file mode 100644 index 0000000..a7f42c9 --- /dev/null +++ b/tests/unit/support/external-contract-normalizers.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest'; +import { + normalizeCliSuccess, + normalizeMcpSuccess, +} from '../../support/external-contract-normalizers.js'; + +describe('external contract normalizers', () => { + it('normalizes the allowed CLI success envelope fields', () => { + expect( + normalizeCliSuccess({ schemaVersion: 1, ok: true, data: { count: 0 }, warnings: [] }), + ).toEqual({ schemaVersion: 1, data: { count: 0 }, warnings: [] }); + }); + + it('rejects renamed or unexpected CLI success envelope fields', () => { + expect(() => + normalizeCliSuccess({ schemaVersion: 1, ok: true, data: {}, warning: [] }), + ).toThrow(/CLI result.*keys/i); + }); + + it('normalizes the allowed MCP structured success envelope fields', () => { + expect( + normalizeMcpSuccess({ + structuredContent: { schemaVersion: 1, data: { count: 0 }, warnings: [] }, + content: [], + }), + ).toEqual({ schemaVersion: 1, data: { count: 0 }, warnings: [] }); + }); + + it('rejects renamed or unexpected MCP structured success fields', () => { + expect(() => + normalizeMcpSuccess({ + structuredContent: { schemaVersion: 1, data: {}, warning: [] }, + content: [], + }), + ).toThrow(/MCP structured result.*keys/i); + }); +}); diff --git a/tests/unit/support/mcp-test-client.test.ts b/tests/unit/support/mcp-test-client.test.ts index e36c196..bcda3c4 100644 --- a/tests/unit/support/mcp-test-client.test.ts +++ b/tests/unit/support/mcp-test-client.test.ts @@ -1,14 +1,17 @@ import { describe, expect, it } from 'vitest'; +import type { McpTestClient } from '../../support/mcp-test-client.js'; import { createAgentTestRuntime } from '../../support/agent-test-runtime.js'; import { createMcpTestClient } from '../../support/mcp-test-client.js'; describe('createMcpTestClient', () => { it('discovers Relay tools while keeping server stdout protocol-owned', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime, { - cwd: await runtime.createWorkingDirectory('mcp/nested'), - }); + let runtime: Awaited> | undefined; + let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('mcp/nested'), + }); const tools = await client.listTools(); expect(tools.map((tool) => tool.name)).toEqual( expect.arrayContaining([ @@ -22,20 +25,22 @@ describe('createMcpTestClient', () => { ); expect(client.stderr()).not.toContain('Content-Length'); } finally { - await client.close(); - await runtime.close(); + await client?.close(); + await runtime?.close(); } }); it('surfaces unknown-tool protocol failures and still closes the child', async () => { - const runtime = await createAgentTestRuntime(); - const client = await createMcpTestClient(runtime); + let runtime: Awaited> | undefined; + let client: McpTestClient | undefined; try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); await expect(client.callTool('unknown_tool', {})).resolves.toMatchObject({ isError: true }); - expect(client.stderr()).not.toMatch(/SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\/i); + expect(client.stderr()).toBe(''); } finally { - await client.close(); - await runtime.close(); + await client?.close(); + await runtime?.close(); } }); });