diff --git a/docs/agent-integration-verification.md b/docs/agent-integration-verification.md new file mode 100644 index 0000000..7890e16 --- /dev/null +++ b/docs/agent-integration-verification.md @@ -0,0 +1,90 @@ +# Agent Integration Verification + +## Scope and safety statement + +This evidence covers the source-checkout Relay MCP and CLI adapters, their shared task contract, the HTTP database path, and the canonical integration assets for issue #25. All automated scenarios use a fresh disposable database under the repository's `tmp/` directory and arbitrary disposable working directories. No test invokes an external LLM, reads or writes the default Relay database, or changes real Codex, Claude Code, or other client configuration. MCP diagnostics are captured from stderr; protocol data remains on stdout. + +## Automated scenario matrix + +| Scenario | Automated test or manual step | Result | Evidence | +| ------------------------------------------------------------------- | ---------------------------------------------------------------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1. MCP capture then CLI retrieval | `mcp-cli-parity.test.ts` — MCP capture followed by built CLI get | PASS | Built `dist/mcp/main.js` and `dist/cli/main.js`, same disposable `RELAY_DB_PATH` | +| 2. CLI capture then MCP retrieval | `mcp-cli-parity.test.ts` — CLI capture followed by MCP get | PASS | Complete public task DTO equality | +| 3. Task list/get fields and ordering | `mcp-cli-parity.test.ts` — list DTO comparison and persisted ordering | PASS | Transport-only envelope normalization | +| 4. Session-ID isolation | `agent-workflow-e2e.test.ts` — alpha and beta captures | PASS | Exact session filtering | +| 5. Completed and archived session review | `agent-workflow-e2e.test.ts` — all-status review | PASS | Open, DONE, and ARCHIVED captures returned | +| 6. Missing session behavior | `mcp-cli-parity.test.ts` and existing adapter contract tests | PASS | Stable missing-task/session contract coverage | +| 7. Malformed session behavior | `mcp-cli-parity.test.ts` and existing strict schema tests | PASS | MCP validation boundary and CLI validation envelope | +| 8. Duplicate candidates, warnings, and match reasons | `mcp-cli-parity.test.ts` — duplicate capture and find-similar | PASS | Advisory warning and deterministic candidate assertions | +| 9. Edit parity | `mcp-cli-parity.test.ts` — CLI edit and MCP readback | PASS | Complete task and change metadata | +| 10. Triage/start/complete/archive parity | `mcp-cli-parity.test.ts` — cross-adapter lifecycle sequence | PASS | Focused lifecycle actions and statuses | +| 11. No-op metadata | `mcp-cli-parity.test.ts` — repeated edit | PASS | `NO_CHANGE`, empty fields, unchanged timestamps | +| 12. Validation, transition, archived, not-found, and storage errors | `mcp-cli-parity.test.ts` — stable errors and locked-database write failure | PASS | Built MCP and CLI writes ran concurrently against one initialized disposable SQLite database while a separate connection held `BEGIN IMMEDIATE`; both returned equal normalized `STORAGE_ERROR` contracts, CLI exit code 5, and sanitized external output | +| 13. CLI JSON schemas and exit codes | `cli-test-process.test.ts` and built parity tests | PASS | One JSON document, separated stderr, exit codes 0/2/3/4/5 | +| 14. One database across HTTP, MCP, and CLI | `database-path-parity.test.ts` | PASS | Arbitrary CWDs, HTTP runtime, restart persistence, no CWD-local DB | +| 15. Skill and vendor-wrapper drift | `validate-agent-integration-assets.test.ts` and `validate:assets` | PASS | 33 validator tests; canonical policy and entry-point checks | +| 16. Integration removal preserves data | `agent-workflow-e2e.test.ts` config-driven disposable MCP launch and removal | PASS | Parsed `.mcp.json` launches the built server; removal is followed by retrieval from the same DB | + +## Clean-checkout environment + +- OS: Microsoft Windows NT 10.0.26200.0 +- Node: v24.18.0 +- pnpm: 10.2.0 through Corepack (`corepack pnpm --version`); direct global pnpm was 11.9.0 and was not used for authoritative final commands. +- Branch: `feature/issue-25-mcp-cli-compatibility` +- Verification base SHA: `e51307d066e70b95f8072ac52d2679b8e63c5244` +- Database strategy: each test calls `createAgentTestRuntime()` and uses a unique disposable `/data/relay.db`; client CWDs are created below the same disposable root. + +Exact setup and validation commands: + +```text +corepack pnpm install --frozen-lockfile +corepack pnpm build +corepack pnpm exec vitest run tests/integration/mcp-cli-parity.test.ts +corepack pnpm exec vitest run tests/integration/agent-workflow-e2e.test.ts +corepack pnpm exec vitest run tests/integration/database-path-parity.test.ts +corepack pnpm validate:assets +corepack pnpm verify +git status --short +``` + +The disposable strategy and assertions confirm that the default database and real client configuration were not touched. + +## Codex validation + +Live validation is unverified. `Get-Command codex` resolved the installed desktop executable, but `codex --version` failed with Windows `Access is denied`. No Codex process was started, no MCP discovery result is claimed, and no Codex configuration or profile was changed. + +The required isolated workflow remains: + +1. Use a clean checkout and `corepack pnpm install --frozen-lockfile`. +2. Run `corepack pnpm build`. +3. Create an isolated profile/configuration and disposable `RELAY_DB_PATH`. +4. Add the canonical `node /dist/mcp/main.js` server using the documented Codex configuration. +5. Install the canonical skills unchanged under `.agents/skills/`. +6. Restart Codex, discover Relay tools, capture two follow-ups with one session ID, review that session, perform one explicitly directed lifecycle action, use the CLI JSON fallback, remove only config/skill references, and confirm data remains. + +No step above is represented as executed in this environment. + +## Claude Code validation + +Live validation is unverified. `Get-Command claude` and `Get-Command claude-code` returned no executable. No Claude configuration was edited and no client result is claimed. + +The equivalent isolated workflow is documented in `integrations/claude-code/README.md`: project-scoped stdio configuration, canonical `.claude/skills/` directories, tool discovery, two same-session captures, exact session review, one explicitly directed mutation, CLI fallback, configuration-only removal, and post-removal data retrieval. + +## Cross-client differences and limitations + +The automated contract is client-neutral: built MCP uses protocol-owned stdout and CLI uses one JSON envelope plus stable exit codes. Codex and Claude Code syntax, skill-discovery locations, and availability could not be exercised live here. The issue #24 documentation records the official-source verification date and the current unavailable-client limitations. + +## Data and configuration preservation + +Automated tests only create disposable files under the test runtime root. They remove temporary configuration fixtures and restart against the same database, then retrieve persisted tasks. They never remove a database to disable an integration. The validator rejects removal guidance that deletes SQLite data and requires explicit configuration-only wording. + +## Epic #2 closure checklist + +- [x] Issues #19, #20, #21, #22, #23, #24, and #26 are closed on GitHub and their required artifacts exist locally. +- [x] Built MCP and CLI entry points are exercised from arbitrary working directories. +- [x] MCP stdout remains protocol-clean; diagnostics are stderr-only. +- [x] Automated contract, lifecycle, session, error, storage, restart, shared-path, and asset checks pass. +- [x] Default database, real client configuration, and external LLMs were not touched by automation. +- [ ] Live Codex workflow — blocked by executable access denied; human review required. +- [ ] Live Claude Code workflow — blocked because the client is unavailable; human review required. +- [ ] Human reviewer must inspect cleanup, normalizers, source-context safety, and one independent client workflow before merging. diff --git a/docs/superpowers/plans/2026-07-29-issue-25-mcp-cli-compatibility-verification.md b/docs/superpowers/plans/2026-07-29-issue-25-mcp-cli-compatibility-verification.md index 89ba085..46fbbb7 100644 --- a/docs/superpowers/plans/2026-07-29-issue-25-mcp-cli-compatibility-verification.md +++ b/docs/superpowers/plans/2026-07-29-issue-25-mcp-cli-compatibility-verification.md @@ -668,12 +668,19 @@ Use these exact top-level sections: # Agent Integration Verification ## Scope and safety statement + ## Automated scenario matrix + ## Clean-checkout environment + ## Codex validation + ## Claude Code validation + ## Cross-client differences and limitations + ## Data and configuration preservation + ## Epic #2 closure checklist ``` diff --git a/docs/superpowers/tasks/2026-07-30-pr-34-review-tracker.md b/docs/superpowers/tasks/2026-07-30-pr-34-review-tracker.md new file mode 100644 index 0000000..368d086 --- /dev/null +++ b/docs/superpowers/tasks/2026-07-30-pr-34-review-tracker.md @@ -0,0 +1,87 @@ +# PR #34 Review Task Tracker + +Source: [Luna remediation implementation plan](https://github.com/krishna916/relay/pull/34#issuecomment-5122267820) + +Scope: complete every actionable task in the linked plan while preserving issue #25's acceptance claims, application contracts, adapter schemas, database defaults, canonical skills, and vendor policies. + +## Working rules + +- [x] Verify each task against the current checkout before editing. +- [x] Keep implementation changes limited to the files named by the plan, plus this requested local tracker. +- [x] Use test-first changes and record focused evidence for each task. +- [x] Do not commit `dist/`, coverage output, temporary runtimes, or default Relay data. +- [x] Do not modify client configuration or claim live Codex/Claude validation without evidence. +- [x] Do not reply to or resolve the GitHub review thread unless explicitly requested after local work is complete. + +## Task 1 - Build Node artifacts before process tests + +- [x] Record the clean-state baseline failure: with `dist/` absent, the pre-fix command ran Vitest without built entries and reported 19 failed tests across 4 failed suites. +- [x] Update only `test` and `test:coverage` scripts to build Node artifacts first. +- [x] Run the focused CLI process test with `dist/` initially absent. +- [x] Run the focused MCP coverage test with `dist/` initially absent. +- [x] Commit the focused Task 1 change as `5142625`. + +## Task 2 - Create the repository temporary parent + +- [x] Add the missing-parent regression test before implementation. +- [x] Confirm the regression test fails with the expected `ENOENT` baseline. +- [x] Create `/tmp` recursively before `mkdtemp()`. +- [x] Preserve generated-root-only cleanup and path-escape protections. +- [x] Run focused runtime tests and typecheck. +- [x] Commit the focused Task 2 change as `b3703ca`. + +## Task 3 - Verify structured MCP/CLI storage-error parity + +- [x] Replace startup-rejection coverage with a post-initialization locked-database test. +- [x] Run CLI and MCP writes concurrently against the same `BEGIN IMMEDIATE` lock. +- [x] Assert CLI exit code `5`, structured `STORAGE_ERROR`, MCP structured `STORAGE_ERROR`, normalized equality, and sanitized output. +- [x] Run the complete MCP/CLI parity file. +- [x] Commit the focused Task 3 change as `cc3380a`. + +## Task 4 - Correct evidence claims + +- [x] Update scenario 12 to describe the verified locked-database parity evidence. +- [x] Remove the old unusable-parent/startup-rejection parity claim. +- [x] Run formatting and repository asset validation. +- [x] Commit the focused Task 4 change as `e190f1f`. + +## Task 5 - Clean-state verification and reconciliation + +- [x] Run `corepack pnpm install --frozen-lockfile` successfully with the pinned pnpm 10.2.0 toolchain after the initial sandbox relink permission failure was retried with filesystem escalation. +- [x] Verify `corepack pnpm test` works after generated output is removed: 39 files/512 tests passed and `dist/` was recreated by the script. +- [x] Verify `corepack pnpm test:coverage` works after generated output is removed: `verify` started from deleted `dist/`, rebuilt Node artifacts first, and completed coverage successfully. +- [x] Run `corepack pnpm verify` from a clean generated-output state: passed formatting, lint, typecheck, 512 tests, coverage, builds, asset validation, and the high-severity audit gate. +- [x] Run the three focused issue #25 integration suites: MCP/CLI parity 25 passed; agent workflow 3 passed; database-path parity 2 passed. +- [x] Confirm the worktree contains no tracked generated output or disposable runtime changes; only pre-existing `.codegraph/` remains outside the task and the tracker is the requested local artifact. +- [x] Reconcile this tracker against the final acceptance checklist below. +- [x] Keep PR claims/documentation honest; do not publish GitHub replies or resolve the thread in this pass. + +## Verification log + +| Check | Result | Evidence | +| --------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Baseline | Pass | PR #34 comment and review context fetched; branch is `feature/issue-25-mcp-cli-compatibility`; only pre-existing untracked `.codegraph/` is outside this task. | +| Task 1 focused checks | Pass | Clean `dist/` then `corepack pnpm test -- tests/unit/support/cli-test-process.test.ts`: 39 files/511 tests passed; clean `dist/` then `corepack pnpm test:coverage -- tests/unit/support/mcp-test-client.test.ts`: 39 files/511 tests passed; coverage 88.74/81.34/88.94/90.90. | +| Task 2 focused checks | Pass | Red regression produced `ENOENT`; `vitest run tests/unit/support/agent-test-runtime.test.ts`: 3 tests passed; `corepack pnpm typecheck`: passed. | +| Task 3 focused checks | Pass | Focused lock test: 1 passed/24 skipped; complete `mcp-cli-parity.test.ts`: 25 passed. | +| Task 4 checks | Pass | `corepack pnpm format:check` passed; `corepack pnpm validate:assets` passed. | +| Full clean-state gate | Pass | `corepack pnpm verify` from deleted `dist/`: 39 files, 512 tests, 88.74% statements / 81.34% branches / 88.94% functions / 90.90% lines; build, assets, and audit gate passed. | + +## Final acceptance checklist + +- [x] `pnpm test` works when `dist/` is initially absent. +- [x] `pnpm test:coverage` works when `dist/` is initially absent. +- [x] `corepack pnpm verify` passes from a clean generated-output state. +- [x] `createAgentTestRuntime()` succeeds when `/tmp` is initially absent. +- [x] Runtime cleanup removes the generated unique directory and SQLite sidecars. +- [x] Storage failure occurs after MCP initialization, not during transport startup. +- [x] CLI returns exit code `5` and structured `STORAGE_ERROR`. +- [x] MCP returns structured `STORAGE_ERROR`. +- [x] Normalized CLI and MCP storage errors are equal. +- [x] External output contains no SQL, stack, database path, or user-directory leakage. +- [x] Documentation claims match the verified behavior; live Codex/Claude validation remains explicitly unverified. +- [x] Existing 80% coverage thresholds and all quality gates remain unchanged. + +## Publication status + +Verified commits `5142625`, `b3703ca`, `cc3380a`, `e190f1f`, and `c64efa8` are pushed to `origin/feature/issue-25-mcp-cli-compatibility`. PR #34 remains draft and its description records the clean-checkout verification evidence. The GitHub review thread remains unresolved; no follow-up comment was posted because that remote write requires an explicit user request. diff --git a/integrations/claude-code/README.md b/integrations/claude-code/README.md index 4b87d0e..31f7e9c 100644 --- a/integrations/claude-code/README.md +++ b/integrations/claude-code/README.md @@ -4,6 +4,6 @@ Build Relay and substitute an absolute checkout path. Add the stdio server with Install the canonical [Relay Capture](../../skills/relay-capture/SKILL.md) and [Relay Session Review](../../skills/relay-session-review/SKILL.md) skill directories by copying or symlinking them unchanged to `.claude/skills/relay-capture/` and `.claude/skills/relay-session-review/`. For a personal installation across projects, use the client’s documented user-scoped skills directory. Do not copy the policy text into Claude-specific documentation or use instruction-file imports as skill discovery. -Use `claude mcp list`, `claude mcp get relay`, and `/mcp` to validate and authorize the server. Confirm `relay_health`, the task tools, a disposable capture, and the exact session lookup when live validation is available. Remove it with `claude mcp remove relay` or by deleting the Relay entry and skill directories; the SQLite database remains untouched. +Use `claude mcp list`, `claude mcp get relay`, and `/mcp` to validate and authorize the server. Confirm `relay_health`, the task tools, a disposable capture, and the exact session lookup when live validation is available. Remove only the client configuration with `claude mcp remove relay` or by deleting the Relay entry and skill directories; the SQLite database remains untouched. For current syntax and skill discovery behavior, see the [Claude Code MCP documentation](https://code.claude.com/docs/en/mcp) and [Claude Code skills documentation](https://code.claude.com/docs/en/skills). diff --git a/integrations/generic-cli/README.md b/integrations/generic-cli/README.md index 563af0d..1b060c3 100644 --- a/integrations/generic-cli/README.md +++ b/integrations/generic-cli/README.md @@ -12,4 +12,4 @@ node __RELAY_CHECKOUT__/dist/cli/main.js task complete TASK_ID --output json node __RELAY_CHECKOUT__/dist/cli/main.js task archive TASK_ID --output json ``` -Exit codes are documented in [the CLI reference](../../docs/cli-reference.md). Capture can be autonomous; edit, triage, start, complete, and archive require explicit user direction. See [Relay Capture](../../skills/relay-capture/SKILL.md) and [Relay Session Review](../../skills/relay-session-review/SKILL.md). Removing a client integration does not delete data; the SQLite database remains untouched. +Exit codes are documented in [the CLI reference](../../docs/cli-reference.md). See [Relay Capture](../../skills/relay-capture/SKILL.md) and [Relay Session Review](../../skills/relay-session-review/SKILL.md) for behavioural policy. Removing a client integration does not delete data; the SQLite database remains untouched. diff --git a/package.json b/package.json index 181cb07..73361d6 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,8 @@ "relay-mcp": "./dist/mcp/main.js" }, "scripts": { - "test": "vitest run", - "test:coverage": "vitest run --coverage", + "test": "pnpm build:node && vitest run", + "test:coverage": "pnpm build:node && vitest run --coverage", "lint": "eslint . --max-warnings=0", "typecheck": "tsc --build --noEmit", "format": "prettier --write .", diff --git a/scripts/validate-agent-integration-assets.ts b/scripts/validate-agent-integration-assets.ts index facaa99..c3408e0 100644 --- a/scripts/validate-agent-integration-assets.ts +++ b/scripts/validate-agent-integration-assets.ts @@ -75,6 +75,20 @@ function validateCompatibilityClaims(shared: string): void { function validateVendorClaims(rootDir: string, shared: string): void { for (const readme of vendorReadmes) { const text = readAsset(rootDir, `integrations/${readme}/README.md`); + const affirmativeAutonomyText = text.replace( + /\b(?:(?:(?:must|should|shall|may|can|will|do(?:es)?|did)\s+)?not|never)\s+autonomously\s+(?:edit|triage|start|complete|archive|delete|merge)\b/gi, + '', + ); + if ( + /^## Autonomy boundaries$/im.test(text) || + /autonomously\s+(?:edit|triage|start|complete|archive|delete|merge)\b/i.test( + affirmativeAutonomyText, + ) + ) { + fail( + `${readme} must reference canonical behavioural policy instead of redefining mutation autonomy.`, + ); + } const claimsNoLiveTest = /(?:live smoke test|live validation)[^\n]*not completed/i.test(text); const claimsLiveEvidence = /(?:live|manual)[^\n]*(?:tested|performed|verified|discovered|captured)/i.test(text); @@ -99,6 +113,52 @@ function validateVendorClaims(rootDir: string, shared: string): void { } } +function validateCanonicalSkills(rootDir: string): void { + const capture = readAsset(rootDir, canonicalSkills[0]); + if (!/autonomously create only a new Relay task in `?INBOX`?/i.test(capture)) + fail('Canonical capture skill must define autonomous creation as INBOX-only.'); + if ( + !/must not edit, triage, start, complete, archive, delete, merge, or move any task/i.test( + capture, + ) + ) + fail('Canonical capture skill must prohibit autonomous lifecycle mutation.'); + if (/autonomously\s+(?:edit|triage|start|complete|archive|delete|merge)/i.test(capture)) + fail('Canonical capture skill must not grant autonomous lifecycle mutation.'); + + const review = readAsset(rootDir, canonicalSkills[1]); + if (!/completed and archived captures/i.test(review)) + fail( + 'Canonical session-review skill must require all-status retrieval including completed and archived captures.', + ); + if (!/explicit user direction/i.test(review)) + fail('Canonical session-review skill must require explicit user direction for mutations.'); +} + +function validateRemovalGuidance(rootDir: string): void { + for (const readme of vendorReadmes) { + const text = readAsset(rootDir, `integrations/${readme}/README.md`); + const affirmativeRemovalText = text.replace( + /\b(?:(?:(?:must|should|shall|may|can|will|do(?:es)?|did)\s+)?not|never)\s+(?:remove|delete)\s+(?:the\s+)?(?:SQLite\s+)?database\b/gi, + '', + ); + if ( + /(?:remove|delete)\s+(?:the\s+)?(?:SQLite\s+)?database\b|(?:SQLite\s+)?database\b[^\n]{0,80}\b(?:remove|delete)\b/i.test( + affirmativeRemovalText, + ) + ) + fail(`${readme} removal guidance must not delete the SQLite database.`); + if ( + !/(?:remov(?:e|ing)|delete)[\s\S]{0,160}(?:configuration|integration|client assets?)/i.test( + text, + ) + ) + fail(`${readme} removal guidance must distinguish configuration removal from stored data.`); + if (!/SQLite database remains untouched/i.test(text)) + fail(`${readme} removal guidance must state that the SQLite database remains untouched.`); + } +} + function validateTemplateShape(rootDir: string): void { const expectedMcpPath = '/tmp/relay-checkout/dist/mcp/main.js'; const jsonTemplates = [ @@ -121,7 +181,7 @@ function validateTemplateShape(rootDir: string): void { fail(`${path} must separate command and arguments for a stdio server.`); } if (server.args[0] !== expectedMcpPath) { - fail(`${path} must use the exact dist/mcp/main.js entry path.`); + fail(`${path} must use the canonical dist/mcp/main.js entry path.`); } if (typeof server.command !== 'string' || /[\\/\s]/.test(server.command)) { fail(`${path} must not embed a shell command in command.`); @@ -149,7 +209,7 @@ function validateTemplateShape(rootDir: string): void { codexServer.args[0] !== expectedMcpPath ) { fail( - 'integrations/codex/config.toml.example must use node plus dist/mcp/main.js as separate fields.', + 'integrations/codex/config.toml.example must use node plus the canonical dist/mcp/main.js entry as separate fields.', ); } if (codexServer.env?.RELAY_DB_PATH !== '/tmp/relay-checkout/.relay-validation/relay.db') { @@ -164,6 +224,9 @@ export function validateAgentIntegrationAssets( for (const path of requiredPaths) { if (!existsSync(join(rootDir, path))) fail(`Required path missing: ${path}`); } + for (const path of canonicalSkills) { + if (!existsSync(join(rootDir, path))) fail(`Canonical skill path missing: ${path}`); + } const integrationRoot = join(rootDir, 'integrations'); const contents = filesUnder(integrationRoot) @@ -173,6 +236,8 @@ export function validateAgentIntegrationAssets( const troubleshooting = readAsset(rootDir, 'docs/troubleshooting-agent-integration.md'); const all = `${contents}\n${shared}\n${troubleshooting}`; + validateCanonicalSkills(rootDir); + if (/(?:[A-Z]:[\\/]Users[\\/]|\/Users\/|\/home\/|~\/)[^\s"'`]+/i.test(all)) fail('Machine-specific absolute path found.'); for (const skill of canonicalSkills) { @@ -220,11 +285,7 @@ export function validateAgentIntegrationAssets( fail('Claude README must preserve complete canonical skill directories unchanged.'); if (/^## Autonomy boundaries$/m.test(contents)) fail('Vendor assets must not copy behavioural policy.'); - for (const readme of vendorReadmes) { - const text = readAsset(rootDir, `integrations/${readme}/README.md`); - if (!/SQLite database remains untouched/i.test(text)) - fail(`${readme} removal guidance must state that the SQLite database remains untouched.`); - } + validateRemovalGuidance(rootDir); for (const match of all.matchAll(/relay mcp/gi)) { const context = all.slice(Math.max(0, match.index! - 80), match.index! + 100); if (!/(future|not available|Epic #18)/i.test(context)) diff --git a/tests/fixtures/agent-integrations/valid/integrations/claude-code/README.md b/tests/fixtures/agent-integrations/valid/integrations/claude-code/README.md index 1dfedfa..65afaaf 100644 --- a/tests/fixtures/agent-integrations/valid/integrations/claude-code/README.md +++ b/tests/fixtures/agent-integrations/valid/integrations/claude-code/README.md @@ -1 +1 @@ -skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md .claude/skills/relay-capture/ .claude/skills/relay-session-review/ Copy or symlink the complete canonical skill directories and preserve each SKILL.md unchanged. SQLite database remains untouched. +skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md .claude/skills/relay-capture/ .claude/skills/relay-session-review/ Copy or symlink the complete canonical skill directories and preserve each SKILL.md unchanged. Remove only the client configuration and skill references; the SQLite database remains untouched. diff --git a/tests/fixtures/agent-integrations/valid/integrations/codex/README.md b/tests/fixtures/agent-integrations/valid/integrations/codex/README.md index ee032d2..4ea651f 100644 --- a/tests/fixtures/agent-integrations/valid/integrations/codex/README.md +++ b/tests/fixtures/agent-integrations/valid/integrations/codex/README.md @@ -1 +1 @@ -skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md .agents/skills/relay-capture/ .agents/skills/relay-session-review/ SQLite database remains untouched. +skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md .agents/skills/relay-capture/ .agents/skills/relay-session-review/ Remove only the client configuration and skill references; the SQLite database remains untouched. diff --git a/tests/fixtures/agent-integrations/valid/integrations/generic-cli/README.md b/tests/fixtures/agent-integrations/valid/integrations/generic-cli/README.md index 9d777a8..cebde18 100644 --- a/tests/fixtures/agent-integrations/valid/integrations/generic-cli/README.md +++ b/tests/fixtures/agent-integrations/valid/integrations/generic-cli/README.md @@ -1 +1 @@ -skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md `export RELAY_DB_PATH="__RELAY_CHECKOUT__/.relay-validation/relay.db"` SQLite database remains untouched. +skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md `export RELAY_DB_PATH="__RELAY_CHECKOUT__/.relay-validation/relay.db"` Remove only the client configuration; the SQLite database remains untouched. diff --git a/tests/fixtures/agent-integrations/valid/integrations/generic-mcp/README.md b/tests/fixtures/agent-integrations/valid/integrations/generic-mcp/README.md index 571f6ef..9f7a815 100644 --- a/tests/fixtures/agent-integrations/valid/integrations/generic-mcp/README.md +++ b/tests/fixtures/agent-integrations/valid/integrations/generic-mcp/README.md @@ -1 +1 @@ -skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md Validation requires explicit isolated RELAY_DB_PATH; omission is permitted only for non-validation use. relay_health task_capture task_list task_get task_find_similar session_captures_list SQLite database remains untouched. +skills/relay-capture/SKILL.md skills/relay-session-review/SKILL.md Validation requires explicit isolated RELAY_DB_PATH; omission is permitted only for non-validation use. Remove only the client configuration; the SQLite database remains untouched. relay_health task_capture task_list task_get task_find_similar session_captures_list SQLite database remains untouched. diff --git a/tests/fixtures/contracts/agent-workflow.ts b/tests/fixtures/contracts/agent-workflow.ts new file mode 100644 index 0000000..05d7b7d --- /dev/null +++ b/tests/fixtures/contracts/agent-workflow.ts @@ -0,0 +1,22 @@ +export const AGENT_WORKFLOW_FIXTURES = { + sessions: { + alpha: 'session-alpha', + beta: 'session-beta', + }, + agents: { + codex: 'Codex', + claudeCode: 'Claude Code', + }, + workspace: 'relay-verification', + titles: { + alphaOpen: 'Alpha open capture', + alphaCompleted: 'Alpha completed capture', + alphaArchived: 'Alpha archived capture', + betaOpen: 'Beta open capture', + duplicate: 'Duplicate candidate capture', + }, + malformed: { + session: 'bad session id!', + taskId: 'not-a-valid-task-id', + }, +} as const; diff --git a/tests/integration/agent-workflow-e2e.test.ts b/tests/integration/agent-workflow-e2e.test.ts new file mode 100644 index 0000000..23c0eca --- /dev/null +++ b/tests/integration/agent-workflow-e2e.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, it } from 'vitest'; +import { readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import type { McpTestClient } from '../support/mcp-test-client.js'; +import type { AgentTestRuntime } from '../support/agent-test-runtime.js'; +import { createAgentTestRuntime } from '../support/agent-test-runtime.js'; +import { runRelayCli } from '../support/cli-test-process.js'; +import { createMcpTestClient } from '../support/mcp-test-client.js'; +import { + normalizeCliSuccess, + normalizeMcpSuccess, +} from '../support/external-contract-normalizers.js'; + +describe('built agent workflow end to end', () => { + it('isolates session review and includes open, completed, and archived captures', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + const connectedClient = await createMcpTestClient(runtime); + client = connectedClient; + const capture = async (title: string, sessionId: string) => + normalizeMcpSuccess( + await connectedClient.callTool('task_capture', { + title, + createdByName: 'Codex', + sessionId, + workspace: 'relay-verification', + }), + ); + await capture('Alpha open capture', 'session-alpha'); + const completed = await capture('Alpha completed capture', 'session-alpha'); + const archived = await capture('Alpha archived capture', 'session-alpha'); + const beta = await runRelayCli(runtime, [ + 'task', + 'capture', + '--title', + 'Beta open capture', + '--agent', + 'Claude Code', + '--session', + 'session-beta', + '--output', + 'json', + ]); + expect(beta.exitCode).toBe(0); + + const completedId = String((completed.data as { task: { id: string } }).task.id); + const archivedId = String((archived.data as { task: { id: string } }).task.id); + await connectedClient.callTool('task_triage', { taskId: completedId, target: 'ACTIVE' }); + await connectedClient.callTool('task_start', { taskId: completedId }); + await connectedClient.callTool('task_triage', { taskId: archivedId, target: 'ACTIVE' }); + await connectedClient.callTool('task_start', { taskId: archivedId }); + expect( + await connectedClient.callTool('task_complete', { taskId: completedId }), + ).toMatchObject({ + structuredContent: { data: { task: { status: 'DONE' } } }, + }); + await connectedClient.callTool('task_complete', { taskId: archivedId }); + expect(await connectedClient.callTool('task_archive', { taskId: archivedId })).toMatchObject({ + structuredContent: { data: { task: { status: 'ARCHIVED' } } }, + }); + + const mcpReview = normalizeMcpSuccess( + await connectedClient.callTool('session_captures_list', { + sessionId: 'session-alpha', + limit: 100, + }), + ); + const cliReview = await runRelayCli(runtime, [ + 'session', + 'captures', + '--session', + 'session-alpha', + '--output', + 'json', + ]); + expect(cliReview.exitCode).toBe(0); + expect(normalizeCliSuccess(cliReview.json)).toEqual(mcpReview); + expect(mcpReview.data).toMatchObject({ + sessionId: 'session-alpha', + count: 3, + tasks: [ + { title: 'Alpha open capture', status: 'INBOX' }, + { title: 'Alpha completed capture', status: 'DONE' }, + { title: 'Alpha archived capture', status: 'ARCHIVED' }, + ], + }); + expect(mcpReview.data).not.toMatchObject({ + tasks: [expect.objectContaining({ sessionId: 'session-beta' })], + }); + } finally { + await client?.close(); + await runtime?.close(); + } + }); + + it('persists tasks and mutations across short-lived MCP and CLI restarts', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + let taskId: string | undefined; + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('restart/mcp-1'), + }); + const capture = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'Restart persistence task', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ); + taskId = String((capture.data as { task: { id: string } }).task.id); + await client.close(); + client = undefined; + + const get = await runRelayCli(runtime, ['task', 'get', taskId, '--output', 'json'], { + cwd: await runtime.createWorkingDirectory('restart/cli-1'), + }); + expect(get.exitCode).toBe(0); + const edit = await runRelayCli(runtime, [ + 'task', + 'edit', + taskId, + '--description', + 'survives restart', + '--output', + 'json', + ]); + expect(edit.exitCode).toBe(0); + + client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('restart/mcp-2'), + }); + try { + const get = normalizeMcpSuccess(await client.callTool('task_get', { taskId })); + expect(get.data).toMatchObject({ + task: { id: taskId, description: 'survives restart', sessionId: 'session-alpha' }, + }); + } finally { + await client.close(); + client = undefined; + } + } finally { + await client?.close(); + await runtime?.close(); + } + }); + + it('preserves stored data when a disposable integration configuration is removed', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + const configDirectory = await runtime.createWorkingDirectory('integration-config'); + const configPath = join(configDirectory, '.mcp.json'); + const config = { + mcpServers: { + relay: { + command: process.execPath, + args: [join(runtime.checkoutPath, 'dist', 'mcp', 'main.js')], + }, + }, + }; + await writeFile(configPath, JSON.stringify(config)); + const loadedConfig = JSON.parse(await readFile(configPath, 'utf8')) as typeof config; + const server = loadedConfig.mcpServers.relay; + client = await createMcpTestClient(runtime, { + cwd: configDirectory, + command: server.command, + args: server.args, + }); + const capture = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'Configuration removal preserves data', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ); + const taskId = String((capture.data as { task: { id: string } }).task.id); + await client.close(); + client = undefined; + await rm(configPath); + await expect(stat(configPath)).rejects.toMatchObject({ code: 'ENOENT' }); + + client = await createMcpTestClient(runtime, { cwd: configDirectory }); + const retrieved = normalizeMcpSuccess(await client.callTool('task_get', { taskId })); + expect(retrieved.data).toMatchObject({ + task: { id: taskId, title: 'Configuration removal preserves data' }, + }); + } finally { + await client?.close(); + await runtime?.close(); + } + }); +}); diff --git a/tests/integration/database-path-parity.test.ts b/tests/integration/database-path-parity.test.ts new file mode 100644 index 0000000..c72a384 --- /dev/null +++ b/tests/integration/database-path-parity.test.ts @@ -0,0 +1,175 @@ +import { readdir, stat } from 'node:fs/promises'; +import { basename, dirname, join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { createTaskRuntime } from '../../src/interfaces/shared/create-task-runtime.js'; +import { + createHttpServer, + type HttpServerInstance, +} from '../../src/interfaces/http/create-http-server.js'; +import { createAgentTestRuntime, type AgentTestRuntime } from '../support/agent-test-runtime.js'; +import { runRelayCli } from '../support/cli-test-process.js'; +import { createMcpTestClient, type McpTestClient } from '../support/mcp-test-client.js'; +import { + normalizeCliSuccess, + normalizeMcpSuccess, +} from '../support/external-contract-normalizers.js'; + +describe('shared database path across HTTP, MCP, and CLI', () => { + it('uses one configured database from arbitrary CWDs and preserves data after all adapters restart', async () => { + let runtime: AgentTestRuntime | undefined; + let applicationRuntime: ReturnType | undefined; + let server: HttpServerInstance | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + applicationRuntime = createTaskRuntime({ databasePath: runtime.databasePath }); + server = await createHttpServer({ + host: '127.0.0.1', + port: 0, + taskApplication: applicationRuntime.taskApplication, + }); + client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('cwd/mcp'), + }); + + const capture = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'Shared MCP task', + createdByName: 'Codex', + sessionId: 'session-alpha', + workspace: 'relay-verification', + }), + ); + const capturedTask = (capture.data as { task: { id: string } }).task; + const cliGet = await runRelayCli( + runtime, + ['task', 'get', capturedTask.id, '--output', 'json'], + { cwd: await runtime.createWorkingDirectory('cwd/cli') }, + ); + expect(cliGet.exitCode).toBe(0); + expect(normalizeCliSuccess(cliGet.json).data).toEqual({ task: capturedTask }); + + const cliEdit = await runRelayCli(runtime, [ + 'task', + 'edit', + capturedTask.id, + '--title', + 'Shared edited task', + '--output', + 'json', + ]); + expect(cliEdit.exitCode).toBe(0); + const httpEdited = await fetch(`${server.url}/api/tasks/${capturedTask.id}`); + expect(httpEdited.status).toBe(200); + await expect(httpEdited.json()).resolves.toMatchObject({ + task: { id: capturedTask.id, title: 'Shared edited task' }, + }); + + const humanResponse = await fetch(`${server.url}/api/tasks`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ title: 'Shared human task' }), + }); + expect(humanResponse.status).toBe(201); + const humanTask = (await humanResponse.json()) as { + task: { id: string; createdByType: string }; + }; + expect(humanTask.task.createdByType).toBe('HUMAN'); + + const mcpHuman = normalizeMcpSuccess( + await client.callTool('task_get', { taskId: humanTask.task.id }), + ); + expect(mcpHuman.data).toMatchObject({ + task: { id: humanTask.task.id, createdByType: 'HUMAN' }, + }); + const cliList = await runRelayCli(runtime, ['task', 'list', '--output', 'json']); + expect(cliList.exitCode).toBe(0); + expect(normalizeCliSuccess(cliList.json).data).toMatchObject({ + tasks: expect.arrayContaining([ + expect.objectContaining({ id: capturedTask.id, title: 'Shared edited task' }), + expect.objectContaining({ id: humanTask.task.id, title: 'Shared human task' }), + ]), + count: 2, + }); + + await client.close(); + client = undefined; + await server.stop(); + server = undefined; + applicationRuntime.close(); + applicationRuntime = undefined; + + applicationRuntime = createTaskRuntime({ databasePath: runtime.databasePath }); + server = await createHttpServer({ + host: '127.0.0.1', + port: 0, + taskApplication: applicationRuntime.taskApplication, + }); + client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('cwd/mcp-restart'), + }); + const restartedHttp = await fetch(`${server.url}/api/tasks/${capturedTask.id}`); + expect(restartedHttp.status).toBe(200); + await expect(restartedHttp.json()).resolves.toMatchObject({ + task: { id: capturedTask.id, title: 'Shared edited task' }, + }); + const restartedMcp = normalizeMcpSuccess( + await client.callTool('task_get', { taskId: humanTask.task.id }), + ); + expect(restartedMcp.data).toMatchObject({ + task: { id: humanTask.task.id, title: 'Shared human task', createdByType: 'HUMAN' }, + }); + } finally { + await client?.close(); + await server?.stop(); + applicationRuntime?.close(); + await runtime?.close(); + } + }); + + it('does not create a default or CWD-local database file', async () => { + let runtime: AgentTestRuntime | undefined; + let applicationRuntime: ReturnType | undefined; + let server: HttpServerInstance | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + applicationRuntime = createTaskRuntime({ databasePath: runtime.databasePath }); + server = await createHttpServer({ + host: '127.0.0.1', + port: 0, + taskApplication: applicationRuntime.taskApplication, + }); + const mcpCwd = await runtime.createWorkingDirectory('mcp'); + const cliCwd = await runtime.createWorkingDirectory('cli'); + client = await createMcpTestClient(runtime, { cwd: mcpCwd }); + await client.callTool('task_capture', { + title: 'Database location task', + createdByName: 'Codex', + sessionId: 'session-alpha', + }); + const cli = await runRelayCli(runtime, ['task', 'list', '--output', 'json'], { cwd: cliCwd }); + expect(cli.exitCode).toBe(0); + expect(await stat(runtime.databasePath)).toBeDefined(); + expect(await databaseFilesUnder(mcpCwd)).toEqual([]); + expect(await databaseFilesUnder(cliCwd)).toEqual([]); + expect(basename(dirname(runtime.databasePath))).toBe('data'); + } finally { + await client?.close(); + await server?.stop(); + applicationRuntime?.close(); + await runtime?.close(); + } + }); +}); + +async function databaseFilesUnder(root: string): Promise { + const entries = await readdir(root, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries) { + const path = join(root, entry.name); + if (entry.isDirectory()) files.push(...(await databaseFilesUnder(path))); + else if (/\.db(?:-(?:wal|shm))?$|^relay\.db$/i.test(entry.name)) files.push(path); + } + return files; +} diff --git a/tests/integration/mcp-cli-parity.test.ts b/tests/integration/mcp-cli-parity.test.ts index c2b2906..15f0969 100644 --- a/tests/integration/mcp-cli-parity.test.ts +++ b/tests/integration/mcp-cli-parity.test.ts @@ -1,3 +1,4 @@ +import Database from 'better-sqlite3'; import { describe, expect, it, vi } from 'vitest'; import { createTaskApplication, @@ -12,7 +13,18 @@ import { FixedIdGenerator, InMemoryTaskRepository, } from '../unit/application/tasks/task-test-fixtures.js'; +import type { AgentTestRuntime } from '../support/agent-test-runtime.js'; import { connectMcp } from '../unit/interfaces/mcp/mcp-test-utils.js'; +import type { McpTestClient } from '../support/mcp-test-client.js'; +import { createAgentTestRuntime } from '../support/agent-test-runtime.js'; +import { runRelayCli } from '../support/cli-test-process.js'; +import { createMcpTestClient } from '../support/mcp-test-client.js'; +import { + normalizeCliError, + normalizeCliSuccess, + normalizeMcpError, + normalizeMcpSuccess, +} from '../support/external-contract-normalizers.js'; const FIXED_NOW = new Date('2026-07-27T10:00:00.000Z'); @@ -244,3 +256,484 @@ describe('MCP and CLI semantic parity', () => { expect(cliConflict.error).toEqual(mcpConflict.error); }); }); + +describe('built MCP and CLI contract parity', () => { + it('captures through MCP and retrieves the identical public task through CLI', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('mcp-capture'), + }); + const capture = await client.callTool('task_capture', { + title: 'Built MCP capture', + createdByName: 'Codex', + sessionId: 'session-alpha', + workspace: 'relay-verification', + sourceContext: 'issue-25', + }); + const capturedTask = (normalizeMcpSuccess(capture).data as { task: Record }) + .task; + const cli = await runRelayCli( + runtime, + ['task', 'get', String(capturedTask.id), '--output', 'json'], + { cwd: await runtime.createWorkingDirectory('cli-get') }, + ); + + expect(cli.exitCode).toBe(0); + expect(normalizeCliSuccess(cli.json).data).toEqual({ task: capturedTask }); + expect(cli.stderr).toBe(''); + } finally { + await client?.close(); + await runtime?.close(); + } + }); + + it('captures through CLI and retrieves the identical public task through MCP', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); + const cli = await runRelayCli(runtime, [ + 'task', + 'capture', + '--title', + 'Built CLI capture', + '--agent', + 'Claude Code', + '--session', + 'session-beta', + '--workspace', + 'relay-verification', + '--source-context', + 'issue-25', + '--output', + 'json', + ]); + expect(cli.exitCode).toBe(0); + const capturedTask = (normalizeCliSuccess(cli.json).data as { task: Record }) + .task; + const mcp = await client.callTool('task_get', { taskId: capturedTask.id }); + + expect(normalizeMcpSuccess(mcp).data).toEqual({ task: capturedTask }); + expect(cli.stderr).toBe(''); + } finally { + await client?.close(); + await runtime?.close(); + } + }); + + it('keeps list and get DTO fields and persisted ordering identical across adapters', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); + for (const [title, sessionId] of [ + ['First built task', 'session-alpha'], + ['Second built task', 'session-beta'], + ] as const) { + await client.callTool('task_capture', { + title, + createdByName: 'Codex', + sessionId, + workspace: 'relay-verification', + }); + } + const mcp = normalizeMcpSuccess(await client.callTool('task_list', { limit: 100 })); + const cli = await runRelayCli(runtime, ['task', 'list', '--output', 'json']); + expect(cli.exitCode).toBe(0); + expect(normalizeCliSuccess(cli.json)).toEqual(mcp); + expect((mcp.data as { tasks: readonly unknown[] }).tasks).toHaveLength(2); + } finally { + await client?.close(); + await runtime?.close(); + } + }); + + it('preserves duplicate candidates, warnings, and match reasons without rejecting capture', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); + const existing = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'Duplicate candidate capture', + createdByName: 'Codex', + sessionId: 'session-alpha', + workspace: 'relay-verification', + }), + ); + const candidate = (existing.data as { task: { id: string } }).task.id; + const similar = normalizeMcpSuccess( + await client.callTool('task_find_similar', { + title: 'Duplicate candidate capture', + workspace: 'relay-verification', + }), + ); + const duplicate = await runRelayCli(runtime, [ + 'task', + 'capture', + '--title', + 'Duplicate candidate capture', + '--agent', + 'Claude Code', + '--session', + 'session-beta', + '--workspace', + 'relay-verification', + '--output', + 'json', + ]); + + expect(duplicate.exitCode).toBe(0); + expect(duplicate.json).toMatchObject({ + warnings: [ + { + code: 'POSSIBLE_DUPLICATE', + candidates: [{ id: candidate }], + }, + ], + }); + expect(similar.data).toMatchObject({ + candidates: [{ task: { id: candidate }, matchReason: expect.any(String) }], + }); + } finally { + await client?.close(); + await runtime?.close(); + } + }); + + it('returns identical mutation results when each adapter reads the other adapter’s persisted state', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); + const capture = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'Mutation parity task', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ); + const taskId = String((capture.data as { task: { id: string } }).task.id); + const edit = await runRelayCli(runtime, [ + 'task', + 'edit', + taskId, + '--title', + 'Mutation parity task edited', + '--output', + 'json', + ]); + expect(edit.exitCode).toBe(0); + expect(normalizeMcpSuccess(await client.callTool('task_get', { taskId })).data).toMatchObject( + { + task: { id: taskId, title: 'Mutation parity task edited' }, + }, + ); + + const triage = normalizeMcpSuccess( + await client.callTool('task_triage', { taskId, target: 'ACTIVE' }), + ); + expect(triage.data).toMatchObject({ + change: { action: 'TRIAGED', from: 'INBOX', to: 'ACTIVE' }, + }); + const start = await runRelayCli(runtime, ['task', 'start', taskId, '--output', 'json']); + expect(start.exitCode).toBe(0); + expect(normalizeMcpSuccess(await client.callTool('task_get', { taskId })).data).toMatchObject( + { + task: { status: 'IN_PROGRESS' }, + }, + ); + const complete = normalizeMcpSuccess(await client.callTool('task_complete', { taskId })); + expect(complete.data).toMatchObject({ + task: { status: 'DONE' }, + change: { action: 'COMPLETED' }, + }); + const archive = await runRelayCli(runtime, ['task', 'archive', taskId, '--output', 'json']); + expect(archive.exitCode).toBe(0); + expect(normalizeMcpSuccess(await client.callTool('task_get', { taskId })).data).toMatchObject( + { + task: { status: 'ARCHIVED' }, + }, + ); + } finally { + await client?.close(); + await runtime?.close(); + } + }); + + it('returns identical no-op change metadata without changing task timestamps', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); + const capture = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'No-op parity task', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ); + const task = (capture.data as { task: Record }).task; + const taskId = String(task.id); + const cli = await runRelayCli(runtime, [ + 'task', + 'edit', + taskId, + '--title', + 'No-op parity task', + '--output', + 'json', + ]); + const mcp = normalizeMcpSuccess( + await client.callTool('task_edit', { taskId, title: 'No-op parity task' }), + ); + + expect(cli.exitCode).toBe(0); + expect(normalizeCliSuccess(cli.json)).toEqual(mcp); + expect(mcp.data).toMatchObject({ + task: { id: taskId, createdAt: task.createdAt, updatedAt: task.updatedAt }, + change: { action: 'NO_CHANGE', fields: [] }, + }); + } finally { + await client?.close(); + await runtime?.close(); + } + }); + + it('maps not-found, invalid transition, and archived mutation errors consistently', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); + const notFoundCli = await runRelayCli(runtime, [ + 'task', + 'get', + 'missing-id', + '--output', + 'json', + ]); + const notFoundMcp = await client.callTool('task_get', { taskId: 'missing-id' }); + expect(notFoundCli.exitCode).toBe(3); + expect(notFoundCli.json).toMatchObject({ error: { code: 'NOT_FOUND' } }); + expect(notFoundMcp).toMatchObject({ + isError: true, + structuredContent: { error: { code: 'NOT_FOUND' } }, + }); + expect(normalizeCliError(notFoundCli.json)).toEqual(normalizeMcpError(notFoundMcp)); + + const capture = normalizeMcpSuccess( + await client.callTool('task_capture', { + title: 'Error parity task', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ); + const taskId = String((capture.data as { task: { id: string } }).task.id); + const invalidCli = await runRelayCli(runtime, [ + 'task', + 'complete', + taskId, + '--output', + 'json', + ]); + const invalidMcp = await client.callTool('task_complete', { taskId }); + expect(invalidCli.exitCode).toBe(4); + expect(invalidCli.json).toMatchObject({ error: { code: 'CONFLICT' } }); + expect(invalidMcp).toMatchObject({ + isError: true, + structuredContent: { error: { code: 'CONFLICT' } }, + }); + expect(normalizeCliError(invalidCli.json)).toEqual(normalizeMcpError(invalidMcp)); + + await client.callTool('task_triage', { taskId, target: 'ACTIVE' }); + await client.callTool('task_start', { taskId }); + await client.callTool('task_complete', { taskId }); + await client.callTool('task_archive', { taskId }); + const archivedCli = await runRelayCli(runtime, [ + 'task', + 'edit', + taskId, + '--title', + 'Archived edit', + '--output', + 'json', + ]); + const archivedMcp = await client.callTool('task_edit', { taskId, title: 'Archived edit' }); + expect(archivedCli.exitCode).toBe(4); + expect(archivedCli.json).toMatchObject({ error: { code: 'ARCHIVED_TASK' } }); + expect(archivedMcp).toMatchObject({ + isError: true, + structuredContent: { error: { code: 'ARCHIVED_TASK' } }, + }); + expect(normalizeCliError(archivedCli.json)).toEqual(normalizeMcpError(archivedMcp)); + for (const value of [ + notFoundCli.json, + notFoundMcp, + invalidCli.json, + invalidMcp, + archivedCli.json, + archivedMcp, + ]) { + expect(JSON.stringify(value)).not.toMatch(/SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\/i); + } + } finally { + await client?.close(); + await runtime?.close(); + } + }); + + it('keeps malformed inputs at validation/protocol boundaries and preserves exit code 2', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); + const cli = await runRelayCli(runtime, [ + 'task', + 'capture', + '--title', + '', + '--agent', + 'Codex', + '--session', + 'session-alpha', + '--output', + 'json', + ]); + expect(cli.exitCode).toBe(2); + expect(cli.json).toMatchObject({ error: { code: 'VALIDATION_ERROR' } }); + await expect( + client.callTool('task_capture', { + title: '', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ).resolves.toMatchObject({ + isError: true, + content: [{ type: 'text', text: expect.stringMatching(/invalid arguments|title/i) }], + }); + expect(cli.stderr).not.toMatch(/SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\/i); + } finally { + await client?.close(); + await runtime?.close(); + } + }); + + it('matches built empty-session results and rejects malformed session IDs at each adapter boundary', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); + const mcpMissing = normalizeMcpSuccess( + await client.callTool('session_captures_list', { + sessionId: 'missing-session', + limit: 100, + }), + ); + const cliMissing = await runRelayCli(runtime, [ + 'session', + 'captures', + '--session', + 'missing-session', + '--output', + 'json', + ]); + expect(cliMissing.exitCode).toBe(0); + expect(normalizeCliSuccess(cliMissing.json)).toEqual(mcpMissing); + expect(mcpMissing.data).toMatchObject({ sessionId: 'missing-session', tasks: [], count: 0 }); + + const cliMalformed = await runRelayCli(runtime, [ + 'session', + 'captures', + '--session', + 'bad session', + '--output', + 'json', + ]); + expect(cliMalformed.exitCode).toBe(2); + expect(cliMalformed.json).toMatchObject({ error: { code: 'VALIDATION_ERROR' } }); + await expect( + client.callTool('session_captures_list', { sessionId: 'bad session', limit: 100 }), + ).resolves.toMatchObject({ + isError: true, + content: [{ type: 'text', text: expect.stringMatching(/invalid arguments|sessionId/i) }], + }); + } finally { + await client?.close(); + await runtime?.close(); + } + }); + + it('maps a locked-database write failure to the same structured storage error', async () => { + let runtime: AgentTestRuntime | undefined; + let client: McpTestClient | undefined; + let lock: Database.Database | undefined; + + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); + lock = new Database(runtime.databasePath); + await client.callTool('relay_health', {}); + + lock.pragma('busy_timeout = 100'); + lock.exec('BEGIN IMMEDIATE'); + + const [cli, mcp] = await Promise.all([ + runRelayCli(runtime, [ + 'task', + 'capture', + '--title', + 'Locked CLI capture', + '--agent', + 'Codex', + '--session', + 'session-alpha', + '--output', + 'json', + ]), + client.callTool('task_capture', { + title: 'Locked MCP capture', + createdByName: 'Codex', + sessionId: 'session-alpha', + }), + ]); + + expect(cli.exitCode).toBe(5); + expect(cli.json).toMatchObject({ + error: { code: 'STORAGE_ERROR', message: expect.any(String) }, + }); + expect(mcp).toMatchObject({ + isError: true, + structuredContent: { + error: { code: 'STORAGE_ERROR', message: expect.any(String) }, + }, + }); + expect(normalizeCliError(cli.json)).toEqual(normalizeMcpError(mcp)); + + for (const value of [cli.json, mcp, cli.stderr, client.stderr()]) { + expect(JSON.stringify(value)).not.toMatch( + /SQL|stack|RELAY_DB_PATH|[A-Z]:\\Users\\|\/Users\//i, + ); + } + } finally { + try { + lock?.exec('ROLLBACK'); + } catch { + // The transaction may already be closed after a setup failure. + } + lock?.close(); + await client?.close(); + await runtime?.close(); + } + }, 15_000); +}); diff --git a/tests/support/agent-test-runtime.ts b/tests/support/agent-test-runtime.ts new file mode 100644 index 0000000..9b50f44 --- /dev/null +++ b/tests/support/agent-test-runtime.ts @@ -0,0 +1,58 @@ +import { mkdir, mkdtemp, rm } from 'node:fs/promises'; +import { dirname, isAbsolute, relative, resolve, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export interface AgentTestRuntimeOptions { + readonly environment?: NodeJS.ProcessEnv; +} + +export interface AgentTestRuntime { + readonly checkoutPath: string; + readonly databasePath: string; + createWorkingDirectory(name: string): Promise; + environment(overrides?: NodeJS.ProcessEnv): NodeJS.ProcessEnv; + close(): Promise; +} + +const checkoutPath = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); + +export async function createAgentTestRuntime( + options: AgentTestRuntimeOptions = {}, +): Promise { + const temporaryRoot = join(checkoutPath, 'tmp'); + await mkdir(temporaryRoot, { recursive: true }); + const root = await mkdtemp(join(temporaryRoot, 'relay-agent-verification-')); + const dataDirectory = join(root, 'data'); + const workingDirectoryRoot = join(root, 'cwd'); + await Promise.all([mkdir(dataDirectory), mkdir(workingDirectoryRoot)]); + + const databasePath = join(dataDirectory, 'relay.db'); + let closed = false; + + return { + checkoutPath, + databasePath, + async createWorkingDirectory(name: string): Promise { + const workingDirectory = resolve(workingDirectoryRoot, name); + const escape = relative(workingDirectoryRoot, workingDirectory); + if (escape.startsWith('..') || isAbsolute(escape)) { + throw new Error(`Working directory escapes the disposable root: ${name}`); + } + await mkdir(workingDirectory, { recursive: true }); + return workingDirectory; + }, + environment(overrides: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...process.env, + ...options.environment, + ...overrides, + RELAY_DB_PATH: databasePath, + }; + }, + async close(): Promise { + if (closed) return; + closed = true; + await rm(root, { recursive: true, force: true }); + }, + }; +} diff --git a/tests/support/cli-test-process.ts b/tests/support/cli-test-process.ts new file mode 100644 index 0000000..9533119 --- /dev/null +++ b/tests/support/cli-test-process.ts @@ -0,0 +1,111 @@ +import { spawn, type ChildProcess } from 'node:child_process'; +import { join } from 'node:path'; +import type { AgentTestRuntime } from './agent-test-runtime.js'; + +export interface CliProcessOptions { + readonly cwd?: string; + readonly environment?: NodeJS.ProcessEnv; + readonly timeoutMs?: number; +} + +export interface CliProcessResult { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; + readonly json: unknown; +} + +export function runRelayCli( + runtime: AgentTestRuntime, + args: readonly string[], + options: CliProcessOptions = {}, +): Promise { + const command = join(runtime.checkoutPath, 'dist', 'cli', 'main.js'); + const child = spawn(process.execPath, [command, ...args], { + cwd: options.cwd ?? runtime.checkoutPath, + env: runtime.environment(options.environment), + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk: Buffer | string) => { + stdout += chunk.toString(); + }); + child.stderr.on('data', (chunk: Buffer | string) => { + stderr += chunk.toString(); + }); + + return new Promise((resolve, reject) => { + let settled = false; + const timeout = setTimeout(() => { + settled = true; + const timeoutError = new Error(`Relay CLI timed out after ${options.timeoutMs ?? 30_000}ms.`); + void terminateChild(child).then( + () => reject(timeoutError), + () => reject(timeoutError), + ); + }, options.timeoutMs ?? 30_000); + child.once('error', (error) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + reject(error); + }); + child.once('close', (code) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + try { + resolve({ + exitCode: code ?? 1, + stdout, + stderr, + json: parseSingleJson(stdout), + }); + } catch (error) { + reject(error); + } + }); + }); +} + +async function terminateChild(child: ChildProcess): Promise { + if (hasExited(child)) return; + + child.kill(); + if (await waitForClose(child, 250)) return; + + child.kill('SIGKILL'); + await waitForClose(child); +} + +function hasExited(child: ChildProcess): boolean { + return child.exitCode !== null || child.signalCode !== null; +} + +function waitForClose(child: ChildProcess, timeoutMs?: number): Promise { + if (hasExited(child)) return Promise.resolve(true); + + return new Promise((resolve) => { + let timeout: NodeJS.Timeout | undefined; + const onClose = () => { + if (timeout !== undefined) clearTimeout(timeout); + resolve(true); + }; + child.once('close', onClose); + if (timeoutMs !== undefined) { + timeout = setTimeout(() => { + child.removeListener('close', onClose); + resolve(false); + }, timeoutMs); + } + }); +} + +function parseSingleJson(stdout: string): unknown { + const trimmed = stdout.trim(); + if (trimmed.length === 0) throw new Error('Relay CLI produced no JSON output.'); + return JSON.parse(trimmed) as unknown; +} diff --git a/tests/support/external-contract-normalizers.ts b/tests/support/external-contract-normalizers.ts new file mode 100644 index 0000000..0a87e6f --- /dev/null +++ b/tests/support/external-contract-normalizers.ts @@ -0,0 +1,93 @@ +export interface ExternalError { + readonly code: string; + readonly message: string; + readonly details?: unknown; +} + +export interface ExternalOperationResult { + readonly schemaVersion: number; + readonly data: unknown; + readonly warnings: readonly unknown[]; +} + +export function normalizeCliSuccess(value: unknown): ExternalOperationResult { + const envelope = record(value, 'CLI result'); + assertKeys(envelope, ['schemaVersion', 'ok', 'data', 'warnings'], 'CLI result'); + return { + schemaVersion: number(envelope.schemaVersion), + data: envelope.data, + warnings: array(envelope.warnings), + }; +} + +export function normalizeMcpSuccess(value: unknown): ExternalOperationResult { + const envelope = record(value, 'MCP result'); + const structuredContent = record(envelope.structuredContent, 'MCP structured result'); + assertKeys(structuredContent, ['schemaVersion', 'data', 'warnings'], 'MCP structured result'); + return { + schemaVersion: number(structuredContent.schemaVersion), + data: structuredContent.data, + warnings: array(structuredContent.warnings), + }; +} + +export function normalizeCliError(value: unknown): ExternalError { + return normalizeError(record(value, 'CLI error').error); +} + +export function normalizeMcpError(value: unknown): ExternalError { + if (value instanceof Error) { + return { code: 'MCP_PROTOCOL_ERROR', message: value.message }; + } + const envelope = record(value, 'MCP error'); + const structuredContent = envelope.structuredContent; + return normalizeError(record(structuredContent, 'MCP structured error').error); +} + +function normalizeError(value: unknown): ExternalError { + const error = record(value, 'error'); + return { + code: string(error.code), + message: string(error.message), + ...(error.details === undefined ? {} : { details: error.details }), + }; +} + +function record(value: unknown, label: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`${label} must be an object.`); + } + return value as Record; +} + +function assertKeys( + value: Record, + expectedKeys: readonly string[], + label: string, +): void { + const actualKeys = Object.keys(value).sort(); + const sortedExpectedKeys = [...expectedKeys].sort(); + if ( + actualKeys.length !== sortedExpectedKeys.length || + actualKeys.some((key, index) => key !== sortedExpectedKeys[index]) + ) { + throw new Error( + `${label} keys must be exactly ${sortedExpectedKeys.join(', ')}; received ${actualKeys.join(', ')}.`, + ); + } +} + +function string(value: unknown): string { + if (typeof value !== 'string') throw new Error('Expected a string contract field.'); + return value; +} + +function number(value: unknown): number { + if (typeof value !== 'number') throw new Error('Expected a numeric schema version.'); + return value; +} + +function array(value: unknown): readonly unknown[] { + if (!Array.isArray(value)) throw new Error('Expected an array contract field.'); + return value; +} diff --git a/tests/support/mcp-test-client.ts b/tests/support/mcp-test-client.ts new file mode 100644 index 0000000..934874a --- /dev/null +++ b/tests/support/mcp-test-client.ts @@ -0,0 +1,74 @@ +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import { join } from 'node:path'; +import type { AgentTestRuntime } from './agent-test-runtime.js'; + +export interface McpTestClient { + listTools(): Promise; + callTool(name: string, args: Record): Promise; + stderr(): string; + close(): Promise; +} + +export interface McpTestClientOptions { + readonly cwd?: string; + readonly command?: string; + readonly args?: readonly string[]; + readonly environment?: NodeJS.ProcessEnv; +} + +export async function createMcpTestClient( + runtime: AgentTestRuntime, + options: McpTestClientOptions = {}, +): Promise { + const transport = new StdioClientTransport({ + command: options.command ?? process.execPath, + args: [...(options.args ?? [join(runtime.checkoutPath, 'dist', 'mcp', 'main.js')])], + cwd: options.cwd ?? runtime.checkoutPath, + env: stringEnvironment(runtime.environment(options.environment)), + stderr: 'pipe', + }); + let serverStderr = ''; + transport.stderr?.on('data', (chunk: Buffer | string) => { + serverStderr += chunk.toString(); + }); + + const client = new Client({ name: 'relay-issue-25-test-client', version: '1.0.0' }); + try { + await client.connect(transport); + } catch (error) { + await transport.close().catch(() => undefined); + throw error; + } + + let closed = false; + return { + async listTools() { + const result = await client.listTools(); + return result.tools.map(({ name }) => ({ name })); + }, + async callTool(name, args) { + return client.callTool({ name, arguments: args }); + }, + stderr() { + return serverStderr; + }, + async close() { + if (closed) return; + closed = true; + try { + await client.close(); + } finally { + await transport.close(); + } + }, + }; +} + +function stringEnvironment(environment: NodeJS.ProcessEnv): Record { + return Object.fromEntries( + Object.entries(environment).filter( + (entry): entry is [string, string] => entry[1] !== undefined, + ), + ); +} diff --git a/tests/unit/scripts/validate-agent-integration-assets.test.ts b/tests/unit/scripts/validate-agent-integration-assets.test.ts index eebe312..618a318 100644 --- a/tests/unit/scripts/validate-agent-integration-assets.test.ts +++ b/tests/unit/scripts/validate-agent-integration-assets.test.ts @@ -12,6 +12,7 @@ describe('validateAgentIntegrationAssets', () => { function createRoot(): string { const rootDir = mkdtempSync(join(tmpdir(), 'relay-agent-integration-assets-')); cpSync(fixtureRoot, rootDir, { recursive: true }); + cpSync(join(process.cwd(), 'skills'), join(rootDir, 'skills'), { recursive: true }); roots.push(rootDir); return rootDir; } @@ -226,4 +227,140 @@ describe('validateAgentIntegrationAssets', () => { expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/future-only/i); }); + + it('rejects a canonical capture skill without autonomous-create permission', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'skills/relay-capture/SKILL.md'); + writeFileSync( + path, + readFileSync(path, 'utf8').replace('autonomously create only', 'may create only'), + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow( + /canonical capture.*autonom/i, + ); + }); + + it('rejects a canonical capture skill that permits lifecycle mutation', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'skills/relay-capture/SKILL.md'); + writeFileSync( + path, + readFileSync(path, 'utf8').replace( + 'It must not edit, triage, start, complete, archive, delete, merge, or move any task', + 'It may edit, triage, start, complete, archive, delete, merge, or move any task', + ), + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow( + /canonical capture.*lifecycle/i, + ); + }); + + it('rejects a canonical session-review skill that omits completed and archived captures', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'skills/relay-session-review/SKILL.md'); + writeFileSync( + path, + readFileSync(path, 'utf8').replace('completed and archived captures', 'INBOX tasks'), + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/session-review.*status/i); + }); + + it('rejects a canonical session-review skill without explicit user-action guidance', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'skills/relay-session-review/SKILL.md'); + writeFileSync( + path, + readFileSync(path, 'utf8').replaceAll('explicit user direction', 'automatic action'), + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/session-review.*explicit/i); + }); + + it('rejects a vendor wrapper that copies mutation autonomy policy', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-cli/README.md'); + writeFileSync( + path, + `${readFileSync(path, 'utf8')}\n## Autonomy boundaries\nThe agent may autonomously edit and archive tasks.`, + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow( + /vendor.*policy|behavioural policy/i, + ); + }); + + it('accepts vendor guidance that explicitly prohibits autonomous edits', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-cli/README.md'); + writeFileSync( + path, + `${readFileSync(path, 'utf8')}\nThe agent must not autonomously edit tasks.`, + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).not.toThrow(); + }); + + it('rejects a vendor wrapper without both canonical skill references', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-mcp/README.md'); + writeFileSync( + path, + readFileSync(path, 'utf8').replace('skills/relay-session-review/SKILL.md', ''), + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow(/generic-mcp.*skill/i); + }); + + it('rejects a configuration template that points at a non-canonical MCP entry', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-mcp/server-config.json.example'); + writeFileSync( + path, + readFileSync(path, 'utf8').replace('dist/mcp/main.js', 'dist/other-mcp.js'), + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow( + /canonical.*dist\/mcp\/main\.js/i, + ); + }); + + it('rejects removal guidance that deletes the SQLite database', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-cli/README.md'); + writeFileSync( + path, + `${readFileSync(path, 'utf8')} Remove the SQLite database when disabling Relay.`, + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow( + /removal.*database|destructive/i, + ); + }); + + it('accepts removal guidance that explicitly prohibits deleting the SQLite database', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-cli/README.md'); + writeFileSync(path, `${readFileSync(path, 'utf8')}\nDo not delete the SQLite database.`); + + expect(() => validateAgentIntegrationAssets({ rootDir })).not.toThrow(); + }); + + it('rejects removal guidance that does not distinguish configuration from stored data', () => { + const rootDir = createRoot(); + const path = join(rootDir, 'integrations/generic-cli/README.md'); + writeFileSync( + path, + readFileSync(path, 'utf8').replace( + 'SQLite database remains untouched', + 'Relay data may be deleted', + ), + ); + + expect(() => validateAgentIntegrationAssets({ rootDir })).toThrow( + /removal.*configuration|database remains/i, + ); + }); }); diff --git a/tests/unit/scripts/validate-repository-assets.test.ts b/tests/unit/scripts/validate-repository-assets.test.ts index 20ac27b..a2035ff 100644 --- a/tests/unit/scripts/validate-repository-assets.test.ts +++ b/tests/unit/scripts/validate-repository-assets.test.ts @@ -74,11 +74,11 @@ function createFixtureRoot(): string { }); writeFileSync( join(rootDir, 'skills/relay-capture/SKILL.md'), - `## Purpose\n\nCapture a concrete, actionable follow-up.\n\n## When to capture\n\nUse it for a concrete, actionable follow-up.\n\n## Adapter selection\n\nMCP is preferred. CLI is the fallback with --output json and one adapter.\n\n## Session and provenance\n\nThe agent supplies createdByName and the exact active session ID. Relay supplies createdByType: AGENT and status: INBOX.\n\n## Capture procedure\n\nContinue the original work.\n\n## Duplicate handling\n\nA duplicate is advisory.\n\n## Context safety\n\nKeep context concise.\n\n## Autonomy boundaries\n\nAn agent must not edit, triage, start, complete, or archive tasks. Leave captures in INBOX.\n\n## Do not capture\n\nDo not capture speculation.\n`, + `## Purpose\n\nCapture a concrete, actionable follow-up.\n\n## When to capture\n\nUse it for a concrete, actionable follow-up.\n\n## Adapter selection\n\nMCP is preferred. CLI is the fallback with --output json and one adapter.\n\n## Session and provenance\n\nThe agent supplies createdByName and the exact active session ID. Relay supplies createdByType: AGENT and status: INBOX.\n\n## Capture procedure\n\nContinue the original work. An agent may autonomously create only a new Relay task in INBOX.\n\n## Duplicate handling\n\nA duplicate is advisory.\n\n## Context safety\n\nKeep context concise.\n\n## Autonomy boundaries\n\nAn agent must not edit, triage, start, complete, archive, delete, merge, or move any task. Leave captures in INBOX.\n\n## Do not capture\n\nDo not capture speculation.\n`, ); writeFileSync( join(rootDir, 'skills/relay-session-review/SKILL.md'), - `## Purpose\n\nReview before final completion.\n\n## When to review\n\nAlways perform the exact active session lookup before final completion.\n\n## Session lookup\n\nUse the exact active session ID. Include completed and archived tasks; never mix sessions. An empty authoritative result is valid.\n\n## Review presentation\n\nPresent captures.\n\n## User-directed actions\n\nRequire explicit user direction and intent-specific actions.\n\n## Unresolved captures\n\nLeave unresolved tasks in INBOX.\n\n## Adapter selection\n\nUse the same adapter.\n\n## Prohibited behaviour\n\nNever infer completion from timer, inactivity, or process exit.\n`, + `## Purpose\n\nReview before final completion.\n\n## When to review\n\nAlways perform the exact active session lookup before final completion.\n\n## Session lookup\n\nUse the exact active session ID. Include completed and archived captures; never mix sessions. An empty authoritative result is valid.\n\n## Review presentation\n\nPresent captures.\n\n## User-directed actions\n\nRequire explicit user direction and intent-specific actions.\n\n## Unresolved captures\n\nLeave unresolved tasks in INBOX.\n\n## Adapter selection\n\nUse the same adapter.\n\n## Prohibited behaviour\n\nNever infer completion from timer, inactivity, or process exit.\n`, ); for (const [path, name, description] of [ ['skills/relay-capture/SKILL.md', 'relay-capture', 'Use when testing capture.'], diff --git a/tests/unit/support/agent-test-runtime.test.ts b/tests/unit/support/agent-test-runtime.test.ts new file mode 100644 index 0000000..abc4316 --- /dev/null +++ b/tests/unit/support/agent-test-runtime.test.ts @@ -0,0 +1,68 @@ +import { rm, stat } from 'node:fs/promises'; +import { dirname, isAbsolute, join, relative, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { getDefaultDatabasePath } from '../../../src/database/database-config.js'; +import { createAgentTestRuntime } from '../../support/agent-test-runtime.js'; + +describe('createAgentTestRuntime', () => { + it('creates an isolated absolute database path and arbitrary working directories', async () => { + const runtime = await createAgentTestRuntime(); + try { + const disposableRoot = dirname(dirname(runtime.databasePath)); + const repositoryTemporaryRoot = resolve( + dirname(fileURLToPath(import.meta.url)), + '../../..', + 'tmp', + ); + + expect(runtime.databasePath).toBe(join(disposableRoot, 'data', 'relay.db')); + expect(isAbsolute(runtime.databasePath)).toBe(true); + expect(relative(repositoryTemporaryRoot, disposableRoot)).not.toMatch(/^\.\.(?:[\\/]|$)/); + expect(runtime.databasePath).not.toBe(getDefaultDatabasePath()); + + const cwd = await runtime.createWorkingDirectory('nested/client'); + expect(isAbsolute(cwd)).toBe(true); + expect(cwd).not.toBe(runtime.checkoutPath); + + const environment = runtime.environment({ RELAY_TEST_MARKER: 'isolated' }); + expect(environment.RELAY_DB_PATH).toBe(runtime.databasePath); + expect(environment.RELAY_TEST_MARKER).toBe('isolated'); + } finally { + await runtime.close(); + } + }); + + it('creates the shared temporary parent when it is absent', async () => { + const repositoryTemporaryRoot = resolve( + dirname(fileURLToPath(import.meta.url)), + '../../..', + 'tmp', + ); + await rm(repositoryTemporaryRoot, { recursive: true, force: true }); + + const runtime = await createAgentTestRuntime(); + try { + expect(runtime.databasePath).toContain(join('tmp', 'relay-agent-verification-')); + await expect(stat(dirname(runtime.databasePath))).resolves.toBeDefined(); + } finally { + await runtime.close(); + } + }); + + it('removes the disposable directory including SQLite sidecars and closes idempotently', async () => { + const runtime = await createAgentTestRuntime(); + const root = dirname(dirname(runtime.databasePath)); + await import('node:fs/promises').then(({ writeFile }) => + Promise.all([ + writeFile(`${runtime.databasePath}-wal`, 'test'), + writeFile(`${runtime.databasePath}-shm`, 'test'), + ]), + ); + + await runtime.close(); + await runtime.close(); + + await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' }); + }); +}); diff --git a/tests/unit/support/cli-test-process.test.ts b/tests/unit/support/cli-test-process.test.ts new file mode 100644 index 0000000..d369fb9 --- /dev/null +++ b/tests/unit/support/cli-test-process.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest'; +import { createAgentTestRuntime } from '../../support/agent-test-runtime.js'; +import { runRelayCli } from '../../support/cli-test-process.js'; + +describe('runRelayCli', () => { + it('runs the built CLI from an arbitrary cwd and parses one JSON document', async () => { + const runtime = await createAgentTestRuntime(); + try { + const cwd = await runtime.createWorkingDirectory('cli/nested'); + const result = await runRelayCli(runtime, ['task', 'list', '--output', 'json'], { cwd }); + + expect(result.exitCode).toBe(0); + expect(result.stderr).toBe(''); + expect(result.json).toMatchObject({ schemaVersion: expect.any(Number) }); + } finally { + await runtime.close(); + } + }); + + it('retains stdout, stderr, and a stable non-zero exit code on failure', async () => { + const runtime = await createAgentTestRuntime(); + try { + const result = await runRelayCli(runtime, ['task', 'get', 'missing-id', '--output', 'json']); + + expect(result.exitCode).toBe(3); + expect(result.json).toMatchObject({ + schemaVersion: expect.any(Number), + error: { code: 'NOT_FOUND', message: expect.any(String) }, + }); + expect(result.stdout.trim().split(/\r?\n/)).toHaveLength(1); + expect(result.stderr).toContain('Task was not found.'); + } finally { + await runtime.close(); + } + }); + + it('terminates a child process that exceeds the configured timeout', async () => { + const runtime = await createAgentTestRuntime(); + try { + await expect( + runRelayCli(runtime, ['task', 'list', '--output', 'json'], { timeoutMs: 1 }), + ).rejects.toThrow(/timed out/i); + } finally { + await runtime.close(); + } + }); +}); diff --git a/tests/unit/support/external-contract-normalizers.test.ts b/tests/unit/support/external-contract-normalizers.test.ts new file mode 100644 index 0000000..a7f42c9 --- /dev/null +++ b/tests/unit/support/external-contract-normalizers.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest'; +import { + normalizeCliSuccess, + normalizeMcpSuccess, +} from '../../support/external-contract-normalizers.js'; + +describe('external contract normalizers', () => { + it('normalizes the allowed CLI success envelope fields', () => { + expect( + normalizeCliSuccess({ schemaVersion: 1, ok: true, data: { count: 0 }, warnings: [] }), + ).toEqual({ schemaVersion: 1, data: { count: 0 }, warnings: [] }); + }); + + it('rejects renamed or unexpected CLI success envelope fields', () => { + expect(() => + normalizeCliSuccess({ schemaVersion: 1, ok: true, data: {}, warning: [] }), + ).toThrow(/CLI result.*keys/i); + }); + + it('normalizes the allowed MCP structured success envelope fields', () => { + expect( + normalizeMcpSuccess({ + structuredContent: { schemaVersion: 1, data: { count: 0 }, warnings: [] }, + content: [], + }), + ).toEqual({ schemaVersion: 1, data: { count: 0 }, warnings: [] }); + }); + + it('rejects renamed or unexpected MCP structured success fields', () => { + expect(() => + normalizeMcpSuccess({ + structuredContent: { schemaVersion: 1, data: {}, warning: [] }, + content: [], + }), + ).toThrow(/MCP structured result.*keys/i); + }); +}); diff --git a/tests/unit/support/mcp-test-client.test.ts b/tests/unit/support/mcp-test-client.test.ts new file mode 100644 index 0000000..bcda3c4 --- /dev/null +++ b/tests/unit/support/mcp-test-client.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import type { McpTestClient } from '../../support/mcp-test-client.js'; +import { createAgentTestRuntime } from '../../support/agent-test-runtime.js'; +import { createMcpTestClient } from '../../support/mcp-test-client.js'; + +describe('createMcpTestClient', () => { + it('discovers Relay tools while keeping server stdout protocol-owned', async () => { + let runtime: Awaited> | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime, { + cwd: await runtime.createWorkingDirectory('mcp/nested'), + }); + const tools = await client.listTools(); + expect(tools.map((tool) => tool.name)).toEqual( + expect.arrayContaining([ + 'relay_health', + 'task_capture', + 'task_list', + 'task_get', + 'task_find_similar', + 'session_captures_list', + ]), + ); + expect(client.stderr()).not.toContain('Content-Length'); + } finally { + await client?.close(); + await runtime?.close(); + } + }); + + it('surfaces unknown-tool protocol failures and still closes the child', async () => { + let runtime: Awaited> | undefined; + let client: McpTestClient | undefined; + try { + runtime = await createAgentTestRuntime(); + client = await createMcpTestClient(runtime); + await expect(client.callTool('unknown_tool', {})).resolves.toMatchObject({ isError: true }); + expect(client.stderr()).toBe(''); + } finally { + await client?.close(); + await runtime?.close(); + } + }); +});