diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 27037623..a01d306b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -155,6 +155,92 @@ jobs: retention-days: 14 if-no-files-found: ignore + researcher-e2e: + name: Researcher / Unit + Playwright E2E + runs-on: ubuntu-latest + timeout-minutes: 25 + + services: + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_USER: memwal + POSTGRES_PASSWORD: memwal_secret + POSTGRES_DB: memwal + ports: ["5432:5432"] + options: >- + --health-cmd "pg_isready -U memwal" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + redis: + image: redis:7-alpine + ports: ["6379:6379"] + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-retries 10 + + env: + POSTGRES_URL: postgresql://memwal:memwal_secret@localhost:5432/memwal + REDIS_URL: redis://localhost:6379 + AUTH_SECRET: ci-test-secret-not-for-production + OPENROUTER_API_KEY: mock-not-used-tests-use-local-mocks + # Only read by the delegate-account binding mock to fabricate the + # MemWalAccount type; any non-empty 0x id works. + NEXT_PUBLIC_MEMWAL_PACKAGE_ID: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + PORT: "3000" + PLAYWRIGHT: "True" + NODE_ENV: test + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: pnpm + + - name: Install deps + run: pnpm install --frozen-lockfile + + - name: Test and build SDK (workspace dep of researcher) + run: pnpm --filter @mysten-incubation/memwal test + + # Fast node:test unit tests (no DB/browser). Includes the retired-model + # regression tests for the title-model production incident. + - name: Unit tests + run: pnpm --filter researcher test:unit + + - name: Cache Playwright browsers + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: pw-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: pw-${{ runner.os }}- + + - name: Install Playwright (Chromium + OS deps) + run: pnpm --filter researcher playwright:install + + - name: Run Playwright E2E + run: pnpm --filter researcher test:e2e + + - name: Upload Playwright report + traces + if: always() + uses: actions/upload-artifact@v4 + with: + name: playwright-report-researcher + path: | + apps/researcher/playwright-report + apps/researcher/test-results + retention-days: 14 + if-no-files-found: ignore + server-e2e: name: Server / E2E runs-on: ubuntu-latest diff --git a/apps/researcher/.gitignore b/apps/researcher/.gitignore index 864e97de..b3e2d7be 100644 --- a/apps/researcher/.gitignore +++ b/apps/researcher/.gitignore @@ -42,3 +42,4 @@ yarn-error.log* /playwright-report/ /blob-report/ /playwright/* +/tests/playwright/.auth/ diff --git a/apps/researcher/lib/ai/models.mock.ts b/apps/researcher/lib/ai/models.mock.ts index 3babc204..6b907c0f 100644 --- a/apps/researcher/lib/ai/models.mock.ts +++ b/apps/researcher/lib/ai/models.mock.ts @@ -121,6 +121,21 @@ const createMockReasoningModel = (): LanguageModel => { } as unknown as LanguageModel; }; +/** + * When a user message contains this sentinel, the mock title model rejects. + * The e2e suite uses it to reproduce the retired-title-model production + * failure and assert the chat stream still completes without an error part + * (the try/catch guard in app/(chat)/api/chat/route.ts). + * Mirrored in tests/playwright/helpers.ts — keep the two in sync. + */ +export const TITLE_FAILURE_SENTINEL = "FAIL_TITLE_GENERATION"; + +function assertTitlePromptOk(prompt: unknown): void { + if (JSON.stringify(prompt).includes(TITLE_FAILURE_SENTINEL)) { + throw new Error("Mock title model failure (TITLE_FAILURE_SENTINEL)"); + } +} + const createMockTitleModel = (): LanguageModel => { return { specificationVersion: "v3", @@ -128,18 +143,22 @@ const createMockTitleModel = (): LanguageModel => { modelId: "mock-title-model", defaultObjectGenerationMode: "tool", supportedUrls: {}, - doGenerate: async () => ({ - finishReason: "stop", - usage: { - inputTokens: { total: 5, noCache: 5, cacheRead: 0, cacheWrite: 0 }, - outputTokens: { total: 5, text: 5, reasoning: 0 }, - }, - content: [{ type: "text", text: "Test Conversation" }], - warnings: [], - }), - doStream: () => ({ + doGenerate: async ({ prompt }: { prompt: unknown }) => { + assertTitlePromptOk(prompt); + return { + finishReason: "stop", + usage: { + inputTokens: { total: 5, noCache: 5, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 5, text: 5, reasoning: 0 }, + }, + content: [{ type: "text", text: "Test Conversation" }], + warnings: [], + }; + }, + doStream: ({ prompt }: { prompt: unknown }) => ({ stream: new ReadableStream({ start(controller) { + assertTitlePromptOk(prompt); controller.enqueue({ type: "text-start", id: "t1" }); controller.enqueue({ type: "text-delta", diff --git a/apps/researcher/lib/ai/providers.ts b/apps/researcher/lib/ai/providers.ts index 29c26008..2375aa55 100644 --- a/apps/researcher/lib/ai/providers.ts +++ b/apps/researcher/lib/ai/providers.ts @@ -32,14 +32,33 @@ export const myProvider = isTestEnvironment })() : null; +function isReasoningModelId(modelId: string) { + return ( + modelId.endsWith("-thinking") || + (modelId.includes("reasoning") && !modelId.includes("non-reasoning")) + ); +} + +const MOCK_MODEL_IDS = new Set([ + "chat-model", + "chat-model-reasoning", + "title-model", +]); + export function getLanguageModel(modelId: string) { if (isTestEnvironment && myProvider) { - return myProvider.languageModel(modelId); + // The picker sends real OpenRouter ids; the mock provider only registers + // its own three, so anything else must map onto them or languageModel() + // throws NoSuchModelError on the first test message. + if (MOCK_MODEL_IDS.has(modelId)) { + return myProvider.languageModel(modelId); + } + return myProvider.languageModel( + isReasoningModelId(modelId) ? "chat-model-reasoning" : "chat-model" + ); } - const isReasoningModel = - modelId.endsWith("-thinking") || - (modelId.includes("reasoning") && !modelId.includes("non-reasoning")); + const isReasoningModel = isReasoningModelId(modelId); if (isReasoningModel) { const gatewayModelId = modelId.replace(THINKING_SUFFIX_REGEX, ""); diff --git a/apps/researcher/lib/auth/delegate-account.mock.ts b/apps/researcher/lib/auth/delegate-account.mock.ts new file mode 100644 index 00000000..fcc1cacd --- /dev/null +++ b/apps/researcher/lib/auth/delegate-account.mock.ts @@ -0,0 +1,69 @@ +import * as ed from "@noble/ed25519"; +import { sha512 } from "@noble/hashes/sha512"; +import { enokiConfig } from "@/lib/enoki/config"; + +if (!ed.etc.sha512Sync) { + ed.etc.sha512Sync = (...m: Uint8Array[]) => { + const h = sha512.create(); + for (const msg of m) h.update(msg); + return h.digest(); + }; +} + +/** + * Fixture identities for Playwright runs (lib/constants.ts isTestEnvironment). + * + * Each account id maps to exactly one delegate private key, so the real + * binding validation in delegate-account.ts still runs meaningfully against + * the fabricated object: an unknown account id fails lookup, and a key that + * doesn't derive the registered public key is rejected — the same failure + * modes as the on-chain path. + * + * Mirrored in tests/playwright/fixtures/test-accounts.ts — keep in sync. + */ +const TEST_DELEGATE_ACCOUNTS: ReadonlyArray<{ + accountId: string; + privateKey: string; +}> = [ + { accountId: `0x${"aa".repeat(32)}`, privateKey: "a".repeat(64) }, + { accountId: `0x${"bb".repeat(32)}`, privateKey: "b".repeat(64) }, +]; + +function hexToBytes(hex: string): Uint8Array { + const bytes = new Uint8Array(hex.length / 2); + for (let i = 0; i < bytes.length; i++) { + bytes[i] = Number.parseInt(hex.slice(i * 2, i * 2 + 2), 16); + } + return bytes; +} + +function bytesToHex(bytes: Uint8Array): string { + return Array.from(bytes) + .map((b) => b.toString(16).padStart(2, "0")) + .join(""); +} + +/** + * Fabricated stand-in for the Sui gRPC getObject response, shaped exactly + * like the fields delegate-account.ts validates. Returns null for account + * ids outside the fixture list (the "object not found" case). + */ +export function mockDelegateAccountObject( + accountId: string +): { type: string; json: unknown } | null { + const fixture = TEST_DELEGATE_ACCOUNTS.find( + (account) => account.accountId.toLowerCase() === accountId.toLowerCase() + ); + if (!fixture) { + return null; + } + + const publicKey = bytesToHex(ed.getPublicKey(hexToBytes(fixture.privateKey))); + return { + type: `${enokiConfig.memwalPackageId}::account::MemWalAccount`, + json: { + active: true, + delegate_keys: [{ public_key: publicKey }], + }, + }; +} diff --git a/apps/researcher/lib/auth/delegate-account.ts b/apps/researcher/lib/auth/delegate-account.ts index 677c2142..eb64bfaf 100644 --- a/apps/researcher/lib/auth/delegate-account.ts +++ b/apps/researcher/lib/auth/delegate-account.ts @@ -1,5 +1,6 @@ import { SuiGrpcClient } from "@mysten/sui/grpc"; import { fromBase64, normalizeSuiAddress, toHex } from "@mysten/sui/utils"; +import { isTestEnvironment } from "@/lib/constants"; import { enokiConfig } from "@/lib/enoki/config"; export class DelegateAccountBindingError extends Error { @@ -85,6 +86,27 @@ export async function assertDelegateAccountBinding(input: { ); } + if (isTestEnvironment) { + // Playwright runs have no chain to read. Serve a fixture object instead + // of the gRPC fetch so the real validation below still executes — an + // unknown account or unregistered key fails the same way it would live. + const { mockDelegateAccountObject } = await import( + "./delegate-account.mock" + ); + const mocked = mockDelegateAccountObject(input.accountId); + if (!mocked) { + throw new DelegateAccountBindingError( + "Unable to verify Walrus Memory account" + ); + } + const mockError = delegateAccountBindingError(mocked.type, mocked.json, { + publicKeyHex: input.publicKeyHex, + packageId: enokiConfig.memwalPackageId, + }); + if (mockError) throw new DelegateAccountBindingError(mockError); + return; + } + const network = enokiConfig.suiNetwork; const client = new SuiGrpcClient({ network, diff --git a/apps/researcher/lib/sprint/memwal.ts b/apps/researcher/lib/sprint/memwal.ts index 3bf7aa64..1dd2ef9e 100644 --- a/apps/researcher/lib/sprint/memwal.ts +++ b/apps/researcher/lib/sprint/memwal.ts @@ -1,10 +1,20 @@ import "server-only"; -import { MemWal } from "@mysten-incubation/memwal"; +import { MemWal, MemWalMock } from "@mysten-incubation/memwal"; import type { RememberResult } from "@mysten-incubation/memwal"; +import { isTestEnvironment } from "@/lib/constants"; import type { Citation, SourceMeta } from "./types"; -function getMemWalClient(key: string, accountId: string) { +// Shared across requests so a sprint remembered in one Playwright request is +// recallable in the next — and so test runs can never write to the real +// relayer configured in MEMWAL_SERVER_URL. +let testMemWalClient: MemWalMock | null = null; + +function getMemWalClient(key: string, accountId: string): MemWal | MemWalMock { + if (isTestEnvironment) { + testMemWalClient ??= MemWalMock.create({ owner: "playwright" }); + return testMemWalClient; + } return MemWal.create({ key, accountId, diff --git a/apps/researcher/package.json b/apps/researcher/package.json index a77ee0cc..9af9ccb2 100644 --- a/apps/researcher/package.json +++ b/apps/researcher/package.json @@ -18,7 +18,10 @@ "verify:memwal": "tsx ../../scripts/verify-memwal-credentials.ts", "test:redis-outage": "node --conditions=react-server --test --import tsx lib/shared-redis.test.ts", "test:unit": "node --test --import tsx 'lib/**/*.unit.test.ts'", - "test": "export PLAYWRIGHT=True && pnpm exec playwright test" + "test": "pnpm test:e2e", + "playwright:install": "playwright install --with-deps chromium", + "test:e2e": "PLAYWRIGHT=True playwright test", + "test:e2e:ui": "PLAYWRIGHT=True playwright test --ui" }, "dependencies": { "@ai-sdk/gateway": "^3.0.15", diff --git a/apps/researcher/playwright.config.ts b/apps/researcher/playwright.config.ts new file mode 100644 index 00000000..279a99d1 --- /dev/null +++ b/apps/researcher/playwright.config.ts @@ -0,0 +1,105 @@ +import path from "node:path"; +import { defineConfig, devices } from "@playwright/test"; +import { config } from "dotenv"; + +// Later files never override earlier ones, matching Next's own precedence. +config({ path: ".env.local" }); +config({ path: ".env" }); + +// Match the dev script's default port (next dev, no --port flag). +const PORT = process.env.PORT || "3000"; +const baseURL = `http://localhost:${PORT}`; + +const isCI = !!process.env.CI; + +export const STORAGE_STATE_USER_A = path.join( + __dirname, + "tests/playwright/.auth/user-a.json" +); + +// A second identity for cross-account tests. Signed in once by the setup +// project rather than per-test: the auth limiter allows only 10 verify +// attempts per IP per minute, which retries would otherwise exhaust. +export const STORAGE_STATE_USER_B = path.join( + __dirname, + "tests/playwright/.auth/user-b.json" +); + +export default defineConfig({ + testDir: "./tests/playwright", + outputDir: "./test-results", + fullyParallel: true, + forbidOnly: isCI, + retries: isCI ? 2 : 0, + workers: 2, + reporter: isCI + ? [ + ["html", { open: "never", outputFolder: "playwright-report" }], + ["github"], + ["list"], + ["junit", { outputFile: "playwright-report/junit.xml" }], + ] + : [["html", { open: "never", outputFolder: "playwright-report" }], ["list"]], + + globalSetup: require.resolve("./tests/playwright/global-setup"), + + use: { + baseURL, + trace: "retain-on-failure", + video: isCI ? "retain-on-failure" : "off", + screenshot: "only-on-failure", + actionTimeout: 10_000, + // 30s to tolerate cold Next.js/Turbopack compile on 2-vCPU CI runners; + // globalSetup also warms key routes to make first-nav fast. + navigationTimeout: 30_000, + }, + + timeout: 60_000, + expect: { timeout: 10_000 }, + + projects: [ + { + // Signs in through the real login form once and saves the session + // cookie; every e2e spec starts from that storage state. + name: "setup", + testMatch: /auth\.setup\.ts$/, + use: { ...devices["Desktop Chrome"] }, + }, + { + name: "e2e", + testMatch: /e2e\/.*\.test\.ts$/, + dependencies: ["setup"], + use: { + ...devices["Desktop Chrome"], + storageState: STORAGE_STATE_USER_A, + }, + }, + ], + + webServer: { + command: "pnpm dev", + url: `${baseURL}/ping`, + timeout: 120_000, + reuseExistingServer: !isCI, + stdout: "pipe", + stderr: "pipe", + env: { + // Force every mock seam (lib/constants.ts → isTestEnvironment): + // AI models, the delegate-account binding check, and the Walrus client. + PLAYWRIGHT: "True", + // Ensure the webServer binds the port baseURL targets. + PORT, + // The session JWT is minted and verified inside this one process, so + // any non-empty secret works when the environment doesn't provide one. + AUTH_SECRET: + process.env.AUTH_SECRET ?? "playwright-e2e-secret-not-for-production", + // The binding mock fabricates the MemWalAccount type from this id; it + // only has to be non-empty and stable for the run. + NEXT_PUBLIC_MEMWAL_PACKAGE_ID: + process.env.NEXT_PUBLIC_MEMWAL_PACKAGE_ID ?? `0x${"ee".repeat(32)}`, + // The auth rate limiter is fail-closed (503 without Redis); default to + // the conventional local instance when the environment doesn't say. + REDIS_URL: process.env.REDIS_URL ?? "redis://localhost:6379", + }, + }, +}); diff --git a/apps/researcher/tests/README.md b/apps/researcher/tests/README.md new file mode 100644 index 00000000..db7bf1ab --- /dev/null +++ b/apps/researcher/tests/README.md @@ -0,0 +1,108 @@ +# Researcher E2E tests (Playwright) + +Playwright tests live under `tests/playwright/`. They start a Next.js dev +server on `localhost:3000` and drive Chromium against it. Every external +dependency is mocked at the module layer (see "What is mocked") so a test run +never calls OpenRouter, Sui, or the Walrus relayer. + +## Prerequisites + +- Node 22 (or 20) + pnpm 9. +- PostgreSQL with the schema migrations applied: + ```bash + docker compose -f apps/researcher/docker-compose.yml up -d + ``` +- Redis. The auth rate limiter is fail-closed — without it, every login + returns 503 "Authentication service temporarily unavailable": + ```bash + docker run -d --name researcher-redis -p 127.0.0.1:6379:6379 redis:7-alpine + ``` + +## Local run + +```bash +# one-time: install Playwright browsers + system deps +pnpm --filter researcher playwright:install + +# run the E2E suite +pnpm --filter researcher test:e2e + +# same, interactive UI +pnpm --filter researcher test:e2e:ui + +# debug a single test with Playwright Inspector +PWDEBUG=1 pnpm --filter researcher test:e2e --grep "P0 regression" +``` + +`playwright.config.ts` loads `.env.local` then `.env` from `apps/researcher/`, +so a local `POSTGRES_URL` and `REDIS_URL` are picked up automatically. It also +supplies safe defaults for `AUTH_SECRET`, `NEXT_PUBLIC_MEMWAL_PACKAGE_ID`, and +`REDIS_URL` when the environment doesn't set them. + +`global-setup.ts` applies Drizzle migrations (idempotent), clears the auth +rate-limit keys from Redis, and warms `/login` and `/` so the first +navigations don't race Turbopack's cold compile. + +The rate-limit reset matters: the auth limiter allows 10 verify attempts per +IP per minute and a run spends about five, so without it a second run inside +a minute fails at sign-in with a 429 that looks nothing like the real problem. +For the same reason both identities sign in once in the setup project rather +than per-test. + +## What is mocked + +All four seams key off `isTestEnvironment` in `lib/constants.ts`, true whenever +`PLAYWRIGHT`, `PLAYWRIGHT_TEST_BASE_URL`, or `CI_PLAYWRIGHT` is set. The +Playwright config exports `PLAYWRIGHT=True` to both the runner and the +webServer, so the Next.js process takes the mock path too. + +| Seam | File | Behavior under test | +|------|------|---------------------| +| LLMs | `lib/ai/providers.ts` → `lib/ai/models.mock.ts` | Deterministic streamed text; picker model ids map onto the three mock models. | +| Title model failure | `lib/ai/models.mock.ts` | A user message containing `FAIL_TITLE_GENERATION` makes the title model reject — reproduces the retired-model production P0. | +| Delegate-account binding | `lib/auth/delegate-account.ts` → `.mock.ts` | Fabricates the `MemWalAccount` object for two fixture accounts. The real validation still runs, so wrong-key and unknown-account logins fail exactly as they would on-chain. | +| Walrus Memory | `lib/sprint/memwal.ts` | `MemWalMock` from the SDK, one instance per server process, so remember → recall round-trips in memory and CI can never write to the real relayer. | + +Fixture identities live in `tests/playwright/fixtures/test-accounts.ts` and are +mirrored in `lib/auth/delegate-account.mock.ts` — change one, change both. + +There is no live-model or live-Walrus canary in this suite. The real +remember → recall loop is verified manually against the production relayer. + +## Suite layout + +| File | Covers | +|------|--------| +| `auth.setup.ts` | Signs in as account A through the real delegate-key form (doubles as the happy-path login test) and as account B through the API, saving both storage states. | +| `e2e/auth.test.ts` | Anonymous redirects; unregistered key, unknown account, and malformed key rejections. | +| `e2e/chat.test.ts` | Input and suggestions render; a message streams an assistant reply; a finished chat survives reload with both rows. Asserts zero uncaught page errors (guards the auto-resume `TypeError`). | +| `e2e/chat-stream.test.ts` | Raw SSE frames from `/api/chat`: text streams, a title arrives, and a **failing title model never injects an `error` frame** (the P0 regression test). | +| `e2e/visibility.test.ts` | Private chats render Next's not-found page for other users with no content leak; public chats are readable by other signed-in users; anonymous visitors always land on `/login`. | + +## CI + +The `researcher-e2e` job in `.github/workflows/test.yml` provisions Postgres +and Redis as service containers, runs the `node:test` unit suite, installs +Chromium, and runs `pnpm test:e2e`. On failure these artifacts are uploaded and +kept for 14 days: + +- `playwright-report-researcher/playwright-report/index.html` — HTML report + with trace links +- `playwright-report-researcher/playwright-report/junit.xml` — JUnit results +- `playwright-report-researcher/test-results/**` — raw traces, screenshots, + videos for failed tests + +## Adding new tests + +- Location: `tests/playwright/e2e/.test.ts`. +- Selectors: prefer `data-testid`, then role + accessible name. Avoid CSS + class selectors — they break on design refactors. +- Specs run signed in as account A by default. For anonymous coverage, opt out + with `test.use({ storageState: { cookies: [], origins: [] } })`; for a second + identity, create a context with `storageState: STORAGE_STATE_USER_B`. +- `browser.newContext()` inherits the project's `use.storageState` (account A), + so any context that must be someone else — including an anonymous one — has + to pass its own `storageState` explicitly. Miss it and the test still passes, + for the wrong reason. +- Keep tests independent: never depend on ordering or on side effects from + other tests. `fullyParallel: true` will expose any implicit coupling. diff --git a/apps/researcher/tests/playwright/auth.setup.ts b/apps/researcher/tests/playwright/auth.setup.ts new file mode 100644 index 00000000..6dd8ca73 --- /dev/null +++ b/apps/researcher/tests/playwright/auth.setup.ts @@ -0,0 +1,45 @@ +import { mkdirSync } from "node:fs"; +import path from "node:path"; +import { expect, test as setup } from "@playwright/test"; +import { + STORAGE_STATE_USER_A, + STORAGE_STATE_USER_B, +} from "../../playwright.config"; +import { TEST_ACCOUNT_A, TEST_ACCOUNT_B } from "./fixtures/test-accounts"; +import { loginViaApi } from "./helpers"; + +/** + * Signs in once through the real delegate-key form and saves the session + * cookie as storage state for the e2e project. Doubles as the happy-path + * login test: if the form, the /api/auth/key route, or the binding check + * regress, everything downstream fails here with a precise error. + */ +setup("sign in with delegate key", async ({ page }) => { + await page.goto("/login"); + + await page + .getByRole("button", { name: "Sign in with delegate key" }) + .click(); + await page.locator("#accountId").fill(TEST_ACCOUNT_A.accountId); + await page.locator("#privateKey").fill(TEST_ACCOUNT_A.privateKey); + await page.getByRole("button", { name: "Sign In", exact: true }).click(); + + await page.waitForURL("/"); + await expect(page.getByTestId("multimodal-input")).toBeVisible(); + + mkdirSync(path.dirname(STORAGE_STATE_USER_A), { recursive: true }); + await page.context().storageState({ path: STORAGE_STATE_USER_A }); +}); + +/** + * Second identity for cross-account tests, signed in through the API rather + * than the form (the form is already covered above). Done once here so the + * visibility specs don't spend a verify-bucket slot each — the limiter allows + * 10 per IP per minute, and CI retries would exhaust that. + */ +setup("sign in as the second account", async ({ request, baseURL }) => { + await loginViaApi(request, baseURL!, TEST_ACCOUNT_B); + + mkdirSync(path.dirname(STORAGE_STATE_USER_B), { recursive: true }); + await request.storageState({ path: STORAGE_STATE_USER_B }); +}); diff --git a/apps/researcher/tests/playwright/e2e/auth.test.ts b/apps/researcher/tests/playwright/e2e/auth.test.ts new file mode 100644 index 00000000..5f6d6209 --- /dev/null +++ b/apps/researcher/tests/playwright/e2e/auth.test.ts @@ -0,0 +1,67 @@ +import { expect, test } from "@playwright/test"; +import { + TEST_ACCOUNT_A, + UNKNOWN_ACCOUNT_ID, + UNREGISTERED_PRIVATE_KEY, +} from "../fixtures/test-accounts"; + +// Auth negatives run without the shared signed-in session. +test.use({ storageState: { cookies: [], origins: [] } }); + +async function submitDelegateLogin( + page: import("@playwright/test").Page, + accountId: string, + privateKey: string +) { + await page.goto("/login"); + await page.getByRole("button", { name: "Sign in with delegate key" }).click(); + await page.locator("#accountId").fill(accountId); + await page.locator("#privateKey").fill(privateKey); + await page.getByRole("button", { name: "Sign In", exact: true }).click(); +} + +test.describe("Authentication", () => { + test("anonymous visitor is redirected to /login", async ({ page }) => { + await page.goto("/"); + await expect(page).toHaveURL(/\/login$/); + }); + + test("anonymous visitor cannot open a chat URL", async ({ page }) => { + await page.goto("/chat/00000000-0000-4000-8000-000000000000"); + await expect(page).toHaveURL(/\/login$/); + }); + + test("unregistered delegate key is rejected", async ({ page }) => { + await submitDelegateLogin( + page, + TEST_ACCOUNT_A.accountId, + UNREGISTERED_PRIVATE_KEY + ); + await expect(page.getByTestId("toast")).toContainText( + /not registered/i + ); + await expect(page).toHaveURL(/\/login$/); + }); + + test("unknown account id is rejected", async ({ page }) => { + await submitDelegateLogin( + page, + UNKNOWN_ACCOUNT_ID, + TEST_ACCOUNT_A.privateKey + ); + await expect(page.getByTestId("toast")).toContainText( + /unable to verify/i + ); + await expect(page).toHaveURL(/\/login$/); + }); + + test("malformed private key is rejected with a clear message", async ({ + page, + }) => { + await submitDelegateLogin(page, TEST_ACCOUNT_A.accountId, "not-a-key"); + await expect(page.getByTestId("toast")).toContainText( + /invalid private key/i + ); + await expect(page).toHaveURL(/\/login$/); + }); +}); diff --git a/apps/researcher/tests/playwright/e2e/chat-stream.test.ts b/apps/researcher/tests/playwright/e2e/chat-stream.test.ts new file mode 100644 index 00000000..510bef3d --- /dev/null +++ b/apps/researcher/tests/playwright/e2e/chat-stream.test.ts @@ -0,0 +1,63 @@ +import { expect, test } from "@playwright/test"; +import { + framesOfType, + postChatMessage, + TITLE_FAILURE_SENTINEL, +} from "../helpers"; + +/** + * Raw SSE assertions against /api/chat. These pin the exact production P0: + * a rejected title generation used to surface as an `error` frame on the + * stream, making every new chat render as failed even though the answer had + * streamed fine underneath. + */ +test.describe("Chat stream protocol", () => { + test("new chat streams text and a title with no error frames", async ({ + request, + baseURL, + }) => { + const { response, frames } = await postChatMessage(request, baseURL!, { + text: "Stream protocol check", + }); + + expect(response.status()).toBe(200); + expect(framesOfType(frames, "text-delta").length).toBeGreaterThan(0); + expect(framesOfType(frames, "error")).toHaveLength(0); + + const titleFrames = framesOfType(frames, "data-chat-title"); + expect(titleFrames).toHaveLength(1); + expect(titleFrames[0].data).toBe("Test Conversation"); + }); + + test("a failing title model never injects an error frame (P0 regression)", async ({ + request, + baseURL, + page, + }) => { + const { chatId, response, frames } = await postChatMessage( + request, + baseURL!, + { + // The sentinel makes the mock title model reject, reproducing the + // retired-model 404 that broke production. The guard in + // app/(chat)/api/chat/route.ts must swallow it. + text: `Please explain ${TITLE_FAILURE_SENTINEL} to me`, + } + ); + + expect(response.status()).toBe(200); + // The answer still streams… + expect(framesOfType(frames, "text-delta").length).toBeGreaterThan(0); + // …the failed title never becomes a client-visible error… + expect(framesOfType(frames, "error")).toHaveLength(0); + // …and no title frame is emitted for the failed generation. + expect(framesOfType(frames, "data-chat-title")).toHaveLength(0); + + // The chat itself is intact: it reloads with both messages and no + // failure UI. + await page.goto(`/chat/${chatId}`); + await expect(page.getByTestId("message-content")).toHaveCount(2, { + timeout: 15_000, + }); + }); +}); diff --git a/apps/researcher/tests/playwright/e2e/chat.test.ts b/apps/researcher/tests/playwright/e2e/chat.test.ts new file mode 100644 index 00000000..42c9ccd1 --- /dev/null +++ b/apps/researcher/tests/playwright/e2e/chat.test.ts @@ -0,0 +1,69 @@ +import { expect, test } from "@playwright/test"; + +// The mock chat model streams one of a small fixed set of responses +// (lib/ai/models.mock.ts), so an assistant bubble with stable text proves the +// full request → stream → render path. +test.describe("Chat", () => { + let pageErrors: Error[]; + + test.beforeEach(({ page }) => { + pageErrors = []; + page.on("pageerror", (error) => pageErrors.push(error)); + }); + + test.afterEach(() => { + expect( + pageErrors, + `Uncaught page errors: ${pageErrors.map((e) => e.message).join("; ")}` + ).toHaveLength(0); + }); + + test("home page shows input and suggestions", async ({ page }) => { + await page.goto("/"); + await expect(page.getByTestId("multimodal-input")).toBeVisible(); + await expect(page.getByTestId("suggested-actions").first()).toBeVisible(); + }); + + test("sending a message streams an assistant reply", async ({ page }) => { + await page.goto("/"); + + await page.getByTestId("multimodal-input").fill("Hello there"); + await page.getByTestId("send-button").click(); + + // User bubble + assistant bubble. + await expect(page.getByTestId("message-content")).toHaveCount(2, { + timeout: 15_000, + }); + await expect(page.getByTestId("multimodal-input")).toHaveValue(""); + + // Chat URL is claimed so the conversation is shareable/reloadable. + await expect(page).toHaveURL(/\/chat\/[0-9a-f-]{36}/); + }); + + test("a finished chat survives reload with both messages", async ({ + page, + }) => { + await page.goto("/"); + await page.getByTestId("multimodal-input").fill("Hello persistence"); + await page.getByTestId("send-button").click(); + await expect(page.getByTestId("message-content")).toHaveCount(2, { + timeout: 15_000, + }); + await expect(page).toHaveURL(/\/chat\/[0-9a-f-]{36}/); + + // Streaming has rendered; give persistence a beat, then reload the + // permalink. Both rows must come back from the database — this guards + // the missing-assistant-row symptom seen in production, and the + // afterEach page-error assertion guards the auto-resume crash that + // used to throw on reopening chats. + await expect + .poll( + async () => { + await page.reload(); + return page.getByTestId("message-content").count(); + }, + { timeout: 15_000 } + ) + .toBe(2); + }); +}); diff --git a/apps/researcher/tests/playwright/e2e/visibility.test.ts b/apps/researcher/tests/playwright/e2e/visibility.test.ts new file mode 100644 index 00000000..83e27e46 --- /dev/null +++ b/apps/researcher/tests/playwright/e2e/visibility.test.ts @@ -0,0 +1,112 @@ +import { expect, test } from "@playwright/test"; +import { + STORAGE_STATE_USER_A, + STORAGE_STATE_USER_B, +} from "../../../playwright.config"; +import { postChatMessage } from "../helpers"; + +/** + * The Private/Public boundary, pinned as it behaves today. + * + * Contexts here are created explicitly rather than via the `page` fixture, + * because each test needs two identities. Note that `browser.newContext()` + * inherits this project's `use.storageState` (user A), so every context that + * must NOT be user A passes its own storage state explicitly — without it + * the "other user" is silently user A and these tests pass for the wrong + * reason. + */ +const ANONYMOUS = { cookies: [], origins: [] }; + +test.describe("Chat visibility", () => { + test("private chat is not readable by another user", async ({ + browser, + baseURL, + }) => { + const canary = `private-canary-${Date.now()}`; + + const contextA = await browser.newContext({ + baseURL, + storageState: STORAGE_STATE_USER_A, + }); + const { chatId } = await postChatMessage(contextA.request, baseURL!, { + text: canary, + visibility: "private", + }); + + const contextB = await browser.newContext({ + baseURL, + storageState: STORAGE_STATE_USER_B, + }); + const pageB = await contextB.newPage(); + await pageB.goto(`/chat/${chatId}`); + + // The route renders Next's not-found page. It does so with HTTP 200 + // rather than 404 because app/(chat)/chat/[id]/page.tsx wraps the async + // component in , so the streaming shell is already flushed by + // the time notFound() runs — assert on what the user actually gets. + await expect( + pageB.getByText(/This page could not be found/i) + ).toBeVisible(); + await expect(pageB.getByText(canary)).toHaveCount(0); + + await contextA.close(); + await contextB.close(); + }); + + test("public chat is readable by another signed-in user", async ({ + browser, + baseURL, + }) => { + const canary = `public-canary-${Date.now()}`; + + const contextA = await browser.newContext({ + baseURL, + storageState: STORAGE_STATE_USER_A, + }); + const { chatId } = await postChatMessage(contextA.request, baseURL!, { + text: canary, + visibility: "public", + }); + + const contextB = await browser.newContext({ + baseURL, + storageState: STORAGE_STATE_USER_B, + }); + const pageB = await contextB.newPage(); + const response = await pageB.goto(`/chat/${chatId}`); + + expect(response?.status()).toBe(200); + await expect(pageB.getByText(canary)).toBeVisible(); + + await contextA.close(); + await contextB.close(); + }); + + test("anonymous visitor is redirected to login even for a public chat", async ({ + browser, + baseURL, + }) => { + const contextA = await browser.newContext({ + baseURL, + storageState: STORAGE_STATE_USER_A, + }); + const { chatId } = await postChatMessage(contextA.request, baseURL!, { + text: `anon-canary-${Date.now()}`, + visibility: "public", + }); + + // Known product gap: "Public — Anyone with the link" is not true for + // logged-out visitors, because proxy.ts's auth check precedes the + // visibility check. Pinned deliberately; update if that changes. + const anonContext = await browser.newContext({ + baseURL, + storageState: ANONYMOUS, + }); + const anonPage = await anonContext.newPage(); + await anonPage.goto(`/chat/${chatId}`); + await expect(anonPage).toHaveURL(/\/login$/); + + await contextA.close(); + await anonContext.close(); + }); +}); diff --git a/apps/researcher/tests/playwright/fixtures/test-accounts.ts b/apps/researcher/tests/playwright/fixtures/test-accounts.ts new file mode 100644 index 00000000..c22a2f5d --- /dev/null +++ b/apps/researcher/tests/playwright/fixtures/test-accounts.ts @@ -0,0 +1,27 @@ +/** + * Fixture identities the delegate-account binding mock registers + * (lib/auth/delegate-account.mock.ts — keep the two lists in sync). + * + * Each account id maps to exactly one delegate private key, so wrong-key and + * unknown-account logins fail the same way they would against the chain. + */ +export type TestAccount = { + accountId: string; + privateKey: string; +}; + +export const TEST_ACCOUNT_A: TestAccount = { + accountId: `0x${"aa".repeat(32)}`, + privateKey: "a".repeat(64), +}; + +export const TEST_ACCOUNT_B: TestAccount = { + accountId: `0x${"bb".repeat(32)}`, + privateKey: "b".repeat(64), +}; + +/** Valid 64-hex key that is NOT registered on any fixture account. */ +export const UNREGISTERED_PRIVATE_KEY = "c".repeat(64); + +/** Well-formed account id the binding mock has never heard of. */ +export const UNKNOWN_ACCOUNT_ID = `0x${"cc".repeat(32)}`; diff --git a/apps/researcher/tests/playwright/global-setup.ts b/apps/researcher/tests/playwright/global-setup.ts new file mode 100644 index 00000000..e127554d --- /dev/null +++ b/apps/researcher/tests/playwright/global-setup.ts @@ -0,0 +1,96 @@ +/** + * Playwright global setup. + * + * Runs once before any test (after the webServer is spawned). Responsible for: + * 1. Applying Drizzle migrations so chat/user tables exist. + * 2. Failing fast with a clear error if POSTGRES_URL is missing in CI. + * 3. Clearing the auth rate-limit counters left by a previous run. + * 4. Warming `/login` and `/` so the suite's first navigations don't race + * Turbopack's lazy cold compile against the navigationTimeout on CI. + */ +import { spawnSync } from "node:child_process"; +import { createClient } from "redis"; + +/** + * The auth limiter allows 10 verify attempts per IP per minute and the suite + * spends about five. Without this, two runs inside a minute — routine while + * iterating locally — fail at sign-in with a 429 that looks nothing like the + * real problem. Only this app's own limiter keys are touched. + */ +async function clearAuthRateLimits(): Promise { + const url = process.env.REDIS_URL; + if (!url) return; + + const client = createClient({ url }); + client.on("error", () => { + // Handled by the catch below; without a listener node-redis throws. + }); + + try { + await client.connect(); + const keys = await client.keys("auth-rate-limit:*"); + if (keys.length > 0) { + await client.del(keys); + console.log(`[playwright] Cleared ${keys.length} auth rate-limit keys`); + } + } catch (err) { + console.warn("[playwright] Could not clear auth rate limits:", err); + } finally { + try { + client.destroy(); + } catch { + // already closed + } + } +} + +export default async function globalSetup(): Promise { + const url = process.env.POSTGRES_URL; + + if (!url) { + if (process.env.CI) { + throw new Error( + "POSTGRES_URL is required in CI. Start a Postgres service container and export the URL." + ); + } + console.warn( + "[playwright] POSTGRES_URL not set — skipping migrations (local dev only)" + ); + } else { + console.log("[playwright] Applying Drizzle migrations..."); + const result = spawnSync("pnpm", ["exec", "tsx", "lib/db/migrate.ts"], { + stdio: "inherit", + env: process.env, + }); + + if (result.status !== 0) { + throw new Error( + `[playwright] Migration failed with exit code ${result.status}` + ); + } + } + + await clearAuthRateLimits(); + + // Prime Next.js/Turbopack's per-route compile cache. On a cold CI runner + // the first navigation to a lazily-compiled route can take 15-30s, which + // flakes tests until retries kick in. Fetching here moves the cost to + // setup time so the first real navigations hit a warm cache. + const port = process.env.PORT ?? "3000"; + for (const route of ["/login", "/"]) { + const target = `http://localhost:${port}${route}`; + console.log(`[playwright] Warming ${target} ...`); + try { + const started = Date.now(); + const res = await fetch(target, { + redirect: "manual", // `/` 307s to /login pre-auth; we just want the compile + signal: AbortSignal.timeout(60_000), + }); + console.log( + `[playwright] Warm-up done in ${Date.now() - started}ms (status ${res.status})` + ); + } catch (err) { + console.warn("[playwright] Warm-up failed, continuing:", err); + } + } +} diff --git a/apps/researcher/tests/playwright/helpers.ts b/apps/researcher/tests/playwright/helpers.ts new file mode 100644 index 00000000..14b9dbd3 --- /dev/null +++ b/apps/researcher/tests/playwright/helpers.ts @@ -0,0 +1,104 @@ +import { randomUUID } from "node:crypto"; +import type { APIRequestContext, APIResponse } from "@playwright/test"; +import type { TestAccount } from "./fixtures/test-accounts"; + +/** + * Mirrors TITLE_FAILURE_SENTINEL in lib/ai/models.mock.ts (kept as a copy so + * specs don't import app code through Playwright's transpiler). A user + * message containing it makes the mock title model reject, reproducing the + * retired-title-model production failure. + */ +export const TITLE_FAILURE_SENTINEL = "FAIL_TITLE_GENERATION"; + +/** Default id from lib/ai/models.ts — must be in the route's allowlist. */ +export const DEFAULT_CHAT_MODEL_ID = "google/gemini-2.5-flash"; + +/** + * Log in through the API from a browser-context request so the session + * cookie lands on the context. The route enforces same-origin, so the + * Origin header must match the app host. + */ +export async function loginViaApi( + request: APIRequestContext, + baseURL: string, + account: TestAccount +): Promise { + const res = await request.post("/api/auth/key", { + headers: { origin: baseURL }, + data: { + privateKey: account.privateKey, + accountId: account.accountId, + }, + }); + if (!res.ok()) { + throw new Error( + `Login failed for ${account.accountId}: ${res.status()} ${await res.text()}` + ); + } +} + +export type PostChatResult = { + chatId: string; + response: APIResponse; + body: string; + frames: Array>; +}; + +/** + * Send one user message to /api/chat and return the parsed SSE data frames. + * The response only resolves once the stream has fully drained, so callers + * can assert on the complete frame sequence. + */ +export async function postChatMessage( + request: APIRequestContext, + baseURL: string, + { + text, + chatId = randomUUID(), + visibility = "private", + }: { + text: string; + chatId?: string; + visibility?: "public" | "private"; + } +): Promise { + const response = await request.post("/api/chat", { + headers: { origin: baseURL }, + data: { + id: chatId, + message: { + id: randomUUID(), + role: "user", + parts: [{ type: "text", text }], + }, + selectedChatModel: DEFAULT_CHAT_MODEL_ID, + selectedVisibilityType: visibility, + }, + }); + + const body = await response.text(); + return { chatId, response, body, frames: parseSseFrames(body) }; +} + +/** Parse `data: {...}` SSE lines into JSON frames, skipping non-JSON lines. */ +export function parseSseFrames(body: string): Array> { + const frames: Array> = []; + for (const line of body.split("\n")) { + if (!line.startsWith("data: ")) continue; + const payload = line.slice("data: ".length).trim(); + if (!payload || payload === "[DONE]") continue; + try { + frames.push(JSON.parse(payload)); + } catch { + // ignore non-JSON data lines + } + } + return frames; +} + +export function framesOfType( + frames: Array>, + type: string +): Array> { + return frames.filter((frame) => frame.type === type); +}