diff --git a/.gitignore b/.gitignore index 98cce1ee4..d8a33e0a3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ node_modules # Build outputs dist/ out/ +packages/*/build/ dist-e2e-mac/ dist-e2e-win/ release/ diff --git a/docs/design.md b/docs/design.md index dc2905cf8..7bb72fc64 100644 --- a/docs/design.md +++ b/docs/design.md @@ -247,18 +247,20 @@ Workspace tokens share the same visual intent as several shadcn tokens. Workspac Workspace-only tokens without a shadcn counterpart, plus shadow tokens. For shared surface colors, see **Workspace ↔ shadcn Equivalence** above. -| Token | Tailwind class | Light value | Usage | -| ----------------------------------- | ---------------------------------- | ----------------------------------------------------------------- | ------------------------------------------- | -| `--bg-400` | `bg-bg-400` | `hsl(45 10% 88%)` | Sidebar row action hover | -| `--text-300` | `text-text-300` | `hsl(43 3% 57%)` | Action icon default color and loading dots | -| `--rail-card-bg` | `bg-rail-card-bg` | `hsl(0 0% 100%)` | Sidebar rail card | -| `--danger-000` / `--danger-900` | `text-danger-000`, `bg-danger-900` | `hsl(0 45% 38%)`, `hsl(0 55% 95%)` | Destructive session menu and dialog actions | -| `--action-panel-toggle` | `text-action-panel-toggle` | `hsl(0 0% 42%)` | Collapsed preview toggle | -| `--surface-control-hover` | `hover:bg-surface-control-hover` | `hsl(38 20% 90%)` | Header icon control hover | -| `--message-user-text` | `text-message-user-text` | `hsl(0 0% 12%)` | User message bubble text | -| `--shadow-card` | `shadow-card` | `0 0 0 1px rgb(10 10 10 / 0.06), 0 4px 24px rgb(10 10 10 / 0.04)` | Sidebar rail card and composer dock | -| `--shadow-card-opaque` | `shadow-card-opaque` | `0 0 0 1px rgb(10 10 10 / 0.08), 0 8px 28px rgb(10 10 10 / 0.1)` | Composer form | -| `--shadow-menu` / `--shadow-dialog` | `shadow-menu`, `shadow-dialog` | `0 2px 8px rgb(0 0 0 / 0.08)`, `0 8px 32px rgb(10 10 10 / 12%)` | Menus and modal dialogs | +| Token | Tailwind class | Light value | Usage | +| ----------------------------------- | -------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | ---------------------------------------------------- | +| `--bg-400` | `bg-bg-400` | `hsl(45 10% 88%)` | Sidebar row action hover | +| `--text-300` | `text-text-300` | `hsl(43 3% 57%)` | Action icon default color and loading dots | +| `--rail-card-bg` | `bg-rail-card-bg` | `hsl(0 0% 100%)` | Sidebar rail card | +| `--danger-000` / `--danger-900` | `text-danger-000`, `bg-danger-900` | `hsl(0 45% 38%)`, `hsl(0 55% 95%)` | Destructive session menu and dialog actions | +| `--action-panel-toggle` | `text-action-panel-toggle` | `hsl(0 0% 42%)` | Collapsed preview toggle | +| `--surface-control-hover` | `hover:bg-surface-control-hover` | `hsl(38 20% 90%)` | Header icon control hover | +| `--message-user-text` | `text-message-user-text` | `hsl(0 0% 12%)` | User message bubble text | +| `--diff-added-*` | `bg-diff-added-surface`, `bg-diff-added-highlight`, `text-diff-added-foreground` | Light green surfaces with `hsl(145 60% 24%)` foreground | Added Version-diff rows, inline spans, and markers | +| `--diff-removed-*` | `bg-diff-removed-surface`, `bg-diff-removed-highlight`, `text-diff-removed-foreground` | Light red surfaces with `hsl(0 55% 32%)` foreground | Removed Version-diff rows, inline spans, and markers | +| `--shadow-card` | `shadow-card` | `0 0 0 1px rgb(10 10 10 / 0.06), 0 4px 24px rgb(10 10 10 / 0.04)` | Sidebar rail card and composer dock | +| `--shadow-card-opaque` | `shadow-card-opaque` | `0 0 0 1px rgb(10 10 10 / 0.08), 0 8px 28px rgb(10 10 10 / 0.1)` | Composer form | +| `--shadow-menu` / `--shadow-dialog` | `shadow-menu`, `shadow-dialog` | `0 2px 8px rgb(0 0 0 / 0.08)`, `0 8px 32px rgb(10 10 10 / 12%)` | Menus and modal dialogs | ### Settings Status and Category Tokens diff --git a/e2e/fixtures/electron-app.ts b/e2e/fixtures/electron-app.ts index 6bba7c68f..abdb29307 100644 --- a/e2e/fixtures/electron-app.ts +++ b/e2e/fixtures/electron-app.ts @@ -16,6 +16,8 @@ const APP_ROOT = resolve(process.cwd()) const FAKE_AGENT_PATH = resolve(APP_ROOT, 'e2e', 'fixtures', 'fake-opencode.mjs') const FAKE_REMOTEIT_PATH = resolve(APP_ROOT, 'e2e', 'fixtures', 'fake-remoteit.cjs') const FAKE_PROVIDER_NAME = 'Electron E2E provider' +const E2E_LOCALE_ARGUMENT = '--lang=en-US' +const E2E_SETTINGS = `${JSON.stringify({ localePreference: 'en' }, null, 2)}\n` type E2eWindowMode = 'hidden' | 'normal' const electronLaunchTarget = ( @@ -27,6 +29,7 @@ const electronLaunchTarget = ( return { args: [ `--user-data-dir=${userDataRoot}`, + E2E_LOCALE_ARGUMENT, ...(platform === 'linux' ? ['--password-store=basic'] : []), ...(executablePath ? [] : [APP_ROOT]) ], @@ -290,6 +293,7 @@ class ElectronAppHarness implements ElectronApp { ) try { await mkdir(harness.roots.storageRoot, { recursive: true }) + await writeFile(join(harness.roots.storageRoot, 'settings.json'), E2E_SETTINGS, 'utf8') await writeFile(harness.roots.fakeRemoteItState, JSON.stringify({ services: [] }), 'utf8') await writeFakeAgentLauncher(harness.roots.fakeAgentBinRoot) await writeFakeRemoteItCommands(harness.roots.fakeRemoteItRoot) @@ -438,6 +442,7 @@ class ElectronAppHarness implements ElectronApp { executable, [ `--user-data-dir=${this.roots.userDataRoot}`, + E2E_LOCALE_ARGUMENT, ...(process.env.OPEN_SCIENCE_E2E_EXECUTABLE ? [] : [appPath]) ], { diff --git a/e2e/launch-environment.spec.ts b/e2e/launch-environment.spec.ts index 3394189de..9718050c0 100644 --- a/e2e/launch-environment.spec.ts +++ b/e2e/launch-environment.spec.ts @@ -31,6 +31,10 @@ test('allows native window-system tests to opt into normal presentation', () => expect(environment.OPEN_SCIENCE_E2E_WINDOW_MODE).toBe('normal') }) +test('pins the Electron UI to English for stable accessibility selectors', () => { + expect(electronLaunchTarget('profile-root', {}, 'darwin').args).toContain('--lang=en-US') +}) + test('enables Session CPU tracing only for an active local performance profile', () => { const ordinary = launchEnvironment('storage-root', undefined, {}) const profiled = launchEnvironment('storage-root', undefined, {}, undefined, 'hidden', true) @@ -41,13 +45,18 @@ test('enables Session CPU tracing only for an active local performance profile', test('enables the basic password store only for Linux E2E profiles', () => { expect(electronLaunchTarget('profile-root', {}, 'linux')).toEqual({ - args: ['--user-data-dir=profile-root', '--password-store=basic', expect.any(String)] + args: [ + '--user-data-dir=profile-root', + '--lang=en-US', + '--password-store=basic', + expect.any(String) + ] }) expect(electronLaunchTarget('profile-root', {}, 'darwin')).toEqual({ - args: ['--user-data-dir=profile-root', expect.any(String)] + args: ['--user-data-dir=profile-root', '--lang=en-US', expect.any(String)] }) expect(electronLaunchTarget('profile-root', {}, 'win32')).toEqual({ - args: ['--user-data-dir=profile-root', expect.any(String)] + args: ['--user-data-dir=profile-root', '--lang=en-US', expect.any(String)] }) }) @@ -61,10 +70,15 @@ test('launches packaged and source applications with the expected Linux argument 'linux' ) ).toEqual({ - args: ['--user-data-dir=profile-root', '--password-store=basic'], + args: ['--user-data-dir=profile-root', '--lang=en-US', '--password-store=basic'], executablePath: '/artifacts/Open Science.app/Contents/MacOS/Open Science' }) expect(electronLaunchTarget('profile-root', {}, 'linux')).toEqual({ - args: ['--user-data-dir=profile-root', '--password-store=basic', expect.any(String)] + args: [ + '--user-data-dir=profile-root', + '--lang=en-US', + '--password-store=basic', + expect.any(String) + ] }) }) diff --git a/e2e/workspace-files.spec.ts b/e2e/workspace-files.spec.ts index 548f18efe..d1bdb7380 100644 --- a/e2e/workspace-files.spec.ts +++ b/e2e/workspace-files.spec.ts @@ -1,5 +1,5 @@ import { expect } from '@playwright/test' -import type { Page } from 'playwright' +import type { Locator, Page } from 'playwright' import { test } from './fixtures/electron-app' const PROJECT_NAME = 'Project files journey' @@ -10,6 +10,11 @@ const IMAGE_CONTENT = Buffer.from( 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=', 'base64' ) +const VERSION_TWO_CONTENT = '# Fixture findings\n\nFirst edited version.' +const VERSION_THREE_CONTENT = '## Fixture findings\n\nSecond edited version.' +const SCRIPT_NAME = 'analysis.sh' +const SCRIPT_CONTENT = '#!/bin/bash\n# stable\necho "old"\n' +const SCRIPT_VERSION_TWO_CONTENT = '#!/bin/bash\n# stable\necho "new"\n' const createProject = async (page: Page): Promise => { await page.getByRole('button', { name: 'New project' }).click() @@ -19,7 +24,54 @@ const createProject = async (page: Page): Promise => { await expect(page.getByRole('heading', { name: 'New conversation' })).toBeVisible() } -test('uploads an attachment and previews it from Project files', async ({ app }) => { +const saveTextVersion = async ( + preview: Locator, + baseline: string, + nextContent: string, + fileName = FILE_NAME +): Promise => { + await preview.getByRole('button', { name: `Edit ${fileName}` }).click() + const editor = preview.getByRole('textbox', { name: `Edit ${fileName} source` }) + await expect(editor).toHaveValue(baseline) + const saveButton = preview.getByRole('button', { name: 'Save changes' }) + await expect(preview.getByRole('button', { name: 'Cancel', exact: true })).toBeVisible() + await expect(saveButton).toHaveText('Save') + await expect(saveButton.locator('svg')).toHaveCount(0) + await expect(preview.getByRole('button', { name: `Download ${fileName}` })).toHaveCount(0) + await expect(preview.getByRole('button', { name: `Close preview of ${fileName}` })).toHaveCount(0) + await editor.fill(nextContent) + await saveButton.click() + await expect(editor).toBeHidden() + await expect(preview.getByRole('button', { name: `Download ${fileName}` })).toBeVisible() + await expect(preview.getByRole('button', { name: `Close preview of ${fileName}` })).toBeVisible() +} + +const visibleChangeTextContents = async (changes: Locator): Promise => + changes.evaluateAll((elements) => + elements.map((element) => { + const copy = element.cloneNode(true) as HTMLElement + copy.querySelectorAll('.sr-only').forEach((label) => label.remove()) + return copy.textContent ?? '' + }) + ) + +const reconstructedDiffText = async ( + container: Locator +): Promise<{ before: string; after: string }> => + container.evaluate((element) => { + const textWithout = (selector: string): string => { + const copy = element.cloneNode(true) as HTMLElement + copy.querySelectorAll('.sr-only').forEach((label) => label.remove()) + copy.querySelectorAll(selector).forEach((change) => change.remove()) + return copy.textContent ?? '' + } + return { + before: textWithout('ins[data-managed-diff="added"]'), + after: textWithout('del[data-managed-diff="removed"]') + } + }) + +test('edits uploaded Markdown versions and keeps diff navigation coherent', async ({ app }) => { let page = await app.completeOnboarding() page = await app.configureFakeAgent() await createProject(page) @@ -43,10 +95,141 @@ test('uploads an attachment and previews it from Project files', async ({ app }) await expect(preview).toBeVisible() await expect(preview.getByText('Fixture findings', { exact: true })).toBeVisible() await expect(preview.getByText('Deterministic preview content.', { exact: true })).toBeVisible() + + // Three immutable versions let this journey prove that an active diff follows version changes. + const versionNavigation = preview.getByTestId('managed-preview-version-navigation') + await saveTextVersion(preview, FILE_CONTENT, VERSION_TWO_CONTENT) + await expect(versionNavigation.getByText('v2', { exact: true })).toBeVisible() + await expect(preview.getByText('First edited version.', { exact: true })).toBeVisible() + + await saveTextVersion(preview, VERSION_TWO_CONTENT, VERSION_THREE_CONTENT) + await expect(versionNavigation.getByText('v3', { exact: true })).toBeVisible() + await expect(preview.getByText('Second edited version.', { exact: true })).toBeVisible() + + await preview + .getByRole('button', { name: `Compare ${FILE_NAME} with its source version` }) + .click() + const differences = preview.getByRole('region', { name: 'File version differences' }) + await expect(differences.getByRole('heading', { name: 'Fixture findings' })).toHaveCount(0) + const rawHeading = differences.locator('[data-diff-kind="mixed"] pre').filter({ + hasText: 'Fixture findings' + }) + const rawHeadingAdded = rawHeading.locator('ins[data-managed-diff="added"]') + await expect(rawHeading).toBeVisible() + expect(await visibleChangeTextContents(rawHeadingAdded)).toEqual(['#']) + expect( + await rawHeading.evaluate((element) => { + const copy = element.cloneNode(true) as HTMLElement + copy.querySelectorAll('.sr-only').forEach((label) => label.remove()) + return copy.textContent + }) + ).toBe('## Fixture findings') + expect( + await rawHeading.evaluate((element) => getComputedStyle(element.parentElement!).backgroundColor) + ).toBe('rgba(0, 0, 0, 0)') + const removedChange = differences.locator('p del[data-managed-diff="removed"]') + const addedChange = differences.locator('p ins[data-managed-diff="added"]') + await expect(removedChange).toBeVisible() + await expect(addedChange).toBeVisible() + expect(await visibleChangeTextContents(removedChange)).toEqual(['First']) + expect(await visibleChangeTextContents(addedChange)).toEqual(['Second']) + await expect(removedChange.locator('.sr-only')).toHaveText('Removed:') + await expect(addedChange.locator('.sr-only')).toHaveText('Added:') + expect( + await removedChange.evaluate( + (element) => + getComputedStyle(element.closest('[data-diff-kind]')!).backgroundColor + ) + ).toBe('rgba(0, 0, 0, 0)') + const diffColors = await differences.evaluate((region) => { + const added = getComputedStyle(region.querySelector('p ins')!) + const removed = getComputedStyle(region.querySelector('p del')!) + return { + addedBackground: added.backgroundColor, + removedBackground: removed.backgroundColor, + addedDecoration: added.textDecorationLine, + removedDecoration: removed.textDecorationLine + } + }) + expect(diffColors.addedBackground).not.toBe(diffColors.removedBackground) + expect(diffColors.addedBackground).not.toBe('rgba(0, 0, 0, 0)') + expect(diffColors.removedBackground).not.toBe('rgba(0, 0, 0, 0)') + expect(diffColors.addedDecoration).not.toContain('underline') + expect(diffColors.removedDecoration).toContain('line-through') + await versionNavigation.getByRole('button', { name: 'Previous file version' }).click() + await expect(versionNavigation.getByText('v2', { exact: true })).toBeVisible() + await expect(preview.getByRole('button', { name: `Stop comparing ${FILE_NAME}` })).toBeVisible() + await expect(differences.getByRole('heading', { name: 'Fixture findings' })).toBeVisible() + const versionTwoParagraph = differences.locator( + 'p:has(del[data-managed-diff="removed"]):has(ins[data-managed-diff="added"])' + ) + await expect(versionTwoParagraph).toBeVisible() + expect(await reconstructedDiffText(versionTwoParagraph)).toEqual({ + before: 'Deterministic preview content.', + after: 'First edited version.' + }) + + await versionNavigation.getByRole('button', { name: 'Previous file version' }).click() + await expect(versionNavigation.getByText('v1', { exact: true })).toBeVisible() + await expect(preview.getByRole('button', { name: `Stop comparing ${FILE_NAME}` })).toBeVisible() + await expect(differences).toBeHidden() + await expect(preview.getByText('Deterministic preview content.', { exact: true })).toBeVisible() + await expect(preview.locator('[data-diff-kind]')).toHaveCount(0) + + await versionNavigation.getByRole('button', { name: 'Next file version' }).click() + await expect(versionNavigation.getByText('v2', { exact: true })).toBeVisible() + await expect(preview.getByRole('button', { name: `Stop comparing ${FILE_NAME}` })).toBeVisible() + await expect(differences).toBeVisible() + await expect(versionTwoParagraph).toBeVisible() + expect(await reconstructedDiffText(versionTwoParagraph)).toEqual({ + before: 'Deterministic preview content.', + after: 'First edited version.' + }) + await preview.getByRole('button', { name: `Close preview of ${FILE_NAME}` }).click() await expect(preview).toBeHidden() }) +test('shows structured text replacements with character-level highlights', async ({ app }) => { + let page = await app.completeOnboarding() + page = await app.configureFakeAgent() + await createProject(page) + + await page.locator('input[type="file"][multiple]').setInputFiles({ + name: SCRIPT_NAME, + mimeType: 'text/x-shellscript', + buffer: Buffer.from(SCRIPT_CONTENT) + }) + await page.getByRole('textbox', { name: 'Ask anything' }).fill('Use the attached script.') + await page.getByRole('button', { name: 'Send message' }).click() + await expect(page.getByText('Deterministic reply:', { exact: false })).toBeVisible() + + await page.getByRole('button', { name: 'Files', exact: true }).click() + await page.getByRole('button', { name: `Preview uploaded file ${SCRIPT_NAME}` }).click() + const preview = page.getByRole('dialog', { name: `Preview ${SCRIPT_NAME}` }) + await saveTextVersion(preview, SCRIPT_CONTENT, SCRIPT_VERSION_TWO_CONTENT, SCRIPT_NAME) + + await preview + .getByRole('button', { name: `Compare ${SCRIPT_NAME} with its source version` }) + .click() + const differences = preview.getByRole('region', { name: 'File version differences' }) + const mixedLine = differences.locator('[data-diff-kind="mixed"]') + const removedText = mixedLine.locator('del[data-diff-segment="removed"]') + const addedText = mixedLine.locator('ins[data-diff-segment="added"]') + await expect(removedText.locator('[data-managed-diff-content]')).toHaveText('old') + await expect(addedText.locator('[data-managed-diff-content]')).toHaveText('new') + await expect(removedText.locator('.sr-only')).toHaveText('Removed:') + await expect(addedText.locator('.sr-only')).toHaveText('Added:') + await expect(mixedLine.locator('pre > span')).toHaveText(['echo "', '"']) + await expect(mixedLine).toHaveCSS('background-color', 'rgba(0, 0, 0, 0)') + await expect(differences.locator('[data-diff-kind="removed"]')).toHaveCount(0) + await expect(differences.locator('[data-diff-kind="added"]')).toHaveCount(0) + await expect(differences.getByTestId('source-line-number')).toHaveCount(0) + await expect( + differences.locator('[aria-label="Added line"], [aria-label="Removed line"]') + ).toHaveCount(0) +}) + test('loads managed image previews from Project files', async ({ app }) => { let page = await app.completeOnboarding() page = await app.configureFakeAgent() diff --git a/package-lock.json b/package-lock.json index df965f16b..85b444013 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,12 +23,14 @@ "@radix-ui/react-focus-scope": "^1.1.11", "@rc-component/qrcode": "^2.0.0", "ajv": "^8.20.0", + "diff": "9.0.0", "electron-updater": "^6.3.9", "fflate": "0.8.3", "js-tiktoken": "^1.0.21", "js-yaml": "^4.1.0", "marked": "^17.0.6", "openchemlib": "^9.24.0", + "parse5": "^7.3.0", "pdfjs-dist": "5.4.624", "sharp": "0.35.3", "tar": "^7.5.22", @@ -4359,16 +4361,6 @@ "react-dom": "^18.3.1 || ^19.0.0" } }, - "node_modules/@pierre/diffs/node_modules/diff": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-9.0.0.tgz", - "integrity": "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw==", - "license": "BSD-3-Clause", - "peer": true, - "engines": { - "node": ">=0.3.1" - } - }, "node_modules/@pierre/theme": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@pierre/theme/-/theme-1.1.0.tgz", @@ -11045,8 +11037,9 @@ } }, "node_modules/diff": { - "version": "8.0.4", - "dev": true, + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-9.0.0.tgz", + "integrity": "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw==", "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" @@ -17595,6 +17588,8 @@ }, "node_modules/parse5": { "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", "license": "MIT", "dependencies": { "entities": "^6.0.0" @@ -20190,6 +20185,16 @@ "node": ">=20" } }, + "node_modules/shadcn/node_modules/diff": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/diff/-/diff-8.0.4.tgz", + "integrity": "sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/shadcn/node_modules/fs-extra": { "version": "11.3.6", "dev": true, diff --git a/package.json b/package.json index 6c35bf2b8..542107ef2 100644 --- a/package.json +++ b/package.json @@ -68,12 +68,14 @@ "@radix-ui/react-focus-scope": "^1.1.11", "@rc-component/qrcode": "^2.0.0", "ajv": "^8.20.0", + "diff": "9.0.0", "electron-updater": "^6.3.9", "fflate": "0.8.3", "js-tiktoken": "^1.0.21", "js-yaml": "^4.1.0", "marked": "^17.0.6", "openchemlib": "^9.24.0", + "parse5": "^7.3.0", "pdfjs-dist": "5.4.624", "sharp": "0.35.3", "tar": "^7.5.22", diff --git a/packages/safe-file-publisher-native/binding.gyp b/packages/safe-file-publisher-native/binding.gyp index 1b3a32d96..0904a4f1b 100644 --- a/packages/safe-file-publisher-native/binding.gyp +++ b/packages/safe-file-publisher-native/binding.gyp @@ -14,6 +14,9 @@ }], ["OS!='win'", { "cflags_cc": ["-std=c++17"] + }], + ["OS=='linux'", { + "libraries": ["-lcrypto"] }] ] } diff --git a/packages/safe-file-publisher-native/index.d.ts b/packages/safe-file-publisher-native/index.d.ts index f9532f8b3..b124e2303 100644 --- a/packages/safe-file-publisher-native/index.d.ts +++ b/packages/safe-file-publisher-native/index.d.ts @@ -1,3 +1,5 @@ +export const supportsAnchoredWrites: boolean + export function publishNoReplace( rootPath: string, relativeParentPath: string, @@ -5,6 +7,58 @@ export function publishNoReplace( destinationName: string ): void +export function writeAndPublishNoReplace( + rootPath: string, + relativeParentPath: string, + temporaryName: string, + destinationName: string, + bytes: Buffer +): void + +export function readFile(rootPath: string, relativeParentPath: string, name: string): Buffer + +export function readFileBounded( + rootPath: string, + relativeParentPath: string, + name: string, + maxBytes: number +): Buffer + +export function publishVerifiedNoReplace( + rootPath: string, + relativeParentPath: string, + temporaryName: string, + destinationName: string, + expectedBytes: Buffer +): void + +export function verifyFile( + rootPath: string, + relativeParentPath: string, + name: string, + expectedSizeBytes: number, + expectedSha256: string +): boolean + +export function statFile( + rootPath: string, + relativeParentPath: string, + name: string +): { sizeBytes: number } + +export function removeFile(rootPath: string, relativeParentPath: string, name: string): boolean + +export type AnchoredDirectoryEntry = { + name: string + isFile: boolean + mtimeMs: number +} + +export function listDirectory( + rootPath: string, + relativeParentPath: string +): AnchoredDirectoryEntry[] + export type StoragePathCapabilities = { isRemote: boolean supportsHardLinks: boolean diff --git a/packages/safe-file-publisher-native/src/safe_file_publisher_native.cc b/packages/safe-file-publisher-native/src/safe_file_publisher_native.cc index f08018b11..de063759b 100644 --- a/packages/safe-file-publisher-native/src/safe_file_publisher_native.cc +++ b/packages/safe-file-publisher-native/src/safe_file_publisher_native.cc @@ -1,7 +1,11 @@ #include +#include #include +#include #include +#include +#include #include #include #include @@ -11,20 +15,28 @@ #define WIN32_LEAN_AND_MEAN #include #else +#include #include #include #include #ifdef __linux__ #include +#ifndef AT_EMPTY_PATH +#define AT_EMPTY_PATH 0x1000 +#endif #ifndef RENAME_NOREPLACE #define RENAME_NOREPLACE (1 << 0) #endif #endif #ifdef __APPLE__ +#include #include +#include #ifndef RENAME_EXCL #define RENAME_EXCL 0x00000004 #endif +#elif defined(__linux__) +#include #endif #endif @@ -64,6 +76,14 @@ bool IsSimpleName(const std::string& value) { return true; } +bool IsSha256Hex(const std::string& value) { + if (value.size() != 64) return false; + return std::all_of(value.begin(), value.end(), [](unsigned char character) { + return (character >= '0' && character <= '9') || + (character >= 'a' && character <= 'f'); + }); +} + bool SplitRelativePath(const std::string& value, std::vector* components) { if (value.empty()) return true; size_t start = 0; @@ -83,6 +103,26 @@ bool SplitRelativePath(const std::string& value, std::vector* compo return false; } +bool ReadPathArguments( + napi_env env, + napi_callback_info info, + size_t expected_argc, + napi_value* argv, + std::string* root, + std::vector* parent_components, + std::string* name) { + size_t argc = expected_argc; + if (napi_get_cb_info(env, info, &argc, argv, nullptr, nullptr) != napi_ok || + argc != expected_argc) { + return false; + } + std::string relative_parent; + return ReadString(env, argv[0], root) && !root->empty() && + ReadString(env, argv[1], &relative_parent) && + SplitRelativePath(relative_parent, parent_components) && ReadString(env, argv[2], name) && + IsSimpleName(*name); +} + #ifdef _WIN32 std::wstring Utf8ToWide(const std::string& value) { @@ -402,12 +442,826 @@ const char* PosixErrorCode(int error) { case EPERM: return "EPERM"; case ELOOP: + case ENOTDIR: return "ELOOP"; default: return "EIO"; } } +void CloseAnchoredDirectories(int root_fd, int parent_fd) { + if (parent_fd != root_fd) close(parent_fd); + close(root_fd); +} + +bool UnlinkNameIfIdentityMatches( + int parent_fd, + const std::string& name, + const struct stat& expected_info) { + struct stat current_info {}; + if (fstatat(parent_fd, name.c_str(), ¤t_info, AT_SYMLINK_NOFOLLOW) != 0) { + return errno == ENOENT; + } + if (!S_ISREG(current_info.st_mode) || current_info.st_dev != expected_info.st_dev || + current_info.st_ino != expected_info.st_ino) { + return false; + } + return unlinkat(parent_fd, name.c_str(), 0) == 0 || errno == ENOENT; +} + +bool NativeTestHooksEnabled() { + const char* enabled = std::getenv("OPEN_SCIENCE_NATIVE_TEST_HOOKS"); + const char* node_env = std::getenv("NODE_ENV"); + const char* vitest = std::getenv("VITEST"); + return enabled != nullptr && std::strcmp(enabled, "1") == 0 && node_env != nullptr && + std::strcmp(node_env, "test") == 0 && vitest != nullptr && + std::strcmp(vitest, "true") == 0; +} + +void ExitAfterDurableTempForTest() { + if (!NativeTestHooksEnabled()) return; + const char* test_exit = std::getenv("OPEN_SCIENCE_TEST_EXIT_AFTER_DURABLE_TEMP"); + if (test_exit != nullptr && std::strcmp(test_exit, "86") == 0) _exit(86); +} + +void PauseAfterVerifiedTempForTest() { + if (!NativeTestHooksEnabled()) return; + const char* marker = std::getenv("OPEN_SCIENCE_TEST_VERIFIED_TEMP_MARKER"); + const char* resume = std::getenv("OPEN_SCIENCE_TEST_VERIFIED_TEMP_RESUME"); + if (marker == nullptr || resume == nullptr || marker[0] == '\0' || resume[0] == '\0') return; + const int marker_fd = open(marker, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0600); + if (marker_fd < 0) _exit(87); + static constexpr char kVerified[] = "verified"; + if (write(marker_fd, kVerified, sizeof(kVerified) - 1) != + static_cast(sizeof(kVerified) - 1) || + fsync(marker_fd) != 0) { + close(marker_fd); + _exit(87); + } + close(marker_fd); + for (size_t attempt = 0; attempt < 10'000; attempt += 1) { + if (access(resume, F_OK) == 0) return; + usleep(1'000); + } + _exit(88); +} + +void PauseAfterBoundedReadSizeForTest() { + if (!NativeTestHooksEnabled()) return; + const char* marker = std::getenv("OPEN_SCIENCE_TEST_BOUNDED_READ_MARKER"); + const char* resume = std::getenv("OPEN_SCIENCE_TEST_BOUNDED_READ_RESUME"); + if (marker == nullptr || resume == nullptr || marker[0] == '\0' || resume[0] == '\0') return; + const int marker_fd = open(marker, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0600); + if (marker_fd < 0) _exit(89); + close(marker_fd); + for (size_t attempt = 0; attempt < 10'000; attempt += 1) { + if (access(resume, F_OK) == 0) return; + usleep(1'000); + } + _exit(90); +} + +bool OpenAnchoredParent( + const std::string& root, + const std::vector& parent_components, + bool create, + int* root_fd_out, + int* parent_fd_out, + int* error_out) { + const int root_fd = open(root.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (root_fd < 0) { + *error_out = errno; + return false; + } + + int parent_fd = root_fd; + for (const std::string& component : parent_components) { + int next_fd = + openat(parent_fd, component.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + bool created = false; + if (next_fd < 0 && errno == ENOENT && create) { + if (mkdirat(parent_fd, component.c_str(), 0700) != 0) { + *error_out = errno; + CloseAnchoredDirectories(root_fd, parent_fd); + return false; + } + created = true; + next_fd = + openat(parent_fd, component.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + } + if (next_fd < 0) { + *error_out = errno; + CloseAnchoredDirectories(root_fd, parent_fd); + return false; + } + if (created && (fsync(next_fd) != 0 || fsync(parent_fd) != 0)) { + *error_out = errno; + close(next_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return false; + } + if (parent_fd != root_fd) close(parent_fd); + parent_fd = next_fd; + } + *root_fd_out = root_fd; + *parent_fd_out = parent_fd; + return true; +} + +bool VerifyFdSha256( + int file_fd, + size_t expected_size, + const std::string& expected_sha256, + bool* matches, + int* error_out) { + *matches = false; + struct stat info {}; + if (fstat(file_fd, &info) != 0) { + *error_out = errno; + return false; + } + if (!S_ISREG(info.st_mode)) { + *error_out = ELOOP; + return false; + } + if (info.st_size < 0 || static_cast(info.st_size) != expected_size) return true; + if (lseek(file_fd, 0, SEEK_SET) < 0) { + *error_out = errno; + return false; + } + +#ifdef __APPLE__ + CC_SHA256_CTX context; + if (CC_SHA256_Init(&context) != 1) { + *error_out = EIO; + return false; + } +#elif defined(__linux__) + SHA256_CTX context; + if (SHA256_Init(&context) != 1) { + *error_out = EIO; + return false; + } +#endif + std::vector chunk(64 * 1024); + size_t total = 0; + while (true) { + const ssize_t count = read(file_fd, chunk.data(), chunk.size()); + if (count < 0 && errno == EINTR) continue; + if (count < 0) { + *error_out = errno; + return false; + } + if (count == 0) break; + const size_t byte_count = static_cast(count); + if (byte_count > expected_size - (std::min)(total, expected_size)) return true; +#ifdef __APPLE__ + if (CC_SHA256_Update(&context, chunk.data(), static_cast(byte_count)) != 1) { + *error_out = EIO; + return false; + } +#elif defined(__linux__) + if (SHA256_Update(&context, chunk.data(), byte_count) != 1) { + *error_out = EIO; + return false; + } +#endif + total += byte_count; + } + if (total != expected_size) return true; + +#ifdef __APPLE__ + unsigned char digest[CC_SHA256_DIGEST_LENGTH]; + if (CC_SHA256_Final(digest, &context) != 1) { + *error_out = EIO; + return false; + } +#elif defined(__linux__) + unsigned char digest[SHA256_DIGEST_LENGTH]; + if (SHA256_Final(digest, &context) != 1) { + *error_out = EIO; + return false; + } +#endif + static constexpr char kHex[] = "0123456789abcdef"; + std::string actual_sha256; + actual_sha256.resize(64); + for (size_t index = 0; index < 32; index += 1) { + actual_sha256[index * 2] = kHex[digest[index] >> 4]; + actual_sha256[index * 2 + 1] = kHex[digest[index] & 0x0f]; + } + *matches = actual_sha256 == expected_sha256; + return true; +} + +bool FileDescriptorMatchesBytes( + int file_fd, + const void* expected_bytes, + size_t expected_size, + int* error_out) { + struct stat info {}; + if (fstat(file_fd, &info) != 0) { + *error_out = errno; + return false; + } + if (!S_ISREG(info.st_mode) || info.st_size < 0 || + static_cast(info.st_size) != expected_size) { + *error_out = EIO; + return false; + } + if (lseek(file_fd, 0, SEEK_SET) < 0) { + *error_out = errno; + return false; + } + const auto* expected = static_cast(expected_bytes); + std::vector chunk(64 * 1024); + size_t offset = 0; + while (offset < expected_size) { + const size_t requested = (std::min)(chunk.size(), expected_size - offset); + const ssize_t count = read(file_fd, chunk.data(), requested); + if (count < 0 && errno == EINTR) continue; + if (count <= 0 || + std::memcmp(chunk.data(), expected + offset, static_cast(count)) != 0) { + *error_out = count < 0 ? errno : EIO; + return false; + } + offset += static_cast(count); + } + unsigned char growth_probe = 0; + ssize_t probe_count = 0; + do { + probe_count = read(file_fd, &growth_probe, 1); + } while (probe_count < 0 && errno == EINTR); + if (probe_count != 0) { + *error_out = probe_count < 0 ? errno : EIO; + return false; + } + return true; +} + +#ifdef __linux__ +int LinkOpenFileDescriptorNoReplace( + int file_fd, + int parent_fd, + const std::string& destination_name) { + // AT_EMPTY_PATH binds publication to the descriptor that was verified. Some kernels require a + // capability for this form, so the fallback resolves the same open descriptor through procfs. + int result = linkat(file_fd, "", parent_fd, destination_name.c_str(), AT_EMPTY_PATH); + if (result == 0) return 0; + const int direct_error = errno; + if (direct_error != EPERM && direct_error != EINVAL && direct_error != ENOENT && + direct_error != ENOSYS && direct_error != EOPNOTSUPP) { + errno = direct_error; + return -1; + } + const std::string source_fd_path = "/proc/self/fd/" + std::to_string(file_fd); + result = linkat( + AT_FDCWD, + source_fd_path.c_str(), + parent_fd, + destination_name.c_str(), + AT_SYMLINK_FOLLOW); + if (result != 0 && errno == ENOENT) errno = ENOTSUP; + return result; +} +#endif + +napi_value WriteAndPublishNoReplacePosix( + napi_env env, + const std::string& root, + const std::vector& parent_components, + const std::string& temporary_name, + const std::string& destination_name, + const void* bytes, + size_t byte_length) { + int root_fd = -1; + int parent_fd = -1; + int error = 0; + if (!OpenAnchoredParent(root, parent_components, true, &root_fd, &parent_fd, &error)) { + return ThrowError(env, "Could not create the anchored temporary parent.", PosixErrorCode(error)); + } + const int file_fd = openat( + parent_fd, + temporary_name.c_str(), + O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, + 0600); + if (file_fd < 0) { + error = errno; + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not exclusively create the temporary file.", PosixErrorCode(error)); + } + + const auto* cursor = static_cast(bytes); + size_t remaining = byte_length; + while (remaining > 0) { + const ssize_t written = write(file_fd, cursor, remaining); + if (written < 0) { + if (errno == EINTR) continue; + error = errno; + close(file_fd); + (void)unlinkat(parent_fd, temporary_name.c_str(), 0); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not write the temporary file.", PosixErrorCode(error)); + } + cursor += written; + remaining -= static_cast(written); + } + if (fsync(file_fd) != 0) { + error = errno; + close(file_fd); + (void)unlinkat(parent_fd, temporary_name.c_str(), 0); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not sync the temporary file.", PosixErrorCode(error)); + } + // Persist the temporary directory entry before publication so startup recovery can find the + // complete file even if the process exits between this barrier and the no-replace publish. + if (fsync(parent_fd) != 0) { + error = errno; + close(file_fd); + (void)unlinkat(parent_fd, temporary_name.c_str(), 0); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not sync the temporary file parent.", PosixErrorCode(error)); + } + ExitAfterDurableTempForTest(); + struct stat source_info {}; + if (fstat(file_fd, &source_info) != 0 || !S_ISREG(source_info.st_mode)) { + error = errno == 0 ? ELOOP : errno; + close(file_fd); + (void)unlinkat(parent_fd, temporary_name.c_str(), 0); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "The temporary file identity is invalid.", PosixErrorCode(error)); + } + PauseAfterVerifiedTempForTest(); + +#ifdef __linux__ + const int result = LinkOpenFileDescriptorNoReplace(file_fd, parent_fd, destination_name); + const int publish_error = result == 0 ? 0 : errno; +#elif defined(__APPLE__) + const int result = fclonefileat(file_fd, parent_fd, destination_name.c_str(), 0); + const int publish_error = result == 0 ? 0 : errno; +#else +#error Unsupported platform for anchored managed-file publication +#endif + if (result != 0) { + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Atomic no-replace publication failed.", PosixErrorCode(publish_error)); + } + + const int destination_fd = + openat(parent_fd, destination_name.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + struct stat destination_info {}; + bool destination_is_owned = + destination_fd >= 0 && fstat(destination_fd, &destination_info) == 0 && + S_ISREG(destination_info.st_mode); +#ifdef __linux__ + destination_is_owned = destination_is_owned && source_info.st_dev == destination_info.st_dev && + source_info.st_ino == destination_info.st_ino; +#elif defined(__APPLE__) + if (destination_is_owned) { + destination_is_owned = + FileDescriptorMatchesBytes(destination_fd, bytes, byte_length, &error); + } +#endif + if (!destination_is_owned) { + if (destination_fd >= 0) close(destination_fd); + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Published destination identity changed.", "ELOOP"); + } + if (fsync(destination_fd) != 0) { + error = errno; + close(destination_fd); + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not sync the published destination.", PosixErrorCode(error)); + } + if (fsync(parent_fd) != 0) { + error = errno; + close(destination_fd); + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not sync the published destination.", PosixErrorCode(error)); + } + close(destination_fd); + (void)UnlinkNameIfIdentityMatches(parent_fd, temporary_name, source_info); + if (fsync(parent_fd) != 0) { + error = errno; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not sync temporary file cleanup.", PosixErrorCode(error)); + } + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + napi_value undefined; + napi_get_undefined(env, &undefined); + return undefined; +} + +napi_value ReadFilePosix( + napi_env env, + const std::string& root, + const std::vector& parent_components, + const std::string& name, + size_t max_bytes = std::numeric_limits::max()) { + int root_fd = -1; + int parent_fd = -1; + int error = 0; + if (!OpenAnchoredParent(root, parent_components, false, &root_fd, &parent_fd, &error)) { + return ThrowError(env, "Could not open the anchored file parent.", PosixErrorCode(error)); + } + const int file_fd = openat(parent_fd, name.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (file_fd < 0) { + error = errno; + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not open the anchored file.", PosixErrorCode(error)); + } + struct stat info {}; + if (fstat(file_fd, &info) != 0 || !S_ISREG(info.st_mode) || info.st_size < 0) { + error = errno == 0 ? ELOOP : errno; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "The anchored file is not regular.", PosixErrorCode(error)); + } + if (static_cast(info.st_size) > std::numeric_limits::max()) { + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "The anchored file is too large.", "EIO"); + } + if (static_cast(info.st_size) > max_bytes) { + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "The anchored file exceeds the bounded read limit.", "EFBIG"); + } + PauseAfterBoundedReadSizeForTest(); + void* output = nullptr; + napi_value buffer; + const size_t size = static_cast(info.st_size); + if (napi_create_buffer(env, size, &output, &buffer) != napi_ok) { + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not allocate the anchored file buffer.", "EIO"); + } + auto* cursor = static_cast(output); + size_t remaining = size; + while (remaining > 0) { + const ssize_t count = read(file_fd, cursor, remaining); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) { + error = count == 0 ? EIO : errno; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not read the anchored file.", PosixErrorCode(error)); + } + cursor += count; + remaining -= static_cast(count); + } + unsigned char growth_probe = 0; + while (true) { + const ssize_t count = read(file_fd, &growth_probe, 1); + if (count < 0 && errno == EINTR) continue; + if (count > 0) { + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "The anchored file grew during the bounded read.", "EFBIG"); + } + if (count < 0) { + error = errno; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not complete the anchored bounded read.", PosixErrorCode(error)); + } + break; + } + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return buffer; +} + +napi_value PublishVerifiedNoReplacePosix( + napi_env env, + const std::string& root, + const std::vector& parent_components, + const std::string& temporary_name, + const std::string& destination_name, + const void* expected_bytes, + size_t expected_size) { + int root_fd = -1; + int parent_fd = -1; + int error = 0; + if (!OpenAnchoredParent(root, parent_components, false, &root_fd, &parent_fd, &error)) { + return ThrowError(env, "Could not open the anchored recovery parent.", PosixErrorCode(error)); + } + const int file_fd = openat(parent_fd, temporary_name.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (file_fd < 0) { + error = errno; + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not open the anchored recovery temp.", PosixErrorCode(error)); + } + struct stat info {}; + if (fstat(file_fd, &info) != 0 || !S_ISREG(info.st_mode) || info.st_size < 0 || + static_cast(info.st_size) != expected_size) { + error = errno == 0 ? EIO : errno; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "The anchored recovery temp size is invalid.", PosixErrorCode(error)); + } + const auto* expected = static_cast(expected_bytes); + std::vector chunk(64 * 1024); + size_t offset = 0; + while (offset < expected_size) { + const size_t requested = (std::min)(chunk.size(), expected_size - offset); + const ssize_t count = read(file_fd, chunk.data(), requested); + if (count < 0 && errno == EINTR) continue; + if (count <= 0 || std::memcmp(chunk.data(), expected + offset, static_cast(count)) != 0) { + error = count < 0 ? errno : EIO; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "The anchored recovery temp content is invalid.", PosixErrorCode(error)); + } + offset += static_cast(count); + } + unsigned char growth_probe = 0; + ssize_t probe_count = 0; + do { + probe_count = read(file_fd, &growth_probe, 1); + } while (probe_count < 0 && errno == EINTR); + if (probe_count != 0) { + error = probe_count < 0 ? errno : EIO; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "The anchored recovery temp content is invalid.", PosixErrorCode(error)); + } + if (fsync(file_fd) != 0) { + error = errno; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not sync the anchored recovery temp.", PosixErrorCode(error)); + } + PauseAfterVerifiedTempForTest(); + +#ifdef __linux__ + const int result = LinkOpenFileDescriptorNoReplace(file_fd, parent_fd, destination_name); + const int publish_error = result == 0 ? 0 : errno; +#elif defined(__APPLE__) + const int result = fclonefileat(file_fd, parent_fd, destination_name.c_str(), 0); + const int publish_error = result == 0 ? 0 : errno; +#else +#error Unsupported platform for anchored managed-file recovery publication +#endif + if (result != 0) { + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Atomic no-replace recovery publication failed.", PosixErrorCode(publish_error)); + } + + const int destination_fd = + openat(parent_fd, destination_name.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + struct stat destination_info {}; + bool destination_is_owned = + destination_fd >= 0 && fstat(destination_fd, &destination_info) == 0 && + S_ISREG(destination_info.st_mode); +#ifdef __linux__ + destination_is_owned = destination_is_owned && info.st_dev == destination_info.st_dev && + info.st_ino == destination_info.st_ino; +#elif defined(__APPLE__) + if (destination_is_owned) { + destination_is_owned = + FileDescriptorMatchesBytes(destination_fd, expected_bytes, expected_size, &error); + } +#endif + if (!destination_is_owned) { + if (destination_fd >= 0) close(destination_fd); + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Published recovery destination identity changed.", "ELOOP"); + } + if (fsync(destination_fd) != 0) { + error = errno; + close(destination_fd); + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not sync the recovered destination.", PosixErrorCode(error)); + } + if (fsync(parent_fd) != 0) { + error = errno; + close(destination_fd); + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not sync the recovered destination.", PosixErrorCode(error)); + } + close(destination_fd); + (void)UnlinkNameIfIdentityMatches(parent_fd, temporary_name, info); + if (fsync(parent_fd) != 0) { + error = errno; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not sync recovery temp cleanup.", PosixErrorCode(error)); + } + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + napi_value undefined; + napi_get_undefined(env, &undefined); + return undefined; +} + +napi_value VerifyFilePosix( + napi_env env, + const std::string& root, + const std::vector& parent_components, + const std::string& name, + size_t expected_size, + const std::string& expected_sha256) { + int root_fd = -1; + int parent_fd = -1; + int error = 0; + if (!OpenAnchoredParent(root, parent_components, false, &root_fd, &parent_fd, &error)) { + return ThrowError(env, "Could not open the anchored verification parent.", PosixErrorCode(error)); + } + const int file_fd = openat(parent_fd, name.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (file_fd < 0) { + error = errno; + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not open the anchored verification file.", PosixErrorCode(error)); + } + bool matches = false; + const bool verified = VerifyFdSha256(file_fd, expected_size, expected_sha256, &matches, &error); + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + if (!verified) { + return ThrowError(env, "Could not verify the anchored file.", PosixErrorCode(error)); + } + napi_value result; + napi_get_boolean(env, matches, &result); + return result; +} + +napi_value StatFilePosix( + napi_env env, + const std::string& root, + const std::vector& parent_components, + const std::string& name) { + int root_fd = -1; + int parent_fd = -1; + int error = 0; + if (!OpenAnchoredParent(root, parent_components, false, &root_fd, &parent_fd, &error)) { + return ThrowError(env, "Could not open the anchored file parent.", PosixErrorCode(error)); + } + const int file_fd = openat(parent_fd, name.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (file_fd < 0) { + error = errno; + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not open the anchored file.", PosixErrorCode(error)); + } + struct stat info {}; + if (fstat(file_fd, &info) != 0 || !S_ISREG(info.st_mode) || info.st_size < 0) { + error = errno == 0 ? ELOOP : errno; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "The anchored file is not regular.", PosixErrorCode(error)); + } + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + napi_value result; + napi_create_object(env, &result); + napi_value size; + napi_create_double(env, static_cast(info.st_size), &size); + napi_set_named_property(env, result, "sizeBytes", size); + return result; +} + +napi_value RemoveFilePosix( + napi_env env, + const std::string& root, + const std::vector& parent_components, + const std::string& name) { + int root_fd = -1; + int parent_fd = -1; + int error = 0; + if (!OpenAnchoredParent(root, parent_components, false, &root_fd, &parent_fd, &error)) { + if (error == ENOENT) { + napi_value removed; + napi_get_boolean(env, false, &removed); + return removed; + } + return ThrowError(env, "Could not open the anchored cleanup parent.", PosixErrorCode(error)); + } + const int file_fd = openat(parent_fd, name.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (file_fd < 0) { + error = errno; + CloseAnchoredDirectories(root_fd, parent_fd); + if (error == ENOENT) { + napi_value removed; + napi_get_boolean(env, false, &removed); + return removed; + } + return ThrowError(env, "Could not open the anchored cleanup file.", PosixErrorCode(error)); + } + struct stat info {}; + if (fstat(file_fd, &info) != 0 || !S_ISREG(info.st_mode)) { + error = errno == 0 ? ELOOP : errno; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "The anchored cleanup target is not regular.", PosixErrorCode(error)); + } + struct stat current_info {}; + if (fstatat(parent_fd, name.c_str(), ¤t_info, AT_SYMLINK_NOFOLLOW) != 0) { + error = errno; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not verify the anchored cleanup target.", PosixErrorCode(error)); + } + if (!S_ISREG(current_info.st_mode) || current_info.st_dev != info.st_dev || + current_info.st_ino != info.st_ino) { + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "The anchored cleanup target identity changed.", "EAGAIN"); + } + if (unlinkat(parent_fd, name.c_str(), 0) != 0) { + error = errno; + close(file_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not remove the anchored file.", PosixErrorCode(error)); + } + close(file_fd); + if (fsync(parent_fd) != 0) { + error = errno; + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not sync anchored cleanup.", PosixErrorCode(error)); + } + CloseAnchoredDirectories(root_fd, parent_fd); + napi_value removed; + napi_get_boolean(env, true, &removed); + return removed; +} + +napi_value ListDirectoryPosix( + napi_env env, + const std::string& root, + const std::vector& parent_components) { + int root_fd = -1; + int parent_fd = -1; + int error = 0; + if (!OpenAnchoredParent(root, parent_components, false, &root_fd, &parent_fd, &error)) { + return ThrowError(env, "Could not open the anchored directory.", PosixErrorCode(error)); + } + const int directory_fd = dup(parent_fd); + if (directory_fd < 0) { + error = errno; + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not duplicate the anchored directory.", PosixErrorCode(error)); + } + DIR* directory = fdopendir(directory_fd); + if (directory == nullptr) { + error = errno; + close(directory_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not enumerate the anchored directory.", PosixErrorCode(error)); + } + + napi_value entries; + napi_create_array(env, &entries); + uint32_t index = 0; + errno = 0; + while (dirent* entry = readdir(directory)) { + const std::string name(entry->d_name); + if (name == "." || name == "..") continue; + struct stat info {}; + if (fstatat(parent_fd, name.c_str(), &info, AT_SYMLINK_NOFOLLOW) != 0) { + error = errno; + closedir(directory); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Could not inspect an anchored directory entry.", PosixErrorCode(error)); + } + napi_value item; + napi_create_object(env, &item); + napi_value name_value; + napi_create_string_utf8(env, name.c_str(), name.size(), &name_value); + napi_set_named_property(env, item, "name", name_value); + napi_value is_file; + napi_get_boolean(env, S_ISREG(info.st_mode), &is_file); + napi_set_named_property(env, item, "isFile", is_file); +#ifdef __APPLE__ + const double mtime_ms = static_cast(info.st_mtimespec.tv_sec) * 1000.0 + + static_cast(info.st_mtimespec.tv_nsec) / 1000000.0; +#else + const double mtime_ms = static_cast(info.st_mtim.tv_sec) * 1000.0 + + static_cast(info.st_mtim.tv_nsec) / 1000000.0; +#endif + napi_value mtime_value; + napi_create_double(env, mtime_ms, &mtime_value); + napi_set_named_property(env, item, "mtimeMs", mtime_value); + napi_set_element(env, entries, index++, item); + } + error = errno; + closedir(directory); + CloseAnchoredDirectories(root_fd, parent_fd); + if (error != 0) { + return ThrowError(env, "Could not enumerate the anchored directory.", PosixErrorCode(error)); + } + return entries; +} + napi_value PublishPosix( napi_env env, const std::string& root, @@ -481,13 +1335,21 @@ napi_value PublishPosix( #else #error Unsupported platform for atomic no-replace publication #endif - close(source_fd); - if (parent_fd != root_fd) close(parent_fd); - close(root_fd); if (result != 0) { + close(source_fd); + CloseAnchoredDirectories(root_fd, parent_fd); return ThrowError(env, "Atomic no-replace publication failed.", PosixErrorCode(rename_error)); } + if (fsync(parent_fd) != 0) { + const int sync_error = errno; + close(source_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + return ThrowError(env, "Atomic publication directory sync failed.", PosixErrorCode(sync_error)); + } + close(source_fd); + CloseAnchoredDirectories(root_fd, parent_fd); + napi_value undefined; napi_get_undefined(env, &undefined); return undefined; @@ -576,7 +1438,189 @@ napi_value PublishNoReplace(napi_env env, napi_callback_info info) { #endif } +napi_value WriteAndPublishNoReplace(napi_env env, napi_callback_info info) { + napi_value argv[5]; + std::string root; + std::string temporary_name; + std::string destination_name; + std::vector parent_components; + if (!ReadPathArguments(env, info, 5, argv, &root, &parent_components, &temporary_name) || + !ReadString(env, argv[3], &destination_name) || !IsSimpleName(destination_name)) { + return ThrowError( + env, + "writeAndPublishNoReplace requires a safe root, parent, temporary name, destination, and Buffer.", + "EINVAL"); + } + bool is_buffer = false; + void* bytes = nullptr; + size_t byte_length = 0; + if (napi_is_buffer(env, argv[4], &is_buffer) != napi_ok || !is_buffer || + napi_get_buffer_info(env, argv[4], &bytes, &byte_length) != napi_ok) { + return ThrowError(env, "writeAndPublishNoReplace bytes must be a Buffer.", "EINVAL"); + } +#ifdef _WIN32 + return ThrowError(env, "Anchored write publication is unavailable on this platform.", + "ENOTSUP"); +#else + return WriteAndPublishNoReplacePosix( + env, root, parent_components, temporary_name, destination_name, bytes, byte_length); +#endif +} + +napi_value ReadAnchoredFile(napi_env env, napi_callback_info info) { + napi_value argv[3]; + std::string root; + std::string name; + std::vector parent_components; + if (!ReadPathArguments(env, info, 3, argv, &root, &parent_components, &name)) { + return ThrowError(env, "readFile requires a safe root, parent, and name.", "EINVAL"); + } +#ifdef _WIN32 + return ThrowError(env, "Anchored file reading is unavailable on this platform.", "ENOTSUP"); +#else + return ReadFilePosix(env, root, parent_components, name); +#endif +} + +napi_value ReadAnchoredFileBounded(napi_env env, napi_callback_info info) { + napi_value argv[4]; + std::string root; + std::string name; + std::vector parent_components; + if (!ReadPathArguments(env, info, 4, argv, &root, &parent_components, &name)) { + return ThrowError(env, "readFileBounded requires a safe root, parent, name, and maxBytes.", "EINVAL"); + } + double max_bytes = 0; + if (napi_get_value_double(env, argv[3], &max_bytes) != napi_ok || !std::isfinite(max_bytes) || + std::floor(max_bytes) != max_bytes || max_bytes < 0 || + max_bytes > 9007199254740991.0 || + max_bytes > static_cast(std::numeric_limits::max())) { + return ThrowError(env, "readFileBounded maxBytes is invalid.", "EINVAL"); + } +#ifdef _WIN32 + return ThrowError(env, "Anchored bounded reading is unavailable on this platform.", "ENOTSUP"); +#else + return ReadFilePosix(env, root, parent_components, name, static_cast(max_bytes)); +#endif +} + +napi_value PublishVerifiedNoReplace(napi_env env, napi_callback_info info) { + napi_value argv[5]; + std::string root; + std::string temporary_name; + std::string destination_name; + std::vector parent_components; + if (!ReadPathArguments(env, info, 5, argv, &root, &parent_components, &temporary_name) || + !ReadString(env, argv[3], &destination_name) || !IsSimpleName(destination_name)) { + return ThrowError(env, "publishVerifiedNoReplace requires safe anchored names and expected bytes.", "EINVAL"); + } + bool is_buffer = false; + void* bytes = nullptr; + size_t byte_length = 0; + if (napi_is_buffer(env, argv[4], &is_buffer) != napi_ok || !is_buffer || + napi_get_buffer_info(env, argv[4], &bytes, &byte_length) != napi_ok) { + return ThrowError(env, "publishVerifiedNoReplace expected bytes must be a Buffer.", "EINVAL"); + } +#ifdef _WIN32 + return ThrowError(env, "Anchored recovery publication is unavailable on this platform.", "ENOTSUP"); +#else + return PublishVerifiedNoReplacePosix( + env, root, parent_components, temporary_name, destination_name, bytes, byte_length); +#endif +} + +napi_value VerifyAnchoredFile(napi_env env, napi_callback_info info) { + napi_value argv[5]; + std::string root; + std::string name; + std::string expected_sha256; + std::vector parent_components; + if (!ReadPathArguments(env, info, 5, argv, &root, &parent_components, &name) || + !ReadString(env, argv[4], &expected_sha256) || !IsSha256Hex(expected_sha256)) { + return ThrowError( + env, "verifyFile requires a safe root, parent, name, size, and lowercase SHA-256.", "EINVAL"); + } + double expected_size = 0; + if (napi_get_value_double(env, argv[3], &expected_size) != napi_ok || + !std::isfinite(expected_size) || std::floor(expected_size) != expected_size || + expected_size < 0 || expected_size > 9007199254740991.0 || + expected_size > static_cast(std::numeric_limits::max())) { + return ThrowError(env, "verifyFile expected size is invalid.", "EINVAL"); + } +#ifdef _WIN32 + return ThrowError(env, "Anchored streaming verification is unavailable on this platform.", + "ENOTSUP"); +#else + return VerifyFilePosix( + env, + root, + parent_components, + name, + static_cast(expected_size), + expected_sha256); +#endif +} + +napi_value StatAnchoredFile(napi_env env, napi_callback_info info) { + napi_value argv[3]; + std::string root; + std::string name; + std::vector parent_components; + if (!ReadPathArguments(env, info, 3, argv, &root, &parent_components, &name)) { + return ThrowError(env, "statFile requires a safe root, parent, and name.", "EINVAL"); + } +#ifdef _WIN32 + return ThrowError(env, "Anchored file metadata is unavailable on this platform.", "ENOTSUP"); +#else + return StatFilePosix(env, root, parent_components, name); +#endif +} + +napi_value RemoveAnchoredFile(napi_env env, napi_callback_info info) { + napi_value argv[3]; + std::string root; + std::string name; + std::vector parent_components; + if (!ReadPathArguments(env, info, 3, argv, &root, &parent_components, &name)) { + return ThrowError(env, "removeFile requires a safe root, parent, and name.", "EINVAL"); + } +#ifdef _WIN32 + return ThrowError(env, "Anchored file cleanup is unavailable on this platform.", "ENOTSUP"); +#else + return RemoveFilePosix(env, root, parent_components, name); +#endif +} + +napi_value ListAnchoredDirectory(napi_env env, napi_callback_info info) { + size_t argc = 2; + napi_value argv[2]; + if (napi_get_cb_info(env, info, &argc, argv, nullptr, nullptr) != napi_ok || argc != 2) { + return ThrowError(env, "listDirectory requires a safe root and parent.", "EINVAL"); + } + std::string root; + std::string relative_parent; + std::vector parent_components; + if (!ReadString(env, argv[0], &root) || root.empty() || + !ReadString(env, argv[1], &relative_parent) || + !SplitRelativePath(relative_parent, &parent_components)) { + return ThrowError(env, "listDirectory requires a safe root and parent.", "EINVAL"); + } +#ifdef _WIN32 + return ThrowError(env, "Anchored directory enumeration is unavailable on this platform.", + "ENOTSUP"); +#else + return ListDirectoryPosix(env, root, parent_components); +#endif +} + napi_value Init(napi_env env, napi_value exports) { + napi_value supports_anchored_writes; +#ifdef _WIN32 + napi_get_boolean(env, false, &supports_anchored_writes); +#else + napi_get_boolean(env, true, &supports_anchored_writes); +#endif + napi_set_named_property(env, exports, "supportsAnchoredWrites", supports_anchored_writes); napi_value publish; napi_create_function( env, "publishNoReplace", NAPI_AUTO_LENGTH, PublishNoReplace, nullptr, &publish); @@ -585,6 +1629,38 @@ napi_value Init(napi_env env, napi_value exports) { napi_create_function( env, "inspectPath", NAPI_AUTO_LENGTH, InspectPath, nullptr, &inspect_path); napi_set_named_property(env, exports, "inspectPath", inspect_path); + napi_value write_and_publish; + napi_create_function(env, "writeAndPublishNoReplace", NAPI_AUTO_LENGTH, + WriteAndPublishNoReplace, nullptr, &write_and_publish); + napi_set_named_property(env, exports, "writeAndPublishNoReplace", write_and_publish); + napi_value read_file; + napi_create_function( + env, "readFile", NAPI_AUTO_LENGTH, ReadAnchoredFile, nullptr, &read_file); + napi_set_named_property(env, exports, "readFile", read_file); + napi_value read_file_bounded; + napi_create_function(env, "readFileBounded", NAPI_AUTO_LENGTH, + ReadAnchoredFileBounded, nullptr, &read_file_bounded); + napi_set_named_property(env, exports, "readFileBounded", read_file_bounded); + napi_value publish_verified; + napi_create_function(env, "publishVerifiedNoReplace", NAPI_AUTO_LENGTH, + PublishVerifiedNoReplace, nullptr, &publish_verified); + napi_set_named_property(env, exports, "publishVerifiedNoReplace", publish_verified); + napi_value verify_file; + napi_create_function( + env, "verifyFile", NAPI_AUTO_LENGTH, VerifyAnchoredFile, nullptr, &verify_file); + napi_set_named_property(env, exports, "verifyFile", verify_file); + napi_value stat_file; + napi_create_function( + env, "statFile", NAPI_AUTO_LENGTH, StatAnchoredFile, nullptr, &stat_file); + napi_set_named_property(env, exports, "statFile", stat_file); + napi_value remove_file; + napi_create_function( + env, "removeFile", NAPI_AUTO_LENGTH, RemoveAnchoredFile, nullptr, &remove_file); + napi_set_named_property(env, exports, "removeFile", remove_file); + napi_value list_directory; + napi_create_function( + env, "listDirectory", NAPI_AUTO_LENGTH, ListAnchoredDirectory, nullptr, &list_directory); + napi_set_named_property(env, exports, "listDirectory", list_directory); return exports; } diff --git a/prisma/schema.prisma b/prisma/schema.prisma index b6ae25a99..ddcbb148c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -226,12 +226,15 @@ model ArtifactLineage { sessionId String normalizedFilename String filename String + currentVersionId String? @unique createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - versions ArtifactVersion[] + versions ArtifactVersion[] @relation("ArtifactLineageVersions") + currentVersion ArtifactVersion? @relation("ArtifactLineageCurrentVersion", fields: [id, currentVersionId], references: [artifactId, id], onDelete: Restrict) originSession FileOriginSession @relation(fields: [projectId, sessionId], references: [projectId, sessionId], onDelete: Restrict) @@unique([projectId, sessionId, normalizedFilename]) + @@unique([id, currentVersionId]) @@index([projectId, sessionId]) } @@ -241,12 +244,15 @@ model UploadFile { sessionId String filename String originalFilename String + currentVersionId String? @unique createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - versions UploadVersion[] + versions UploadVersion[] @relation("UploadFileVersions") + currentVersion UploadVersion? @relation("UploadFileCurrentVersion", fields: [id, currentVersionId], references: [uploadFileId, id], onDelete: Restrict) originSession FileOriginSession @relation(fields: [projectId, sessionId], references: [projectId, sessionId], onDelete: Restrict) @@index([projectId, sessionId]) + @@unique([id, currentVersionId]) } model UploadVersion { @@ -254,7 +260,12 @@ model UploadVersion { uploadFileId String versionNumber Int state String @default("staging") - contentStorageKey String + originKind String @default("user_upload") + basedOnVersionId String? + storageTag String? + storedFilename String? + writeOperationId String? @unique + contentStorageKey String @unique filename String originalFilename String contentType String? @@ -263,11 +274,15 @@ model UploadVersion { createdAt DateTime? registeredAt DateTime @default(now()) updatedAt DateTime @updatedAt - uploadFile UploadFile @relation(fields: [uploadFileId], references: [id], onDelete: Cascade) + uploadFile UploadFile @relation("UploadFileVersions", fields: [uploadFileId], references: [id], onDelete: Cascade) + currentForFile UploadFile? @relation("UploadFileCurrentVersion") + basedOnVersion UploadVersion? @relation("UploadVersionDerivations", fields: [uploadFileId, basedOnVersionId], references: [uploadFileId, id], onDelete: Restrict) + derivedVersions UploadVersion[] @relation("UploadVersionDerivations") sourceInputs ArtifactVersionInput[] @relation("UploadVersionSourceInputs") visionEvidence VisionEvidence[] @@unique([uploadFileId, versionNumber]) + @@unique([uploadFileId, id]) @@index([uploadFileId, state, registeredAt]) } @@ -321,44 +336,53 @@ model ArtifactMessageSnapshot { // One immutable generated Artifact save event. Evidence/content fields become immutable once the // staging row is published; finalize only advances state and message ownership. model ArtifactVersion { - id String @id + id String @id artifactId String versionNumber Int filename String - artifactRunId String - writeOperationId String? @unique + originKind String @default("agent_generated") + basedOnVersionId String? + storageTag String? + storedFilename String? + artifactRunId String? + writeOperationId String? @unique writeRequestChecksum String? - rootFrameId String - agentFrameId String - messageBranchId String - runtimeSegmentId String - promptMessageId String + rootFrameId String? + agentFrameId String? + messageBranchId String? + runtimeSegmentId String? + promptMessageId String? notebookSessionId String? producerRunId String? producerRunIndex Int? messageId String? messageSnapshotId String? - state String @default("staging") - contentStorageKey String - evidenceStorageKey String + state String @default("staging") + managedVisibleAt DateTime? + contentStorageKey String @unique + evidenceStorageKey String? contentType String? sizeBytes BigInt checksum String - evidenceJson String - evidenceChecksum String - evidenceSchemaVersion Int @default(1) + evidenceJson String? + evidenceChecksum String? + evidenceSchemaVersion Int? executionSnapshotJson String? executionSnapshotChecksum String? executionSnapshotStorageKey String? executionSnapshotSchemaVersion Int? - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - artifact ArtifactLineage @relation(fields: [artifactId], references: [id], onDelete: Cascade) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + artifact ArtifactLineage @relation("ArtifactLineageVersions", fields: [artifactId], references: [id], onDelete: Cascade) + currentForLineage ArtifactLineage? @relation("ArtifactLineageCurrentVersion") + basedOnVersion ArtifactVersion? @relation("ArtifactVersionDerivations", fields: [artifactId, basedOnVersionId], references: [artifactId, id], onDelete: Restrict) + derivedVersions ArtifactVersion[] @relation("ArtifactVersionDerivations") messageSnapshot ArtifactMessageSnapshot? @relation(fields: [messageSnapshotId], references: [id], onDelete: SetNull) - inputs ArtifactVersionInput[] @relation("ProducedArtifactInputs") - sourceInputs ArtifactVersionInput[] @relation("ArtifactVersionSourceInputs") + inputs ArtifactVersionInput[] @relation("ProducedArtifactInputs") + sourceInputs ArtifactVersionInput[] @relation("ArtifactVersionSourceInputs") @@unique([artifactId, versionNumber]) + @@unique([artifactId, id]) @@index([artifactId, createdAt]) @@index([artifactRunId, state]) @@index([rootFrameId, agentFrameId, messageBranchId, promptMessageId]) @@ -366,6 +390,31 @@ model ArtifactVersion { @@index([messageSnapshotId]) } +// Recovery journal for cross-resource text edit publication. Staging operations do not allocate a +// content version number and are invisible to normal version readers. +model ManagedFileVersionWriteOperation { + operationId String @id + source String + projectId String + sourceFileId String + basedOnVersionId String + expectedHeadVersionId String + state String @default("staging") + storageTag String + storedFilename String + contentStorageKey String @unique + checksum String + sizeBytes BigInt + textFormatJson String + resultVersionId String? @unique + errorCode String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([source, sourceFileId, state]) + @@index([projectId, state, createdAt]) +} + model ArtifactVersionInput { id String @id artifactVersionId String diff --git a/prisma/sqlite-check-constraints.json b/prisma/sqlite-check-constraints.json index 46d3d8488..067ff5640 100644 --- a/prisma/sqlite-check-constraints.json +++ b/prisma/sqlite-check-constraints.json @@ -130,6 +130,16 @@ "name": "UploadVersion_state_check", "expression": "\"state\" IN ('staging', 'ready')" }, + { + "tableName": "UploadVersion", + "name": "UploadVersion_originKind_check", + "expression": "\"originKind\" IN ('user_upload', 'user_edit', 'legacy')" + }, + { + "tableName": "UploadVersion", + "name": "UploadVersion_userEdit_check", + "expression": "(\"originKind\" <> 'user_edit' OR (\"state\" = 'ready' AND \"basedOnVersionId\" IS NOT NULL AND \"storageTag\" IS NOT NULL AND \"storedFilename\" IS NOT NULL))" + }, { "tableName": "ArtifactMessageSnapshot", "name": "ArtifactMessageSnapshot_state_check", @@ -145,10 +155,30 @@ "name": "ArtifactVersion_filename_check", "expression": "length(\"filename\") > 0" }, + { + "tableName": "ArtifactVersion", + "name": "ArtifactVersion_originKind_check", + "expression": "\"originKind\" IN ('agent_generated', 'user_edit', 'legacy')" + }, + { + "tableName": "ArtifactVersion", + "name": "ArtifactVersion_provenance_check", + "expression": "((\"originKind\" = 'agent_generated' AND \"artifactRunId\" IS NOT NULL AND \"rootFrameId\" IS NOT NULL AND \"agentFrameId\" IS NOT NULL AND \"messageBranchId\" IS NOT NULL AND \"runtimeSegmentId\" IS NOT NULL AND \"promptMessageId\" IS NOT NULL AND \"evidenceStorageKey\" IS NOT NULL AND \"evidenceJson\" IS NOT NULL AND \"evidenceChecksum\" IS NOT NULL AND \"evidenceSchemaVersion\" IS NOT NULL) OR (\"originKind\" = 'user_edit' AND \"state\" = 'finalized' AND \"basedOnVersionId\" IS NOT NULL AND \"storageTag\" IS NOT NULL AND \"storedFilename\" IS NOT NULL AND \"artifactRunId\" IS NULL AND \"writeRequestChecksum\" IS NULL AND \"rootFrameId\" IS NULL AND \"agentFrameId\" IS NULL AND \"messageBranchId\" IS NULL AND \"runtimeSegmentId\" IS NULL AND \"promptMessageId\" IS NULL AND \"notebookSessionId\" IS NULL AND \"producerRunId\" IS NULL AND \"producerRunIndex\" IS NULL AND \"messageId\" IS NULL AND \"messageSnapshotId\" IS NULL AND \"evidenceStorageKey\" IS NULL AND \"evidenceJson\" IS NULL AND \"evidenceChecksum\" IS NULL AND \"evidenceSchemaVersion\" IS NULL AND \"executionSnapshotJson\" IS NULL AND \"executionSnapshotChecksum\" IS NULL AND \"executionSnapshotStorageKey\" IS NULL AND \"executionSnapshotSchemaVersion\" IS NULL) OR \"originKind\" = 'legacy')" + }, + { + "tableName": "ManagedFileVersionWriteOperation", + "name": "ManagedFileVersionWriteOperation_source_check", + "expression": "\"source\" IN ('artifact', 'upload')" + }, + { + "tableName": "ManagedFileVersionWriteOperation", + "name": "ManagedFileVersionWriteOperation_state_check", + "expression": "\"state\" IN ('staging', 'file_ready', 'published', 'conflict', 'failed')" + }, { "tableName": "ArtifactVersion", "name": "ArtifactVersion_evidenceJson_check", - "expression": "json_valid(\"evidenceJson\") AND json_type(\"evidenceJson\") = 'object'" + "expression": "\"evidenceJson\" IS NULL OR (json_valid(\"evidenceJson\") AND json_type(\"evidenceJson\") = 'object')" }, { "tableName": "ArtifactVersion", diff --git a/resources/skills/literature-review/kernel.test.ts b/resources/skills/literature-review/kernel.test.ts index 5753f30ac..bd83a441b 100644 --- a/resources/skills/literature-review/kernel.test.ts +++ b/resources/skills/literature-review/kernel.test.ts @@ -5,9 +5,12 @@ import { fileURLToPath } from 'node:url' import { describe, expect, it } from 'vitest' -const python3 = ['/opt/homebrew/bin/python3', '/usr/local/bin/python3', '/usr/bin/python3'].find( - existsSync -) +const python3 = [ + process.env.OPEN_SCIENCE_TEST_PY_ENV, + '/opt/homebrew/bin/python3', + '/usr/local/bin/python3', + '/usr/bin/python3' +].find((candidate): candidate is string => typeof candidate === 'string' && existsSync(candidate)) const gate = python3 ? describe : describe.skip const testFile = resolve(dirname(fileURLToPath(import.meta.url)), 'test_kernel.py') diff --git a/scripts/database-migration-ledger-smoke.mjs b/scripts/database-migration-ledger-smoke.mjs index 9978f3f54..0627b6461 100644 --- a/scripts/database-migration-ledger-smoke.mjs +++ b/scripts/database-migration-ledger-smoke.mjs @@ -52,6 +52,10 @@ const EXPECTED_MIGRATION_LEDGER = [ { id: '0012_tag_ordering', checksum: '2cbc89454c8642d65806366add598ef4547fb7f513e04c267d1ad0274a472e2f' + }, + { + id: '0013_managed_file_version_foundation', + checksum: 'ed8b5f4ad1a326a98f844193ec182396cc486d86c90969b19996e9a485aaf6c7' } ] const LEGACY_PROJECT_ID = 'package-smoke-legacy-project' diff --git a/scripts/database-migration-ledger-smoke.test.ts b/scripts/database-migration-ledger-smoke.test.ts index 1aaa5b231..b210cc245 100644 --- a/scripts/database-migration-ledger-smoke.test.ts +++ b/scripts/database-migration-ledger-smoke.test.ts @@ -20,7 +20,7 @@ import { PrismaClient } from '@prisma/client' describe('packaged database migration ledger smoke', () => { it('pins every packaged application migration identity and checksum', () => { expect(MIGRATION_MANIFEST.at(-1)?.checksum).toBe( - '2cbc89454c8642d65806366add598ef4547fb7f513e04c267d1ad0274a472e2f' + 'ed8b5f4ad1a326a98f844193ec182396cc486d86c90969b19996e9a485aaf6c7' ) expect(() => assertApplicationMigrationLedger(MIGRATION_MANIFEST)).not.toThrow() expect(() => assertApplicationMigrationLedger(MIGRATION_MANIFEST.slice(0, -1))).toThrow( diff --git a/src/main/acp/file-reference-resolver.test.ts b/src/main/acp/file-reference-resolver.test.ts index 2412be945..1caf11cf5 100644 --- a/src/main/acp/file-reference-resolver.test.ts +++ b/src/main/acp/file-reference-resolver.test.ts @@ -27,6 +27,158 @@ afterEach(async () => { }) describe('managed file reference resolver', () => { + it('opens an exact trusted lease for a logical reference without reopening its path', async () => { + const close = vi.fn(async () => undefined) + const openResolved = vi.fn().mockResolvedValue({ + path: '/replaced-after-open.txt', + size: 12, + read: vi.fn(), + readRange: vi.fn(), + verifyUnchanged: vi.fn(), + close, + logicalFile: { id: 'artifact-file', displayName: 'notes.txt' }, + version: { + id: 'artifact-version-2', + checksum: '2'.repeat(64), + contentType: 'text/plain' + } + }) + const resolver = createManagedFileReferenceResolver({ + managedFileVersions: { openResolved } as never + }) + + const resolved = await resolver.resolve( + { projectId: 'project-1', sessionId: 'session-1' }, + { + id: 'artifact-row', + sourceFileId: 'artifact-file', + versionId: 'artifact-version-2', + name: 'stale.txt', + path: 'artifact-version:stale', + source: 'artifact' + } + ) + + expect(openResolved).toHaveBeenCalledWith({ + source: 'artifact', + projectId: 'project-1', + fileId: 'artifact-file', + versionId: 'artifact-version-2' + }) + expect(resolved).toMatchObject({ + absolutePath: '/replaced-after-open.txt', + size: 12, + sourceFileId: 'artifact-file', + versionId: 'artifact-version-2', + checksum: '2'.repeat(64), + trustedLease: { close } + }) + expect(close).not.toHaveBeenCalled() + }) + + it.each(['artifact', 'upload'] as const)( + 'resolves a default %s reference through the current DB head at prompt preparation', + async (source) => { + root = await mkdtemp(join(tmpdir(), 'file-reference-head-')) + const headPath = join(root, `${source}-v2.csv`) + await writeFile(headPath, 'head bytes') + const close = vi.fn(async () => undefined) + const openResolved = vi.fn().mockResolvedValue({ + path: headPath, + size: 10, + read: vi.fn(), + readRange: vi.fn(), + verifyUnchanged: vi.fn(), + close, + logicalFile: { id: `${source}-file`, displayName: 'study.csv' }, + version: { + id: `${source}-version-2`, + checksum: '2'.repeat(64), + contentType: 'text/csv' + } + }) + const resolver = createManagedFileReferenceResolver({ + managedFileVersions: { openResolved } as never + }) + + await expect( + resolver.resolve( + { projectId: 'project-1', sessionId: 'target-session' }, + { + id: `${source}-row`, + sourceFileId: `${source}-file`, + name: 'stale-name.csv', + path: `${source}-version:stale-projection`, + source + } + ) + ).resolves.toMatchObject({ + absolutePath: headPath, + name: 'study.csv', + mimeType: 'text/csv', + size: 10, + sourceFileId: `${source}-file`, + versionId: `${source}-version-2`, + checksum: '2'.repeat(64) + }) + expect(openResolved).toHaveBeenCalledWith({ + source, + projectId: 'project-1', + fileId: `${source}-file` + }) + expect(close).not.toHaveBeenCalled() + } + ) + + it('preserves an explicit historical Version when preparing an Agent reference', async () => { + root = await mkdtemp(join(tmpdir(), 'file-reference-exact-')) + const historicalPath = join(root, 'artifact-v1.csv') + await writeFile(historicalPath, 'v1 bytes') + const close = vi.fn(async () => undefined) + const openResolved = vi.fn().mockResolvedValue({ + path: historicalPath, + size: 8, + read: vi.fn(), + readRange: vi.fn(), + verifyUnchanged: vi.fn(), + close, + logicalFile: { id: 'artifact-file', displayName: 'study.csv' }, + version: { + id: 'artifact-version-1', + checksum: '1'.repeat(64), + contentType: 'text/csv' + } + }) + const resolver = createManagedFileReferenceResolver({ + managedFileVersions: { openResolved } as never + }) + + await expect( + resolver.resolve( + { projectId: 'project-1', sessionId: 'target-session' }, + { + id: 'artifact-row', + sourceFileId: 'artifact-file', + versionId: 'artifact-version-1', + name: 'study.csv', + path: 'artifact-version:stale-projection', + source: 'artifact' + } + ) + ).resolves.toMatchObject({ + sourceFileId: 'artifact-file', + versionId: 'artifact-version-1', + checksum: '1'.repeat(64) + }) + + expect(openResolved).toHaveBeenCalledWith({ + source: 'artifact', + projectId: 'project-1', + fileId: 'artifact-file', + versionId: 'artifact-version-1' + }) + expect(close).not.toHaveBeenCalled() + }) it('validates upload paths and returns trusted on-disk metadata', async () => { root = await mkdtemp(join(tmpdir(), 'file-reference-resolver-')) const uploads = new UploadRepository(root) diff --git a/src/main/acp/file-reference-resolver.ts b/src/main/acp/file-reference-resolver.ts index 974a64cfa..65db60c08 100644 --- a/src/main/acp/file-reference-resolver.ts +++ b/src/main/acp/file-reference-resolver.ts @@ -15,6 +15,10 @@ import type { ArtifactRepository } from '../artifacts/repository' import type { ArtifactProvenanceRepository } from '../artifacts/provenance-repository' import { createLogger, errorLogFields } from '../logger' import type { UploadRepository } from '../uploads/repository' +import type { + ManagedFileReadLease, + ManagedFileVersionService +} from '../managed-file-versions/service' const log = createLogger('acp-file-reference-resolver') @@ -24,6 +28,11 @@ export type FileReferenceContext = { connectionGeneration?: number } +export type TrustedFileReferenceLease = Pick< + ManagedFileReadLease, + 'size' | 'read' | 'readRange' | 'copyTo' | 'verifyUnchanged' | 'close' +> + export type ResolvedFileReference = { absolutePath: string uri: string @@ -31,6 +40,10 @@ export type ResolvedFileReference = { mimeType?: string size: number allowSkillImportReference: boolean + sourceFileId?: string + versionId?: string + checksum?: string + trustedLease?: TrustedFileReferenceLease } // This adapter is the deliberate extension seam for linked folders and other future file origins. @@ -288,13 +301,18 @@ export class FileReferenceResolver { if (!adapter) throw new Error(`File reference source is not configured: ${reference.source}`) const resolved = await adapter.resolve(context, reference) - const fileInfo = await stat(resolved.absolutePath) - if (!fileInfo.isFile()) throw new Error('Referenced path is not a file.') + try { + const fileInfo = resolved.trustedLease ? undefined : await stat(resolved.absolutePath) + if (fileInfo && !fileInfo.isFile()) throw new Error('Referenced path is not a file.') - return { - ...resolved, - uri: pathToFileURL(resolved.absolutePath).href, - size: fileInfo.size + return { + ...resolved, + uri: pathToFileURL(resolved.absolutePath).href, + size: resolved.trustedLease?.size ?? fileInfo!.size + } + } catch (error) { + await resolved.trustedLease?.close().catch(() => undefined) + throw error } } @@ -321,17 +339,45 @@ export const createManagedFileReferenceResolver = (dependencies: { grantedRoots?: { resolveRoot: (rootId: string) => Promise | undefined> } + managedFileVersions?: Pick }): FileReferenceResolver => { const adapters: FileReferenceAdapter[] = [] const readOnlyProjection = dependencies.grantedRoots ? new ReadOnlyLinkedFileProjection(dependencies.readOnlyProjectionMaxSessionBytes) : undefined - if (dependencies.uploads) { + const resolveLogicalReference = async ( + projectId: string, + reference: Extract + ): Promise => { + if (!reference.sourceFileId || !dependencies.managedFileVersions) return undefined + return dependencies.managedFileVersions.openResolved({ + source: reference.source, + projectId, + fileId: reference.sourceFileId, + ...(reference.versionId ? { versionId: reference.versionId } : {}) + }) + } + + if (dependencies.uploads || dependencies.managedFileVersions) { adapters.push({ source: 'upload', resolve: async ({ projectId, sessionId }, reference) => { if (reference.source !== 'upload') throw new Error('Invalid upload reference.') + const logical = await resolveLogicalReference(projectId, reference) + if (logical) { + return { + absolutePath: logical.path, + name: logical.logicalFile.displayName, + mimeType: logical.version.contentType ?? reference.mimeType, + allowSkillImportReference: true, + sourceFileId: logical.logicalFile.id, + versionId: logical.version.id, + checksum: logical.version.checksum, + trustedLease: logical + } + } + if (!dependencies.uploads) throw new Error('Upload repository is not configured.') let absolutePath: string try { absolutePath = await dependencies.uploads!.resolveSessionUploadPath( @@ -358,11 +404,24 @@ export const createManagedFileReferenceResolver = (dependencies: { }) } - if (dependencies.artifacts) { + if (dependencies.artifacts || dependencies.managedFileVersions) { adapters.push({ source: 'artifact', resolve: async ({ projectId }, reference) => { if (reference.source !== 'artifact') throw new Error('Invalid artifact reference.') + const logical = await resolveLogicalReference(projectId, reference) + if (logical) { + return { + absolutePath: logical.path, + name: logical.logicalFile.displayName, + mimeType: logical.version.contentType ?? reference.mimeType, + allowSkillImportReference: false, + sourceFileId: logical.logicalFile.id, + versionId: logical.version.id, + checksum: logical.version.checksum, + trustedLease: logical + } + } const versionIdentity = parseArtifactVersionLocator(reference.path) if (versionIdentity) { if (versionIdentity.projectId !== projectId) { @@ -380,6 +439,7 @@ export const createManagedFileReferenceResolver = (dependencies: { allowSkillImportReference: false } } + if (!dependencies.artifacts) throw new Error('Artifact repository is not configured.') return { absolutePath: await dependencies.artifacts!.resolveManagedFilePath({ path: reference.path diff --git a/src/main/acp/interrupted-turn-continuation.test.ts b/src/main/acp/interrupted-turn-continuation.test.ts index 3f915ee40..87dc38dfb 100644 --- a/src/main/acp/interrupted-turn-continuation.test.ts +++ b/src/main/acp/interrupted-turn-continuation.test.ts @@ -100,6 +100,9 @@ describe('continueInterruptedTurn', () => { { type: 'artifact', id: 'artifact-1', + sourceFileId: 'artifact-lineage-1', + versionId: 'artifact-version-3', + checksum: 'a'.repeat(64), name: 'evidence.csv', path: '/workspace/evidence.csv', source: 'artifact' @@ -132,7 +135,15 @@ describe('continueInterruptedTurn', () => { text: expect.stringMatching(/continue the interrupted turn/i), turnIntent: 'plan-first', forcedSkillIds: ['skill-1'], - referencedArtifacts: [expect.objectContaining({ id: 'artifact-1' })], + referencedArtifacts: [ + expect.objectContaining({ + id: 'artifact-1', + source: 'artifact', + sourceFileId: 'artifact-lineage-1', + versionId: 'artifact-version-3', + checksum: 'a'.repeat(64) + }) + ], suppressUserMessage: true, provenanceContext: expect.objectContaining({ promptMessageId: 'prompt-1' }) }) diff --git a/src/main/acp/interrupted-turn-continuation.ts b/src/main/acp/interrupted-turn-continuation.ts index d8f57f6a9..9c451fb52 100644 --- a/src/main/acp/interrupted-turn-continuation.ts +++ b/src/main/acp/interrupted-turn-continuation.ts @@ -131,8 +131,10 @@ const buildContinuationRequest = ( name: part.name, path: part.path, source: part.source, + sourceFileId: part.sourceFileId, mimeType: part.mimeType, - versionId: part.versionId + versionId: part.versionId, + checksum: part.checksum } ) return references diff --git a/src/main/acp/prompt-content-owner.test.ts b/src/main/acp/prompt-content-owner.test.ts index 3a9fde66c..b3976ee00 100644 --- a/src/main/acp/prompt-content-owner.test.ts +++ b/src/main/acp/prompt-content-owner.test.ts @@ -1,17 +1,21 @@ import type { ContentBlock } from '@agentclientprotocol/sdk' -import { mkdtemp, rm, truncate, writeFile } from 'node:fs/promises' +import { access, mkdtemp, readFile, rm, stat, truncate, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { pathToFileURL } from 'node:url' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' import { afterEach, describe, expect, it, vi } from 'vitest' import type { UploadedAttachment } from '../../shared/uploads' import { estimateHistoryTokens } from '../../shared/history-preamble' +import { MAX_AUTO_PROCESS_IMAGE_BYTES } from '../uploads/attachment-media' import { UploadRepository } from '../uploads/repository' import { stageUploadFixtures } from '../uploads/repository.test-utils' -import { MAX_AUTO_PROCESS_IMAGE_BYTES } from '../uploads/attachment-media' -import { createManagedFileReferenceResolver } from './file-reference-resolver' +import { + createManagedFileReferenceResolver, + FileReferenceResolver +} from './file-reference-resolver' import { AcpPromptContentOwner } from './prompt-content-owner' +import { TurnResourceSnapshotStore } from './turn-resource-snapshot-store' const roots: string[] = [] @@ -26,56 +30,462 @@ const contentBlocks = (content: string | ContentBlock[]): ContentBlock[] => { return content as ContentBlock[] } +type TrustedLeaseFixture = { + size: number + read: ReturnType + readRange: ReturnType + copyTo: ReturnType + verifyUnchanged: ReturnType + close: ReturnType +} + +const createTrustedLease = (bytes: Buffer): TrustedLeaseFixture => { + const readRange = vi.fn(async (begin: number, end: number) => bytes.subarray(begin, end)) + const read = vi.fn( + async (buffer: Uint8Array, offset: number, length: number, position: number) => { + const chunk = bytes.subarray(position, position + length) + buffer.set(chunk, offset) + return { bytesRead: chunk.byteLength } + } + ) + const verifyUnchanged = vi.fn(async () => undefined) + const copyTo = vi.fn(async (destinationPath: string) => { + await writeFile(destinationPath, bytes, { flag: 'wx' }) + }) + const close = vi.fn(async () => undefined) + return { size: bytes.byteLength, read, readRange, copyTo, verifyUnchanged, close } +} + afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) }) describe('AcpPromptContentOwner', () => { - it('sends a read-only linked file from its disposable snapshot URI', async () => { + it('keeps resource links on a private verified snapshot until prepared content closes', async () => { const root = await createRoot() - const sourcePath = join(root, 'study.csv') - await writeFile(sourcePath, 'id,value\n1,2\n') - const resolver = createManagedFileReferenceResolver({ - grantedRoots: { resolveRoot: async () => ({ path: root, access: 'ro' }) } + const replacedPath = join(root, 'replaced.txt') + await writeFile(replacedPath, 'untrusted path bytes') + const trustedBytes = Buffer.from('verified lease bytes') + const trustedLease = createTrustedLease(trustedBytes) + const owner = new AcpPromptContentOwner({ + fileReferenceResolver: new FileReferenceResolver([ + { + source: 'artifact', + resolve: async () => + ({ + absolutePath: replacedPath, + name: 'notes.txt', + mimeType: 'text/plain', + allowSkillImportReference: false, + sourceFileId: 'artifact-file', + versionId: 'artifact-version-2', + trustedLease + }) as never + } + ]) }) - const resolveReference = vi.spyOn(resolver, 'resolve') - const owner = new AcpPromptContentOwner({ fileReferenceResolver: resolver }) - const prepared = await owner.prepare({ + const result = await owner.prepare({ appSessionId: 'session-1', - projectId: 'default-project', - connectionGeneration: 2, - text: 'analyze this file', + projectId: 'project-1', + text: 'read this', historyImages: [], historyUploads: [], currentUploads: [], references: [ { - id: 'linked-1', - name: 'study.csv', - source: 'linked-folder', - rootId: 'root-1', - relativePath: 'study.csv', - mimeType: 'text/csv' + id: 'artifact-row', + sourceFileId: 'artifact-file', + name: 'notes.txt', + path: 'artifact-version:stale', + source: 'artifact' + } + ], + codexSkillInputs: [], + skillImportEnabled: false, + fileTextBudget: 1 + }) + + const resourceLink = contentBlocks(result.content).find( + (block): block is Extract => + block.type === 'resource_link' + ) + expect(resourceLink).toMatchObject({ name: 'notes.txt', mimeType: 'text/plain' }) + const snapshotPath = fileURLToPath(resourceLink!.uri) + expect(snapshotPath).not.toBe(replacedPath) + await writeFile(replacedPath, 'replaced again after prepare') + await expect(readFile(snapshotPath)).resolves.toEqual(trustedBytes) + expect((await stat(dirname(snapshotPath))).mode & 0o777).toBe(0o700) + expect((await stat(snapshotPath)).mode & 0o777).toBe(0o600) + expect(trustedLease.copyTo).toHaveBeenCalledWith(snapshotPath, { exclusive: true }) + expect(trustedLease.close).toHaveBeenCalledOnce() + + result.close() + await expect(access(snapshotPath)).rejects.toMatchObject({ code: 'ENOENT' }) + }) + + it('preserves snapshot copy errors when lease close and cleanup also fail', async () => { + const trustedLease = createTrustedLease(Buffer.from('verified lease bytes')) + const copyError = new Error('snapshot copy failed') + trustedLease.copyTo.mockRejectedValueOnce(copyError) + trustedLease.close.mockRejectedValueOnce(new Error('lease close failed')) + const owner = new AcpPromptContentOwner({ + fileReferenceResolver: new FileReferenceResolver([ + { + source: 'artifact', + resolve: async () => + ({ + absolutePath: '/replaced.txt', + name: 'notes.txt', + mimeType: 'text/plain', + allowSkillImportReference: false, + trustedLease + }) as never + } + ]) + }) + + await expect( + owner.prepare({ + appSessionId: 'session-1', + projectId: 'project-1', + text: 'read this', + historyImages: [], + historyUploads: [], + currentUploads: [], + references: [ + { + id: 'artifact-row', + sourceFileId: 'artifact-file', + name: 'notes.txt', + path: 'artifact-version:stale', + source: 'artifact' + } + ], + codexSkillInputs: [], + skillImportEnabled: false + }) + ).rejects.toBe(copyError) + expect(trustedLease.close).toHaveBeenCalledOnce() + }) + + it('keeps prepared-content close best-effort and idempotent when snapshot cleanup fails', async () => { + const root = await createRoot() + const sourcePath = join(root, 'source.txt') + await writeFile(sourcePath, 'path bytes') + const trustedLease = createTrustedLease(Buffer.from('verified bytes')) + const removeDirectory = vi.fn(() => { + throw new Error('snapshot cleanup failed') + }) + const owner = new AcpPromptContentOwner({ + createResourceSnapshotStore: () => new TurnResourceSnapshotStore({ removeDirectory }), + fileReferenceResolver: new FileReferenceResolver([ + { + source: 'artifact', + resolve: async () => + ({ + absolutePath: sourcePath, + name: 'notes.txt', + mimeType: 'text/plain', + allowSkillImportReference: false, + trustedLease + }) as never + } + ]) + }) + + const result = await owner.prepare({ + appSessionId: 'session-1', + projectId: 'project-1', + text: 'read this', + historyImages: [], + historyUploads: [], + currentUploads: [], + references: [ + { + id: 'artifact-row', + sourceFileId: 'artifact-file', + name: 'notes.txt', + path: 'artifact-version:stale', + source: 'artifact' + } + ], + codexSkillInputs: [], + skillImportEnabled: false, + fileTextBudget: 1 + }) + const resourceLink = contentBlocks(result.content).find( + (block): block is Extract => + block.type === 'resource_link' + ) + const snapshotRoot = dirname(fileURLToPath(resourceLink!.uri)) + + expect(() => result.close()).not.toThrow() + expect(() => result.close()).not.toThrow() + expect(removeDirectory).toHaveBeenCalledOnce() + await rm(snapshotRoot, { recursive: true, force: true }) + }) + + it('consumes small managed reference text from the trusted lease and closes it', async () => { + const root = await createRoot() + const replacedPath = join(root, 'replaced.txt') + await writeFile(replacedPath, 'wrong path bytes') + const trustedBytes = Buffer.from('trusted lease bytes') + const trustedLease = createTrustedLease(trustedBytes) + const owner = new AcpPromptContentOwner({ + fileReferenceResolver: new FileReferenceResolver([ + { + source: 'artifact', + resolve: async () => + ({ + absolutePath: replacedPath, + name: 'notes.txt', + mimeType: 'text/plain', + allowSkillImportReference: false, + sourceFileId: 'artifact-file', + versionId: 'artifact-version-2', + checksum: '2'.repeat(64), + trustedLease + }) as never + } + ]) + }) + + const result = await owner.prepare({ + appSessionId: 'session-1', + projectId: 'project-1', + text: 'read this', + historyImages: [], + historyUploads: [], + currentUploads: [], + references: [ + { + id: 'artifact-row', + sourceFileId: 'artifact-file', + versionId: 'artifact-version-2', + name: 'notes.txt', + path: 'artifact-version:stale', + source: 'artifact' } ], codexSkillInputs: [], skillImportEnabled: false }) - const resource = contentBlocks(prepared.content).find((block) => block.type === 'resource') - expect(resource).toMatchObject({ + expect(contentBlocks(result.content)).toContainEqual({ type: 'resource', - resource: { text: 'id,value\n1,2\n' } + resource: expect.objectContaining({ text: 'trusted lease bytes' }) }) - if (resource?.type === 'resource') { - expect(resource.resource.uri).not.toBe(pathToFileURL(sourcePath).href) - } - expect(resolveReference).toHaveBeenCalledWith( - expect.objectContaining({ connectionGeneration: 2 }), - expect.anything() - ) - owner.clear() + expect(trustedLease.readRange).toHaveBeenCalledWith(0, trustedBytes.byteLength) + expect(trustedLease.close).toHaveBeenCalledOnce() + result.close() + }) + + it('builds budgeted managed text previews from trusted ranges and closes the lease', async () => { + const root = await createRoot() + const trustedBytes = Buffer.from(`TRUSTED-BEGIN\n${'a'.repeat(600_000)}\nTRUSTED-END`) + const replacedPath = join(root, 'replaced-large.txt') + await writeFile(replacedPath, Buffer.alloc(trustedBytes.byteLength, 0x78)) + const trustedLease = createTrustedLease(trustedBytes) + const owner = new AcpPromptContentOwner({ + fileReferenceResolver: new FileReferenceResolver([ + { + source: 'artifact', + resolve: async () => + ({ + absolutePath: replacedPath, + name: 'large.txt', + mimeType: 'text/plain', + allowSkillImportReference: false, + sourceFileId: 'artifact-file', + versionId: 'artifact-version-2', + checksum: '2'.repeat(64), + trustedLease + }) as never + } + ]) + }) + + const result = await owner.prepare({ + appSessionId: 'session-1', + projectId: 'project-1', + text: 'summarize this', + historyImages: [], + historyUploads: [], + currentUploads: [], + references: [ + { + id: 'artifact-row', + sourceFileId: 'artifact-file', + name: 'large.txt', + path: 'artifact-version:stale', + source: 'artifact' + } + ], + codexSkillInputs: [], + skillImportEnabled: false, + fileTextBudget: 2_000 + }) + + const renderedText = contentBlocks(result.content) + .flatMap((block) => (block.type === 'text' ? [block.text] : [])) + .join('\n') + expect(renderedText).toContain('TRUSTED-BEGIN') + expect(renderedText).toContain('TRUSTED-END') + expect(trustedLease.read.mock.calls.length).toBeGreaterThanOrEqual(2) + expect(trustedLease.close).toHaveBeenCalledOnce() + result.close() + }) + + it('inlines managed images from the trusted lease and closes it', async () => { + const root = await createRoot() + const replacedPath = join(root, 'replaced.png') + await writeFile(replacedPath, 'wrong image bytes') + const trustedBytes = Buffer.from('trusted image bytes') + const trustedLease = createTrustedLease(trustedBytes) + const owner = new AcpPromptContentOwner({ + fileReferenceResolver: new FileReferenceResolver([ + { + source: 'artifact', + resolve: async () => + ({ + absolutePath: replacedPath, + name: 'figure.png', + mimeType: 'image/png', + allowSkillImportReference: false, + trustedLease + }) as never + } + ]) + }) + + const result = await owner.prepare({ + appSessionId: 'session-1', + projectId: 'project-1', + text: 'inspect this', + historyImages: [], + historyUploads: [], + currentUploads: [], + references: [ + { + id: 'artifact-row', + sourceFileId: 'artifact-file', + name: 'figure.png', + path: 'artifact-version:stale', + source: 'artifact', + mimeType: 'image/png' + } + ], + codexSkillInputs: [], + skillImportEnabled: false + }) + + expect(contentBlocks(result.content)).toContainEqual({ + type: 'image', + data: trustedBytes.toString('base64'), + mimeType: 'image/png', + uri: expect.any(String) + }) + expect(trustedLease.close).toHaveBeenCalledOnce() + result.close() + }) + + it('closes the trusted lease when automatic consumption fails', async () => { + const trustedLease = createTrustedLease(Buffer.from('unreadable')) + trustedLease.readRange.mockRejectedValueOnce(new Error('anchored read failed')) + trustedLease.close.mockRejectedValueOnce(new Error('close failed')) + const owner = new AcpPromptContentOwner({ + fileReferenceResolver: new FileReferenceResolver([ + { + source: 'artifact', + resolve: async () => + ({ + absolutePath: '/missing.txt', + name: 'notes.txt', + mimeType: 'text/plain', + allowSkillImportReference: false, + trustedLease + }) as never + } + ]) + }) + + await expect( + owner.prepare({ + appSessionId: 'session-1', + projectId: 'project-1', + text: 'read this', + historyImages: [], + historyUploads: [], + currentUploads: [], + references: [ + { + id: 'artifact-row', + sourceFileId: 'artifact-file', + name: 'notes.txt', + path: 'artifact-version:stale', + source: 'artifact' + } + ], + codexSkillInputs: [], + skillImportEnabled: false + }) + ).rejects.toThrow('anchored read failed') + expect(trustedLease.close).toHaveBeenCalledOnce() + }) + + it('registers the exact head identity resolved at Agent turn start', async () => { + const root = await createRoot() + const path = join(root, 'head.csv') + await writeFile(path, 'id,value\n1,2\n') + const owner = new AcpPromptContentOwner({ + fileReferenceResolver: new FileReferenceResolver([ + { + source: 'artifact', + resolve: async () => ({ + absolutePath: path, + name: 'head.csv', + mimeType: 'text/csv', + allowSkillImportReference: false, + sourceFileId: 'artifact-file', + versionId: 'artifact-version-2', + checksum: '2'.repeat(64) + }) + } + ]) + }) + + const result = await owner.prepare({ + appSessionId: 'session-1', + projectId: 'project-1', + text: 'analyze', + historyImages: [], + historyUploads: [], + currentUploads: [], + references: [ + { + id: 'stale-row', + sourceFileId: 'artifact-file', + name: 'stale.csv', + path: 'artifact-version:stale', + source: 'artifact' + } + ], + codexSkillInputs: [], + skillImportEnabled: false + }) + + expect(result.turnInputs?.references).toEqual([ + { + id: 'stale-row', + sourceFileId: 'artifact-file', + name: 'head.csv', + path: 'artifact-version:stale', + source: 'artifact', + versionId: 'artifact-version-2', + checksum: '2'.repeat(64) + } + ]) }) it('keeps the text fast path isolated from ambient resolvers and defensively owns Codex metadata', async () => { @@ -101,9 +511,14 @@ describe('AcpPromptContentOwner', () => { onSkillImportAttachmentEligible }) - expect(plain).toEqual({ content: ' plain text is preserved ', historyImageCount: 0 }) + expect(plain).toEqual({ + content: ' plain text is preserved ', + historyImageCount: 0, + close: expect.any(Function) + }) expect(resolveReference).not.toHaveBeenCalled() expect(onSkillImportAttachmentEligible).not.toHaveBeenCalled() + plain.close() const codexSkillInputs = [{ name: 'research', path: '/skills/research/SKILL.md' }] const withCodexMetadata = await owner.prepare({ @@ -132,8 +547,10 @@ describe('AcpPromptContentOwner', () => { } } ], - historyImageCount: 0 + historyImageCount: 0, + close: expect.any(Function) }) + withCodexMetadata.close() expect(resolveReference).not.toHaveBeenCalled() }) diff --git a/src/main/acp/prompt-content-owner.ts b/src/main/acp/prompt-content-owner.ts index 4c99be81e..0a8f4cd99 100644 --- a/src/main/acp/prompt-content-owner.ts +++ b/src/main/acp/prompt-content-owner.ts @@ -10,7 +10,11 @@ import { PENDING_UPLOAD_SESSION_ID, type UploadedAttachment } from '../../shared/uploads' -import { readBoundedManagedFilePreview } from '../managed-file-preview' +import { + readBoundedManagedFilePreview, + readBoundedManagedFilePreviewLease +} from '../managed-file-preview' +import { createLogger, errorLogFields } from '../logger' import { buildImageContentData, canInlineImageInSession, @@ -23,7 +27,10 @@ import { type InlineImageBudget } from '../uploads/attachment-media' import type { UploadRepository } from '../uploads/repository' -import { isImportableSkillArchivePath } from '../skills/skill-archive-sniffer' +import { + isImportableSkillArchive, + isImportableSkillArchivePath +} from '../skills/skill-archive-sniffer' import { ATTACHMENT_PREVIEW_BYTES, MAX_EMBEDDED_TEXT_UPLOAD_BYTES, @@ -36,9 +43,12 @@ import { isTextLikeAttachment, mimeEssence } from './attachment-content' -import type { FileReferenceResolver } from './file-reference-resolver' +import type { FileReferenceResolver, TrustedFileReferenceLease } from './file-reference-resolver' +import { TurnResourceSnapshotStore } from './turn-resource-snapshot-store' import type { VisionEvidenceSource } from './vision-evidence-repository' +const log = createLogger('acp-prompt-content-owner') + type CodexSkillInput = { name: string path: string @@ -48,6 +58,7 @@ type AcpPromptContentOwnerOptions = { uploadRepository?: UploadRepository fileReferenceResolver: FileReferenceResolver inlineImageBudgetBytes?: number + createResourceSnapshotStore?: () => TurnResourceSnapshotStore } type PrepareAcpPromptContentInput = { @@ -77,6 +88,7 @@ type PreparedAcpPromptContent = { historyImageCount: number imageSources?: ReadonlyArray turnInputs?: AcpPromptTurnInputs + close: () => void } type ResolvedPromptFile = { @@ -86,6 +98,7 @@ type ResolvedPromptFile = { mimeType?: string size: number allowSkillImportReference: boolean + trustedLease?: TrustedFileReferenceLease } type PromptFileTextBudget = { @@ -112,6 +125,7 @@ const errorMessage = (error: unknown): string => { // with the runtime; every piece of content resolved here is supplied explicitly by the caller. class AcpPromptContentOwner { private readonly sessionInlineImageBytes = new Map>() + private readonly activePreparedResources = new Set<() => void>() private readonly inlineImageBudgetBytes: number constructor(private readonly options: AcpPromptContentOwnerOptions) { @@ -119,8 +133,40 @@ class AcpPromptContentOwner { } async prepare(input: PrepareAcpPromptContentInput): Promise { + const snapshots = + this.options.createResourceSnapshotStore?.() ?? new TurnResourceSnapshotStore() + let closed = false + const close = (): void => { + if (closed) return + closed = true + this.activePreparedResources.delete(close) + try { + snapshots.close() + } catch (error) { + try { + log.error('turn resource snapshot cleanup failed', errorLogFields(error)) + } catch { + // Snapshot cleanup cannot replace the provider outcome. + } + } + } + this.activePreparedResources.add(close) + try { + return await this.prepareOwned(input, snapshots, close) + } catch (error) { + close() + throw error + } + } + + private async prepareOwned( + input: PrepareAcpPromptContentInput, + snapshots: TurnResourceSnapshotStore, + close: () => void + ): Promise { const hasUploads = input.historyUploads.length > 0 || input.currentUploads.length > 0 let promptUploads: UploadedAttachment[] = [] + const resolvedReferences: FileReference[] = [] let historyImageCount = 0 const imageSources: Array = [] @@ -238,12 +284,14 @@ class AcpPromptContentOwner { } for (const reference of input.references) { - const blocks = await this.createReferencedArtifactContentBlocks( + const resolved = await this.createReferencedArtifactContentBlocks( input, reference, - fileTextBudget + fileTextBudget, + snapshots ) - for (const block of blocks) { + resolvedReferences.push(resolved.reference) + for (const block of resolved.blocks) { appendBlock(block, this.imageOverflowResourceLink(block, reference.name)) } } @@ -255,17 +303,18 @@ class AcpPromptContentOwner { const turnInputUploads = promptUploads.filter( (upload, index) => index >= input.historyUploads.length || upload.versionId ) - const hasTurnInputs = turnInputUploads.length > 0 || input.references.length > 0 + const hasTurnInputs = turnInputUploads.length > 0 || resolvedReferences.length > 0 return { content: preparedContent, + close, historyImageCount, ...(imageSources.length > 0 ? { imageSources } : {}), ...(hasTurnInputs ? { turnInputs: { uploads: turnInputUploads, - references: [...input.references] + references: resolvedReferences } } : {}) @@ -283,6 +332,7 @@ class AcpPromptContentOwner { clear(): void { this.options.fileReferenceResolver.clear() this.sessionInlineImageBytes.clear() + for (const close of [...this.activePreparedResources]) close() } clearGeneration(connectionGeneration: number): void { @@ -371,8 +421,9 @@ class AcpPromptContentOwner { private async createReferencedArtifactContentBlocks( input: PrepareAcpPromptContentInput, reference: FileReference, - fileTextBudget: PromptFileTextBudget - ): Promise { + fileTextBudget: PromptFileTextBudget, + snapshots: TurnResourceSnapshotStore + ): Promise<{ blocks: ContentBlock[]; reference: FileReference }> { const resolvedReference = await this.options.fileReferenceResolver.resolve( { sessionId: input.appSessionId, @@ -382,7 +433,49 @@ class AcpPromptContentOwner { reference ) - return this.buildFileContentBlocks(input, resolvedReference, fileTextBudget, false) + let prepared: { blocks: ContentBlock[]; reference: FileReference } + try { + const snapshot = resolvedReference.trustedLease + ? await snapshots.create(resolvedReference.name, resolvedReference.trustedLease) + : undefined + const promptReference = snapshot + ? { + ...resolvedReference, + absolutePath: snapshot.absolutePath, + uri: snapshot.uri + } + : resolvedReference + const exactReference: FileReference = + reference.source !== 'linked-folder' && + resolvedReference.sourceFileId && + resolvedReference.versionId + ? { + ...reference, + sourceFileId: resolvedReference.sourceFileId, + name: resolvedReference.name, + versionId: resolvedReference.versionId, + ...(resolvedReference.checksum ? { checksum: resolvedReference.checksum } : {}) + } + : reference + prepared = { + blocks: await this.buildFileContentBlocks(input, promptReference, fileTextBudget, false), + reference: exactReference + } + } catch (error) { + await resolvedReference.trustedLease?.close().catch(() => undefined) + throw error + } + await resolvedReference.trustedLease?.close() + return prepared + } + + private async readPromptFileBytes(descriptor: ResolvedPromptFile): Promise { + if (!descriptor.trustedLease) return readFile(descriptor.absolutePath) + if (descriptor.size === 0) { + await descriptor.trustedLease.verifyUnchanged() + return Buffer.alloc(0) + } + return Buffer.from(await descriptor.trustedLease.readRange(0, descriptor.size)) } private async buildFileContentBlocks( @@ -448,7 +541,7 @@ class AcpPromptContentOwner { if ( input.skillImportEnabled && allowSkillImportReference && - (await this.isSkillPackageFile(name, absolutePath)) + (await this.isSkillPackageFile(name, descriptor)) ) { const turnToken = input.skillImportTurnToken if (turnToken) { @@ -487,7 +580,8 @@ class AcpPromptContentOwner { const { data, mimeType: outMimeType } = await buildImageContentData( absolutePath, imageMimeType, - size + size, + descriptor.trustedLease ? () => this.readPromptFileBytes(descriptor) : undefined ) if (!input.imageCompatibilityRelay) { @@ -513,7 +607,7 @@ class AcpPromptContentOwner { { type: 'resource_link', uri, name, title: name, mimeType: 'application/pdf', size } ] } - const block = await this.createPdfContentBlock(name, absolutePath, uri) + const block = await this.createPdfContentBlock(name, descriptor, uri) return this.admitTextResource(block, descriptor, fileTextBudget, false) } @@ -521,7 +615,11 @@ class AcpPromptContentOwner { if (size <= MAX_EMBEDDED_TEXT_UPLOAD_BYTES) { const block: ContentBlock = { type: 'resource', - resource: { uri, mimeType, text: await readFile(absolutePath, 'utf8') } + resource: { + uri, + mimeType, + text: (await this.readPromptFileBytes(descriptor)).toString('utf8') + } } return this.admitTextResource( block, @@ -600,25 +698,35 @@ class AcpPromptContentOwner { } else { const previewBytes = Math.min(ATTACHMENT_PREVIEW_BYTES, Math.max(256, previewBudget * 3)) const startBytes = tabular ? previewBytes : Math.ceil(previewBytes / 2) - const start = await readBoundedManagedFilePreview( - absolutePath, - { path: absolutePath, maxBytes: startBytes, encoding: 'utf8' }, - 'Attachment preview requires UTF-8 encoding.' - ) + const readPreview = ( + request: Parameters[1] + ): ReturnType => + descriptor.trustedLease + ? readBoundedManagedFilePreviewLease( + descriptor.trustedLease, + request, + 'Attachment preview requires UTF-8 encoding.' + ) + : readBoundedManagedFilePreview( + absolutePath, + request, + 'Attachment preview requires UTF-8 encoding.' + ) + const start = await readPreview({ + path: absolutePath, + maxBytes: startBytes, + encoding: 'utf8' + }) if (tabular) { rawPreview = start.content } else { const endBytes = Math.max(1, previewBytes - startBytes) - const end = await readBoundedManagedFilePreview( - absolutePath, - { - path: absolutePath, - offset: Math.max(0, size - endBytes), - maxBytes: endBytes, - encoding: 'utf8' - }, - 'Attachment preview requires UTF-8 encoding.' - ) + const end = await readPreview({ + path: absolutePath, + offset: Math.max(0, size - endBytes), + maxBytes: endBytes, + encoding: 'utf8' + }) rawPreview = `${start.content}\n\n[…middle of file omitted…]\n\n${end.content}` } } @@ -667,15 +775,37 @@ class AcpPromptContentOwner { return name.toLowerCase().endsWith('.pdf') } - private async isSkillPackageFile(name: string, filePath: string): Promise { + private async isSkillPackageFile(name: string, descriptor: ResolvedPromptFile): Promise { const normalizedName = name.toLowerCase() if (!normalizedName.endsWith('.skill') && !normalizedName.endsWith('.zip')) return false - return isImportableSkillArchivePath(filePath) + if (!descriptor.trustedLease) { + return isImportableSkillArchivePath(descriptor.absolutePath) + } + const lease = descriptor.trustedLease + return isImportableSkillArchive({ + size: descriptor.size, + read: async (position, length) => { + if ( + !Number.isSafeInteger(position) || + !Number.isSafeInteger(length) || + position < 0 || + length < 0 || + position + length > descriptor.size + ) { + return undefined + } + if (length === 0) { + await lease.verifyUnchanged() + return Buffer.alloc(0) + } + return Buffer.from(await lease.readRange(position, position + length)) + } + }) } private async createPdfContentBlock( name: string, - filePath: string, + descriptor: ResolvedPromptFile, uri: string ): Promise { const toResource = (text: string): ContentBlock => ({ @@ -684,7 +814,15 @@ class AcpPromptContentOwner { }) try { - const { text, pageCount, truncated } = await extractPdfText(filePath) + const { text, pageCount, truncated } = await extractPdfText( + descriptor.absolutePath, + descriptor.trustedLease + ? { + size: descriptor.size, + readBytes: () => this.readPromptFileBytes(descriptor) + } + : undefined + ) if (!text) { return toResource( `[No selectable text could be extracted from "${name}" (${pageCount} page(s)). It may be a scanned or image-only PDF.]` diff --git a/src/main/acp/prompt-outcome-finalizer.test.ts b/src/main/acp/prompt-outcome-finalizer.test.ts index 92bfb2689..b74b306d1 100644 --- a/src/main/acp/prompt-outcome-finalizer.test.ts +++ b/src/main/acp/prompt-outcome-finalizer.test.ts @@ -376,6 +376,7 @@ describe('AcpPromptOutcomeFinalizer', () => { expect(harness.handles.failPendingSkillActivities).toHaveBeenCalledOnce() expect(harness.context.supersede).toHaveBeenCalledOnce() expect(harness.handles.skill.close).toHaveBeenCalledWith('failed') + expect(harness.handles.prepared?.close).toHaveBeenCalledOnce() }) it('keeps a replacement interaction current when the old provider outcome is superseded', async () => { @@ -395,6 +396,7 @@ describe('AcpPromptOutcomeFinalizer', () => { expect(harness.handles.permission.clearCorrelationsForSession).not.toHaveBeenCalled() expect(harness.handles.onPromptEnded).not.toHaveBeenCalled() expect(harness.context.supersede).toHaveBeenCalledOnce() + expect(harness.handles.prepared?.close).toHaveBeenCalledOnce() }) it('publishes a cancelled stop for a current prompt that was not dispatched', async () => { @@ -421,5 +423,6 @@ describe('AcpPromptOutcomeFinalizer', () => { ) expect(harness.context.fail).toHaveBeenCalledOnce() expect(harness.handles.skill.close).toHaveBeenCalledWith('cancelled') + expect(harness.handles.prepared?.close).toHaveBeenCalledOnce() }) }) diff --git a/src/main/acp/prompt-preparation-owner.test.ts b/src/main/acp/prompt-preparation-owner.test.ts index ed53851ca..49f040164 100644 --- a/src/main/acp/prompt-preparation-owner.test.ts +++ b/src/main/acp/prompt-preparation-owner.test.ts @@ -23,6 +23,7 @@ type Fixture = { authorizeReferencedUploads: Mock releaseGrant: Mock registerTurnInputs: Mock + promptClose: Mock } const request = (overrides: Partial = {}): AcpPromptRequest => ({ @@ -44,11 +45,13 @@ const setup = ( imageInputCompatibility?: Pick ): Fixture => { const turn = contextTurn() + const promptClose = vi.fn() const promptContent = { prepare: vi.fn(async () => ({ content: 'provider-content', historyImageCount: 0, - turnInputs: { uploads: [], references: [] } + turnInputs: { uploads: [], references: [] }, + close: promptClose })) } const contextUsage = { @@ -141,7 +144,8 @@ const setup = ( turnSkill, authorizeReferencedUploads, releaseGrant, - registerTurnInputs + registerTurnInputs, + promptClose } } @@ -200,6 +204,7 @@ describe('AcpPromptPreparationOwner', () => { handle.close() handle.close() expect(fixture.releaseGrant).toHaveBeenCalledTimes(1) + expect(fixture.promptClose).toHaveBeenCalledTimes(1) expect(fixture.turn.fail).not.toHaveBeenCalled() }) @@ -230,7 +235,8 @@ describe('AcpPromptPreparationOwner', () => { resolve({ content: 'stale-provider-content', historyImageCount: 0, - turnInputs: { uploads: [], references: [] } + turnInputs: { uploads: [], references: [] }, + close: fixture.promptClose }) }) ) @@ -244,6 +250,7 @@ describe('AcpPromptPreparationOwner', () => { expect(handle.status).toBe('cancelled') expect(fixture.releaseGrant).toHaveBeenCalledTimes(1) + expect(fixture.promptClose).toHaveBeenCalledTimes(1) expect(fixture.contextUsage.beginTurn).not.toHaveBeenCalled() expect(fixture.registerTurnInputs).not.toHaveBeenCalled() }) @@ -255,7 +262,8 @@ describe('AcpPromptPreparationOwner', () => { const fixture = setup(imageInputCompatibility) fixture.promptContent.prepare.mockResolvedValueOnce({ content: [{ type: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' }], - historyImageCount: 1 + historyImageCount: 1, + close: fixture.promptClose }) const handle = await fixture.prepare({ @@ -298,5 +306,20 @@ describe('AcpPromptPreparationOwner', () => { expect(fixture.turn.fail).toHaveBeenCalledTimes(1) expect(fixture.turn.supersede).toHaveBeenCalledTimes(1) expect(fixture.releaseGrant).toHaveBeenCalledTimes(1) + expect(fixture.promptClose).toHaveBeenCalledTimes(1) + }) + + it('preserves preparation errors when prepared-content cleanup also fails', async () => { + const fixture = setup() + const registrationError = new Error('turn input registration failed') + fixture.registerTurnInputs.mockRejectedValueOnce(registrationError) + fixture.promptClose.mockImplementationOnce(() => { + throw new Error('snapshot cleanup failed') + }) + + await expect(fixture.prepare()).rejects.toBe(registrationError) + + expect(fixture.promptClose).toHaveBeenCalledOnce() + expect(fixture.releaseGrant).toHaveBeenCalledOnce() }) }) diff --git a/src/main/acp/prompt-preparation-owner.ts b/src/main/acp/prompt-preparation-owner.ts index f12a98bbb..abd23f606 100644 --- a/src/main/acp/prompt-preparation-owner.ts +++ b/src/main/acp/prompt-preparation-owner.ts @@ -113,6 +113,7 @@ class AcpPromptPreparationOwner { async prepare(input: AcpPromptPreparationInput): Promise { let releaseGrant: (() => void) | undefined + let releasePromptContent: (() => void) | undefined let contextTurn: ContextWindowTurnHandle | undefined let closed = false @@ -122,14 +123,28 @@ class AcpPromptPreparationOwner { const ownedContext = contextTurn contextTurn = undefined try { - if (ownedContext && failContext) ownedContext.fail() + const releaseContent = releasePromptContent + releasePromptContent = undefined + try { + releaseContent?.() + } catch (error) { + try { + log.error('prepared prompt content cleanup failed', errorLogFields(error)) + } catch { + // Cleanup diagnostics cannot replace the preparation or provider outcome. + } + } } finally { try { - ownedContext?.supersede() + if (ownedContext && failContext) ownedContext.fail() } finally { - const release = releaseGrant - releaseGrant = undefined - release?.() + try { + ownedContext?.supersede() + } finally { + const release = releaseGrant + releaseGrant = undefined + release?.() + } } } } @@ -228,6 +243,7 @@ class AcpPromptPreparationOwner { skillImportTurnToken: input.skillImportTurnToken, onSkillImportAttachmentEligible: input.onSkillImportAttachmentEligible }) + releasePromptContent = prepared.close if (await cancelled()) return cancelPrepared() const providerContent = this.options.imageInputCompatibility ? await this.options.imageInputCompatibility.prepare({ diff --git a/src/main/acp/prompt-turn-workflow.test.ts b/src/main/acp/prompt-turn-workflow.test.ts index 6458833cf..458ab34a5 100644 --- a/src/main/acp/prompt-turn-workflow.test.ts +++ b/src/main/acp/prompt-turn-workflow.test.ts @@ -1,4 +1,9 @@ import type { ActiveSession, PromptResponse } from '@agentclientprotocol/sdk' +import { rmSync } from 'node:fs' +import { access, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' import { describe, expect, it, vi, type Mock } from 'vitest' import type { AcpPromptRequest } from '../../shared/acp' @@ -7,7 +12,7 @@ import { opencodeFramework } from '../agent-framework' import type { ArtifactTurnHandle } from './artifact-turn-owner' import type { AcpBackendGenerationView } from './backend-generation-owner' import type { ContextWindowTurnHandle } from './context-usage-tracker' -import type { AcpPromptOutcomeFinalizer } from './prompt-outcome-finalizer' +import { AcpPromptOutcomeFinalizer } from './prompt-outcome-finalizer' import type { ReadyPreparedPromptHandle } from './prompt-preparation-owner' import { AcpPromptTurnWorkflow, type AcpPromptTurnWorkflowOptions } from './prompt-turn-workflow' import { AcpSessionAggregate } from './session-aggregate' @@ -364,6 +369,62 @@ const request = (): AcpPromptRequest => ({ }) describe('AcpPromptTurnWorkflow', () => { + it('keeps a prepared resource snapshot through provider dispatch and removes it at terminal', async () => { + const root = await mkdtemp(join(tmpdir(), 'acp-workflow-snapshot-')) + const snapshotPath = join(root, 'snapshot.txt') + await writeFile(snapshotPath, 'verified bytes') + const providerPrompt = vi.fn(async (content) => { + await expect(access(snapshotPath)).resolves.toBeUndefined() + expect(content).toEqual([ + expect.objectContaining({ type: 'resource_link', uri: pathToFileURL(snapshotPath).href }) + ]) + }) + const harness = createHarness({ + execute: async (input) => { + expect(await input.beforeDispatch()).toBe('active') + await input.session.prompt(input.content) + await input.onAccepted() + input.captureStop() + return { + kind: 'stopped', + response: { stopReason: 'end_turn' }, + facts: {} + } + }, + finalize: (handles, outcome) => new AcpPromptOutcomeFinalizer().finalize(handles, outcome) + }) + harness.setSession({ + sessionId: 'provider-2', + prompt: providerPrompt, + nextUpdate: vi.fn() + } as unknown as ActiveSession) + Object.assign(harness.prepared, { + content: [ + { + type: 'resource_link', + uri: pathToFileURL(snapshotPath).href, + name: 'notes.txt', + mimeType: 'text/plain' + } + ] + }) + vi.mocked(harness.prepared.close).mockImplementation(() => + rmSync(root, { recursive: true, force: true }) + ) + Object.assign(harness.context, { captureTerminal: vi.fn(() => undefined) }) + + try { + await expect(harness.workflow.run(request(), { kind: 'user' })).resolves.toEqual({ + stopReason: 'end_turn' + }) + expect(providerPrompt).toHaveBeenCalledOnce() + await expect(readFile(snapshotPath)).rejects.toMatchObject({ code: 'ENOENT' }) + expect(harness.prepared.close).toHaveBeenCalledOnce() + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + it('admits and executes one user turn in owner order with its opaque handles', async () => { const harness = createHarness() diff --git a/src/main/acp/runtime-base-composition.ts b/src/main/acp/runtime-base-composition.ts index d8b71e269..a1081fdac 100644 --- a/src/main/acp/runtime-base-composition.ts +++ b/src/main/acp/runtime-base-composition.ts @@ -177,7 +177,8 @@ const composeAcpRuntimeBaseOwners = (options: AcpRuntimeOptions) => { uploads: uploadRepository, artifacts: artifactRepository, artifactVersions: options.artifacts?.provenance, - grantedRoots: options.grantedRoots + grantedRoots: options.grantedRoots, + managedFileVersions: options.artifacts?.managedFileVersions }) return Object.freeze({ diff --git a/src/main/acp/runtime-composition.ts b/src/main/acp/runtime-composition.ts index cd954cf21..087e95585 100644 --- a/src/main/acp/runtime-composition.ts +++ b/src/main/acp/runtime-composition.ts @@ -120,6 +120,10 @@ type AcpRuntimeCompositionOptions = AcpRuntimeArtifacts & { NotificationInboxController, 'record' | 'settleAction' | 'settleAuthorization' > + managedFileVersions?: Pick< + import('../managed-file-versions/service').ManagedFileVersionService, + 'openResolved' + > onSessionTurnStarted?: (sessionId: string, turnToken: string) => void onSessionTurnEnded?: (sessionId: string, turnToken: string) => void onSkillImportAttachmentEligible?: ( @@ -160,6 +164,7 @@ const createAcpRuntime = ({ repository, runRegistry, provenanceRepository, + managedFileVersions, uploadRepository, notebookRpcServer, peekNotebookHandoffContext, @@ -263,6 +268,7 @@ const createAcpRuntime = ({ repository, runRegistry, provenance: provenanceRepository, + managedFileVersions, getRpcConnection: () => notebookRpcServer.ensureStarted(), issueRpcCapability: (binding) => notebookRpcServer.issueArtifactRunCapability(binding), diff --git a/src/main/acp/runtime.test.ts b/src/main/acp/runtime.test.ts index 926872f57..c87602cda 100644 --- a/src/main/acp/runtime.test.ts +++ b/src/main/acp/runtime.test.ts @@ -20286,7 +20286,8 @@ describe('ACP runtime session management', () => { const version = await client.artifactVersion.findFirstOrThrow({ where: { artifactRunId: claim.runId } }) - expect(JSON.parse(version.evidenceJson)).toMatchObject({ + expect(version.evidenceJson).not.toBeNull() + expect(JSON.parse(version.evidenceJson!)).toMatchObject({ producer: { state: 'available', kind: 'connector', @@ -20704,7 +20705,8 @@ describe('ACP runtime session management', () => { where: { artifactRunId: claim.runId } }) expect(finalizedVersion).toMatchObject({ state: 'finalized', messageId: 'assistant-current' }) - expect(JSON.parse(finalizedVersion.evidenceJson)).toMatchObject({ + expect(finalizedVersion.evidenceJson).not.toBeNull() + expect(JSON.parse(finalizedVersion.evidenceJson!)).toMatchObject({ producer: { state: 'available', kind: 'connector', diff --git a/src/main/acp/runtime.ts b/src/main/acp/runtime.ts index 3143eb4b3..1939b2df0 100644 --- a/src/main/acp/runtime.ts +++ b/src/main/acp/runtime.ts @@ -280,6 +280,10 @@ type AcpRuntimeArtifactOptions = { 'resolveVersionContent' > > + managedFileVersions?: Pick< + import('../managed-file-versions/service').ManagedFileVersionService, + 'openResolved' + > } type AcpRuntimeUploadOptions = { diff --git a/src/main/acp/turn-resource-snapshot-store.ts b/src/main/acp/turn-resource-snapshot-store.ts new file mode 100644 index 000000000..25e226372 --- /dev/null +++ b/src/main/acp/turn-resource-snapshot-store.ts @@ -0,0 +1,78 @@ +import { randomUUID } from 'node:crypto' +import { rmSync } from 'node:fs' +import { chmod, mkdtemp } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { extname, join } from 'node:path' +import { pathToFileURL } from 'node:url' + +type TurnResourceSnapshotSource = Readonly<{ + copyTo: (destinationPath: string, options?: { exclusive?: boolean }) => Promise +}> + +type TurnResourceSnapshot = Readonly<{ + absolutePath: string + uri: string +}> + +type TurnResourceSnapshotStoreOptions = Readonly<{ + temporaryRoot?: string + createId?: () => string + removeDirectory?: typeof rmSync +}> + +const safeExtension = (name: string): string => { + const extension = extname(name) + return /^\.[A-Za-z0-9]{1,16}$/.test(extension) ? extension : '' +} + +class TurnResourceSnapshotStore { + private rootPath: string | undefined + private closed = false + private readonly createId: () => string + private readonly removeDirectory: typeof rmSync + + constructor(private readonly options: TurnResourceSnapshotStoreOptions = {}) { + this.createId = options.createId ?? randomUUID + this.removeDirectory = options.removeDirectory ?? rmSync + } + + async create(name: string, source: TurnResourceSnapshotSource): Promise { + if (this.closed) throw new Error('Turn resource snapshot store is closed.') + + try { + const rootPath = await this.ensureRoot() + const absolutePath = join(rootPath, `${this.createId()}${safeExtension(name)}`) + await source.copyTo(absolutePath, { exclusive: true }) + await chmod(absolutePath, 0o600) + return Object.freeze({ absolutePath, uri: pathToFileURL(absolutePath).href }) + } catch (error) { + try { + this.close() + } catch { + // Snapshot creation failure is authoritative; cleanup is best-effort. + } + throw error + } + } + + close(): void { + if (this.closed) return + this.closed = true + const rootPath = this.rootPath + this.rootPath = undefined + if (rootPath) this.removeDirectory(rootPath, { recursive: true, force: true }) + } + + private async ensureRoot(): Promise { + if (this.rootPath) return this.rootPath + const rootPath = await mkdtemp( + join(this.options.temporaryRoot ?? tmpdir(), 'open-science-acp-turn-') + ) + this.rootPath = rootPath + await chmod(rootPath, 0o700) + return rootPath + } +} + +export { TurnResourceSnapshotStore } +export type { TurnResourceSnapshot, TurnResourceSnapshotSource, TurnResourceSnapshotStoreOptions } diff --git a/src/main/application-command-wiring.test.ts b/src/main/application-command-wiring.test.ts index 793c8aa20..abad00e9a 100644 --- a/src/main/application-command-wiring.test.ts +++ b/src/main/application-command-wiring.test.ts @@ -49,6 +49,22 @@ const dependencyBlock = compact( ) describe('production application command wiring', () => { + it('restores durable deletion barriers before managed file version recovery', () => { + expect( + ipcSource.indexOf('await projectDeletionCoordinator.restorePendingDeletionBarriers()') + ).toBeLessThan(ipcSource.indexOf('managedFileVersionService.recoverPendingWrites()')) + expect(ipcSource).toMatch( + /withDataRootWrite\(\(\)\s*=>\s*managedFileVersionService\.recoverPendingWrites\(\)\)/ + ) + }) + + it('does not block application startup on managed file content integrity scanning', () => { + expect(ipcSource).toMatch(/managedFileVersionService\s*\.auditActiveVersionIntegrity\(\)/) + expect(ipcSource).not.toMatch( + /await\s+managedFileVersionService\s*\.auditActiveVersionIntegrity\(\)/ + ) + }) + it('injects each stateful owner into its Electron adapter and command composition', () => { const sharedOwners = [ [ diff --git a/src/main/artifacts/ipc.test.ts b/src/main/artifacts/ipc.test.ts index a80c4afcb..98f424d5f 100644 --- a/src/main/artifacts/ipc.test.ts +++ b/src/main/artifacts/ipc.test.ts @@ -190,7 +190,11 @@ describe('artifact IPC handlers', () => { } as unknown as ArtifactRepository const provenance = { finalizeRun: vi.fn(async () => { - callOrder.push('sqlite') + callOrder.push('sqlite-finalize') + return [finalizedArtifact] + }), + activateFinalizedRun: vi.fn(async () => { + callOrder.push('sqlite-activate') return [finalizedArtifact] }) } @@ -226,7 +230,7 @@ describe('artifact IPC handlers', () => { await handlers.finalizeRunArtifacts({ claimId, messageId: 'message-1' }) expect(mutationScopes).toEqual([{ projectId: 'default-project', sessionId: 'session-1' }]) - expect(callOrder).toEqual(['sqlite', 'compatibility']) + expect(callOrder).toEqual(['sqlite-finalize', 'compatibility', 'sqlite-activate']) expect(provenance.finalizeRun).toHaveBeenCalledWith( expect.objectContaining({ messageId: 'message-1', @@ -519,7 +523,8 @@ describe('artifact IPC handlers', () => { .mockResolvedValue([finalizedArtifact]) } as unknown as ArtifactRepository const provenance = { - finalizeRun: vi.fn().mockResolvedValue([finalizedArtifact]) + finalizeRun: vi.fn().mockResolvedValue([finalizedArtifact]), + activateFinalizedRun: vi.fn().mockResolvedValue([finalizedArtifact]) } const registry = new ArtifactRunRegistry() const claimId = registry.register({ @@ -570,6 +575,7 @@ describe('artifact IPC handlers', () => { ).resolves.toEqual([finalizedArtifact]) expect(repository.finalizeRunArtifacts).toHaveBeenCalledTimes(2) expect(provenance.finalizeRun).toHaveBeenCalledTimes(2) + expect(provenance.activateFinalizedRun).toHaveBeenCalledOnce() expect(registry.resolve(claimId).finalizedMessageId).toBe('message-1') expect(diagnosticLogger.error).toHaveBeenCalledOnce() }) @@ -689,6 +695,93 @@ describe('artifact IPC handlers', () => { }) }) + it('resolves a logical Artifact preview at read time and preserves an explicit Version', async () => { + const root = await createStorageRoot() + const currentPath = join(root, 'current.txt') + await writeFile(currentPath, 'current head') + const resolveManagedFilePath = vi.fn().mockResolvedValue(currentPath) + const handlers = createArtifactHandlers({} as ArtifactRepository, new ArtifactRunRegistry(), { + resolveManagedFilePath + }) + const request = { + path: '/stale/projection.txt', + projectId: 'project-1', + fileId: 'artifact-1', + versionId: 'artifact-v1', + maxBytes: 1024 + } + + await expect(handlers.readPreview(request)).resolves.toMatchObject({ + content: 'current head' + }) + expect(resolveManagedFilePath).toHaveBeenCalledWith(request) + }) + + it('reads a logical Artifact preview through the verified lease and always closes it', async () => { + const bytes = Buffer.from('verified artifact bytes') + const close = vi.fn().mockResolvedValue(undefined) + const verifyUnchanged = vi.fn().mockResolvedValue(undefined) + const openManagedFileVersion = vi.fn().mockResolvedValue({ + size: bytes.byteLength, + read: vi.fn(async (buffer: Uint8Array, offset: number, length: number, position: number) => { + const chunk = bytes.subarray(position, position + length) + buffer.set(chunk, offset) + return { bytesRead: chunk.byteLength } + }), + verifyUnchanged, + close + }) + const resolveManagedFilePath = vi.fn().mockRejectedValue(new Error('must not resolve a path')) + const handlers = createArtifactHandlers({} as ArtifactRepository, new ArtifactRunRegistry(), { + openManagedFileVersion, + resolveManagedFilePath + }) + const request = { + path: '/replaceable/artifact.txt', + projectId: 'project-1', + fileId: 'artifact-1', + versionId: 'artifact-v1', + maxBytes: 1024 + } + + await expect(handlers.readPreview(request)).resolves.toMatchObject({ + content: 'verified artifact bytes' + }) + expect(openManagedFileVersion).toHaveBeenCalledWith(request) + expect(resolveManagedFilePath).not.toHaveBeenCalled() + expect(verifyUnchanged).toHaveBeenCalledOnce() + expect(close).toHaveBeenCalledOnce() + }) + + it('closes the logical Artifact lease when its post-read integrity check fails', async () => { + const bytes = Buffer.from('changed artifact bytes') + const close = vi.fn().mockResolvedValue(undefined) + const handlers = createArtifactHandlers({} as ArtifactRepository, new ArtifactRunRegistry(), { + openManagedFileVersion: vi.fn().mockResolvedValue({ + size: bytes.byteLength, + read: vi.fn( + async (buffer: Uint8Array, offset: number, length: number, position: number) => { + const chunk = bytes.subarray(position, position + length) + buffer.set(chunk, offset) + return { bytesRead: chunk.byteLength } + } + ), + verifyUnchanged: vi.fn().mockRejectedValue(new Error('managed version changed')), + close + }) + }) + + await expect( + handlers.readPreview({ + path: '/replaceable/artifact.txt', + projectId: 'project-1', + fileId: 'artifact-1', + versionId: 'artifact-v1' + }) + ).rejects.toThrow('managed version changed') + expect(close).toHaveBeenCalledOnce() + }) + it('reads bounded base64 previews for small managed image artifacts', async () => { const repository = new ArtifactRepository(await createStorageRoot()) const handlers = createArtifactHandlers(repository, new ArtifactRunRegistry()) diff --git a/src/main/artifacts/ipc.ts b/src/main/artifacts/ipc.ts index 44727e8b0..4e440a29c 100644 --- a/src/main/artifacts/ipc.ts +++ b/src/main/artifacts/ipc.ts @@ -35,7 +35,11 @@ import type { } from '../../shared/artifacts' import { resolveDataRoot } from '../storage-root' import { withDataRootWrite } from '../storage/migration-state' -import { readBoundedManagedFilePreview } from '../managed-file-preview' +import { + readBoundedManagedFilePreview, + readBoundedManagedFilePreviewLease, + type ManagedFilePreviewReadLease +} from '../managed-file-preview' import { createLogger, type Logger } from '../logger' import { ArtifactRepository } from './repository' import { ArtifactRunRegistry } from './run-registry' @@ -80,6 +84,10 @@ type ArtifactHandlers = { type ArtifactHandlerDependencies = { openPath?: (path: string) => Promise logger?: Pick + resolveManagedFilePath?: (request: ReadArtifactPreviewRequest) => Promise + openManagedFileVersion?: ( + request: ReadArtifactPreviewRequest + ) => Promise // Run ids of turns in flight right now (live runtime state). Their pending files are still being // written, so the orphan scan excludes them; a crashed run is absent here and correctly surfaces. getActiveArtifactRunIds?: () => string[] @@ -91,6 +99,7 @@ type ArtifactHandlerDependencies = { provenance?: Pick< ArtifactProvenanceRepository, | 'finalizeRun' + | 'activateFinalizedRun' | 'getLineage' | 'getVersionProvenance' | 'getVersionCore' @@ -199,6 +208,22 @@ const createArtifactHandlers = ( } }, readPreview: async (request) => { + if (request.projectId && request.fileId && dependencies.openManagedFileVersion) { + const lease = await dependencies.openManagedFileVersion(request) + try { + return await readBoundedManagedFilePreviewLease( + lease, + request, + 'Invalid artifact preview encoding.' + ) + } finally { + await lease.close() + } + } + if (request.projectId && request.fileId && dependencies.resolveManagedFilePath) { + const path = await dependencies.resolveManagedFilePath(request) + return readBoundedManagedFilePreview(path, request, 'Invalid artifact preview encoding.') + } const versionIdentity = parseArtifactVersionLocator(request.path) if (!versionIdentity) return repository.readManagedFilePreview(request) if (!dependencies.provenance) throw new Error('Artifact Provenance is not configured.') @@ -252,7 +277,7 @@ const finalizeRunArtifacts = async ( repository: ArtifactRepository, runRegistry: ArtifactRunRegistry, request: FinalizeRunArtifactsRequest, - provenance?: Pick, + provenance?: Pick, logger: Pick = log ): Promise => { const claim = runRegistry.resolve(request.claimId) @@ -344,6 +369,10 @@ const finalizeRunArtifacts = async ( }) compatibilityPublicationCompleted = true + if (provenance && provenanceRequest) { + provenanceArtifacts = await provenance.activateFinalizedRun(provenanceRequest) + } + runRegistry.markFinalized(request.claimId, request.messageId) return provenanceArtifacts ?? artifacts @@ -389,6 +418,7 @@ const registerArtifactIpcHandlers = ( provenance?: Pick< ArtifactProvenanceRepository, | 'finalizeRun' + | 'activateFinalizedRun' | 'getLineage' | 'getVersionProvenance' | 'getVersionCore' diff --git a/src/main/artifacts/provenance-finalization-recovery.ts b/src/main/artifacts/provenance-finalization-recovery.ts index 7b241fb6a..307710551 100644 --- a/src/main/artifacts/provenance-finalization-recovery.ts +++ b/src/main/artifacts/provenance-finalization-recovery.ts @@ -15,6 +15,7 @@ import { validateDurableMessageOwnership } from './provenance-message-finalization' import type { ArtifactRepository, PendingArtifactRunPublication } from './repository' +import { requireAgentArtifactVersion } from './provenance-version-kind' type ArtifactProjectReconciliationState = { readonly projectId: string @@ -45,7 +46,10 @@ type ArtifactProvenanceFinalizationRecoveryOptions = { ArtifactRepository, 'listPendingRunPublications' | 'findRunFinalizationMarker' | 'finalizeRunArtifacts' > - messageFinalizer: Pick + messageFinalizer: Pick< + ArtifactProvenanceMessageFinalizer, + 'finalizeRunWithDurableSession' | 'activateFinalizedRunWithDurableSession' + > } // Resolves the one agent message produced by the prepared prompt turn. It deliberately considers @@ -157,12 +161,15 @@ class ArtifactProvenanceFinalizationRecovery { } const client = await this.options.getClient() - const allFinalizationVersions = await client.artifactVersion.findMany({ - where: { - state: { in: ['pending', 'finalized'] }, - artifact: { is: { projectId, sessionId: appSessionId } } - } - }) + const allFinalizationVersions = ( + await client.artifactVersion.findMany({ + where: { + originKind: 'agent_generated', + state: { in: ['pending', 'finalized'] }, + artifact: { is: { projectId, sessionId: appSessionId } } + } + }) + ).map(requireAgentArtifactVersion) const candidateVersions = allFinalizationVersions.filter( (version) => version.state === 'pending' || @@ -274,6 +281,10 @@ class ArtifactProvenanceFinalizationRecovery { artifactVersionIds: markerVersionIds, provenanceContext: markerContext }) + await this.options.messageFinalizer.activateFinalizedRunWithDurableSession( + finalizationRequest, + durableSession + ) result.recoveredVersionIds.push( ...finalized .filter((version) => pendingVersionIds.has(version.versionId!)) diff --git a/src/main/artifacts/provenance-lifecycle-contract.test.ts b/src/main/artifacts/provenance-lifecycle-contract.test.ts index 1871d5126..fb8371bd9 100644 --- a/src/main/artifacts/provenance-lifecycle-contract.test.ts +++ b/src/main/artifacts/provenance-lifecycle-contract.test.ts @@ -292,6 +292,10 @@ describe('artifact provenance durable lifecycle contract', () => { const coreRow = await value.client.artifactVersion.findUniqueOrThrow({ where: { id: first.versionId } }) + if (!coreRow.evidenceJson || !coreRow.evidenceStorageKey) { + throw new Error('Expected the agent-generated lifecycle fixture to retain evidence.') + } + const coreEvidenceStorageKey = coreRow.evidenceStorageKey const corruptEvidence = JSON.parse(coreRow.evidenceJson) as ArtifactVersionEvidence const persistCorruptEvidence = async (): Promise => { const corruptEvidenceJson = canonicalJson(corruptEvidence as unknown as CanonicalJson) @@ -303,7 +307,7 @@ describe('artifact provenance durable lifecycle contract', () => { } }) await writeFile( - join(value.storageRoot, ...coreRow.evidenceStorageKey.split('/')), + join(value.storageRoot, ...coreEvidenceStorageKey.split('/')), corruptEvidenceJson, 'utf8' ) @@ -388,6 +392,9 @@ describe('artifact provenance durable lifecycle contract', () => { const row = await value.client.artifactVersion.findUniqueOrThrow({ where: { id: version.versionId } }) + if (!row.evidenceJson || !row.evidenceStorageKey) { + throw new Error('Expected the agent-generated lifecycle fixture to retain evidence.') + } const corruptEvidence = JSON.parse(row.evidenceJson) as ArtifactVersionEvidence expect(corruptEvidence).toMatchObject({ producer: { state: 'available' }, diff --git a/src/main/artifacts/provenance-message-finalization.ts b/src/main/artifacts/provenance-message-finalization.ts index f2ae2febc..f1a1b7693 100644 --- a/src/main/artifacts/provenance-message-finalization.ts +++ b/src/main/artifacts/provenance-message-finalization.ts @@ -1,4 +1,4 @@ -import type { PrismaClient } from '@prisma/client' +import type { Prisma, PrismaClient } from '@prisma/client' import type { ArtifactVersionEvidence, @@ -15,6 +15,7 @@ import { } from './provenance-execution-evidence' import { connectorEvidenceIsValid } from './provenance-producer-capture' import type { PersistedVersionFileRecord } from './provenance-version-writer' +import { requireAgentArtifactVersion } from './provenance-version-kind' const SAFE_SEGMENT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/ @@ -34,6 +35,7 @@ type ArtifactFinalizationProofRequest = FinalizeArtifactVersionsRequest & { } type ArtifactFinalizationProofVersion = PersistedVersionFileRecord & { + state: string messageId: string | null rootFrameId: string agentFrameId: string @@ -51,6 +53,7 @@ type ArtifactFinalizationProofVersion = PersistedVersionFileRecord & { type ArtifactProvenanceMessageFinalizerOptions = { getClient: () => Promise + now: () => Date loadSession?: ( projectId: string, appSessionId: string @@ -145,20 +148,23 @@ const loadArtifactFinalizationProofVersions = async ( client: Pick, request: ArtifactFinalizationProofRequest ): Promise => - client.artifactVersion.findMany({ - where: { - artifactRunId: request.artifactRunId, - rootFrameId: request.rootFrameId, - agentFrameId: request.agentFrameId, - messageBranchId: request.messageBranchId, - runtimeSegmentId: request.runtimeSegmentId, - promptMessageId: request.promptMessageId, - state: { in: ['pending', 'finalized'] }, - artifact: { is: { projectId: request.projectId, sessionId: request.appSessionId } } - }, - include: { artifact: true }, - orderBy: [{ artifactId: 'asc' }, { versionNumber: 'asc' }] - }) + ( + await client.artifactVersion.findMany({ + where: { + originKind: 'agent_generated', + artifactRunId: request.artifactRunId, + rootFrameId: request.rootFrameId, + agentFrameId: request.agentFrameId, + messageBranchId: request.messageBranchId, + runtimeSegmentId: request.runtimeSegmentId, + promptMessageId: request.promptMessageId, + state: { in: ['pending', 'finalized'] }, + artifact: { is: { projectId: request.projectId, sessionId: request.appSessionId } } + }, + include: { artifact: true }, + orderBy: [{ artifactId: 'asc' }, { versionNumber: 'asc' }] + }) + ).map(requireAgentArtifactVersion) const validateArtifactFinalizationProof = ( matching: readonly ArtifactFinalizationProofVersion[], @@ -410,6 +416,47 @@ export class ArtifactProvenanceMessageFinalizer { return this.finalizeVerifiedRun({ ...request, ...ancestry }) } + async activateFinalizedRun( + request: FinalizeArtifactVersionsRequest + ): Promise { + const durableSession = await this.loadFinalizationSession(request) + return this.activateFinalizedRunWithDurableSession(request, durableSession) + } + + async activateFinalizedRunWithDurableSession( + request: FinalizeArtifactVersionsRequest, + durableSession: PersistedChatSession + ): Promise { + const ancestry = validateDurableMessageOwnership(durableSession, request) + const normalizedRequest = normalizeArtifactFinalizationProofRequest({ ...request, ...ancestry }) + const client = await this.options.getClient() + const versions = await client.$transaction(async (transaction) => { + const matching = await loadArtifactFinalizationProofVersions(transaction, normalizedRequest) + validateArtifactFinalizationProof(matching, normalizedRequest) + if (matching.some((version) => version.state !== 'finalized')) { + throw new ArtifactFinalizationProofError( + 'version-not-eligible', + 'Artifact Versions must be finalized before their visible head can advance.' + ) + } + await transaction.artifactVersion.updateMany({ + where: { id: { in: matching.map((version) => version.id) }, managedVisibleAt: null }, + data: { managedVisibleAt: this.options.now() } + }) + await this.activateLineageHeads(transaction, matching) + return matching + }) + return Promise.all( + versions.map((version) => + this.options.projectVersionFile( + version, + version.artifact.projectId, + version.artifact.sessionId + ) + ) + ) + } + private async finalizeVerifiedRun( request: ArtifactFinalizationProofRequest ): Promise { @@ -429,14 +476,17 @@ export class ArtifactProvenanceMessageFinalizer { data: { state: 'finalized', messageId: normalizedRequest.messageId } }) - return transaction.artifactVersion.findMany({ - where: { - id: { in: matching.map((version) => version.id) }, - state: 'finalized' - }, - include: { artifact: true }, - orderBy: [{ artifactId: 'asc' }, { versionNumber: 'asc' }] - }) + return ( + await transaction.artifactVersion.findMany({ + where: { + id: { in: matching.map((version) => version.id) }, + originKind: 'agent_generated', + state: 'finalized' + }, + include: { artifact: true }, + orderBy: [{ artifactId: 'asc' }, { versionNumber: 'asc' }] + }) + ).map(requireAgentArtifactVersion) }) return Promise.all( @@ -449,4 +499,33 @@ export class ArtifactProvenanceMessageFinalizer { ) ) } + + private async activateLineageHeads( + transaction: Prisma.TransactionClient, + matching: readonly ArtifactFinalizationProofVersion[] + ): Promise { + for (const artifactId of new Set(matching.map((version) => version.artifactId))) { + const lineage = await transaction.artifactLineage.findUniqueOrThrow({ + where: { id: artifactId }, + include: { currentVersion: true } + }) + const latestMatching = matching + .filter((version) => version.artifactId === artifactId) + .reduce( + (latest, version) => + !latest || version.versionNumber > latest.versionNumber ? version : latest, + undefined + ) + if (!latestMatching) continue + if ( + !lineage.currentVersion || + lineage.currentVersion.versionNumber < latestMatching.versionNumber + ) { + await transaction.artifactLineage.update({ + where: { id: artifactId }, + data: { currentVersionId: latestMatching.id } + }) + } + } + } } diff --git a/src/main/artifacts/provenance-message-snapshot.test.ts b/src/main/artifacts/provenance-message-snapshot.test.ts index 7b4fe0abe..416e44d4d 100644 --- a/src/main/artifacts/provenance-message-snapshot.test.ts +++ b/src/main/artifacts/provenance-message-snapshot.test.ts @@ -178,6 +178,7 @@ describe('Provenance Message snapshots', () => { expect(findVersions).toHaveBeenLastCalledWith( expect.objectContaining({ select: { + originKind: true, rootFrameId: true, agentFrameId: true, messageBranchId: true, diff --git a/src/main/artifacts/provenance-message-snapshot.ts b/src/main/artifacts/provenance-message-snapshot.ts index 9832ac816..476d888db 100644 --- a/src/main/artifacts/provenance-message-snapshot.ts +++ b/src/main/artifacts/provenance-message-snapshot.ts @@ -19,6 +19,7 @@ import type { ProvenanceMessage, ProvenanceMessagePart } from '../../shared/artifact-provenance' +import { requireAgentArtifactVersion } from './provenance-version-kind' type ProvenanceMessageSnapshotOptions = { storageRoot: string @@ -31,6 +32,38 @@ type SessionDeletionReceipt = | { kind: 'ordinary'; projectId: string; sessionId: string } | { kind: 'retained'; projectId: string; sessionId: string; operationId: string } +type AgentMessageScope = { + originKind: string + rootFrameId: string | null + agentFrameId: string | null + messageBranchId: string | null + messageId: string | null +} + +const requireAgentMessageScope = ( + version: T +): T & { + originKind: 'agent_generated' + rootFrameId: string + agentFrameId: string + messageBranchId: string +} => { + if ( + version.originKind !== 'agent_generated' || + !version.rootFrameId || + !version.agentFrameId || + !version.messageBranchId + ) { + throw new Error('Artifact Version does not contain complete Agent message ownership.') + } + return version as T & { + originKind: 'agent_generated' + rootFrameId: string + agentFrameId: string + messageBranchId: string + } +} + class FinalizedArtifactBindingConflictError extends Error { constructor(message: string) { super(message) @@ -139,19 +172,23 @@ class ProvenanceMessageSnapshotRepository { throw new FinalizedArtifactBindingConflictError('Session conversation graph is unavailable.') } const client = await this.options.getClient() - const versions = await client.artifactVersion.findMany({ - where: { - state: 'finalized', - messageId: { not: null }, - artifact: { is: { projectId: session.projectId, sessionId: session.id } } - }, - select: { - rootFrameId: true, - agentFrameId: true, - messageBranchId: true, - messageId: true - } - }) + const versions = await client.artifactVersion + .findMany({ + where: { + originKind: 'agent_generated', + state: 'finalized', + messageId: { not: null }, + artifact: { is: { projectId: session.projectId, sessionId: session.id } } + }, + select: { + originKind: true, + rootFrameId: true, + agentFrameId: true, + messageBranchId: true, + messageId: true + } + }) + .then((rows) => rows.map(requireAgentMessageScope)) const scopes = new Map() for (const version of versions) { if (!version.messageId) continue @@ -174,20 +211,24 @@ class ProvenanceMessageSnapshotRepository { const graph = session.conversationGraph if (!graph) throw new Error('Session conversation graph is unavailable.') const client = await this.options.getClient() - const versions = await client.artifactVersion.findMany({ - where: { - state: 'finalized', - messageId: { not: null }, - messageSnapshotId: null, - artifact: { is: { projectId: session.projectId, sessionId: session.id } } - }, - select: { - rootFrameId: true, - agentFrameId: true, - messageBranchId: true, - messageId: true - } - }) + const versions = await client.artifactVersion + .findMany({ + where: { + originKind: 'agent_generated', + state: 'finalized', + messageId: { not: null }, + messageSnapshotId: null, + artifact: { is: { projectId: session.projectId, sessionId: session.id } } + }, + select: { + originKind: true, + rootFrameId: true, + agentFrameId: true, + messageBranchId: true, + messageId: true + } + }) + .then((rows) => rows.map(requireAgentMessageScope)) const scopes = new Map() for (const version of versions) { if (!version.messageId) continue @@ -227,13 +268,16 @@ class ProvenanceMessageSnapshotRepository { // The live graph is the final opportunity to freeze branch-scoped Messages. Deletion fails // closed if any finalized output cannot be linked to immutable Message evidence. await this.captureFinalizedMessages(session) - const finalizedVersions = await client.artifactVersion.findMany({ - where: { - state: 'finalized', - artifact: { is: { projectId: session.projectId, sessionId: session.id } } - }, - include: { messageSnapshot: true } - }) + const finalizedVersions = ( + await client.artifactVersion.findMany({ + where: { + originKind: 'agent_generated', + state: 'finalized', + artifact: { is: { projectId: session.projectId, sessionId: session.id } } + }, + include: { messageSnapshot: true } + }) + ).map(requireAgentArtifactVersion) if (finalizedVersions.some((version) => !version.messageSnapshot)) { throw new Error( 'Session deletion requires Message snapshots for every finalized Artifact Version.' diff --git a/src/main/artifacts/provenance-read-model.ts b/src/main/artifacts/provenance-read-model.ts index 3384ab2a5..92317d643 100644 --- a/src/main/artifacts/provenance-read-model.ts +++ b/src/main/artifacts/provenance-read-model.ts @@ -43,6 +43,7 @@ import { } from './provenance-snapshot-decoder' import { readOptionalFile, resolveStorageKey } from './provenance-storage' import type { PersistedVersionFileRecord } from './provenance-version-writer' +import { requireAgentArtifactVersion } from './provenance-version-kind' const SAFE_SEGMENT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/ @@ -129,9 +130,12 @@ const validateArtifactExecutionInputs = ( } } -type VersionDescriptorRecord = PersistedVersionFileRecord & { +type VersionDescriptorRecord = Omit & { + artifactRunId: string | null state: string messageId: string | null + originKind: string + basedOnVersionId: string | null } type ArtifactProvenanceReadModelOptions = { @@ -186,7 +190,16 @@ class ArtifactProvenanceReadModel { include: { originSession: true, versions: { - where: { state: { in: ['pending', 'finalized'] } }, + where: { + OR: [ + { + originKind: 'agent_generated', + state: { in: ['pending', 'finalized'] } + }, + { originKind: 'user_edit', state: 'finalized' }, + { originKind: 'legacy', state: 'finalized' } + ] + }, orderBy: [{ versionNumber: 'asc' as const }, { id: 'asc' as const }] } } @@ -235,6 +248,7 @@ class ArtifactProvenanceReadModel { where: { id: versionId, artifactId, + originKind: 'agent_generated', state: { in: ['pending', 'finalized'] }, artifact: { is: { projectId, sessionId: appSessionId } } }, @@ -250,11 +264,12 @@ class ArtifactProvenanceReadModel { version = await findVersion() } if (!version) throw new Error(`Artifact Version not found: ${versionId}`) + const agentVersion = requireAgentArtifactVersion(version) const evidenceMirror = await this.readCanonicalMirror( - resolveStorageKey(this.options.storageRoot, version.evidenceStorageKey), - version.evidenceJson, - version.evidenceChecksum, + resolveStorageKey(this.options.storageRoot, agentVersion.evidenceStorageKey), + agentVersion.evidenceJson, + agentVersion.evidenceChecksum, `Artifact Version evidence is corrupt: ${versionId}` ) const evidence = JSON.parse(evidenceMirror) as ArtifactVersionEvidence @@ -293,10 +308,10 @@ class ArtifactProvenanceReadModel { ) const persistedExecution = parseArtifactExecutionSnapshot(executionMirror) validateArtifactExecutionSnapshot(persistedExecution, { - rootFrameId: version.rootFrameId, - agentFrameId: version.agentFrameId, - messageBranchId: version.messageBranchId, - promptMessageId: version.promptMessageId, + rootFrameId: agentVersion.rootFrameId, + agentFrameId: agentVersion.agentFrameId, + messageBranchId: agentVersion.messageBranchId, + promptMessageId: agentVersion.promptMessageId, producerRunId: version.producerRunId, producerRunIndex: version.producerRunIndex, executionSnapshotChecksum: version.executionSnapshotChecksum, @@ -514,7 +529,7 @@ class ArtifactProvenanceReadModel { return { descriptor: await this.options.projectVersionDescriptor( - version, + agentVersion, projectId, version.artifact.sessionId ), diff --git a/src/main/artifacts/provenance-repository.architecture.test.ts b/src/main/artifacts/provenance-repository.architecture.test.ts index f5b751acf..bf2be5526 100644 --- a/src/main/artifacts/provenance-repository.architecture.test.ts +++ b/src/main/artifacts/provenance-repository.architecture.test.ts @@ -226,6 +226,7 @@ describe('Artifact Provenance repository architecture', () => { it('keeps the established public facade and private projection helpers', () => { expect(methods(facade, 'public')).toEqual( [ + 'activateFinalizedRun', 'createVersion', 'deleteProjectProvenance', 'finalizeRun', @@ -310,6 +311,7 @@ describe('Artifact Provenance repository architecture', () => { expect( Object.fromEntries( [ + 'activateFinalizedRun', 'createVersion', 'finalizeRun', 'getLineage', @@ -329,6 +331,7 @@ describe('Artifact Provenance repository architecture', () => { ].map((method) => [method, delegationTarget(facade, facadeFile, method)]) ) ).toEqual({ + activateFinalizedRun: 'this.messageFinalizer.activateFinalizedRun', createVersion: 'this.versionWriter.writeVersion', finalizeRun: 'this.messageFinalizer.finalizeRun', getLineage: 'this.readModel.getLineage', diff --git a/src/main/artifacts/provenance-repository.test.ts b/src/main/artifacts/provenance-repository.test.ts index 672cf78b8..e64d05738 100644 --- a/src/main/artifacts/provenance-repository.test.ts +++ b/src/main/artifacts/provenance-repository.test.ts @@ -26,6 +26,7 @@ import { ArtifactOwnershipPersistenceRaceError, ArtifactProvenanceRepository } from './provenance-repository' +import { requireAgentArtifactVersion } from './provenance-version-kind' import { ArtifactRepository } from './repository' import { ArtifactWriteBudgetOwner } from './write-budget-owner' @@ -43,6 +44,100 @@ afterEach(async () => { }) describe('artifact provenance repository', () => { + it('projects finalized user edits in lineage without inventing Agent provenance', async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-artifact-user-edit-lineage-')) + const client = createProjectDbClient(storageRoot) + disconnect = () => client.$disconnect() + await migrateApplicationDatabase(client) + const repository = new ArtifactProvenanceRepository({ + storageRoot, + getClient: () => Promise.resolve(client) + }) + await client.fileOriginSession.create({ + data: { projectId: 'project-1', sessionId: 'session-1' } + }) + await client.artifactLineage.create({ + data: { + id: 'artifact-1', + projectId: 'project-1', + sessionId: 'session-1', + normalizedFilename: 'notes.md', + filename: 'notes.md' + } + }) + const baseStorageKey = 'artifacts/project-1/session-1/version-1/content' + const editStorageKey = 'artifacts/project-1/session-1/version-2/content' + await mkdir(dirname(join(storageRoot, baseStorageKey)), { recursive: true }) + await mkdir(dirname(join(storageRoot, editStorageKey)), { recursive: true }) + await writeFile(join(storageRoot, baseStorageKey), 'base') + await writeFile(join(storageRoot, editStorageKey), 'edited') + await client.artifactVersion.create({ + data: { + id: 'version-1', + artifactId: 'artifact-1', + versionNumber: 1, + filename: 'notes.md', + originKind: 'legacy', + state: 'finalized', + contentStorageKey: baseStorageKey, + sizeBytes: 4n, + checksum: 'a'.repeat(64) + } + }) + await client.artifactVersion.create({ + data: { + id: 'version-2', + artifactId: 'artifact-1', + versionNumber: 2, + filename: 'notes.md', + originKind: 'user_edit', + basedOnVersionId: 'version-1', + storageTag: 'v1a2b3c4d', + storedFilename: 'v1a2b3c4d_notes.md', + writeOperationId: 'edit-operation-1', + state: 'finalized', + managedVisibleAt: new Date('2026-08-14T00:00:00.000Z'), + contentStorageKey: editStorageKey, + sizeBytes: 6n, + checksum: 'c'.repeat(64), + createdAt: new Date('2026-08-14T00:00:00.000Z') + } + }) + + await expect( + repository.getLineage({ + projectId: 'project-1', + appSessionId: 'session-1', + artifactId: 'artifact-1' + }) + ).resolves.toMatchObject({ + versions: [ + { versionId: 'version-1', originKind: 'legacy' }, + { + versionId: 'version-2', + originKind: 'user_edit', + basedOnVersionId: 'version-1' + } + ] + }) + await expect( + repository.getVersionProvenance({ + projectId: 'project-1', + appSessionId: 'session-1', + artifactId: 'artifact-1', + versionId: 'version-2' + }) + ).rejects.toThrow('Artifact Version not found: version-2') + await expect( + repository.getVersionProvenance({ + projectId: 'project-1', + appSessionId: 'session-1', + artifactId: 'artifact-1', + versionId: 'version-1' + }) + ).rejects.toThrow('Artifact Version not found: version-1') + }) + it('stores reconstruction cache beside the exact owned immutable Version', async () => { storageRoot = await mkdtemp(join(tmpdir(), 'open-science-artifact-reconstruction-cache-')) const client = createProjectDbClient(storageRoot) @@ -84,6 +179,7 @@ describe('artifact provenance repository', () => { contentStorageKey, evidenceStorageKey: 'artifacts/project-1/session-1/.provenance/versions/version-1/evidence.json', + evidenceSchemaVersion: 1, sizeBytes: BigInt(14), checksum: 'a'.repeat(64), evidenceJson: '{}', @@ -493,9 +589,9 @@ describe('artifact provenance repository', () => { versionId: first.versionId }) ).rejects.toThrow(`Artifact Version not found: ${first.versionId}`) - const firstRow = await client.artifactVersion.findUniqueOrThrow({ - where: { id: first.versionId } - }) + const firstRow = requireAgentArtifactVersion( + await client.artifactVersion.findUniqueOrThrow({ where: { id: first.versionId } }) + ) expect(JSON.parse(firstRow.evidenceJson)).toMatchObject({ agent_name: 'Codex' }) const versions = await client.artifactVersion.findMany({ @@ -656,9 +752,9 @@ describe('artifact provenance repository', () => { artifactRunId: 'artifact-run-1' }) ).resolves.toEqual([expect.objectContaining({ versionId: version.versionId })]) - const row = await client.artifactVersion.findUniqueOrThrow({ - where: { id: version.versionId } - }) + const row = requireAgentArtifactVersion( + await client.artifactVersion.findUniqueOrThrow({ where: { id: version.versionId } }) + ) expect(JSON.parse(row.evidenceJson)).toMatchObject({ producer: { state: 'unavailable', reason: 'producer-not-supplied' }, execution_status: { state: 'unavailable', reason: 'producer-not-supplied' } @@ -1493,6 +1589,7 @@ describe('artifact provenance repository', () => { state: 'staging', contentStorageKey, evidenceStorageKey, + evidenceSchemaVersion: 1, sizeBytes: BigInt(Buffer.byteLength(content)), checksum: contentChecksum, evidenceJson, @@ -1541,6 +1638,7 @@ describe('artifact provenance repository', () => { 'artifacts/project-1/session-1/.provenance/artifact-corrupt/versions/version-recovery-corrupt/content', evidenceStorageKey: 'artifacts/project-1/session-1/.provenance/artifact-corrupt/versions/version-recovery-corrupt/evidence.json', + evidenceSchemaVersion: 1, sizeBytes: BigInt(Buffer.byteLength('expected')), checksum: createHash('sha256').update('expected').digest('hex'), evidenceJson, @@ -1851,7 +1949,9 @@ describe('artifact provenance repository', () => { filename: 'sin.png', contentType: 'image/png' }) - const row = await client.artifactVersion.findUniqueOrThrow({ where: { id: version.versionId } }) + const row = requireAgentArtifactVersion( + await client.artifactVersion.findUniqueOrThrow({ where: { id: version.versionId } }) + ) const evidence = JSON.parse(row.evidenceJson) as Record const execution = JSON.parse(row.executionSnapshotJson ?? '{}') as { producerRunId: string @@ -2077,9 +2177,9 @@ describe('artifact provenance repository', () => { filename: 'inferred.png', contentType: 'image/png' }) - const inferredRow = await client.artifactVersion.findUniqueOrThrow({ - where: { id: inferred.versionId } - }) + const inferredRow = requireAgentArtifactVersion( + await client.artifactVersion.findUniqueOrThrow({ where: { id: inferred.versionId } }) + ) expect(inferredRow).toMatchObject({ producerRunId: null, producerRunIndex: null }) expect(JSON.parse(inferredRow.evidenceJson)).toMatchObject({ producer: { state: 'unavailable', reason: 'producer-not-supplied' }, @@ -2124,9 +2224,9 @@ describe('artifact provenance repository', () => { filename: 'ambiguous.png', contentType: 'image/png' }) - const ambiguousRow = await client.artifactVersion.findUniqueOrThrow({ - where: { id: ambiguous.versionId } - }) + const ambiguousRow = requireAgentArtifactVersion( + await client.artifactVersion.findUniqueOrThrow({ where: { id: ambiguous.versionId } }) + ) expect(ambiguousRow).toMatchObject({ producerRunId: null, producerRunIndex: null }) expect(JSON.parse(ambiguousRow.evidenceJson)).toMatchObject({ producer: { state: 'unavailable', reason: 'producer-source-unverifiable' }, @@ -2461,9 +2561,9 @@ describe('artifact provenance repository', () => { filename: 'sin.png', contentType: 'image/png' }) - const row = await client.artifactVersion.findUniqueOrThrow({ - where: { id: version.versionId } - }) + const row = requireAgentArtifactVersion( + await client.artifactVersion.findUniqueOrThrow({ where: { id: version.versionId } }) + ) expect(row).toMatchObject({ producerRunId: null, producerRunIndex: null }) expect(JSON.parse(row.evidenceJson)).toMatchObject({ @@ -2718,9 +2818,9 @@ describe('artifact provenance repository', () => { filename: 'plot.png', contentType: 'image/png' }) - const inferredRow = await client.artifactVersion.findUniqueOrThrow({ - where: { id: inferred.versionId } - }) + const inferredRow = requireAgentArtifactVersion( + await client.artifactVersion.findUniqueOrThrow({ where: { id: inferred.versionId } }) + ) expect(inferredRow).toMatchObject({ producerRunId: 'notebook-run-owner', producerRunIndex: 0 @@ -2799,9 +2899,11 @@ describe('artifact provenance repository', () => { filename: 'spoof.png', contentType: 'image/png' }) - const spoofedRow = await client.artifactVersion.findUniqueOrThrow({ - where: { id: spoofedObservation.versionId } - }) + const spoofedRow = requireAgentArtifactVersion( + await client.artifactVersion.findUniqueOrThrow({ + where: { id: spoofedObservation.versionId } + }) + ) expect(spoofedRow).toMatchObject({ producerRunId: null, producerRunIndex: null }) expect(JSON.parse(spoofedRow.evidenceJson)).toMatchObject({ producer: { state: 'unavailable', reason: 'producer-source-unverifiable' } @@ -2828,9 +2930,9 @@ describe('artifact provenance repository', () => { filename: 'inline.png', contentType: 'image/png' }) - const inlineRow = await client.artifactVersion.findUniqueOrThrow({ - where: { id: inline.versionId } - }) + const inlineRow = requireAgentArtifactVersion( + await client.artifactVersion.findUniqueOrThrow({ where: { id: inline.versionId } }) + ) expect(inlineRow).toMatchObject({ producerRunId: 'notebook-run-owner', producerRunIndex: 0 @@ -2865,9 +2967,9 @@ describe('artifact provenance repository', () => { filename: 'unobserved-local.png', contentType: 'image/png' }) - const unobservedLocalRow = await client.artifactVersion.findUniqueOrThrow({ - where: { id: unobservedLocal.versionId } - }) + const unobservedLocalRow = requireAgentArtifactVersion( + await client.artifactVersion.findUniqueOrThrow({ where: { id: unobservedLocal.versionId } }) + ) expect(unobservedLocalRow).toMatchObject({ producerRunId: null, producerRunIndex: null }) expect(JSON.parse(unobservedLocalRow.evidenceJson)).toMatchObject({ producer: { state: 'unavailable', reason: 'producer-source-unverifiable' }, @@ -3329,6 +3431,92 @@ describe('artifact provenance repository', () => { expect(replayed.map((version) => version.versionId)).toEqual( finalized.map((version) => version.versionId) ) + await expect( + client.artifactVersion.findMany({ + where: { id: { in: finalized.map((version) => version.versionId) } }, + select: { managedVisibleAt: true } + }) + ).resolves.toEqual([{ managedVisibleAt: null }, { managedVisibleAt: null }]) + await expect( + client.artifactLineage.findUniqueOrThrow({ where: { id: finalizableVersions[0].artifactId } }) + ).resolves.toMatchObject({ currentVersionId: null }) + const competing = await client.artifactVersion.findUniqueOrThrow({ + where: { id: finalized[1].versionId } + }) + await client.artifactVersion.create({ + data: { + id: 'artifact-competing-run-v4', + artifactId: competing.artifactId, + versionNumber: 4, + filename: competing.filename, + originKind: 'agent_generated', + artifactRunId: 'artifact-run-competing', + writeOperationId: 'write-competing-v4', + writeRequestChecksum: 'f'.repeat(64), + rootFrameId: competing.rootFrameId, + agentFrameId: competing.agentFrameId, + messageBranchId: competing.messageBranchId, + runtimeSegmentId: competing.runtimeSegmentId, + promptMessageId: competing.promptMessageId, + notebookSessionId: competing.notebookSessionId, + producerRunId: competing.producerRunId, + producerRunIndex: competing.producerRunIndex, + messageId: null, + state: 'finalized', + managedVisibleAt: null, + contentStorageKey: 'artifacts/project-1/session-1/competing-v4/content', + evidenceStorageKey: 'artifacts/project-1/session-1/competing-v4/evidence.json', + contentType: competing.contentType, + sizeBytes: competing.sizeBytes, + checksum: competing.checksum, + evidenceJson: competing.evidenceJson, + evidenceChecksum: competing.evidenceChecksum, + evidenceSchemaVersion: competing.evidenceSchemaVersion, + executionSnapshotJson: competing.executionSnapshotJson, + executionSnapshotChecksum: competing.executionSnapshotChecksum, + executionSnapshotStorageKey: competing.executionSnapshotStorageKey, + executionSnapshotSchemaVersion: competing.executionSnapshotSchemaVersion + } + }) + await repository.activateFinalizedRun(finalizeRequest) + await expect( + client.artifactLineage.findUniqueOrThrow({ where: { id: finalizableVersions[0].artifactId } }) + ).resolves.toMatchObject({ currentVersionId: finalized[1].versionId }) + await expect( + client.artifactVersion.findMany({ + where: { id: { in: finalized.map((version) => version.versionId) } }, + select: { managedVisibleAt: true } + }) + ).resolves.toEqual([ + { managedVisibleAt: expect.any(Date) }, + { managedVisibleAt: expect.any(Date) } + ]) + const laterUserEditId = 'artifact-user-edit-v5' + await client.artifactVersion.create({ + data: { + id: laterUserEditId, + artifactId: finalizableVersions[0].artifactId, + versionNumber: 5, + filename: common.filename, + originKind: 'user_edit', + basedOnVersionId: finalized[1].versionId, + storageTag: 'vlate0001', + storedFilename: 'vlate0001_sin.png', + state: 'finalized', + contentStorageKey: 'artifacts/project-1/session-1/user-edit-v5/content', + contentType: 'image/png', + sizeBytes: BigInt(4), + checksum: 'd'.repeat(64) + } + }) + await client.artifactLineage.update({ + where: { id: finalizableVersions[0].artifactId }, + data: { currentVersionId: laterUserEditId } + }) + await repository.activateFinalizedRun(finalizeRequest) + await expect( + client.artifactLineage.findUniqueOrThrow({ where: { id: finalizableVersions[0].artifactId } }) + ).resolves.toMatchObject({ currentVersionId: laterUserEditId }) expect( await client.artifactVersion.count({ where: { state: 'finalized', messageId: 'message-1' } @@ -3911,9 +4099,9 @@ describe('artifact provenance repository', () => { source: createPngInlineSource('plot bytes') }) const version = await repository.createVersion(request) - const versionRow = await client.artifactVersion.findUniqueOrThrow({ - where: { id: version.versionId } - }) + const versionRow = requireAgentArtifactVersion( + await client.artifactVersion.findUniqueOrThrow({ where: { id: version.versionId } }) + ) await rm(version.path) await rm(join(storageRoot, ...versionRow.evidenceStorageKey.split('/'))) @@ -4068,6 +4256,17 @@ describe('artifact provenance repository', () => { where: { id: version.versionId }, data: { messageSnapshotId: snapshotId } }) + await client.artifactLineage.update({ + where: { id: version.artifactId }, + data: { currentVersionId: null } + }) + await client.managedFile.deleteMany({ + where: { + projectId: request.projectId, + source: 'artifact', + sourceFileId: version.artifactId + } + }) await client.artifactVersion.delete({ where: { id: version.versionId } }) await expect( @@ -4407,6 +4606,249 @@ describe('artifact provenance repository', () => { ) }) + it('deletes migrated Artifact and Upload graphs with multiple derived Versions', async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-version-chain-delete-')) + const client = createProjectDbClient(storageRoot) + disconnect = () => client.$disconnect() + await migrateApplicationDatabase(client) + const repository = new ArtifactProvenanceRepository({ + storageRoot, + getClient: () => Promise.resolve(client), + compatibilityRepository: new ArtifactRepository(storageRoot) + }) + + await client.fileOriginSession.create({ + data: { projectId: 'project-1', sessionId: 'session-1' } + }) + await client.artifactLineage.create({ + data: { + id: 'artifact-1', + projectId: 'project-1', + sessionId: 'session-1', + normalizedFilename: 'report.md', + filename: 'report.md' + } + }) + await client.artifactVersion.createMany({ + data: [ + { + id: 'artifact-v1', + artifactId: 'artifact-1', + versionNumber: 1, + filename: 'report.md', + originKind: 'legacy', + state: 'finalized', + contentStorageKey: 'artifacts/project-1/session-1/artifact-v1/content', + sizeBytes: 1, + checksum: 'a'.repeat(64) + }, + { + id: 'artifact-v2', + artifactId: 'artifact-1', + versionNumber: 2, + filename: 'report.md', + originKind: 'legacy', + basedOnVersionId: 'artifact-v1', + state: 'finalized', + contentStorageKey: 'artifacts/project-1/session-1/artifact-v2/content', + sizeBytes: 2, + checksum: 'b'.repeat(64) + } + ] + }) + await client.artifactLineage.update({ + where: { id: 'artifact-1' }, + data: { currentVersionId: 'artifact-v2' } + }) + + await client.uploadFile.create({ + data: { + id: 'upload-1', + projectId: 'project-1', + sessionId: 'session-1', + filename: 'input.csv', + originalFilename: 'Input.csv' + } + }) + await client.uploadVersion.createMany({ + data: [ + { + id: 'upload-v1', + uploadFileId: 'upload-1', + versionNumber: 1, + state: 'ready', + contentStorageKey: 'uploads/project-1/session-1/upload-1/v1/content', + filename: 'input.csv', + originalFilename: 'Input.csv', + sizeBytes: 1, + checksum: 'c'.repeat(64) + }, + { + id: 'upload-v2', + uploadFileId: 'upload-1', + versionNumber: 2, + state: 'ready', + originKind: 'user_edit', + basedOnVersionId: 'upload-v1', + storageTag: 'v1a2b3c4d', + storedFilename: 'v1a2b3c4d_input.csv', + contentStorageKey: 'uploads/project-1/session-1/upload-1/v2/content', + filename: 'input.csv', + originalFilename: 'Input.csv', + sizeBytes: 2, + checksum: 'd'.repeat(64) + } + ] + }) + await client.uploadFile.update({ + where: { id: 'upload-1' }, + data: { currentVersionId: 'upload-v2' } + }) + for (const storageKeyValue of [ + 'uploads/project-1/session-1/upload-1/v1/content', + 'uploads/project-1/session-1/upload-1/v2/content' + ]) { + const path = join(storageRoot, ...storageKeyValue.split('/')) + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, 'x') + } + + await expect(repository.deleteProjectProvenance('project-1')).resolves.toBeUndefined() + await expect(client.artifactVersion.count()).resolves.toBe(0) + await expect(client.artifactLineage.count()).resolves.toBe(0) + await expect(client.uploadVersion.count()).resolves.toBe(0) + await expect(client.uploadFile.count()).resolves.toBe(0) + await expect(client.fileOriginSession.count()).resolves.toBe(0) + }) + + it('retains every Version write journal until all Artifact and Upload journal bytes are deleted', async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-version-journal-delete-')) + const client = createProjectDbClient(storageRoot) + disconnect = () => client.$disconnect() + await migrateApplicationDatabase(client) + const repository = new ArtifactProvenanceRepository({ + storageRoot, + getClient: () => Promise.resolve(client), + compatibilityRepository: new ArtifactRepository(storageRoot) + }) + const operations = [ + { + operationId: 'operation-artifact-a-staging', + source: 'artifact', + state: 'staging', + contentStorageKey: 'managed-version-journals/project-1/artifact/staging/content' + }, + { + operationId: 'operation-artifact-b-file-ready', + source: 'artifact', + state: 'file_ready', + contentStorageKey: 'managed-version-journals/project-1/artifact/file-ready/content' + }, + { + operationId: 'operation-artifact-c-published', + source: 'artifact', + state: 'published', + contentStorageKey: 'managed-version-journals/project-1/artifact/published/content' + }, + { + operationId: 'operation-upload-a-staging', + source: 'upload', + state: 'staging', + contentStorageKey: 'uploads/project-1/session-1/upload-1/staging/content' + }, + { + operationId: 'operation-upload-b-file-ready', + source: 'upload', + state: 'file_ready', + contentStorageKey: 'uploads/project-1/session-1/upload-1/file-ready/content' + }, + { + operationId: 'operation-upload-c-published', + source: 'upload', + state: 'published', + contentStorageKey: 'uploads/project-1/session-1/upload-1/published/content' + } + ] + await client.fileOriginSession.create({ + data: { projectId: 'project-1', sessionId: 'session-1' } + }) + await client.uploadFile.create({ + data: { + id: 'upload-1', + projectId: 'project-1', + sessionId: 'session-1', + filename: 'notes.txt', + originalFilename: 'Notes.txt', + currentVersionId: null, + versions: { + create: { + id: 'upload-v1', + versionNumber: 1, + state: 'ready', + contentStorageKey: operations[5]!.contentStorageKey, + filename: 'notes.txt', + originalFilename: 'Notes.txt', + sizeBytes: 1, + checksum: 'f'.repeat(64) + } + } + } + }) + await client.uploadFile.update({ + where: { id: 'upload-1' }, + data: { currentVersionId: 'upload-v1' } + }) + await client.managedFileVersionWriteOperation.createMany({ + data: operations.map((operation, index) => ({ + ...operation, + projectId: 'project-1', + sourceFileId: `${operation.source}-1`, + basedOnVersionId: `${operation.source}-v1`, + expectedHeadVersionId: `${operation.source}-v1`, + storageTag: `vjournal${index}`, + storedFilename: `vjournal${index}_notes.txt`, + checksum: String(index).repeat(64), + sizeBytes: 1, + textFormatJson: '{}' + })) + }) + for (const operation of operations) { + const path = join(storageRoot, ...operation.contentStorageKey.split('/')) + await mkdir(dirname(path), { recursive: true }) + if (operation.operationId === 'operation-artifact-b-file-ready') { + await mkdir(path) + } else { + await writeFile(path, 'x') + } + } + + await expect(repository.deleteProjectProvenance('project-1')).rejects.toThrow() + await expect( + client.managedFileVersionWriteOperation.count({ where: { projectId: 'project-1' } }) + ).resolves.toBe(6) + await expect(client.uploadFile.count({ where: { projectId: 'project-1' } })).resolves.toBe(1) + await expect( + readFile(join(storageRoot, ...operations[0]!.contentStorageKey.split('/'))) + ).rejects.toMatchObject({ code: 'ENOENT' }) + await expect( + readFile(join(storageRoot, ...operations[2]!.contentStorageKey.split('/'))) + ).resolves.toEqual(Buffer.from('x')) + + const blockingPath = join(storageRoot, ...operations[1]!.contentStorageKey.split('/')) + await rm(blockingPath, { recursive: true }) + await writeFile(blockingPath, 'x') + await expect(repository.deleteProjectProvenance('project-1')).resolves.toBeUndefined() + + await expect( + client.managedFileVersionWriteOperation.count({ where: { projectId: 'project-1' } }) + ).resolves.toBe(0) + for (const operation of operations) { + await expect( + readFile(join(storageRoot, ...operation.contentStorageKey.split('/'))) + ).rejects.toMatchObject({ code: 'ENOENT' }) + } + }) + it('retains Upload authority when Project byte deletion must be retried', async () => { storageRoot = await mkdtemp(join(tmpdir(), 'open-science-project-upload-delete-retry-')) const client = createProjectDbClient(storageRoot) diff --git a/src/main/artifacts/provenance-repository.ts b/src/main/artifacts/provenance-repository.ts index 8798e5121..d36d315db 100644 --- a/src/main/artifacts/provenance-repository.ts +++ b/src/main/artifacts/provenance-repository.ts @@ -52,6 +52,7 @@ import { ArtifactProvenanceReadModel } from './provenance-read-model' import type { PersistedChatSession } from '../../shared/session-persistence' import { ArtifactProvenanceDependencyReader } from './provenance-dependency-reader' import type { HostLineageDependencyRelation, HostLineageDirection } from '../../shared/host-lineage' +import { requireAgentArtifactVersion } from './provenance-version-kind' import { LOCAL_RESOURCE_BUDGETS, type LocalResourceBudgetOverrides } from '../resource-budget' import { ArtifactWriteBudgetOwner } from './write-budget-owner' import { digestFileWithinBudget } from '../bounded-file-io' @@ -74,6 +75,17 @@ type ArtifactProvenanceRepositoryOptions = { resourceBudgets?: LocalResourceBudgetOverrides } +type ProjectableVersionFileRecord = Omit & { + artifactRunId: string | null +} + +type VersionDescriptorRecord = ProjectableVersionFileRecord & { + state: string + messageId: string | null + originKind: string + basedOnVersionId: string | null +} + export type WriteAppGeneratedArtifactVersionRequest = Omit< CreateArtifactVersionRequest, | 'writeOperationId' @@ -177,6 +189,7 @@ class ArtifactProvenanceRepository { }) this.messageFinalizer = new ArtifactProvenanceMessageFinalizer({ getClient: options.getClient, + now: this.now, loadSession: options.loadSession, projectVersionFile: (version, projectId, appSessionId) => this.toArtifactVersionFile(version, projectId, appSessionId) @@ -354,42 +367,44 @@ class ArtifactProvenanceRepository { include: { artifact: true } }) if (!existing) return undefined + const agentVersion = requireAgentArtifactVersion(existing) const producerMatches = request.producerRunId !== undefined - ? (existing.producerRunId ?? undefined) === request.producerRunId - : existing.producerRunId === null || hasServerInferredProducer(existing.evidenceJson) + ? (agentVersion.producerRunId ?? undefined) === request.producerRunId + : agentVersion.producerRunId === null || + hasServerInferredProducer(agentVersion.evidenceJson) if ( - existing.artifact.projectId !== projectId || - existing.artifact.sessionId !== appSessionId || - existing.artifactRunId !== artifactRunId || - existing.artifact.normalizedFilename !== normalizedFilename || - (existing.contentType ?? undefined) !== request.contentType || + agentVersion.artifact.projectId !== projectId || + agentVersion.artifact.sessionId !== appSessionId || + agentVersion.artifactRunId !== artifactRunId || + agentVersion.artifact.normalizedFilename !== normalizedFilename || + (agentVersion.contentType ?? undefined) !== request.contentType || !producerMatches ) { throw new Error( `Artifact write operation was reused for a different request: ${writeOperationId}` ) } - if (existing.state === 'staging') { + if (agentVersion.state === 'staging') { return this.stagingRecovery.recoverVersion( - existing, + agentVersion, projectId, appSessionId, request.filename, this.stagingRecovery.routingPublisher(projectId, artifactStorageSessionId, request.filename) ) } - if (existing.state !== 'pending' && existing.state !== 'finalized') { + if (agentVersion.state !== 'pending' && agentVersion.state !== 'finalized') { throw new Error(`Artifact write has an invalid lifecycle state: ${writeOperationId}`) } - if (existing.state === 'pending') { + if (agentVersion.state === 'pending') { await this.stagingRecovery.routingPublisher( projectId, artifactStorageSessionId, request.filename - )(existing, { replaceUnroutedBytes: true }) + )(agentVersion, { replaceUnroutedBytes: true }) } - return this.toArtifactVersionFile(existing, projectId, appSessionId) + return this.toArtifactVersionFile(agentVersion, projectId, appSessionId) } async validateFinalizationOwnership(request: FinalizeArtifactVersionsRequest): Promise { @@ -400,6 +415,12 @@ class ArtifactProvenanceRepository { return this.messageFinalizer.finalizeRun(request) } + async activateFinalizedRun( + request: FinalizeArtifactVersionsRequest + ): Promise { + return this.messageFinalizer.activateFinalizedRun(request) + } + async listRunVersions(request: { projectId: string appSessionId: string @@ -411,6 +432,7 @@ class ArtifactProvenanceRepository { const client = await this.options.getClient() const versions = await client.artifactVersion.findMany({ where: { + originKind: 'agent_generated', artifactRunId, state: { in: ['pending', 'finalized'] }, artifact: { is: { projectId, sessionId: appSessionId } } @@ -420,7 +442,9 @@ class ArtifactProvenanceRepository { }) return Promise.all( - versions.map((version) => this.toArtifactVersionFile(version, projectId, appSessionId)) + versions.map((version) => + this.toArtifactVersionFile(requireAgentArtifactVersion(version), projectId, appSessionId) + ) ) } @@ -520,12 +544,18 @@ class ArtifactProvenanceRepository { const versions = await client.artifactVersion.findMany({ where: { id: { in: versionIds }, + originKind: 'agent_generated', state: 'finalized', artifact: { is: { projectId } } }, include: { artifact: true } }) - const versionsById = new Map(versions.map((version) => [version.id, version])) + const versionsById = new Map( + versions.map((version) => { + const agentVersion = requireAgentArtifactVersion(version) + return [agentVersion.id, agentVersion] as const + }) + ) return Promise.all( versionIds.flatMap((versionId) => { @@ -676,15 +706,26 @@ class ArtifactProvenanceRepository { async deleteProjectProvenance(projectIdValue: string): Promise { const projectId = assertSafeSegment(projectIdValue, 'project id') const client = await this.options.getClient() - const uploadVersions = await client.uploadVersion.findMany({ - where: { uploadFile: { is: { projectId } } }, - select: { contentStorageKey: true } - }) + const [uploadVersions, versionWriteOperations] = await Promise.all([ + client.uploadVersion.findMany({ + where: { uploadFile: { is: { projectId } } }, + select: { contentStorageKey: true } + }), + client.managedFileVersionWriteOperation.findMany({ + where: { projectId, source: { in: ['artifact', 'upload'] } }, + orderBy: { operationId: 'asc' }, + select: { contentStorageKey: true } + }) + ]) - // Delete managed Upload bytes while their authority rows still make the operation replayable. - // Any failure leaves the Project deletion intent and storage keys available for a later retry. - for (const version of uploadVersions) { - await rm(resolveStorageKey(this.options.storageRoot, version.contentStorageKey), { + // Journal paths are explicit authority and need not live under a source's conventional root. + // Keep every row until all unique Upload Version and Artifact/Upload journal paths are removed, + // so a partial filesystem failure remains idempotently replayable from the complete journal. + const managedStorageKeys = new Set( + [...uploadVersions, ...versionWriteOperations].map((entry) => entry.contentStorageKey) + ) + for (const contentStorageKey of managedStorageKeys) { + await rm(resolveStorageKey(this.options.storageRoot, contentStorageKey), { force: true }) } @@ -698,6 +739,31 @@ class ArtifactProvenanceRepository { ] } }) + await tx.managedFileVersionWriteOperation.deleteMany({ where: { projectId } }) + await tx.artifactLineage.updateMany({ + where: { projectId }, + data: { currentVersionId: null } + }) + await tx.uploadFile.updateMany({ + where: { projectId }, + data: { currentVersionId: null } + }) + const artifactVersions = await tx.artifactVersion.findMany({ + where: { artifact: { is: { projectId } } }, + orderBy: [{ versionNumber: 'desc' }, { id: 'desc' }], + select: { id: true } + }) + for (const version of artifactVersions) { + await tx.artifactVersion.delete({ where: { id: version.id } }) + } + const uploadVersionRows = await tx.uploadVersion.findMany({ + where: { uploadFile: { is: { projectId } } }, + orderBy: [{ versionNumber: 'desc' }, { id: 'desc' }], + select: { id: true } + }) + for (const version of uploadVersionRows) { + await tx.uploadVersion.delete({ where: { id: version.id } }) + } await tx.artifactLineage.deleteMany({ where: { projectId } }) await tx.uploadFile.deleteMany({ where: { projectId } }) await tx.artifactMessageSnapshot.deleteMany({ where: { projectId } }) @@ -711,7 +777,7 @@ class ArtifactProvenanceRepository { } private async toArtifactVersionFile( - version: PersistedVersionFileRecord, + version: ProjectableVersionFileRecord, projectId: string, appSessionId: string ): Promise { @@ -750,7 +816,7 @@ class ArtifactProvenanceRepository { environment, projectId, sessionId: appSessionId, - runId: version.artifactRunId, + runId: version.artifactRunId ?? undefined, name: version.filename, path: filePath, fileUrl: pathToFileURL(filePath).toString(), @@ -761,7 +827,7 @@ class ArtifactProvenanceRepository { } private async toDescriptor( - version: PersistedVersionFileRecord & { state: string; messageId: string | null }, + version: VersionDescriptorRecord, projectId: string, appSessionId: string ): Promise { @@ -772,7 +838,9 @@ class ArtifactProvenanceRepository { return { ...relocatableFile, state: version.state as 'pending' | 'finalized', - messageId: version.messageId ?? undefined + messageId: version.messageId ?? undefined, + originKind: version.originKind as 'agent_generated' | 'user_edit' | 'legacy', + basedOnVersionId: version.basedOnVersionId ?? undefined } } } diff --git a/src/main/artifacts/provenance-staging-recovery.ts b/src/main/artifacts/provenance-staging-recovery.ts index e6306bb96..7bc7d3327 100644 --- a/src/main/artifacts/provenance-staging-recovery.ts +++ b/src/main/artifacts/provenance-staging-recovery.ts @@ -14,6 +14,7 @@ import type { PublishCompatibilityRouting, StagingArtifactVersionRecord } from './provenance-version-writer' +import { requireAgentArtifactVersion } from './provenance-version-kind' const SAFE_SEGMENT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/ // Reconciliation can also run from a read path while an active writer is between copying bytes and @@ -182,7 +183,11 @@ export class ArtifactProvenanceStagingRecovery { data: { state: 'pending' } }) }) - return this.options.projectVersionFile(recovered, projectId, appSessionId) + return this.options.projectVersionFile( + requireAgentArtifactVersion(recovered), + projectId, + appSessionId + ) } async reconcileSession( @@ -199,13 +204,16 @@ export class ArtifactProvenanceStagingRecovery { quarantinedVersionIds: [] } const client = await this.options.getClient() - const stagingVersions = await client.artifactVersion.findMany({ - where: { - state: 'staging', - artifact: { is: { projectId, sessionId: appSessionId } } - }, - include: { artifact: true } - }) + const stagingVersions = await client.artifactVersion + .findMany({ + where: { + originKind: 'agent_generated', + state: 'staging', + artifact: { is: { projectId, sessionId: appSessionId } } + }, + include: { artifact: true } + }) + .then((rows) => rows.map(requireAgentArtifactVersion)) // A crash can leave a complete staging row after its immutable bytes were copied but before the // final state update. Resume those rows from SQLite authority before scanning unindexed folders. diff --git a/src/main/artifacts/provenance-unindexed-recovery.ts b/src/main/artifacts/provenance-unindexed-recovery.ts index 88eb1e368..e226af0da 100644 --- a/src/main/artifacts/provenance-unindexed-recovery.ts +++ b/src/main/artifacts/provenance-unindexed-recovery.ts @@ -429,6 +429,7 @@ class ArtifactProvenanceUnindexedRecovery { checksum, evidenceJson, evidenceChecksum: sha256(evidenceJson), + evidenceSchemaVersion: 1, executionSnapshotJson, executionSnapshotChecksum, executionSnapshotStorageKey: executionSnapshotJson diff --git a/src/main/artifacts/provenance-version-kind.test.ts b/src/main/artifacts/provenance-version-kind.test.ts new file mode 100644 index 000000000..8a5d34746 --- /dev/null +++ b/src/main/artifacts/provenance-version-kind.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest' + +import { requireAgentArtifactVersion } from './provenance-version-kind' + +const completeAgentVersion = { + originKind: 'agent_generated', + artifactRunId: 'run-1', + rootFrameId: 'root-1', + agentFrameId: 'agent-1', + messageBranchId: 'branch-1', + runtimeSegmentId: 'segment-1', + promptMessageId: 'prompt-1', + evidenceStorageKey: 'artifact/evidence.json', + evidenceJson: '{}', + evidenceChecksum: 'checksum', + evidenceSchemaVersion: 1 +} + +describe('Agent Artifact Version narrowing', () => { + it('returns a complete Agent-generated version with non-null provenance', () => { + expect(requireAgentArtifactVersion({ id: 'version-1', ...completeAgentVersion })).toEqual({ + id: 'version-1', + ...completeAgentVersion + }) + }) + + it.each([ + 'artifactRunId', + 'rootFrameId', + 'agentFrameId', + 'messageBranchId', + 'runtimeSegmentId', + 'promptMessageId', + 'evidenceStorageKey', + 'evidenceJson', + 'evidenceChecksum', + 'evidenceSchemaVersion' + ] as const)('rejects agent_generated when required provenance %s is null', (field) => { + expect(() => + requireAgentArtifactVersion({ + id: 'version-agent-incomplete', + ...completeAgentVersion, + [field]: null + }) + ).toThrow(/does not contain complete Agent provenance/) + }) + + it('rejects a complete non-Agent version based on originKind alone', () => { + expect(() => + requireAgentArtifactVersion({ + id: 'version-edit', + ...completeAgentVersion, + originKind: 'user_edit' + }) + ).toThrow(/does not contain complete Agent provenance/) + }) +}) diff --git a/src/main/artifacts/provenance-version-kind.ts b/src/main/artifacts/provenance-version-kind.ts new file mode 100644 index 000000000..984352dbb --- /dev/null +++ b/src/main/artifacts/provenance-version-kind.ts @@ -0,0 +1,54 @@ +type AgentArtifactVersionProvenance = { + originKind: 'agent_generated' + artifactRunId: string + rootFrameId: string + agentFrameId: string + messageBranchId: string + runtimeSegmentId: string + promptMessageId: string + evidenceStorageKey: string + evidenceJson: string + evidenceChecksum: string + evidenceSchemaVersion: number +} + +type ArtifactVersionWithNullableAgentProvenance = { + originKind: string + artifactRunId: string | null + rootFrameId: string | null + agentFrameId: string | null + messageBranchId: string | null + runtimeSegmentId: string | null + promptMessageId: string | null + evidenceStorageKey: string | null + evidenceJson: string | null + evidenceChecksum: string | null + evidenceSchemaVersion: number | null +} + +const requireAgentArtifactVersion = ( + version: T +): T & AgentArtifactVersionProvenance => { + if ( + version.originKind !== 'agent_generated' || + !version.artifactRunId || + !version.rootFrameId || + !version.agentFrameId || + !version.messageBranchId || + !version.runtimeSegmentId || + !version.promptMessageId || + !version.evidenceStorageKey || + !version.evidenceJson || + !version.evidenceChecksum || + version.evidenceSchemaVersion === null + ) { + throw new Error('Artifact Version does not contain complete Agent provenance.') + } + return version as T & AgentArtifactVersionProvenance +} + +export { + requireAgentArtifactVersion, + type AgentArtifactVersionProvenance, + type ArtifactVersionWithNullableAgentProvenance +} diff --git a/src/main/artifacts/provenance-version-writer.ts b/src/main/artifacts/provenance-version-writer.ts index 6828209b2..e3ba5e923 100644 --- a/src/main/artifacts/provenance-version-writer.ts +++ b/src/main/artifacts/provenance-version-writer.ts @@ -14,6 +14,7 @@ import type { PreparedArtifactVersionPersistence } from './provenance-producer-capture' import type { ArtifactRepository } from './repository' +import { requireAgentArtifactVersion } from './provenance-version-kind' import type { ArtifactWriteBudgetOwner } from './write-budget-owner' import { assertDiskReserve, copyFileWithinBudget, digestFileWithinBudget } from '../bounded-file-io' import { @@ -288,32 +289,33 @@ class ArtifactProvenanceVersionWriter { }) if (existing) { + const agentVersion = requireAgentArtifactVersion(existing) if ( - existing.writeRequestChecksum !== writeRequestChecksum || - existing.artifact.projectId !== projectId || - existing.artifact.sessionId !== appSessionId + agentVersion.writeRequestChecksum !== writeRequestChecksum || + agentVersion.artifact.projectId !== projectId || + agentVersion.artifact.sessionId !== appSessionId ) { throw new Error( `Artifact write operation was reused for a different request: ${writeOperationId}` ) } - if (existing.state === 'staging') { + if (agentVersion.state === 'staging') { return this.options.recoverStagingVersion( - existing, + agentVersion, projectId, appSessionId, request.filename, publishCompatibilityRouting ) } - if (existing.state !== 'pending' && existing.state !== 'finalized') { + if (agentVersion.state !== 'pending' && agentVersion.state !== 'finalized') { throw new Error(`Artifact write has an invalid lifecycle state: ${writeOperationId}`) } - if (existing.state === 'pending') { - await publishCompatibilityRouting(existing, { replaceUnroutedBytes: true, signal }) + if (agentVersion.state === 'pending') { + await publishCompatibilityRouting(agentVersion, { replaceUnroutedBytes: true, signal }) } - return this.options.projectVersionFile(existing, projectId, appSessionId) + return this.options.projectVersionFile(agentVersion, projectId, appSessionId) } const pendingFiles = await this.options.compatibilityRepository.listPendingRunFiles({ @@ -411,10 +413,23 @@ class ArtifactProvenanceVersionWriter { }) } - const latest = await transaction.artifactVersion.aggregate({ - where: { artifactId: lineage.id }, - _max: { versionNumber: true } - }) + const [latest, basedOnVersion] = await Promise.all([ + transaction.artifactVersion.aggregate({ + where: { artifactId: lineage.id }, + _max: { versionNumber: true } + }), + transaction.artifactVersion.findFirst({ + where: { + artifactId: lineage.id, + OR: [ + { artifactRunId, state: { in: ['pending', 'finalized'] } }, + ...(lineage.currentVersionId ? [{ id: lineage.currentVersionId }] : []) + ] + }, + orderBy: { versionNumber: 'desc' }, + select: { id: true } + }) + ]) const versionNumber = (latest._max.versionNumber ?? 0) + 1 const contentStorageKey = storageKey( 'artifacts', @@ -486,39 +501,43 @@ class ArtifactProvenanceVersionWriter { LOCAL_RESOURCE_BUDGETS.artifactSessionBytes ) - return transaction.artifactVersion.create({ - data: { - id: versionId, - artifactId: lineage.id, - versionNumber, - filename: request.filename, - artifactRunId, - writeOperationId, - writeRequestChecksum, - rootFrameId: assertSafeSegment(request.rootFrameId, 'root frame id'), - agentFrameId: assertSafeSegment(request.agentFrameId, 'agent frame id'), - messageBranchId: assertSafeSegment(request.messageBranchId, 'message branch id'), - runtimeSegmentId: assertSafeSegment(request.runtimeSegmentId, 'runtime segment id'), - promptMessageId: assertSafeSegment(request.promptMessageId, 'prompt message id'), - notebookSessionId: prepared.notebookSessionId, - producerRunId: prepared.producerRunId, - producerRunIndex: prepared.producerRunIndex, - state: 'staging', - contentStorageKey, - evidenceStorageKey, - contentType: request.contentType, - sizeBytes: BigInt(sizeBytes), - checksum, - evidenceJson: prepared.evidenceJson, - evidenceChecksum: prepared.evidenceChecksum, - executionSnapshotJson: prepared.executionSnapshotJson, - executionSnapshotChecksum: prepared.executionSnapshotChecksum, - executionSnapshotStorageKey, - executionSnapshotSchemaVersion: prepared.executionSnapshotJson ? 2 : undefined, - ...(prepared.inputs ? { inputs: prepared.inputs } : {}), - createdAt - } - }) + return requireAgentArtifactVersion( + await transaction.artifactVersion.create({ + data: { + id: versionId, + artifactId: lineage.id, + versionNumber, + filename: request.filename, + basedOnVersionId: basedOnVersion?.id, + artifactRunId, + writeOperationId, + writeRequestChecksum, + rootFrameId: assertSafeSegment(request.rootFrameId, 'root frame id'), + agentFrameId: assertSafeSegment(request.agentFrameId, 'agent frame id'), + messageBranchId: assertSafeSegment(request.messageBranchId, 'message branch id'), + runtimeSegmentId: assertSafeSegment(request.runtimeSegmentId, 'runtime segment id'), + promptMessageId: assertSafeSegment(request.promptMessageId, 'prompt message id'), + notebookSessionId: prepared.notebookSessionId, + producerRunId: prepared.producerRunId, + producerRunIndex: prepared.producerRunIndex, + state: 'staging', + contentStorageKey, + evidenceStorageKey, + contentType: request.contentType, + sizeBytes: BigInt(sizeBytes), + checksum, + evidenceJson: prepared.evidenceJson, + evidenceChecksum: prepared.evidenceChecksum, + evidenceSchemaVersion: 1, + executionSnapshotJson: prepared.executionSnapshotJson, + executionSnapshotChecksum: prepared.executionSnapshotChecksum, + executionSnapshotStorageKey, + executionSnapshotSchemaVersion: prepared.executionSnapshotJson ? 2 : undefined, + ...(prepared.inputs ? { inputs: prepared.inputs } : {}), + createdAt + } + }) + ) }) ) stagingRowPersisted = true @@ -557,10 +576,12 @@ class ArtifactProvenanceVersionWriter { where: { id: persisted.artifactId }, data: { filename: request.filename } }) - return transaction.artifactVersion.update({ - where: { id: persisted.id }, - data: { state: 'pending' } - }) + return requireAgentArtifactVersion( + await transaction.artifactVersion.update({ + where: { id: persisted.id }, + data: { state: 'pending' } + }) + ) }) return this.options.projectVersionFile(finalized, projectId, appSessionId) } catch (error) { diff --git a/src/main/artifacts/provenance-write-contract.test.ts b/src/main/artifacts/provenance-write-contract.test.ts index b888d9779..65a6fd9cf 100644 --- a/src/main/artifacts/provenance-write-contract.test.ts +++ b/src/main/artifacts/provenance-write-contract.test.ts @@ -1,15 +1,21 @@ import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' import { dirname, join, posix } from 'node:path' import { mkdir, readFile, realpath, stat, writeFile } from 'node:fs/promises' import { afterEach, describe, expect, it } from 'vitest' +import type { ArtifactVersionFile } from '../../shared/artifact-provenance' +import { createLinearConversationGraph } from '../../shared/conversation-graph' import type { NotebookRunInputFile } from '../../shared/notebook' +import type { PersistedChatSession } from '../../shared/session-persistence' import { ImmutableInputAuthority } from '../immutable-input-authority' +import { ManagedFileVersionService } from '../managed-file-versions/service' import { createFrameNotebookLane } from '../notebook/lane-identity' import { createPngBytes } from './artifact-test-fixtures' import * as provenanceModule from './provenance-repository' import { ArtifactProvenanceRepository } from './provenance-repository' +import { requireAgentArtifactVersion } from './provenance-version-kind' import { createArtifactVersionRequest, createProvenanceTestFixture, @@ -51,6 +57,7 @@ const PUBLIC_METHODS = [ 'replayVersion', 'validateFinalizationOwnership', 'finalizeRun', + 'activateFinalizedRun', 'listRunVersions', 'prepareProjectReconciliation', 'reconcileSession', @@ -124,22 +131,25 @@ describe('artifact provenance allocation and write identity', () => { }) expect(rows.map(({ versionNumber }) => versionNumber)).toEqual([1, 2]) for (const [index, row] of rows.entries()) { + const agentRow = requireAgentArtifactVersion(row) const expectedBytes = createPngBytes(index === 0 ? 'version one' : 'version two') - expect(row.checksum).toBe(sha256(expectedBytes)) - expect(row.evidenceChecksum).toBe(sha256(row.evidenceJson)) - expect(row.evidenceJson).toBe(JSON.stringify(canonicalize(JSON.parse(row.evidenceJson)))) - expect(JSON.parse(row.evidenceJson)).toMatchObject({ + expect(agentRow.checksum).toBe(sha256(expectedBytes)) + expect(agentRow.evidenceChecksum).toBe(sha256(agentRow.evidenceJson)) + expect(agentRow.evidenceJson).toBe( + JSON.stringify(canonicalize(JSON.parse(agentRow.evidenceJson))) + ) + expect(JSON.parse(agentRow.evidenceJson)).toMatchObject({ artifact_id: first.artifactId, - version_id: row.id, + version_id: agentRow.id, version_number: index + 1, - checksum: row.checksum + checksum: agentRow.checksum }) await expect( - readFile(join(storageRoot, ...row.contentStorageKey.split('/'))) + readFile(join(storageRoot, ...agentRow.contentStorageKey.split('/'))) ).resolves.toEqual(expectedBytes) await expect( - readFile(join(storageRoot, ...row.evidenceStorageKey.split('/')), 'utf8') - ).resolves.toBe(row.evidenceJson) + readFile(join(storageRoot, ...agentRow.evidenceStorageKey.split('/')), 'utf8') + ).resolves.toBe(agentRow.evidenceJson) } }) @@ -164,6 +174,157 @@ describe('artifact provenance allocation and write identity', () => { await expect(client.artifactVersion.count()).resolves.toBe(1) }) + it('diffs a generated text Version against its published or same-run predecessor', async () => { + const value = await fixture() + const prompt = { + id: 'prompt-1', + role: 'user' as const, + content: 'write a README', + status: 'complete' as const, + eventIds: [], + createdAt: 1, + updatedAt: 1 + } + const assistant = { + id: 'message-1', + role: 'agent' as const, + content: 'done', + status: 'complete' as const, + eventIds: [], + createdAt: 2, + updatedAt: 2 + } + const conversationGraph = createLinearConversationGraph({ + sessionId: 'session-1', + messages: [prompt, assistant], + frameworkId: 'codex', + createdAt: 1, + updatedAt: 2 + }) + const session: PersistedChatSession = { + id: 'session-1', + projectId: 'project-1', + title: 'Generated text versions', + cwd: '/workspace', + status: 'idle', + messages: [prompt, assistant], + conversationGraph, + createdAt: 1, + updatedAt: 2 + } + const repository = new ArtifactProvenanceRepository({ + ...value.repositoryOptions, + loadSession: async () => session + }) + const context = { + rootFrameId: conversationGraph.rootFrameId, + agentFrameId: conversationGraph.activeFrameId, + messageBranchId: conversationGraph.branches[0].id, + runtimeSegmentId: conversationGraph.runtimeSegments[0].id, + promptMessageId: prompt.id + } + const writeVersion = async ( + artifactRunId: string, + writeOperationId: string, + content: string + ): Promise => { + await value.compatibilityRepository.writePendingFile({ + projectId: 'project-1', + sessionId: 'artifact-session-1', + runId: artifactRunId, + filename: 'README.md', + mimeType: 'text/markdown', + source: { kind: 'inline', content, encoding: 'utf8' } + }) + return repository.createVersion({ + projectId: 'project-1', + appSessionId: 'session-1', + artifactStorageSessionId: 'artifact-session-1', + artifactRunId, + writeOperationId, + writeRequestChecksum: sha256(writeOperationId), + ...context, + filename: 'README.md', + contentType: 'text/markdown' + }) + } + const finishRun = async ( + artifactRunId: string, + artifactVersionIds: string[], + activate: boolean + ): Promise => { + const request = { + projectId: 'project-1', + appSessionId: 'session-1', + artifactRunId, + artifactVersionIds, + ...context, + messageId: assistant.id + } + await repository.finalizeRun(request) + if (activate) await repository.activateFinalizedRun(request) + } + + const first = await writeVersion('artifact-run-public', 'write-markdown-1', '# First\n') + await finishRun('artifact-run-public', [first.versionId], true) + + const hidden = await writeVersion('artifact-run-hidden', 'write-markdown-2', '# Hidden\n') + await finishRun('artifact-run-hidden', [hidden.versionId], false) + + const third = await writeVersion('artifact-run-active', 'write-markdown-3', '# Third\n') + const fourth = await writeVersion('artifact-run-active', 'write-markdown-4', '# Fourth\n') + await expect( + value.client.artifactVersion.findMany({ + where: { id: { in: [third.versionId, fourth.versionId] } }, + orderBy: { versionNumber: 'asc' }, + select: { basedOnVersionId: true } + }) + ).resolves.toEqual([ + { basedOnVersionId: first.versionId }, + { basedOnVersionId: third.versionId } + ]) + + await finishRun('artifact-run-active', [third.versionId, fourth.versionId], false) + await expect( + value.client.artifactLineage.findUniqueOrThrow({ where: { id: first.artifactId } }) + ).resolves.toMatchObject({ currentVersionId: first.versionId }) + await expect( + value.client.artifactVersion.findUniqueOrThrow({ where: { id: hidden.versionId } }) + ).resolves.toMatchObject({ state: 'finalized', managedVisibleAt: null }) + + await repository.activateFinalizedRun({ + projectId: 'project-1', + appSessionId: 'session-1', + artifactRunId: 'artifact-run-active', + artifactVersionIds: [third.versionId, fourth.versionId], + ...context, + messageId: assistant.id + }) + + const service = new ManagedFileVersionService({ + storageRoot: value.storageRoot, + getClient: () => Promise.resolve(value.client), + nativeWriteAvailable: true, + readAnchored: (_rootPath, parentPath, name) => readFileSync(join(parentPath, name)) + }) + await expect( + service.diffText({ + source: 'artifact', + projectId: 'project-1', + fileId: fourth.artifactId, + versionId: fourth.versionId, + requestId: 'generated-markdown-diff' + }) + ).resolves.toMatchObject({ + baseVersionId: third.versionId, + selectedVersionId: fourth.versionId, + lines: expect.arrayContaining([ + expect.objectContaining({ kind: 'removed', oldLineNumber: 1 }), + expect.objectContaining({ kind: 'added', newLineNumber: 1 }) + ]) + }) + }) + it('keeps the SQLite lifecycle in staging when an immutable evidence barrier fails', async () => { const value = await fixture() const repository = new ArtifactProvenanceRepository({ @@ -332,9 +493,9 @@ describe('artifact provenance producer and source validation', () => { }) const version = await value.repository.createVersion(request) - const row = await value.client.artifactVersion.findUniqueOrThrow({ - where: { id: version.versionId } - }) + const row = requireAgentArtifactVersion( + await value.client.artifactVersion.findUniqueOrThrow({ where: { id: version.versionId } }) + ) expect(row).toMatchObject({ producerRunId: 'producer-run', producerRunIndex: 0 }) expect(JSON.parse(row.evidenceJson)).toMatchObject({ producer: { @@ -398,9 +559,9 @@ describe('artifact provenance producer and source validation', () => { writeOperationId: 'mtime-operation', writeRequestChecksum: '6'.repeat(64) }) - const row = await value.client.artifactVersion.findUniqueOrThrow({ - where: { id: version.versionId } - }) + const row = requireAgentArtifactVersion( + await value.client.artifactVersion.findUniqueOrThrow({ where: { id: version.versionId } }) + ) expect(row).toMatchObject({ producerRunId: null, producerRunIndex: null }) expect(JSON.parse(row.evidenceJson)).toMatchObject({ producer: { state: 'unavailable', reason: 'producer-source-unverifiable' }, @@ -427,9 +588,9 @@ describe('artifact provenance producer and source validation', () => { sourceFileObservation: { ...observation, sizeBytes: observation.sizeBytes + 1 } }) ) - const row = await value.client.artifactVersion.findUniqueOrThrow({ - where: { id: version.versionId } - }) + const row = requireAgentArtifactVersion( + await value.client.artifactVersion.findUniqueOrThrow({ where: { id: version.versionId } }) + ) expect(row).toMatchObject({ producerRunId: null, producerRunIndex: null }) expect(JSON.parse(row.evidenceJson)).toMatchObject({ producer: { state: 'unavailable', reason: 'producer-source-unverifiable' } @@ -478,9 +639,9 @@ describe('artifact provenance producer and source validation', () => { } }) ) - const row = await value.client.artifactVersion.findUniqueOrThrow({ - where: { id: version.versionId } - }) + const row = requireAgentArtifactVersion( + await value.client.artifactVersion.findUniqueOrThrow({ where: { id: version.versionId } }) + ) expect(row).toMatchObject({ producerRunId: null, producerRunIndex: null }) expect(JSON.parse(row.evidenceJson)).toMatchObject({ diff --git a/src/main/database/application-database.integration.test.ts b/src/main/database/application-database.integration.test.ts index 09d927989..9906c9f30 100644 --- a/src/main/database/application-database.integration.test.ts +++ b/src/main/database/application-database.integration.test.ts @@ -113,7 +113,8 @@ describe('application database (integration)', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ] }) @@ -465,7 +466,8 @@ describe('application database (integration)', () => { sizeBytes: 3n, checksum: 'c'.repeat(64), evidenceJson: '{}', - evidenceChecksum: 'd'.repeat(64) + evidenceChecksum: 'd'.repeat(64), + evidenceSchemaVersion: 1 } }) await client.artifactVersionInput.create({ @@ -587,6 +589,8 @@ describe('application database (integration)', () => { await client.$executeRawUnsafe('DROP TABLE "ComputeAuthOperation"') await client.$executeRawUnsafe('DROP TABLE "ComputeHost"') await client.$executeRawUnsafe('DROP TABLE "VisionEvidence"') + await client.$executeRawUnsafe('DROP TABLE "TagAssignment"') + await client.$executeRawUnsafe('DROP TABLE "Tag"') // Simulate a pre-ledger database: it predates both the migration ledger and Agent Context. await client.$executeRawUnsafe('DROP TABLE "_open_science_migrations"') await client.$executeRawUnsafe('ALTER TABLE "Project" DROP COLUMN "agentContext"') @@ -643,7 +647,8 @@ describe('application database (integration)', () => { sizeBytes: 3n, checksum: 'a'.repeat(64), evidenceJson: '{}', - evidenceChecksum: 'b'.repeat(64) + evidenceChecksum: 'b'.repeat(64), + evidenceSchemaVersion: 1 } }) @@ -668,6 +673,8 @@ describe('application database (integration)', () => { await client.$executeRawUnsafe('DROP TABLE "ComputeAuthOperation"') await client.$executeRawUnsafe('DROP TABLE "ComputeHost"') await client.$executeRawUnsafe('DROP TABLE "VisionEvidence"') + await client.$executeRawUnsafe('DROP TABLE "TagAssignment"') + await client.$executeRawUnsafe('DROP TABLE "Tag"') // Simulate a pre-ledger database: it predates both the migration ledger and Agent Context. await client.$executeRawUnsafe('DROP TABLE "_open_science_migrations"') await client.$executeRawUnsafe('ALTER TABLE "Project" DROP COLUMN "agentContext"') @@ -703,9 +710,12 @@ describe('application database (integration)', () => { $queryRawUnsafe: vi.fn(async () => []) } as unknown as PrismaClient - await expect(applyRuntimeSchemaBaseline(client, { pendingCheckConstraints: [] })).rejects.toBe( - migrationFailure - ) + await expect( + applyRuntimeSchemaBaseline(client, { + pendingCheckConstraints: [], + verificationTarget: 'baseline' + }) + ).rejects.toBe(migrationFailure) }) it('releases and recreates the shared client for exclusive migration validation', async () => { @@ -846,7 +856,8 @@ describe('application database (integration)', () => { sizeBytes: 3n, checksum: 'b'.repeat(64), evidenceJson: '{"schema_version":1}', - evidenceChecksum: 'c'.repeat(64) + evidenceChecksum: 'c'.repeat(64), + evidenceSchemaVersion: 1 } }) @@ -1035,7 +1046,8 @@ describe('application database (integration)', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ] }) diff --git a/src/main/database/database-json-constraints-migration.test.ts b/src/main/database/database-json-constraints-migration.test.ts index e8de0887c..3d98487c1 100644 --- a/src/main/database/database-json-constraints-migration.test.ts +++ b/src/main/database/database-json-constraints-migration.test.ts @@ -145,18 +145,22 @@ describe('database JSON constraints migration', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ], from: '0007_notification_attention_metadata', - to: '0012_tag_ordering' + to: '0013_managed_file_version_foundation' }) await expect(access(`${databasePath}.before-${MIGRATION_ID}.backup`)).rejects.toMatchObject({ code: 'ENOENT' }) await expect( access(`${databasePath}.before-0011_cross_resource_tags.backup`) - ).resolves.toBeUndefined() + ).rejects.toMatchObject({ code: 'ENOENT' }) await expect(access(`${databasePath}.before-0012_tag_ordering.backup`)).resolves.toBeUndefined() + await expect( + access(`${databasePath}.before-0013_managed_file_version_foundation.backup`) + ).resolves.toBeUndefined() await expect( client.$queryRaw>` diff --git a/src/main/database/database-json-constraints.test.ts b/src/main/database/database-json-constraints.test.ts index 521dc176a..0a2e44e4f 100644 --- a/src/main/database/database-json-constraints.test.ts +++ b/src/main/database/database-json-constraints.test.ts @@ -53,10 +53,11 @@ describe('database JSON and remaining domain constraints', () => { `INSERT INTO "ArtifactVersion" ( "id","artifactId","versionNumber","filename","artifactRunId","rootFrameId","agentFrameId", "messageBranchId","runtimeSegmentId","promptMessageId","state","contentStorageKey", - "evidenceStorageKey","sizeBytes","checksum","evidenceJson","evidenceChecksum","updatedAt" + "evidenceStorageKey","sizeBytes","checksum","evidenceJson","evidenceChecksum", + "evidenceSchemaVersion","updatedAt" ) VALUES ( 'version','lineage',1,'result.txt','run','root','agent','branch','segment','prompt','staging', - 'content','evidence.json',0,'checksum','{}','evidence-checksum',CURRENT_TIMESTAMP + 'content','evidence.json',0,'checksum','{}','evidence-checksum',1,CURRENT_TIMESTAMP )` ) await client.$executeRawUnsafe( diff --git a/src/main/database/database-startup-logging.test.ts b/src/main/database/database-startup-logging.test.ts index b50138b0a..a2b2f590d 100644 --- a/src/main/database/database-startup-logging.test.ts +++ b/src/main/database/database-startup-logging.test.ts @@ -109,7 +109,8 @@ describe('database startup logging', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ], adoptedLegacy: true }) diff --git a/src/main/database/generated/runtime-schema.ts b/src/main/database/generated/runtime-schema.ts index a5fa0c6f3..bca468baf 100644 --- a/src/main/database/generated/runtime-schema.ts +++ b/src/main/database/generated/runtime-schema.ts @@ -175,8 +175,10 @@ const RUNTIME_SCHEMA_TABLE_DDLS = [ "sessionId" TEXT NOT NULL, "normalizedFilename" TEXT NOT NULL, "filename" TEXT NOT NULL, + "currentVersionId" TEXT, "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, "updatedAt" DATETIME NOT NULL, + CONSTRAINT "ArtifactLineage_id_currentVersionId_fkey" FOREIGN KEY ("id", "currentVersionId") REFERENCES "ArtifactVersion" ("artifactId", "id") ON DELETE RESTRICT ON UPDATE CASCADE, CONSTRAINT "ArtifactLineage_projectId_sessionId_fkey" FOREIGN KEY ("projectId", "sessionId") REFERENCES "FileOriginSession" ("projectId", "sessionId") ON DELETE RESTRICT ON UPDATE CASCADE );`, `CREATE TABLE IF NOT EXISTS "UploadFile" ( @@ -185,8 +187,10 @@ const RUNTIME_SCHEMA_TABLE_DDLS = [ "sessionId" TEXT NOT NULL, "filename" TEXT NOT NULL, "originalFilename" TEXT NOT NULL, + "currentVersionId" TEXT, "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, "updatedAt" DATETIME NOT NULL, + CONSTRAINT "UploadFile_id_currentVersionId_fkey" FOREIGN KEY ("id", "currentVersionId") REFERENCES "UploadVersion" ("uploadFileId", "id") ON DELETE RESTRICT ON UPDATE CASCADE, CONSTRAINT "UploadFile_projectId_sessionId_fkey" FOREIGN KEY ("projectId", "sessionId") REFERENCES "FileOriginSession" ("projectId", "sessionId") ON DELETE RESTRICT ON UPDATE CASCADE );`, `CREATE TABLE IF NOT EXISTS "UploadVersion" ( @@ -194,6 +198,11 @@ const RUNTIME_SCHEMA_TABLE_DDLS = [ "uploadFileId" TEXT NOT NULL, "versionNumber" INTEGER NOT NULL, "state" TEXT NOT NULL DEFAULT 'staging', + "originKind" TEXT NOT NULL DEFAULT 'user_upload', + "basedOnVersionId" TEXT, + "storageTag" TEXT, + "storedFilename" TEXT, + "writeOperationId" TEXT, "contentStorageKey" TEXT NOT NULL, "filename" TEXT NOT NULL, "originalFilename" TEXT NOT NULL, @@ -204,7 +213,10 @@ const RUNTIME_SCHEMA_TABLE_DDLS = [ "registeredAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, "updatedAt" DATETIME NOT NULL, CONSTRAINT "UploadVersion_uploadFileId_fkey" FOREIGN KEY ("uploadFileId") REFERENCES "UploadFile" ("id") ON DELETE CASCADE ON UPDATE CASCADE, - CONSTRAINT "UploadVersion_state_check" CHECK ("state" IN ('staging', 'ready')) + CONSTRAINT "UploadVersion_uploadFileId_basedOnVersionId_fkey" FOREIGN KEY ("uploadFileId", "basedOnVersionId") REFERENCES "UploadVersion" ("uploadFileId", "id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "UploadVersion_state_check" CHECK ("state" IN ('staging', 'ready')), + CONSTRAINT "UploadVersion_originKind_check" CHECK ("originKind" IN ('user_upload', 'user_edit', 'legacy')), + CONSTRAINT "UploadVersion_userEdit_check" CHECK (("originKind" <> 'user_edit' OR ("state" = 'ready' AND "basedOnVersionId" IS NOT NULL AND "storageTag" IS NOT NULL AND "storedFilename" IS NOT NULL))) );`, `CREATE TABLE IF NOT EXISTS "VisionEvidence" ( "id" TEXT NOT NULL PRIMARY KEY, @@ -254,28 +266,33 @@ const RUNTIME_SCHEMA_TABLE_DDLS = [ "artifactId" TEXT NOT NULL, "versionNumber" INTEGER NOT NULL, "filename" TEXT NOT NULL, - "artifactRunId" TEXT NOT NULL, + "originKind" TEXT NOT NULL DEFAULT 'agent_generated', + "basedOnVersionId" TEXT, + "storageTag" TEXT, + "storedFilename" TEXT, + "artifactRunId" TEXT, "writeOperationId" TEXT, "writeRequestChecksum" TEXT, - "rootFrameId" TEXT NOT NULL, - "agentFrameId" TEXT NOT NULL, - "messageBranchId" TEXT NOT NULL, - "runtimeSegmentId" TEXT NOT NULL, - "promptMessageId" TEXT NOT NULL, + "rootFrameId" TEXT, + "agentFrameId" TEXT, + "messageBranchId" TEXT, + "runtimeSegmentId" TEXT, + "promptMessageId" TEXT, "notebookSessionId" TEXT, "producerRunId" TEXT, "producerRunIndex" INTEGER, "messageId" TEXT, "messageSnapshotId" TEXT, "state" TEXT NOT NULL DEFAULT 'staging', + "managedVisibleAt" DATETIME, "contentStorageKey" TEXT NOT NULL, - "evidenceStorageKey" TEXT NOT NULL, + "evidenceStorageKey" TEXT, "contentType" TEXT, "sizeBytes" BIGINT NOT NULL, "checksum" TEXT NOT NULL, - "evidenceJson" TEXT NOT NULL, - "evidenceChecksum" TEXT NOT NULL, - "evidenceSchemaVersion" INTEGER NOT NULL DEFAULT 1, + "evidenceJson" TEXT, + "evidenceChecksum" TEXT, + "evidenceSchemaVersion" INTEGER, "executionSnapshotJson" TEXT, "executionSnapshotChecksum" TEXT, "executionSnapshotStorageKey" TEXT, @@ -283,12 +300,36 @@ const RUNTIME_SCHEMA_TABLE_DDLS = [ "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, "updatedAt" DATETIME NOT NULL, CONSTRAINT "ArtifactVersion_artifactId_fkey" FOREIGN KEY ("artifactId") REFERENCES "ArtifactLineage" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT "ArtifactVersion_artifactId_basedOnVersionId_fkey" FOREIGN KEY ("artifactId", "basedOnVersionId") REFERENCES "ArtifactVersion" ("artifactId", "id") ON DELETE RESTRICT ON UPDATE CASCADE, CONSTRAINT "ArtifactVersion_messageSnapshotId_fkey" FOREIGN KEY ("messageSnapshotId") REFERENCES "ArtifactMessageSnapshot" ("id") ON DELETE SET NULL ON UPDATE CASCADE, CONSTRAINT "ArtifactVersion_state_check" CHECK ("state" IN ('staging', 'pending', 'finalized')), CONSTRAINT "ArtifactVersion_filename_check" CHECK (length("filename") > 0), - CONSTRAINT "ArtifactVersion_evidenceJson_check" CHECK (json_valid("evidenceJson") AND json_type("evidenceJson") = 'object'), + CONSTRAINT "ArtifactVersion_originKind_check" CHECK ("originKind" IN ('agent_generated', 'user_edit', 'legacy')), + CONSTRAINT "ArtifactVersion_provenance_check" CHECK ((("originKind" = 'agent_generated' AND "artifactRunId" IS NOT NULL AND "rootFrameId" IS NOT NULL AND "agentFrameId" IS NOT NULL AND "messageBranchId" IS NOT NULL AND "runtimeSegmentId" IS NOT NULL AND "promptMessageId" IS NOT NULL AND "evidenceStorageKey" IS NOT NULL AND "evidenceJson" IS NOT NULL AND "evidenceChecksum" IS NOT NULL AND "evidenceSchemaVersion" IS NOT NULL) OR ("originKind" = 'user_edit' AND "state" = 'finalized' AND "basedOnVersionId" IS NOT NULL AND "storageTag" IS NOT NULL AND "storedFilename" IS NOT NULL AND "artifactRunId" IS NULL AND "writeRequestChecksum" IS NULL AND "rootFrameId" IS NULL AND "agentFrameId" IS NULL AND "messageBranchId" IS NULL AND "runtimeSegmentId" IS NULL AND "promptMessageId" IS NULL AND "notebookSessionId" IS NULL AND "producerRunId" IS NULL AND "producerRunIndex" IS NULL AND "messageId" IS NULL AND "messageSnapshotId" IS NULL AND "evidenceStorageKey" IS NULL AND "evidenceJson" IS NULL AND "evidenceChecksum" IS NULL AND "evidenceSchemaVersion" IS NULL AND "executionSnapshotJson" IS NULL AND "executionSnapshotChecksum" IS NULL AND "executionSnapshotStorageKey" IS NULL AND "executionSnapshotSchemaVersion" IS NULL) OR "originKind" = 'legacy')), + CONSTRAINT "ArtifactVersion_evidenceJson_check" CHECK ("evidenceJson" IS NULL OR (json_valid("evidenceJson") AND json_type("evidenceJson") = 'object')), CONSTRAINT "ArtifactVersion_executionSnapshotJson_check" CHECK ("executionSnapshotJson" IS NULL OR (json_valid("executionSnapshotJson") AND json_type("executionSnapshotJson") = 'object')), CONSTRAINT "ArtifactVersion_executionSnapshotBundle_check" CHECK ((("executionSnapshotJson" IS NULL AND "executionSnapshotChecksum" IS NULL AND "executionSnapshotStorageKey" IS NULL AND "executionSnapshotSchemaVersion" IS NULL) OR ("executionSnapshotJson" IS NOT NULL AND "executionSnapshotChecksum" IS NOT NULL AND "executionSnapshotStorageKey" IS NOT NULL AND "executionSnapshotSchemaVersion" IS NOT NULL))) +);`, + `CREATE TABLE IF NOT EXISTS "ManagedFileVersionWriteOperation" ( + "operationId" TEXT NOT NULL PRIMARY KEY, + "source" TEXT NOT NULL, + "projectId" TEXT NOT NULL, + "sourceFileId" TEXT NOT NULL, + "basedOnVersionId" TEXT NOT NULL, + "expectedHeadVersionId" TEXT NOT NULL, + "state" TEXT NOT NULL DEFAULT 'staging', + "storageTag" TEXT NOT NULL, + "storedFilename" TEXT NOT NULL, + "contentStorageKey" TEXT NOT NULL, + "checksum" TEXT NOT NULL, + "sizeBytes" BIGINT NOT NULL, + "textFormatJson" TEXT NOT NULL, + "resultVersionId" TEXT, + "errorCode" TEXT, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "ManagedFileVersionWriteOperation_source_check" CHECK ("source" IN ('artifact', 'upload')), + CONSTRAINT "ManagedFileVersionWriteOperation_state_check" CHECK ("state" IN ('staging', 'file_ready', 'published', 'conflict', 'failed')) );`, `CREATE TABLE IF NOT EXISTS "ArtifactVersionInput" ( "id" TEXT NOT NULL PRIMARY KEY, @@ -503,23 +544,36 @@ const RUNTIME_SCHEMA_INDEX_DDLS = [ `CREATE UNIQUE INDEX IF NOT EXISTS "ManagedFile_projectId_source_storageKey_key" ON "ManagedFile"("projectId", "source", "storageKey");`, `CREATE INDEX IF NOT EXISTS "ManagedFileSessionSync_projectId_deletedAt_groupSortAtMs_sessionId_idx" ON "ManagedFileSessionSync"("projectId", "deletedAt", "groupSortAtMs", "sessionId");`, `CREATE INDEX IF NOT EXISTS "FileOriginSession_projectId_state_idx" ON "FileOriginSession"("projectId", "state");`, + `CREATE UNIQUE INDEX IF NOT EXISTS "ArtifactLineage_currentVersionId_key" ON "ArtifactLineage"("currentVersionId");`, `CREATE INDEX IF NOT EXISTS "ArtifactLineage_projectId_sessionId_idx" ON "ArtifactLineage"("projectId", "sessionId");`, `CREATE UNIQUE INDEX IF NOT EXISTS "ArtifactLineage_projectId_sessionId_normalizedFilename_key" ON "ArtifactLineage"("projectId", "sessionId", "normalizedFilename");`, + `CREATE UNIQUE INDEX IF NOT EXISTS "ArtifactLineage_id_currentVersionId_key" ON "ArtifactLineage"("id", "currentVersionId");`, + `CREATE UNIQUE INDEX IF NOT EXISTS "UploadFile_currentVersionId_key" ON "UploadFile"("currentVersionId");`, `CREATE INDEX IF NOT EXISTS "UploadFile_projectId_sessionId_idx" ON "UploadFile"("projectId", "sessionId");`, + `CREATE UNIQUE INDEX IF NOT EXISTS "UploadFile_id_currentVersionId_key" ON "UploadFile"("id", "currentVersionId");`, + `CREATE UNIQUE INDEX IF NOT EXISTS "UploadVersion_writeOperationId_key" ON "UploadVersion"("writeOperationId");`, + `CREATE UNIQUE INDEX IF NOT EXISTS "UploadVersion_contentStorageKey_key" ON "UploadVersion"("contentStorageKey");`, `CREATE INDEX IF NOT EXISTS "UploadVersion_uploadFileId_state_registeredAt_idx" ON "UploadVersion"("uploadFileId", "state", "registeredAt");`, `CREATE UNIQUE INDEX IF NOT EXISTS "UploadVersion_uploadFileId_versionNumber_key" ON "UploadVersion"("uploadFileId", "versionNumber");`, + `CREATE UNIQUE INDEX IF NOT EXISTS "UploadVersion_uploadFileId_id_key" ON "UploadVersion"("uploadFileId", "id");`, `CREATE INDEX IF NOT EXISTS "VisionEvidence_projectId_sessionId_idx" ON "VisionEvidence"("projectId", "sessionId");`, `CREATE INDEX IF NOT EXISTS "VisionEvidence_sessionId_idx" ON "VisionEvidence"("sessionId");`, `CREATE INDEX IF NOT EXISTS "VisionEvidence_uploadVersionId_idx" ON "VisionEvidence"("uploadVersionId");`, `CREATE INDEX IF NOT EXISTS "ArtifactMessageSnapshot_projectId_sessionId_state_idx" ON "ArtifactMessageSnapshot"("projectId", "sessionId", "state");`, `CREATE UNIQUE INDEX IF NOT EXISTS "ArtifactMessageSnapshot_projectId_sessionId_agentFrameId_messageBranchId_terminalMessageId_key" ON "ArtifactMessageSnapshot"("projectId", "sessionId", "agentFrameId", "messageBranchId", "terminalMessageId");`, `CREATE UNIQUE INDEX IF NOT EXISTS "ArtifactVersion_writeOperationId_key" ON "ArtifactVersion"("writeOperationId");`, + `CREATE UNIQUE INDEX IF NOT EXISTS "ArtifactVersion_contentStorageKey_key" ON "ArtifactVersion"("contentStorageKey");`, `CREATE INDEX IF NOT EXISTS "ArtifactVersion_artifactId_createdAt_idx" ON "ArtifactVersion"("artifactId", "createdAt");`, `CREATE INDEX IF NOT EXISTS "ArtifactVersion_artifactRunId_state_idx" ON "ArtifactVersion"("artifactRunId", "state");`, `CREATE INDEX IF NOT EXISTS "ArtifactVersion_rootFrameId_agentFrameId_messageBranchId_promptMessageId_idx" ON "ArtifactVersion"("rootFrameId", "agentFrameId", "messageBranchId", "promptMessageId");`, `CREATE INDEX IF NOT EXISTS "ArtifactVersion_messageId_idx" ON "ArtifactVersion"("messageId");`, `CREATE INDEX IF NOT EXISTS "ArtifactVersion_messageSnapshotId_idx" ON "ArtifactVersion"("messageSnapshotId");`, `CREATE UNIQUE INDEX IF NOT EXISTS "ArtifactVersion_artifactId_versionNumber_key" ON "ArtifactVersion"("artifactId", "versionNumber");`, + `CREATE UNIQUE INDEX IF NOT EXISTS "ArtifactVersion_artifactId_id_key" ON "ArtifactVersion"("artifactId", "id");`, + `CREATE UNIQUE INDEX IF NOT EXISTS "ManagedFileVersionWriteOperation_contentStorageKey_key" ON "ManagedFileVersionWriteOperation"("contentStorageKey");`, + `CREATE UNIQUE INDEX IF NOT EXISTS "ManagedFileVersionWriteOperation_resultVersionId_key" ON "ManagedFileVersionWriteOperation"("resultVersionId");`, + `CREATE INDEX IF NOT EXISTS "ManagedFileVersionWriteOperation_source_sourceFileId_state_idx" ON "ManagedFileVersionWriteOperation"("source", "sourceFileId", "state");`, + `CREATE INDEX IF NOT EXISTS "ManagedFileVersionWriteOperation_projectId_state_createdAt_idx" ON "ManagedFileVersionWriteOperation"("projectId", "state", "createdAt");`, `CREATE INDEX IF NOT EXISTS "ArtifactVersionInput_sourceKind_inputFileVersionId_idx" ON "ArtifactVersionInput"("sourceKind", "inputFileVersionId");`, `CREATE INDEX IF NOT EXISTS "ArtifactVersionInput_sourceArtifactVersionId_idx" ON "ArtifactVersionInput"("sourceArtifactVersionId");`, `CREATE INDEX IF NOT EXISTS "ArtifactVersionInput_sourceUploadVersionId_idx" ON "ArtifactVersionInput"("sourceUploadVersionId");`, @@ -567,6 +621,7 @@ const RUNTIME_SCHEMA_TABLES = [ 'VisionEvidence', 'ArtifactMessageSnapshot', 'ArtifactVersion', + 'ManagedFileVersionWriteOperation', 'ArtifactVersionInput', 'ReviewFindingDisposition', 'ReviewScopeSnapshot', diff --git a/src/main/database/legacy-baseline-adapter.ts b/src/main/database/legacy-baseline-adapter.ts index 2c9d6f0cc..85b3b62cf 100644 --- a/src/main/database/legacy-baseline-adapter.ts +++ b/src/main/database/legacy-baseline-adapter.ts @@ -1,6 +1,8 @@ import { Prisma, type PrismaClient } from '@prisma/client' import { + RUNTIME_SCHEMA_INDEX_DDLS as CURRENT_RUNTIME_SCHEMA_INDEX_DDLS, + RUNTIME_SCHEMA_TABLE_DDLS as CURRENT_RUNTIME_SCHEMA_TABLE_DDLS, RUNTIME_SCHEMA_TABLES as CURRENT_RUNTIME_SCHEMA_TABLES, RUNTIME_SCHEMA_TARGET_SQL as CURRENT_RUNTIME_SCHEMA_TARGET_SQL } from './generated/runtime-schema' @@ -16,7 +18,8 @@ import { import { applySqliteCheckConstraints, findPendingSqliteCheckConstraints, - type SqliteCheckConstraintMigration + type SqliteCheckConstraintMigration, + type SqliteMigrationOperation } from './sqlite-schema-migrations' import { DatabaseValidationError } from './database-validation-error' import { migrationSqlExecutor } from './migration-sql-executor' @@ -342,6 +345,12 @@ const TARGET_TABLES = createTargetTables(RUNTIME_SCHEMA_BASELINE_TARGET_SQL) const TARGET_INDEXES = createTargetIndexes(RUNTIME_SCHEMA_BASELINE_TARGET_SQL) const CURRENT_TARGET_TABLES = createTargetTables(CURRENT_RUNTIME_SCHEMA_TARGET_SQL) const CURRENT_TARGET_INDEXES = createTargetIndexes(CURRENT_RUNTIME_SCHEMA_TARGET_SQL) +const CURRENT_RUNTIME_SCHEMA_TABLE_DDL_BY_NAME = new Map( + CURRENT_RUNTIME_SCHEMA_TABLE_DDLS.flatMap((ddl) => { + const parsed = parseTargetTable(ddl) + return parsed ? [[parsed[0], ddl] as const] : [] + }) +) // Non-exact baseline adoption may recognize only released suffix FKs named here. Do not derive this // list from the generated current target: doing so would silently admit every future suffix FK. const NON_EXACT_BASELINE_FOREIGN_KEY_ALLOWLIST: ReadonlyMap = @@ -438,6 +447,98 @@ const addColumnIfMissing = async ( // Creates the schema if missing. Idempotent; no projects are seeded, so a fresh install starts empty. type PreparedRuntimeSchemaBaseline = { pendingCheckConstraints: readonly SqliteCheckConstraintMigration[] + verificationTarget: 'baseline' | 'current' +} + +const CURRENT_PROVENANCE_CHECK_CONSTRAINT_MIGRATIONS: readonly SqliteCheckConstraintMigration[] = + PROVENANCE_CHECK_CONSTRAINT_MIGRATIONS.map((migration) => { + const target = CURRENT_TARGET_TABLES.get(migration.tableName) + const canonicalTableDdl = CURRENT_RUNTIME_SCHEMA_TABLE_DDL_BY_NAME.get(migration.tableName) + if (!target || !canonicalTableDdl) { + throw new Error(`Current SQLite schema is missing ${migration.tableName}.`) + } + return { + ...migration, + constraintNames: [...target.checks.keys()], + canonicalTableDdl + } + }) + +const hasCurrentManagedFileVersionFoundation = async (client: PrismaClient): Promise => { + const requiredColumns = [ + ['ArtifactLineage', 'currentVersionId'], + ['UploadFile', 'currentVersionId'], + ['ArtifactVersion', 'originKind'], + ['ArtifactVersion', 'basedOnVersionId'], + ['ArtifactVersion', 'storageTag'], + ['ArtifactVersion', 'storedFilename'], + ['UploadVersion', 'originKind'], + ['UploadVersion', 'basedOnVersionId'], + ['UploadVersion', 'storageTag'], + ['UploadVersion', 'storedFilename'] + ] as const + const present = await Promise.all( + requiredColumns.map(([tableName, columnName]) => hasTableColumn(client, tableName, columnName)) + ) + return present.every(Boolean) +} + +type AdaptedMigrationOperations = { + operations: readonly SqliteMigrationOperation[] + currentTableNames: readonly string[] +} + +const adaptMigrationOperationsForCurrentSchema = async ( + client: PrismaClient, + operations: readonly SqliteMigrationOperation[] +): Promise => { + const adaptedOperations: SqliteMigrationOperation[] = [] + const currentTableNames = new Set() + for (const operation of operations) { + if (operation.kind !== 'rebuild-table-set') { + adaptedOperations.push(operation) + continue + } + const adaptedTableNames = new Set() + const tables: (typeof operation.tables)[number][] = [] + for (const table of operation.tables) { + const current = CURRENT_TARGET_TABLES.get(table.tableName) + const canonicalTableDdl = CURRENT_RUNTIME_SCHEMA_TABLE_DDL_BY_NAME.get(table.tableName) + if (!current || !canonicalTableDdl) { + throw new Error(`Current SQLite schema is missing ${table.tableName}.`) + } + const columns = await migrationSqlExecutor.query>( + client, + `PRAGMA table_info(${quoteSqliteIdentifier(table.tableName)})` + ) + const existingColumns = new Set(columns.map(({ name }) => name)) + if ([...current.columns.keys()].every((column) => existingColumns.has(column))) { + adaptedTableNames.add(table.tableName) + currentTableNames.add(table.tableName) + tables.push({ + ...table, + canonicalTableDdl, + columns: [...current.columns.keys()] + }) + } else { + tables.push(table) + } + } + const retainedIndexes = operation.indexes.filter((ddl) => { + const target = [...createTargetIndexes([ddl]).values()][0] + return target ? !adaptedTableNames.has(target.tableName) : true + }) + const currentIndexes = CURRENT_RUNTIME_SCHEMA_INDEX_DDLS.filter((ddl) => { + const current = [...createTargetIndexes([ddl]).values()][0] + return current ? adaptedTableNames.has(current.tableName) : false + }) + adaptedOperations.push({ + ...operation, + tables, + indexes: [...retainedIndexes, ...currentIndexes] + }) + } + return { operations: adaptedOperations, currentTableNames: [...currentTableNames] } } const classifyLegacySchema = async (client: PrismaClient): Promise => { @@ -531,11 +632,17 @@ const prepareRuntimeSchemaBaseline = async ( client: PrismaClient ): Promise => { await classifyLegacySchema(client) + const verificationTarget = (await hasCurrentManagedFileVersionFoundation(client)) + ? 'current' + : 'baseline' return { pendingCheckConstraints: await findPendingSqliteCheckConstraints( client, - PROVENANCE_CHECK_CONSTRAINT_MIGRATIONS - ) + verificationTarget === 'current' + ? CURRENT_PROVENANCE_CHECK_CONSTRAINT_MIGRATIONS + : PROVENANCE_CHECK_CONSTRAINT_MIGRATIONS + ), + verificationTarget } } @@ -878,11 +985,35 @@ const verifyRuntimeSchemaBaseline = ( const verifyCurrentRuntimeSchema = (client: PrismaClient): Promise => verifyRuntimeSchemaTarget(client, CURRENT_SCHEMA_TARGET, true) +const verifyCurrentRuntimeSchemaTables = ( + client: PrismaClient, + tableNames: readonly string[] +): Promise => { + const selectedTables = new Set(tableNames) + return verifyRuntimeSchemaTarget( + client, + { + tableNames, + tables: new Map( + [...CURRENT_TARGET_TABLES].filter(([tableName]) => selectedTables.has(tableName)) + ), + indexes: new Map( + [...CURRENT_TARGET_INDEXES].filter(([, index]) => selectedTables.has(index.tableName)) + ) + }, + false + ) +} + const applyRuntimeSchemaBaseline = async ( client: PrismaClient, prepared: PreparedRuntimeSchemaBaseline ): Promise => { - for (const ddl of RUNTIME_SCHEMA_TABLE_DDLS) { + const tableDdls = + prepared.verificationTarget === 'current' + ? CURRENT_RUNTIME_SCHEMA_TABLE_DDLS + : RUNTIME_SCHEMA_TABLE_DDLS + for (const ddl of tableDdls) { await migrationSqlExecutor.execute(client, ddl) } @@ -931,7 +1062,11 @@ const applyRuntimeSchemaBaseline = async ( COMPUTE_JOB_ADD_NOTIFICATION_CONSUMED_AT_DDL ) - await applySqliteCheckConstraints(client, prepared.pendingCheckConstraints) + await applySqliteCheckConstraints( + client, + prepared.pendingCheckConstraints, + prepared.verificationTarget === 'current' ? CURRENT_RUNTIME_SCHEMA_INDEX_DDLS : [] + ) for (const ddl of RUNTIME_SCHEMA_INDEX_DDLS) { await migrationSqlExecutor.execute(client, ddl) } @@ -941,8 +1076,15 @@ export { RUNTIME_SCHEMA_BASELINE_CONTRACT, RUNTIME_SCHEMA_BASELINE_TARGET_SQL, applyRuntimeSchemaBaseline, + adaptMigrationOperationsForCurrentSchema, + hasCurrentManagedFileVersionFoundation, prepareRuntimeSchemaBaseline, verifyCurrentRuntimeSchema, + verifyCurrentRuntimeSchemaTables, verifyRuntimeSchemaBaseline } -export type { AllowedSuffixCheckConstraints, PreparedRuntimeSchemaBaseline } +export type { + AdaptedMigrationOperations, + AllowedSuffixCheckConstraints, + PreparedRuntimeSchemaBaseline +} diff --git a/src/main/database/migration-service.test.ts b/src/main/database/migration-service.test.ts index 51a1f39e9..e515b465d 100644 --- a/src/main/database/migration-service.test.ts +++ b/src/main/database/migration-service.test.ts @@ -6,8 +6,13 @@ import { PrismaClient } from '@prisma/client' import { afterEach, describe, expect, it } from 'vitest' import { createProjectDbClient } from '../projects/prisma-client' -import { verifyCurrentRuntimeSchema } from './legacy-baseline-adapter' +import { + adaptMigrationOperationsForCurrentSchema, + verifyCurrentRuntimeSchema +} from './legacy-baseline-adapter' import { RUNTIME_SCHEMA_TABLE_DDL_BY_NAME } from './migrations/0001-runtime-schema-baseline' +import { databaseJsonConstraintsMigration } from './migrations/0008-database-json-constraints' +import { applySqliteMigrationOperations } from './sqlite-schema-migrations' import { BASELINE_CHECKSUM, MIGRATION_MANIFEST, @@ -20,7 +25,7 @@ import { } from './migration-service' const futureTestMigration = (): MigrationManifestEntry => { - const id = '0013_test_suffix' + const id = '0014_test_suffix' const statements = [`UPDATE "Project" SET "name" = "name" WHERE 0`] as const const verifiers = [{ kind: 'table-exists', version: 1, table: 'Project' }] as const return { @@ -33,6 +38,32 @@ const futureTestMigration = (): MigrationManifestEntry => { } } +const LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID = '0009_managed_file_version_foundation' +const LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM = + '54d50c127428b47efcea83e18c30f1dd7b94bfe7f37a3b2aae29a1a7ac43a1f8' + +const legacyDraftMigrationManifest = (): readonly MigrationManifestEntry[] => { + const managedIndex = MIGRATION_MANIFEST.findIndex( + ({ id }) => id === '0013_managed_file_version_foundation' + ) + const upstreamSuffixIndex = MIGRATION_MANIFEST.findIndex( + ({ id }) => id === '0009_vision_evidence' + ) + const managedMigration = MIGRATION_MANIFEST[managedIndex] + if (managedIndex < 0 || upstreamSuffixIndex < 0 || !managedMigration) { + throw new Error('Managed migration test fixture is unavailable.') + } + return [ + ...MIGRATION_MANIFEST.slice(0, upstreamSuffixIndex), + { + ...managedMigration, + id: LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID, + checksum: LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM + }, + ...MIGRATION_MANIFEST.slice(upstreamSuffixIndex, managedIndex) + ] +} + const createDatabaseAtMigration0005 = async (client: PrismaClient): Promise => { const migration0006Index = MIGRATION_MANIFEST.findIndex( (migration) => migration.id === '0006_database_domain_constraints' @@ -60,6 +91,32 @@ const createDatabaseAtMigration0005 = async (client: PrismaClient): Promise => { + await client.$executeRawUnsafe('PRAGMA foreign_keys = OFF') + for (const migration of manifest) { + for (const statement of migration.statements) await client.$executeRawUnsafe(statement) + await applySqliteMigrationOperations(client, migration.operations ?? []) + } + await client.$executeRawUnsafe(`CREATE TABLE "_open_science_migrations" ( + "id" TEXT NOT NULL PRIMARY KEY, + "checksum" TEXT NOT NULL, + "appliedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "_open_science_migrations_checksum_check" + CHECK (length("checksum") = 64 AND "checksum" NOT GLOB '*[^0-9a-f]*') + )`) + for (const migration of manifest) { + await client.$executeRawUnsafe( + `INSERT INTO "_open_science_migrations" ("id", "checksum") VALUES (?, ?)`, + migration.id, + migration.checksum + ) + } + await client.$executeRawUnsafe('PRAGMA foreign_keys = ON') +} + const removeComputePasswordAuthSchema = async (client: PrismaClient): Promise => { await client.$executeRawUnsafe('DROP TABLE "ComputeCredential"') await client.$executeRawUnsafe('DROP TABLE "ComputeAuthOperation"') @@ -249,10 +306,11 @@ describe('application database migrations', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ], from: null, - to: '0012_tag_ordering' + to: '0013_managed_file_version_foundation' }) expect(compatibility).toEqual([{ sqliteVersion: expect.stringMatching(/^\d+\.\d+\.\d+$/) }]) await expect( @@ -265,8 +323,8 @@ describe('application database migrations', () => { await expect(migrateApplicationDatabase(client)).resolves.toEqual({ adoptedLegacy: false, applied: [], - from: '0012_tag_ordering', - to: '0012_tag_ordering' + from: '0013_managed_file_version_foundation', + to: '0013_managed_file_version_foundation' }) }) @@ -279,6 +337,351 @@ describe('application database migrations', () => { await expect(verifyCurrentRuntimeSchema(client)).resolves.toBeUndefined() }) + it('refreshes stale heads and ManagedFile projections while adopting a current pre-ledger schema', async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-database-current-adoption-')) + client = createProjectDbClient(storageRoot) + await migrateApplicationDatabase(client) + await client.$executeRawUnsafe(`INSERT INTO "Project" ("id", "name", "updatedAt") + VALUES ('project-1', 'Project', CURRENT_TIMESTAMP)`) + await client.$executeRawUnsafe(`INSERT INTO "FileOriginSession" ( + "projectId", "sessionId", "updatedAt" + ) VALUES ('project-1', 'session-1', CURRENT_TIMESTAMP)`) + await client.$executeRawUnsafe(`INSERT INTO "ArtifactLineage" ( + "id", "projectId", "sessionId", "normalizedFilename", "filename", "updatedAt" + ) VALUES ( + 'artifact-1', 'project-1', 'session-1', 'report.md', 'report.md', CURRENT_TIMESTAMP + )`) + await client.$executeRawUnsafe(`INSERT INTO "ArtifactVersion" ( + "id", "artifactId", "versionNumber", "filename", "originKind", "state", + "contentStorageKey", "contentType", "sizeBytes", "checksum", "createdAt", "updatedAt" + ) VALUES + ( + 'artifact-version-1', 'artifact-1', 1, 'report-v1.md', 'legacy', 'finalized', + 'artifacts/report-v1.md', 'text/markdown', 10, '${'a'.repeat(64)}', + '2026-08-18T00:00:00.000Z', CURRENT_TIMESTAMP + ), + ( + 'artifact-version-2', 'artifact-1', 2, 'report-v2.md', 'legacy', 'finalized', + 'artifacts/report-v2.md', 'text/markdown', 20, '${'b'.repeat(64)}', + '2026-08-19T00:00:00.000Z', CURRENT_TIMESTAMP + )`) + await client.$executeRawUnsafe(`UPDATE "ArtifactLineage" + SET "currentVersionId" = 'artifact-version-1' + WHERE "id" = 'artifact-1'`) + await client.$executeRawUnsafe(`INSERT INTO "ManagedFile" ( + "source", "sourceFileId", "sourceVersionId", "checksum", "projectId", "sessionId", + "displayName", "storageKey", "mimeType", "sizeBytes", "mtimeMs", "sortAtMs", "updatedAt" + ) VALUES ( + 'artifact', 'artifact-1', 'artifact-version-1', '${'a'.repeat(64)}', + 'project-1', 'session-1', 'report-v1.md', 'artifacts/report-v1.md', + 'text/markdown', 10, 1000, 1000, CURRENT_TIMESTAMP + )`) + await client.$executeRawUnsafe(`INSERT INTO "UploadFile" ( + "id", "projectId", "sessionId", "filename", "originalFilename", "updatedAt" + ) VALUES ( + 'upload-1', 'project-1', 'session-1', 'upload-v1.csv', 'dataset.csv', CURRENT_TIMESTAMP + )`) + await client.$executeRawUnsafe(`INSERT INTO "UploadVersion" ( + "id", "uploadFileId", "versionNumber", "state", "originKind", "contentStorageKey", + "filename", "originalFilename", "contentType", "sizeBytes", "checksum", "createdAt", + "updatedAt" + ) VALUES ( + 'upload-version-1', 'upload-1', 1, 'ready', 'user_upload', 'uploads/upload-v1.csv', + 'upload-v1.csv', 'dataset.csv', 'text/csv', 30, '${'c'.repeat(64)}', + '2026-08-19T01:00:00.000Z', CURRENT_TIMESTAMP + )`) + await client.$executeRawUnsafe('DROP TABLE "_open_science_migrations"') + + await expect(migrateApplicationDatabase(client)).resolves.toMatchObject({ + adoptedLegacy: true, + applied: MIGRATION_MANIFEST.map((migration) => migration.id) + }) + await expect( + client.$queryRawUnsafe>( + `SELECT "currentVersionId" FROM "ArtifactLineage" WHERE "id" = 'artifact-1'` + ) + ).resolves.toEqual([{ currentVersionId: 'artifact-version-2' }]) + await expect( + client.$queryRawUnsafe>( + `SELECT "currentVersionId" FROM "UploadFile" WHERE "id" = 'upload-1'` + ) + ).resolves.toEqual([{ currentVersionId: 'upload-version-1' }]) + await expect( + client.$queryRawUnsafe< + Array<{ + source: string + sourceFileId: string + sourceVersionId: string | null + checksum: string | null + displayName: string + storageKey: string + }> + >(`SELECT "source", "sourceFileId", "sourceVersionId", "checksum", "displayName", "storageKey" + FROM "ManagedFile" WHERE "projectId" = 'project-1' ORDER BY "source"`) + ).resolves.toEqual([ + { + source: 'artifact', + sourceFileId: 'artifact-1', + sourceVersionId: 'artifact-version-2', + checksum: 'b'.repeat(64), + displayName: 'report-v2.md', + storageKey: 'artifacts/report-v2.md' + }, + { + source: 'upload', + sourceFileId: 'upload-1', + sourceVersionId: 'upload-version-1', + checksum: 'c'.repeat(64), + displayName: 'dataset.csv', + storageKey: 'uploads/upload-v1.csv' + } + ]) + }) + + it('replays an upstream suffix before adopting the canonical managed migration', async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-database-managed-prefix-')) + client = createProjectDbClient(storageRoot) + await migrateApplicationDatabase(client) + await client.$executeRawUnsafe(`DELETE FROM "_open_science_migrations" + WHERE "id" IN ('0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', '0012_tag_ordering', '0013_managed_file_version_foundation')`) + await client.$executeRawUnsafe('DROP TABLE "VisionEvidence"') + await removeComputePasswordAuthSchema(client) + await client.$executeRawUnsafe('DROP TABLE "TagAssignment"') + await client.$executeRawUnsafe('DROP TABLE "Tag"') + + await expect(migrateApplicationDatabase(client)).resolves.toEqual({ + adoptedLegacy: false, + applied: [ + '0009_vision_evidence', + '0010_compute_password_auth', + '0011_cross_resource_tags', + '0012_tag_ordering', + '0013_managed_file_version_foundation' + ], + from: '0008_database_json_constraints', + to: '0013_managed_file_version_foundation' + }) + await expect(verifyCurrentRuntimeSchema(client)).resolves.toBeUndefined() + }) + + it('upgrades the real upstream ledger and repairs VisionEvidence after the managed rebuild', async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-database-upstream-ledger-')) + const databasePath = join(storageRoot, 'open-science.db') + const backupPath = `${databasePath}.before-0013_managed_file_version_foundation.backup` + client = createProjectDbClient(storageRoot) + const upstreamManifest = MIGRATION_MANIFEST.slice(0, -1) + await createDatabaseAtReleasedManifest(client, upstreamManifest) + await client.$executeRawUnsafe( + `INSERT INTO "Project" ("id", "name", "updatedAt") + VALUES ('project-1', 'Project', CURRENT_TIMESTAMP)` + ) + await client.$executeRawUnsafe( + `INSERT INTO "FileOriginSession" ("projectId", "sessionId", "updatedAt") + VALUES ('project-1', 'session-1', CURRENT_TIMESTAMP)` + ) + await client.$executeRawUnsafe( + `INSERT INTO "UploadFile" ( + "id", "projectId", "sessionId", "filename", "originalFilename", "updatedAt" + ) VALUES ( + 'upload-1', 'project-1', 'session-1', 'image.png', 'image.png', CURRENT_TIMESTAMP + )` + ) + await client.$executeRawUnsafe( + `INSERT INTO "UploadVersion" ( + "id", "uploadFileId", "versionNumber", "state", "contentStorageKey", "filename", + "originalFilename", "contentType", "sizeBytes", "checksum", "updatedAt" + ) VALUES ( + 'upload-version-1', 'upload-1', 1, 'ready', 'uploads/image.png', 'image.png', + 'image.png', 'image/png', 3, '${'a'.repeat(64)}', CURRENT_TIMESTAMP + )` + ) + await client.$executeRaw` + INSERT INTO "VisionEvidence" ( + "id", "projectId", "sessionId", "sourceKind", "uploadVersionId", "imageChecksum", + "mimeType", "extractorFingerprint", "evidenceSchemaVersion", "evidenceJson", + "evidenceChecksum", "updatedAt" + ) VALUES ( + ${'vision-1'}, ${'project-1'}, ${'session-1'}, ${'upload-version'}, + ${'upload-version-1'}, ${'b'.repeat(64)}, ${'image/png'}, ${'c'.repeat(64)}, + ${1}, ${'{}'}, ${'d'.repeat(64)}, ${new Date('2026-08-19T00:00:00.000Z')} + ) + ` + await expect(migrateApplicationDatabase(client)).resolves.toEqual({ + adoptedLegacy: false, + applied: ['0013_managed_file_version_foundation'], + from: '0012_tag_ordering', + to: '0013_managed_file_version_foundation' + }) + await expect( + client.$queryRaw>` + SELECT "uploadVersionId" FROM "VisionEvidence" WHERE "id" = 'vision-1' + ` + ).resolves.toEqual([{ uploadVersionId: 'upload-version-1' }]) + await expect( + client.$queryRawUnsafe>('PRAGMA foreign_key_check') + ).resolves.toEqual([]) + await expect(verifyCurrentRuntimeSchema(client)).resolves.toBeUndefined() + await expect( + client.$queryRawUnsafe>( + 'SELECT "id" FROM "_open_science_migrations" ORDER BY "id"' + ) + ).resolves.toEqual(MIGRATION_MANIFEST.map(({ id }) => ({ id }))) + await expect(access(backupPath)).resolves.toBeUndefined() + + await expect(migrateApplicationDatabase(client)).resolves.toMatchObject({ applied: [] }) + await expect(access(backupPath)).resolves.toBeUndefined() + }) + + it('normalizes the exact legacy Draft managed ledger and retains its recovery backup', async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-database-draft-ledger-')) + const databasePath = join(storageRoot, 'open-science.db') + const backupPath = `${databasePath}.before-0013_managed_file_version_foundation.backup` + client = createProjectDbClient(storageRoot) + await createDatabaseAtReleasedManifest(client, legacyDraftMigrationManifest()) + await client.$executeRawUnsafe( + `INSERT INTO "Project" ("id", "name", "updatedAt") + VALUES ('legacy-draft-project', 'Preserved Draft project', CURRENT_TIMESTAMP)` + ) + + await expect(migrateApplicationDatabase(client, { databasePath })).resolves.toEqual({ + adoptedLegacy: false, + applied: ['0013_managed_file_version_foundation'], + from: '0012_tag_ordering', + to: '0013_managed_file_version_foundation' + }) + await expect( + client.$queryRawUnsafe>( + 'SELECT "id", "checksum" FROM "_open_science_migrations" ORDER BY "id"' + ) + ).resolves.toEqual(MIGRATION_MANIFEST.map(({ id, checksum }) => ({ id, checksum }))) + await expect( + client.$queryRawUnsafe>( + `SELECT "name" FROM "Project" WHERE "id" = 'legacy-draft-project'` + ) + ).resolves.toEqual([{ name: 'Preserved Draft project' }]) + await expect(access(backupPath)).resolves.toBeUndefined() + + await expect(migrateApplicationDatabase(client, { databasePath })).resolves.toMatchObject({ + applied: [] + }) + await expect(access(backupPath)).resolves.toBeUndefined() + }) + + it('upgrades the shortest exact legacy Draft managed prefix through the canonical suffix', async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-database-draft-prefix-')) + const databasePath = join(storageRoot, 'open-science.db') + const backupPath = `${databasePath}.before-0013_managed_file_version_foundation.backup` + client = createProjectDbClient(storageRoot) + const legacyDraftPrefix = legacyDraftMigrationManifest().slice(0, 9) + await createDatabaseAtReleasedManifest(client, legacyDraftPrefix) + + await expect(migrateApplicationDatabase(client, { databasePath })).resolves.toEqual({ + adoptedLegacy: false, + applied: [ + '0009_vision_evidence', + '0010_compute_password_auth', + '0011_cross_resource_tags', + '0012_tag_ordering', + '0013_managed_file_version_foundation' + ], + from: LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID, + to: '0013_managed_file_version_foundation' + }) + await expect( + client.$queryRawUnsafe>( + 'SELECT "id", "checksum" FROM "_open_science_migrations" ORDER BY "id"' + ) + ).resolves.toEqual(MIGRATION_MANIFEST.map(({ id, checksum }) => ({ id, checksum }))) + await expect(access(backupPath)).resolves.toBeUndefined() + }) + + it('rejects a corrupted legacy Draft managed checksum without changing its ledger', async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-database-draft-checksum-')) + client = createProjectDbClient(storageRoot) + const legacyDraftPrefix = legacyDraftMigrationManifest().slice(0, 9) + await createDatabaseAtReleasedManifest(client, legacyDraftPrefix) + const corruptedChecksum = 'f'.repeat(64) + await client.$executeRawUnsafe( + `UPDATE "_open_science_migrations" SET "checksum" = ? WHERE "id" = ?`, + corruptedChecksum, + LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID + ) + + await expect(migrateApplicationDatabase(client)).rejects.toMatchObject({ + code: 'database_history_invalid' + }) + await expect( + client.$queryRawUnsafe>( + 'SELECT "id", "checksum" FROM "_open_science_migrations" ORDER BY "id"' + ) + ).resolves.toEqual([ + ...MIGRATION_MANIFEST.slice(0, 8).map(({ id, checksum }) => ({ id, checksum })), + { + id: LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID, + checksum: corruptedChecksum + } + ]) + }) + + it('rejects a reordered legacy Draft managed identity without changing its ledger', async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-database-draft-order-')) + client = createProjectDbClient(storageRoot) + const legacyDraftPrefix = legacyDraftMigrationManifest().slice(0, 9) + await createDatabaseAtReleasedManifest(client, legacyDraftPrefix) + const reorderedId = '0007z_managed_file_version_foundation' + await client.$executeRawUnsafe( + `UPDATE "_open_science_migrations" SET "id" = ? WHERE "id" = ?`, + reorderedId, + LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID + ) + + await expect(migrateApplicationDatabase(client)).rejects.toMatchObject({ + code: 'database_history_invalid' + }) + await expect( + client.$queryRawUnsafe>( + `SELECT "id", "checksum" FROM "_open_science_migrations" + WHERE "id" = ?`, + reorderedId + ) + ).resolves.toEqual([ + { + id: reorderedId, + checksum: LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM + } + ]) + }) + + it('keeps the legacy Draft ledger identity when canonical registration fails', async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-database-draft-rollback-')) + const databasePath = join(storageRoot, 'open-science.db') + const backupPath = `${databasePath}.before-0013_managed_file_version_foundation.backup` + client = createProjectDbClient(storageRoot) + await createDatabaseAtReleasedManifest(client, legacyDraftMigrationManifest()) + await client.$executeRawUnsafe(`CREATE TRIGGER "reject_canonical_managed_ledger" + BEFORE INSERT ON "_open_science_migrations" + WHEN NEW."id" = '0013_managed_file_version_foundation' + BEGIN + SELECT RAISE(ABORT, 'canonical ledger registration rejected'); + END`) + + await expect(migrateApplicationDatabase(client, { databasePath })).rejects.toMatchObject({ + code: 'database_migration_failed', + migrationId: '0013_managed_file_version_foundation' + }) + await expect( + client.$queryRawUnsafe>( + `SELECT "id" FROM "_open_science_migrations" + WHERE "id" IN ( + '0009_managed_file_version_foundation', + '0013_managed_file_version_foundation' + ) + ORDER BY "id"` + ) + ).resolves.toEqual([{ id: LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID }]) + await expect(access(backupPath)).resolves.toBeUndefined() + }) + it('upgrades a pre-ledger ComputeJob table while preserving historical rows', async () => { storageRoot = await mkdtemp(join(tmpdir(), 'open-science-jobs-3a-to-current-')) client = createProjectDbClient(storageRoot) @@ -330,7 +733,8 @@ describe('application database migrations', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ] }) await expect(migrateApplicationDatabase(client)).resolves.toMatchObject({ applied: [] }) @@ -375,7 +779,7 @@ describe('application database migrations', () => { await expect(migrateApplicationDatabase(client)).resolves.toMatchObject({ applied: expect.arrayContaining(['0010_compute_password_auth']), - to: '0012_tag_ordering' + to: '0013_managed_file_version_foundation' }) await expect( client.$executeRawUnsafe( @@ -397,7 +801,7 @@ describe('application database migrations', () => { await client.$executeRawUnsafe('DROP INDEX "ComputeJob_status_idx"') await removeComputePasswordAuthSchema(client) await client.$executeRawUnsafe(`DELETE FROM "_open_science_migrations" - WHERE "id" IN ('0006_database_domain_constraints', '0007_notification_attention_metadata', '0008_database_json_constraints', '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', '0012_tag_ordering')`) + WHERE "id" IN ('0006_database_domain_constraints', '0007_notification_attention_metadata', '0008_database_json_constraints', '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', '0012_tag_ordering', '0013_managed_file_version_foundation')`) await expect(migrateApplicationDatabase(client)).resolves.toMatchObject({ applied: [ @@ -407,14 +811,47 @@ describe('application database migrations', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ], from: '0005_project_preview_state_owner_fk', - to: '0012_tag_ordering' + to: '0013_managed_file_version_foundation' }) await expect(verifyCurrentRuntimeSchema(client)).resolves.toBeUndefined() }) + it('adapts replayed tables independently from migrations that have not run yet', async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-database-table-local-adapter-')) + client = createProjectDbClient(storageRoot) + await migrateApplicationDatabase(client) + await removeComputePasswordAuthSchema(client) + + const adapted = await adaptMigrationOperationsForCurrentSchema( + client, + databaseJsonConstraintsMigration.operations + ) + const table = (tableName: string): { columns: readonly string[] } => { + const descriptor = adapted.operations + .filter((operation) => operation.kind === 'rebuild-table-set') + .flatMap((operation) => operation.tables) + .find((candidate) => candidate.tableName === tableName) + if (!descriptor) throw new Error(`Expected a descriptor for ${tableName}.`) + return descriptor + } + + expect(adapted.currentTableNames).toEqual( + expect.arrayContaining(['ArtifactVersion', 'UploadVersion']) + ) + expect(adapted.currentTableNames).not.toContain('ComputeHost') + expect(table('ArtifactVersion').columns).toEqual( + expect.arrayContaining(['originKind', 'basedOnVersionId', 'storageTag', 'storedFilename']) + ) + expect(table('UploadVersion').columns).toEqual( + expect.arrayContaining(['originKind', 'basedOnVersionId', 'storageTag', 'storedFilename']) + ) + expect(table('ComputeHost').columns).not.toContain('authenticationMode') + }) + it('upgrades valid 0005 rows while preserving known retired columns', async () => { storageRoot = await mkdtemp(join(tmpdir(), 'open-science-database-0006-upgrade-')) client = createProjectDbClient(storageRoot) @@ -473,10 +910,11 @@ describe('application database migrations', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ], from: '0005_project_preview_state_owner_fk', - to: '0012_tag_ordering' + to: '0013_managed_file_version_foundation' }) await expect( client.$queryRaw< @@ -595,7 +1033,7 @@ describe('application database migrations', () => { }) ).rejects.toMatchObject({ code: 'database_validation_failed', - migrationId: '0012_tag_ordering' + migrationId: '0013_managed_file_version_foundation' }) expect(retired).toEqual([]) await expect(access(backupPath)).resolves.toBeUndefined() @@ -611,9 +1049,9 @@ describe('application database migrations', () => { migrateApplicationDatabaseWithManifest(client, [...MIGRATION_MANIFEST, future]) ).resolves.toEqual({ adoptedLegacy: false, - applied: ['0013_test_suffix'], - from: '0012_tag_ordering', - to: '0013_test_suffix' + applied: ['0014_test_suffix'], + from: '0013_managed_file_version_foundation', + to: '0014_test_suffix' }) await expect( client.$queryRaw>` @@ -632,7 +1070,8 @@ describe('application database migrations', () => { { id: '0010_compute_password_auth' }, { id: '0011_cross_resource_tags' }, { id: '0012_tag_ordering' }, - { id: '0013_test_suffix' } + { id: '0013_managed_file_version_foundation' }, + { id: '0014_test_suffix' } ]) }) @@ -696,10 +1135,11 @@ describe('application database migrations', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ], from: '0001_runtime_schema_baseline', - to: '0012_tag_ordering' + to: '0013_managed_file_version_foundation' }) expect(backupEvents).toEqual([ { @@ -756,13 +1196,21 @@ describe('application database migrations', () => { migrationId: '0012_tag_ordering', path: `${databasePath}.before-0012_tag_ordering.backup`, reused: false + }, + { + migrationId: '0013_managed_file_version_foundation', + path: `${databasePath}.before-0013_managed_file_version_foundation.backup`, + reused: false } ]) await expect(access(backupPath)).rejects.toMatchObject({ code: 'ENOENT' }) + await expect( + access(`${databasePath}.before-0013_managed_file_version_foundation.backup`) + ).resolves.toBeUndefined() await expect(access(`${databasePath}.before-0012_tag_ordering.backup`)).resolves.toBeUndefined() await expect( access(`${databasePath}.before-0011_cross_resource_tags.backup`) - ).resolves.toBeUndefined() + ).rejects.toMatchObject({ code: 'ENOENT' }) await expect( client.$queryRaw>` SELECT "agentContext", "name" FROM "Project" WHERE "id" = 'project-1' @@ -792,7 +1240,7 @@ describe('application database migrations', () => { migrateApplicationDatabaseWithManifest(client, [...MIGRATION_MANIFEST, future]) ).rejects.toMatchObject({ code: 'database_validation_failed', - migrationId: '0013_test_suffix' + migrationId: '0014_test_suffix' }) await expect( client.$queryRaw>` @@ -816,7 +1264,8 @@ describe('application database migrations', () => { { id: '0009_vision_evidence' }, { id: '0010_compute_password_auth' }, { id: '0011_cross_resource_tags' }, - { id: '0012_tag_ordering' } + { id: '0012_tag_ordering' }, + { id: '0013_managed_file_version_foundation' } ]) }) @@ -860,7 +1309,10 @@ describe('application database migrations', () => { readdir(storageRoot).then((entries) => entries.filter((entry) => entry.endsWith('.backup')).sort() ) - ).resolves.toEqual([`open-science.db.before-${future.id}.backup`]) + ).resolves.toEqual([ + 'open-science.db.before-0011_cross_resource_tags.backup', + `open-science.db.before-${future.id}.backup` + ]) }) it('rejects a migration when its required column is missing', async () => { @@ -881,7 +1333,7 @@ describe('application database migrations', () => { migrateApplicationDatabaseWithManifest(client, [...MIGRATION_MANIFEST, future]) ).rejects.toMatchObject({ code: 'database_validation_failed', - migrationId: '0013_test_suffix' + migrationId: '0014_test_suffix' }) }) @@ -919,9 +1371,10 @@ describe('application database migrations', () => { '0010_compute_password_auth', '0011_cross_resource_tags', '0012_tag_ordering', - '0013_test_suffix' + '0013_managed_file_version_foundation', + '0014_test_suffix' ], - to: '0013_test_suffix' + to: '0014_test_suffix' }) await expect( client.project.findUniqueOrThrow({ where: { id: 'legacy-project' } }) @@ -1037,10 +1490,19 @@ describe('application database migrations', () => { await client.$executeRawUnsafe('PRAGMA foreign_keys = ON') await removeComputePasswordAuthSchema(client) await client.$executeRawUnsafe('DROP TABLE "VisionEvidence"') + await client.$executeRawUnsafe('DROP TABLE "TagAssignment"') + await client.$executeRawUnsafe('DROP TABLE "Tag"') await client.$executeRawUnsafe('DROP TABLE "_open_science_migrations"') + const managedFileMigrationIndex = MIGRATION_MANIFEST.findIndex( + (migration) => migration.id === '0013_managed_file_version_foundation' + ) + await expect( - migrateApplicationDatabaseWithManifest(client, MIGRATION_MANIFEST.slice(0, -3)) + migrateApplicationDatabaseWithManifest( + client, + MIGRATION_MANIFEST.slice(0, managedFileMigrationIndex) + ) ).rejects.toMatchObject({ code: 'database_validation_failed', migrationId: '0001_runtime_schema_baseline' @@ -1148,7 +1610,8 @@ describe('application database migrations', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ] }) await expect( @@ -1255,7 +1718,8 @@ describe('application database migrations', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ] }) await expect(migrateApplicationDatabase(client)).resolves.toMatchObject({ applied: [] }) @@ -1316,7 +1780,8 @@ describe('application database migrations', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ] }) await expect( @@ -1380,7 +1845,8 @@ describe('application database migrations', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ] }) await expect(verifyCurrentRuntimeSchema(client)).resolves.toBeUndefined() @@ -1478,7 +1944,8 @@ describe('application database migrations', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ] }) await expect( @@ -1496,6 +1963,7 @@ describe('application database migrations', () => { const computePasswordAuthBackupPath = `${databasePath}.before-0010_compute_password_auth.backup` const crossResourceTagsBackupPath = `${databasePath}.before-0011_cross_resource_tags.backup` const tagOrderingBackupPath = `${databasePath}.before-0012_tag_ordering.backup` + const managedFileBackupPath = `${databasePath}.before-0013_managed_file_version_foundation.backup` const backupEvents: unknown[] = [] client = createProjectDbClient(storageRoot) await client.$executeRawUnsafe(`CREATE TABLE "Project" ( @@ -1533,7 +2001,8 @@ describe('application database migrations', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ] }) expect(backupEvents).toEqual([ @@ -1596,6 +2065,11 @@ describe('application database migrations', () => { migrationId: '0012_tag_ordering', path: `${databasePath}.before-0012_tag_ordering.backup`, reused: false + }, + { + migrationId: '0013_managed_file_version_foundation', + path: managedFileBackupPath, + reused: false } ]) await expect( @@ -1603,19 +2077,20 @@ describe('application database migrations', () => { entries.filter((entry) => entry.endsWith('.backup')).sort() ) ).resolves.toEqual([ - 'open-science.db.before-0011_cross_resource_tags.backup', - 'open-science.db.before-0012_tag_ordering.backup' + 'open-science.db.before-0012_tag_ordering.backup', + 'open-science.db.before-0013_managed_file_version_foundation.backup' ]) await expect(access(backupPath)).rejects.toMatchObject({ code: 'ENOENT' }) await expect(access(agentContextBackupPath)).rejects.toMatchObject({ code: 'ENOENT' }) await expect(access(visionEvidenceBackupPath)).rejects.toMatchObject({ code: 'ENOENT' }) await expect(access(computePasswordAuthBackupPath)).rejects.toMatchObject({ code: 'ENOENT' }) - await expect(access(crossResourceTagsBackupPath)).resolves.toBeUndefined() + await expect(access(crossResourceTagsBackupPath)).rejects.toMatchObject({ code: 'ENOENT' }) await expect(access(tagOrderingBackupPath)).resolves.toBeUndefined() + await expect(access(managedFileBackupPath)).resolves.toBeUndefined() await expect(client.project.count()).resolves.toBe(1) const backupClient = new PrismaClient({ - datasources: { db: { url: `file:${crossResourceTagsBackupPath.replaceAll('\\', '/')}` } } + datasources: { db: { url: `file:${tagOrderingBackupPath.replaceAll('\\', '/')}` } } }) try { await expect( @@ -1627,7 +2102,7 @@ describe('application database migrations', () => { backupClient.$queryRaw>` SELECT "id" FROM "_open_science_migrations" ORDER BY "id" DESC LIMIT 1 ` - ).resolves.toEqual([{ id: '0010_compute_password_auth' }]) + ).resolves.toEqual([{ id: '0011_cross_resource_tags' }]) } finally { await backupClient.$disconnect() } @@ -2035,6 +2510,11 @@ describe('application database migrations', () => { migrationId: '0012_tag_ordering', path: `${databasePath}.before-0012_tag_ordering.backup`, reused: false + }), + expect.objectContaining({ + migrationId: '0013_managed_file_version_foundation', + path: `${databasePath}.before-0013_managed_file_version_foundation.backup`, + reused: false }) ]) expect(retired).toEqual([ @@ -2079,6 +2559,10 @@ describe('application database migrations', () => { { migrationId: '0012_tag_ordering', path: `${databasePath}.before-0012_tag_ordering.backup` + }, + { + migrationId: '0013_managed_file_version_foundation', + path: `${databasePath}.before-0013_managed_file_version_foundation.backup` } ]) await expect(access(backupPath)).rejects.toMatchObject({ code: 'ENOENT' }) @@ -2112,11 +2596,11 @@ describe('application database migrations', () => { entries.filter((entry) => entry.endsWith('.backup')).sort() ) ).resolves.toEqual([ - 'open-science.db.before-0011_cross_resource_tags.backup', 'open-science.db.before-0012_tag_ordering.backup', + 'open-science.db.before-0013_managed_file_version_foundation.backup', unknownBackupName ]) - expect(retired).toHaveLength(10) + expect(retired).toHaveLength(11) expect(retired).toEqual( expect.arrayContaining( MIGRATION_MANIFEST.slice(0, -2).map((migration) => @@ -2157,6 +2641,16 @@ describe('application database migrations', () => { await migrateApplicationDatabase(client, { databasePath }) await mkdir(backupPath) await writeFile(join(backupPath, 'keep'), 'occupied', 'utf8') + await writeFile( + `${databasePath}.before-0011_cross_resource_tags.backup`, + 'newer recovery snapshot', + 'utf8' + ) + await writeFile( + `${databasePath}.before-0012_tag_ordering.backup`, + 'newest recovery snapshot', + 'utf8' + ) const failures: unknown[] = [] await expect( diff --git a/src/main/database/migration-service.ts b/src/main/database/migration-service.ts index 1eec02452..4c04f9a1b 100644 --- a/src/main/database/migration-service.ts +++ b/src/main/database/migration-service.ts @@ -6,9 +6,12 @@ import type { DatabaseStartupErrorCode } from '../../shared/database-startup' import { RUNTIME_SCHEMA_BASELINE_CONTRACT, + adaptMigrationOperationsForCurrentSchema, applyRuntimeSchemaBaseline, + hasCurrentManagedFileVersionFoundation, prepareRuntimeSchemaBaseline, verifyCurrentRuntimeSchema, + verifyCurrentRuntimeSchemaTables, verifyRuntimeSchemaBaseline, type AllowedSuffixCheckConstraints } from './legacy-baseline-adapter' @@ -26,6 +29,10 @@ import { visionEvidenceMigration } from './migrations/0009-vision-evidence' import { computePasswordAuthMigration } from './migrations/0010-compute-password-auth' import { crossResourceTagsMigration } from './migrations/0011-cross-resource-tags' import { tagOrderingMigration } from './migrations/0012-tag-ordering' +import { + managedFileVersionFoundationCurrentSchemaAdoptionStatements, + managedFileVersionFoundationMigration +} from './migrations/0013-managed-file-version-foundation' import { applySqliteMigrationOperations, type SqliteMigrationOperation @@ -71,6 +78,8 @@ type MigrationVerifierDescriptor = version: 1 indexes: readonly { name: string; sql: string }[] } + | { kind: 'foreign-key-integrity'; version: 1 } + | { kind: 'managed-file-version-domain'; version: 1 } type MigrationVerifiers = readonly [MigrationVerifierDescriptor, ...MigrationVerifierDescriptor[]] @@ -116,6 +125,10 @@ const serializeMigrationVerifier = (verifier: MigrationVerifierDescriptor): stri .flatMap(({ name, sql }) => [name, sql]) .map(lengthPrefixedChecksumText) .join('')}` + case 'foreign-key-integrity': + return `foreign-key-integrity:v${verifier.version}` + case 'managed-file-version-domain': + return `managed-file-version-domain:v${verifier.version}` } } @@ -192,6 +205,17 @@ const DATABASE_JSON_CONSTRAINTS_CHECKSUM = checksumMigrationPayload( databaseJsonConstraintsMigration.verifiers, databaseJsonConstraintsMigration.operations ) +const MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM = checksumMigrationPayload( + managedFileVersionFoundationMigration.id, + managedFileVersionFoundationMigration.statements, + managedFileVersionFoundationMigration.verifiers +) +const LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID = '0009_managed_file_version_foundation' +const LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM = checksumMigrationPayload( + LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID, + managedFileVersionFoundationMigration.statements, + managedFileVersionFoundationMigration.verifiers +) const VISION_EVIDENCE_CHECKSUM = checksumMigrationPayload( visionEvidenceMigration.id, visionEvidenceMigration.statements, @@ -352,6 +376,13 @@ const MIGRATION_MANIFEST = [ checksum: TAG_ORDERING_CHECKSUM, backupOnApply: 'required', backupRetention: 'retain' + }, + { + ...managedFileVersionFoundationMigration, + checksum: MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM, + backupOnApply: 'required', + backupRetention: 'retain', + foreignKeysDuringApply: 'disabled' } ] as const satisfies readonly MigrationManifestEntry[] // schema-locality: begin frozen-0001-repairs @@ -438,6 +469,77 @@ type MigrationManifestEntry = { verifiers: MigrationVerifiers backupOnApply: 'required' | 'none' backupRetention: 'retain' | 'delete-after-success' + foreignKeysDuringApply?: 'enabled' | 'disabled' +} + +const verifyForeignKeyIntegrity = async (client: PrismaClient): Promise => { + const violations = await migrationSqlExecutor.query( + client, + 'PRAGMA foreign_key_check' + ) + if (violations.length > 0) { + throw new Error( + `Database foreign-key integrity audit found orphaned relations: ${violations + .map((violation) => `${violation.table}->${violation.parent}`) + .join(', ')}.` + ) + } +} + +const verifyManagedFileVersionDomain = async (client: PrismaClient): Promise => { + await verifyForeignKeyIntegrity(client) + const violations = await migrationSqlExecutor.query>( + client, + ` + SELECT 'artifact-head' AS "kind", "lineage"."id" AS "id" + FROM "ArtifactLineage" AS "lineage" + WHERE "lineage"."currentVersionId" IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM "ArtifactVersion" AS "version" + WHERE "version"."id" = "lineage"."currentVersionId" + AND "version"."artifactId" = "lineage"."id" + AND "version"."state" = 'finalized' + ) + UNION ALL + SELECT 'upload-head', "file"."id" + FROM "UploadFile" AS "file" + WHERE "file"."currentVersionId" IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM "UploadVersion" AS "version" + WHERE "version"."id" = "file"."currentVersionId" + AND "version"."uploadFileId" = "file"."id" + AND "version"."state" = 'ready' + ) + UNION ALL + SELECT 'artifact-based-on', "version"."id" + FROM "ArtifactVersion" AS "version" + WHERE "version"."basedOnVersionId" IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM "ArtifactVersion" AS "parent" + WHERE "parent"."id" = "version"."basedOnVersionId" + AND "parent"."artifactId" = "version"."artifactId" + AND "parent"."state" = 'finalized' + AND "parent"."versionNumber" < "version"."versionNumber" + ) + UNION ALL + SELECT 'upload-based-on', "version"."id" + FROM "UploadVersion" AS "version" + WHERE "version"."basedOnVersionId" IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM "UploadVersion" AS "parent" + WHERE "parent"."id" = "version"."basedOnVersionId" + AND "parent"."uploadFileId" = "version"."uploadFileId" + AND "parent"."state" = 'ready' + AND "parent"."versionNumber" < "version"."versionNumber" + ) + LIMIT 1 + ` + ) + if (violations.length > 0) { + throw new Error( + `Managed file version domain audit failed for ${violations[0]!.kind}: ${violations[0]!.id}` + ) + } } const RETAINED_DATABASE_MIGRATION_BACKUP_LIMIT = 2 @@ -450,7 +552,8 @@ type DatabaseMigrationBackupRetirementScope = { const runMigrationVerifiers = async ( client: PrismaClient, verifiers: MigrationVerifiers, - allowedSuffixChecks: AllowedSuffixCheckConstraints = {} + allowedSuffixChecks: AllowedSuffixCheckConstraints = {}, + currentTableNames: ReadonlySet = new Set() ): Promise => { for (const verifier of verifiers) { switch (verifier.kind) { @@ -464,6 +567,7 @@ const runMigrationVerifiers = async ( await verifyRuntimeSchemaBaseline(client, allowedSuffixChecks) break case 'table-exists': { + if (currentTableNames.has(verifier.table)) break const rows = await client.$queryRaw>` SELECT "name" FROM "sqlite_schema" WHERE "type" = 'table' AND "name" = ${verifier.table} @@ -474,6 +578,7 @@ const runMigrationVerifiers = async ( break } case 'column-exists': { + if (currentTableNames.has(verifier.table)) break const quotedTable = `"${verifier.table.replaceAll('"', '""')}"` const columns = await migrationSqlExecutor.query>( client, @@ -487,6 +592,7 @@ const runMigrationVerifiers = async ( break } case 'foreign-key-exists': { + if (currentTableNames.has(verifier.table)) break const quotedTable = `"${verifier.table.replaceAll('"', '""')}"` const foreignKeys = await migrationSqlExecutor.query( client, @@ -518,6 +624,7 @@ const runMigrationVerifiers = async ( } case 'check-constraints-exist': { for (const table of verifier.tables) { + if (currentTableNames.has(table.table)) continue const rows = await migrationSqlExecutor.query>( client, `SELECT "sql" FROM "sqlite_schema" WHERE "type" = 'table' AND "name" = ?`, @@ -549,6 +656,8 @@ const runMigrationVerifiers = async ( .replace(/;$/, '') .trim() for (const index of verifier.indexes) { + const tableName = index.sql.match(/\bON\s+"([^"]+)"/i)?.[1] + if (tableName && currentTableNames.has(tableName)) continue const rows = await migrationSqlExecutor.query>( client, `SELECT "sql" FROM "sqlite_schema" WHERE "type" = 'index' AND "name" = ?`, @@ -560,6 +669,12 @@ const runMigrationVerifiers = async ( } break } + case 'foreign-key-integrity': + await verifyForeignKeyIntegrity(client) + break + case 'managed-file-version-domain': + await verifyManagedFileVersionDomain(client) + break } } } @@ -747,30 +862,42 @@ const retireDatabaseMigrationBackups = async ( if (boundaryIndex < 0) { throw new Error(`Unknown database backup retention boundary ${scope.throughMigrationId}.`) } - const retainedMigrationIds = new Set( + const retainedCandidates = manifest + .slice(0, boundaryIndex + 1) + .filter( + (migration) => + migration.backupOnApply === 'required' && migration.backupRetention === 'retain' + ) + const manifestRetainedMigrationIds = new Set( + retainedCandidates + .slice(-RETAINED_DATABASE_MIGRATION_BACKUP_LIMIT) + .map((migration) => migration.id) + ) + const retirementCandidates = ( + retainedMigrationIds: ReadonlySet + ): readonly MigrationManifestEntry[] => manifest .slice(0, boundaryIndex + 1) .filter( (migration) => - migration.backupOnApply === 'required' && migration.backupRetention === 'retain' + (scope.includeDeleteAfterSuccess && + migration.backupRetention === 'delete-after-success') || + (migration.backupOnApply === 'required' && + migration.backupRetention === 'retain' && + !retainedMigrationIds.has(migration.id)) ) - .slice(-RETAINED_DATABASE_MIGRATION_BACKUP_LIMIT) - .map((migration) => migration.id) - ) - const retired = manifest.filter( - (migration) => - (scope.includeDeleteAfterSuccess && migration.backupRetention === 'delete-after-success') || - (migration.backupOnApply === 'required' && - migration.backupRetention === 'retain' && - !retainedMigrationIds.has(migration.id)) - ) - if (retired.length === 0) return - let databasePath: string | undefined + const manifestRetired = retirementCandidates(manifestRetainedMigrationIds) + if (manifestRetired.length === 0) return + + let databasePath: string try { - databasePath = options.databasePath ?? (await readMainDatabasePath(client)) - if (!databasePath) throw new Error('The database backup retention path is unavailable.') + const resolvedDatabasePath = options.databasePath ?? (await readMainDatabasePath(client)) + if (!resolvedDatabasePath) { + throw new Error('The database backup retention path is unavailable.') + } + databasePath = resolvedDatabasePath } catch (error) { - for (const migration of retired) { + for (const migration of manifestRetired) { try { options.onBackupRetirementFailed?.({ migrationId: migration.id, error }) } catch { @@ -779,6 +906,22 @@ const retireDatabaseMigrationBackups = async ( } return } + + // Retain the newest backups that actually exist. A history bridge may create an older backup + // after newer ledger entries are already present, and that recovery point must survive startup. + const existingRetainedCandidates: MigrationManifestEntry[] = [] + for (const migration of retainedCandidates) { + if (await pathExists(`${databasePath}.before-${migration.id}.backup`)) { + existingRetainedCandidates.push(migration) + } + } + const retainedMigrationIds = new Set( + existingRetainedCandidates + .slice(-RETAINED_DATABASE_MIGRATION_BACKUP_LIMIT) + .map((migration) => migration.id) + ) + const retired = retirementCandidates(retainedMigrationIds) + if (retired.length === 0) return for (const migration of retired) { const path = `${databasePath}.before-${migration.id}.backup` try { @@ -847,6 +990,39 @@ const validateLedger = ( return ledger.length } +const requiresLegacyDraftManagedFileVersionHistoryBridge = ( + ledger: readonly LedgerRow[], + manifest: readonly MigrationManifestEntry[] +): boolean => { + const managedIndex = manifest.findIndex( + (migration) => + migration.id === managedFileVersionFoundationMigration.id && + migration.checksum === MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM + ) + const legacyInsertionIndex = manifest.findIndex( + (migration) => migration.id > LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID + ) + if ( + managedIndex < 0 || + legacyInsertionIndex < 0 || + legacyInsertionIndex >= managedIndex || + ledger.length <= legacyInsertionIndex || + ledger.length > managedIndex + 1 + ) { + return false + } + return ledger.every((row, index) => { + if (index === legacyInsertionIndex) { + return ( + row.id === LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID && + row.checksum === LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM + ) + } + const expected = manifest[index < legacyInsertionIndex ? index : index - 1] + return expected !== undefined && row.id === expected.id && row.checksum === expected.checksum + }) +} + const readForeignKeyState = async (client: PrismaClient): Promise => { const rows = await migrationSqlExecutor.query( client, @@ -978,7 +1154,8 @@ const applyBaselineMigration = async ( client: PrismaClient, migration: MigrationManifestEntry, deferPreviewStateForeignKeyViolations: boolean, - allowedSuffixChecks: AllowedSuffixCheckConstraints + allowedSuffixChecks: AllowedSuffixCheckConstraints, + canAdoptCurrentSchema: boolean ): Promise => { let prepared: Awaited> try { @@ -986,6 +1163,13 @@ const applyBaselineMigration = async ( } catch (error) { throw classifyDatabaseFailure(error, 'validation', migration.id) } + if (prepared.verificationTarget === 'current' && !canAdoptCurrentSchema) { + throw classifyDatabaseFailure( + new Error('Current schema adoption requires its versioned migration manifest.'), + 'validation', + migration.id + ) + } const disableForeignKeys = prepared.pendingCheckConstraints.length > 0 let foreignKeysWereEnabled = false let migrationFailure: unknown @@ -1005,7 +1189,9 @@ const applyBaselineMigration = async ( } // The pinned 0005 suffix owns pruning these rows before the migration run completes. } - await runMigrationVerifiers(transactionClient, migration.verifiers, allowedSuffixChecks) + if (prepared.verificationTarget === 'baseline') { + await runMigrationVerifiers(transactionClient, migration.verifiers, allowedSuffixChecks) + } await insertLedgerRow(transactionClient, migration) }) } catch (error) { @@ -1037,9 +1223,44 @@ const applyBaselineMigration = async ( const applyManifestMigration = async ( client: PrismaClient, - migration: MigrationManifestEntry + migration: MigrationManifestEntry, + options: { + repairVisionEvidenceReference?: boolean + legacyLedgerIdentityToReplace?: { id: string; checksum: string } + } = {} ): Promise => { + const preserveCurrentSchema = await hasCurrentManagedFileVersionFoundation(client) + const canAdaptCurrentSchema = + preserveCurrentSchema && + migration.id !== managedFileVersionFoundationMigration.id && + migration.id < managedFileVersionFoundationMigration.id && + MIGRATION_MANIFEST.some( + (candidate) => candidate.id === migration.id && candidate.checksum === migration.checksum + ) + const canVerifyAsCurrentSchema = canAdaptCurrentSchema + const adapted = canAdaptCurrentSchema + ? await adaptMigrationOperationsForCurrentSchema(client, migration.operations ?? []) + : { operations: migration.operations ?? [], currentTableNames: [] } + const currentTableNames = new Set(adapted.currentTableNames) + const verifyMigrationTarget = async (targetClient: PrismaClient): Promise => { + if (!canVerifyAsCurrentSchema) { + await runMigrationVerifiers(targetClient, migration.verifiers) + return + } + await verifyCurrentRuntimeSchemaTables(targetClient, adapted.currentTableNames) + await runMigrationVerifiers(targetClient, migration.verifiers, {}, currentTableNames) + if (migration.id === projectPreviewStateOwnerFkMigration.id) { + await runMigrationVerifiers(targetClient, migration.verifiers) + } + } + const disableForeignKeys = + migration.foreignKeysDuringApply === 'disabled' || + (canAdaptCurrentSchema && (migration.operations?.length ?? 0) > 0) + let foreignKeysWereEnabled = false + let migrationFailure: unknown try { + foreignKeysWereEnabled = disableForeignKeys && (await readForeignKeyState(client)) === 1 + if (foreignKeysWereEnabled) await setForeignKeys(client, false) await client.$transaction(async (transaction) => { const transactionClient = transaction as unknown as PrismaClient // A pre-ledger build may already have emitted the current generated schema. When this @@ -1047,19 +1268,46 @@ const applyManifestMigration = async ( // identity without replaying non-idempotent SQLite ALTER TABLE statements. let contractAlreadySatisfied = false try { - await runMigrationVerifiers(transactionClient, migration.verifiers) + await verifyMigrationTarget(transactionClient) contractAlreadySatisfied = true } catch { // The migration statements below own bringing this schema suffix into compliance. } - if (!contractAlreadySatisfied) { - for (const statement of migration.statements) { + if ( + contractAlreadySatisfied && + migration.id === managedFileVersionFoundationMigration.id && + migration.checksum === MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM + ) { + for (const statement of managedFileVersionFoundationCurrentSchemaAdoptionStatements) { await migrationSqlExecutor.execute(transaction, statement) } - await applySqliteMigrationOperations(transactionClient, migration.operations ?? []) + } + if (!contractAlreadySatisfied) { + if (canVerifyAsCurrentSchema && migration.id === projectPreviewStateOwnerFkMigration.id) { + await migrationSqlExecutor.execute( + transaction, + `DELETE FROM "ProjectPreviewState" + WHERE NOT EXISTS ( + SELECT 1 FROM "Project" WHERE "Project"."id" = "ProjectPreviewState"."projectId" + )` + ) + } else { + for (const statement of migration.statements) { + await migrationSqlExecutor.execute(transaction, statement) + } + } + await applySqliteMigrationOperations(transactionClient, adapted.operations) + if (options.repairVisionEvidenceReference) { + // The upstream history created VisionEvidence before this immutable migration. Rebuild it + // after UploadVersion so SQLite does not retain the temporary rename as its FK target. + await applySqliteMigrationOperations( + transactionClient, + visionEvidenceMigration.operations + ) + } } try { - await runMigrationVerifiers(transactionClient, migration.verifiers) + await verifyMigrationTarget(transactionClient) } catch (error) { throw new DatabaseMigrationError( 'database_validation_failed', @@ -1069,11 +1317,42 @@ const applyManifestMigration = async ( { cause: error } ) } + if (options.legacyLedgerIdentityToReplace) { + const removed = await migrationSqlExecutor.execute( + transaction, + `DELETE FROM "_open_science_migrations" WHERE "id" = ? AND "checksum" = ?`, + options.legacyLedgerIdentityToReplace.id, + options.legacyLedgerIdentityToReplace.checksum + ) + if (removed !== 1) { + throw new Error('The legacy Draft managed migration ledger changed during startup.') + } + } await insertLedgerRow(transactionClient, migration) }) } catch (error) { - throw classifyDatabaseFailure(error, 'migration', migration.id) + migrationFailure = error + } + + let restoreFailure: unknown + try { + if (foreignKeysWereEnabled) await setForeignKeys(client, true) + } catch (error) { + restoreFailure = error + } + + if (migrationFailure && restoreFailure) { + throw classifyDatabaseFailure( + new AggregateError( + [migrationFailure, restoreFailure], + `Database migration failed and foreign-key enforcement could not be restored: ${migrationFailure instanceof Error ? migrationFailure.message : String(migrationFailure)}` + ), + 'migration', + migration.id + ) } + if (migrationFailure) throw classifyDatabaseFailure(migrationFailure, 'migration', migration.id) + if (restoreFailure) throw classifyDatabaseFailure(restoreFailure, 'migration', migration.id) } const reportDatabaseCompatibility = async ( @@ -1115,30 +1394,14 @@ const migrateApplicationDatabaseWithManifest = async ( } catch (error) { throw classifyDatabaseFailure(error, 'open') } - const appliedCount = validateLedger(ledger, manifest) + validateLedger([], manifest) const latest = manifest.at(-1)! + const adoptsManagedFileVersionFoundation = manifest.some( + (migration) => + migration.id === managedFileVersionFoundationMigration.id && + migration.checksum === MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM + ) const from = ledger.at(-1)?.id ?? null - const complete = async (result: SchemaMigrationResult): Promise => { - try { - await verifyCurrentRuntimeSchema(client) - } catch (error) { - throw classifyDatabaseFailure(error, 'validation', latest.id) - } - await reportDatabaseCompatibility(client, options) - await retireDatabaseMigrationBackups(client, manifest, options, { - throughMigrationId: latest.id, - includeDeleteAfterSuccess: true - }) - try { - options.onCompleted?.(result) - } catch { - // A diagnostic sink failure must not invalidate a completed migration. - } - return result - } - if (appliedCount === manifest.length) { - return complete({ adoptedLegacy: false, applied: [], from, to: latest.id }) - } let hadApplicationTablesAtStart: boolean try { @@ -1171,6 +1434,51 @@ const migrateApplicationDatabaseWithManifest = async ( includeDeleteAfterSuccess: false }) } + + const requiresLegacyDraftManagedBridge = requiresLegacyDraftManagedFileVersionHistoryBridge( + ledger, + manifest + ) + let legacyDraftManagedBackupReady = false + if (requiresLegacyDraftManagedBridge) { + const managedMigration = manifest.find( + (migration) => + migration.id === managedFileVersionFoundationMigration.id && + migration.checksum === MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM + )! + options.onProgress?.({ phase: 'migrating', migrationId: managedMigration.id }) + await backupBeforeMigration(managedMigration) + legacyDraftManagedBackupReady = true + // Validate and apply the canonical suffix against an in-memory view. The durable legacy row + // remains in place until the canonical managed migration can replace it atomically. + ledger = ledger.filter(({ id }) => id !== LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID) + } + + const appliedCount = validateLedger(ledger, manifest) + const complete = async (result: SchemaMigrationResult): Promise => { + try { + await verifyCurrentRuntimeSchema(client) + if (adoptsManagedFileVersionFoundation) { + await verifyManagedFileVersionDomain(client) + } + } catch (error) { + throw classifyDatabaseFailure(error, 'validation', latest.id) + } + await reportDatabaseCompatibility(client, options) + await retireDatabaseMigrationBackups(client, manifest, options, { + throughMigrationId: latest.id, + includeDeleteAfterSuccess: true + }) + try { + options.onCompleted?.(result) + } catch { + // A diagnostic sink failure must not invalidate a completed migration. + } + return result + } + if (appliedCount === manifest.length) { + return complete({ adoptedLegacy: false, applied: [], from, to: latest.id }) + } const repairsPreviewStateForeignKeyViolations = manifest.some( (candidate) => candidate.id === projectPreviewStateOwnerFkMigration.id && @@ -1225,7 +1533,8 @@ const migrateApplicationDatabaseWithManifest = async ( client, baseline, repairsPreviewStateForeignKeyViolations, - allowedSuffixChecks + allowedSuffixChecks, + adoptsManagedFileVersionFoundation ) applied.push(baseline.id) nextIndex = 1 @@ -1233,8 +1542,27 @@ const migrateApplicationDatabaseWithManifest = async ( for (const migration of manifest.slice(nextIndex)) { options.onProgress?.({ phase: 'migrating', migrationId: migration.id }) - await backupBeforeMigration(migration) - await applyManifestMigration(client, migration) + if ( + !legacyDraftManagedBackupReady || + migration.id !== managedFileVersionFoundationMigration.id || + migration.checksum !== MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM + ) { + await backupBeforeMigration(migration) + } + await applyManifestMigration(client, migration, { + repairVisionEvidenceReference: + migration.id === managedFileVersionFoundationMigration.id && + migration.checksum === MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM, + legacyLedgerIdentityToReplace: + requiresLegacyDraftManagedBridge && + migration.id === managedFileVersionFoundationMigration.id && + migration.checksum === MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM + ? { + id: LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_ID, + checksum: LEGACY_DRAFT_MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM + } + : undefined + }) applied.push(migration.id) } @@ -1253,6 +1581,7 @@ export { DATABASE_DOMAIN_CONSTRAINTS_CHECKSUM, NOTIFICATION_ATTENTION_METADATA_CHECKSUM, DATABASE_JSON_CONSTRAINTS_CHECKSUM, + MANAGED_FILE_VERSION_FOUNDATION_CHECKSUM, VISION_EVIDENCE_CHECKSUM, COMPUTE_PASSWORD_AUTH_CHECKSUM, TAG_ORDERING_CHECKSUM, diff --git a/src/main/database/migrations/0013-managed-file-version-foundation.ts b/src/main/database/migrations/0013-managed-file-version-foundation.ts new file mode 100644 index 000000000..e322e20dd --- /dev/null +++ b/src/main/database/migrations/0013-managed-file-version-foundation.ts @@ -0,0 +1,510 @@ +const managedFileVersionFoundationStatements = [ + `ALTER TABLE "ArtifactVersionInput" RENAME TO "_0009_old_ArtifactVersionInput";`, + `ALTER TABLE "ArtifactVersion" RENAME TO "_0009_old_ArtifactVersion";`, + `ALTER TABLE "UploadVersion" RENAME TO "_0009_old_UploadVersion";`, + `ALTER TABLE "ArtifactLineage" RENAME TO "_0009_old_ArtifactLineage";`, + `ALTER TABLE "UploadFile" RENAME TO "_0009_old_UploadFile";`, + `CREATE TABLE "ArtifactLineage" ( + "id" TEXT NOT NULL PRIMARY KEY, + "projectId" TEXT NOT NULL, + "sessionId" TEXT NOT NULL, + "normalizedFilename" TEXT NOT NULL, + "filename" TEXT NOT NULL, + "currentVersionId" TEXT, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "ArtifactLineage_id_currentVersionId_fkey" FOREIGN KEY ("id", "currentVersionId") REFERENCES "ArtifactVersion" ("artifactId", "id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "ArtifactLineage_projectId_sessionId_fkey" FOREIGN KEY ("projectId", "sessionId") REFERENCES "FileOriginSession" ("projectId", "sessionId") ON DELETE RESTRICT ON UPDATE CASCADE + );`, + `INSERT INTO "ArtifactLineage" ( + "id", "projectId", "sessionId", "normalizedFilename", "filename", "createdAt", "updatedAt" + ) + SELECT "id", "projectId", "sessionId", "normalizedFilename", "filename", "createdAt", "updatedAt" + FROM "_0009_old_ArtifactLineage";`, + `CREATE TABLE "UploadFile" ( + "id" TEXT NOT NULL PRIMARY KEY, + "projectId" TEXT NOT NULL, + "sessionId" TEXT NOT NULL, + "filename" TEXT NOT NULL, + "originalFilename" TEXT NOT NULL, + "currentVersionId" TEXT, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "UploadFile_id_currentVersionId_fkey" FOREIGN KEY ("id", "currentVersionId") REFERENCES "UploadVersion" ("uploadFileId", "id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "UploadFile_projectId_sessionId_fkey" FOREIGN KEY ("projectId", "sessionId") REFERENCES "FileOriginSession" ("projectId", "sessionId") ON DELETE RESTRICT ON UPDATE CASCADE + );`, + `INSERT INTO "UploadFile" ( + "id", "projectId", "sessionId", "filename", "originalFilename", "createdAt", "updatedAt" + ) + SELECT "id", "projectId", "sessionId", "filename", "originalFilename", "createdAt", "updatedAt" + FROM "_0009_old_UploadFile";`, + `CREATE TABLE "UploadVersion" ( + "id" TEXT NOT NULL PRIMARY KEY, + "uploadFileId" TEXT NOT NULL, + "versionNumber" INTEGER NOT NULL, + "state" TEXT NOT NULL DEFAULT 'staging', + "originKind" TEXT NOT NULL DEFAULT 'user_upload', + "basedOnVersionId" TEXT, + "storageTag" TEXT, + "storedFilename" TEXT, + "writeOperationId" TEXT, + "contentStorageKey" TEXT NOT NULL, + "filename" TEXT NOT NULL, + "originalFilename" TEXT NOT NULL, + "contentType" TEXT, + "sizeBytes" BIGINT NOT NULL, + "checksum" TEXT NOT NULL, + "createdAt" DATETIME, + "registeredAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "UploadVersion_uploadFileId_fkey" FOREIGN KEY ("uploadFileId") REFERENCES "UploadFile" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT "UploadVersion_uploadFileId_basedOnVersionId_fkey" FOREIGN KEY ("uploadFileId", "basedOnVersionId") REFERENCES "UploadVersion" ("uploadFileId", "id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "UploadVersion_state_check" CHECK ("state" IN ('staging', 'ready')), + CONSTRAINT "UploadVersion_originKind_check" CHECK ("originKind" IN ('user_upload', 'user_edit', 'legacy')), + CONSTRAINT "UploadVersion_userEdit_check" CHECK (("originKind" <> 'user_edit' OR ("state" = 'ready' AND "basedOnVersionId" IS NOT NULL AND "storageTag" IS NOT NULL AND "storedFilename" IS NOT NULL))) + );`, + `INSERT INTO "UploadVersion" ( + "id", "uploadFileId", "versionNumber", "state", "originKind", "contentStorageKey", + "filename", "originalFilename", "contentType", "sizeBytes", "checksum", "createdAt", + "registeredAt", "updatedAt" + ) + SELECT "id", "uploadFileId", "versionNumber", "state", 'user_upload', "contentStorageKey", + "filename", "originalFilename", "contentType", "sizeBytes", "checksum", "createdAt", + "registeredAt", "updatedAt" + FROM "_0009_old_UploadVersion";`, + `UPDATE "UploadVersion" AS "current" + SET "basedOnVersionId" = ( + SELECT "previous"."id" + FROM "UploadVersion" AS "previous" + WHERE "previous"."uploadFileId" = "current"."uploadFileId" + AND "previous"."versionNumber" < "current"."versionNumber" + AND "previous"."state" = 'ready' + ORDER BY "previous"."versionNumber" DESC + LIMIT 1 + );`, + `CREATE TABLE "ArtifactVersion" ( + "id" TEXT NOT NULL PRIMARY KEY, + "artifactId" TEXT NOT NULL, + "versionNumber" INTEGER NOT NULL, + "filename" TEXT NOT NULL, + "originKind" TEXT NOT NULL DEFAULT 'agent_generated', + "basedOnVersionId" TEXT, + "storageTag" TEXT, + "storedFilename" TEXT, + "artifactRunId" TEXT, + "writeOperationId" TEXT, + "writeRequestChecksum" TEXT, + "rootFrameId" TEXT, + "agentFrameId" TEXT, + "messageBranchId" TEXT, + "runtimeSegmentId" TEXT, + "promptMessageId" TEXT, + "notebookSessionId" TEXT, + "producerRunId" TEXT, + "producerRunIndex" INTEGER, + "messageId" TEXT, + "messageSnapshotId" TEXT, + "state" TEXT NOT NULL DEFAULT 'staging', + "managedVisibleAt" DATETIME, + "contentStorageKey" TEXT NOT NULL, + "evidenceStorageKey" TEXT, + "contentType" TEXT, + "sizeBytes" BIGINT NOT NULL, + "checksum" TEXT NOT NULL, + "evidenceJson" TEXT, + "evidenceChecksum" TEXT, + "evidenceSchemaVersion" INTEGER, + "executionSnapshotJson" TEXT, + "executionSnapshotChecksum" TEXT, + "executionSnapshotStorageKey" TEXT, + "executionSnapshotSchemaVersion" INTEGER, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "ArtifactVersion_artifactId_fkey" FOREIGN KEY ("artifactId") REFERENCES "ArtifactLineage" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT "ArtifactVersion_artifactId_basedOnVersionId_fkey" FOREIGN KEY ("artifactId", "basedOnVersionId") REFERENCES "ArtifactVersion" ("artifactId", "id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "ArtifactVersion_messageSnapshotId_fkey" FOREIGN KEY ("messageSnapshotId") REFERENCES "ArtifactMessageSnapshot" ("id") ON DELETE SET NULL ON UPDATE CASCADE, + CONSTRAINT "ArtifactVersion_state_check" CHECK ("state" IN ('staging', 'pending', 'finalized')), + CONSTRAINT "ArtifactVersion_filename_check" CHECK (length("filename") > 0), + CONSTRAINT "ArtifactVersion_originKind_check" CHECK ("originKind" IN ('agent_generated', 'user_edit', 'legacy')), + CONSTRAINT "ArtifactVersion_provenance_check" CHECK ((("originKind" = 'agent_generated' AND "artifactRunId" IS NOT NULL AND "rootFrameId" IS NOT NULL AND "agentFrameId" IS NOT NULL AND "messageBranchId" IS NOT NULL AND "runtimeSegmentId" IS NOT NULL AND "promptMessageId" IS NOT NULL AND "evidenceStorageKey" IS NOT NULL AND "evidenceJson" IS NOT NULL AND "evidenceChecksum" IS NOT NULL AND "evidenceSchemaVersion" IS NOT NULL) OR ("originKind" = 'user_edit' AND "state" = 'finalized' AND "basedOnVersionId" IS NOT NULL AND "storageTag" IS NOT NULL AND "storedFilename" IS NOT NULL AND "artifactRunId" IS NULL AND "writeRequestChecksum" IS NULL AND "rootFrameId" IS NULL AND "agentFrameId" IS NULL AND "messageBranchId" IS NULL AND "runtimeSegmentId" IS NULL AND "promptMessageId" IS NULL AND "notebookSessionId" IS NULL AND "producerRunId" IS NULL AND "producerRunIndex" IS NULL AND "messageId" IS NULL AND "messageSnapshotId" IS NULL AND "evidenceStorageKey" IS NULL AND "evidenceJson" IS NULL AND "evidenceChecksum" IS NULL AND "evidenceSchemaVersion" IS NULL AND "executionSnapshotJson" IS NULL AND "executionSnapshotChecksum" IS NULL AND "executionSnapshotStorageKey" IS NULL AND "executionSnapshotSchemaVersion" IS NULL) OR "originKind" = 'legacy')), + CONSTRAINT "ArtifactVersion_evidenceJson_check" CHECK ("evidenceJson" IS NULL OR (json_valid("evidenceJson") AND json_type("evidenceJson") = 'object')), + CONSTRAINT "ArtifactVersion_executionSnapshotJson_check" CHECK ("executionSnapshotJson" IS NULL OR (json_valid("executionSnapshotJson") AND json_type("executionSnapshotJson") = 'object')), + CONSTRAINT "ArtifactVersion_executionSnapshotBundle_check" CHECK ((("executionSnapshotJson" IS NULL AND "executionSnapshotChecksum" IS NULL AND "executionSnapshotStorageKey" IS NULL AND "executionSnapshotSchemaVersion" IS NULL) OR ("executionSnapshotJson" IS NOT NULL AND "executionSnapshotChecksum" IS NOT NULL AND "executionSnapshotStorageKey" IS NOT NULL AND "executionSnapshotSchemaVersion" IS NOT NULL))) + );`, + `INSERT INTO "ArtifactVersion" ( + "id", "artifactId", "versionNumber", "filename", "originKind", "artifactRunId", + "writeOperationId", "writeRequestChecksum", "rootFrameId", "agentFrameId", "messageBranchId", + "runtimeSegmentId", "promptMessageId", "notebookSessionId", "producerRunId", + "producerRunIndex", "messageId", "messageSnapshotId", "state", "managedVisibleAt", "contentStorageKey", + "evidenceStorageKey", "contentType", "sizeBytes", "checksum", "evidenceJson", + "evidenceChecksum", "evidenceSchemaVersion", "executionSnapshotJson", + "executionSnapshotChecksum", "executionSnapshotStorageKey", "executionSnapshotSchemaVersion", + "createdAt", "updatedAt" + ) + SELECT + "id", "artifactId", "versionNumber", "filename", 'agent_generated', "artifactRunId", + "writeOperationId", "writeRequestChecksum", "rootFrameId", "agentFrameId", "messageBranchId", + "runtimeSegmentId", "promptMessageId", "notebookSessionId", "producerRunId", + "producerRunIndex", "messageId", "messageSnapshotId", "state", + CASE WHEN "state" = 'finalized' THEN "createdAt" ELSE NULL END, "contentStorageKey", + "evidenceStorageKey", "contentType", "sizeBytes", "checksum", "evidenceJson", + "evidenceChecksum", "evidenceSchemaVersion", "executionSnapshotJson", + "executionSnapshotChecksum", "executionSnapshotStorageKey", "executionSnapshotSchemaVersion", + "createdAt", "updatedAt" + FROM "_0009_old_ArtifactVersion" + ORDER BY "artifactId", "versionNumber";`, + `UPDATE "ArtifactVersion" AS "current" + SET "basedOnVersionId" = ( + SELECT "previous"."id" + FROM "ArtifactVersion" AS "previous" + WHERE "previous"."artifactId" = "current"."artifactId" + AND "previous"."versionNumber" < "current"."versionNumber" + AND "previous"."state" = 'finalized' + ORDER BY "previous"."versionNumber" DESC + LIMIT 1 + );`, + `CREATE TABLE "ArtifactVersionInput" ( + "id" TEXT NOT NULL PRIMARY KEY, + "artifactVersionId" TEXT NOT NULL, + "ordinal" INTEGER NOT NULL, + "inputFileVersionId" TEXT NOT NULL, + "sourceKind" TEXT NOT NULL, + "sourceFileId" TEXT NOT NULL, + "sourceArtifactVersionId" TEXT, + "sourceUploadVersionId" TEXT, + "sourceVersionNumber" INTEGER, + "sourceCreatedAt" DATETIME, + "sourceProjectId" TEXT NOT NULL, + "sourceSessionId" TEXT NOT NULL, + "filename" TEXT NOT NULL, + "contentType" TEXT, + "sizeBytes" BIGINT NOT NULL, + "checksum" TEXT NOT NULL, + "storageKey" TEXT NOT NULL, + "strongestAssociation" TEXT NOT NULL, + CONSTRAINT "ArtifactVersionInput_artifactVersionId_fkey" FOREIGN KEY ("artifactVersionId") REFERENCES "ArtifactVersion" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT "ArtifactVersionInput_sourceArtifactVersionId_fkey" FOREIGN KEY ("sourceArtifactVersionId") REFERENCES "ArtifactVersion" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "ArtifactVersionInput_sourceUploadVersionId_fkey" FOREIGN KEY ("sourceUploadVersionId") REFERENCES "UploadVersion" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "ArtifactVersionInput_sourceProjectId_sourceSessionId_fkey" FOREIGN KEY ("sourceProjectId", "sourceSessionId") REFERENCES "FileOriginSession" ("projectId", "sessionId") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "ArtifactVersionInput_sourceKind_check" CHECK ("sourceKind" IN ('artifact-version', 'upload-version')), + CONSTRAINT "ArtifactVersionInput_sourceIdentity_check" CHECK ((("sourceKind" = 'artifact-version' AND "sourceArtifactVersionId" IS NOT NULL AND "sourceUploadVersionId" IS NULL AND "inputFileVersionId" = "sourceArtifactVersionId") OR ("sourceKind" = 'upload-version' AND "sourceArtifactVersionId" IS NULL AND "sourceUploadVersionId" IS NOT NULL AND "inputFileVersionId" = "sourceUploadVersionId"))), + CONSTRAINT "ArtifactVersionInput_strongestAssociation_check" CHECK ("strongestAssociation" IN ('turn-attached', 'resolver-accessed', 'captured-version')) + );`, + `INSERT INTO "ArtifactVersionInput" SELECT * FROM "_0009_old_ArtifactVersionInput";`, + `DROP TABLE "_0009_old_ArtifactVersionInput";`, + `DROP TABLE "_0009_old_ArtifactVersion";`, + `DROP TABLE "_0009_old_UploadVersion";`, + `DROP TABLE "_0009_old_ArtifactLineage";`, + `DROP TABLE "_0009_old_UploadFile";`, + `UPDATE "ArtifactLineage" AS "lineage" + SET "currentVersionId" = ( + SELECT "version"."id" + FROM "ArtifactVersion" AS "version" + WHERE "version"."artifactId" = "lineage"."id" AND "version"."state" = 'finalized' + ORDER BY "version"."versionNumber" DESC + LIMIT 1 + );`, + `UPDATE "UploadFile" AS "file" + SET "currentVersionId" = ( + SELECT "version"."id" + FROM "UploadVersion" AS "version" + WHERE "version"."uploadFileId" = "file"."id" AND "version"."state" = 'ready' + ORDER BY "version"."versionNumber" DESC + LIMIT 1 + );`, + `UPDATE "ManagedFileSessionSync" AS "sync" + SET "filesRevision" = -1, "syncedAt" = CURRENT_TIMESTAMP + WHERE "sync"."deletedAt" IS NULL + AND NOT EXISTS ( + SELECT 1 FROM "ProjectDeletionIntent" AS "intent" + WHERE "intent"."projectId" = "sync"."projectId" + ) + AND EXISTS ( + SELECT 1 + FROM "ManagedFile" AS "managed" + WHERE "managed"."projectId" = "sync"."projectId" + AND "managed"."sessionId" = "sync"."sessionId" + AND "managed"."source" = 'upload' + AND "managed"."deletedAt" IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM "UploadFile" AS "file" + JOIN "UploadVersion" AS "version" + ON "version"."uploadFileId" = "file"."id" + AND "version"."id" = "file"."currentVersionId" + AND "version"."state" = 'ready' + WHERE "file"."id" = "managed"."sourceFileId" + AND "file"."projectId" = "managed"."projectId" + ) + );`, + `DELETE FROM "ManagedFile" + WHERE "source" = 'artifact' + AND NOT EXISTS ( + SELECT 1 + FROM "ArtifactLineage" AS "lineage" + JOIN "ArtifactVersion" AS "version" + ON "version"."artifactId" = "lineage"."id" + AND "version"."id" = "lineage"."currentVersionId" + AND "version"."state" = 'finalized' + WHERE "lineage"."id" = "ManagedFile"."sourceFileId" + AND "lineage"."projectId" = "ManagedFile"."projectId" + );`, + `DELETE FROM "ManagedFile" + WHERE "source" = 'upload' + AND NOT EXISTS ( + SELECT 1 + FROM "UploadFile" AS "file" + JOIN "UploadVersion" AS "version" + ON "version"."uploadFileId" = "file"."id" + AND "version"."id" = "file"."currentVersionId" + AND "version"."state" = 'ready' + WHERE "file"."id" = "ManagedFile"."sourceFileId" + AND "file"."projectId" = "ManagedFile"."projectId" + );`, + `UPDATE "ManagedFile" + SET ("sourceVersionId", "checksum", "projectId", "sessionId", "messageId", "displayName", + "storageKey", "mimeType", "sizeBytes", "mtimeMs", "sortAtMs", "updatedAt") = ( + SELECT "version"."id", "version"."checksum", "lineage"."projectId", "lineage"."sessionId", + "version"."messageId", "version"."filename", "version"."contentStorageKey", + "version"."contentType", "version"."sizeBytes", + CASE WHEN typeof("version"."createdAt") IN ('integer', 'real') + THEN CAST("version"."createdAt" AS INTEGER) + ELSE CAST(strftime('%s', "version"."createdAt") AS INTEGER) * 1000 END, + CASE WHEN typeof("version"."createdAt") IN ('integer', 'real') + THEN CAST("version"."createdAt" AS INTEGER) + ELSE CAST(strftime('%s', "version"."createdAt") AS INTEGER) * 1000 END, + CURRENT_TIMESTAMP + FROM "ArtifactLineage" AS "lineage" + JOIN "ArtifactVersion" AS "version" + ON "version"."artifactId" = "lineage"."id" + AND "version"."id" = "lineage"."currentVersionId" + AND "version"."state" = 'finalized' + WHERE "lineage"."id" = "ManagedFile"."sourceFileId" + AND "lineage"."projectId" = "ManagedFile"."projectId" + ) + WHERE "source" = 'artifact';`, + `UPDATE "ManagedFile" + SET ("sourceVersionId", "checksum", "projectId", "sessionId", "displayName", + "storageKey", "mimeType", "sizeBytes", "mtimeMs", "updatedAt") = ( + SELECT "version"."id", "version"."checksum", "file"."projectId", "file"."sessionId", + COALESCE(NULLIF("version"."originalFilename", ''), "version"."filename"), + "version"."contentStorageKey", "version"."contentType", + "version"."sizeBytes", + CASE WHEN typeof(COALESCE("version"."createdAt", "version"."registeredAt")) IN ('integer', 'real') + THEN CAST(COALESCE("version"."createdAt", "version"."registeredAt") AS INTEGER) + ELSE CAST(strftime('%s', COALESCE("version"."createdAt", "version"."registeredAt")) AS INTEGER) * 1000 END, + CURRENT_TIMESTAMP + FROM "UploadFile" AS "file" + JOIN "UploadVersion" AS "version" + ON "version"."uploadFileId" = "file"."id" + AND "version"."id" = "file"."currentVersionId" + AND "version"."state" = 'ready' + WHERE "file"."id" = "ManagedFile"."sourceFileId" + AND "file"."projectId" = "ManagedFile"."projectId" + ) + WHERE "source" = 'upload';`, + `UPDATE "ManagedFileSessionSync" AS "sync" + SET "filesRevision" = -1, "syncedAt" = CURRENT_TIMESTAMP + WHERE "sync"."deletedAt" IS NULL + AND NOT EXISTS ( + SELECT 1 FROM "ProjectDeletionIntent" AS "intent" + WHERE "intent"."projectId" = "sync"."projectId" + ) + AND ( + EXISTS ( + SELECT 1 + FROM "ArtifactLineage" AS "lineage" + JOIN "ArtifactVersion" AS "version" + ON "version"."artifactId" = "lineage"."id" + AND "version"."id" = "lineage"."currentVersionId" + AND "version"."state" = 'finalized' + WHERE "lineage"."projectId" = "sync"."projectId" + AND "lineage"."sessionId" = "sync"."sessionId" + AND NOT EXISTS ( + SELECT 1 FROM "ManagedFile" + WHERE "projectId" = "lineage"."projectId" + AND "source" = 'artifact' + AND "sourceFileId" = "lineage"."id" + ) + ) + OR EXISTS ( + SELECT 1 + FROM "UploadFile" AS "file" + JOIN "UploadVersion" AS "version" + ON "version"."uploadFileId" = "file"."id" + AND "version"."id" = "file"."currentVersionId" + AND "version"."state" = 'ready' + WHERE "file"."projectId" = "sync"."projectId" + AND "file"."sessionId" = "sync"."sessionId" + AND NOT EXISTS ( + SELECT 1 FROM "ManagedFile" + WHERE "projectId" = "file"."projectId" + AND "source" = 'upload' + AND "sourceFileId" = "file"."id" + ) + ) + );`, + `INSERT INTO "ManagedFile" + ("source", "sourceFileId", "sourceVersionId", "checksum", "projectId", "sessionId", + "messageId", "displayName", "storageKey", "mimeType", "sizeBytes", "mtimeMs", "sortAtMs", + "createdAt", "updatedAt") + SELECT 'artifact', "lineage"."id", "version"."id", "version"."checksum", + "lineage"."projectId", "lineage"."sessionId", "version"."messageId", "version"."filename", + "version"."contentStorageKey", "version"."contentType", "version"."sizeBytes", + CASE WHEN typeof("version"."createdAt") IN ('integer', 'real') + THEN CAST("version"."createdAt" AS INTEGER) + ELSE CAST(strftime('%s', "version"."createdAt") AS INTEGER) * 1000 END, + CASE WHEN typeof("version"."createdAt") IN ('integer', 'real') + THEN CAST("version"."createdAt" AS INTEGER) + ELSE CAST(strftime('%s', "version"."createdAt") AS INTEGER) * 1000 END, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM "ArtifactLineage" AS "lineage" + JOIN "ArtifactVersion" AS "version" + ON "version"."artifactId" = "lineage"."id" + AND "version"."id" = "lineage"."currentVersionId" + AND "version"."state" = 'finalized' + WHERE NOT EXISTS ( + SELECT 1 FROM "ManagedFile" + WHERE "projectId" = "lineage"."projectId" + AND "source" = 'artifact' + AND "sourceFileId" = "lineage"."id" + ) + AND NOT EXISTS ( + SELECT 1 FROM "ManagedFileSessionSync" AS "sync" + WHERE "sync"."projectId" = "lineage"."projectId" + AND "sync"."sessionId" = "lineage"."sessionId" + AND "sync"."deletedAt" IS NOT NULL + ) + AND NOT EXISTS ( + SELECT 1 FROM "ProjectDeletionIntent" AS "intent" + WHERE "intent"."projectId" = "lineage"."projectId" + );`, + `INSERT INTO "ManagedFile" + ("source", "sourceFileId", "sourceVersionId", "checksum", "projectId", "sessionId", + "messageId", "displayName", "storageKey", "mimeType", "sizeBytes", "mtimeMs", "sortAtMs", + "createdAt", "updatedAt") + SELECT 'upload', "file"."id", "version"."id", "version"."checksum", "file"."projectId", + "file"."sessionId", NULL, + COALESCE(NULLIF("version"."originalFilename", ''), "version"."filename"), + "version"."contentStorageKey", + "version"."contentType", "version"."sizeBytes", + CASE WHEN typeof(COALESCE("version"."createdAt", "version"."registeredAt")) IN ('integer', 'real') + THEN CAST(COALESCE("version"."createdAt", "version"."registeredAt") AS INTEGER) + ELSE CAST(strftime('%s', COALESCE("version"."createdAt", "version"."registeredAt")) AS INTEGER) * 1000 END, + CASE WHEN typeof(COALESCE("version"."createdAt", "version"."registeredAt")) IN ('integer', 'real') + THEN CAST(COALESCE("version"."createdAt", "version"."registeredAt") AS INTEGER) + ELSE CAST(strftime('%s', COALESCE("version"."createdAt", "version"."registeredAt")) AS INTEGER) * 1000 END, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + FROM "UploadFile" AS "file" + JOIN "UploadVersion" AS "version" + ON "version"."uploadFileId" = "file"."id" + AND "version"."id" = "file"."currentVersionId" + AND "version"."state" = 'ready' + WHERE NOT EXISTS ( + SELECT 1 FROM "ManagedFile" + WHERE "projectId" = "file"."projectId" + AND "source" = 'upload' + AND "sourceFileId" = "file"."id" + ) + AND NOT EXISTS ( + SELECT 1 FROM "ManagedFileSessionSync" AS "sync" + WHERE "sync"."projectId" = "file"."projectId" + AND "sync"."sessionId" = "file"."sessionId" + AND "sync"."deletedAt" IS NOT NULL + ) + AND NOT EXISTS ( + SELECT 1 FROM "ProjectDeletionIntent" AS "intent" + WHERE "intent"."projectId" = "file"."projectId" + );`, + `CREATE TABLE "ManagedFileVersionWriteOperation" ( + "operationId" TEXT NOT NULL PRIMARY KEY, + "source" TEXT NOT NULL, + "projectId" TEXT NOT NULL, + "sourceFileId" TEXT NOT NULL, + "basedOnVersionId" TEXT NOT NULL, + "expectedHeadVersionId" TEXT NOT NULL, + "state" TEXT NOT NULL DEFAULT 'staging', + "storageTag" TEXT NOT NULL, + "storedFilename" TEXT NOT NULL, + "contentStorageKey" TEXT NOT NULL, + "checksum" TEXT NOT NULL, + "sizeBytes" BIGINT NOT NULL, + "textFormatJson" TEXT NOT NULL, + "resultVersionId" TEXT, + "errorCode" TEXT, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" DATETIME NOT NULL, + CONSTRAINT "ManagedFileVersionWriteOperation_source_check" CHECK ("source" IN ('artifact', 'upload')), + CONSTRAINT "ManagedFileVersionWriteOperation_state_check" CHECK ("state" IN ('staging', 'file_ready', 'published', 'conflict', 'failed')) + );`, + `CREATE UNIQUE INDEX "ArtifactLineage_currentVersionId_key" ON "ArtifactLineage"("currentVersionId");`, + `CREATE UNIQUE INDEX "ArtifactLineage_id_currentVersionId_key" ON "ArtifactLineage"("id", "currentVersionId");`, + `CREATE INDEX "ArtifactLineage_projectId_sessionId_idx" ON "ArtifactLineage"("projectId", "sessionId");`, + `CREATE UNIQUE INDEX "ArtifactLineage_projectId_sessionId_normalizedFilename_key" ON "ArtifactLineage"("projectId", "sessionId", "normalizedFilename");`, + `CREATE UNIQUE INDEX "UploadFile_currentVersionId_key" ON "UploadFile"("currentVersionId");`, + `CREATE UNIQUE INDEX "UploadFile_id_currentVersionId_key" ON "UploadFile"("id", "currentVersionId");`, + `CREATE INDEX "UploadFile_projectId_sessionId_idx" ON "UploadFile"("projectId", "sessionId");`, + `CREATE UNIQUE INDEX "UploadVersion_writeOperationId_key" ON "UploadVersion"("writeOperationId");`, + `CREATE UNIQUE INDEX "UploadVersion_contentStorageKey_key" ON "UploadVersion"("contentStorageKey");`, + `CREATE INDEX "UploadVersion_uploadFileId_state_registeredAt_idx" ON "UploadVersion"("uploadFileId", "state", "registeredAt");`, + `CREATE UNIQUE INDEX "UploadVersion_uploadFileId_versionNumber_key" ON "UploadVersion"("uploadFileId", "versionNumber");`, + `CREATE UNIQUE INDEX "UploadVersion_uploadFileId_id_key" ON "UploadVersion"("uploadFileId", "id");`, + `CREATE UNIQUE INDEX "ArtifactVersion_writeOperationId_key" ON "ArtifactVersion"("writeOperationId");`, + `CREATE UNIQUE INDEX "ArtifactVersion_contentStorageKey_key" ON "ArtifactVersion"("contentStorageKey");`, + `CREATE INDEX "ArtifactVersion_artifactId_createdAt_idx" ON "ArtifactVersion"("artifactId", "createdAt");`, + `CREATE INDEX "ArtifactVersion_artifactRunId_state_idx" ON "ArtifactVersion"("artifactRunId", "state");`, + `CREATE INDEX "ArtifactVersion_rootFrameId_agentFrameId_messageBranchId_promptMessageId_idx" ON "ArtifactVersion"("rootFrameId", "agentFrameId", "messageBranchId", "promptMessageId");`, + `CREATE INDEX "ArtifactVersion_messageId_idx" ON "ArtifactVersion"("messageId");`, + `CREATE INDEX "ArtifactVersion_messageSnapshotId_idx" ON "ArtifactVersion"("messageSnapshotId");`, + `CREATE UNIQUE INDEX "ArtifactVersion_artifactId_versionNumber_key" ON "ArtifactVersion"("artifactId", "versionNumber");`, + `CREATE UNIQUE INDEX "ArtifactVersion_artifactId_id_key" ON "ArtifactVersion"("artifactId", "id");`, + `CREATE INDEX "ArtifactVersionInput_sourceKind_inputFileVersionId_idx" ON "ArtifactVersionInput"("sourceKind", "inputFileVersionId");`, + `CREATE INDEX "ArtifactVersionInput_sourceArtifactVersionId_idx" ON "ArtifactVersionInput"("sourceArtifactVersionId");`, + `CREATE INDEX "ArtifactVersionInput_sourceUploadVersionId_idx" ON "ArtifactVersionInput"("sourceUploadVersionId");`, + `CREATE INDEX "ArtifactVersionInput_sourceProjectId_sourceSessionId_idx" ON "ArtifactVersionInput"("sourceProjectId", "sourceSessionId");`, + `CREATE UNIQUE INDEX "ArtifactVersionInput_artifactVersionId_sourceKind_inputFileVersionId_key" ON "ArtifactVersionInput"("artifactVersionId", "sourceKind", "inputFileVersionId");`, + `CREATE UNIQUE INDEX "ArtifactVersionInput_artifactVersionId_ordinal_key" ON "ArtifactVersionInput"("artifactVersionId", "ordinal");`, + `CREATE UNIQUE INDEX "ManagedFileVersionWriteOperation_contentStorageKey_key" ON "ManagedFileVersionWriteOperation"("contentStorageKey");`, + `CREATE UNIQUE INDEX "ManagedFileVersionWriteOperation_resultVersionId_key" ON "ManagedFileVersionWriteOperation"("resultVersionId");`, + `CREATE INDEX "ManagedFileVersionWriteOperation_source_sourceFileId_state_idx" ON "ManagedFileVersionWriteOperation"("source", "sourceFileId", "state");`, + `CREATE INDEX "ManagedFileVersionWriteOperation_projectId_state_createdAt_idx" ON "ManagedFileVersionWriteOperation"("projectId", "state", "createdAt");` +] as const + +const managedFileVersionFoundationMigration = { + id: '0013_managed_file_version_foundation', + statements: managedFileVersionFoundationStatements, + verifiers: [ + { + kind: 'table-exists', + version: 1, + table: 'ManagedFileVersionWriteOperation' + }, + { + kind: 'foreign-key-integrity', + version: 1 + }, + { + kind: 'managed-file-version-domain', + version: 1 + } + ] +} as const + +// These statements are idempotent against an already-current schema. They let a pre-ledger +// current database refresh its derived heads and ManagedFile projection without replaying DDL. +const currentSchemaAdoptionStart = managedFileVersionFoundationStatements.findIndex((statement) => + statement.startsWith('UPDATE "ArtifactLineage" AS "lineage"') +) +const currentSchemaAdoptionEnd = managedFileVersionFoundationStatements.findIndex((statement) => + statement.startsWith('CREATE TABLE "ManagedFileVersionWriteOperation"') +) +if (currentSchemaAdoptionStart < 0 || currentSchemaAdoptionEnd <= currentSchemaAdoptionStart) { + throw new Error('Managed file version migration adoption boundaries are invalid.') +} +const managedFileVersionFoundationCurrentSchemaAdoptionStatements = + managedFileVersionFoundationStatements.slice(currentSchemaAdoptionStart, currentSchemaAdoptionEnd) + +export { + managedFileVersionFoundationCurrentSchemaAdoptionStatements, + managedFileVersionFoundationMigration +} diff --git a/src/main/database/notification-attention-metadata-migration.test.ts b/src/main/database/notification-attention-metadata-migration.test.ts index aaa3da63b..203198496 100644 --- a/src/main/database/notification-attention-metadata-migration.test.ts +++ b/src/main/database/notification-attention-metadata-migration.test.ts @@ -71,18 +71,22 @@ describe('notification attention metadata migration', () => { '0009_vision_evidence', '0010_compute_password_auth', '0011_cross_resource_tags', - '0012_tag_ordering' + '0012_tag_ordering', + '0013_managed_file_version_foundation' ], from: '0006_database_domain_constraints', - to: '0012_tag_ordering' + to: '0013_managed_file_version_foundation' }) await expect( access(`${databasePath}.before-0007_notification_attention_metadata.backup`) ).rejects.toMatchObject({ code: 'ENOENT' }) await expect( access(`${databasePath}.before-0011_cross_resource_tags.backup`) - ).resolves.toBeUndefined() + ).rejects.toMatchObject({ code: 'ENOENT' }) await expect(access(`${databasePath}.before-0012_tag_ordering.backup`)).resolves.toBeUndefined() + await expect( + access(`${databasePath}.before-0013_managed_file_version_foundation.backup`) + ).resolves.toBeUndefined() await expect( client.$queryRaw< diff --git a/src/main/database/sqlite-schema-migrations.ts b/src/main/database/sqlite-schema-migrations.ts index 1293d2095..087443c20 100644 --- a/src/main/database/sqlite-schema-migrations.ts +++ b/src/main/database/sqlite-schema-migrations.ts @@ -383,10 +383,14 @@ const findPendingSqliteCheckConstraints = async ( const applySqliteCheckConstraints = async ( client: SqliteExecutor, - pending: readonly SqliteCheckConstraintMigration[] + pending: readonly SqliteCheckConstraintMigration[], + postRebuildStatements: readonly string[] = [] ): Promise => { for (const migration of pending) await validateExistingValues(client, migration) for (const migration of pending) await rebuildTable(client, migration) + for (const statement of postRebuildStatements) { + await migrationSqlExecutor.execute(client, statement) + } const violations = await migrationSqlExecutor.query( client, diff --git a/src/main/delegation/production-composition.test.ts b/src/main/delegation/production-composition.test.ts index ef6e8462c..5e638e373 100644 --- a/src/main/delegation/production-composition.test.ts +++ b/src/main/delegation/production-composition.test.ts @@ -2159,6 +2159,10 @@ describe('production delegated-work composition', () => { finalizeRun: async (request) => { await ownership.validateFinalizationOwnership(request) return versionsByRun.get(request.artifactRunId) ?? [] + }, + activateFinalizedRun: async (request) => { + await ownership.validateFinalizationOwnership(request) + return versionsByRun.get(request.artifactRunId) ?? [] } } as unknown as ArtifactProvenanceRepository }) diff --git a/src/main/file-save.test.ts b/src/main/file-save.test.ts index 38ce93953..f6d4dc1d2 100644 --- a/src/main/file-save.test.ts +++ b/src/main/file-save.test.ts @@ -1,11 +1,13 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { unzipSync } from 'fflate' +import { createHash } from 'node:crypto' import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { Readable } from 'node:stream' const downloadsPath = join('/Users/example', 'Downloads') +const sha256 = (bytes: string): string => createHash('sha256').update(bytes).digest('hex') const handlers = new Map unknown>() const getAppPath = vi.hoisted(() => vi.fn()) @@ -276,6 +278,210 @@ describe('file save IPC handlers', () => { expect(handlers.has('file:save-managed')).toBe(true) }) + it('resolves a logical managed file only after Save As confirms so a newer DB head is exported', async () => { + let headPath = '/managed/v1-report.csv' + const resolveManagedFilePath = vi.fn(async (_source, request) => { + expect(request).toEqual({ + path: 'artifact-version:stale-v1', + projectId: 'project-1', + fileId: 'artifact-1' + }) + return headPath + }) + const openManagedFile = vi.fn().mockResolvedValue({ + copyTo: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockResolvedValue(undefined) + }) + showSaveDialog.mockImplementation(async () => { + headPath = '/managed/v2-report.csv' + return { canceled: false, filePath: join(downloadsPath, 'report.csv') } + }) + registerFileSaveHandlers({ resolveManagedFilePath, openManagedFile } as never) + + await handlers.get('file:save-managed')!( + { sender: {} }, + { + source: 'artifact', + path: 'artifact-version:stale-v1', + projectId: 'project-1', + fileId: 'artifact-1', + suggestedName: 'report.csv' + } + ) + + expect(openManagedFile).toHaveBeenCalledWith('/managed/v2-report.csv') + }) + + it('opens a trusted logical-file lease after Save As without reopening its resolved path', async () => { + const resolveManagedFilePath = vi.fn().mockResolvedValue('/managed/path-must-not-be-used.csv') + const copyTo = vi.fn().mockResolvedValue(undefined) + const close = vi.fn().mockResolvedValue(undefined) + const openManagedFileVersion = vi.fn().mockResolvedValue({ copyTo, close }) + showSaveDialog.mockResolvedValue({ + canceled: false, + filePath: join(downloadsPath, 'report.csv') + }) + registerFileSaveHandlers({ resolveManagedFilePath, openManagedFileVersion } as never) + + await handlers.get('file:save-managed')!( + { sender: {} }, + { + source: 'artifact', + path: 'artifact-version:stale-v1', + projectId: 'project-1', + fileId: 'artifact-1', + suggestedName: 'report.csv' + } + ) + + expect(openManagedFileVersion).toHaveBeenCalledWith('artifact', { + projectId: 'project-1', + fileId: 'artifact-1' + }) + expect(resolveManagedFilePath).not.toHaveBeenCalled() + expect(copyTo).toHaveBeenCalledWith(join(downloadsPath, 'report.csv')) + expect(close).toHaveBeenCalledOnce() + }) + + it('preserves an explicit historical version when exporting a logical managed file', async () => { + const resolveManagedFilePath = vi.fn().mockResolvedValue('/managed/v1-report.csv') + const openManagedFile = vi.fn().mockResolvedValue({ + copyTo: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockResolvedValue(undefined) + }) + showSaveDialog.mockResolvedValue({ + canceled: false, + filePath: join(downloadsPath, 'report.csv') + }) + registerFileSaveHandlers({ resolveManagedFilePath, openManagedFile } as never) + + await handlers.get('file:save-managed')!( + { sender: {} }, + { + source: 'artifact', + path: 'artifact-version:v1', + projectId: 'project-1', + fileId: 'artifact-1', + versionId: 'version-1', + suggestedName: 'report.csv' + } + ) + + expect(resolveManagedFilePath).toHaveBeenCalledWith('artifact', { + path: 'artifact-version:v1', + projectId: 'project-1', + fileId: 'artifact-1', + versionId: 'version-1' + }) + }) + + it('resolves every logical Session Artifact after the destination folder is chosen', async () => { + const destinationDirectory = '/downloads/session-artifacts' + const resolveManagedFilePath = vi + .fn() + .mockResolvedValueOnce('/managed/a-v2.csv') + .mockResolvedValueOnce('/managed/b-v4.csv') + const openManagedFile = vi.fn().mockResolvedValue({ + copyTo: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockResolvedValue(undefined) + }) + showOpenDialog.mockResolvedValue({ canceled: false, filePaths: [destinationDirectory] }) + registerFileSaveHandlers({ resolveManagedFilePath, openManagedFile } as never) + + await handlers.get('file:save-session-artifacts')!( + { sender: {} }, + { + projectId: 'project-1', + sessionId: 'session-1', + files: [ + { path: 'stale-a', fileId: 'artifact-a', suggestedName: 'a.csv' }, + { path: 'stale-b', fileId: 'artifact-b', suggestedName: 'b.csv' } + ] + } + ) + + expect(resolveManagedFilePath).toHaveBeenNthCalledWith(1, 'artifact', { + path: 'stale-a', + projectId: 'project-1', + sessionId: 'session-1', + fileId: 'artifact-a' + }) + expect(resolveManagedFilePath).toHaveBeenNthCalledWith(2, 'artifact', { + path: 'stale-b', + projectId: 'project-1', + sessionId: 'session-1', + fileId: 'artifact-b' + }) + }) + + it('exports each logical Session Artifact through its own trusted lease and closes every lease', async () => { + const destinationDirectory = '/downloads/session-artifacts' + const first = { + copyTo: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockResolvedValue(undefined) + } + const second = { + copyTo: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockResolvedValue(undefined) + } + const openManagedFileVersion = vi + .fn() + .mockResolvedValueOnce(first) + .mockResolvedValueOnce(second) + const resolveManagedFilePath = vi.fn() + showOpenDialog.mockResolvedValue({ canceled: false, filePaths: [destinationDirectory] }) + registerFileSaveHandlers({ resolveManagedFilePath, openManagedFileVersion } as never) + + await handlers.get('file:save-session-artifacts')!( + { sender: {} }, + { + projectId: 'project-1', + sessionId: 'session-1', + files: [ + { path: 'stale-a', fileId: 'artifact-a', suggestedName: 'a.csv' }, + { + path: 'stale-b', + fileId: 'artifact-b', + versionId: 'artifact-b-v1', + suggestedName: 'b.csv' + } + ] + } + ) + + expect(openManagedFileVersion).toHaveBeenNthCalledWith(1, 'artifact', { + projectId: 'project-1', + fileId: 'artifact-a' + }) + expect(openManagedFileVersion).toHaveBeenNthCalledWith(2, 'artifact', { + projectId: 'project-1', + fileId: 'artifact-b', + versionId: 'artifact-b-v1' + }) + expect(resolveManagedFilePath).not.toHaveBeenCalled() + expect(first.close).toHaveBeenCalledOnce() + expect(second.close).toHaveBeenCalledOnce() + }) + + it('passes an Upload logical identity to the source-neutral export resolver', async () => { + const resolveManagedFilePath = vi.fn().mockResolvedValue('/managed/upload-v3.csv') + showSaveDialog.mockResolvedValue({ canceled: true }) + registerFileSaveHandlers({ resolveManagedFilePath } as never) + + await handlers.get('file:save-managed')!( + { sender: {} }, + { + source: 'upload', + path: 'upload-version:stale-v1', + projectId: 'project-1', + fileId: 'upload-1', + suggestedName: 'study.csv' + } + ) + + expect(resolveManagedFilePath).not.toHaveBeenCalled() + }) + it('opens a managed source once and copies that exact file to the selected destination', async () => { const resolveManagedFilePath = vi.fn().mockResolvedValue('/managed/canonical-report.csv') const copyTo = vi.fn().mockResolvedValue(undefined) @@ -658,6 +864,127 @@ describe('file save IPC handlers', () => { } }) + it('reads each logical Project file from the current managed-file head', async () => { + const root = await mkdtemp(join(tmpdir(), 'open-science-save-project-current-head-')) + const destinationPath = join(root, 'Research-artifacts.zip') + const resolveManagedFilePath = vi.fn().mockRejectedValue(new Error('stale path used')) + const resolveSessionArtifactFilePath = vi.fn().mockRejectedValue(new Error('stale path used')) + const closeArtifact = vi.fn().mockResolvedValue(undefined) + const closeUpload = vi.fn().mockResolvedValue(undefined) + const verifyArtifact = vi.fn().mockResolvedValue(undefined) + const verifyUpload = vi.fn().mockResolvedValue(undefined) + const openManagedFileVersion = vi + .fn() + .mockResolvedValueOnce({ + size: 21, + readRange: vi.fn().mockResolvedValue(Buffer.from('current artifact head')), + verifyUnchanged: verifyArtifact, + copyTo: vi.fn(), + close: closeArtifact + }) + .mockResolvedValueOnce({ + size: 19, + readRange: vi.fn().mockResolvedValue(Buffer.from('current upload head')), + verifyUnchanged: verifyUpload, + copyTo: vi.fn(), + close: closeUpload + }) + showSaveDialog.mockResolvedValue({ canceled: false, filePath: destinationPath }) + registerFileSaveHandlers({ + resolveManagedFilePath, + resolveSessionArtifactFilePath, + openManagedFileVersion + } as never) + + try { + const result = await handlers.get('file:save-project-artifacts')!( + { sender: {} }, + { + projectId: 'project-1', + suggestedArchiveName: 'Research', + files: [ + { + source: 'artifact', + sessionId: 'session-1', + path: '/stale/report.csv', + fileId: 'artifact-file-1', + suggestedName: 'report.csv' + }, + { + source: 'upload', + sessionId: 'session-2', + path: '/stale/data.csv', + fileId: 'upload-file-1', + suggestedName: 'data.csv' + } + ] + } + ) + + expect(result).toEqual({ saved: true, filePath: destinationPath }) + expect(openManagedFileVersion.mock.calls).toEqual([ + ['artifact', { projectId: 'project-1', fileId: 'artifact-file-1' }], + ['upload', { projectId: 'project-1', fileId: 'upload-file-1' }] + ]) + expect(resolveManagedFilePath).not.toHaveBeenCalled() + expect(resolveSessionArtifactFilePath).not.toHaveBeenCalled() + const entries = unzipSync(new Uint8Array(await readFile(destinationPath))) + expect(Buffer.from(entries['generated/report.csv']!).toString('utf8')).toBe( + 'current artifact head' + ) + expect(Buffer.from(entries['uploads/data.csv']!).toString('utf8')).toBe('current upload head') + expect(verifyArtifact).toHaveBeenCalledOnce() + expect(verifyUpload).toHaveBeenCalledOnce() + expect(closeArtifact).toHaveBeenCalledOnce() + expect(closeUpload).toHaveBeenCalledOnce() + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it('closes a retained Project Version lease when temporary archive setup fails', async () => { + const close = vi.fn().mockResolvedValue(undefined) + const readRange = vi.fn() + const verifyUnchanged = vi.fn() + const temporaryRootError = new Error('temporary storage unavailable') + showSaveDialog.mockResolvedValue({ + canceled: false, + filePath: join(downloadsPath, 'Research-artifacts.zip') + }) + registerFileSaveHandlers({ + openManagedFileVersion: vi.fn().mockResolvedValue({ + size: 1, + readRange, + verifyUnchanged, + copyTo: vi.fn(), + close + }), + createProjectArtifactTemporaryRoot: vi.fn().mockRejectedValue(temporaryRootError) + } as never) + + await expect( + handlers.get('file:save-project-artifacts')!( + { sender: {} }, + { + projectId: 'project-1', + suggestedArchiveName: 'Research', + files: [ + { + source: 'artifact', + sessionId: 'session-1', + path: 'artifact://report', + fileId: 'artifact-file-1', + suggestedName: 'report.csv' + } + ] + } + ) + ).rejects.toBe(temporaryRootError) + expect(readRange).not.toHaveBeenCalled() + expect(verifyUnchanged).not.toHaveBeenCalled() + expect(close).toHaveBeenCalledOnce() + }) + it('exports Project Artifacts without reading an entire source into memory', async () => { const root = await mkdtemp(join(tmpdir(), 'open-science-save-project-stream-')) const destinationPath = join(root, 'Research-artifacts.zip') @@ -766,6 +1093,267 @@ describe('file save IPC handlers', () => { } }) + it('keeps an explicit Project file version pinned during zip export', async () => { + const root = await mkdtemp(join(tmpdir(), 'open-science-save-project-exact-version-')) + const destinationPath = join(root, 'Research-artifacts.zip') + const close = vi.fn().mockResolvedValue(undefined) + const openManagedFileVersion = vi.fn().mockResolvedValue({ + size: 18, + readRange: vi.fn().mockResolvedValue(Buffer.from('historical version')), + verifyUnchanged: vi.fn().mockResolvedValue(undefined), + copyTo: vi.fn(), + close + }) + showSaveDialog.mockResolvedValue({ canceled: false, filePath: destinationPath }) + registerFileSaveHandlers({ openManagedFileVersion } as never) + + try { + const result = await handlers.get('file:save-project-artifacts')!( + { sender: {} }, + { + projectId: 'project-1', + suggestedArchiveName: 'Research', + files: [ + { + source: 'artifact', + sessionId: 'session-1', + path: '/stale/report.csv', + fileId: 'artifact-file-1', + versionId: 'artifact-version-2', + suggestedName: 'report.csv' + } + ] + } + ) + + expect(result).toEqual({ saved: true, filePath: destinationPath }) + expect(openManagedFileVersion).toHaveBeenCalledWith('artifact', { + projectId: 'project-1', + fileId: 'artifact-file-1', + versionId: 'artifact-version-2' + }) + const entries = unzipSync(new Uint8Array(await readFile(destinationPath))) + expect(Buffer.from(entries['generated/report.csv']!).toString('utf8')).toBe( + 'historical version' + ) + expect(close).toHaveBeenCalledOnce() + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it('falls back to the legacy managed path only when anchored Version reads are unavailable', async () => { + const root = await mkdtemp(join(tmpdir(), 'open-science-save-project-native-fallback-')) + const sourcePath = join(root, 'managed-report.csv') + const destinationPath = join(root, 'Research-artifacts.zip') + await writeFile(sourcePath, 'legacy fallback bytes') + const resolveManagedFilePath = vi.fn().mockResolvedValue({ + path: sourcePath, + expectedSize: Buffer.byteLength('legacy fallback bytes'), + expectedChecksum: sha256('legacy fallback bytes') + }) + const resolveSessionArtifactFilePath = vi.fn() + const openManagedFileVersion = vi.fn().mockRejectedValue( + Object.assign(new Error('Native anchored managed-file access is unavailable.'), { + code: 'NATIVE_WRITE_REQUIRED' + }) + ) + showSaveDialog.mockResolvedValue({ canceled: false, filePath: destinationPath }) + registerFileSaveHandlers({ + resolveManagedFilePath, + resolveSessionArtifactFilePath, + openManagedFileVersion + } as never) + + try { + const result = await handlers.get('file:save-project-artifacts')!( + { sender: {} }, + { + projectId: 'project-1', + suggestedArchiveName: 'Research', + files: [ + { + source: 'artifact', + sessionId: 'session-1', + path: 'artifact://legacy-report', + fileId: 'artifact-file-1', + suggestedName: 'report.csv' + } + ] + } + ) + + expect(result).toEqual({ saved: true, filePath: destinationPath }) + const entries = unzipSync(new Uint8Array(await readFile(destinationPath))) + expect(Buffer.from(entries['generated/report.csv']!).toString('utf8')).toBe( + 'legacy fallback bytes' + ) + expect(resolveManagedFilePath).toHaveBeenCalledWith('artifact', { + path: 'artifact://legacy-report', + projectId: 'project-1', + sessionId: 'session-1', + fileId: 'artifact-file-1' + }) + expect(resolveSessionArtifactFilePath).not.toHaveBeenCalled() + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it('rejects a same-size replacement during managed Version path fallback', async () => { + const root = await mkdtemp(join(tmpdir(), 'open-science-save-project-corrupt-fallback-')) + const sourcePath = join(root, 'managed-report.csv') + const destinationPath = join(root, 'Research-artifacts.zip') + const expectedBytes = 'trusted version' + const replacementBytes = 'replaced bytes!' + expect(Buffer.byteLength(replacementBytes)).toBe(Buffer.byteLength(expectedBytes)) + await writeFile(sourcePath, replacementBytes) + const resolveManagedFilePath = vi.fn().mockResolvedValue({ + path: sourcePath, + expectedSize: Buffer.byteLength(expectedBytes), + expectedChecksum: sha256(expectedBytes) + }) + const openManagedFileVersion = vi.fn().mockRejectedValue( + Object.assign(new Error('Native anchored managed-file access is unavailable.'), { + code: 'NATIVE_WRITE_REQUIRED' + }) + ) + showSaveDialog.mockResolvedValue({ canceled: false, filePath: destinationPath }) + registerFileSaveHandlers({ resolveManagedFilePath, openManagedFileVersion } as never) + + try { + const result = await handlers.get('file:save-project-artifacts')!( + { sender: {} }, + { + projectId: 'project-1', + suggestedArchiveName: 'Research', + files: [ + { + source: 'artifact', + sessionId: 'session-1', + path: 'artifact://legacy-report', + fileId: 'artifact-file-1', + suggestedName: 'report.csv' + } + ] + } + ) + + expect(result).toEqual({ + saved: true, + failures: [ + { + source: 'artifact', + sessionId: 'session-1', + path: 'artifact://legacy-report', + fileId: 'artifact-file-1', + suggestedName: 'report.csv', + message: 'Project export source does not match the managed Version record.' + } + ] + }) + expect(showSaveDialog).not.toHaveBeenCalled() + await expect(readFile(destinationPath)).rejects.toMatchObject({ code: 'ENOENT' }) + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + it('does not hide managed Version integrity failures behind the legacy path fallback', async () => { + const resolveSessionArtifactFilePath = vi.fn().mockResolvedValue('/managed/legacy-report.csv') + const openManagedFileVersion = vi.fn().mockRejectedValue( + Object.assign(new Error('Managed file version content is unavailable or corrupt.'), { + code: 'CONTENT_INTEGRITY_FAILED' + }) + ) + registerFileSaveHandlers({ resolveSessionArtifactFilePath, openManagedFileVersion } as never) + + const result = await handlers.get('file:save-project-artifacts')!( + { sender: {} }, + { + projectId: 'project-1', + suggestedArchiveName: 'Research', + files: [ + { + source: 'artifact', + sessionId: 'session-1', + path: 'artifact://legacy-report', + fileId: 'artifact-file-1', + suggestedName: 'report.csv' + } + ] + } + ) + + expect(result).toEqual({ + saved: true, + failures: [ + { + source: 'artifact', + sessionId: 'session-1', + path: 'artifact://legacy-report', + fileId: 'artifact-file-1', + suggestedName: 'report.csv', + message: 'Managed file version content is unavailable or corrupt.' + } + ] + }) + expect(resolveSessionArtifactFilePath).not.toHaveBeenCalled() + expect(showSaveDialog).not.toHaveBeenCalled() + }) + + it('does not replace an explicit historical version with the legacy current path', async () => { + const root = await mkdtemp(join(tmpdir(), 'open-science-save-project-exact-no-native-')) + const legacyCurrentPath = join(root, 'current-report.csv') + await writeFile(legacyCurrentPath, 'wrong current bytes') + const resolveSessionArtifactFilePath = vi.fn().mockResolvedValue(legacyCurrentPath) + const openManagedFileVersion = vi.fn().mockRejectedValue( + Object.assign(new Error('Native anchored managed-file access is unavailable.'), { + code: 'NATIVE_WRITE_REQUIRED' + }) + ) + registerFileSaveHandlers({ resolveSessionArtifactFilePath, openManagedFileVersion } as never) + + try { + const result = await handlers.get('file:save-project-artifacts')!( + { sender: {} }, + { + projectId: 'project-1', + suggestedArchiveName: 'Research', + files: [ + { + source: 'artifact', + sessionId: 'session-1', + path: 'artifact://current-report', + fileId: 'artifact-file-1', + versionId: 'artifact-version-1', + suggestedName: 'report.csv' + } + ] + } + ) + + expect(result).toEqual({ + saved: true, + failures: [ + { + source: 'artifact', + sessionId: 'session-1', + path: 'artifact://current-report', + fileId: 'artifact-file-1', + versionId: 'artifact-version-1', + suggestedName: 'report.csv', + message: 'Native anchored managed-file access is unavailable.' + } + ] + }) + expect(resolveSessionArtifactFilePath).not.toHaveBeenCalled() + expect(showSaveDialog).not.toHaveBeenCalled() + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + it('applies collision suffixes within each source category only', async () => { const root = await mkdtemp(join(tmpdir(), 'open-science-save-project-categories-')) const artifactPathA = join(root, 'managed-a.csv') @@ -1751,3 +2339,36 @@ describe('assertSaveManagedFileRequest validation paths', () => { ) }) }) + +describe('assertSaveSessionArtifactsRequest logical identity validation', () => { + beforeEach(() => { + handlers.clear() + showSaveDialog.mockReset() + showOpenDialog.mockReset() + }) + + it.each([ + { identity: { fileId: 42 }, label: 'numeric file id' }, + { identity: { fileId: ' ' }, label: 'blank file id' }, + { identity: { fileId: 'artifact-1', versionId: 42 }, label: 'numeric version id' }, + { identity: { fileId: 'artifact-1', versionId: '' }, label: 'blank version id' }, + { identity: { versionId: 'artifact-v1' }, label: 'version without file id' } + ] as const)('rejects $label before opening a save dialog', async ({ identity }) => { + registerFileSaveHandlers({ + resolveSessionArtifactFilePath: vi.fn().mockResolvedValue('/managed/report.csv') + } as never) + + await expect( + handlers.get('file:save-session-artifacts')!( + { sender: {} }, + { + projectId: 'project-1', + sessionId: 'session-1', + files: [{ path: 'artifact://report', suggestedName: 'report.csv', ...identity }] + } + ) + ).rejects.toThrow('Invalid Session Artifact save request.') + expect(showSaveDialog).not.toHaveBeenCalled() + expect(showOpenDialog).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/file-save.ts b/src/main/file-save.ts index b54d455c6..3fdf6619b 100644 --- a/src/main/file-save.ts +++ b/src/main/file-save.ts @@ -2,6 +2,7 @@ import { BrowserWindow, app, dialog, type OpenDialogOptions } from 'electron' import { Zip, ZipDeflate } from 'fflate' import { ipcMainHandle } from './ipc-handler-registry' +import { createHash } from 'node:crypto' import { constants } from 'node:fs' import { copyFile, mkdtemp, open, rm, writeFile, type FileHandle } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -25,19 +26,38 @@ import { englishNativeTranslator, type NativeTranslator } from './locale/main-pr type RegisterFileSaveHandlersOptions = { resolveManagedFilePath?: ( source: SaveManagedFileRequest['source'], - request: { path: string; projectId?: string; sessionId?: string } - ) => Promise + request: { + path: string + projectId?: string + sessionId?: string + fileId?: string + versionId?: string + } + ) => Promise resolveSessionArtifactFilePath?: ( projectId: string, sessionId: string, path: string ) => Promise openManagedFile?: (sourcePath: string) => Promise + openManagedFileVersion?: ( + source: 'artifact' | 'upload', + request: { projectId: string; fileId: string; versionId?: string } + ) => Promise openProjectArtifactFile?: (sourcePath: string) => Promise + createProjectArtifactTemporaryRoot?: () => Promise projectArtifactExportLimits?: ProjectArtifactExportLimits translate?: NativeTranslator } +type ManagedFilePathResolution = + | string + | { + path: string + expectedSize: number + expectedChecksum: string + } + type ProjectArtifactExportLimits = { maxFiles: number maxFileBytes: number @@ -62,21 +82,41 @@ const PROJECT_ARTIFACT_EXPORT_LIMITS: ProjectArtifactExportLimits = { const PROJECT_ARTIFACT_STREAM_CHUNK_BYTES = 64 * 1024 -type ProjectArtifactExportCandidate = { +type ProjectArtifactExportCandidateBase = { file: SaveProjectArtifactsRequest['files'][number] - sourcePath: string entryName: string - device: number - inode: number } +type ProjectArtifactExportCandidate = ProjectArtifactExportCandidateBase & + ( + | { + kind: 'managed-version' + source: ManagedFileVersionHandle + } + | { + kind: 'path' + sourcePath: string + device: number + inode: number + integrity?: { expectedSize: number; expectedChecksum: string } + } + ) + type ManagedFileHandle = { copyTo: (destinationPath: string, options?: { exclusive?: boolean }) => Promise close: () => Promise } +// The verified Version lease exposes bounded reads from the inode selected by the DB lookup. +type ManagedFileVersionHandle = ManagedFileHandle & { + size: number + readRange: (begin: number, end: number) => Promise + verifyUnchanged: () => Promise +} + // IPC input is renderer-controlled; reject malformed sources and paths before any filesystem work. const assertSaveManagedFileRequest = (request: SaveManagedFileRequest): void => { + const isVersionedSource = request?.source === 'artifact' || request?.source === 'upload' if ( typeof request !== 'object' || request === null || @@ -86,7 +126,20 @@ const assertSaveManagedFileRequest = (request: SaveManagedFileRequest): void => request.source !== 'local') || typeof request.path !== 'string' || request.path.trim().length === 0 || - typeof request.suggestedName !== 'string' + typeof request.suggestedName !== 'string' || + (isVersionedSource && + (('projectId' in request && + request.projectId !== undefined && + (typeof request.projectId !== 'string' || request.projectId.trim().length === 0)) || + ('fileId' in request && + request.fileId !== undefined && + (typeof request.fileId !== 'string' || request.fileId.trim().length === 0)) || + Boolean(request.projectId) !== Boolean(request.fileId) || + ('versionId' in request && + request.versionId !== undefined && + (typeof request.versionId !== 'string' || + request.versionId.trim().length === 0 || + !request.fileId)))) ) { throw new Error('Invalid managed file save request.') } @@ -102,14 +155,26 @@ const assertSaveSessionArtifactsRequest = (request: SaveSessionArtifactsRequest) request.sessionId.trim().length === 0 || !Array.isArray(request.files) || request.files.length === 0 || - request.files.some( - (file) => + request.files.some((file) => { + if ( typeof file !== 'object' || file === null || typeof file.path !== 'string' || file.path.trim().length === 0 || typeof file.suggestedName !== 'string' - ) + ) { + return true + } + const hasFileId = typeof file.fileId === 'string' && file.fileId.trim().length > 0 + if (file.fileId !== undefined && !hasFileId) return true + if ( + file.versionId !== undefined && + (typeof file.versionId !== 'string' || file.versionId.trim().length === 0 || !hasFileId) + ) { + return true + } + return false + }) ) { throw new Error('Invalid Session Artifact save request.') } @@ -125,8 +190,8 @@ const assertSaveProjectArtifactsRequest = (request: SaveProjectArtifactsRequest) !Array.isArray(request.files) || request.files.length === 0 || request.files.length > 10000 || - request.files.some( - (file) => + request.files.some((file) => { + if ( typeof file !== 'object' || file === null || (file.source !== 'artifact' && file.source !== 'upload') || @@ -135,7 +200,16 @@ const assertSaveProjectArtifactsRequest = (request: SaveProjectArtifactsRequest) typeof file.path !== 'string' || file.path.trim().length === 0 || typeof file.suggestedName !== 'string' - ) + ) { + return true + } + const hasFileId = typeof file.fileId === 'string' && file.fileId.trim().length > 0 + if (file.fileId !== undefined && !hasFileId) return true + return ( + file.versionId !== undefined && + (typeof file.versionId !== 'string' || file.versionId.trim().length === 0 || !hasFileId) + ) + }) ) { throw new Error('Invalid Project Artifact save request.') } @@ -179,6 +253,42 @@ const openManagedFile = async (sourcePath: string): Promise = const openProjectArtifactFile = (sourcePath: string): Promise => open(sourcePath, 'r') +const getManagedFilePath = (resolved: ManagedFilePathResolution): string => + typeof resolved === 'string' ? resolved : resolved.path + +const getManagedFileIntegrity = ( + resolved: ManagedFilePathResolution +): { expectedSize: number; expectedChecksum: string } | undefined => { + if (typeof resolved === 'string') return undefined + if ( + !Number.isSafeInteger(resolved.expectedSize) || + resolved.expectedSize < 0 || + !/^[a-f0-9]{64}$/u.test(resolved.expectedChecksum) + ) { + throw new Error('Managed Version integrity metadata is invalid.') + } + return { + expectedSize: resolved.expectedSize, + expectedChecksum: resolved.expectedChecksum + } +} + +const checksumProjectArtifactFile = async ( + source: ProjectArtifactFileHandle +): Promise<{ size: number; checksum: string }> => { + const checksum = createHash('sha256') + let size = 0 + const stream = source.createReadStream({ + autoClose: false, + highWaterMark: PROJECT_ARTIFACT_STREAM_CHUNK_BYTES + }) + for await (const chunk of stream) { + size += chunk.byteLength + checksum.update(chunk) + } + return { size, checksum: checksum.digest('hex') } +} + const appendArchiveChunk = async (handle: FileHandle, chunk: Uint8Array): Promise => { let offset = 0 while (offset < chunk.byteLength) { @@ -193,15 +303,22 @@ const writeProjectArtifactArchive = async (options: { candidates: ProjectArtifactExportCandidate[] failures: SaveProjectArtifactFailure[] limits: ProjectArtifactExportLimits - openSource: (sourcePath: string) => Promise + openPathSource: (sourcePath: string) => Promise + createTemporaryRoot: () => Promise }): Promise => { - const temporaryRoot = await mkdtemp(join(tmpdir(), 'open-science-project-export-')) - const temporaryArchivePath = join(temporaryRoot, 'project-artifacts.zip') + let temporaryRoot: string | undefined let archiveHandle: FileHandle | undefined let archiveHandleClosed = false let zip: Zip | undefined + const pendingManagedSources = new Set( + options.candidates.flatMap((candidate) => + candidate.kind === 'managed-version' ? [candidate.source] : [] + ) + ) try { + temporaryRoot = await options.createTemporaryRoot() + const temporaryArchivePath = join(temporaryRoot, 'project-artifacts.zip') archiveHandle = await open(temporaryArchivePath, 'wx', 0o600) let archiveFailure: Error | undefined let pendingArchiveWrite = Promise.resolve() @@ -226,21 +343,54 @@ const writeProjectArtifactArchive = async (options: { } for (const candidate of options.candidates) { - let source: ProjectArtifactFileHandle | undefined + let closeSource: (() => Promise) | undefined let entryStarted = false try { - source = await options.openSource(candidate.sourcePath) - const metadata = await source.stat() - if (!metadata.isFile()) { - throw new Error('Project export source is not a regular file.') - } - if (metadata.dev !== candidate.device || metadata.ino !== candidate.inode) { - throw new Error('Project export source changed after validation.') + let sourceSize: number + let chunks: AsyncIterable + let verifySource: () => Promise + + if (candidate.kind === 'managed-version') { + const managedSource = candidate.source + sourceSize = managedSource.size + closeSource = () => managedSource.close() + chunks = (async function* (): AsyncGenerator { + for (let begin = 0; begin < managedSource.size;) { + const end = Math.min(begin + PROJECT_ARTIFACT_STREAM_CHUNK_BYTES, managedSource.size) + const chunk = new Uint8Array(await managedSource.readRange(begin, end)) + if (chunk.byteLength !== end - begin) { + throw new Error('Project export source changed while streaming.') + } + begin = end + yield chunk + } + })() + verifySource = () => managedSource.verifyUnchanged() + } else { + const pathSource = await options.openPathSource(candidate.sourcePath) + closeSource = () => pathSource.close() + const metadata = await pathSource.stat() + if (!metadata.isFile()) { + throw new Error('Project export source is not a regular file.') + } + if (metadata.dev !== candidate.device || metadata.ino !== candidate.inode) { + throw new Error('Project export source changed after validation.') + } + if (candidate.integrity && metadata.size !== candidate.integrity.expectedSize) { + throw new Error('Project export source does not match the managed Version record.') + } + sourceSize = metadata.size + chunks = pathSource.createReadStream({ + autoClose: false, + highWaterMark: PROJECT_ARTIFACT_STREAM_CHUNK_BYTES + }) + verifySource = async () => undefined } - if (metadata.size > options.limits.maxFileBytes) { + + if (sourceSize > options.limits.maxFileBytes) { throw new Error('Project export file exceeds the per-file size limit.') } - if (streamedBytes + metadata.size > options.limits.maxTotalBytes) { + if (streamedBytes + sourceSize > options.limits.maxTotalBytes) { throw new Error('Project export exceeds the total size limit.') } @@ -248,11 +398,9 @@ const writeProjectArtifactArchive = async (options: { zip.add(zipEntry) entryStarted = true let entryBytes = 0 - const sourceStream = source.createReadStream({ - autoClose: false, - highWaterMark: PROJECT_ARTIFACT_STREAM_CHUNK_BYTES - }) - for await (const chunk of sourceStream) { + const checksum = + candidate.kind === 'path' && candidate.integrity ? createHash('sha256') : undefined + for await (const chunk of chunks) { entryBytes += chunk.byteLength if (entryBytes > options.limits.maxFileBytes) { throw new Error('Project export file exceeds the per-file size limit.') @@ -261,6 +409,7 @@ const writeProjectArtifactArchive = async (options: { throw new Error('Project export exceeds the total size limit.') } + checksum?.update(chunk) zipEntry.push(chunk, false) await pendingArchiveWrite throwIfArchiveFailed() @@ -268,6 +417,15 @@ const writeProjectArtifactArchive = async (options: { // window, and lifecycle events between source chunks. await yieldToEventLoop() } + if ( + candidate.kind === 'path' && + candidate.integrity && + (entryBytes !== candidate.integrity.expectedSize || + checksum?.digest('hex') !== candidate.integrity.expectedChecksum) + ) { + throw new Error('Project export source does not match the managed Version record.') + } + await verifySource() zipEntry.push(new Uint8Array(0), true) await pendingArchiveWrite throwIfArchiveFailed() @@ -280,7 +438,10 @@ const writeProjectArtifactArchive = async (options: { message: error instanceof Error ? error.message : String(error) }) } finally { - await source?.close() + await closeSource?.() + if (candidate.kind === 'managed-version') { + pendingManagedSources.delete(candidate.source) + } } } @@ -303,13 +464,21 @@ const writeProjectArtifactArchive = async (options: { if (archiveHandle && !archiveHandleClosed) { await archiveHandle.close().catch(() => undefined) } - await rm(temporaryRoot, { recursive: true, force: true }).catch(() => undefined) + await Promise.all( + [...pendingManagedSources].map((source) => source.close().catch(() => undefined)) + ) + if (temporaryRoot) { + await rm(temporaryRoot, { recursive: true, force: true }).catch(() => undefined) + } } } const isAlreadyExistsError = (error: unknown): boolean => error instanceof Error && 'code' in error && error.code === 'EEXIST' +const hasErrorCode = (error: unknown, code: string): boolean => + typeof error === 'object' && error !== null && 'code' in error && error.code === code + const addFilenameCollisionSuffix = (filename: string, suffix: number): string => { const extension = extname(filename) const stem = basename(filename, extension) @@ -427,7 +596,11 @@ const registerFileSaveHandlers = (options: RegisterFileSaveHandlersOptions = {}) ): Promise => { const resolver = options.resolveManagedFilePath if (!resolver) throw new Error('Managed file resolver is not configured.') - return resolver('upload', { path: file.path, projectId, sessionId: file.sessionId }) + return resolver('upload', { + path: file.path, + projectId, + sessionId: file.sessionId + }).then(getManagedFilePath) } ipcMainHandle( @@ -458,37 +631,68 @@ const registerFileSaveHandlers = (options: RegisterFileSaveHandlersOptions = {}) ipcMainHandle( 'file:save-managed', async (event, request: SaveManagedFileRequest): Promise => { - if (!options.resolveManagedFilePath) { + if (!options.resolveManagedFilePath && !options.openManagedFileVersion) { throw new Error('Managed file resolver is not configured.') } assertSaveManagedFileRequest(request) - const sourcePath = await options.resolveManagedFilePath(request.source, { - path: request.path - }) + const versionedRequest = + request.source === 'artifact' || request.source === 'upload' ? request : undefined + const logicalIdentity = versionedRequest?.fileId + const legacySourcePath = logicalIdentity + ? undefined + : getManagedFilePath( + await options.resolveManagedFilePath!(request.source, { path: request.path }) + ) const requestedBaseName = basename(request.suggestedName.trim()) const safeName = requestedBaseName && requestedBaseName !== '.' && requestedBaseName !== '..' ? requestedBaseName - : basename(sourcePath) + : basename(legacySourcePath ?? request.path) const dialogOptions = { defaultPath: join(app.getPath('downloads'), safeName), title: (options.translate ?? englishNativeTranslator)('Save file') } - const managedFile = await (options.openManagedFile ?? openManagedFile)(sourcePath) - + const legacyManagedFile = legacySourcePath + ? await (options.openManagedFile ?? openManagedFile)(legacySourcePath) + : undefined + const parentWindow = BrowserWindow.fromWebContents(event.sender) try { - const parentWindow = BrowserWindow.fromWebContents(event.sender) const { canceled, filePath } = parentWindow ? await dialog.showSaveDialog(parentWindow, dialogOptions) : await dialog.showSaveDialog(dialogOptions) if (canceled || !filePath) return { saved: false } - await managedFile.copyTo(filePath) - return { saved: true, filePath } + // Resolve after user confirmation so a default logical locator observes the current DB head. + const managedFile = logicalIdentity + ? options.openManagedFileVersion + ? await options.openManagedFileVersion(versionedRequest!.source, { + projectId: versionedRequest!.projectId!, + fileId: logicalIdentity, + ...(versionedRequest!.versionId ? { versionId: versionedRequest!.versionId } : {}) + }) + : await (options.openManagedFile ?? openManagedFile)( + getManagedFilePath( + await options.resolveManagedFilePath!(request.source, { + path: request.path, + projectId: versionedRequest!.projectId, + fileId: logicalIdentity, + ...(versionedRequest!.versionId + ? { versionId: versionedRequest!.versionId } + : {}) + }) + ) + ) + : legacyManagedFile! + try { + await managedFile.copyTo(filePath) + return { saved: true, filePath } + } finally { + if (logicalIdentity) await managedFile.close() + } } finally { - await managedFile.close() + await legacyManagedFile?.close() } } ) @@ -497,7 +701,11 @@ const registerFileSaveHandlers = (options: RegisterFileSaveHandlersOptions = {}) 'file:save-session-artifacts', async (event, request: SaveSessionArtifactsRequest): Promise => { const resolveSessionArtifactFilePath = options.resolveSessionArtifactFilePath - if (!resolveSessionArtifactFilePath) { + if ( + !resolveSessionArtifactFilePath && + !options.resolveManagedFilePath && + !options.openManagedFileVersion + ) { throw new Error('Session Artifact file resolver is not configured.') } @@ -506,25 +714,41 @@ const registerFileSaveHandlers = (options: RegisterFileSaveHandlersOptions = {}) if (request.files.length === 1) { const [file] = request.files - const sourcePath = await resolveSessionArtifactFilePath( - request.projectId, - request.sessionId, - file.path - ) - const safeName = getSafeFilename(file.suggestedName, sourcePath) + const legacySourcePath = file.fileId + ? undefined + : await resolveSessionArtifactFilePath!(request.projectId, request.sessionId, file.path) + const safeName = getSafeFilename(file.suggestedName, legacySourcePath ?? file.path) const dialogOptions = { defaultPath: join(app.getPath('downloads'), safeName), title: (options.translate ?? englishNativeTranslator)('Save artifact') } - const managedFile = await (options.openManagedFile ?? openManagedFile)(sourcePath) + const { canceled, filePath } = parentWindow + ? await dialog.showSaveDialog(parentWindow, dialogOptions) + : await dialog.showSaveDialog(dialogOptions) - try { - const { canceled, filePath } = parentWindow - ? await dialog.showSaveDialog(parentWindow, dialogOptions) - : await dialog.showSaveDialog(dialogOptions) + if (canceled || !filePath) return { saved: false } - if (canceled || !filePath) return { saved: false } + const managedFile = file.fileId + ? options.openManagedFileVersion + ? await options.openManagedFileVersion('artifact', { + projectId: request.projectId, + fileId: file.fileId, + ...(file.versionId ? { versionId: file.versionId } : {}) + }) + : await (options.openManagedFile ?? openManagedFile)( + getManagedFilePath( + await options.resolveManagedFilePath!('artifact', { + path: file.path, + projectId: request.projectId, + sessionId: request.sessionId, + fileId: file.fileId, + ...(file.versionId ? { versionId: file.versionId } : {}) + }) + ) + ) + : await (options.openManagedFile ?? openManagedFile)(legacySourcePath!) + try { await managedFile.copyTo(filePath) return { saved: true, filePaths: [filePath] } } finally { @@ -548,13 +772,31 @@ const registerFileSaveHandlers = (options: RegisterFileSaveHandlersOptions = {}) for (const file of request.files) { let managedFile: ManagedFileHandle | undefined try { - const sourcePath = await resolveSessionArtifactFilePath( - request.projectId, - request.sessionId, - file.path - ) - const safeName = getSafeFilename(file.suggestedName, sourcePath) - managedFile = await (options.openManagedFile ?? openManagedFile)(sourcePath) + if (file.fileId && options.openManagedFileVersion) { + managedFile = await options.openManagedFileVersion('artifact', { + projectId: request.projectId, + fileId: file.fileId, + ...(file.versionId ? { versionId: file.versionId } : {}) + }) + } else { + const sourcePath = file.fileId + ? getManagedFilePath( + await options.resolveManagedFilePath!('artifact', { + path: file.path, + projectId: request.projectId, + sessionId: request.sessionId, + fileId: file.fileId, + ...(file.versionId ? { versionId: file.versionId } : {}) + }) + ) + : await resolveSessionArtifactFilePath!( + request.projectId, + request.sessionId, + file.path + ) + managedFile = await (options.openManagedFile ?? openManagedFile)(sourcePath) + } + const safeName = getSafeFilename(file.suggestedName, file.path) savedPaths.push( await copyToAvailableDestination(managedFile, destinationDirectory, safeName) ) @@ -588,27 +830,110 @@ const registerFileSaveHandlers = (options: RegisterFileSaveHandlersOptions = {}) const failures: SaveProjectArtifactFailure[] = [] let totalBytes = 0 for (const file of request.files) { - let exportFile: ProjectArtifactFileHandle | undefined + let exportFile: { close: () => Promise } | undefined + let retainedManagedVersion = false try { if (takenNames.size >= limits.maxFiles) { throw new Error('Project export exceeds the file-count limit.') } - const sourcePath = - file.source === 'upload' - ? await resolveManagedUpload(file, request.projectId) - : await resolveProjectArtifact(file, request.projectId) - exportFile = await (options.openProjectArtifactFile ?? openProjectArtifactFile)( - sourcePath - ) - const metadata = await exportFile.stat() - if (!metadata.isFile()) { - throw new Error('Project export source is not a regular file.') - } - if (metadata.size > limits.maxFileBytes) { - throw new Error('Project export file exceeds the per-file size limit.') + let sourcePath = file.path + let managedVersion: ManagedFileVersionHandle | undefined + let pathSize: number | undefined + let pathCandidate: + | { + sourcePath: string + device: number + inode: number + integrity?: { expectedSize: number; expectedChecksum: string } + } + | undefined + let pathIntegrity: { expectedSize: number; expectedChecksum: string } | undefined + let usePathFallback = !file.fileId || !options.openManagedFileVersion + + if (file.fileId && options.openManagedFileVersion) { + // Omitting versionId intentionally resolves the current DB head at export time. An + // explicit versionId remains an exact historical-version request. + try { + managedVersion = await options.openManagedFileVersion(file.source, { + projectId: request.projectId, + fileId: file.fileId, + ...(file.versionId ? { versionId: file.versionId } : {}) + }) + exportFile = managedVersion + if (!Number.isSafeInteger(managedVersion.size) || managedVersion.size < 0) { + throw new Error('Project export source size is invalid.') + } + if (managedVersion.size > limits.maxFileBytes) { + throw new Error('Project export file exceeds the per-file size limit.') + } + if (totalBytes + managedVersion.size > limits.maxTotalBytes) { + throw new Error('Project export exceeds the total size limit.') + } + } catch (error) { + // Native anchored reads are unavailable on Windows and unsupported installations. + // Only that capability error may use the existing validated path flow; integrity and + // version errors remain failures and must never be hidden by stale-path fallback. + if (file.versionId || !hasErrorCode(error, 'NATIVE_WRITE_REQUIRED')) throw error + await exportFile?.close().catch(() => undefined) + exportFile = undefined + managedVersion = undefined + usePathFallback = true + } } - if (totalBytes + metadata.size > limits.maxTotalBytes) { - throw new Error('Project export exceeds the total size limit.') + + if (usePathFallback) { + if (file.fileId) { + const resolved = await options.resolveManagedFilePath!(file.source, { + path: file.path, + projectId: request.projectId, + sessionId: file.sessionId, + fileId: file.fileId, + ...(file.versionId ? { versionId: file.versionId } : {}) + }) + sourcePath = getManagedFilePath(resolved) + pathIntegrity = getManagedFileIntegrity(resolved) + if (!pathIntegrity) { + throw new Error('Managed Version integrity metadata is unavailable.') + } + } else { + sourcePath = + file.source === 'upload' + ? await resolveManagedUpload(file, request.projectId) + : await resolveProjectArtifact(file, request.projectId) + } + const legacyExportFile = await ( + options.openProjectArtifactFile ?? openProjectArtifactFile + )(sourcePath) + exportFile = legacyExportFile + const metadata = await legacyExportFile.stat() + if (!metadata.isFile()) { + throw new Error('Project export source is not a regular file.') + } + if (pathIntegrity && metadata.size !== pathIntegrity.expectedSize) { + throw new Error('Project export source does not match the managed Version record.') + } + if (metadata.size > limits.maxFileBytes) { + throw new Error('Project export file exceeds the per-file size limit.') + } + if (totalBytes + metadata.size > limits.maxTotalBytes) { + throw new Error('Project export exceeds the total size limit.') + } + if (pathIntegrity) { + const observed = await checksumProjectArtifactFile(legacyExportFile) + if ( + observed.size !== pathIntegrity.expectedSize || + observed.checksum !== pathIntegrity.expectedChecksum + ) { + throw new Error('Project export source does not match the managed Version record.') + } + } + pathCandidate = { + sourcePath, + device: metadata.dev, + inode: metadata.ino, + ...(pathIntegrity ? { integrity: pathIntegrity } : {}) + } + pathSize = metadata.size } // Entries are grouped by origin under constant directory prefixes; the file name part // is sanitized before prefixing and collision suffixes apply within each category. @@ -617,23 +942,25 @@ const registerFileSaveHandlers = (options: RegisterFileSaveHandlersOptions = {}) takenNames, `${categoryDirectory}/${getSafeZipEntryName(file.suggestedName, sourcePath)}` ) - candidates.push({ - file, - sourcePath, - entryName, - device: metadata.dev, - inode: metadata.ino - }) + if (managedVersion) { + candidates.push({ kind: 'managed-version', file, entryName, source: managedVersion }) + retainedManagedVersion = true + totalBytes += managedVersion.size + } else if (pathCandidate) { + candidates.push({ kind: 'path', file, entryName, ...pathCandidate }) + totalBytes += pathSize! + } else { + throw new Error('Project export source could not be opened.') + } // Claim checks are case-insensitive; the entry itself keeps its original casing. takenNames.add(entryName.toLowerCase()) - totalBytes += metadata.size } catch (error) { failures.push({ ...file, message: error instanceof Error ? error.message : String(error) }) } finally { - await exportFile?.close() + if (!retainedManagedVersion) await exportFile?.close() } } if (candidates.length === 0) { @@ -654,17 +981,42 @@ const registerFileSaveHandlers = (options: RegisterFileSaveHandlersOptions = {}) } ] } - const { canceled, filePath } = parentWindow - ? await dialog.showSaveDialog(parentWindow, dialogOptions) - : await dialog.showSaveDialog(dialogOptions) - if (canceled || !filePath) return { saved: false } + let dialogResult: Awaited> + try { + dialogResult = parentWindow + ? await dialog.showSaveDialog(parentWindow, dialogOptions) + : await dialog.showSaveDialog(dialogOptions) + } catch (error) { + await Promise.all( + candidates.flatMap((candidate) => + candidate.kind === 'managed-version' + ? [candidate.source.close().catch(() => undefined)] + : [] + ) + ) + throw error + } + const { canceled, filePath } = dialogResult + if (canceled || !filePath) { + await Promise.all( + candidates.flatMap((candidate) => + candidate.kind === 'managed-version' + ? [candidate.source.close().catch(() => undefined)] + : [] + ) + ) + return { saved: false } + } const wroteArchive = await writeProjectArtifactArchive({ destinationPath: filePath, candidates, failures, limits, - openSource: options.openProjectArtifactFile ?? openProjectArtifactFile + openPathSource: options.openProjectArtifactFile ?? openProjectArtifactFile, + createTemporaryRoot: + options.createProjectArtifactTemporaryRoot ?? + (() => mkdtemp(join(tmpdir(), 'open-science-project-export-'))) }) return { saved: true, diff --git a/src/main/ipc.ts b/src/main/ipc.ts index e16483c7c..b2ea715bf 100644 --- a/src/main/ipc.ts +++ b/src/main/ipc.ts @@ -176,6 +176,11 @@ import { } from './session-persistence/conversation-export' import { createProjectFilesHandlers, registerProjectFilesIpcHandlers } from './project-files/ipc' import { createManagedFileIndexRepository } from './project-files/repository' +import { + createManagedFileVersionHandlers, + registerManagedFileVersionIpcHandlers +} from './managed-file-versions/ipc' +import { ManagedFileVersionService } from './managed-file-versions/service' import { ProjectDeletionCoordinator, ProjectDeletionRecoveryLoop @@ -449,6 +454,10 @@ const createApplicationModules = async ( diagnosticErrorFields(error) ) } + const managedFileVersionService = new ManagedFileVersionService({ + storageRoot: resolveDataRoot(), + getClient: () => getProjectDbClient(resolveStorageRoot()) + }) // Session reads and permission scope validation both need a late-bound view of ACP ownership: // startup runs before the runtime exists, while later reads must preserve live prompt state. const runtimeRef: { current: ReturnType | undefined } = { @@ -541,8 +550,24 @@ const createApplicationModules = async ( // One source-neutral resolver keeps previews and user-requested exports on identical trust checks. const resolveManagedFilePath = ( source: ManagedPreviewSource, - request: { path: string; projectId?: string; sessionId?: string } + request: { + path: string + projectId?: string + sessionId?: string + fileId?: string + versionId?: string + } ): Promise => { + if ((source === 'artifact' || source === 'upload') && request.projectId && request.fileId) { + return managedFileVersionService + .resolve({ + source, + projectId: request.projectId, + fileId: request.fileId, + ...(request.versionId ? { versionId: request.versionId } : {}) + }) + .then((resolved) => resolved.path) + } if (source === 'artifact') { const versionIdentity = parseArtifactVersionLocator(request.path) return versionIdentity @@ -578,7 +603,9 @@ const createApplicationModules = async ( }) // One registry owns short-lived capability URLs for both managed artifact repositories. const previewResources = new ManagedPreviewResources({ - resolvePath: resolveManagedFilePath + resolvePath: resolveManagedFilePath, + openManagedFileVersion: (source, request) => + managedFileVersionService.openResolved({ source, ...request }) }) const managedPreviewOwners = createManagedPreviewOwnerRegistry(previewResources) @@ -744,6 +771,14 @@ const createApplicationModules = async ( onDelivered: (event) => broadcastToRenderers('side-chat:relay-delivered', event) }) const uploadCommandOwner = createUploadCommandOwner(uploadRepository, { + resolveManagedFilePath: (request) => resolveManagedFilePath('upload', request), + openManagedFileVersion: (request) => + managedFileVersionService.openResolved({ + source: 'upload', + projectId: request.projectId!, + fileId: request.fileId!, + ...(request.versionId ? { versionId: request.versionId } : {}) + }), withSessionMutation: (projectId, sessionId, mutation) => sessionPersistenceCoordinator.runSessionMutation(projectId, sessionId, mutation) }) @@ -843,6 +878,10 @@ const createApplicationModules = async ( sessionPersistenceCoordinator, projectDeletionCoordinator ) + const managedFileVersionHandlers = createManagedFileVersionHandlers(managedFileVersionService, { + withDataRootWrite, + onChanged: (event) => broadcastToRenderers('project-files:changed', event) + }) // Stashed host.agents.switch bindings for sessions that are not yet durable (fresh unsent drafts), // flushed to disk on the session's first save so an approved switch survives an app restart before // the next message. Shared by persistSessionSpecialist (stash) and saveSession (flush). @@ -1376,6 +1415,29 @@ const createApplicationModules = async ( sessionEnabledComputeHostsOwnerRef.current = sessionEnabledComputeHostsOwner computeJobDeletionRef.current = jobDeletionOwner await projectDeletionCoordinator.restorePendingDeletionBarriers() + try { + await withDataRootWrite(() => managedFileVersionService.recoverPendingWrites()) + } catch (error) { + storageLog.error( + 'managed file version recovery incomplete; will retry next launch', + diagnosticErrorFields(error) + ) + } + void managedFileVersionService + .auditActiveVersionIntegrity() + .then((integrityErrors) => { + if (integrityErrors.length > 0) { + storageLog.error('managed file version integrity audit found corrupt active content', { + count: integrityErrors.length + }) + } + }) + .catch((error) => + storageLog.error( + 'managed file version integrity audit incomplete; will retry next launch', + diagnosticErrorFields(error) + ) + ) await jobDeletionOwner.restoreOrphanJobDeletionBarriers(isComputeJobOwnerLive) const dataRoot = resolveDataRoot() // Start the JobPoller wired to the shared broadcaster so every state/tail change is pushed to all @@ -1957,8 +2019,24 @@ const createApplicationModules = async ( const logsCommandOwner = createLogsCommandOwner() declareElectronAdapter('desktop-utilities', () => { registerFileSaveHandlers({ - resolveManagedFilePath, + resolveManagedFilePath: (source, request) => + (source === 'artifact' || source === 'upload') && request.projectId && request.fileId + ? managedFileVersionService + .resolvePath({ + source, + projectId: request.projectId, + fileId: request.fileId, + ...(request.versionId ? { versionId: request.versionId } : {}) + }) + .then((resolved) => ({ + path: resolved.path, + expectedSize: Number(resolved.version.sizeBytes), + expectedChecksum: resolved.version.checksum + })) + : resolveManagedFilePath(source, request), resolveSessionArtifactFilePath, + openManagedFileVersion: (source, request) => + managedFileVersionService.openResolved({ source, ...request }), translate }) registerLogsIpcHandlers(logsCommandOwner) @@ -1975,6 +2053,7 @@ const createApplicationModules = async ( repository: artifactRepository, runRegistry: artifactRunRegistry, provenanceRepository: artifactProvenanceRepository, + managedFileVersions: managedFileVersionService, uploadRepository, notebookRpcServer, peekNotebookHandoffContext: (sessionId) => notebookService.peekHandoffContext(sessionId), @@ -2624,11 +2703,18 @@ const createApplicationModules = async ( ) ) const officePreviewSupervisor = new OfficePreviewSupervisor({ - inspectResource: ({ source, path }) => previewResources.inspect({ source, path }), + inspectResource: ({ source, path, projectId, fileId, versionId }) => + previewResources.inspect({ source, path, projectId, fileId, versionId }), acquireResource: (ownerId, request, snapshot, maxBytes) => previewResources.acquire( ownerId, - { source: request.source, path: request.path }, + { + source: request.source, + path: request.path, + projectId: request.projectId, + fileId: request.fileId, + versionId: request.versionId + }, { snapshot, maxBytes } ), releaseResource: (ownerId, resourceId) => previewResources.release(ownerId, { resourceId }), @@ -2780,6 +2866,14 @@ const createApplicationModules = async ( getActiveArtifactRunIds: () => runtimeRef.current ? runtimeRef.current.getActiveArtifactRunIds() : [], provenance: artifactProvenanceRepository, + resolveManagedFilePath: (request) => resolveManagedFilePath('artifact', request), + openManagedFileVersion: (request) => + managedFileVersionService.openResolved({ + source: 'artifact', + projectId: request.projectId!, + fileId: request.fileId!, + ...(request.versionId ? { versionId: request.versionId } : {}) + }), codeReconstruction, withSessionMutation: (projectId, sessionId, mutation) => sessionPersistenceCoordinator.runSessionMutation(projectId, sessionId, mutation) @@ -2865,6 +2959,9 @@ const createApplicationModules = async ( projectFilesHandlers ) ) + declareElectronAdapter('managed-file-versions', () => + registerManagedFileVersionIpcHandlers(managedFileVersionHandlers) + ) // Backs the "This computer" browser; shares localFsService with the managed-preview resolver. declareElectronAdapter('local-fs', () => registerLocalFsIpcHandlers(localFsService)) declareElectronAdapter('preview-state', () => diff --git a/src/main/managed-file-preview.ts b/src/main/managed-file-preview.ts index e094c36fe..ac0f1e180 100644 --- a/src/main/managed-file-preview.ts +++ b/src/main/managed-file-preview.ts @@ -7,13 +7,26 @@ const DEFAULT_PREVIEW_BYTES = 8192 // without truncation; callers that only need a thumbnail keep passing a smaller explicit maxBytes. const MAX_PREVIEW_BYTES = 10 * 1024 * 1024 -// Reads a caller-bounded preview from an already-validated managed file path. -const readBoundedManagedFilePreview = async ( - filePath: string, +type ManagedFilePreviewReadLease = Readonly<{ + size: number + read: ( + buffer: Uint8Array, + offset: number, + length: number, + position: number + ) => Promise<{ bytesRead: number }> + verifyUnchanged: () => Promise + close: () => Promise +}> + +type ManagedFilePreviewReader = Pick & + Partial> + +const readBoundedManagedFilePreviewFromReader = async ( + reader: ManagedFilePreviewReader, request: ReadArtifactPreviewRequest, invalidEncodingMessage: string ): Promise => { - const fileStat = await stat(filePath) // Normalize the optional byte limit before applying the repository-wide hard cap. const requestedBytes = typeof request.maxBytes === 'number' && Number.isFinite(request.maxBytes) @@ -25,7 +38,7 @@ const readBoundedManagedFilePreview = async ( if (encoding !== 'utf8' && encoding !== 'base64') { throw new Error(invalidEncodingMessage) } - if (!Number.isSafeInteger(offset) || offset < 0 || offset > fileStat.size) { + if (!Number.isSafeInteger(offset) || offset < 0 || offset > reader.size) { throw new Error('Invalid managed file preview offset.') } @@ -33,32 +46,55 @@ const readBoundedManagedFilePreview = async ( const includePageMetadata = request.offset !== undefined // UTF-8 pages may read up to three extra bytes so the final character is never split. const readBudget = encoding === 'utf8' ? maxBytes + 3 : maxBytes - const bytesToRead = Math.min(fileStat.size - offset, readBudget) + const bytesToRead = Math.min(reader.size - offset, readBudget) const buffer = Buffer.alloc(bytesToRead) + const { bytesRead } = await reader.read(buffer, 0, bytesToRead, offset) + let contentBytesRead = Math.min(bytesRead, maxBytes) + if (encoding === 'utf8') { + while (contentBytesRead < bytesRead && (buffer[contentBytesRead] & 0xc0) === 0x80) { + contentBytesRead += 1 + } + } + const endOffset = offset + contentBytesRead + await reader.verifyUnchanged?.() + + return { + content: buffer.subarray(0, contentBytesRead).toString(encoding), + encoding, + size: reader.size, + truncated: reader.size > endOffset, + ...(includePageMetadata ? { offset } : {}), + ...(includePageMetadata && reader.size > endOffset ? { nextOffset: endOffset } : {}) + } +} + +// Reads a caller-bounded preview from an already-validated managed file path. +const readBoundedManagedFilePreview = async ( + filePath: string, + request: ReadArtifactPreviewRequest, + invalidEncodingMessage: string +): Promise => { + const fileStat = await stat(filePath) // Use an explicit file handle so the bounded read never streams the whole file by accident. const fileHandle = await open(filePath, 'r') try { - const { bytesRead } = await fileHandle.read(buffer, 0, bytesToRead, offset) - let contentBytesRead = Math.min(bytesRead, maxBytes) - if (encoding === 'utf8') { - while (contentBytesRead < bytesRead && (buffer[contentBytesRead] & 0xc0) === 0x80) { - contentBytesRead += 1 - } - } - const endOffset = offset + contentBytesRead - - return { - content: buffer.subarray(0, contentBytesRead).toString(encoding), - encoding, - size: fileStat.size, - truncated: fileStat.size > endOffset, - ...(includePageMetadata ? { offset } : {}), - ...(includePageMetadata && fileStat.size > endOffset ? { nextOffset: endOffset } : {}) - } + return await readBoundedManagedFilePreviewFromReader( + { size: fileStat.size, read: fileHandle.read.bind(fileHandle) }, + request, + invalidEncodingMessage + ) } finally { await fileHandle.close() } } -export { readBoundedManagedFilePreview } +const readBoundedManagedFilePreviewLease = ( + lease: ManagedFilePreviewReadLease, + request: ReadArtifactPreviewRequest, + invalidEncodingMessage: string +): Promise => + readBoundedManagedFilePreviewFromReader(lease, request, invalidEncodingMessage) + +export { readBoundedManagedFilePreview, readBoundedManagedFilePreviewLease } +export type { ManagedFilePreviewReadLease } diff --git a/src/main/managed-file-versions/diff-task.test.ts b/src/main/managed-file-versions/diff-task.test.ts new file mode 100644 index 000000000..f617357c6 --- /dev/null +++ b/src/main/managed-file-versions/diff-task.test.ts @@ -0,0 +1,957 @@ +import { describe, expect, it } from 'vitest' + +import { ManagedFileVersionError } from './service' +import { ManagedTextDiffTaskRunner } from './diff-task' + +describe('ManagedTextDiffTaskRunner', () => { + it('returns line numbers and intra-line segments for a replacement', async () => { + const runner = new ManagedTextDiffTaskRunner() + + const lines = await runner.run({ + requestId: 'diff-1', + before: 'alpha beta\nkeep\n', + after: 'alpha gamma\nkeep\n' + }) + expect(lines).toMatchObject([ + { + kind: 'removed', + oldLineNumber: 1 + }, + { + kind: 'added', + newLineNumber: 1 + }, + { + kind: 'context', + oldLineNumber: 2, + newLineNumber: 2, + segments: [{ kind: 'context', text: 'keep\n' }] + } + ]) + expect(lines[0]?.segments.map((segment) => segment.text).join('')).toBe('alpha beta\n') + expect(lines[1]?.segments.map((segment) => segment.text).join('')).toBe('alpha gamma\n') + expect(lines[0]?.segments.some((segment) => segment.kind === 'removed')).toBe(true) + expect(lines[1]?.segments.some((segment) => segment.kind === 'added')).toBe(true) + }) + + it('preserves a shared CRLF on an unchanged context line', async () => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'shared-crlf-context-line', + before: 'same\r\n', + after: 'same\r\n' + }) + + expect(lines).toEqual([ + { + kind: 'context', + oldLineNumber: 1, + newLineNumber: 1, + segments: [{ kind: 'context', text: 'same\r\n' }] + } + ]) + }) + + it('preserves an unchanged CRLF as context on both sides of a changed line', async () => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'shared-crlf-changed-line', + before: 'old value\r\n', + after: 'new value\r\n' + }) + + expect(lines).toEqual([ + { + kind: 'removed', + oldLineNumber: 1, + segments: [ + { kind: 'removed', text: 'old' }, + { kind: 'context', text: ' value\r\n' } + ] + }, + { + kind: 'added', + newLineNumber: 1, + segments: [ + { kind: 'added', text: 'new' }, + { kind: 'context', text: ' value\r\n' } + ] + } + ]) + }) + + it.each([ + { + label: 'addition', + before: 'line', + after: 'line\n', + expected: [ + { + kind: 'removed', + oldLineNumber: 1, + segments: [{ kind: 'context', text: 'line' }] + }, + { + kind: 'added', + newLineNumber: 1, + segments: [ + { kind: 'context', text: 'line' }, + { kind: 'added', text: '\n' } + ] + } + ] + }, + { + label: 'removal', + before: 'line\n', + after: 'line', + expected: [ + { + kind: 'removed', + oldLineNumber: 1, + segments: [ + { kind: 'context', text: 'line' }, + { kind: 'removed', text: '\n' } + ] + }, + { + kind: 'added', + newLineNumber: 1, + segments: [{ kind: 'context', text: 'line' }] + } + ] + } + ])('preserves a trailing newline $label as an exact character segment', async (fixture) => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: `trailing-newline-${fixture.label}`, + before: fixture.before, + after: fixture.after + }) + + expect(lines).toEqual(fixture.expected) + }) + + it.each([ + { + label: 'LF to CRLF', + before: 'line\n', + after: 'line\r\n', + removedSegments: [ + { kind: 'context', text: 'line' }, + { kind: 'context', text: '\n' } + ], + addedSegments: [ + { kind: 'context', text: 'line' }, + { kind: 'added', text: '\r' }, + { kind: 'context', text: '\n' } + ] + }, + { + label: 'CRLF to LF', + before: 'line\r\n', + after: 'line\n', + removedSegments: [ + { kind: 'context', text: 'line' }, + { kind: 'removed', text: '\r' }, + { kind: 'context', text: '\n' } + ], + addedSegments: [ + { kind: 'context', text: 'line' }, + { kind: 'context', text: '\n' } + ] + } + ])('preserves the shared newline during a trailing $label conversion', async (fixture) => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: `trailing-ending-conversion-${fixture.label}`, + before: fixture.before, + after: fixture.after + }) + + expect(lines).toEqual([ + { kind: 'removed', oldLineNumber: 1, segments: fixture.removedSegments }, + { kind: 'added', newLineNumber: 1, segments: fixture.addedSegments } + ]) + }) + + it.each([ + { + label: 'bare CR to CRLF', + before: 'line\r', + after: 'line\r\n', + removedSegments: [{ kind: 'context', text: 'line\r' }], + addedSegments: [ + { kind: 'context', text: 'line\r' }, + { kind: 'added', text: '\n' } + ] + }, + { + label: 'CRLF to bare CR', + before: 'line\r\n', + after: 'line\r', + removedSegments: [ + { kind: 'context', text: 'line\r' }, + { kind: 'removed', text: '\n' } + ], + addedSegments: [{ kind: 'context', text: 'line\r' }] + }, + { + label: 'bare CR to LF', + before: 'line\r', + after: 'line\n', + removedSegments: [ + { kind: 'context', text: 'line' }, + { kind: 'removed', text: '\r' } + ], + addedSegments: [ + { kind: 'context', text: 'line' }, + { kind: 'added', text: '\n' } + ] + } + ])('preserves exact characters during a $label conversion', async (fixture) => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: `bare-cr-${fixture.label}`, + before: fixture.before, + after: fixture.after + }) + + expect(lines).toEqual([ + { kind: 'removed', oldLineNumber: 1, segments: fixture.removedSegments }, + { kind: 'added', newLineNumber: 1, segments: fixture.addedSegments } + ]) + }) + + it.each([ + { label: 'LINE SEPARATOR', character: '\u2028' }, + { label: 'PARAGRAPH SEPARATOR', character: '\u2029' } + ])('keeps Unicode $label as a content character', async ({ character }) => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'unicode-line-separator', + before: `alpha${character}old`, + after: `alpha${character}new` + }) + + expect(lines[0]?.segments.map((segment) => segment.text).join('')).toBe(`alpha${character}old`) + expect(lines[1]?.segments.map((segment) => segment.text).join('')).toBe(`alpha${character}new`) + }) + + it('reconstructs both complete sources from a mixed-ending diff DTO', async () => { + const before = 'same\r\nold value\nremove me\r\nunicode\u2029tail' + const after = 'same\r\nnew value\nadded only\r\nunicode\u2029tail\n' + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'source-reconstruction', + before, + after + }) + const reconstruct = (excludedKind: 'added' | 'removed'): string => + lines + .filter((line) => line.kind !== excludedKind) + .flatMap((line) => line.segments) + .map((segment) => segment.text) + .join('') + + expect(reconstruct('added')).toBe(before) + expect(reconstruct('removed')).toBe(after) + }) + + it.each([ + { + label: 'LF addition', + before: '', + after: 'line\n', + expected: [ + { + kind: 'added', + newLineNumber: 1, + segments: [{ kind: 'added', text: 'line\n' }] + } + ] + }, + { + label: 'CRLF addition', + before: '', + after: 'line\r\n', + expected: [ + { + kind: 'added', + newLineNumber: 1, + segments: [{ kind: 'added', text: 'line\r\n' }] + } + ] + }, + { + label: 'LF removal', + before: 'line\n', + after: '', + expected: [ + { + kind: 'removed', + oldLineNumber: 1, + segments: [{ kind: 'removed', text: 'line\n' }] + } + ] + }, + { + label: 'CRLF removal', + before: 'line\r\n', + after: '', + expected: [ + { + kind: 'removed', + oldLineNumber: 1, + segments: [{ kind: 'removed', text: 'line\r\n' }] + } + ] + } + ])('preserves line endings for a pure $label', async (fixture) => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: `pure-ending-${fixture.label}`, + before: fixture.before, + after: fixture.after + }) + + expect(lines).toEqual(fixture.expected) + }) + + it.each([ + { + label: 'addition', + before: 'plain', + after: 'plain\ncontinuation\r\n', + trailingKind: 'added', + trailingText: 'continuation\r\n' + }, + { + label: 'removal', + before: 'plain\ncontinuation\r\n', + after: 'plain', + trailingKind: 'removed', + trailingText: 'continuation\r\n' + } + ])('preserves an unmatched trailing line ending after a line-count $label', async (fixture) => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: `unmatched-ending-${fixture.label}`, + before: fixture.before, + after: fixture.after + }) + + expect(lines.at(-1)).toMatchObject({ + kind: fixture.trailingKind, + segments: [{ kind: fixture.trailingKind, text: fixture.trailingText }] + }) + }) + + it('keeps an inserted line separate from the similar modified line that follows it', async () => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'inserted-line-before-modified-line', + before: '## What Is This?\nOriginal paragraph old.\n', + after: '## What Is This?? ?\nWonderful\nOriginal paragraph new.\n' + }) + + expect( + lines.map((line) => ({ + kind: line.kind, + oldLineNumber: line.oldLineNumber, + newLineNumber: line.newLineNumber, + text: line.segments.map((segment) => segment.text).join(''), + changed: line.segments + .filter((segment) => segment.kind === line.kind) + .map((segment) => segment.text) + .join('') + })) + ).toEqual([ + { + kind: 'removed', + oldLineNumber: 1, + newLineNumber: undefined, + text: '## What Is This?\n', + changed: '' + }, + { + kind: 'added', + oldLineNumber: undefined, + newLineNumber: 1, + text: '## What Is This?? ?\n', + changed: '? ?' + }, + { + kind: 'added', + oldLineNumber: undefined, + newLineNumber: 2, + text: 'Wonderful\n', + changed: 'Wonderful\n' + }, + { + kind: 'removed', + oldLineNumber: 2, + newLineNumber: undefined, + text: 'Original paragraph old.\n', + changed: 'old' + }, + { + kind: 'added', + oldLineNumber: undefined, + newLineNumber: 3, + text: 'Original paragraph new.\n', + changed: 'new' + } + ]) + }) + + it('does not anchor a paragraph insertion to the blank line before its matching paragraph', async () => { + const beforeParagraph = + "If you switch between Claude's official subscription and third-party API routing." + const afterParagraph = + 'If you switch between official subscription and third-party API routing.' + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'blank-line-before-matching-paragraph', + before: `## What Is This?\n\n${beforeParagraph}\n\nStable next.\n`, + after: `## What Is This?？？\n\n### Wonderful\n\n${afterParagraph}\n\nStable next.\n` + }) + const summary = lines.map((line) => ({ + kind: line.kind, + text: line.segments.map((segment) => segment.text).join(''), + changed: line.segments + .filter((segment) => segment.kind === line.kind) + .map((segment) => segment.text) + .join('') + })) + + expect(summary).toContainEqual({ + kind: 'added', + text: '### Wonderful\n', + changed: '### Wonderful\n' + }) + expect(summary).toContainEqual({ + kind: 'removed', + text: `${beforeParagraph}\n`, + changed: "Claude's " + }) + expect(summary).toContainEqual({ + kind: 'added', + text: `${afterParagraph}\n`, + changed: '' + }) + expect(summary.findIndex((line) => line.text === '### Wonderful\n')).toBeLessThan( + summary.findIndex((line) => line.text === `${afterParagraph}\n`) + ) + }) + + it.each([199, 200])( + 'preserves %i unchanged blank lines around a paragraph replacement', + async (blankLineCount) => { + const blankLines = '\n'.repeat(blankLineCount) + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: `blank-line-alignment-budget-${blankLineCount}`, + before: `Paragraph old.\n${blankLines}`, + after: `Paragraph new.\n${blankLines}` + }) + + expect(lines.filter((line) => line.kind === 'context')).toHaveLength(blankLineCount) + expect( + lines + .filter((line) => line.kind === 'context') + .every( + (line) => + line.segments.length === 1 && + line.segments[0].kind === 'context' && + line.segments[0].text === '\n' + ) + ).toBe(true) + expect( + lines + .filter((line) => line.kind !== 'context') + .map((line) => ({ + kind: line.kind, + text: line.segments.map((segment) => segment.text).join(''), + changed: line.segments + .filter((segment) => segment.kind === line.kind) + .map((segment) => segment.text) + .join('') + })) + ).toEqual([ + { kind: 'removed', text: 'Paragraph old.\n', changed: 'old' }, + { kind: 'added', text: 'Paragraph new.\n', changed: 'new' } + ]) + } + ) + + it('preserves a long unchanged whitespace line across a paragraph replacement', async () => { + const whitespaceLine = `${' '.repeat(24_990)}\n` + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'whitespace-line-alignment-character-budget', + before: `Paragraph old.\n${whitespaceLine}`, + after: `Paragraph new.\n${whitespaceLine}` + }) + + expect(lines.filter((line) => line.kind === 'context')).toEqual([ + expect.objectContaining({ + segments: [{ kind: 'context', text: whitespaceLine }] + }) + ]) + expect( + lines + .filter((line) => line.kind !== 'context') + .map((line) => ({ + kind: line.kind, + changed: line.segments + .filter((segment) => segment.kind === line.kind) + .map((segment) => segment.text) + .join('') + })) + ).toEqual([ + { kind: 'removed', changed: 'old' }, + { kind: 'added', changed: 'new' } + ]) + }) + + it('keeps a removed line separate from the similar modified line that follows it', async () => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'removed-line-before-modified-line', + before: '## What Is This?\nWonderful\nOriginal paragraph old.\n', + after: '## What Is This?? ?\nOriginal paragraph new.\n' + }) + + expect( + lines.map((line) => ({ + kind: line.kind, + text: line.segments.map((segment) => segment.text).join(''), + changed: line.segments + .filter((segment) => segment.kind === line.kind) + .map((segment) => segment.text) + .join('') + })) + ).toEqual([ + { kind: 'removed', text: '## What Is This?\n', changed: '' }, + { kind: 'added', text: '## What Is This?? ?\n', changed: '? ?' }, + { kind: 'removed', text: 'Wonderful\n', changed: 'Wonderful\n' }, + { kind: 'removed', text: 'Original paragraph old.\n', changed: 'old' }, + { kind: 'added', text: 'Original paragraph new.\n', changed: 'new' } + ]) + }) + + it('does not force unrelated residual lines into a character replacement', async () => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'unrelated-residual-lines', + before: 'abcdefghij old\n', + after: 'Wonderful\nuvwxyz new\n' + }) + + expect( + lines.map((line) => ({ + kind: line.kind, + text: line.segments.map((segment) => segment.text).join(''), + changed: line.segments + .filter((segment) => segment.kind === line.kind) + .map((segment) => segment.text) + .join('') + })) + ).toEqual([ + { kind: 'removed', text: 'abcdefghij old\n', changed: 'abcdefghij old\n' }, + { kind: 'added', text: 'Wonderful\n', changed: 'Wonderful\n' }, + { kind: 'added', text: 'uvwxyz new\n', changed: 'uvwxyz new\n' } + ]) + }) + + it.each([ + { + label: 'insertion', + before: 'Heading old.\n', + after: 'Heading new.\nWonderful\n', + trailingKind: 'added', + trailingText: 'Wonderful\n' + }, + { + label: 'removal', + before: 'Heading old.\nWonderful\n', + after: 'Heading new.\n', + trailingKind: 'removed', + trailingText: 'Wonderful\n' + } + ])('keeps a line-count $label after a modified line separate', async (fixture) => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: `line-after-modification-${fixture.label}`, + before: fixture.before, + after: fixture.after + }) + + expect(lines.slice(0, 2).map((line) => line.segments.map((segment) => segment.text))).toEqual([ + ['Heading ', 'old', '.\n'], + ['Heading ', 'new', '.\n'] + ]) + expect(lines.at(-1)).toMatchObject({ + kind: fixture.trailingKind, + segments: [{ kind: fixture.trailingKind, text: fixture.trailingText }] + }) + }) + + it('aligns repeated similar lines around a true insertion', async () => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'repeated-lines-around-insertion', + before: 'Repeat alpha.\nRepeat alpha.\n', + after: 'Repeat beta.\nInserted\nRepeat beta.\n' + }) + + expect( + lines.map((line) => ({ + kind: line.kind, + text: line.segments.map((segment) => segment.text).join(''), + changed: line.segments + .filter((segment) => segment.kind === line.kind) + .map((segment) => segment.text) + .join('') + })) + ).toEqual([ + { kind: 'removed', text: 'Repeat alpha.\n', changed: 'lpha' }, + { kind: 'added', text: 'Repeat beta.\n', changed: 'bet' }, + { kind: 'added', text: 'Inserted\n', changed: 'Inserted\n' }, + { kind: 'removed', text: 'Repeat alpha.\n', changed: 'lpha' }, + { kind: 'added', text: 'Repeat beta.\n', changed: 'bet' } + ]) + }) + + it('treats an equal-count unrelated line swap as whole-line removal and addition', async () => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'equal-count-unrelated-line-swap', + before: '## Heading\nObsolete\nParagraph old.\n', + after: '## Heading?\nWonderful\nParagraph new.\n' + }) + + expect( + lines.slice(2, 4).map((line) => ({ + kind: line.kind, + text: line.segments.map((segment) => segment.text).join(''), + changed: line.segments + .filter((segment) => segment.kind === line.kind) + .map((segment) => segment.text) + .join('') + })) + ).toEqual([ + { kind: 'removed', text: 'Obsolete\n', changed: 'Obsolete' }, + { kind: 'added', text: 'Wonderful\n', changed: 'Wonderful' } + ]) + }) + + it.each([ + { label: 'short Markdown heading', before: '# A\n', after: '# B\n', common: '# ' }, + { label: 'short Chinese text', before: '你好甲\n', after: '你好乙\n', common: '你好' }, + { label: 'short prefix extension', before: 'Name\n', after: 'Name extended\n', common: 'Name' } + ])('keeps the shared characters in a $label', async (fixture) => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: `short-line-${fixture.label}`, + before: fixture.before, + after: fixture.after + }) + + expect( + lines[0]?.segments + .filter((segment) => segment.kind === 'context') + .map((segment) => segment.text) + .join('') + ).toContain(fixture.common) + expect( + lines[1]?.segments + .filter((segment) => segment.kind === 'context') + .map((segment) => segment.text) + .join('') + ).toContain(fixture.common) + }) + + it.each([ + { + label: 'split', + before: 'Hello world\n', + after: 'Hello \nworld\n', + removed: '', + added: '\n' + }, + { + label: 'merge', + before: 'Hello \nworld\n', + after: 'Hello world\n', + removed: '\n', + added: '' + } + ])('marks only the changed line ending for a line $label', async (fixture) => { + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: `line-${fixture.label}`, + before: fixture.before, + after: fixture.after + }) + + expect( + lines + .filter((line) => line.kind !== 'added') + .flatMap((line) => line.segments) + .map((segment) => segment.text) + .join('') + ).toBe(fixture.before) + expect( + lines + .filter((line) => line.kind !== 'removed') + .flatMap((line) => line.segments) + .map((segment) => segment.text) + .join('') + ).toBe(fixture.after) + expect( + lines + .flatMap((line) => line.segments) + .filter((segment) => segment.kind === 'removed') + .map((segment) => segment.text) + .join('') + ).toBe(fixture.removed) + expect( + lines + .flatMap((line) => line.segments) + .filter((segment) => segment.kind === 'added') + .map((segment) => segment.text) + .join('') + ).toBe(fixture.added) + }) + + it('preserves context symmetrically across a one-to-many line split', async () => { + const before = 'StartMiddleEnd' + const after = 'Start\ninserted Middle\nEnd' + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'one-to-many-line-split-context', + before, + after + }) + + expect( + lines + .filter((line) => line.kind !== 'added') + .flatMap((line) => line.segments) + .map((segment) => segment.text) + .join('') + ).toBe(before) + expect( + lines + .filter((line) => line.kind !== 'removed') + .flatMap((line) => line.segments) + .map((segment) => segment.text) + .join('') + ).toBe(after) + expect( + lines + .filter((line) => line.kind === 'removed') + .flatMap((line) => line.segments) + .filter((segment) => segment.kind === 'context') + .map((segment) => segment.text) + .join('') + ).toBe('StartMiddleEnd') + expect( + lines + .filter((line) => line.kind === 'added') + .flatMap((line) => line.segments) + .filter((segment) => segment.kind === 'context') + .map((segment) => segment.text) + .join('') + ).toBe('StartMiddleEnd') + }) + + it('preserves cross-line anchors carried by an otherwise matched line', async () => { + const before = 'Line one\nStartMiddleEnd' + const after = 'New first\nMiddle\nEnd' + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'matched-line-cross-line-anchor', + before, + after + }) + const beforeContext = lines + .filter((line) => line.kind !== 'added') + .flatMap((line) => line.segments) + .filter((segment) => segment.kind === 'context') + .map((segment) => segment.text) + .join('') + const afterContext = lines + .filter((line) => line.kind !== 'removed') + .flatMap((line) => line.segments) + .filter((segment) => segment.kind === 'context') + .map((segment) => segment.text) + .join('') + + expect(beforeContext).toBe('\nMiddleEnd') + expect(afterContext).toBe('\nMiddleEnd') + }) + + it('propagates cross-line anchors through all connected natural-language lines', async () => { + const before = 'Intro\nMiddleEnd' + const after = 'Intro Middle\nInserted End\nDone' + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'transitive-cross-line-anchors', + before, + after + }) + + for (const [excludedKind, expected] of [ + ['added', before], + ['removed', after] + ] as const) { + expect( + lines + .filter((line) => line.kind !== excludedKind) + .flatMap((line) => line.segments) + .map((segment) => segment.text) + .join('') + ).toBe(expected) + } + const beforeContext = lines + .filter((line) => line.kind !== 'added') + .flatMap((line) => line.segments) + .filter((segment) => segment.kind === 'context') + .map((segment) => segment.text) + .join('') + const afterContext = lines + .filter((line) => line.kind !== 'removed') + .flatMap((line) => line.segments) + .filter((segment) => segment.kind === 'context') + .map((segment) => segment.text) + .join('') + + expect(beforeContext).toBe('IntroMiddleEnd') + expect(afterContext).toBe('IntroMiddleEnd') + }) + + it('uses conservative alignment when a changed hunk exceeds the line budget', async () => { + const before = 'a\n'.repeat(1_001) + const after = 'b\n'.repeat(1_001) + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'line-alignment-line-budget', + before, + after + }) + + expect(lines).toHaveLength(2_002) + expect(lines[0]).toMatchObject({ + kind: 'removed', + segments: [{ kind: 'removed', text: 'a\n' }] + }) + expect(lines[1_000]).toMatchObject({ kind: 'removed' }) + expect(lines[1_001]).toMatchObject({ + kind: 'added', + segments: [{ kind: 'added', text: 'b\n' }] + }) + }) + + it('uses conservative alignment before a repeated hunk can exhaust worker memory', async () => { + const before = 'Repeat alpha.\n'.repeat(500) + const after = 'Repeat beta.\n'.repeat(500) + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'line-alignment-repeated-hunk-budget', + before, + after + }) + + expect(lines).toHaveLength(1_000) + expect(lines[0]).toMatchObject({ + kind: 'removed', + segments: [{ kind: 'removed', text: 'Repeat alpha.\n' }] + }) + expect(lines[499]).toMatchObject({ kind: 'removed' }) + expect(lines[500]).toMatchObject({ + kind: 'added', + segments: [{ kind: 'added', text: 'Repeat beta.\n' }] + }) + }) + + it('uses conservative alignment when a changed hunk exceeds the character budget', async () => { + const before = 'a'.repeat(125_001) + const after = 'b'.repeat(125_001) + const lines = await new ManagedTextDiffTaskRunner().run({ + requestId: 'line-alignment-character-budget', + before, + after + }) + + expect(lines).toEqual([ + { + kind: 'removed', + oldLineNumber: 1, + segments: [{ kind: 'removed', text: before }] + }, + { + kind: 'added', + newLineNumber: 1, + segments: [{ kind: 'added', text: after }] + } + ]) + }) + + it('terminates an active worker when its request is cancelled', async () => { + let terminated = false + const runner = new ManagedTextDiffTaskRunner({ + createWorker: () => ({ + once: () => undefined, + terminate: async () => { + terminated = true + return 0 + } + }) + }) + + const pending = runner.run({ requestId: 'diff-cancel', before: 'a', after: 'b' }) + expect(runner.cancel('diff-cancel')).toBe(true) + await expect(pending).rejects.toEqual( + expect.objectContaining>({ code: 'DIFF_CANCELLED' }) + ) + expect(terminated).toBe(true) + }) + + it('terminates a worker that exceeds the hard task timeout', async () => { + let terminated = false + const runner = new ManagedTextDiffTaskRunner({ + timeoutMs: 5, + createWorker: () => ({ + once: () => undefined, + terminate: async () => { + terminated = true + return 0 + } + }) + }) + + await expect( + runner.run({ requestId: 'diff-timeout', before: 'a', after: 'b' }) + ).rejects.toEqual( + expect.objectContaining>({ code: 'DIFF_TIMEOUT' }) + ) + expect(terminated).toBe(true) + }) + + it('rejects a complete diff beyond the line limit instead of returning a truncation', async () => { + const runner = new ManagedTextDiffTaskRunner() + const before = Array.from({ length: 20_001 }, (_, index) => `old-${index}`).join('\n') + + await expect( + runner.run({ requestId: 'diff-output-limit', before, after: before }) + ).rejects.toEqual( + expect.objectContaining>({ + code: 'DIFF_OUTPUT_LIMIT_EXCEEDED' + }) + ) + }) + + it('creates workers with bounded heap and stack resources', async () => { + let resourceLimits: unknown + let emitMessage: ((value: unknown) => void) | undefined + const runner = new ManagedTextDiffTaskRunner({ + createWorker: (_task, limits) => { + resourceLimits = limits + return { + once: (event, listener) => { + if (event === 'message') emitMessage = listener as (value: unknown) => void + }, + terminate: async () => 0 + } + } + }) + + const result = runner.run({ requestId: 'diff-limited-worker', before: 'a', after: 'b' }) + emitMessage?.([]) + + await expect(result).resolves.toEqual([]) + expect(resourceLimits).toEqual({ + maxOldGenerationSizeMb: 32, + maxYoungGenerationSizeMb: 8, + stackSizeMb: 2 + }) + }) +}) diff --git a/src/main/managed-file-versions/diff-task.ts b/src/main/managed-file-versions/diff-task.ts new file mode 100644 index 000000000..cd3506254 --- /dev/null +++ b/src/main/managed-file-versions/diff-task.ts @@ -0,0 +1,655 @@ +import { Worker } from 'node:worker_threads' + +import { + MANAGED_DIFF_MAX_OUTPUT_BYTES, + MANAGED_DIFF_MAX_OUTPUT_LINES, + type ManagedFileVersionDiffLine +} from '../../shared/managed-file-versions' +import { ManagedFileVersionError } from './service' + +type DiffTask = { requestId: string; before: string; after: string } +type WorkerLike = { + once(event: 'message', listener: (value: unknown) => void): unknown + once(event: 'error', listener: (error: Error) => void): unknown + once(event: 'exit', listener: (code: number) => void): unknown + terminate(): Promise +} +type DiffWorkerResourceLimits = { + maxOldGenerationSizeMb: number + maxYoungGenerationSizeMb: number + stackSizeMb: number +} +type DiffTaskRunnerOptions = { + createWorker?: (task: DiffTask, resourceLimits: DiffWorkerResourceLimits) => WorkerLike + timeoutMs?: number +} +const DEFAULT_DIFF_TASK_TIMEOUT_MS = 10_000 +const DIFF_WORKER_RESOURCE_LIMITS: DiffWorkerResourceLimits = { + maxOldGenerationSizeMb: 32, + maxYoungGenerationSizeMb: 8, + stackSizeMb: 2 +} + +const WORKER_SOURCE = String.raw` +const { parentPort, workerData } = require('node:worker_threads') +const { diffArrays, diffChars, diffLines } = require('diff') +const LINE_ALIGNMENT_MAX_LINES = 400 +const LINE_ALIGNMENT_MAX_CHARACTERS = 50000 +const LINE_ALIGNMENT_MAX_EDIT_LENGTH = 1000 +const LINE_ALIGNMENT_TIMEOUT_MS = 500 +const CROSS_LINE_CONTINUATION_MIN_CHARACTERS = 3 +const run = () => { +const splitChangedLines = (value) => { + if (value.length === 0) return [] + const lines = [] + let start = 0 + for (let index = 0; index < value.length; index += 1) { + if (value[index] !== '\n') continue + const hasCarriageReturn = index > start && value[index - 1] === '\r' + lines.push({ + text: value.slice(start, hasCarriageReturn ? index - 1 : index), + ending: hasCarriageReturn ? '\r\n' : '\n' + }) + start = index + 1 + } + if (start < value.length) lines.push({ text: value.slice(start), ending: '' }) + return lines +} +const anchorCharacterChanges = (changes) => { + const anchored = [] + let nextAnchor = 0 + for (const change of changes) { + if (change.added || change.removed) { + anchored.push(change) + continue + } + let start = 0 + for (let index = 0; index < change.value.length; index += 1) { + if (change.value[index] !== '\n') continue + const endingStart = index > start && change.value[index - 1] === '\r' ? index - 1 : index + if (endingStart > start) { + anchored.push({ value: change.value.slice(start, endingStart), anchor: nextAnchor++ }) + } + const ending = change.value.slice(endingStart, index + 1) + const previous = anchored.at(-1) + if (previous?.anchor !== undefined && !previous.added && !previous.removed) { + previous.value += ending + } else { + anchored.push({ value: ending }) + } + start = index + 1 + } + if (start < change.value.length) { + anchored.push({ value: change.value.slice(start), anchor: nextAnchor++ }) + } + } + return anchored +} +const projectCharacterChanges = (lines, changes, side) => { + const projected = lines.map((line) => ({ ...line, segments: [] })) + let lineIndex = 0 + let lineOffset = 0 + let forceBoundary = false + for (const change of changes) { + if ((side === 'removed' && change.added) || (side === 'added' && change.removed)) { + forceBoundary = true + continue + } + const kind = change.added || change.removed ? side : 'context' + let valueOffset = 0 + while (valueOffset < change.value.length) { + const line = projected[lineIndex] + if (!line) return undefined + const lineLength = line.text.length + line.ending.length + const take = Math.min(change.value.length - valueOffset, lineLength - lineOffset) + const text = change.value.slice(valueOffset, valueOffset + take) + const previous = line.segments.at(-1) + if (!forceBoundary && previous?.kind === kind && previous.anchor === change.anchor) { + previous.text += text + } else { + line.segments.push({ kind, text, anchor: change.anchor }) + } + forceBoundary = false + valueOffset += take + lineOffset += take + if (lineOffset === lineLength) { + lineIndex += 1 + lineOffset = 0 + } + } + } + return lineIndex === projected.length && lineOffset === 0 ? projected : undefined +} +const segmentsForPair = (before, after, options) => { + const beforeValue = before.text + before.ending + const afterValue = after.text + after.ending + const changes = diffChars(beforeValue, afterValue, options) + if (!changes) return undefined + const removed = [] + const added = [] + for (const change of changes) { + if (change.added) added.push({ kind: 'added', text: change.value }) + else if (change.removed) removed.push({ kind: 'removed', text: change.value }) + else { + removed.push({ kind: 'context', text: change.value }) + added.push({ kind: 'context', text: change.value }) + } + } + return { removed, added } +} +const segmentsForReplacement = (before, after) => { + const markdownPrefix = (text) => + text.match(/^(?:(?: {0,3}> ?)|(?: {0,3}(?:[-+*] |\d{1,9}[.)] )))+/)?.[0] ?? + text.match(/^(?:[ \t]*#{1,6}[ \t]+|[ \t]+)/)?.[0] ?? + '' + const beforePrefix = markdownPrefix(before.text) + const afterPrefix = markdownPrefix(after.text) + const contextPrefix = beforePrefix === afterPrefix ? beforePrefix : '' + const endingSegments = segmentsForPair( + { text: '', ending: before.ending }, + { text: '', ending: after.ending } + ) + return { + removed: [ + ...(contextPrefix.length > 0 ? [{ kind: 'context', text: contextPrefix }] : []), + ...(before.text.length > contextPrefix.length + ? [{ kind: 'removed', text: before.text.slice(contextPrefix.length) }] + : []), + ...endingSegments.removed + ], + added: [ + ...(contextPrefix.length > 0 ? [{ kind: 'context', text: contextPrefix }] : []), + ...(after.text.length > contextPrefix.length + ? [{ kind: 'added', text: after.text.slice(contextPrefix.length) }] + : []), + ...endingSegments.added + ] + } +} +const changedSegments = (line, kind) => [{ kind, text: line.text + line.ending }] +const conservativeLineAlignment = (beforeLines, afterLines) => [ + ...beforeLines.map((before) => ({ kind: 'removed', before })), + ...afterLines.map((after) => ({ kind: 'added', after })) +] +const alignChangedLines = (beforeLines, afterLines) => { + if (beforeLines.length === 0 || afterLines.length === 0) { + return conservativeLineAlignment(beforeLines, afterLines) + } + const characterCount = [...beforeLines, ...afterLines].reduce( + (total, line) => total + line.text.length + line.ending.length, + 0 + ) + if ( + beforeLines.length + afterLines.length > LINE_ALIGNMENT_MAX_LINES || + characterCount > LINE_ALIGNMENT_MAX_CHARACTERS + ) { + return conservativeLineAlignment(beforeLines, afterLines) + } + + const deadline = Date.now() + LINE_ALIGNMENT_TIMEOUT_MS + const characterChanges = diffChars( + beforeLines.map((line) => line.text + line.ending).join(''), + afterLines.map((line) => line.text + line.ending).join(''), + { maxEditLength: LINE_ALIGNMENT_MAX_EDIT_LENGTH, timeout: LINE_ALIGNMENT_TIMEOUT_MS } + ) + if (!characterChanges) return conservativeLineAlignment(beforeLines, afterLines) + const anchoredCharacterChanges = anchorCharacterChanges(characterChanges) + const projectedBeforeLines = projectCharacterChanges(beforeLines, anchoredCharacterChanges, 'removed') + const projectedAfterLines = projectCharacterChanges(afterLines, anchoredCharacterChanges, 'added') + if (!projectedBeforeLines || !projectedAfterLines) { + return conservativeLineAlignment(beforeLines, afterLines) + } + const toToken = (line, index) => { + const anchors = new Map() + for (const segment of line.segments) { + if (segment.kind !== 'context' || segment.anchor === undefined) continue + const text = segment.text.replace(/\r?\n$/, '') + if (text.length > 0) anchors.set(segment.anchor, [...text].length) + } + return { index, line, anchors } + } + const beforeTokens = projectedBeforeLines.map(toToken) + const afterTokens = projectedAfterLines.map(toToken) + const pairDetails = beforeTokens.map((before) => + afterTokens.map((after) => { + const anchors = [] + let commonCharacters = 0 + let longestAnchor = 0 + for (const [candidate, length] of before.anchors) { + if (!after.anchors.has(candidate)) continue + anchors.push(candidate) + commonCharacters += length + longestAnchor = Math.max(longestAnchor, length) + } + const shortestLine = Math.min([...before.line.text].length, [...after.line.text].length) + const isSameBlankLine = + before.line.text.trim().length === 0 && + before.line.text === after.line.text && + before.line.ending === after.line.ending + return { + anchors, + related: isSameBlankLine || (shortestLine > 0 && commonCharacters * 2 >= shortestLine), + connectsCrossLine: longestAnchor >= CROSS_LINE_CONTINUATION_MIN_CHARACTERS + } + }) + ) + const linesAreRelated = (before, after) => pairDetails[before.index][after.index].related + + const parent = Array.from( + { length: beforeTokens.length + afterTokens.length }, + (_, index) => index + ) + const find = (node) => { + while (parent[node] !== node) { + parent[node] = parent[parent[node]] + node = parent[node] + } + return node + } + const union = (left, right) => { + const leftRoot = find(left) + const rightRoot = find(right) + if (leftRoot !== rightRoot) parent[rightRoot] = leftRoot + } + const beforeDegrees = beforeTokens.map(() => 0) + const afterDegrees = afterTokens.map(() => 0) + const anchorEdges = [] + for (let beforeIndex = 0; beforeIndex < beforeTokens.length; beforeIndex += 1) { + for (let afterIndex = 0; afterIndex < afterTokens.length; afterIndex += 1) { + const detail = pairDetails[beforeIndex][afterIndex] + if (!detail.related && !detail.connectsCrossLine) continue + const beforeNode = beforeIndex + const afterNode = beforeTokens.length + afterIndex + union(beforeNode, afterNode) + beforeDegrees[beforeIndex] += 1 + afterDegrees[afterIndex] += 1 + anchorEdges.push({ beforeNode, afterNode, anchors: detail.anchors }) + } + } + const crossLineRoots = new Set() + for (let index = 0; index < beforeDegrees.length; index += 1) { + if (beforeDegrees[index] > 1) crossLineRoots.add(find(index)) + } + for (let index = 0; index < afterDegrees.length; index += 1) { + if (afterDegrees[index] > 1) crossLineRoots.add(find(beforeTokens.length + index)) + } + const crossLineAnchors = new Set() + for (const edge of anchorEdges) { + if (!crossLineRoots.has(find(edge.beforeNode))) continue + for (const candidate of edge.anchors) crossLineAnchors.add(candidate) + } + const remainingMs = deadline - Date.now() + if (remainingMs <= 0) return conservativeLineAlignment(beforeLines, afterLines) + const changes = diffArrays(beforeTokens, afterTokens, { + comparator: linesAreRelated, + maxEditLength: beforeLines.length + afterLines.length, + timeout: remainingMs + }) + if (!changes) return conservativeLineAlignment(beforeLines, afterLines) + + const hasCrossLineAnchor = (token) => { + for (const candidate of token.anchors.keys()) { + if (crossLineAnchors.has(candidate)) return true + } + return false + } + const crossLineSegments = (line, kind) => { + const segments = [] + for (const segment of line.segments) { + const segmentKind = + segment.kind === 'context' && + ((segment.anchor !== undefined && crossLineAnchors.has(segment.anchor)) || + (segment.anchor === undefined && /^(?:\r?\n)+$/.test(segment.text))) + ? 'context' + : kind + const previous = segments.at(-1) + if (previous?.kind === segmentKind) previous.text += segment.text + else segments.push({ kind: segmentKind, text: segment.text }) + } + return segments + } + const crossLinePairSegments = (beforeIndex, afterIndex) => + hasCrossLineAnchor(beforeTokens[beforeIndex]) || hasCrossLineAnchor(afterTokens[afterIndex]) + ? { + removed: crossLineSegments(projectedBeforeLines[beforeIndex], 'removed'), + added: crossLineSegments(projectedAfterLines[afterIndex], 'added') + } + : undefined + + const alignment = [] + let beforeIndex = 0 + let afterIndex = 0 + for (let changeIndex = 0; changeIndex < changes.length; changeIndex += 1) { + const change = changes[changeIndex] + const next = changes[changeIndex + 1] + if (change.removed && next?.added) { + const beforeRun = projectedBeforeLines.slice(beforeIndex, beforeIndex + change.value.length) + const afterRun = projectedAfterLines.slice(afterIndex, afterIndex + next.value.length) + const beforeNonBlankCount = beforeRun.filter((line) => line.text.length > 0).length + const afterNonBlankCount = afterRun.filter((line) => line.text.length > 0).length + if ( + beforeRun.length === afterRun.length || + beforeNonBlankCount === afterNonBlankCount + ) { + let beforeRunIndex = 0 + let afterRunIndex = 0 + while (beforeRunIndex < beforeRun.length || afterRunIndex < afterRun.length) { + const before = beforeRun[beforeRunIndex] + const after = afterRun[afterRunIndex] + const beforeRemaining = beforeRun.length - beforeRunIndex + const afterRemaining = afterRun.length - afterRunIndex + if (before?.text.length === 0 && beforeRemaining > afterRemaining) { + alignment.push({ kind: 'removed', before }) + beforeRunIndex += 1 + } else if (after?.text.length === 0 && afterRemaining > beforeRemaining) { + alignment.push({ kind: 'added', after }) + afterRunIndex += 1 + } else if (before && after) { + alignment.push({ + kind: 'replacement', + before, + after, + segments: crossLinePairSegments( + beforeIndex + beforeRunIndex, + afterIndex + afterRunIndex + ) + }) + beforeRunIndex += 1 + afterRunIndex += 1 + } else if (before) { + alignment.push({ kind: 'removed', before }) + beforeRunIndex += 1 + } else if (after) { + alignment.push({ kind: 'added', after }) + afterRunIndex += 1 + } + } + beforeIndex += change.value.length + afterIndex += next.value.length + changeIndex += 1 + continue + } + } + for (let index = 0; index < change.value.length; index += 1) { + if (change.removed) { + alignment.push({ + kind: 'removed', + before: projectedBeforeLines[beforeIndex], + segments: hasCrossLineAnchor(beforeTokens[beforeIndex]) + ? crossLineSegments(projectedBeforeLines[beforeIndex], 'removed') + : undefined + }) + beforeIndex += 1 + } else if (change.added) { + alignment.push({ + kind: 'added', + after: projectedAfterLines[afterIndex], + segments: hasCrossLineAnchor(afterTokens[afterIndex]) + ? crossLineSegments(projectedAfterLines[afterIndex], 'added') + : undefined + }) + afterIndex += 1 + } + else { + alignment.push({ + kind: 'paired', + before: projectedBeforeLines[beforeIndex], + after: projectedAfterLines[afterIndex], + segments: crossLinePairSegments(beforeIndex, afterIndex), + deadline + }) + beforeIndex += 1 + afterIndex += 1 + } + } + } + return alignment +} +let oldLine = 1 +let newLine = 1 +const lines = [] +const changes = diffLines(workerData.before, workerData.after, { timeout: 9000, maxEditLength: 20000 }) +if (!changes) { + parentPort.postMessage({ error: 'DIFF_TIMEOUT' }) + return +} +let outputBytes = 2 +const pushLine = (line) => { + const nextBytes = Buffer.byteLength(JSON.stringify(line), 'utf8') + (lines.length === 0 ? 0 : 1) + if (lines.length + 1 > workerData.maxOutputLines || outputBytes + nextBytes > workerData.maxOutputBytes) { + parentPort.postMessage({ error: 'DIFF_OUTPUT_LIMIT_EXCEEDED' }) + return false + } + lines.push(line) + outputBytes += nextBytes + return true +} +const pushAlignedLine = (aligned) => { + if (aligned.kind === 'paired') { + const beforeText = aligned.before.text + aligned.before.ending + const afterText = aligned.after.text + aligned.after.ending + if (beforeText === afterText) { + return pushLine({ + kind: 'context', + oldLineNumber: oldLine++, + newLineNumber: newLine++, + segments: [{ kind: 'context', text: beforeText }] + }) + } + const remainingMs = aligned.deadline - Date.now() + const exactSegments = + aligned.segments ?? + (remainingMs > 0 + ? segmentsForPair(aligned.before, aligned.after, { + maxEditLength: LINE_ALIGNMENT_MAX_EDIT_LENGTH, + timeout: remainingMs + }) + : undefined) + const segments = exactSegments ?? segmentsForReplacement(aligned.before, aligned.after) + return ( + pushLine({ kind: 'removed', oldLineNumber: oldLine++, segments: segments.removed }) && + pushLine({ kind: 'added', newLineNumber: newLine++, segments: segments.added }) + ) + } + if (aligned.kind === 'replacement') { + const segments = aligned.segments ?? segmentsForReplacement(aligned.before, aligned.after) + return ( + pushLine({ kind: 'removed', oldLineNumber: oldLine++, segments: segments.removed }) && + pushLine({ kind: 'added', newLineNumber: newLine++, segments: segments.added }) + ) + } + if (aligned.kind === 'removed') { + return pushLine({ + kind: 'removed', + oldLineNumber: oldLine++, + segments: aligned.segments ?? changedSegments(aligned.before, 'removed') + }) + } + return pushLine({ + kind: 'added', + newLineNumber: newLine++, + segments: aligned.segments ?? changedSegments(aligned.after, 'added') + }) +} + +let lineGroup = [] +const pushContextEntry = (entry) => + pushLine({ + kind: 'context', + oldLineNumber: oldLine++, + newLineNumber: newLine++, + segments: [{ kind: 'context', text: entry.line.text + entry.line.ending }] + }) +const alignLineGroup = (group) => { + const hasChanges = group.some((entry) => entry.kind !== 'context') + if (!hasChanges) return group.every(pushContextEntry) + const beforeLines = group + .filter((entry) => entry.kind !== 'added') + .map((entry) => entry.line) + const afterLines = group + .filter((entry) => entry.kind !== 'removed') + .map((entry) => entry.line) + const characterCount = [...beforeLines, ...afterLines].reduce( + (total, line) => total + line.text.length + line.ending.length, + 0 + ) + if ( + beforeLines.length + afterLines.length > LINE_ALIGNMENT_MAX_LINES || + characterCount > LINE_ALIGNMENT_MAX_CHARACTERS + ) { + const middle = (group.length - 1) / 2 + let splitIndex = -1 + let splitDistance = Number.POSITIVE_INFINITY + for (let index = 0; index < group.length; index += 1) { + if (group[index].kind !== 'context') continue + const distance = Math.abs(index - middle) + if (distance < splitDistance) { + splitIndex = index + splitDistance = distance + } + } + if (splitIndex >= 0) { + return ( + alignLineGroup(group.slice(0, splitIndex)) && + pushContextEntry(group[splitIndex]) && + alignLineGroup(group.slice(splitIndex + 1)) + ) + } + } + return alignChangedLines(beforeLines, afterLines).every(pushAlignedLine) +} +const flushLineGroup = () => { + if (lineGroup.length === 0) return true + const group = lineGroup + lineGroup = [] + return alignLineGroup(group) +} + +// Blank lines are weak diff anchors: repeated separators can make a newly inserted heading match +// the preceding paragraph. Re-align changes and their blank separators as one semantic region; +// unchanged non-blank lines remain hard boundaries and never enter the more expensive alignment. +for (const change of changes) { + const kind = change.removed ? 'removed' : change.added ? 'added' : 'context' + for (const line of splitChangedLines(change.value)) { + if (kind === 'context' && line.text.trim().length > 0) { + if (!flushLineGroup()) return + if ( + !pushLine({ + kind: 'context', + oldLineNumber: oldLine++, + newLineNumber: newLine++, + segments: [{ kind: 'context', text: line.text + line.ending }] + }) + ) return + continue + } + lineGroup.push({ kind, line }) + } +} +if (!flushLineGroup()) return +parentPort.postMessage(lines) +} +run() +` + +class ManagedTextDiffTaskRunner { + private readonly active = new Map< + string, + { worker: WorkerLike; reject: (error: ManagedFileVersionError) => void } + >() + + constructor(private readonly options: DiffTaskRunnerOptions = {}) {} + + run(task: DiffTask): Promise { + if (this.active.has(task.requestId)) { + return Promise.reject( + new ManagedFileVersionError('INVALID_REQUEST', 'Diff request id is already active.') + ) + } + const worker = + this.options.createWorker?.(task, DIFF_WORKER_RESOURCE_LIMITS) ?? + new Worker(WORKER_SOURCE, { + eval: true, + resourceLimits: DIFF_WORKER_RESOURCE_LIMITS, + workerData: { + before: task.before, + after: task.after, + maxOutputLines: MANAGED_DIFF_MAX_OUTPUT_LINES, + maxOutputBytes: MANAGED_DIFF_MAX_OUTPUT_BYTES + } + }) + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + if (!this.active.delete(task.requestId)) return + reject(new ManagedFileVersionError('DIFF_TIMEOUT', 'Diff task exceeded the time limit.')) + void worker.terminate() + }, this.options.timeoutMs ?? DEFAULT_DIFF_TASK_TIMEOUT_MS) + const clear = (): boolean => { + clearTimeout(timeout) + return this.active.delete(task.requestId) + } + this.active.set(task.requestId, { + worker, + reject: (error) => { + clearTimeout(timeout) + reject(error) + } + }) + worker.once('message', (value: unknown) => { + clear() + if (typeof value === 'object' && value !== null && 'error' in value) { + const code = value.error === 'DIFF_OUTPUT_LIMIT_EXCEEDED' ? value.error : 'DIFF_TIMEOUT' + reject( + new ManagedFileVersionError( + code, + code === 'DIFF_TIMEOUT' + ? 'Diff task exceeded the time limit.' + : 'The complete diff exceeds the display limit.' + ) + ) + return + } + const lines = value as ManagedFileVersionDiffLine[] + if ( + lines.length > MANAGED_DIFF_MAX_OUTPUT_LINES || + Buffer.byteLength(JSON.stringify(lines), 'utf8') > MANAGED_DIFF_MAX_OUTPUT_BYTES + ) { + reject( + new ManagedFileVersionError( + 'DIFF_OUTPUT_LIMIT_EXCEEDED', + 'The complete diff exceeds the display limit.' + ) + ) + return + } + resolve(lines) + }) + worker.once('error', (error: Error) => { + if (!clear()) return + reject( + new ManagedFileVersionError('CONTENT_INTEGRITY_FAILED', 'Diff task failed.', { + cause: error + }) + ) + }) + worker.once('exit', (code: number) => { + if (code === 0 || !clear()) return + reject( + new ManagedFileVersionError('CONTENT_INTEGRITY_FAILED', 'Diff task exited unexpectedly.') + ) + }) + }) + } + + cancel(requestId: string): boolean { + const active = this.active.get(requestId) + if (!active) return false + this.active.delete(requestId) + active.reject(new ManagedFileVersionError('DIFF_CANCELLED', 'Diff request was cancelled.')) + void active.worker.terminate() + return true + } +} + +export { ManagedTextDiffTaskRunner } +export type { DiffTask, DiffTaskRunnerOptions } diff --git a/src/main/managed-file-versions/ipc.test.ts b/src/main/managed-file-versions/ipc.test.ts new file mode 100644 index 000000000..669ce709b --- /dev/null +++ b/src/main/managed-file-versions/ipc.test.ts @@ -0,0 +1,536 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import type { + ManagedFileVersionCancelDiffRequest, + ManagedFileVersionDiffRequest, + ManagedFileVersionDiffResult, + ManagedFileVersionIpcResult, + ManagedFileVersionInspectRequest, + ManagedFileVersionInspectResult, + ManagedFileVersionSaveTextEditRequest, + SaveTextEditResult +} from '../../shared/managed-file-versions' + +const { registered } = vi.hoisted(() => ({ + registered: new Map unknown>() +})) + +vi.mock('../ipc-handler-registry', () => ({ + ipcMainHandle: vi.fn((channel: string, handler: (event: unknown, request: never) => unknown) => { + registered.set(channel, handler) + }) +})) + +import { ManagedFileVersionError } from './service' +import { createManagedFileVersionHandlers, registerManagedFileVersionIpcHandlers } from './ipc' + +const inspectRequest: ManagedFileVersionInspectRequest = { + source: 'artifact', + projectId: 'project-1', + fileId: 'artifact-1' +} +const saveRequest: ManagedFileVersionSaveTextEditRequest = { + ...inspectRequest, + basedOnVersionId: 'version-1', + expectedHeadVersionId: 'version-1', + operationId: 'operation-1', + content: 'changed\n' +} +const diffRequest: ManagedFileVersionDiffRequest = { + ...inspectRequest, + versionId: 'version-2', + requestId: 'diff-request-1' +} +const cancelDiffRequest: ManagedFileVersionCancelDiffRequest = { requestId: 'diff-request-1' } +const diffResult: ManagedFileVersionDiffResult = { + baseVersionId: 'version-1', + selectedVersionId: 'version-2', + lines: [ + { + kind: 'removed', + oldLineNumber: 1, + segments: [{ kind: 'removed', text: 'before' }] + }, + { + kind: 'added', + newLineNumber: 1, + segments: [{ kind: 'added', text: 'after' }] + } + ] +} +const inspectResult: ManagedFileVersionInspectResult = { + ...inspectRequest, + sessionId: 'session-1', + displayName: 'README.md', + headVersionId: 'version-1', + selectedVersionId: 'version-1', + versions: [], + canEdit: true, + canDiff: false, + text: 'before\n', + textFormat: { hasUtf8Bom: false, newline: 'lf', hasTrailingNewline: true } +} +const saveResult: SaveTextEditResult = { + kind: 'created', + replayed: false, + version: { + id: 'version-2', + source: 'artifact', + fileId: 'artifact-1', + versionNumber: 2, + displayName: 'README.md', + originKind: 'user_edit', + basedOnVersionId: 'version-1', + contentType: 'text/markdown', + sizeBytes: 8, + checksum: 'a'.repeat(64), + createdAt: '2026-08-11T00:00:00.000Z' + }, + headVersionId: 'version-2' +} + +describe('managed file version IPC', () => { + beforeEach(() => registered.clear()) + + it('returns renderer-safe discriminated envelopes and gates writes through the data-root lease', async () => { + const service = { + getCapability: vi.fn(() => ({ available: true as const })), + inspect: vi.fn(async () => inspectResult), + diffText: vi.fn(async () => diffResult), + cancelDiff: vi.fn(() => true), + saveTextEdit: vi.fn(async () => saveResult) + } + const writeGateEntered = vi.fn() + const withDataRootWrite = async (write: () => Promise): Promise => { + writeGateEntered() + return write() + } + const onChanged = vi.fn() + const handlers = createManagedFileVersionHandlers(service, { withDataRootWrite, onChanged }) + + await expect(handlers.inspect(inspectRequest)).resolves.toEqual({ + ok: true, + value: inspectResult + }) + await expect(handlers.diffText(diffRequest)).resolves.toEqual({ ok: true, value: diffResult }) + expect(handlers.cancelDiff(cancelDiffRequest)).toEqual({ ok: true, value: { cancelled: true } }) + await expect(handlers.saveTextEdit(saveRequest)).resolves.toEqual({ + ok: true, + value: saveResult + }) + expect(writeGateEntered).toHaveBeenCalledTimes(1) + expect(onChanged).toHaveBeenCalledWith({ + projectId: 'project-1', + sources: ['artifact'], + kind: 'upsert' + }) + }) + + it('preserves stable expected error codes instead of relying on Electron Error serialization', async () => { + const handlers = createManagedFileVersionHandlers( + { + getCapability: vi.fn(() => ({ + available: false as const, + reason: 'NATIVE_WRITE_REQUIRED' as const + })), + inspect: vi.fn(async () => { + throw new ManagedFileVersionError('INVALID_UTF8', 'Not valid UTF-8.') + }), + diffText: vi.fn(async () => { + throw new ManagedFileVersionError('DIFF_OUTPUT_LIMIT_EXCEEDED', 'Diff is too large.') + }), + cancelDiff: vi.fn(() => false), + saveTextEdit: vi.fn(async () => { + throw new Error('unexpected implementation failure') + }) + }, + { withDataRootWrite: async (write) => write() } + ) + + await expect(handlers.inspect(inspectRequest)).resolves.toEqual({ + ok: false, + error: { code: 'INVALID_UTF8', message: 'Not valid UTF-8.' } + }) + await expect(handlers.diffText(diffRequest)).resolves.toEqual({ + ok: false, + error: { code: 'DIFF_OUTPUT_LIMIT_EXCEEDED', message: 'Diff is too large.' } + }) + await expect(handlers.saveTextEdit(saveRequest)).resolves.toEqual({ + ok: false, + error: { code: 'CONTENT_INTEGRITY_FAILED', message: 'Managed file operation failed.' } + }) + }) + + it('does not emit another Files change event for a replayed published operation', async () => { + const onChanged = vi.fn() + const handlers = createManagedFileVersionHandlers( + { + getCapability: vi.fn(() => ({ available: true as const })), + inspect: vi.fn(async () => inspectResult), + diffText: vi.fn(async () => diffResult), + cancelDiff: vi.fn(() => false), + saveTextEdit: vi.fn(async () => ({ ...saveResult, replayed: true })) + }, + { withDataRootWrite: async (write) => write(), onChanged } + ) + + await expect(handlers.saveTextEdit(saveRequest)).resolves.toMatchObject({ + ok: true, + value: { kind: 'created', replayed: true } + }) + expect(onChanged).not.toHaveBeenCalled() + }) + + it('registers the exact typed channels and forwards requests', async () => { + const handlers = { + getCapability: vi.fn(() => ({ available: true as const })), + inspect: vi.fn(async () => ({ ok: true as const, value: inspectResult })), + diffText: vi.fn(async () => ({ ok: true as const, value: diffResult })), + cancelDiff: vi.fn(() => ({ ok: true as const, value: { cancelled: true } })), + saveTextEdit: vi.fn(async () => ({ ok: true as const, value: saveResult })) + } + registerManagedFileVersionIpcHandlers(handlers) + + expect([...registered.keys()].sort()).toEqual([ + 'managed-file-versions:cancel-diff', + 'managed-file-versions:diff-text', + 'managed-file-versions:get-capability', + 'managed-file-versions:inspect', + 'managed-file-versions:save-text-edit' + ]) + await registered.get('managed-file-versions:get-capability')?.({}, undefined as never) + await registered.get('managed-file-versions:inspect')?.({}, inspectRequest as never) + const sender = { id: 1, once: vi.fn() } + await registered.get('managed-file-versions:diff-text')?.({ sender }, diffRequest as never) + await registered.get('managed-file-versions:cancel-diff')?.( + { sender }, + cancelDiffRequest as never + ) + await registered.get('managed-file-versions:save-text-edit')?.({}, saveRequest as never) + expect(handlers.inspect).toHaveBeenCalledWith(inspectRequest) + expect(handlers.diffText).toHaveBeenCalledWith(diffRequest) + expect(handlers.cancelDiff).not.toHaveBeenCalled() + expect(handlers.saveTextEdit).toHaveBeenCalledWith(saveRequest) + expect(handlers.getCapability).toHaveBeenCalledTimes(1) + }) + + it('scopes diff cancellation to its sender and cancels all tasks when that renderer is destroyed', async () => { + let resolveDiff!: () => void + const pendingDiff = new Promise>( + (resolve) => { + resolveDiff = () => resolve({ ok: true, value: diffResult }) + } + ) + registerManagedFileVersionIpcHandlers({ + getCapability: vi.fn(() => ({ available: true as const })), + inspect: vi.fn(async () => ({ ok: true as const, value: inspectResult })), + diffText: vi.fn(() => pendingDiff), + cancelDiff: vi.fn(() => ({ ok: true as const, value: { cancelled: true } })), + saveTextEdit: vi.fn(async () => ({ ok: true as const, value: saveResult })) + }) + const destroyedListeners: Array<() => void> = [] + const senderA = { + id: 11, + once: vi.fn((_event: string, listener: () => void) => destroyedListeners.push(listener)) + } + const senderB = { id: 22, once: vi.fn() } + + const diff = registered.get('managed-file-versions:diff-text')! + const cancel = registered.get('managed-file-versions:cancel-diff')! + void diff({ sender: senderA }, diffRequest as never) + + expect(cancel({ sender: senderB }, cancelDiffRequest as never)).toEqual({ + ok: true, + value: { cancelled: false } + }) + expect(cancel({ sender: senderA }, cancelDiffRequest as never)).toEqual({ + ok: true, + value: { cancelled: true } + }) + + void diff({ sender: senderA }, { ...diffRequest, requestId: 'diff-request-2' } as never) + destroyedListeners.at(-1)?.() + expect(cancel({ sender: senderA }, { requestId: 'diff-request-2' } as never)).toEqual({ + ok: true, + value: { cancelled: false } + }) + resolveDiff() + }) + + it('rejects a colliding diff request id without transferring ownership to another sender', async () => { + let resolveDiff!: () => void + const pendingDiff = new Promise>( + (resolve) => { + resolveDiff = () => resolve({ ok: true, value: diffResult }) + } + ) + const handlers = { + getCapability: vi.fn(() => ({ available: true as const })), + inspect: vi.fn(async () => ({ ok: true as const, value: inspectResult })), + diffText: vi.fn(() => pendingDiff), + cancelDiff: vi.fn(() => ({ ok: true as const, value: { cancelled: true } })), + saveTextEdit: vi.fn(async () => ({ ok: true as const, value: saveResult })) + } + registerManagedFileVersionIpcHandlers(handlers) + const senderA = { id: 11, once: vi.fn() } + const senderB = { id: 22, once: vi.fn() } + const diff = registered.get('managed-file-versions:diff-text')! + const cancel = registered.get('managed-file-versions:cancel-diff')! + + const senderAResult = diff({ sender: senderA }, diffRequest as never) + await expect(diff({ sender: senderB }, diffRequest as never)).resolves.toEqual({ + ok: false, + error: { + code: 'INVALID_REQUEST', + message: 'Diff request id is already active.' + } + }) + expect(handlers.diffText).toHaveBeenCalledTimes(1) + expect(cancel({ sender: senderB }, cancelDiffRequest as never)).toEqual({ + ok: true, + value: { cancelled: false } + }) + + resolveDiff() + await senderAResult + }) + + it('does not allow a request id to be reused until the cancelled owner settles', async () => { + let resolveFirst!: () => void + let resolveSecond!: () => void + const first = new Promise>( + (resolve) => { + resolveFirst = () => resolve({ ok: true, value: diffResult }) + } + ) + const second = new Promise>( + (resolve) => { + resolveSecond = () => resolve({ ok: true, value: diffResult }) + } + ) + const handlers = { + getCapability: vi.fn(() => ({ available: true as const })), + inspect: vi.fn(async () => ({ ok: true as const, value: inspectResult })), + diffText: vi.fn().mockReturnValueOnce(first).mockReturnValueOnce(second), + cancelDiff: vi.fn(() => ({ ok: true as const, value: { cancelled: true } })), + saveTextEdit: vi.fn(async () => ({ ok: true as const, value: saveResult })) + } + registerManagedFileVersionIpcHandlers(handlers) + const senderA = { id: 11, once: vi.fn() } + const senderB = { id: 22, once: vi.fn() } + const diff = registered.get('managed-file-versions:diff-text')! + const cancel = registered.get('managed-file-versions:cancel-diff')! + + const firstResult = diff({ sender: senderA }, diffRequest as never) + expect(cancel({ sender: senderA }, cancelDiffRequest as never)).toEqual({ + ok: true, + value: { cancelled: true } + }) + await expect(diff({ sender: senderB }, diffRequest as never)).resolves.toMatchObject({ + ok: false, + error: { code: 'INVALID_REQUEST' } + }) + + resolveFirst() + await firstResult + const secondResult = diff({ sender: senderB }, diffRequest as never) + expect(cancel({ sender: senderB }, cancelDiffRequest as never)).toEqual({ + ok: true, + value: { cancelled: true } + }) + + resolveSecond() + await secondResult + }) + + it('bounds active diff work per sender and globally with a stable concurrency error', async () => { + const pendingResolvers: Array<() => void> = [] + const handlers = { + getCapability: vi.fn(() => ({ available: true as const })), + inspect: vi.fn(async () => ({ ok: true as const, value: inspectResult })), + diffText: vi.fn( + () => + new Promise>((resolve) => { + pendingResolvers.push(() => resolve({ ok: true, value: diffResult })) + }) + ), + cancelDiff: vi.fn(() => ({ ok: true as const, value: { cancelled: true } })), + saveTextEdit: vi.fn(async () => ({ ok: true as const, value: saveResult })) + } + registerManagedFileVersionIpcHandlers(handlers) + const diff = registered.get('managed-file-versions:diff-text')! + const senderA = { id: 11, once: vi.fn() } + const senderB = { id: 22, once: vi.fn() } + const senderC = { id: 33, once: vi.fn() } + + const active = [ + diff({ sender: senderA }, { ...diffRequest, requestId: 'a-1' } as never), + diff({ sender: senderA }, { ...diffRequest, requestId: 'a-2' } as never), + diff({ sender: senderB }, { ...diffRequest, requestId: 'b-1' } as never), + diff({ sender: senderB }, { ...diffRequest, requestId: 'b-2' } as never) + ] + + await expect( + diff({ sender: senderA }, { ...diffRequest, requestId: 'a-3' } as never) + ).resolves.toEqual({ + ok: false, + error: { + code: 'DIFF_CONCURRENCY_LIMIT', + message: 'Too many diff requests are active.' + } + }) + await expect( + diff({ sender: senderC }, { ...diffRequest, requestId: 'c-1' } as never) + ).resolves.toEqual({ + ok: false, + error: { + code: 'DIFF_CONCURRENCY_LIMIT', + message: 'Too many diff requests are active.' + } + }) + expect(handlers.diffText).toHaveBeenCalledTimes(4) + + for (const resolve of pendingResolvers) resolve() + await Promise.all(active) + }) + + it('retains ownership when cancellation arrives before the worker starts', async () => { + let resolveDiff!: () => void + const pendingDiff = new Promise>( + (resolve) => { + resolveDiff = () => resolve({ ok: true, value: diffResult }) + } + ) + const handlers = { + getCapability: vi.fn(() => ({ available: true as const })), + inspect: vi.fn(async () => ({ ok: true as const, value: inspectResult })), + diffText: vi.fn(() => pendingDiff), + cancelDiff: vi.fn(() => ({ ok: true as const, value: { cancelled: false } })), + saveTextEdit: vi.fn(async () => ({ ok: true as const, value: saveResult })) + } + registerManagedFileVersionIpcHandlers(handlers) + const senderA = { id: 11, once: vi.fn() } + const senderB = { id: 22, once: vi.fn() } + const diff = registered.get('managed-file-versions:diff-text')! + const cancel = registered.get('managed-file-versions:cancel-diff')! + + const first = diff({ sender: senderA }, diffRequest as never) + expect(cancel({ sender: senderA }, cancelDiffRequest as never)).toEqual({ + ok: true, + value: { cancelled: true } + }) + const collision = diff({ sender: senderB }, diffRequest as never) + await Promise.resolve() + expect(handlers.diffText).toHaveBeenCalledTimes(1) + + resolveDiff() + await first + await expect(collision).resolves.toMatchObject({ + ok: false, + error: { code: 'INVALID_REQUEST' } + }) + await expect(diff({ sender: senderB }, diffRequest as never)).resolves.toMatchObject({ + ok: true + }) + }) + + it('keeps a cancelled pre-worker request in the sender concurrency count until settle', async () => { + const pendingResolvers: Array<() => void> = [] + const handlers = { + getCapability: vi.fn(() => ({ available: true as const })), + inspect: vi.fn(async () => ({ ok: true as const, value: inspectResult })), + diffText: vi.fn( + () => + new Promise>((resolve) => { + pendingResolvers.push(() => resolve({ ok: true, value: diffResult })) + }) + ), + cancelDiff: vi.fn(() => ({ ok: true as const, value: { cancelled: false } })), + saveTextEdit: vi.fn(async () => ({ ok: true as const, value: saveResult })) + } + registerManagedFileVersionIpcHandlers(handlers) + const sender = { id: 11, once: vi.fn() } + const diff = registered.get('managed-file-versions:diff-text')! + const cancel = registered.get('managed-file-versions:cancel-diff')! + + const first = diff({ sender }, { ...diffRequest, requestId: 'slot-1' } as never) + const second = diff({ sender }, { ...diffRequest, requestId: 'slot-2' } as never) + expect(cancel({ sender }, { requestId: 'slot-1' } as never)).toMatchObject({ + ok: true, + value: { cancelled: true } + }) + await expect( + diff({ sender }, { ...diffRequest, requestId: 'slot-3' } as never) + ).resolves.toMatchObject({ + ok: false, + error: { code: 'DIFF_CONCURRENCY_LIMIT' } + }) + + for (const resolve of pendingResolvers) resolve() + await Promise.all([first, second]) + }) + + it('keeps destroyed sender requests owned until their pending diff settles', async () => { + let resolveDiff!: () => void + const pendingDiff = new Promise>( + (resolve) => { + resolveDiff = () => resolve({ ok: true, value: diffResult }) + } + ) + const handlers = { + getCapability: vi.fn(() => ({ available: true as const })), + inspect: vi.fn(async () => ({ ok: true as const, value: inspectResult })), + diffText: vi.fn(() => pendingDiff), + cancelDiff: vi.fn(() => ({ ok: true as const, value: { cancelled: false } })), + saveTextEdit: vi.fn(async () => ({ ok: true as const, value: saveResult })) + } + registerManagedFileVersionIpcHandlers(handlers) + const destroyedListeners: Array<() => void> = [] + const senderA = { + id: 11, + once: vi.fn((_event: string, listener: () => void) => destroyedListeners.push(listener)) + } + const senderB = { id: 22, once: vi.fn() } + const diff = registered.get('managed-file-versions:diff-text')! + + const first = diff({ sender: senderA }, diffRequest as never) + destroyedListeners[0]?.() + await expect(diff({ sender: senderB }, diffRequest as never)).resolves.toMatchObject({ + ok: false, + error: { code: 'INVALID_REQUEST' } + }) + + resolveDiff() + await first + await expect(diff({ sender: senderB }, diffRequest as never)).resolves.toMatchObject({ + ok: true + }) + }) + + it('releases sender lifecycle observation when destruction happens after all work settles', async () => { + const handlers = { + getCapability: vi.fn(() => ({ available: true as const })), + inspect: vi.fn(async () => ({ ok: true as const, value: inspectResult })), + diffText: vi.fn(async () => ({ ok: true as const, value: diffResult })), + cancelDiff: vi.fn(() => ({ ok: true as const, value: { cancelled: false } })), + saveTextEdit: vi.fn(async () => ({ ok: true as const, value: saveResult })) + } + registerManagedFileVersionIpcHandlers(handlers) + const destroyedListeners: Array<() => void> = [] + const settledSender = { + id: 41, + once: vi.fn((_event: string, listener: () => void) => destroyedListeners.push(listener)) + } + const replacementSender = { id: 41, once: vi.fn() } + const diff = registered.get('managed-file-versions:diff-text')! + + await diff({ sender: settledSender }, diffRequest as never) + destroyedListeners[0]?.() + await diff({ sender: replacementSender }, { + ...diffRequest, + requestId: 'replacement-request' + } as never) + + expect(replacementSender.once).toHaveBeenCalledOnce() + }) +}) diff --git a/src/main/managed-file-versions/ipc.ts b/src/main/managed-file-versions/ipc.ts new file mode 100644 index 000000000..0304638bd --- /dev/null +++ b/src/main/managed-file-versions/ipc.ts @@ -0,0 +1,205 @@ +import type { ProjectFilesChangedEvent } from '../../shared/project-files' +import type { + ManagedFileVersionInspectRequest, + ManagedFileVersionInspectResult, + ManagedFileVersionHostCapability, + ManagedFileVersionIpcResult, + ManagedFileVersionSaveTextEditRequest, + ManagedFileVersionCancelDiffRequest, + ManagedFileVersionDiffRequest, + ManagedFileVersionDiffResult, + SaveTextEditResult +} from '../../shared/managed-file-versions' +import { ipcMainHandle } from '../ipc-handler-registry' +import { ManagedFileVersionError } from './service' + +type ManagedFileVersionIpcService = { + getCapability(): ManagedFileVersionHostCapability + inspect(request: ManagedFileVersionInspectRequest): Promise + diffText(request: ManagedFileVersionDiffRequest): Promise + cancelDiff(requestId: string): boolean + saveTextEdit(request: ManagedFileVersionSaveTextEditRequest): Promise +} + +type ManagedFileVersionHandlerDependencies = { + withDataRootWrite(write: () => Promise): Promise + onChanged?(event: ProjectFilesChangedEvent): void +} + +type ManagedFileVersionHandlers = { + getCapability(): ManagedFileVersionHostCapability + inspect( + request: ManagedFileVersionInspectRequest + ): Promise> + diffText( + request: ManagedFileVersionDiffRequest + ): Promise> + cancelDiff( + request: ManagedFileVersionCancelDiffRequest + ): ManagedFileVersionIpcResult<{ cancelled: boolean }> + saveTextEdit( + request: ManagedFileVersionSaveTextEditRequest + ): Promise> +} + +const rendererResult = async ( + operation: () => Promise +): Promise> => { + try { + return { ok: true, value: await operation() } + } catch (error) { + if (error instanceof ManagedFileVersionError) { + return { ok: false, error: { code: error.code, message: error.message } } + } + return { + ok: false, + error: { + code: 'CONTENT_INTEGRITY_FAILED', + message: 'Managed file operation failed.' + } + } + } +} + +const createManagedFileVersionHandlers = ( + service: ManagedFileVersionIpcService, + dependencies: ManagedFileVersionHandlerDependencies +): ManagedFileVersionHandlers => ({ + getCapability: () => service.getCapability(), + inspect: (request) => rendererResult(() => service.inspect(request)), + diffText: (request) => rendererResult(() => service.diffText(request)), + cancelDiff: ({ requestId }) => ({ + ok: true, + value: { cancelled: service.cancelDiff(requestId) } + }), + saveTextEdit: (request) => + rendererResult(async () => { + const result = await dependencies.withDataRootWrite(() => service.saveTextEdit(request)) + if (result.kind === 'created' && !result.replayed) { + dependencies.onChanged?.({ + projectId: request.projectId, + sources: [request.source], + kind: 'upsert' + }) + } + return result + }) +}) + +const registerManagedFileVersionIpcHandlers = (handlers: ManagedFileVersionHandlers): void => { + const maxActiveDiffsPerSender = 2 + const maxActiveDiffsGlobal = 4 + const requestOwner = new Map() + const senderRequests = new Map>() + const observedSenders = new Set() + const destroyedSenders = new Set() + const cancellationRequested = new Set() + + const ownRequest = ( + sender: { id: number; once(event: 'destroyed', listener: () => void): unknown }, + requestId: string + ): 'owned' | 'collision' | 'limit' => { + if (requestOwner.has(requestId)) return 'collision' + if ( + requestOwner.size >= maxActiveDiffsGlobal || + (senderRequests.get(sender.id)?.size ?? 0) >= maxActiveDiffsPerSender + ) + return 'limit' + requestOwner.set(requestId, sender.id) + let requests = senderRequests.get(sender.id) + if (!requests) { + requests = new Set() + senderRequests.set(sender.id, requests) + } + requests.add(requestId) + if (observedSenders.has(sender.id)) return 'owned' + observedSenders.add(sender.id) + sender.once('destroyed', () => { + if ((senderRequests.get(sender.id)?.size ?? 0) === 0) { + observedSenders.delete(sender.id) + destroyedSenders.delete(sender.id) + return + } + destroyedSenders.add(sender.id) + for (const ownedRequestId of senderRequests.get(sender.id) ?? []) { + if (requestOwner.get(ownedRequestId) !== sender.id) continue + if (cancellationRequested.has(ownedRequestId)) continue + cancellationRequested.add(ownedRequestId) + handlers.cancelDiff({ requestId: ownedRequestId }) + } + }) + return 'owned' + } + + const releaseRequest = (senderId: number, requestId: string): void => { + if (requestOwner.get(requestId) !== senderId) return + requestOwner.delete(requestId) + cancellationRequested.delete(requestId) + const requests = senderRequests.get(senderId) + requests?.delete(requestId) + if (requests?.size === 0) { + senderRequests.delete(senderId) + if (destroyedSenders.delete(senderId)) observedSenders.delete(senderId) + } + } + + ipcMainHandle('managed-file-versions:get-capability', () => handlers.getCapability()) + ipcMainHandle( + 'managed-file-versions:inspect', + (_event, request: ManagedFileVersionInspectRequest) => handlers.inspect(request) + ) + ipcMainHandle( + 'managed-file-versions:diff-text', + async ( + event: { sender: { id: number; once(event: 'destroyed', listener: () => void): unknown } }, + request: ManagedFileVersionDiffRequest + ) => { + const ownership = ownRequest(event.sender, request.requestId) + if (ownership !== 'owned') { + return { + ok: false as const, + error: { + code: + ownership === 'collision' + ? ('INVALID_REQUEST' as const) + : ('DIFF_CONCURRENCY_LIMIT' as const), + message: + ownership === 'collision' + ? 'Diff request id is already active.' + : 'Too many diff requests are active.' + } + } + } + try { + return await handlers.diffText(request) + } finally { + releaseRequest(event.sender.id, request.requestId) + } + } + ) + ipcMainHandle( + 'managed-file-versions:cancel-diff', + (event: { sender: { id: number } }, request: ManagedFileVersionCancelDiffRequest) => { + if (requestOwner.get(request.requestId) !== event.sender.id) { + return { ok: true as const, value: { cancelled: false } } + } + if (cancellationRequested.has(request.requestId)) { + return { ok: true as const, value: { cancelled: false } } + } + cancellationRequested.add(request.requestId) + handlers.cancelDiff(request) + return { ok: true as const, value: { cancelled: true } } + } + ) + ipcMainHandle( + 'managed-file-versions:save-text-edit', + (_event, request: ManagedFileVersionSaveTextEditRequest) => handlers.saveTextEdit(request) + ) +} + +export { createManagedFileVersionHandlers, registerManagedFileVersionIpcHandlers } +export type { + ManagedFileVersionHandlerDependencies, + ManagedFileVersionHandlers, + ManagedFileVersionIpcService +} diff --git a/src/main/managed-file-versions/service.integration.test.ts b/src/main/managed-file-versions/service.integration.test.ts new file mode 100644 index 000000000..9d295fae5 --- /dev/null +++ b/src/main/managed-file-versions/service.integration.test.ts @@ -0,0 +1,2241 @@ +import { createHash } from 'node:crypto' +import { spawn } from 'node:child_process' +import { + linkSync, + mkdirSync, + readFileSync, + renameSync, + symlinkSync, + unlinkSync, + writeFileSync +} from 'node:fs' +import { + mkdir, + mkdtemp, + readFile, + readdir, + rename, + rm, + stat, + symlink, + utimes, + writeFile +} from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join, relative } from 'node:path' + +import type { PrismaClient } from '@prisma/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { createProjectDbClient, migrateApplicationDatabase } from '../projects/prisma-client' +import { MANAGED_TEXT_EDIT_MAX_BYTES } from '../../shared/managed-file-versions' +import { ManagedFileVersionError, ManagedFileVersionService } from './service' + +const checksum = (bytes: Buffer): string => createHash('sha256').update(bytes).digest('hex') +const testPublish = ( + _rootPath: string, + parentPath: string, + sourceName: string, + destinationName: string +): void => { + linkSync(join(parentPath, sourceName), join(parentPath, destinationName)) + unlinkSync(join(parentPath, sourceName)) +} +const testWriteAndPublish = ( + rootPath: string, + parentPath: string, + temporaryName: string, + destinationName: string, + bytes: Buffer +): void => { + mkdirSync(parentPath, { recursive: true }) + writeFileSync(join(parentPath, temporaryName), bytes) + testPublish(rootPath, parentPath, temporaryName, destinationName) +} + +type SourceFixture = { + source: 'artifact' | 'upload' + fileId: string + versionIds: [string, string] +} + +describe('ManagedFileVersionService (SQLite + filesystem)', () => { + let storageRoot: string + let outsideRoot: string | undefined + let client: PrismaClient + + beforeEach(async () => { + storageRoot = await mkdtemp(join(tmpdir(), 'open-science-managed-version-')) + client = createProjectDbClient(storageRoot) + await migrateApplicationDatabase(client) + await client.project.create({ data: { id: 'project-1', name: 'Project one' } }) + await client.fileOriginSession.create({ + data: { projectId: 'project-1', sessionId: 'session-1' } + }) + }) + + afterEach(async () => { + await client.$disconnect() + await rm(storageRoot, { recursive: true, force: true }) + if (outsideRoot) await rm(outsideRoot, { recursive: true, force: true }) + outsideRoot = undefined + }) + + const createFixture = async (source: 'artifact' | 'upload'): Promise => { + const fileId = `${source}-file-1` + const versionIds: [string, string] = [`${source}-v1`, `${source}-v2`] + const first = Buffer.from('\ufefffirst\r\nline\r\n') + const second = Buffer.from('second\n') + const storageKeys = versionIds.map( + (versionId) => `${source}s/project-1/session-1/${fileId}/versions/${versionId}/content` + ) + for (const [index, storageKey] of storageKeys.entries()) { + const path = join(storageRoot, ...storageKey.split('/')) + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, index === 0 ? first : second) + } + + if (source === 'artifact') { + await client.artifactLineage.create({ + data: { + id: fileId, + projectId: 'project-1', + sessionId: 'session-1', + normalizedFilename: 'readme.md', + filename: 'README.md' + } + }) + await client.artifactVersion.createMany({ + data: versionIds.map((id, index) => ({ + id, + artifactId: fileId, + versionNumber: index + 1, + filename: 'README.md', + originKind: 'legacy', + basedOnVersionId: index === 0 ? null : versionIds[0], + state: 'finalized', + contentStorageKey: storageKeys[index]!, + contentType: 'text/markdown', + sizeBytes: BigInt(index === 0 ? first.byteLength : second.byteLength), + checksum: checksum(index === 0 ? first : second) + })) + }) + await client.artifactLineage.update({ + where: { id: fileId }, + data: { currentVersionId: versionIds[1] } + }) + } else { + await client.uploadFile.create({ + data: { + id: fileId, + projectId: 'project-1', + sessionId: 'session-1', + filename: 'README.md', + originalFilename: 'README.md' + } + }) + await client.uploadVersion.createMany({ + data: versionIds.map((id, index) => ({ + id, + uploadFileId: fileId, + versionNumber: index + 1, + state: 'ready', + originKind: 'legacy', + basedOnVersionId: index === 0 ? null : versionIds[0], + contentStorageKey: storageKeys[index]!, + filename: 'README.md', + originalFilename: 'README.md', + contentType: 'text/markdown', + sizeBytes: BigInt(index === 0 ? first.byteLength : second.byteLength), + checksum: checksum(index === 0 ? first : second), + createdAt: new Date(`2026-08-0${index + 1}T00:00:00.000Z`) + })) + }) + await client.uploadFile.update({ + where: { id: fileId }, + data: { currentVersionId: versionIds[1] } + }) + } + + // Deliberately stale: default resolution must use the logical file head, not this projection. + await client.managedFile.create({ + data: { + source, + sourceFileId: fileId, + sourceVersionId: versionIds[0], + checksum: checksum(first), + projectId: 'project-1', + sessionId: 'session-1', + displayName: 'README.md', + storageKey: storageKeys[0]!, + mimeType: 'text/markdown', + sizeBytes: BigInt(first.byteLength), + mtimeMs: BigInt(1), + sortAtMs: BigInt(1) + } + }) + return { source, fileId, versionIds } + } + + it.each(['artifact', 'upload'] as const)( + 'resolves the %s DB head by default and an explicit owned historical version exactly', + async (source) => { + const fixture = await createFixture(source) + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }) + + const head = await service.inspect({ + source, + projectId: 'project-1', + fileId: fixture.fileId + }) + expect(head).toMatchObject({ + displayName: 'README.md', + headVersionId: fixture.versionIds[1], + selectedVersionId: fixture.versionIds[1], + text: 'second\n', + canEdit: true, + canDiff: true + }) + expect(head.versions.map((version) => version.id)).toEqual(fixture.versionIds) + + const historical = await service.inspect({ + source, + projectId: 'project-1', + fileId: fixture.fileId, + versionId: fixture.versionIds[0] + }) + expect(historical).toMatchObject({ + headVersionId: fixture.versionIds[1], + selectedVersionId: fixture.versionIds[0], + text: 'first\r\nline\r\n', + textFormat: { hasUtf8Bom: true, newline: 'crlf', hasTrailingNewline: true }, + canDiff: false + }) + + await expect( + service.resolve({ + source, + projectId: 'project-1', + fileId: fixture.fileId, + versionId: `${source}-other-version` + }) + ).rejects.toMatchObject({ code: 'VERSION_NOT_FOUND' }) + } + ) + + it.each(['artifact', 'upload'] as const)( + 'keeps the verified %s inode pinned when its storage path is replaced before consumption', + async (source) => { + const fixture = await createFixture(source) + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + nativeWriteAvailable: true, + verifyAnchored: (_rootPath, parentPath, name, expectedSizeBytes, expectedChecksum) => { + const bytes = readFileSync(join(parentPath, name)) + return bytes.byteLength === expectedSizeBytes && checksum(bytes) === expectedChecksum + } + }) + const lease = await service.openResolved({ + source, + projectId: 'project-1', + fileId: fixture.fileId + }) + const replacementPath = `${lease.path}.verified` + const copiedPath = join(storageRoot, `${source}-downloaded.md`) + + await rename(lease.path, replacementPath) + await writeFile(lease.path, 'attacker-controlled replacement') + + try { + await expect(lease.readRange(0, lease.size)).resolves.toEqual( + new Uint8Array(Buffer.from('second\n')) + ) + await lease.copyTo(copiedPath) + await expect(readFile(copiedPath, 'utf8')).resolves.toBe('second\n') + await expect(readFile(lease.path, 'utf8')).resolves.toBe('attacker-controlled replacement') + } finally { + await lease.close() + await lease.close() + } + } + ) + + it.each(['artifact', 'upload'] as const)( + 'diffs the selected %s version against its explicit basedOn version', + async (source) => { + const fixture = await createFixture(source) + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }) + + await expect( + service.diffText({ + source, + projectId: 'project-1', + fileId: fixture.fileId, + versionId: fixture.versionIds[1], + requestId: `${source}-diff` + }) + ).resolves.toMatchObject({ + baseVersionId: fixture.versionIds[0], + selectedVersionId: fixture.versionIds[1], + lines: expect.arrayContaining([ + expect.objectContaining({ kind: 'removed', oldLineNumber: 1 }), + expect.objectContaining({ kind: 'added', newLineNumber: 1 }) + ]) + }) + + await expect( + service.diffText({ + source, + projectId: 'project-1', + fileId: fixture.fileId, + versionId: fixture.versionIds[0], + requestId: `${source}-v1-diff` + }) + ).rejects.toMatchObject({ code: 'DIFF_BASE_NOT_FOUND' }) + } + ) + + it('cancels during asynchronous resolution before starting a diff worker', async () => { + const fixture = await createFixture('upload') + let releaseClient!: () => void + const clientGate = new Promise((resolve) => { + releaseClient = resolve + }) + const run = vi.fn() + const cancel = vi.fn(() => false) + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: async () => { + await clientGate + return client + }, + diffTaskRunner: { run, cancel } + }) + + const pending = service.diffText({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + versionId: fixture.versionIds[1], + requestId: 'cancel-before-worker' + }) + expect(service.cancelDiff('cancel-before-worker')).toBe(true) + releaseClient() + + await expect(pending).rejects.toMatchObject({ code: 'DIFF_CANCELLED' }) + expect(run).not.toHaveBeenCalled() + expect(cancel).not.toHaveBeenCalled() + }) + + it('cancels after a diff worker result is queued but before the service settles', async () => { + const fixture = await createFixture('upload') + let signalRunStarted!: () => void + const runStarted = new Promise((resolve) => { + signalRunStarted = resolve + }) + let releaseRun!: () => void + const run = vi.fn( + () => + new Promise((resolve) => { + releaseRun = () => resolve([]) + signalRunStarted() + }) + ) + const cancel = vi.fn(() => false) + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + diffTaskRunner: { run, cancel } + }) + + const pending = service.diffText({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + versionId: fixture.versionIds[1], + requestId: 'cancel-after-worker-result' + }) + await runStarted + releaseRun() + expect(service.cancelDiff('cancel-after-worker-result')).toBe(true) + + await expect(pending).rejects.toMatchObject({ code: 'DIFF_CANCELLED' }) + expect(cancel).toHaveBeenCalledWith('cancel-after-worker-result') + }) + + it('fails closed when inspect reaches an anchored reader after a version ancestor is replaced', async () => { + const fixture = await createFixture('upload') + outsideRoot = await mkdtemp(join(tmpdir(), 'open-science-managed-version-outside-')) + const versionsPath = join( + storageRoot, + 'uploads', + 'project-1', + 'session-1', + fixture.fileId, + 'versions' + ) + let readAttempts = 0 + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + readAnchored: () => { + readAttempts += 1 + renameSync(versionsPath, `${versionsPath}-replaced`) + symlinkSync(outsideRoot!, versionsPath, process.platform === 'win32' ? 'junction' : 'dir') + throw Object.assign(new Error('anchored parent changed'), { code: 'ELOOP' }) + } + }) + + await expect( + service.inspect({ source: 'upload', projectId: 'project-1', fileId: fixture.fileId }) + ).rejects.toMatchObject({ code: 'CONTENT_INTEGRITY_FAILED' }) + expect(readAttempts).toBe(1) + }) + + it('fails closed when save reads its baseline after a version ancestor is replaced', async () => { + const fixture = await createFixture('upload') + outsideRoot = await mkdtemp(join(tmpdir(), 'open-science-managed-version-outside-')) + const versionsPath = join( + storageRoot, + 'uploads', + 'project-1', + 'session-1', + fixture.fileId, + 'versions' + ) + let readAttempts = 0 + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + readAnchored: () => { + readAttempts += 1 + renameSync(versionsPath, `${versionsPath}-replaced`) + symlinkSync(outsideRoot!, versionsPath, process.platform === 'win32' ? 'junction' : 'dir') + throw Object.assign(new Error('anchored parent changed'), { code: 'ELOOP' }) + } + }) + + await expect( + service.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'changed\n', + operationId: 'anchored-baseline-read' + }) + ).rejects.toMatchObject({ code: 'CONTENT_INTEGRITY_FAILED' }) + expect(readAttempts).toBe(1) + }) + + it('uses anchored metadata to reject a large version without reading its body', async () => { + const fixture = await createFixture('upload') + const version = await client.uploadVersion.findUniqueOrThrow({ + where: { id: fixture.versionIds[1] } + }) + const bytes = Buffer.alloc(MANAGED_TEXT_EDIT_MAX_BYTES + 1, 0x61) + await writeFile(join(storageRoot, ...version.contentStorageKey.split('/')), bytes) + await client.uploadVersion.update({ + where: { id: version.id }, + data: { sizeBytes: BigInt(bytes.byteLength), checksum: checksum(bytes) } + }) + let bodyReads = 0 + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + readAnchored: () => { + bodyReads += 1 + throw new Error('large body must not be read') + } + }) + + await expect( + service.inspect({ source: 'upload', projectId: 'project-1', fileId: fixture.fileId }) + ).resolves.toMatchObject({ + canEdit: false, + canDiff: false, + unavailableReason: 'EDIT_LIMIT_EXCEEDED' + }) + expect(bodyReads).toBe(0) + }) + + it('maps an atomic bounded-read overflow to EDIT_LIMIT_EXCEEDED', async () => { + const fixture = await createFixture('upload') + const service = new ManagedFileVersionService({ + storageRoot, + getClient: () => Promise.resolve(client), + readAnchoredBounded: () => { + throw Object.assign(new Error('bounded read overflow'), { code: 'EFBIG' }) + } + }) + + await expect( + service.inspect({ source: 'upload', projectId: 'project-1', fileId: fixture.fileId }) + ).resolves.toMatchObject({ + canEdit: false, + canDiff: false, + unavailableReason: 'EDIT_LIMIT_EXCEEDED' + }) + }) + + it('hides durable but not managed-visible agent Artifact versions from list and exact inspect', async () => { + const fixture = await createFixture('artifact') + const bytes = Buffer.from('not activated\n') + const storageKey = + 'artifacts/project-1/session-1/artifact-file-1/versions/artifact-hidden-v3/content' + const contentPath = join(storageRoot, ...storageKey.split('/')) + await mkdir(dirname(contentPath), { recursive: true }) + await writeFile(contentPath, bytes) + await client.artifactVersion.create({ + data: { + id: 'artifact-hidden-v3', + artifactId: fixture.fileId, + versionNumber: 3, + filename: 'README.md', + originKind: 'agent_generated', + artifactRunId: 'compatibility-failed-run', + rootFrameId: 'root-1', + agentFrameId: 'agent-1', + messageBranchId: 'branch-1', + runtimeSegmentId: 'runtime-1', + promptMessageId: 'prompt-1', + state: 'finalized', + managedVisibleAt: null, + contentStorageKey: storageKey, + evidenceStorageKey: `${storageKey}.evidence`, + contentType: 'text/markdown', + sizeBytes: BigInt(bytes.byteLength), + checksum: checksum(bytes), + evidenceJson: '{}', + evidenceChecksum: checksum(Buffer.from('{}')), + evidenceSchemaVersion: 1 + } + }) + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }) + + await expect( + service.inspect({ source: 'artifact', projectId: 'project-1', fileId: fixture.fileId }) + ).resolves.toMatchObject({ + versions: [ + expect.objectContaining({ id: fixture.versionIds[0] }), + expect.objectContaining({ id: fixture.versionIds[1] }) + ] + }) + await expect( + service.inspect({ + source: 'artifact', + projectId: 'project-1', + fileId: fixture.fileId, + versionId: 'artifact-hidden-v3' + }) + ).rejects.toMatchObject({ code: 'VERSION_NOT_FOUND' }) + await expect( + service.saveTextEdit({ + source: 'artifact', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: 'artifact-hidden-v3', + expectedHeadVersionId: fixture.versionIds[1], + content: 'must not derive from a hidden version\n', + operationId: 'hidden-baseline-edit' + }) + ).rejects.toMatchObject({ code: 'VERSION_NOT_FOUND' }) + expect( + await client.managedFileVersionWriteOperation.count({ + where: { operationId: 'hidden-baseline-edit' } + }) + ).toBe(0) + + await client.artifactVersion.update({ + where: { id: 'artifact-hidden-v3' }, + data: { managedVisibleAt: new Date('2026-08-13T00:00:00.000Z') } + }) + await expect( + service.inspect({ + source: 'artifact', + projectId: 'project-1', + fileId: fixture.fileId, + versionId: 'artifact-hidden-v3' + }) + ).resolves.toMatchObject({ selectedVersionId: 'artifact-hidden-v3' }) + }) + + it('rechecks an Agent edit baseline visibility inside the publication transaction', async () => { + const fixture = await createFixture('artifact') + await client.artifactVersion.update({ + where: { id: fixture.versionIds[1] }, + data: { + originKind: 'agent_generated', + artifactRunId: 'visible-run', + rootFrameId: 'root-1', + agentFrameId: 'agent-1', + messageBranchId: 'branch-1', + runtimeSegmentId: 'runtime-1', + promptMessageId: 'prompt-1', + managedVisibleAt: new Date('2026-08-13T00:00:00.000Z'), + evidenceStorageKey: 'artifacts/project-1/session-1/evidence/v2.json', + evidenceJson: '{}', + evidenceChecksum: checksum(Buffer.from('{}')), + evidenceSchemaVersion: 1 + } + }) + let hidBaseline = false + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => 'artifact-racing-v3', + createStorageTag: () => 'vrace0001', + durability: { + syncFile: () => Promise.resolve(), + syncDirectory: async () => { + if (hidBaseline) return + hidBaseline = true + await client.artifactVersion.update({ + where: { id: fixture.versionIds[1] }, + data: { managedVisibleAt: null } + }) + } + } + }) + + await expect( + service.saveTextEdit({ + source: 'artifact', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'must not publish after the base becomes hidden\n', + operationId: 'visibility-race-operation' + }) + ).rejects.toMatchObject({ code: 'VERSION_NOT_FOUND' }) + await expect( + client.artifactLineage.findUniqueOrThrow({ where: { id: fixture.fileId } }) + ).resolves.toMatchObject({ currentVersionId: fixture.versionIds[1] }) + expect(await client.artifactVersion.count({ where: { id: 'artifact-racing-v3' } })).toBe(0) + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: 'visibility-race-operation' } + }) + ).resolves.toMatchObject({ state: 'file_ready', resultVersionId: null }) + }) + + it.each(['artifact', 'upload'] as const)( + 'saves a %s historical edit as the next immutable head and synchronizes the Files projection', + async (source) => { + const fixture = await createFixture(source) + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => `${source}-v3`, + createStorageTag: () => 'va1b2c3d4' + }) + + await client.managedFile.update({ + where: { + projectId_source_sourceFileId: { + projectId: 'project-1', + source, + sourceFileId: fixture.fileId + } + }, + data: { messageId: 'message-before-edit' } + }) + + const result = await service.saveTextEdit({ + source, + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[0], + expectedHeadVersionId: fixture.versionIds[1], + content: 'changed\nfrom history\n', + operationId: `${source}-operation-1` + }) + + expect(result).toMatchObject({ + kind: 'created', + headVersionId: `${source}-v3`, + version: { + id: `${source}-v3`, + versionNumber: 3, + basedOnVersionId: fixture.versionIds[0], + originKind: 'user_edit', + displayName: 'README.md' + } + }) + const resolved = await service.resolve({ + source, + projectId: 'project-1', + fileId: fixture.fileId + }) + expect(resolved.version.id).toBe(`${source}-v3`) + expect(resolved.version.storedFilename).toBe('va1b2c3d4_README.md') + expect(await readFile(resolved.path)).toEqual( + Buffer.from('\ufeffchanged\r\nfrom history\r\n') + ) + await expect(stat(resolved.path)).resolves.toMatchObject({ size: 26 }) + + const projection = await client.managedFile.findUniqueOrThrow({ + where: { + projectId_source_sourceFileId: { + projectId: 'project-1', + source, + sourceFileId: fixture.fileId + } + } + }) + expect(projection).toMatchObject({ + sourceVersionId: `${source}-v3`, + storageKey: resolved.version.contentStorageKey, + checksum: resolved.version.checksum, + displayName: 'README.md', + deletedAt: null, + messageId: null + }) + } + ) + + it.each(['artifact', 'upload'] as const)( + 'returns a no-op for unchanged %s bytes without creating a journal, file, or version', + async (source) => { + const fixture = await createFixture(source) + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }) + const before = + source === 'artifact' + ? await client.artifactVersion.count() + : await client.uploadVersion.count() + + const result = await service.saveTextEdit({ + source, + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[0], + expectedHeadVersionId: fixture.versionIds[1], + content: 'first\nline\n', + operationId: `${source}-noop-operation` + }) + + expect(result).toMatchObject({ kind: 'noop', headVersionId: fixture.versionIds[1] }) + expect(await client.managedFileVersionWriteOperation.count()).toBe(0) + expect( + source === 'artifact' + ? await client.artifactVersion.count() + : await client.uploadVersion.count() + ).toBe(before) + } + ) + + it.each([ + ['CONTAINS_NUL', 'unsafe\0content'], + ['EDIT_LIMIT_EXCEEDED', 'x'.repeat(MANAGED_TEXT_EDIT_MAX_BYTES + 1)] + ] as const)( + 'rejects normalized save bytes with %s before creating a journal', + async (code, content) => { + const fixture = await createFixture('upload') + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }) + + await expect( + service.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content, + operationId: `invalid-output-${code}` + }) + ).rejects.toMatchObject({ code }) + expect(await client.managedFileVersionWriteOperation.count()).toBe(0) + expect(await client.uploadVersion.count({ where: { uploadFileId: fixture.fileId } })).toBe(2) + } + ) + + it('rejects an oversized edit at the service boundary before opening the database', async () => { + const getClient = vi.fn().mockRejectedValue(new Error('database must not be opened')) + const service = new ManagedFileVersionService({ + storageRoot, + getClient, + nativeWriteAvailable: true + }) + + await expect( + service.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: 'upload-file-1', + basedOnVersionId: 'upload-v1', + expectedHeadVersionId: 'upload-v1', + content: 'x'.repeat(MANAGED_TEXT_EDIT_MAX_BYTES + 1), + operationId: 'oversized-before-database' + }) + ).rejects.toMatchObject({ code: 'EDIT_LIMIT_EXCEEDED' }) + expect(getClient).not.toHaveBeenCalled() + }) + + it('allows only one of two concurrent saves against the same head to publish', async () => { + const fixture = await createFixture('upload') + let id = 2 + let tag = 0 + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => `upload-v${++id}`, + createStorageTag: () => `v0000000${++tag}` + }) + const base = { + source: 'upload' as const, + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1] + } + + const results = await Promise.all([ + service.saveTextEdit({ ...base, content: 'left\n', operationId: 'operation-left' }), + service.saveTextEdit({ ...base, content: 'right\n', operationId: 'operation-right' }) + ]) + + expect(results.map((result) => result.kind).sort()).toEqual(['conflict', 'created']) + expect(await client.uploadVersion.count()).toBe(3) + expect( + await client.managedFileVersionWriteOperation.count({ where: { state: 'conflict' } }) + ).toBe(1) + }) + + it('retries a colliding physical storage tag without clobbering existing bytes', async () => { + const fixture = await createFixture('artifact') + const collidingKey = `artifacts/project-1/session-1/${fixture.fileId}/managed-versions/vaaaaaaaa_README.md` + const collidingPath = join(storageRoot, ...collidingKey.split('/')) + await mkdir(dirname(collidingPath), { recursive: true }) + await writeFile(collidingPath, 'do not replace') + const tags = ['vaaaaaaaa', 'vbbbbbbbb'] + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => 'artifact-v3', + createStorageTag: () => tags.shift()! + }) + + const result = await service.saveTextEdit({ + source: 'artifact', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'new bytes\n', + operationId: 'artifact-collision-operation' + }) + + expect(result).toMatchObject({ kind: 'created' }) + await expect( + service.resolve({ + source: 'artifact', + projectId: 'project-1', + fileId: fixture.fileId + }) + ).resolves.toMatchObject({ version: { storedFilename: 'vbbbbbbbb_README.md' } }) + await expect(readFile(collidingPath, 'utf8')).resolves.toBe('do not replace') + }) + + it('reallocates the journal destination when a no-clobber publication loses a filesystem race', async () => { + const fixture = await createFixture('upload') + const tags = ['vrace0001', 'vrace0002'] + let publicationAttempts = 0 + const service = new ManagedFileVersionService({ + storageRoot, + getClient: () => Promise.resolve(client), + createId: () => 'upload-v3', + createStorageTag: () => tags.shift()!, + writeAndPublish: (rootPath, parentPath, temporaryName, destinationName, bytes) => { + publicationAttempts += 1 + if (publicationAttempts === 1) { + throw Object.assign(new Error('simulated no-replace race'), { code: 'EEXIST' }) + } + testWriteAndPublish(rootPath, parentPath, temporaryName, destinationName, bytes) + } + }) + + await expect( + service.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'race-safe\n', + operationId: 'race-operation' + }) + ).resolves.toMatchObject({ kind: 'created' }) + expect(publicationAttempts).toBe(2) + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: 'race-operation' } + }) + ).resolves.toMatchObject({ + state: 'published', + storageTag: 'vrace0002', + storedFilename: 'vrace0002_README.md' + }) + }) + + it('does not delete an existing destination when every no-clobber publication collides', async () => { + const fixture = await createFixture('upload') + const tags = Array.from({ length: 16 }, (_, index) => `vcoll${String(index).padStart(4, '0')}`) + const collidingPaths: string[] = [] + const service = new ManagedFileVersionService({ + storageRoot, + getClient: () => Promise.resolve(client), + createStorageTag: () => tags.shift()!, + writeAndPublish: (_rootPath, parentPath, _temporaryName, destinationName) => { + mkdirSync(parentPath, { recursive: true }) + const destinationPath = join(parentPath, destinationName) + writeFileSync(destinationPath, 'existing bytes') + collidingPaths.push(destinationPath) + throw Object.assign(new Error('simulated no-replace collision'), { code: 'EEXIST' }) + } + }) + + await expect( + service.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'never published\n', + operationId: 'exhausted-collision-operation' + }) + ).rejects.toMatchObject({ code: 'STORAGE_COLLISION' }) + + expect(collidingPaths).toHaveLength(16) + for (const collidingPath of collidingPaths) { + await expect(readFile(collidingPath, 'utf8')).resolves.toBe('existing bytes') + } + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: 'exhausted-collision-operation' } + }) + ).resolves.toMatchObject({ state: 'failed', errorCode: 'STORAGE_COLLISION' }) + }) + + it('never writes temporary or final bytes outside the storage root through a symlinked ancestor', async () => { + const fixture = await createFixture('upload') + outsideRoot = await mkdtemp(join(tmpdir(), 'open-science-managed-version-outside-')) + const managedVersionsPath = join( + storageRoot, + 'uploads', + 'project-1', + 'session-1', + fixture.fileId, + 'managed-versions' + ) + await symlink( + outsideRoot, + managedVersionsPath, + process.platform === 'win32' ? 'junction' : 'dir' + ) + const service = new ManagedFileVersionService({ + storageRoot, + getClient: () => Promise.resolve(client), + createStorageTag: () => 'vsymlink1', + writeAndPublish: () => { + throw Object.assign(new Error('anchored publisher rejected symlink'), { code: 'ELOOP' }) + } + }) + + await expect( + service.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'must stay in root\n', + operationId: 'symlink-escape-operation' + }) + ).rejects.toMatchObject({ code: 'ELOOP' }) + + expect(await readdir(outsideRoot)).toEqual([]) + }) + + it('returns one published result and preserves its bytes for concurrent replay of one operation', async () => { + const fixture = await createFixture('upload') + let releaseFirstAfterPublish!: () => void + let signalFirstAfterPublish!: () => void + const firstAfterPublish = new Promise((resolve) => { + signalFirstAfterPublish = resolve + }) + const firstMayContinue = new Promise((resolve) => { + releaseFirstAfterPublish = resolve + }) + let directorySyncCount = 0 + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => 'upload-v3', + createStorageTag: () => 'vreplay01', + durability: { + syncFile: () => Promise.resolve(), + syncDirectory: async () => { + directorySyncCount += 1 + if (directorySyncCount === 1) { + signalFirstAfterPublish() + await firstMayContinue + } + } + } + }) + const request = { + source: 'upload' as const, + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'one durable publication\n', + operationId: 'same-operation' + } + + const first = service.saveTextEdit(request) + await firstAfterPublish + const secondResult = await service.saveTextEdit(request) + releaseFirstAfterPublish() + const firstResult = await first + + expect([firstResult, secondResult]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: 'created', headVersionId: 'upload-v3', replayed: false }), + expect.objectContaining({ kind: 'created', headVersionId: 'upload-v3', replayed: true }) + ]) + ) + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: request.operationId } + }) + ).resolves.toMatchObject({ state: 'published', resultVersionId: 'upload-v3' }) + const resolved = await service.resolve({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId + }) + await expect(readFile(resolved.path, 'utf8')).resolves.toBe('one durable publication\n') + }) + + it('replays the original published result after a later head and rejects corrupt result bytes', async () => { + const fixture = await createFixture('upload') + const tags = ['vreplay02', 'vreplay03'] + const ids = ['upload-v3', 'upload-v4'] + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => ids.shift()!, + createStorageTag: () => tags.shift()! + }) + const firstRequest = { + source: 'upload' as const, + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'published result\n', + operationId: 'published-operation' + } + await expect(service.saveTextEdit(firstRequest)).resolves.toMatchObject({ + kind: 'created', + headVersionId: 'upload-v3', + replayed: false + }) + await service.saveTextEdit({ + ...firstRequest, + basedOnVersionId: 'upload-v3', + expectedHeadVersionId: 'upload-v3', + content: 'later head\n', + operationId: 'later-operation' + }) + + const publishedVersion = await client.uploadVersion.findUniqueOrThrow({ + where: { id: 'upload-v3' } + }) + const replayWithoutBaseline = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + readAnchored: (_rootPath, parentPath, name) => { + if (parentPath.endsWith('/versions/upload-v2')) { + throw new Error('published replay must not read the baseline') + } + return Buffer.from(readFileSync(join(parentPath, name))) + } + }) + expect(publishedVersion.storedFilename).not.toBe('content') + await expect(replayWithoutBaseline.saveTextEdit(firstRequest)).resolves.toMatchObject({ + kind: 'created', + headVersionId: 'upload-v3', + version: { id: 'upload-v3' }, + replayed: true + }) + const original = await service.resolve({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + versionId: 'upload-v3' + }) + await writeFile(original.path, 'corrupt') + await expect(service.saveTextEdit(firstRequest)).rejects.toMatchObject({ + code: 'CONTENT_INTEGRITY_FAILED' + }) + }) + + it('rejects a published journal whose result Version was not created by that operation', async () => { + const fixture = await createFixture('upload') + const version = await client.uploadVersion.findUniqueOrThrow({ + where: { id: fixture.versionIds[1] } + }) + const forgedBytes = Buffer.from('forged\n') + await client.managedFileVersionWriteOperation.create({ + data: { + operationId: 'forged-published-operation', + source: 'upload', + projectId: 'project-1', + sourceFileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + state: 'published', + storageTag: 'vforged1', + storedFilename: 'vforged1_README.md', + contentStorageKey: + 'uploads/project-1/session-1/upload-file-1/managed-versions/vforged1_README.md', + checksum: checksum(forgedBytes), + sizeBytes: BigInt(forgedBytes.byteLength), + textFormatJson: JSON.stringify({ + hasUtf8Bom: false, + newline: 'lf', + hasTrailingNewline: true + }), + resultVersionId: version.id + } + }) + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }) + + await expect( + service.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'forged\n', + operationId: 'forged-published-operation' + }) + ).rejects.toMatchObject({ code: 'CONTENT_INTEGRITY_FAILED' }) + }) + + it('recovers an intact published file after a crash before file_ready and publishes once', async () => { + const fixture = await createFixture('upload') + const crashing = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => 'upload-v3', + createStorageTag: () => 'vcrash001', + testFaultAt: 'after-file-publish' + }) + await expect( + crashing.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'survives crash\n', + operationId: 'recover-operation' + }) + ).rejects.toThrow('simulated managed version crash') + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: 'recover-operation' } + }) + ).resolves.toMatchObject({ state: 'staging' }) + + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => 'upload-v3' + }) + const recovery = await service.recoverPendingWrites() + expect(recovery).toEqual({ recovered: 1, conflicted: 0, failed: 0, integrityErrors: [] }) + expect(await client.uploadVersion.count()).toBe(3) + await expect( + client.uploadFile.findUniqueOrThrow({ where: { id: fixture.fileId } }) + ).resolves.toMatchObject({ currentVersionId: 'upload-v3' }) + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: 'recover-operation' } + }) + ).resolves.toMatchObject({ state: 'published', resultVersionId: 'upload-v3' }) + }) + + it('publishes an intact deterministic temp left before rename instead of failing recovery', async () => { + const fixture = await createFixture('upload') + const crashing = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createStorageTag: () => 'vtmprec01', + testFaultAt: 'after-journal' + }) + await expect( + crashing.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'recover deterministic temp\n', + operationId: 'temp-before-rename-operation' + }) + ).rejects.toThrow() + const operation = await client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: 'temp-before-rename-operation' } + }) + const parentPath = dirname(join(storageRoot, ...operation.contentStorageKey.split('/'))) + const operationDigest = createHash('sha256') + .update('temp-before-rename-operation') + .digest('hex') + .slice(0, 16) + const tempName = `.${operation.storedFilename}.${operationDigest}.tmp` + await mkdir(parentPath, { recursive: true }) + const relativeParentPath = relative(storageRoot, parentPath) + const child = spawn( + process.execPath, + [ + '-e', + ` + const binding = require(process.argv[1]) + binding.writeAndPublishNoReplace( + process.argv[2], + process.argv[3], + process.argv[4], + process.argv[5], + Buffer.from('recover deterministic temp\\n') + ) + `, + join(process.cwd(), 'packages/safe-file-publisher-native'), + storageRoot, + relativeParentPath, + tempName, + operation.storedFilename + ], + { + env: { + ...process.env, + NODE_ENV: 'test', + VITEST: 'true', + OPEN_SCIENCE_NATIVE_TEST_HOOKS: '1', + OPEN_SCIENCE_TEST_EXIT_AFTER_DURABLE_TEMP: '86' + }, + stdio: 'ignore' + } + ) + const childExit = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolveExit, rejectExit) => { + const timeout = setTimeout(() => { + child.kill('SIGKILL') + rejectExit(new Error('durable-temp child timed out')) + }, 5_000) + child.once('exit', (code, signal) => { + clearTimeout(timeout) + resolveExit({ code, signal }) + }) + child.once('error', rejectExit) + } + ) + expect(childExit).toEqual({ code: 86, signal: null }) + await expect(readFile(join(parentPath, tempName), 'utf8')).resolves.toBe( + 'recover deterministic temp\n' + ) + await expect(readFile(join(parentPath, operation.storedFilename))).rejects.toMatchObject({ + code: 'ENOENT' + }) + + const recovered = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => 'upload-temp-recovered-v3' + }) + await expect(recovered.recoverPendingWrites()).resolves.toMatchObject({ + recovered: 1, + failed: 0 + }) + }) + + it('keeps a transient recovery read failure pending for the next startup retry', async () => { + const fixture = await createFixture('upload') + const crashing = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createStorageTag: () => 'vtrans001', + testFaultAt: 'after-file-publish' + }) + await expect( + crashing.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'retry me\n', + operationId: 'transient-recovery-operation' + }) + ).rejects.toThrow('simulated managed version crash') + + const retryable = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + readAnchored: () => { + throw Object.assign(new Error('temporary filesystem outage'), { code: 'EIO' }) + } + }) + await expect(retryable.recoverPendingWrites()).resolves.toMatchObject({ + recovered: 0, + failed: 0 + }) + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: 'transient-recovery-operation' } + }) + ).resolves.toMatchObject({ state: 'staging' }) + }) + + it.each(['after-temp-write', 'after-file-ready'] as const)( + 'idempotently recovers a save interrupted at %s', + async (testFaultAt) => { + const fixture = await createFixture('artifact') + const crashing = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => 'artifact-v3', + createStorageTag: () => `v${testFaultAt === 'after-temp-write' ? 'temp0001' : 'ready001'}`, + testFaultAt + }) + const request = { + source: 'artifact' as const, + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: `${testFaultAt}\n`, + operationId: `${testFaultAt}-operation` + } + await expect(crashing.saveTextEdit(request)).rejects.toThrow( + 'simulated managed version crash' + ) + + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => 'artifact-v3' + }) + await expect(service.recoverPendingWrites()).resolves.toMatchObject({ recovered: 1 }) + await expect(service.saveTextEdit(request)).resolves.toMatchObject({ + kind: 'created', + headVersionId: 'artifact-v3' + }) + expect(await client.artifactVersion.count()).toBe(3) + } + ) + + it.each([ + ['artifact', 'project-intent'], + ['upload', 'session-tombstone'] + ] as const)( + 'does not publish or revive a deleted %s after a %s appears at file_ready', + async (source, barrier) => { + const fixture = await createFixture(source) + const crashing = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => `${source}-v3`, + createStorageTag: () => 'vdelete01', + testFaultAt: 'after-file-ready' + }) + await expect( + crashing.saveTextEdit({ + source, + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'must not publish\n', + operationId: `${source}-deletion-operation` + }) + ).rejects.toThrow('simulated managed version crash') + + if (barrier === 'project-intent') { + await client.projectDeletionIntent.create({ data: { projectId: 'project-1' } }) + } else { + const deletedAt = new Date('2026-08-12T00:00:00.000Z') + await client.managedFileSessionSync.create({ + data: { + projectId: 'project-1', + sessionId: 'session-1', + filesRevision: 1, + groupSortAtMs: BigInt(1), + deletedAt, + deleteOperationId: 'delete-session-1' + } + }) + await client.managedFile.update({ + where: { + projectId_source_sourceFileId: { + projectId: 'project-1', + source, + sourceFileId: fixture.fileId + } + }, + data: { deletedAt, deleteOperationId: 'delete-session-1' } + }) + } + + const recovery = await new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }).recoverPendingWrites() + + expect(recovery).toMatchObject({ recovered: 0, failed: 1 }) + expect( + source === 'artifact' + ? await client.artifactVersion.count({ where: { artifactId: fixture.fileId } }) + : await client.uploadVersion.count({ where: { uploadFileId: fixture.fileId } }) + ).toBe(2) + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: `${source}-deletion-operation` } + }) + ).resolves.toMatchObject({ state: 'failed' }) + if (barrier === 'session-tombstone') { + await expect( + client.managedFileSessionSync.findUniqueOrThrow({ + where: { + projectId_sessionId: { projectId: 'project-1', sessionId: 'session-1' } + } + }) + ).resolves.toMatchObject({ + deletedAt: new Date('2026-08-12T00:00:00.000Z'), + deleteOperationId: 'delete-session-1' + }) + await expect( + client.managedFile.findFirstOrThrow({ where: { source, sourceFileId: fixture.fileId } }) + ).resolves.toMatchObject({ deleteOperationId: 'delete-session-1' }) + } + } + ) + + it('rejects a pre-existing Session tombstone before creating a journal or publishing bytes', async () => { + const fixture = await createFixture('upload') + await client.managedFileSessionSync.create({ + data: { + projectId: 'project-1', + sessionId: 'session-1', + filesRevision: 4, + groupSortAtMs: BigInt(1), + deletedAt: new Date('2026-08-12T00:00:00.000Z'), + deleteOperationId: 'delete-session-1' + } + }) + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createStorageTag: () => 'vblocked1' + }) + + await expect( + service.inspect({ source: 'upload', projectId: 'project-1', fileId: fixture.fileId }) + ).resolves.toMatchObject({ canEdit: false, unavailableReason: 'FILE_DELETED' }) + await expect( + service.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'must not stage\n', + operationId: 'preexisting-tombstone-operation' + }) + ).rejects.toMatchObject({ code: 'FILE_DELETED' }) + expect(await client.managedFileVersionWriteOperation.count()).toBe(0) + expect(await client.uploadVersion.count({ where: { uploadFileId: fixture.fileId } })).toBe(2) + }) + + it('does not rebuild an active projection inside a tombstoned session', async () => { + const fixture = await createFixture('upload') + await client.managedFile.deleteMany({ where: { sourceFileId: fixture.fileId } }) + await client.managedFileSessionSync.create({ + data: { + projectId: 'project-1', + sessionId: 'session-1', + filesRevision: 4, + groupSortAtMs: BigInt(1), + deletedAt: new Date('2026-08-12T00:00:00.000Z'), + deleteOperationId: 'delete-session-1' + } + }) + + await new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }).recoverPendingWrites() + + expect(await client.managedFile.count({ where: { sourceFileId: fixture.fileId } })).toBe(0) + }) + + it('does not expose a completed Artifact head before its Files projection becomes visible', async () => { + const fixture = await createFixture('artifact') + await client.managedFile.deleteMany({ where: { sourceFileId: fixture.fileId } }) + + await new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }).recoverPendingWrites() + + await expect( + client.artifactLineage.findUniqueOrThrow({ where: { id: fixture.fileId } }) + ).resolves.toMatchObject({ currentVersionId: fixture.versionIds[1] }) + expect(await client.managedFile.count({ where: { sourceFileId: fixture.fileId } })).toBe(0) + }) + + it('fails a journal-only interrupted save without allocating a visible version number', async () => { + const fixture = await createFixture('upload') + const crashing = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createStorageTag: () => 'vjournal1', + testFaultAt: 'after-journal' + }) + await expect( + crashing.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'journal only\n', + operationId: 'journal-only-operation' + }) + ).rejects.toThrow('simulated managed version crash') + + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }) + await expect(service.recoverPendingWrites()).resolves.toMatchObject({ failed: 1 }) + expect(await client.uploadVersion.count()).toBe(2) + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: 'journal-only-operation' } + }) + ).resolves.toMatchObject({ state: 'failed' }) + }) + + it('recovers pending and cleans terminal journals beyond the first page', async () => { + const fixture = await createFixture('upload') + const format = JSON.stringify({ + hasUtf8Bom: false, + newline: 'lf', + hasTrailingNewline: true + }) + await client.managedFileVersionWriteOperation.createMany({ + data: Array.from({ length: 101 }, (_, index) => { + const suffix = index.toString().padStart(3, '0') + return { + operationId: `paged-pending-${suffix}`, + source: 'upload', + projectId: 'project-1', + sourceFileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + state: 'staging', + storageTag: `vp${suffix}x001`, + storedFilename: `vp${suffix}x001_README.md`, + contentStorageKey: `uploads/project-1/session-1/${fixture.fileId}/managed-versions/vp${suffix}x001_README.md`, + checksum: checksum(Buffer.from(`missing ${suffix}\n`)), + sizeBytes: BigInt(Buffer.byteLength(`missing ${suffix}\n`)), + textFormatJson: format + } + }) + }) + const terminalPath = join( + storageRoot, + 'uploads/project-1/session-1', + fixture.fileId, + 'managed-versions/vterminal_README.md' + ) + await mkdir(dirname(terminalPath), { recursive: true }) + await writeFile(terminalPath, 'terminal cleanup\n') + await client.managedFileVersionWriteOperation.create({ + data: { + operationId: 'zz-paged-terminal', + source: 'upload', + projectId: 'project-1', + sourceFileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + state: 'failed', + storageTag: 'vterminal', + storedFilename: 'vterminal_README.md', + contentStorageKey: `uploads/project-1/session-1/${fixture.fileId}/managed-versions/vterminal_README.md`, + checksum: checksum(Buffer.from('terminal cleanup\n')), + sizeBytes: BigInt(Buffer.byteLength('terminal cleanup\n')), + textFormatJson: format, + errorCode: 'CONTENT_INTEGRITY_FAILED' + } + }) + + const recovery = await new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }).recoverPendingWrites() + + expect(recovery).toMatchObject({ failed: 101 }) + expect( + await client.managedFileVersionWriteOperation.count({ where: { state: 'failed' } }) + ).toBe(102) + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: 'paged-pending-100' } + }) + ).resolves.toMatchObject({ state: 'failed' }) + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: 'zz-paged-terminal' } + }) + ).resolves.toMatchObject({ state: 'failed' }) + await expect(readFile(terminalPath)).rejects.toMatchObject({ code: 'ENOENT' }) + }) + + it('marks corrupt staged publication bytes failed and never advances the head', async () => { + const fixture = await createFixture('artifact') + const crashing = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + createId: () => 'artifact-v3', + createStorageTag: () => 'vcrash002', + testFaultAt: 'after-file-publish' + }) + await expect( + crashing.saveTextEdit({ + source: 'artifact', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'will corrupt\n', + operationId: 'corrupt-operation' + }) + ).rejects.toThrow() + const operation = await client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: 'corrupt-operation' } + }) + await writeFile(join(storageRoot, ...operation.contentStorageKey.split('/')), 'corrupt') + + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }) + const recovery = await service.recoverPendingWrites() + expect(recovery).toMatchObject({ recovered: 0, conflicted: 0, failed: 1 }) + await expect( + client.artifactLineage.findUniqueOrThrow({ where: { id: fixture.fileId } }) + ).resolves.toMatchObject({ currentVersionId: fixture.versionIds[1] }) + expect(await client.artifactVersion.count()).toBe(2) + }) + + it('never cleans a conflict path that is already owned by a ready Version', async () => { + const fixture = await createFixture('upload') + const owned = await client.uploadVersion.findUniqueOrThrow({ + where: { id: fixture.versionIds[1] } + }) + await client.managedFileVersionWriteOperation.create({ + data: { + operationId: 'owned-conflict-operation', + source: 'upload', + projectId: 'project-1', + sourceFileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[0], + expectedHeadVersionId: fixture.versionIds[0], + state: 'file_ready', + storageTag: 'vowned001', + storedFilename: 'content', + contentStorageKey: owned.contentStorageKey, + checksum: owned.checksum, + sizeBytes: owned.sizeBytes, + textFormatJson: JSON.stringify({ + hasUtf8Bom: false, + newline: 'lf', + hasTrailingNewline: true + }) + } + }) + const ownedPath = join(storageRoot, ...owned.contentStorageKey.split('/')) + + const recovery = await new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }).recoverPendingWrites() + + expect(recovery).toMatchObject({ conflicted: 1 }) + await expect(readFile(ownedPath, 'utf8')).resolves.toBe('second\n') + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: 'owned-conflict-operation' } + }) + ).resolves.toMatchObject({ state: 'conflict' }) + }) + + it.each(['conflict', 'failed'] as const)( + 'retries cleanup of an unowned %s final without changing terminal journal state', + async (state) => { + const fixture = await createFixture('upload') + const contentStorageKey = `uploads/project-1/session-1/${fixture.fileId}/managed-versions/vcleanup1_README.md` + const finalPath = join(storageRoot, ...contentStorageKey.split('/')) + await mkdir(dirname(finalPath), { recursive: true }) + await writeFile(finalPath, 'orphan final\n') + await client.managedFileVersionWriteOperation.create({ + data: { + operationId: `${state}-cleanup-operation`, + source: 'upload', + projectId: 'project-1', + sourceFileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + state, + storageTag: 'vcleanup1', + storedFilename: 'vcleanup1_README.md', + contentStorageKey, + checksum: checksum(Buffer.from('orphan final\n')), + sizeBytes: BigInt(Buffer.byteLength('orphan final\n')), + textFormatJson: JSON.stringify({ + hasUtf8Bom: false, + newline: 'lf', + hasTrailingNewline: true + }), + errorCode: state === 'conflict' ? 'HEAD_CHANGED' : 'CONTENT_INTEGRITY_FAILED' + } + }) + + await new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }).recoverPendingWrites() + + await expect(readFile(finalPath)).rejects.toMatchObject({ code: 'ENOENT' }) + await expect( + client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: `${state}-cleanup-operation` } + }) + ).resolves.toMatchObject({ state }) + } + ) + + it('does not clean unowned terminal paths whose bytes do not match the journal', async () => { + const fixture = await createFixture('upload') + const contentStorageKey = `uploads/project-1/session-1/${fixture.fileId}/managed-versions/vforeign1_README.md` + const finalPath = join(storageRoot, ...contentStorageKey.split('/')) + await mkdir(dirname(finalPath), { recursive: true }) + await writeFile(finalPath, 'foreign bytes\n') + await client.managedFileVersionWriteOperation.create({ + data: { + operationId: 'foreign-cleanup-operation', + source: 'upload', + projectId: 'project-1', + sourceFileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + state: 'failed', + storageTag: 'vforeign1', + storedFilename: 'vforeign1_README.md', + contentStorageKey, + checksum: checksum(Buffer.from('journal bytes\n')), + sizeBytes: BigInt(Buffer.byteLength('journal bytes\n')), + textFormatJson: JSON.stringify({ + hasUtf8Bom: false, + newline: 'lf', + hasTrailingNewline: true + }), + errorCode: 'CONTENT_INTEGRITY_FAILED' + } + }) + + await new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }).recoverPendingWrites() + + await expect(readFile(finalPath, 'utf8')).resolves.toBe('foreign bytes\n') + }) + + it('removes only stale managed-version temporary files that have no journal', async () => { + const fixture = await createFixture('artifact') + const parentPath = join( + storageRoot, + 'artifacts', + 'project-1', + 'session-1', + fixture.fileId, + 'managed-versions' + ) + await mkdir(parentPath, { recursive: true }) + const staleName = '.vorphan01_README.md.0123456789abcdef.tmp' + const freshName = '.vfresh001_README.md.0123456789abcdef.tmp' + await writeFile(join(parentPath, staleName), 'stale') + await writeFile(join(parentPath, freshName), 'fresh') + await utimes(join(parentPath, staleName), new Date(0), new Date(0)) + + await new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + now: () => new Date('2026-08-12T00:00:00.000Z') + }).recoverPendingWrites() + + await expect(readFile(join(parentPath, staleName))).rejects.toMatchObject({ code: 'ENOENT' }) + await expect(readFile(join(parentPath, freshName), 'utf8')).resolves.toBe('fresh') + }) + + it('paginates file roots while preserving a stale temp with exact journal ownership', async () => { + const fixture = await createFixture('upload') + const fileRows = Array.from({ length: 101 }, (_, index) => { + const suffix = index.toString().padStart(3, '0') + return { + id: `paged-upload-${suffix}`, + projectId: 'project-1', + sessionId: 'session-1', + filename: `${suffix}.txt`, + originalFilename: `${suffix}.txt` + } + }) + await client.uploadFile.createMany({ data: fileRows }) + await client.uploadVersion.createMany({ + data: fileRows.map((file, index) => ({ + id: `${file.id}-v1`, + uploadFileId: file.id, + versionNumber: 1, + state: 'ready', + originKind: 'legacy', + contentStorageKey: `uploads/project-1/session-1/${file.id}/content`, + filename: file.filename, + originalFilename: file.originalFilename, + contentType: 'text/plain', + sizeBytes: BigInt(1), + checksum: checksum(Buffer.from('x')), + createdAt: new Date(1_000 + index) + })) + }) + for (const file of fileRows) { + await client.uploadFile.update({ + where: { id: file.id }, + data: { currentVersionId: `${file.id}-v1` } + }) + } + await client.managedFile.createMany({ + data: fileRows.map((file, index) => ({ + source: 'upload', + sourceFileId: file.id, + sourceVersionId: fixture.versionIds[0], + checksum: 'stale', + projectId: 'project-1', + sessionId: 'session-1', + displayName: file.originalFilename, + storageKey: `stale/${file.id}`, + sizeBytes: BigInt(0), + sortAtMs: BigInt(index) + })) + }) + const last = fileRows.at(-1)! + const storedFilename = 'vprotect1_100.txt' + const operationId = 'protected-temp-operation' + const digest = createHash('sha256').update(operationId).digest('hex').slice(0, 16) + const protectedTemp = `.${storedFilename}.${digest}.tmp` + const orphanTemp = '.vorphan02_100.txt.0123456789abcdef.tmp' + const lastParent = join( + storageRoot, + 'uploads', + 'project-1', + 'session-1', + last.id, + 'managed-versions' + ) + await mkdir(lastParent, { recursive: true }) + await writeFile(join(lastParent, protectedTemp), 'protected') + await writeFile(join(lastParent, orphanTemp), 'orphan') + await utimes(join(lastParent, protectedTemp), new Date(0), new Date(0)) + await utimes(join(lastParent, orphanTemp), new Date(0), new Date(0)) + await client.managedFileVersionWriteOperation.create({ + data: { + operationId, + source: 'upload', + projectId: 'project-1', + sourceFileId: last.id, + basedOnVersionId: `${last.id}-v1`, + expectedHeadVersionId: `${last.id}-v1`, + state: 'published', + storageTag: 'vprotect1', + storedFilename, + contentStorageKey: `uploads/project-1/session-1/${last.id}/managed-versions/${storedFilename}`, + checksum: checksum(Buffer.from('protected')), + sizeBytes: BigInt(Buffer.byteLength('protected')), + textFormatJson: JSON.stringify({ + hasUtf8Bom: false, + newline: 'lf', + hasTrailingNewline: false + }), + resultVersionId: `${last.id}-v1` + } + }) + + const transactionSpy = vi.spyOn(client, '$transaction') + await new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + now: () => new Date('2026-08-13T00:00:00.000Z') + }).recoverPendingWrites() + + await expect(readFile(join(lastParent, protectedTemp), 'utf8')).resolves.toBe('protected') + await expect(readFile(join(lastParent, orphanTemp))).rejects.toMatchObject({ code: 'ENOENT' }) + await expect( + client.managedFile.findUniqueOrThrow({ + where: { + projectId_source_sourceFileId: { + projectId: 'project-1', + source: 'upload', + sourceFileId: last.id + } + } + }) + ).resolves.toMatchObject({ sourceVersionId: `${last.id}-v1` }) + expect(transactionSpy).toHaveBeenCalledTimes(fileRows.length + 1) + transactionSpy.mockRestore() + }) + + it('fails closed when stale temporary-file recovery encounters a replaced symlink parent', async () => { + const fixture = await createFixture('artifact') + outsideRoot = await mkdtemp(join(tmpdir(), 'open-science-managed-version-outside-')) + const fileRoot = join(storageRoot, 'artifacts', 'project-1', 'session-1', fixture.fileId) + const originalRoot = `${fileRoot}-original` + await mkdir(join(fileRoot, 'managed-versions'), { recursive: true }) + await rename(fileRoot, originalRoot) + await symlink(outsideRoot, fileRoot) + await writeFile(join(outsideRoot, '.vorphan01_README.md.0123456789abcdef.tmp'), 'outside') + + await expect( + new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + now: () => new Date('2026-08-12T00:00:00.000Z') + }).recoverPendingWrites() + ).rejects.toMatchObject({ code: 'ELOOP' }) + await expect( + readFile(join(outsideRoot, '.vorphan01_README.md.0123456789abcdef.tmp'), 'utf8') + ).resolves.toBe('outside') + }) + + it('rejects archived projects and corrupted completed head bytes with stable error codes', async () => { + const fixture = await createFixture('artifact') + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }) + await client.project.update({ + where: { id: 'project-1' }, + data: { archivedAt: new Date() } + }) + await expect( + service.inspect({ source: 'artifact', projectId: 'project-1', fileId: fixture.fileId }) + ).resolves.toMatchObject({ + canEdit: false, + canDiff: true, + unavailableReason: 'PROJECT_NOT_WRITABLE' + }) + await expect( + service.saveTextEdit({ + source: 'artifact', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'blocked\n', + operationId: 'blocked-operation' + }) + ).rejects.toEqual( + expect.objectContaining>({ + code: 'PROJECT_NOT_WRITABLE' + }) + ) + + await client.project.update({ where: { id: 'project-1' }, data: { archivedAt: null } }) + const resolved = await service.resolve({ + source: 'artifact', + projectId: 'project-1', + fileId: fixture.fileId + }) + await writeFile(resolved.path, 'corrupt') + await expect( + service.inspect({ source: 'artifact', projectId: 'project-1', fileId: fixture.fileId }) + ).rejects.toMatchObject({ code: 'CONTENT_INTEGRITY_FAILED' }) + }) + + it('reports an unsafe stable basename as ineligible instead of failing during save allocation', async () => { + const fixture = await createFixture('upload') + await client.uploadFile.update({ + where: { id: fixture.fileId }, + data: { filename: 'CON.md', originalFilename: 'CON.md' } + }) + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }) + + await expect( + service.inspect({ source: 'upload', projectId: 'project-1', fileId: fixture.fileId }) + ).resolves.toMatchObject({ canEdit: false, unavailableReason: 'UNSAFE_FILENAME' }) + }) + + it('keeps trusted reads available when anchored writes are unavailable', async () => { + const fixture = await createFixture('upload') + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + nativeWriteAvailable: false, + nativeReadFallbackAvailable: true, + readAnchored: () => { + throw Object.assign(new Error('anchored reads unavailable'), { code: 'ENOTSUP' }) + }, + verifyAnchored: () => { + throw Object.assign(new Error('anchored verification unavailable'), { code: 'ENOTSUP' }) + } + }) + + await expect( + service.inspect({ source: 'upload', projectId: 'project-1', fileId: fixture.fileId }) + ).resolves.toMatchObject({ + canEdit: false, + canDiff: true, + text: 'second\n', + unavailableReason: 'NATIVE_WRITE_REQUIRED' + }) + const lease = await service.openResolved({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId + }) + await expect(lease.readRange(0, lease.size)).resolves.toEqual( + new Uint8Array(Buffer.from('second\n')) + ) + await lease.close() + await expect( + service.diffText({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + versionId: fixture.versionIds[1], + requestId: 'native-read-fallback' + }) + ).resolves.toMatchObject({ + baseVersionId: fixture.versionIds[0], + selectedVersionId: fixture.versionIds[1] + }) + await expect(service.auditActiveVersionIntegrity()).resolves.toEqual([]) + const currentPath = await service.resolvePath({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId + }) + await expect(readFile(currentPath.path, 'utf8')).resolves.toBe('second\n') + await expect( + service.resolve({ source: 'upload', projectId: 'project-1', fileId: fixture.fileId }) + ).rejects.toMatchObject({ code: 'NATIVE_WRITE_REQUIRED' }) + await expect( + service.saveTextEdit({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + basedOnVersionId: fixture.versionIds[1], + expectedHeadVersionId: fixture.versionIds[1], + content: 'blocked native write\n', + operationId: 'native-write-unavailable' + }) + ).rejects.toMatchObject({ code: 'NATIVE_WRITE_REQUIRED' }) + }) + + it('fails closed when the native binding is unavailable rather than explicitly read-only', async () => { + const fixture = await createFixture('upload') + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client), + nativeWriteAvailable: false, + nativeReadFallbackAvailable: false + }) + + await expect( + service.inspect({ source: 'upload', projectId: 'project-1', fileId: fixture.fileId }) + ).resolves.toMatchObject({ + canEdit: false, + canDiff: false, + unavailableReason: 'NATIVE_WRITE_REQUIRED' + }) + await expect( + service.inspect({ source: 'upload', projectId: 'project-1', fileId: fixture.fileId }) + ).resolves.not.toHaveProperty('text') + await expect( + service.openResolved({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId + }) + ).rejects.toMatchObject({ code: 'NATIVE_WRITE_REQUIRED' }) + await expect( + service.diffText({ + source: 'upload', + projectId: 'project-1', + fileId: fixture.fileId, + versionId: fixture.versionIds[1], + requestId: 'missing-native-binding' + }) + ).rejects.toMatchObject({ code: 'NATIVE_WRITE_REQUIRED' }) + }) + + it('audits only active heads during startup and validates historical bytes lazily', async () => { + const fixture = await createFixture('artifact') + const historical = await new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }).resolve({ + source: 'artifact', + projectId: 'project-1', + fileId: fixture.fileId, + versionId: fixture.versionIds[0] + }) + await writeFile(historical.path, 'corrupt historical bytes') + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }) + + await expect(service.recoverPendingWrites()).resolves.toMatchObject({ integrityErrors: [] }) + await expect( + service.inspect({ + source: 'artifact', + projectId: 'project-1', + fileId: fixture.fileId, + versionId: fixture.versionIds[0] + }) + ).rejects.toMatchObject({ code: 'CONTENT_INTEGRITY_FAILED' }) + }) + + it('keeps blocking journal recovery separate from the explicit active-head integrity audit', async () => { + const fixture = await createFixture('artifact') + const service = new ManagedFileVersionService({ + storageRoot, + writeAndPublish: testWriteAndPublish, + getClient: () => Promise.resolve(client) + }) + const head = await service.resolve({ + source: 'artifact', + projectId: 'project-1', + fileId: fixture.fileId + }) + await writeFile(head.path, 'corrupt active head') + + await expect(service.recoverPendingWrites()).resolves.toMatchObject({ integrityErrors: [] }) + await expect(service.auditActiveVersionIntegrity()).resolves.toEqual([ + { + source: 'artifact', + fileId: fixture.fileId, + versionId: fixture.versionIds[1], + code: 'CONTENT_INTEGRITY_FAILED' + } + ]) + }) + + it('audits a large binary head without invoking the body reader', async () => { + const fixture = await createFixture('upload') + await client.uploadVersion.update({ + where: { id: fixture.versionIds[1] }, + data: { contentType: 'video/mp4' } + }) + const service = new ManagedFileVersionService({ + storageRoot, + getClient: () => Promise.resolve(client), + verifyAnchored: () => true, + readAnchored: () => { + throw new Error('audit must not allocate the file body') + } + }) + + await expect(service.auditActiveVersionIntegrity()).resolves.toEqual([]) + }) +}) diff --git a/src/main/managed-file-versions/service.ts b/src/main/managed-file-versions/service.ts new file mode 100644 index 000000000..002e20c42 --- /dev/null +++ b/src/main/managed-file-versions/service.ts @@ -0,0 +1,2142 @@ +import { createHash, randomInt, randomUUID } from 'node:crypto' +import { constants, type BigIntStats } from 'node:fs' +import { open, type FileHandle } from 'node:fs/promises' +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' + +import type { Prisma, PrismaClient } from '@prisma/client' + +import { + MANAGED_TEXT_EDIT_MAX_BYTES, + MANAGED_DIFF_MAX_INPUT_BYTES, + buildManagedVersionStoredFilename, + createManagedVersionStorageTag, + inspectManagedTextEditEligibility, + isManagedVersionStoredFilename, + type ManagedFileSource, + type ManagedFileVersionDescriptor, + type ManagedFileVersionDiffRequest, + type ManagedFileVersionDiffResult, + type ManagedFileVersionErrorCode, + type ManagedFileVersionInspectRequest, + type ManagedFileVersionInspectResult, + type ManagedFileVersionHostCapability, + type ManagedFileVersionResolveRequest, + type ManagedFileVersionSaveTextEditRequest, + type ManagedTextFormat, + type SaveTextEditResult +} from '../../shared/managed-file-versions' +import { ManagedTextDiffTaskRunner } from './diff-task' +import { defaultArtifactDurability, type ArtifactDurability } from '../artifacts/durability' +import { sha256 } from '../artifacts/provenance-canonical' +import { + readAnchoredFile, + readAnchoredFileBounded, + listAnchoredDirectory, + managedFileVersionNativeCapability, + removeAnchoredFile, + publishVerifiedAnchoredFileNoReplace, + verifyAnchoredFile, + writeAndPublishNoReplace +} from '../uploads/atomic-no-replace-publisher' + +const COMPLETE_STATE = { artifact: 'finalized', upload: 'ready' } as const +const STORAGE_COLLISION_MAX_ATTEMPTS = 16 +const ORPHAN_TEMP_MIN_AGE_MS = 24 * 60 * 60 * 1000 +const INTEGRITY_AUDIT_BATCH_SIZE = 100 +const INTEGRITY_AUDIT_MAX_ERRORS = 1000 +const SAFE_STORAGE_SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]*$/u +const MANAGED_VERSION_TEMP_PATTERN = /^\.(v[a-z0-9]{8}_.+)\.[a-f0-9]{16}\.tmp$/u + +type ManagedFileVersionRecord = { + id: string + fileId: string + versionNumber: number + state: string + managedVisibleAt?: Date | null + originKind: string + basedOnVersionId: string | null + storageTag: string | null + storedFilename: string | null + writeOperationId: string | null + contentStorageKey: string + filename: string + originalFilename: string | null + contentType: string | null + sizeBytes: bigint + checksum: string + createdAt: Date +} + +type ManagedLogicalFile = { + source: ManagedFileSource + id: string + projectId: string + sessionId: string + displayName: string + currentVersionId: string | null +} + +type ResolvedManagedFileVersion = { + logicalFile: ManagedLogicalFile + version: ManagedFileVersionRecord + path: string +} + +type ManagedFileReadLease = ResolvedManagedFileVersion & { + size: number + versionToken: number + snapshot: ManagedFileLeaseSnapshot + read: ( + buffer: Uint8Array, + offset: number, + length: number, + position: number + ) => Promise<{ bytesRead: number }> + readRange: (begin: number, end: number) => Promise + copyTo: (destinationPath: string, options?: { exclusive?: boolean }) => Promise + verifyUnchanged: () => Promise + close: () => Promise +} + +type ManagedFileLeaseSnapshot = Pick + +const leaseSnapshotMatches = (snapshot: ManagedFileLeaseSnapshot, current: BigIntStats): boolean => + current.isFile() && + current.dev === snapshot.dev && + current.ino === snapshot.ino && + current.size === snapshot.size && + current.mtimeNs === snapshot.mtimeNs + +const readExactFromHandle = async ( + fileHandle: FileHandle, + buffer: Uint8Array, + position: number +): Promise => { + let offset = 0 + while (offset < buffer.byteLength) { + const { bytesRead } = await fileHandle.read( + buffer, + offset, + buffer.byteLength - offset, + position + offset + ) + if (bytesRead <= 0) throw new Error('Managed file changed during trusted consumption.') + offset += bytesRead + } +} + +const checksumOpenHandle = async (fileHandle: FileHandle, size: number): Promise => { + const hash = createHash('sha256') + let position = 0 + while (position < size) { + const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, size - position)) + await readExactFromHandle(fileHandle, buffer, position) + hash.update(buffer) + position += buffer.byteLength + } + return hash.digest('hex') +} + +const openManagedFileReadLease = async ( + resolved: ResolvedManagedFileVersion +): Promise => { + const expectedSize = Number(resolved.version.sizeBytes) + if (!Number.isSafeInteger(expectedSize) || expectedSize < 0) { + operationError('CONTENT_INTEGRITY_FAILED', 'Managed file version size is invalid.') + } + + const fileHandle = await open(resolved.path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)) + let closed = false + let snapshot: ManagedFileLeaseSnapshot + try { + const before = await fileHandle.stat({ bigint: true }) + if (!before.isFile() || before.size !== BigInt(expectedSize)) { + throw new Error('Managed file version size or type changed before trusted consumption.') + } + snapshot = { + dev: before.dev, + ino: before.ino, + size: before.size, + mtimeNs: before.mtimeNs + } + const actualChecksum = await checksumOpenHandle(fileHandle, expectedSize) + const after = await fileHandle.stat({ bigint: true }) + if (actualChecksum !== resolved.version.checksum || !leaseSnapshotMatches(snapshot, after)) { + throw new Error('Managed file version changed during trusted verification.') + } + } catch (error) { + await fileHandle.close().catch(() => undefined) + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Managed file version content is unavailable or corrupt.', + { cause: error } + ) + } + + const assertOpen = (): void => { + if (closed) throw new Error('Managed file read lease is closed.') + } + const verifyUnchanged = async (): Promise => { + assertOpen() + const current = await fileHandle.stat({ bigint: true }) + if (!leaseSnapshotMatches(snapshot, current)) { + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Managed file version changed during trusted consumption.' + ) + } + } + const readRange = async (begin: number, end: number): Promise => { + assertOpen() + if ( + !Number.isSafeInteger(begin) || + !Number.isSafeInteger(end) || + begin < 0 || + end <= begin || + end > expectedSize + ) { + throw new Error('Invalid managed file lease range.') + } + const buffer = Buffer.allocUnsafe(end - begin) + await readExactFromHandle(fileHandle, buffer, begin) + await verifyUnchanged() + return new Uint8Array(buffer) + } + const copyTo = async ( + destinationPath: string, + options?: { exclusive?: boolean } + ): Promise => { + assertOpen() + const destinationHandle = await open( + destinationPath, + constants.O_CREAT | constants.O_RDWR | (options?.exclusive ? constants.O_EXCL : 0), + 0o666 + ) + try { + const sourceStat = await fileHandle.stat() + const destinationStat = await destinationHandle.stat() + if (destinationStat.dev === sourceStat.dev && destinationStat.ino === sourceStat.ino) { + throw new Error('Cannot save a managed file over its source.') + } + await destinationHandle.truncate(0) + const hash = createHash('sha256') + let position = 0 + while (position < expectedSize) { + const buffer = Buffer.allocUnsafe(Math.min(64 * 1024, expectedSize - position)) + await readExactFromHandle(fileHandle, buffer, position) + hash.update(buffer) + let written = 0 + while (written < buffer.byteLength) { + const result = await destinationHandle.write( + buffer, + written, + buffer.byteLength - written, + position + written + ) + if (result.bytesWritten <= 0) throw new Error('Managed file destination write stalled.') + written += result.bytesWritten + } + position += buffer.byteLength + } + if (hash.digest('hex') !== resolved.version.checksum) { + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Managed file version changed during export.' + ) + } + await verifyUnchanged() + } finally { + await destinationHandle.close() + } + } + + return { + ...resolved, + size: expectedSize, + versionToken: Number(snapshot.mtimeNs) / 1_000_000, + snapshot: { ...snapshot }, + read: async (buffer, offset, length, position) => { + assertOpen() + return fileHandle.read(buffer, offset, length, position) + }, + readRange, + copyTo, + verifyUnchanged, + close: async () => { + if (closed) return + closed = true + await fileHandle.close() + } + } +} + +type ManagedFileVersionRecoveryResult = { + recovered: number + conflicted: number + failed: number + integrityErrors: ManagedFileVersionIntegrityError[] +} + +type ManagedFileVersionIntegrityError = { + source: ManagedFileSource + fileId: string + versionId: string + code: 'CONTENT_INTEGRITY_FAILED' +} + +type ManagedFileVersionTestFault = + 'after-journal' | 'after-temp-write' | 'after-file-publish' | 'after-file-ready' + +type ManagedFileVersionServiceOptions = { + storageRoot: string + getClient: () => Promise + createId?: () => string + createStorageTag?: () => string + now?: () => Date + durability?: ArtifactDurability + writeAndPublish?: typeof writeAndPublishNoReplace + readAnchored?: typeof readAnchoredFile + readAnchoredBounded?: typeof readAnchoredFileBounded + verifyAnchored?: typeof verifyAnchoredFile + publishVerified?: typeof publishVerifiedAnchoredFileNoReplace + listAnchored?: typeof listAnchoredDirectory + removeAnchored?: typeof removeAnchoredFile + testFaultAt?: ManagedFileVersionTestFault + nativeWriteAvailable?: boolean + nativeReadFallbackAvailable?: boolean + diffTaskRunner?: Pick +} + +type WriteOperationRecord = Prisma.ManagedFileVersionWriteOperationGetPayload + +class ManagedFileVersionError extends Error { + readonly name = 'ManagedFileVersionError' + + constructor( + readonly code: ManagedFileVersionErrorCode, + message: string, + options?: ErrorOptions + ) { + super(message, options) + } +} + +const operationError = (code: ManagedFileVersionErrorCode, message: string): never => { + throw new ManagedFileVersionError(code, message) +} + +const isMissing = (error: unknown): boolean => + typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT' + +const isExists = (error: unknown): boolean => + typeof error === 'object' && error !== null && 'code' in error && error.code === 'EEXIST' + +const isRetryableRecoveryError = (error: unknown): boolean => { + if (typeof error !== 'object' || error === null) return false + const code = 'code' in error ? error.code : undefined + if (code === 'EIO' || code === 'EBUSY' || code === 'ETIMEDOUT') return true + return 'cause' in error && isRetryableRecoveryError(error.cause) +} + +const assertSafeStorageSegment = (value: string, label: string): string => { + if (!SAFE_STORAGE_SEGMENT.test(value)) { + operationError('INVALID_REQUEST', `Invalid ${label}.`) + } + return value +} + +const toDescriptor = ( + source: ManagedFileSource, + displayName: string, + version: ManagedFileVersionRecord +): ManagedFileVersionDescriptor => ({ + id: version.id, + source, + fileId: version.fileId, + versionNumber: version.versionNumber, + displayName, + originKind: version.originKind as ManagedFileVersionDescriptor['originKind'], + basedOnVersionId: version.basedOnVersionId, + contentType: version.contentType, + sizeBytes: Number(version.sizeBytes), + checksum: version.checksum, + createdAt: version.createdAt.toISOString() +}) + +const normalizeTextBytes = (content: string, format: ManagedTextFormat): Buffer => { + if (content.includes('\0')) operationError('CONTAINS_NUL', 'Text content contains NUL bytes.') + const newline = format.newline === 'crlf' ? '\r\n' : '\n' + const normalized = content.replace(/\r\n|\r|\n/gu, '\n').replace(/\n/gu, newline) + const body = Buffer.from(normalized, 'utf8') + if (new TextDecoder('utf-8', { fatal: true }).decode(body) !== normalized) { + operationError('INVALID_UTF8', 'Text content is not valid UTF-8.') + } + const bytes = format.hasUtf8Bom ? Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), body]) : body + if (bytes.byteLength > MANAGED_TEXT_EDIT_MAX_BYTES) { + operationError('EDIT_LIMIT_EXCEEDED', 'Text content exceeds the edit size limit.') + } + return bytes +} + +const temporaryFilename = (operationId: string, storedFilename: string): string => + `.${storedFilename}.${createHash('sha256').update(operationId).digest('hex').slice(0, 16)}.tmp` + +const isManagedVisibleArtifactVersion = (version: ManagedFileVersionRecord): boolean => + version.originKind !== 'agent_generated' || version.managedVisibleAt != null + +class ManagedFileVersionService { + private readonly createId: () => string + private readonly createStorageTag: () => string + private readonly now: () => Date + private readonly durability: ArtifactDurability + private readonly writeAndPublish: typeof writeAndPublishNoReplace + private readonly readAnchoredBounded: typeof readAnchoredFileBounded + private readonly verifyAnchored: typeof verifyAnchoredFile + private readonly publishVerified: typeof publishVerifiedAnchoredFileNoReplace + private readonly listAnchored: typeof listAnchoredDirectory + private readonly removeAnchored: typeof removeAnchoredFile + private readonly nativeWriteAvailable: boolean + private readonly nativeReadFallbackAvailable: boolean + private readonly diffTaskRunner: Pick + private readonly activeDiffs = new Map() + + constructor(private readonly options: ManagedFileVersionServiceOptions) { + this.createId = options.createId ?? randomUUID + this.createStorageTag = + options.createStorageTag ?? + (() => createManagedVersionStorageTag((limit) => randomInt(limit))) + this.now = options.now ?? (() => new Date()) + this.durability = options.durability ?? defaultArtifactDurability + this.writeAndPublish = options.writeAndPublish ?? writeAndPublishNoReplace + this.readAnchoredBounded = + options.readAnchoredBounded ?? + (options.readAnchored + ? (rootPath, parentPath, name, maxBytes) => { + const bytes = options.readAnchored!(rootPath, parentPath, name) + if (bytes.byteLength > maxBytes) { + throw Object.assign(new Error('bounded read overflow'), { code: 'EFBIG' }) + } + return bytes + } + : readAnchoredFileBounded) + this.verifyAnchored = + options.verifyAnchored ?? + (options.readAnchored + ? (rootPath, parentPath, name, expectedSizeBytes, expectedSha256) => { + const bytes = options.readAnchored!(rootPath, parentPath, name) + return bytes.byteLength === expectedSizeBytes && sha256(bytes) === expectedSha256 + } + : verifyAnchoredFile) + this.publishVerified = options.publishVerified ?? publishVerifiedAnchoredFileNoReplace + this.listAnchored = options.listAnchored ?? listAnchoredDirectory + this.removeAnchored = options.removeAnchored ?? removeAnchoredFile + const nativeCapability = managedFileVersionNativeCapability() + this.nativeWriteAvailable = options.nativeWriteAvailable ?? nativeCapability.available + this.nativeReadFallbackAvailable = + options.nativeReadFallbackAvailable ?? + (options.nativeWriteAvailable === undefined ? nativeCapability.readFallbackAvailable : false) + this.diffTaskRunner = options.diffTaskRunner ?? new ManagedTextDiffTaskRunner() + } + + getCapability(): ManagedFileVersionHostCapability { + return this.nativeWriteAvailable + ? { available: true } + : { available: false, reason: 'NATIVE_WRITE_REQUIRED' } + } + + async inspect( + request: ManagedFileVersionInspectRequest + ): Promise { + const resolved = await this.resolveRecord(request) + const versions = await this.listVersions(resolved.logicalFile) + const writeUnavailableReason = await this.writeUnavailableReason(resolved.logicalFile) + if (writeUnavailableReason === 'NATIVE_WRITE_REQUIRED' && !this.nativeReadFallbackAvailable) { + return { + source: request.source, + projectId: request.projectId, + fileId: request.fileId, + sessionId: resolved.logicalFile.sessionId, + displayName: resolved.logicalFile.displayName, + headVersionId: resolved.logicalFile.currentVersionId!, + selectedVersionId: resolved.version.id, + versions: versions.map((version) => + toDescriptor(request.source, resolved.logicalFile.displayName, version) + ), + canEdit: false, + canDiff: false, + unavailableReason: writeUnavailableReason + } + } + const eligibility = await this.readTextEligibility(resolved) + + return { + source: request.source, + projectId: request.projectId, + fileId: request.fileId, + sessionId: resolved.logicalFile.sessionId, + displayName: resolved.logicalFile.displayName, + headVersionId: resolved.logicalFile.currentVersionId!, + selectedVersionId: resolved.version.id, + versions: versions.map((version) => + toDescriptor(request.source, resolved.logicalFile.displayName, version) + ), + canEdit: eligibility.editable && writeUnavailableReason === undefined, + canDiff: eligibility.editable && resolved.version.basedOnVersionId !== null, + ...(eligibility.editable ? { text: eligibility.text, textFormat: eligibility.format } : {}), + ...(writeUnavailableReason + ? { unavailableReason: writeUnavailableReason } + : eligibility.editable + ? {} + : { unavailableReason: eligibility.reason }) + } + } + + async resolve(request: ManagedFileVersionResolveRequest): Promise { + const resolved = await this.resolveRecord(request) + if (!this.nativeWriteAvailable) { + operationError('NATIVE_WRITE_REQUIRED', 'Native anchored managed-file access is unavailable.') + } + this.verifyVersion(resolved.version) + return resolved + } + + // Resolves only the authoritative DB identity. Export callers use this after anchored reads report + // that the host lacks native support, then pin and bound the resulting path with their own handle. + async resolvePath( + request: ManagedFileVersionResolveRequest + ): Promise { + return this.resolveRecord(request) + } + + async openResolved(request: ManagedFileVersionResolveRequest): Promise { + const resolved = await this.resolveRecord(request) + if (this.nativeWriteAvailable) this.verifyVersion(resolved.version) + else if (!this.nativeReadFallbackAvailable) { + operationError('NATIVE_WRITE_REQUIRED', 'Native anchored managed-file access is unavailable.') + } + return openManagedFileReadLease(resolved) + } + + async diffText(request: ManagedFileVersionDiffRequest): Promise { + if (!request.requestId) operationError('INVALID_REQUEST', 'Diff request id is required.') + if (this.activeDiffs.has(request.requestId)) { + operationError('INVALID_REQUEST', 'Diff request id is already active.') + } + const active = { cancelled: false, workerStarted: false } + this.activeDiffs.set(request.requestId, active) + const assertNotCancelled = (): void => { + if (active.cancelled) operationError('DIFF_CANCELLED', 'Diff request was cancelled.') + } + try { + const selected = await this.resolveRecord(request) + assertNotCancelled() + const baseVersionId = selected.version.basedOnVersionId + if (!baseVersionId) + operationError('DIFF_BASE_NOT_FOUND', 'Selected version has no diff base.') + const ownedBaseVersionId = baseVersionId as string + const base = await this.resolveRecord({ ...request, versionId: ownedBaseVersionId }) + assertNotCancelled() + const before = await this.readTextForDiff(base) + const after = await this.readTextForDiff(selected) + assertNotCancelled() + active.workerStarted = true + const lines = await this.diffTaskRunner.run({ requestId: request.requestId, before, after }) + assertNotCancelled() + return { baseVersionId: ownedBaseVersionId, selectedVersionId: selected.version.id, lines } + } finally { + if (this.activeDiffs.get(request.requestId) === active) { + this.activeDiffs.delete(request.requestId) + } + } + } + + cancelDiff(requestId: string): boolean { + const active = this.activeDiffs.get(requestId) + if (!active) return false + active.cancelled = true + if (active.workerStarted) this.diffTaskRunner.cancel(requestId) + return true + } + + private async resolveRecord( + request: ManagedFileVersionResolveRequest + ): Promise { + this.assertIdentity(request) + const client = await this.options.getClient() + const logicalFile = await this.loadLogicalFile(client, request) + const headVersionId = logicalFile.currentVersionId + if (!headVersionId) { + throw new ManagedFileVersionError( + 'VERSION_NOT_FOUND', + 'Managed file has no published version.' + ) + } + const versionId = request.versionId ?? headVersionId + const version = await this.loadVersion(client, logicalFile, versionId) + if (!version) { + throw new ManagedFileVersionError('VERSION_NOT_FOUND', 'Managed file version was not found.') + } + if (request.source === 'artifact' && !isManagedVisibleArtifactVersion(version)) { + operationError('VERSION_NOT_FOUND', 'Managed file version is not published.') + } + if (version.fileId !== logicalFile.id) { + operationError('VERSION_NOT_IN_FILE', 'Managed file version belongs to another file.') + } + if (version.state !== COMPLETE_STATE[request.source]) { + operationError('VERSION_NOT_FOUND', 'Managed file version is not published.') + } + return { logicalFile, version, path: this.resolveStoragePath(version.contentStorageKey) } + } + + async saveTextEdit(request: ManagedFileVersionSaveTextEditRequest): Promise { + this.assertSaveRequest(request) + if (!this.nativeWriteAvailable) { + operationError('NATIVE_WRITE_REQUIRED', 'Native anchored file writes are unavailable.') + } + const client = await this.options.getClient() + await this.assertProjectWritable(client, request.projectId) + const logicalFile = await this.loadLogicalFile(client, request) + await this.assertFileWritable(client, logicalFile) + // Keep the fast path honest, while publishDatabaseTransaction repeats this barrier under its + // write transaction to close the race with a concurrent deletion. + await this.assertPublicationAllowed(client, logicalFile) + const existing = await client.managedFileVersionWriteOperation.findUnique({ + where: { operationId: request.operationId } + }) + if (existing) { + const format = this.parseOperationFormat(existing.textFormatJson) + const replayBytes = normalizeTextBytes(request.content, format) + this.assertOperationMatches(existing, request, sha256(replayBytes), replayBytes.byteLength) + return this.resumeOperation(client, logicalFile, existing, replayBytes) + } + const headVersionId = logicalFile.currentVersionId + if (!headVersionId) { + throw new ManagedFileVersionError( + 'VERSION_NOT_FOUND', + 'Managed file has no published version.' + ) + } + + const basedOn = await this.loadVersion(client, logicalFile, request.basedOnVersionId) + if (!basedOn) { + throw new ManagedFileVersionError('VERSION_NOT_FOUND', 'Base version was not found.') + } + if (request.source === 'artifact' && !isManagedVisibleArtifactVersion(basedOn)) { + operationError('VERSION_NOT_FOUND', 'Base version is not published.') + } + if (basedOn.fileId !== logicalFile.id) { + operationError('VERSION_NOT_IN_FILE', 'Base version belongs to another file.') + } + if (basedOn.state !== COMPLETE_STATE[request.source]) { + operationError('VERSION_NOT_FOUND', 'Base version is not published.') + } + const eligibility = await this.readTextEligibility({ + logicalFile, + version: basedOn, + path: this.resolveStoragePath(basedOn.contentStorageKey) + }) + if (!eligibility.editable) { + throw new ManagedFileVersionError( + eligibility.reason, + 'Managed file is not editable as UTF-8 text.' + ) + } + const bytes = normalizeTextBytes(request.content, eligibility.format) + const outputEligibility = inspectManagedTextEditEligibility(logicalFile.displayName, bytes) + if (!outputEligibility.editable) { + throw new ManagedFileVersionError( + outputEligibility.reason, + 'Edited managed file content is not valid UTF-8 text.' + ) + } + const contentChecksum = sha256(bytes) + const head = await this.loadVersion(client, logicalFile, headVersionId) + if (!head || head.state !== COMPLETE_STATE[request.source]) { + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Managed file head is not a published version.' + ) + } + + if (contentChecksum === basedOn.checksum && bytes.byteLength === Number(basedOn.sizeBytes)) { + return { + kind: 'noop', + version: toDescriptor(request.source, logicalFile.displayName, basedOn), + headVersionId: head.id + } + } + + const operation = await this.createOperation( + client, + logicalFile, + request, + contentChecksum, + bytes.byteLength, + eligibility.format + ) + this.maybeCrash('after-journal') + return this.resumeOperation(client, logicalFile, operation, bytes) + } + + async recoverPendingWrites(): Promise { + const client = await this.options.getClient() + const result: ManagedFileVersionRecoveryResult = { + recovered: 0, + conflicted: 0, + failed: 0, + integrityErrors: [] + } + let operationCursor: string | undefined + for (;;) { + const operations = await client.managedFileVersionWriteOperation.findMany({ + where: { + state: { in: ['staging', 'file_ready'] }, + ...(operationCursor ? { operationId: { gt: operationCursor } } : {}) + }, + orderBy: { operationId: 'asc' }, + take: INTEGRITY_AUDIT_BATCH_SIZE + }) + for (const operation of operations) { + try { + const logicalFile = await this.loadLogicalFile(client, { + source: operation.source as ManagedFileSource, + projectId: operation.projectId, + fileId: operation.sourceFileId + }) + const resumed = await this.resumeOperation(client, logicalFile, operation) + if (resumed.kind === 'created') result.recovered += 1 + else if (resumed.kind === 'conflict') result.conflicted += 1 + } catch (error) { + if (isRetryableRecoveryError(error)) continue + await this.failOperation(client, operation, 'CONTENT_INTEGRITY_FAILED') + result.failed += 1 + } + } + if (operations.length < INTEGRITY_AUDIT_BATCH_SIZE) break + operationCursor = operations.at(-1)?.operationId + if (!operationCursor) break + await new Promise((resolveRecoveryYield) => setImmediate(resolveRecoveryYield)) + } + + await this.cleanupTerminalOperations(client) + await this.cleanupOrphanTemporaryFiles(client) + await this.rebuildHeadProjections(client) + return result + } + + async auditActiveVersionIntegrity(): Promise { + const client = await this.options.getClient() + return this.auditActiveVersions(client) + } + + private assertIdentity(request: ManagedFileVersionResolveRequest): void { + if (!request || (request.source !== 'artifact' && request.source !== 'upload')) { + operationError('INVALID_REQUEST', 'Managed file source is invalid.') + } + assertSafeStorageSegment(request.projectId, 'project id') + assertSafeStorageSegment(request.fileId, 'file id') + if (request.versionId !== undefined) assertSafeStorageSegment(request.versionId, 'version id') + } + + private assertSaveRequest(request: ManagedFileVersionSaveTextEditRequest): void { + this.assertIdentity(request) + assertSafeStorageSegment(request.basedOnVersionId, 'base version id') + assertSafeStorageSegment(request.expectedHeadVersionId, 'expected head version id') + assertSafeStorageSegment(request.operationId, 'operation id') + if (typeof request.content !== 'string') { + operationError('INVALID_REQUEST', 'Text edit content must be a string.') + } + // Every UTF-16 code unit produces at least one UTF-8 byte. Rejecting this conservative bound + // here prevents oversized renderer input from being copied by newline normalization or Buffer. + if (request.content.length > MANAGED_TEXT_EDIT_MAX_BYTES) { + operationError('EDIT_LIMIT_EXCEEDED', 'Text content exceeds the edit size limit.') + } + } + + private async assertProjectWritable(client: PrismaClient, projectId: string): Promise { + const project = await client.project.findUnique({ + where: { id: projectId }, + select: { archivedAt: true } + }) + const deleting = await client.projectDeletionIntent.findUnique({ where: { projectId } }) + if (!project || project.archivedAt || deleting) { + operationError('PROJECT_NOT_WRITABLE', 'Project is not writable.') + } + } + + private async assertFileWritable( + client: PrismaClient, + logicalFile: ManagedLogicalFile + ): Promise { + const projection = await client.managedFile.findUnique({ + where: { + projectId_source_sourceFileId: { + projectId: logicalFile.projectId, + source: logicalFile.source, + sourceFileId: logicalFile.id + } + }, + select: { deletedAt: true } + }) + if (projection?.deletedAt) operationError('FILE_DELETED', 'Managed file is deleted.') + } + + private async assertPublicationAllowed( + client: PrismaClient | Prisma.TransactionClient, + logicalFile: ManagedLogicalFile + ): Promise { + const [project, deleting, origin, sync, projection] = await Promise.all([ + client.project.findUnique({ + where: { id: logicalFile.projectId }, + select: { archivedAt: true } + }), + client.projectDeletionIntent.findUnique({ + where: { projectId: logicalFile.projectId }, + select: { projectId: true } + }), + client.fileOriginSession.findUnique({ + where: { + projectId_sessionId: { + projectId: logicalFile.projectId, + sessionId: logicalFile.sessionId + } + }, + select: { state: true, deletedAt: true, deletionOperationId: true } + }), + client.managedFileSessionSync.findUnique({ + where: { + projectId_sessionId: { + projectId: logicalFile.projectId, + sessionId: logicalFile.sessionId + } + }, + select: { deletedAt: true, deleteOperationId: true } + }), + client.managedFile.findUnique({ + where: { + projectId_source_sourceFileId: { + projectId: logicalFile.projectId, + source: logicalFile.source, + sourceFileId: logicalFile.id + } + }, + select: { deletedAt: true, deleteOperationId: true } + }) + ]) + if (!project || project.archivedAt || deleting) { + operationError('PROJECT_NOT_WRITABLE', 'Project is not writable.') + } + if ( + (origin && (origin.state !== 'active' || origin.deletedAt || origin.deletionOperationId)) || + sync?.deletedAt || + sync?.deleteOperationId || + projection?.deletedAt || + projection?.deleteOperationId + ) { + operationError('FILE_DELETED', 'Managed file or its Session is deleted.') + } + } + + private async writeUnavailableReason( + logicalFile: ManagedLogicalFile + ): Promise<'NATIVE_WRITE_REQUIRED' | 'PROJECT_NOT_WRITABLE' | 'FILE_DELETED' | undefined> { + if (!this.nativeWriteAvailable) return 'NATIVE_WRITE_REQUIRED' + const client = await this.options.getClient() + const [project, deleting, origin, sync, projection] = await Promise.all([ + client.project.findUnique({ + where: { id: logicalFile.projectId }, + select: { archivedAt: true } + }), + client.projectDeletionIntent.findUnique({ + where: { projectId: logicalFile.projectId }, + select: { projectId: true } + }), + client.fileOriginSession.findUnique({ + where: { + projectId_sessionId: { + projectId: logicalFile.projectId, + sessionId: logicalFile.sessionId + } + }, + select: { state: true, deletedAt: true, deletionOperationId: true } + }), + client.managedFileSessionSync.findUnique({ + where: { + projectId_sessionId: { + projectId: logicalFile.projectId, + sessionId: logicalFile.sessionId + } + }, + select: { deletedAt: true, deleteOperationId: true } + }), + client.managedFile.findUnique({ + where: { + projectId_source_sourceFileId: { + projectId: logicalFile.projectId, + source: logicalFile.source, + sourceFileId: logicalFile.id + } + }, + select: { deletedAt: true, deleteOperationId: true } + }) + ]) + if ( + (origin && (origin.state !== 'active' || origin.deletedAt || origin.deletionOperationId)) || + sync?.deletedAt || + sync?.deleteOperationId || + projection?.deletedAt || + projection?.deleteOperationId + ) { + return 'FILE_DELETED' + } + if (!project || project.archivedAt || deleting) return 'PROJECT_NOT_WRITABLE' + return undefined + } + + private async loadLogicalFile( + client: PrismaClient | Prisma.TransactionClient, + request: { source: ManagedFileSource; projectId: string; fileId: string } + ): Promise { + if (request.source === 'artifact') { + const file = await client.artifactLineage.findFirst({ + where: { id: request.fileId, projectId: request.projectId }, + select: { + id: true, + projectId: true, + sessionId: true, + filename: true, + currentVersionId: true + } + }) + if (!file) { + throw new ManagedFileVersionError('FILE_NOT_FOUND', 'Managed Artifact was not found.') + } + return { source: 'artifact', displayName: file.filename, ...file } + } + const file = await client.uploadFile.findFirst({ + where: { id: request.fileId, projectId: request.projectId }, + select: { + id: true, + projectId: true, + sessionId: true, + filename: true, + originalFilename: true, + currentVersionId: true + } + }) + if (!file) { + throw new ManagedFileVersionError('FILE_NOT_FOUND', 'Managed Upload was not found.') + } + return { + source: 'upload', + id: file.id, + projectId: file.projectId, + sessionId: file.sessionId, + displayName: file.originalFilename || file.filename, + currentVersionId: file.currentVersionId + } + } + + private async loadVersion( + client: PrismaClient | Prisma.TransactionClient, + logicalFile: ManagedLogicalFile, + versionId: string + ): Promise { + if (logicalFile.source === 'artifact') { + const version = await client.artifactVersion.findUnique({ where: { id: versionId } }) + return version + ? { + ...version, + fileId: version.artifactId, + originalFilename: null, + createdAt: version.createdAt + } + : null + } + const version = await client.uploadVersion.findUnique({ where: { id: versionId } }) + return version + ? { + ...version, + fileId: version.uploadFileId, + createdAt: version.createdAt ?? version.registeredAt + } + : null + } + + private async listVersions(logicalFile: ManagedLogicalFile): Promise { + const client = await this.options.getClient() + if (logicalFile.source === 'artifact') { + const versions = await client.artifactVersion.findMany({ + where: { + artifactId: logicalFile.id, + state: 'finalized', + OR: [{ originKind: { not: 'agent_generated' } }, { managedVisibleAt: { not: null } }] + }, + orderBy: { versionNumber: 'asc' } + }) + return versions.map((version) => ({ + ...version, + fileId: version.artifactId, + originalFilename: null, + createdAt: version.createdAt + })) + } + const versions = await client.uploadVersion.findMany({ + where: { uploadFileId: logicalFile.id, state: 'ready' }, + orderBy: { versionNumber: 'asc' } + }) + return versions.map((version) => ({ + ...version, + fileId: version.uploadFileId, + createdAt: version.createdAt ?? version.registeredAt + })) + } + + private resolveStoragePath(contentStorageKey: string): string { + if ( + isAbsolute(contentStorageKey) || + contentStorageKey.includes('\\') || + contentStorageKey + .split('/') + .some((segment) => !segment || segment === '.' || segment === '..') + ) { + operationError('CONTENT_INTEGRITY_FAILED', 'Managed storage key is invalid.') + } + const path = resolve(this.options.storageRoot, ...contentStorageKey.split('/')) + const relativePath = relative(resolve(this.options.storageRoot), path) + if ( + relativePath === '' || + relativePath === '..' || + relativePath.startsWith(`..${sep}`) || + isAbsolute(relativePath) + ) { + operationError('CONTENT_INTEGRITY_FAILED', 'Managed storage key escapes its root.') + } + return path + } + + private versionAnchor(version: ManagedFileVersionRecord): { + path: string + parentPath: string + name: string + } { + const path = this.resolveStoragePath(version.contentStorageKey) + return { path, parentPath: dirname(path), name: basename(path) } + } + + private verifyVersion(version: ManagedFileVersionRecord): void { + const { parentPath, name } = this.versionAnchor(version) + try { + const matches = this.verifyAnchored( + this.options.storageRoot, + parentPath, + name, + Number(version.sizeBytes), + version.checksum + ) + if (!matches) { + throw new Error('checksum or size mismatch') + } + } catch (error) { + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Managed file version content is unavailable or corrupt.', + { cause: error } + ) + } + } + + private async readTextEligibility( + resolved: ResolvedManagedFileVersion + ): Promise> { + if (resolved.version.sizeBytes > BigInt(MANAGED_TEXT_EDIT_MAX_BYTES)) { + return { editable: false, reason: 'EDIT_LIMIT_EXCEEDED' } + } + if (this.nativeWriteAvailable) { + const { parentPath, name } = this.versionAnchor(resolved.version) + try { + const bytes = this.readAnchoredBounded( + this.options.storageRoot, + parentPath, + name, + MANAGED_TEXT_EDIT_MAX_BYTES + ) + if ( + bytes.byteLength !== Number(resolved.version.sizeBytes) || + sha256(bytes) !== resolved.version.checksum + ) { + throw new Error('checksum or size mismatch') + } + return inspectManagedTextEditEligibility(resolved.logicalFile.displayName, bytes) + } catch (error) { + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + error.code === 'EFBIG' + ) { + return { editable: false, reason: 'EDIT_LIMIT_EXCEEDED' } + } + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Managed file version content is unavailable or corrupt.', + { cause: error } + ) + } + } + if (!this.nativeReadFallbackAvailable) { + operationError('NATIVE_WRITE_REQUIRED', 'Native anchored managed-file access is unavailable.') + } + let lease: ManagedFileReadLease | undefined + try { + lease = await openManagedFileReadLease(resolved) + const bytes = lease.size === 0 ? new Uint8Array() : await lease.readRange(0, lease.size) + return inspectManagedTextEditEligibility(resolved.logicalFile.displayName, bytes) + } catch (error) { + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + error.code === 'EFBIG' + ) { + return { editable: false, reason: 'EDIT_LIMIT_EXCEEDED' } + } + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Managed file version content is unavailable or corrupt.', + { cause: error } + ) + } finally { + await lease?.close().catch(() => undefined) + } + } + + private async readTextForDiff(resolved: ResolvedManagedFileVersion): Promise { + if (resolved.version.sizeBytes > BigInt(MANAGED_DIFF_MAX_INPUT_BYTES)) { + operationError('DIFF_INPUT_LIMIT_EXCEEDED', 'Managed file exceeds the diff input limit.') + } + const eligibility = await this.readTextEligibility(resolved) + if (!eligibility.editable) { + if (eligibility.reason === 'EDIT_LIMIT_EXCEEDED') { + operationError('DIFF_INPUT_LIMIT_EXCEEDED', 'Managed file exceeds the diff input limit.') + } + operationError(eligibility.reason, 'Managed file is not eligible for text diff.') + } + return (eligibility as Extract).text + } + + private async verifyResolvedVersion(resolved: ResolvedManagedFileVersion): Promise { + if (this.nativeWriteAvailable) { + this.verifyVersion(resolved.version) + return + } + if (!this.nativeReadFallbackAvailable) { + operationError('NATIVE_WRITE_REQUIRED', 'Native anchored managed-file access is unavailable.') + } + const lease = await openManagedFileReadLease(resolved) + await lease.close() + } + + private async createOperation( + client: PrismaClient, + logicalFile: ManagedLogicalFile, + request: ManagedFileVersionSaveTextEditRequest, + checksum: string, + sizeBytes: number, + format: ManagedTextFormat + ): Promise { + for (let attempt = 0; attempt < STORAGE_COLLISION_MAX_ATTEMPTS; attempt += 1) { + const storageTag = this.createStorageTag() + const storedFilename = buildManagedVersionStoredFilename(logicalFile.displayName, storageTag) + const contentStorageKey = [ + `${logicalFile.source}s`, + logicalFile.projectId, + logicalFile.sessionId, + logicalFile.id, + 'managed-versions', + storedFilename + ].join('/') + const existingKey = await client.managedFileVersionWriteOperation.findFirst({ + where: { contentStorageKey }, + select: { operationId: true } + }) + if (existingKey) continue + try { + return await client.managedFileVersionWriteOperation.create({ + data: { + operationId: request.operationId, + source: logicalFile.source, + projectId: logicalFile.projectId, + sourceFileId: logicalFile.id, + basedOnVersionId: request.basedOnVersionId, + expectedHeadVersionId: request.expectedHeadVersionId, + state: 'staging', + storageTag, + storedFilename, + contentStorageKey, + checksum, + sizeBytes: BigInt(sizeBytes), + textFormatJson: JSON.stringify(format) + } + }) + } catch (error) { + const existing = await client.managedFileVersionWriteOperation.findUnique({ + where: { operationId: request.operationId } + }) + if (existing) { + this.assertOperationMatches(existing, request, checksum, sizeBytes) + return existing + } + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + error.code === 'P2002' + ) { + continue + } + throw error + } + } + throw new ManagedFileVersionError( + 'STORAGE_COLLISION', + 'Could not allocate immutable managed file storage.' + ) + } + + private assertOperationMatches( + operation: WriteOperationRecord, + request: ManagedFileVersionSaveTextEditRequest, + checksum: string, + sizeBytes: number + ): void { + if ( + operation.source !== request.source || + operation.projectId !== request.projectId || + operation.sourceFileId !== request.fileId || + operation.basedOnVersionId !== request.basedOnVersionId || + operation.expectedHeadVersionId !== request.expectedHeadVersionId || + operation.checksum !== checksum || + operation.sizeBytes !== BigInt(sizeBytes) + ) { + operationError('OPERATION_REUSED', 'Write operation id was reused for another edit.') + } + } + + private parseOperationFormat(value: string): ManagedTextFormat { + try { + const parsed = JSON.parse(value) as Partial + if ( + (parsed.newline !== 'lf' && parsed.newline !== 'crlf') || + typeof parsed.hasUtf8Bom !== 'boolean' || + typeof parsed.hasTrailingNewline !== 'boolean' + ) { + throw new Error('invalid text format') + } + return parsed as ManagedTextFormat + } catch (error) { + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Managed file write operation has an invalid text format.', + { cause: error } + ) + } + } + + private async resumeOperation( + client: PrismaClient, + logicalFile: ManagedLogicalFile, + initialOperation: WriteOperationRecord, + bytes?: Buffer, + collisionAttempt = 0 + ): Promise { + let operation = initialOperation + if (operation.state === 'published') return this.publishedResult(client, logicalFile, operation) + if (operation.state === 'conflict') return this.conflictResult(client, logicalFile, operation) + if (operation.state === 'failed') { + operationError('CONTENT_INTEGRITY_FAILED', 'Managed file write operation failed recovery.') + } + + const finalPath = this.resolveStoragePath(operation.contentStorageKey) + const parentPath = dirname(finalPath) + const tempName = temporaryFilename(operation.operationId, operation.storedFilename) + + if (operation.state === 'staging') { + if (!this.isOperationFileValid(operation, parentPath, operation.storedFilename)) { + if (!bytes) { + try { + const tempBytes = this.readAnchoredBounded( + this.options.storageRoot, + parentPath, + tempName, + Number(operation.sizeBytes) + ) + if ( + tempBytes.byteLength === Number(operation.sizeBytes) && + sha256(tempBytes) === operation.checksum + ) { + this.publishVerified( + this.options.storageRoot, + parentPath, + tempName, + operation.storedFilename, + tempBytes + ) + } + } catch (error) { + if (!isMissing(error)) throw error + } + } + if (!bytes) { + if (!this.isOperationFileValid(operation, parentPath, operation.storedFilename)) { + await this.failOperation(client, operation, 'CONTENT_INTEGRITY_FAILED') + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Managed file write bytes are unavailable for recovery.' + ) + } + } + if (bytes) + try { + this.writeAndPublish( + this.options.storageRoot, + parentPath, + tempName, + operation.storedFilename, + bytes + ) + this.maybeCrash('after-temp-write') + } catch (error) { + if (isExists(error)) { + if (collisionAttempt + 1 >= STORAGE_COLLISION_MAX_ATTEMPTS) { + await this.failOperation(client, operation, 'STORAGE_COLLISION', false) + operationError('STORAGE_COLLISION', 'Managed version destination already exists.') + } + const reallocated = await this.reallocateOperationDestination( + client, + logicalFile, + operation + ) + return this.resumeOperation( + client, + logicalFile, + reallocated, + bytes, + collisionAttempt + 1 + ) + } + throw error + } + await this.durability.syncDirectory(parentPath) + } + this.maybeCrash('after-file-publish') + const advanced = await client.managedFileVersionWriteOperation.updateMany({ + where: { operationId: operation.operationId, state: 'staging' }, + data: { state: 'file_ready', errorCode: null } + }) + if (advanced.count !== 1) { + operation = await client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: operation.operationId } + }) + if (operation.state === 'published') { + return this.publishedResult(client, logicalFile, operation) + } + if (operation.state === 'conflict') { + return this.conflictResult(client, logicalFile, operation) + } + if (operation.state !== 'file_ready') { + operationError('CONTENT_INTEGRITY_FAILED', 'Managed file write state is invalid.') + } + } else { + operation = await client.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: operation.operationId } + }) + } + this.maybeCrash('after-file-ready') + } + + if (!this.isOperationFileValid(operation, parentPath, operation.storedFilename)) { + await this.failOperation(client, operation, 'CONTENT_INTEGRITY_FAILED') + operationError('CONTENT_INTEGRITY_FAILED', 'Managed version publication is corrupt.') + } + const result = await this.publishDatabaseTransaction(client, logicalFile, operation) + if (result.kind === 'conflict') { + await this.removeFinalIfUnowned(client, operation) + } + this.tryRemoveAnchored(parentPath, tempName) + return result + } + + private async reallocateOperationDestination( + client: PrismaClient, + logicalFile: ManagedLogicalFile, + operation: WriteOperationRecord + ): Promise { + for (let attempt = 0; attempt < STORAGE_COLLISION_MAX_ATTEMPTS; attempt += 1) { + const storageTag = this.createStorageTag() + const storedFilename = buildManagedVersionStoredFilename(logicalFile.displayName, storageTag) + const contentStorageKey = [ + `${logicalFile.source}s`, + logicalFile.projectId, + logicalFile.sessionId, + logicalFile.id, + 'managed-versions', + storedFilename + ].join('/') + try { + return await client.managedFileVersionWriteOperation.update({ + where: { operationId: operation.operationId, state: 'staging' }, + data: { storageTag, storedFilename, contentStorageKey } + }) + } catch (error) { + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + error.code === 'P2002' + ) { + continue + } + throw error + } + } + await this.failOperation(client, operation, 'STORAGE_COLLISION', false) + throw new ManagedFileVersionError( + 'STORAGE_COLLISION', + 'Could not reallocate immutable managed file storage.' + ) + } + + private async publishDatabaseTransaction( + client: PrismaClient, + logicalFile: ManagedLogicalFile, + operation: WriteOperationRecord + ): Promise { + return client.$transaction(async (tx) => { + const currentFile = await this.loadLogicalFile(tx, { + source: logicalFile.source, + projectId: logicalFile.projectId, + fileId: logicalFile.id + }) + await this.assertPublicationAllowed(tx, currentFile) + if (currentFile.currentVersionId !== operation.expectedHeadVersionId) { + const conflicted = await tx.managedFileVersionWriteOperation.updateMany({ + where: { operationId: operation.operationId, state: 'file_ready' }, + data: { state: 'conflict', errorCode: 'HEAD_CHANGED' } + }) + if (conflicted.count !== 1) { + const currentOperation = await tx.managedFileVersionWriteOperation.findUniqueOrThrow({ + where: { operationId: operation.operationId } + }) + if (currentOperation.state === 'published') { + const publishedVersion = await this.loadVersion( + tx, + currentFile, + currentOperation.resultVersionId ?? '' + ) + this.assertPublishedVersionMatches(currentOperation, currentFile, publishedVersion) + return { + kind: 'created', + replayed: true, + version: toDescriptor(logicalFile.source, logicalFile.displayName, publishedVersion), + headVersionId: publishedVersion.id + } + } + } + const actualHead = currentFile.currentVersionId + ? await this.loadVersion(tx, currentFile, currentFile.currentVersionId) + : null + if (!actualHead) { + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Actual head is unavailable.' + ) + } + return { + kind: 'conflict', + expectedHeadVersionId: operation.expectedHeadVersionId, + actualHead: toDescriptor(logicalFile.source, logicalFile.displayName, actualHead) + } + } + const basedOn = await this.loadVersion(tx, currentFile, operation.basedOnVersionId) + if ( + !basedOn || + basedOn.state !== COMPLETE_STATE[logicalFile.source] || + (logicalFile.source === 'artifact' && !isManagedVisibleArtifactVersion(basedOn)) + ) { + throw new ManagedFileVersionError( + 'VERSION_NOT_FOUND', + 'Base version is unavailable during publication.' + ) + } + const maxVersionNumber = await this.maxVersionNumber(tx, logicalFile) + const versionId = this.createId() + const createdAt = this.now() + const version = await this.insertUserEditVersion( + tx, + logicalFile, + operation, + versionId, + maxVersionNumber + 1, + basedOn, + createdAt + ) + await this.advanceHead(tx, logicalFile, operation.expectedHeadVersionId, versionId) + await this.upsertProjection(tx, logicalFile, version, createdAt) + const published = await tx.managedFileVersionWriteOperation.updateMany({ + where: { operationId: operation.operationId, state: 'file_ready' }, + data: { state: 'published', resultVersionId: versionId, errorCode: null } + }) + if (published.count !== 1) { + operationError('CONTENT_INTEGRITY_FAILED', 'Managed file write lost publication ownership.') + } + return { + kind: 'created', + replayed: false, + version: toDescriptor(logicalFile.source, logicalFile.displayName, version), + headVersionId: versionId + } + }) + } + + private async maxVersionNumber( + tx: Prisma.TransactionClient, + logicalFile: ManagedLogicalFile + ): Promise { + if (logicalFile.source === 'artifact') { + return ( + ( + await tx.artifactVersion.aggregate({ + where: { artifactId: logicalFile.id }, + _max: { versionNumber: true } + }) + )._max.versionNumber ?? 0 + ) + } + return ( + ( + await tx.uploadVersion.aggregate({ + where: { uploadFileId: logicalFile.id }, + _max: { versionNumber: true } + }) + )._max.versionNumber ?? 0 + ) + } + + private async insertUserEditVersion( + tx: Prisma.TransactionClient, + logicalFile: ManagedLogicalFile, + operation: WriteOperationRecord, + versionId: string, + versionNumber: number, + basedOn: ManagedFileVersionRecord, + createdAt: Date + ): Promise { + if (logicalFile.source === 'artifact') { + const version = await tx.artifactVersion.create({ + data: { + id: versionId, + artifactId: logicalFile.id, + versionNumber, + filename: logicalFile.displayName, + originKind: 'user_edit', + basedOnVersionId: basedOn.id, + storageTag: operation.storageTag, + storedFilename: operation.storedFilename, + writeOperationId: operation.operationId, + state: 'finalized', + managedVisibleAt: createdAt, + contentStorageKey: operation.contentStorageKey, + contentType: basedOn.contentType, + sizeBytes: operation.sizeBytes, + checksum: operation.checksum, + createdAt + } + }) + return { ...version, fileId: version.artifactId, originalFilename: null, createdAt } + } + const version = await tx.uploadVersion.create({ + data: { + id: versionId, + uploadFileId: logicalFile.id, + versionNumber, + state: 'ready', + originKind: 'user_edit', + basedOnVersionId: basedOn.id, + storageTag: operation.storageTag, + storedFilename: operation.storedFilename, + writeOperationId: operation.operationId, + contentStorageKey: operation.contentStorageKey, + filename: logicalFile.displayName, + originalFilename: logicalFile.displayName, + contentType: basedOn.contentType, + sizeBytes: operation.sizeBytes, + checksum: operation.checksum, + createdAt + } + }) + return { ...version, fileId: version.uploadFileId, createdAt } + } + + private async advanceHead( + tx: Prisma.TransactionClient, + logicalFile: ManagedLogicalFile, + expectedHeadVersionId: string, + resultVersionId: string + ): Promise { + const updated = + logicalFile.source === 'artifact' + ? await tx.artifactLineage.updateMany({ + where: { id: logicalFile.id, currentVersionId: expectedHeadVersionId }, + data: { currentVersionId: resultVersionId } + }) + : await tx.uploadFile.updateMany({ + where: { id: logicalFile.id, currentVersionId: expectedHeadVersionId }, + data: { currentVersionId: resultVersionId } + }) + if (updated.count !== 1) operationError('HEAD_CHANGED', 'Managed file head changed.') + } + + private async upsertProjection( + tx: Prisma.TransactionClient, + logicalFile: ManagedLogicalFile, + version: ManagedFileVersionRecord, + timestamp: Date + ): Promise { + await tx.managedFile.upsert({ + where: { + projectId_source_sourceFileId: { + projectId: logicalFile.projectId, + source: logicalFile.source, + sourceFileId: logicalFile.id + } + }, + create: { + source: logicalFile.source, + sourceFileId: logicalFile.id, + sourceVersionId: version.id, + checksum: version.checksum, + projectId: logicalFile.projectId, + sessionId: logicalFile.sessionId, + displayName: logicalFile.displayName, + storageKey: version.contentStorageKey, + mimeType: version.contentType, + sizeBytes: version.sizeBytes, + mtimeMs: BigInt(timestamp.getTime()), + sortAtMs: BigInt(timestamp.getTime()) + }, + update: { + sourceVersionId: version.id, + checksum: version.checksum, + sessionId: logicalFile.sessionId, + displayName: logicalFile.displayName, + storageKey: version.contentStorageKey, + mimeType: version.contentType, + sizeBytes: version.sizeBytes, + mtimeMs: BigInt(timestamp.getTime()), + sortAtMs: BigInt(timestamp.getTime()), + messageId: null, + deletedAt: null, + deleteOperationId: null + } + }) + } + + private async publishedResult( + client: PrismaClient, + logicalFile: ManagedLogicalFile, + operation: WriteOperationRecord + ): Promise { + const resultVersionId = operation.resultVersionId + if (!resultVersionId) { + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Published operation has no result version.' + ) + } + const version = await this.loadVersion(client, logicalFile, resultVersionId) + this.assertPublishedVersionMatches(operation, logicalFile, version) + this.verifyVersion(version) + return { + kind: 'created', + replayed: true, + version: toDescriptor(logicalFile.source, logicalFile.displayName, version), + headVersionId: version.id + } + } + + private assertPublishedVersionMatches( + operation: WriteOperationRecord, + logicalFile: ManagedLogicalFile, + version: ManagedFileVersionRecord | null + ): asserts version is ManagedFileVersionRecord { + if ( + !version || + version.fileId !== logicalFile.id || + version.state !== COMPLETE_STATE[logicalFile.source] || + version.writeOperationId !== operation.operationId || + version.contentStorageKey !== operation.contentStorageKey || + version.checksum !== operation.checksum || + version.sizeBytes !== operation.sizeBytes + ) { + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Published result version does not match its write operation.' + ) + } + } + + private async conflictResult( + client: PrismaClient, + logicalFile: ManagedLogicalFile, + operation: WriteOperationRecord + ): Promise { + const headVersionId = logicalFile.currentVersionId + if (!headVersionId) { + throw new ManagedFileVersionError( + 'CONTENT_INTEGRITY_FAILED', + 'Managed file has no actual head.' + ) + } + const actualHead = await this.loadVersion(client, logicalFile, headVersionId) + if (!actualHead) { + throw new ManagedFileVersionError('CONTENT_INTEGRITY_FAILED', 'Actual head is missing.') + } + return { + kind: 'conflict', + expectedHeadVersionId: operation.expectedHeadVersionId, + actualHead: toDescriptor(logicalFile.source, logicalFile.displayName, actualHead) + } + } + + private isOperationFileValid( + operation: WriteOperationRecord, + parentPath: string, + name: string + ): boolean { + try { + return this.verifyAnchored( + this.options.storageRoot, + parentPath, + name, + Number(operation.sizeBytes), + operation.checksum + ) + } catch (error) { + if (isMissing(error)) return false + throw error + } + } + + private tryRemoveAnchored(parentPath: string, name: string): void { + try { + this.removeAnchored(this.options.storageRoot, parentPath, name) + } catch (error) { + if (!isMissing(error)) throw error + } + } + + private async failOperation( + client: PrismaClient, + operation: WriteOperationRecord, + errorCode: ManagedFileVersionErrorCode, + removeFinal = true + ): Promise { + const finalPath = this.resolveStoragePath(operation.contentStorageKey) + const tempPath = join( + dirname(finalPath), + temporaryFilename(operation.operationId, operation.storedFilename) + ) + const failed = await client.managedFileVersionWriteOperation.updateMany({ + where: { + operationId: operation.operationId, + state: { in: ['staging', 'file_ready'] } + }, + data: { state: 'failed', errorCode } + }) + if (failed.count !== 1) return + if (removeFinal) await this.removeFinalIfUnowned(client, operation) + this.tryRemoveAnchored( + dirname(tempPath), + temporaryFilename(operation.operationId, operation.storedFilename) + ) + } + + private async removeFinalIfUnowned( + client: PrismaClient, + operation: WriteOperationRecord + ): Promise { + const removable = await client.$transaction(async (tx) => { + const [journal, artifactOwner, uploadOwner] = await Promise.all([ + tx.managedFileVersionWriteOperation.findUnique({ + where: { operationId: operation.operationId }, + select: { state: true, resultVersionId: true, contentStorageKey: true } + }), + tx.artifactVersion.findUnique({ + where: { contentStorageKey: operation.contentStorageKey }, + select: { id: true } + }), + tx.uploadVersion.findUnique({ + where: { contentStorageKey: operation.contentStorageKey }, + select: { id: true } + }) + ]) + return ( + !!journal && + journal.contentStorageKey === operation.contentStorageKey && + journal.state !== 'published' && + journal.resultVersionId === null && + !artifactOwner && + !uploadOwner + ) + }) + if (!removable) return + const finalPath = this.resolveStoragePath(operation.contentStorageKey) + // A stale journal must not remove a path that now contains unrelated bytes. The database + // ownership proof above prevents deleting a referenced Version; the byte proof prevents a + // failed retry from deleting a reused or externally replaced destination. + let fileMatchesJournal = false + try { + fileMatchesJournal = this.isOperationFileValid( + operation, + dirname(finalPath), + operation.storedFilename + ) + } catch { + return + } + if (!fileMatchesJournal) return + this.tryRemoveAnchored(dirname(finalPath), operation.storedFilename) + } + + private async auditActiveVersions( + client: PrismaClient + ): Promise { + const integrityErrors: ManagedFileVersionIntegrityError[] = [] + let artifactCursor: string | undefined + for (;;) { + const artifacts = await client.artifactLineage.findMany({ + where: { currentVersionId: { not: null } }, + include: { currentVersion: true }, + orderBy: { id: 'asc' }, + take: INTEGRITY_AUDIT_BATCH_SIZE, + ...(artifactCursor ? { cursor: { id: artifactCursor }, skip: 1 } : {}) + }) + for (const file of artifacts) { + const version = file.currentVersion + if (!version || version.state !== 'finalized') continue + const record: ManagedFileVersionRecord = { + ...version, + fileId: version.artifactId, + originalFilename: null, + createdAt: version.createdAt + } + try { + await this.verifyResolvedVersion({ + logicalFile: { + source: 'artifact', + id: file.id, + projectId: file.projectId, + sessionId: file.sessionId, + displayName: file.filename, + currentVersionId: file.currentVersionId + }, + version: record, + path: this.resolveStoragePath(record.contentStorageKey) + }) + } catch { + integrityErrors.push({ + source: 'artifact', + fileId: file.id, + versionId: version.id, + code: 'CONTENT_INTEGRITY_FAILED' + }) + } + if (integrityErrors.length >= INTEGRITY_AUDIT_MAX_ERRORS) return integrityErrors + } + if (artifacts.length < INTEGRITY_AUDIT_BATCH_SIZE) break + artifactCursor = artifacts.at(-1)?.id + if (!artifactCursor) break + await new Promise((resolveAuditYield) => setImmediate(resolveAuditYield)) + } + + let uploadCursor: string | undefined + for (;;) { + const uploads = await client.uploadFile.findMany({ + where: { currentVersionId: { not: null } }, + include: { currentVersion: true }, + orderBy: { id: 'asc' }, + take: INTEGRITY_AUDIT_BATCH_SIZE, + ...(uploadCursor ? { cursor: { id: uploadCursor }, skip: 1 } : {}) + }) + for (const file of uploads) { + const version = file.currentVersion + if (!version || version.state !== 'ready') continue + const record: ManagedFileVersionRecord = { + ...version, + fileId: version.uploadFileId, + createdAt: version.createdAt ?? version.registeredAt + } + try { + await this.verifyResolvedVersion({ + logicalFile: { + source: 'upload', + id: file.id, + projectId: file.projectId, + sessionId: file.sessionId, + displayName: file.originalFilename || file.filename, + currentVersionId: file.currentVersionId + }, + version: record, + path: this.resolveStoragePath(record.contentStorageKey) + }) + } catch { + integrityErrors.push({ + source: 'upload', + fileId: file.id, + versionId: version.id, + code: 'CONTENT_INTEGRITY_FAILED' + }) + } + if (integrityErrors.length >= INTEGRITY_AUDIT_MAX_ERRORS) return integrityErrors + } + if (uploads.length < INTEGRITY_AUDIT_BATCH_SIZE) break + uploadCursor = uploads.at(-1)?.id + if (!uploadCursor) break + await new Promise((resolveAuditYield) => setImmediate(resolveAuditYield)) + } + return integrityErrors + } + + private async cleanupTerminalOperations(client: PrismaClient): Promise { + let cursor: string | undefined + for (;;) { + const operations = await client.managedFileVersionWriteOperation.findMany({ + where: { state: { in: ['conflict', 'failed'] } }, + orderBy: { operationId: 'asc' }, + take: INTEGRITY_AUDIT_BATCH_SIZE, + ...(cursor ? { cursor: { operationId: cursor }, skip: 1 } : {}) + }) + for (const operation of operations) { + await this.removeFinalIfUnowned(client, operation) + this.tryRemoveAnchored( + dirname(this.resolveStoragePath(operation.contentStorageKey)), + temporaryFilename(operation.operationId, operation.storedFilename) + ) + } + if (operations.length < INTEGRITY_AUDIT_BATCH_SIZE) break + cursor = operations.at(-1)?.operationId + if (!cursor) break + await new Promise((resolveCleanupYield) => setImmediate(resolveCleanupYield)) + } + } + + private async cleanupOrphanTemporaryFiles(client: PrismaClient): Promise { + const cutoff = this.now().getTime() - ORPHAN_TEMP_MIN_AGE_MS + for (const source of ['upload', 'artifact'] as const) { + let cursor: string | undefined + for (;;) { + const files = + source === 'upload' + ? await client.uploadFile.findMany({ + orderBy: { id: 'asc' }, + take: INTEGRITY_AUDIT_BATCH_SIZE, + ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + select: { projectId: true, sessionId: true, id: true } + }) + : await client.artifactLineage.findMany({ + orderBy: { id: 'asc' }, + take: INTEGRITY_AUDIT_BATCH_SIZE, + ...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}), + select: { projectId: true, sessionId: true, id: true } + }) + for (const file of files) { + const parentPath = this.resolveStoragePath( + `${source}s/${file.projectId}/${file.sessionId}/${file.id}/managed-versions` + ) + let entries: Array<{ name: string; isFile: boolean; mtimeMs: number }> + try { + entries = this.listAnchored(this.options.storageRoot, parentPath) + } catch (error) { + if (isMissing(error)) continue + throw error + } + for (const entry of entries) { + if (!entry.isFile) continue + const match = MANAGED_VERSION_TEMP_PATTERN.exec(entry.name) + if (!match || !isManagedVersionStoredFilename(match[1]!)) continue + if (entry.mtimeMs > cutoff) continue + const owners = await client.managedFileVersionWriteOperation.findMany({ + where: { + source, + projectId: file.projectId, + sourceFileId: file.id, + storedFilename: match[1]! + }, + select: { operationId: true, storedFilename: true } + }) + if ( + owners.some( + (operation) => + temporaryFilename(operation.operationId, operation.storedFilename) === entry.name + ) + ) { + continue + } + this.tryRemoveAnchored(parentPath, entry.name) + } + } + if (files.length < INTEGRITY_AUDIT_BATCH_SIZE) break + cursor = files.at(-1)?.id + if (!cursor) break + await new Promise((resolveOrphanYield) => setImmediate(resolveOrphanYield)) + } + } + } + + private async rebuildHeadProjections(client: PrismaClient): Promise { + let artifactCursor: string | undefined + for (;;) { + const artifacts = await client.artifactLineage.findMany({ + where: { currentVersionId: { not: null } }, + include: { currentVersion: true }, + orderBy: { id: 'asc' }, + take: INTEGRITY_AUDIT_BATCH_SIZE, + ...(artifactCursor ? { cursor: { id: artifactCursor }, skip: 1 } : {}) + }) + for (const file of artifacts) { + await client.$transaction(async (tx) => { + const version = file.currentVersion + if (!version || version.state !== 'finalized') return + if (await this.hasProjectionBarrier(tx, file.projectId, file.sessionId)) return + const existing = await tx.managedFile.findUnique({ + where: { + projectId_source_sourceFileId: { + projectId: file.projectId, + source: 'artifact', + sourceFileId: file.id + } + }, + select: { deletedAt: true } + }) + // Runtime recovery repairs an already-visible Files tile. It must not create one for an Agent + // head whose compatibility bytes or durable Message graph have not become visible yet. + if (!existing || existing.deletedAt) return + await this.upsertProjection( + tx, + { + source: 'artifact', + id: file.id, + projectId: file.projectId, + sessionId: file.sessionId, + displayName: file.filename, + currentVersionId: version.id + }, + { + ...version, + fileId: version.artifactId, + originalFilename: null, + createdAt: version.createdAt + }, + version.createdAt + ) + }) + } + if (artifacts.length < INTEGRITY_AUDIT_BATCH_SIZE) break + artifactCursor = artifacts.at(-1)?.id + if (!artifactCursor) break + await new Promise((resolveProjectionYield) => setImmediate(resolveProjectionYield)) + } + + let uploadCursor: string | undefined + for (;;) { + const uploads = await client.uploadFile.findMany({ + where: { currentVersionId: { not: null } }, + include: { currentVersion: true }, + orderBy: { id: 'asc' }, + take: INTEGRITY_AUDIT_BATCH_SIZE, + ...(uploadCursor ? { cursor: { id: uploadCursor }, skip: 1 } : {}) + }) + for (const file of uploads) { + await client.$transaction(async (tx) => { + const version = file.currentVersion + if (!version || version.state !== 'ready') return + if (await this.hasProjectionBarrier(tx, file.projectId, file.sessionId)) return + const existing = await tx.managedFile.findUnique({ + where: { + projectId_source_sourceFileId: { + projectId: file.projectId, + source: 'upload', + sourceFileId: file.id + } + }, + select: { deletedAt: true } + }) + if (!existing || existing.deletedAt) return + const createdAt = version.createdAt ?? version.registeredAt + await this.upsertProjection( + tx, + { + source: 'upload', + id: file.id, + projectId: file.projectId, + sessionId: file.sessionId, + displayName: file.originalFilename || file.filename, + currentVersionId: version.id + }, + { ...version, fileId: version.uploadFileId, createdAt }, + createdAt + ) + }) + } + if (uploads.length < INTEGRITY_AUDIT_BATCH_SIZE) break + uploadCursor = uploads.at(-1)?.id + if (!uploadCursor) break + await new Promise((resolveProjectionYield) => setImmediate(resolveProjectionYield)) + } + } + + private async hasProjectionBarrier( + tx: Prisma.TransactionClient, + projectId: string, + sessionId: string + ): Promise { + const [project, deleting, origin, sync] = await Promise.all([ + tx.project.findUnique({ where: { id: projectId }, select: { archivedAt: true } }), + tx.projectDeletionIntent.findUnique({ where: { projectId }, select: { projectId: true } }), + tx.fileOriginSession.findUnique({ + where: { projectId_sessionId: { projectId, sessionId } }, + select: { state: true, deletedAt: true, deletionOperationId: true } + }), + tx.managedFileSessionSync.findUnique({ + where: { projectId_sessionId: { projectId, sessionId } }, + select: { deletedAt: true, deleteOperationId: true } + }) + ]) + return ( + !project || + !!project.archivedAt || + !!deleting || + !origin || + origin.state !== 'active' || + !!origin.deletedAt || + !!origin.deletionOperationId || + !!sync?.deletedAt || + !!sync?.deleteOperationId + ) + } + + private maybeCrash(phase: ManagedFileVersionTestFault): void { + if (this.options.testFaultAt === phase) { + throw new Error(`simulated managed version crash: ${phase}`) + } + } +} + +export { ManagedFileVersionError, ManagedFileVersionService } +export type { + ManagedFileReadLease, + ManagedFileVersionRecoveryResult, + ManagedFileVersionServiceOptions, + ResolvedManagedFileVersion +} diff --git a/src/main/managed-preview-protocol.test.ts b/src/main/managed-preview-protocol.test.ts index a57462ff5..5e35f484d 100644 --- a/src/main/managed-preview-protocol.test.ts +++ b/src/main/managed-preview-protocol.test.ts @@ -202,6 +202,46 @@ describe('managed preview protocol', () => { } }) + it('streams a logical managed resource from its verified lease after the path is replaced', async () => { + const verified = Buffer.from('verified managed version') + const close = vi.fn().mockResolvedValue(undefined) + const lease = { + path: '/managed/report.xlsx', + size: verified.byteLength, + versionToken: 42, + snapshot: { dev: 1n, ino: 2n, size: BigInt(verified.byteLength), mtimeNs: 3n }, + read: vi.fn(async (buffer: Uint8Array, offset: number, length: number, position: number) => { + buffer.set(verified.subarray(position, position + length), offset) + return { bytesRead: length } + }), + readRange: vi.fn(), + copyTo: vi.fn(), + verifyUnchanged: vi.fn().mockResolvedValue(undefined), + close + } + const resources = new (await import('./managed-preview-resources')).ManagedPreviewResources({ + resolvePath: vi.fn(), + openManagedFileVersion: vi.fn().mockResolvedValue(lease), + createId: () => 'trusted-resource' + } as never) + await resources.acquire(17, { + source: 'artifact', + path: 'stale-projection', + projectId: 'project-1', + fileId: 'artifact-1' + }) + + const response = await createManagedPreviewProtocolHandler(resources)( + new Request('open-science-preview://trusted-resource/report.xlsx') + ) + + await expect(response.text()).resolves.toBe('verified managed version') + expect(lease.read).toHaveBeenCalled() + expect(close).not.toHaveBeenCalled() + resources.release(17, { resourceId: 'trusted-resource' }) + expect(close).toHaveBeenCalledOnce() + }) + it('rejects URLs that are not an acquired resource capability', async () => { const resources = { resolveProtocolResource: vi diff --git a/src/main/managed-preview-resources.test.ts b/src/main/managed-preview-resources.test.ts index 305a198b6..5a6da5a29 100644 --- a/src/main/managed-preview-resources.test.ts +++ b/src/main/managed-preview-resources.test.ts @@ -77,6 +77,104 @@ describe('ManagedPreviewResources', () => { }) }) + it('passes a default logical Artifact identity to the resolver at capability acquisition', async () => { + const filePath = await createFile(Buffer.from('head bytes')) + const resolvePath = vi.fn().mockResolvedValue(filePath) + const resources = new ManagedPreviewResources({ resolvePath, createId: () => 'head-resource' }) + + await resources.acquire(17, { + source: 'artifact', + path: 'artifact-version:stale-projection', + projectId: 'project-1', + fileId: 'artifact-1' + }) + + expect(resolvePath).toHaveBeenCalledWith('artifact', { + source: 'artifact', + path: 'artifact-version:stale-projection', + projectId: 'project-1', + fileId: 'artifact-1' + }) + }) + + it('reads a logical managed version through its trusted lease and closes it on release', async () => { + const filePath = await createFile(Buffer.from('path replacement')) + const trustedBytes = Buffer.from('verified inode') + const close = vi.fn().mockResolvedValue(undefined) + const openManagedFileVersion = vi.fn().mockResolvedValue({ + path: '/managed/verified.pdf', + size: trustedBytes.byteLength, + versionToken: 42, + snapshot: { dev: 1n, ino: 2n, size: BigInt(trustedBytes.byteLength), mtimeNs: 3n }, + read: vi.fn(), + readRange: vi.fn(async (begin: number, end: number) => trustedBytes.subarray(begin, end)), + copyTo: vi.fn(), + verifyUnchanged: vi.fn().mockResolvedValue(undefined), + close + }) + const resolvePath = vi.fn().mockResolvedValue(filePath) + const resources = new ManagedPreviewResources({ + resolvePath, + openManagedFileVersion, + createId: () => 'trusted-resource' + } as never) + + const resource = await resources.acquire(17, { + source: 'upload', + path: 'stale-projection', + projectId: 'project-1', + fileId: 'upload-1', + versionId: 'upload-v2' + }) + + await expect( + resources.readRange(17, { resourceId: resource.id, begin: 0, end: trustedBytes.byteLength }) + ).resolves.toEqual({ + begin: 0, + end: trustedBytes.byteLength, + total: trustedBytes.byteLength, + data: new Uint8Array(trustedBytes) + }) + expect(openManagedFileVersion).toHaveBeenCalledWith('upload', { + projectId: 'project-1', + fileId: 'upload-1', + versionId: 'upload-v2' + }) + expect(resolvePath).not.toHaveBeenCalled() + + resources.release(17, { resourceId: resource.id }) + expect(close).toHaveBeenCalledOnce() + }) + + it('closes the temporary trusted lease used for Office admission inspection', async () => { + const close = vi.fn().mockResolvedValue(undefined) + const openManagedFileVersion = vi.fn().mockResolvedValue({ + path: '/managed/report.xlsx', + size: 6, + versionToken: 42, + snapshot: { dev: 1n, ino: 2n, size: 6n, mtimeNs: 3n }, + read: vi.fn(), + readRange: vi.fn(), + copyTo: vi.fn(), + verifyUnchanged: vi.fn(), + close + }) + const resources = new ManagedPreviewResources({ + resolvePath: vi.fn(), + openManagedFileVersion + } as never) + + await expect( + resources.inspect({ + source: 'artifact', + path: 'stale-projection', + projectId: 'project-1', + fileId: 'artifact-1' + }) + ).resolves.toEqual({ size: 6, version: 42, dev: 1n, ino: 2n, mtimeNs: 3n }) + expect(close).toHaveBeenCalledOnce() + }) + it('inspects authoritative metadata without minting a resource capability', async () => { const filePath = await createFile(Buffer.from('office')) const createId = vi.fn(() => 'resource-1') diff --git a/src/main/managed-preview-resources.ts b/src/main/managed-preview-resources.ts index e25043c09..13c4df474 100644 --- a/src/main/managed-preview-resources.ts +++ b/src/main/managed-preview-resources.ts @@ -1,7 +1,6 @@ import { randomUUID } from 'node:crypto' import type { BigIntStats } from 'node:fs' import { open, stat } from 'node:fs/promises' -import type { FileHandle } from 'node:fs/promises' import { basename, extname } from 'node:path' import type { OfficePreviewAdmissionError } from '../shared/office-preview' @@ -13,6 +12,7 @@ import type { ReadManagedPreviewRangeRequest, ReleaseManagedPreviewRequest } from '../shared/preview-resources' +import type { ManagedFileReadLease } from './managed-file-versions/service' const MAX_PREVIEW_RANGE_BYTES = 1024 * 1024 const MAX_RELEASED_RESOURCE_TOMBSTONES = 1024 @@ -62,6 +62,10 @@ type ManagedPreviewResourcesOptions = { source: ManagedPreviewSource, request: AcquireManagedPreviewRequest ) => Promise + openManagedFileVersion?: ( + source: 'artifact' | 'upload', + request: { projectId: string; fileId: string; versionId?: string } + ) => Promise createId?: () => string } @@ -81,6 +85,7 @@ type AcquireManagedPreviewOptions = { type ResourceEntry = ManagedPreviewResource & { ownerId: number filePath: string + trustedLease?: ManagedFileReadLease strictSnapshot?: { dev: bigint ino: bigint @@ -89,10 +94,12 @@ type ResourceEntry = ManagedPreviewResource & { } } +type PreviewProtocolFileHandle = RangeReader & { close: () => Promise } + type PreviewProtocolResource = | Pick | { - fileHandle: FileHandle + fileHandle: PreviewProtocolFileHandle mimeType: string size: number verifyUnchanged: () => Promise @@ -152,6 +159,20 @@ class ManagedPreviewResources { } async inspect(request: AcquireManagedPreviewRequest): Promise { + const trustedLease = await this.openTrustedLease(request) + if (trustedLease) { + try { + return { + size: trustedLease.size, + version: trustedLease.versionToken, + dev: trustedLease.snapshot.dev, + ino: trustedLease.snapshot.ino, + mtimeNs: trustedLease.snapshot.mtimeNs + } + } finally { + await trustedLease.close() + } + } // Resolve through the managed repository so metadata checks never accept an arbitrary path. const filePath = await this.options.resolvePath(request.source, request) const fileStat = await stat(filePath, { bigint: true }) @@ -165,61 +186,77 @@ class ManagedPreviewResources { request: AcquireManagedPreviewRequest, options?: AcquireManagedPreviewOptions ): Promise { - // Resolve through the managed repository before minting an owner-scoped capability URL. - const filePath = await this.options.resolvePath(request.source, request) - const fileStat = await stat(filePath, { bigint: true }) - - if (!fileStat.isFile()) { - throw new Error('Managed preview path is not a file.') - } - const fileSnapshot = snapshotFileStat(fileStat) - if (options && fileSnapshot.size > options.maxBytes) { - const error: OfficePreviewAdmissionError = Object.assign( - new Error('Managed preview file is too large.'), - { - code: 'FILE_TOO_LARGE' as const, - size: fileSnapshot.size, - limit: options.maxBytes - } - ) - throw error - } - if ( - options && - (fileSnapshot.size !== options.snapshot.size || - fileSnapshot.mtimeNs !== options.snapshot.mtimeNs || - fileSnapshot.dev !== options.snapshot.dev || - fileSnapshot.ino !== options.snapshot.ino) - ) { - throw new Error('Managed preview file changed after admission.') - } + const trustedLease = await this.openTrustedLease(request) + let admitted = false + try { + // Resolve through the managed repository before minting an owner-scoped capability URL. + const filePath = trustedLease + ? trustedLease.path + : await this.options.resolvePath(request.source, request) + const fileSnapshot = trustedLease + ? { + size: trustedLease.size, + version: trustedLease.versionToken, + dev: trustedLease.snapshot.dev, + ino: trustedLease.snapshot.ino, + mtimeNs: trustedLease.snapshot.mtimeNs + } + : await stat(filePath, { bigint: true }).then((fileStat) => { + if (!fileStat.isFile()) throw new Error('Managed preview path is not a file.') + return snapshotFileStat(fileStat) + }) + if (options && fileSnapshot.size > options.maxBytes) { + const error: OfficePreviewAdmissionError = Object.assign( + new Error('Managed preview file is too large.'), + { + code: 'FILE_TOO_LARGE' as const, + size: fileSnapshot.size, + limit: options.maxBytes + } + ) + throw error + } + if ( + options && + (fileSnapshot.size !== options.snapshot.size || + fileSnapshot.mtimeNs !== options.snapshot.mtimeNs || + fileSnapshot.dev !== options.snapshot.dev || + fileSnapshot.ino !== options.snapshot.ino) + ) { + throw new Error('Managed preview file changed after admission.') + } - const id = this.createId() - const resource: ManagedPreviewResource = { - id, - url: `${PREVIEW_SCHEME}://${id}/${encodeURIComponent(basename(filePath))}`, - size: fileSnapshot.size, - mimeType: inferMimeType(filePath, request.mimeType), - version: fileSnapshot.version - } + const id = this.createId() + const resource: ManagedPreviewResource = { + id, + url: `${PREVIEW_SCHEME}://${id}/${encodeURIComponent(basename(filePath))}`, + size: fileSnapshot.size, + mimeType: inferMimeType(filePath, request.mimeType), + version: fileSnapshot.version + } - this.releasedOwners.delete(id) - this.resources.set(id, { - ...resource, - ownerId, - filePath, - ...(options - ? { - strictSnapshot: { - dev: options.snapshot.dev, - ino: options.snapshot.ino, - mtimeNs: options.snapshot.mtimeNs, - maxBytes: options.maxBytes + this.releasedOwners.delete(id) + this.resources.set(id, { + ...resource, + ownerId, + filePath, + ...(trustedLease ? { trustedLease } : {}), + ...(options + ? { + strictSnapshot: { + dev: options.snapshot.dev, + ino: options.snapshot.ino, + mtimeNs: options.snapshot.mtimeNs, + maxBytes: options.maxBytes + } } - } - : {}) - }) - return resource + : {}) + }) + admitted = true + return resource + } finally { + if (trustedLease && !admitted) await trustedLease.close() + } } async readRange( @@ -240,9 +277,13 @@ class ManagedPreviewResources { throw new Error('Managed preview range exceeds the maximum size.') } + if (resource.trustedLease) { + const data = await resource.trustedLease.readRange(begin, end) + return { begin, end, total: resource.size, data: new Uint8Array(data) } + } + const buffer = Buffer.allocUnsafe(end - begin) const fileHandle = await open(resource.filePath, 'r') - try { await readExactRange(fileHandle, buffer, begin) @@ -287,6 +328,21 @@ class ManagedPreviewResources { throw new Error('Managed preview resource is not available.') } + if (resource.trustedLease) { + return { + fileHandle: { + read: (buffer, offset, length, position) => + resource.trustedLease!.read(buffer, offset, length, position), + // One capability may serve several concurrent HTTP range requests. The resource owner, + // not an individual response, closes the pinned handle. + close: async () => undefined + }, + mimeType: resource.mimeType, + size: resource.size, + verifyUnchanged: () => resource.trustedLease!.verifyUnchanged() + } + } + if (!resource.strictSnapshot) { return { filePath: resource.filePath, mimeType: resource.mimeType } } @@ -336,7 +392,9 @@ class ManagedPreviewResources { } private revokeResource(resourceId: string, ownerId: number): void { + const resource = this.resources.get(resourceId) this.resources.delete(resourceId) + if (resource?.trustedLease) void resource.trustedLease.close().catch(() => undefined) this.releasedOwners.set(resourceId, ownerId) while (this.releasedOwners.size > MAX_RELEASED_RESOURCE_TOMBSTONES) { const oldestResourceId = this.releasedOwners.keys().next().value @@ -354,6 +412,24 @@ class ManagedPreviewResources { return resource } + + private openTrustedLease( + request: AcquireManagedPreviewRequest + ): Promise { + if ( + !this.options.openManagedFileVersion || + (request.source !== 'artifact' && request.source !== 'upload') || + !request.projectId || + !request.fileId + ) { + return Promise.resolve(undefined) + } + return this.options.openManagedFileVersion(request.source, { + projectId: request.projectId, + fileId: request.fileId, + ...(request.versionId ? { versionId: request.versionId } : {}) + }) + } } export { diff --git a/src/main/notebook/host-artifacts-service.test.ts b/src/main/notebook/host-artifacts-service.test.ts index 271ffef58..6823fa480 100644 --- a/src/main/notebook/host-artifacts-service.test.ts +++ b/src/main/notebook/host-artifacts-service.test.ts @@ -251,6 +251,38 @@ describe('HostArtifactsService', () => { await expect(service.list({ limit: 101 }, context)).rejects.toThrow('between 1 and 100') }) + it('rejects an obsolete offset cursor and a changed catalog snapshot with stable errors', async () => { + let items = [ + artifact({ sourceFileId: 'C', versionId: 'C-v1', sortAtMs: 3 }), + artifact({ sourceFileId: 'A', versionId: 'A-v1', sortAtMs: 2 }), + artifact({ sourceFileId: 'B', versionId: 'B-v1', sortAtMs: 1 }) + ] + const catalog: HostArtifactCatalog = { + readHostArtifactCatalog: vi.fn(async () => items) + } + const service = new HostArtifactsService(catalog, { + artifact: { resolveVersionContent: vi.fn() }, + upload: { resolveManagedUploadPath: vi.fn() } + }) + + const first = await service.list({ limit: 2 }, context) + expect(first.artifacts.map((item) => item.id)).toEqual(['C', 'A']) + const unchangedSecond = await service.list({ limit: 2, cursor: first.nextCursor }, context) + expect(unchangedSecond.artifacts.map((item) => item.id)).toEqual(['B']) + items = [artifact({ sourceFileId: 'A', versionId: 'A-v2', sortAtMs: 4 }), items[0]!, items[2]!] + await expect(service.list({ limit: 2, cursor: first.nextCursor }, context)).rejects.toThrow( + /HOST_ARTIFACTS_CURSOR_SNAPSHOT_CHANGED/u + ) + + const obsolete = Buffer.from( + JSON.stringify({ version: 1, queryKey: 'obsolete', offset: 2 }), + 'utf8' + ).toString('base64url') + await expect(service.list({ cursor: obsolete }, context)).rejects.toThrow( + /cursor format is obsolete.*first page/iu + ) + }) + it('supports direct Version lookup and rejects every mixed or malformed option', async () => { const { service, readHostArtifactCatalog } = harness() diff --git a/src/main/notebook/host-artifacts-service.ts b/src/main/notebook/host-artifacts-service.ts index b534ff39d..3a61ff101 100644 --- a/src/main/notebook/host-artifacts-service.ts +++ b/src/main/notebook/host-artifacts-service.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto' import { basename, isAbsolute } from 'node:path' import { fuzzyScore } from '../../shared/fuzzy-match' @@ -47,7 +48,22 @@ type NormalizedOptions = { limit: number } -type Cursor = { version: 1; queryKey: string; offset: number } +type Cursor = { + version: 2 + queryKey: string + snapshotFingerprint: string + score: number + sortAtMs: number + identity: string +} + +type RankedArtifact = { + item: HostArtifactCatalogItem + score: number + identity: string +} + +const HOST_ARTIFACTS_CURSOR_SNAPSHOT_CHANGED = 'HOST_ARTIFACTS_CURSOR_SNAPSHOT_CHANGED' const OPTION_KEYS = new Set([ 'version_id', @@ -180,18 +196,60 @@ const decodeCursor = (value: string, queryKey: string): Cursor => { } catch { throw new Error('host.artifacts cursor is invalid.') } + if (isRecord(cursor) && cursor.version === 1) { + throw new Error('host.artifacts cursor format is obsolete; restart from the first page.') + } if ( !isRecord(cursor) || - cursor.version !== 1 || + cursor.version !== 2 || cursor.queryKey !== queryKey || - !Number.isInteger(cursor.offset) || - (cursor.offset as number) < 0 + typeof cursor.snapshotFingerprint !== 'string' || + !/^[a-f0-9]{64}$/u.test(cursor.snapshotFingerprint) || + typeof cursor.score !== 'number' || + !Number.isFinite(cursor.score) || + typeof cursor.sortAtMs !== 'number' || + !Number.isFinite(cursor.sortAtMs) || + typeof cursor.identity !== 'string' || + cursor.identity.length === 0 ) { throw new Error('host.artifacts cursor does not match the requested filters.') } return cursor as Cursor } +const artifactIdentity = (item: HostArtifactCatalogItem): string => + `${item.source}:${item.sourceFileId}` + +const compareRankedArtifacts = ( + left: Pick & { + item: Pick + }, + right: Pick & { + item: Pick + } +): number => + right.score - left.score || + right.item.sortAtMs - left.item.sortAtMs || + left.identity.localeCompare(right.identity) + +const catalogSnapshotFingerprint = (items: HostArtifactCatalogItem[]): string => { + const identities = items + .map((item) => ({ + identity: artifactIdentity(item), + versionId: item.versionId, + checksum: item.checksum ?? '', + projectId: item.projectId, + sessionId: item.sessionId, + filename: item.filename, + contentType: item.contentType ?? '', + sizeBytes: item.sizeBytes, + sortAtMs: item.sortAtMs, + rootFrameId: item.rootFrameId + })) + .sort((left, right) => left.identity.localeCompare(right.identity)) + return createHash('sha256').update(JSON.stringify(identities)).digest('hex') +} + const toHostArtifact = (item: HostArtifactCatalogItem): HostArtifact => { if (!item.sourceFileCreatedAt) { throw new Error(`Host Artifact source file metadata is incomplete: ${item.versionId}`) @@ -234,7 +292,8 @@ class HostArtifactsService { projectId: context.projectId, ...(normalized.versionId ? { versionId: normalized.versionId } : {}) }) - const ranked = candidates.flatMap((item) => { + const snapshotFingerprint = catalogSnapshotFingerprint(candidates) + const ranked: RankedArtifact[] = candidates.flatMap((item) => { if ( normalized.frameId && (item.source !== 'artifact' || item.agentFrameId !== normalized.frameId) @@ -254,19 +313,9 @@ class HostArtifactsService { if (normalized.beforeMs !== undefined && item.sortAtMs >= normalized.beforeMs) return [] const match = normalized.search ? fuzzyScore(normalized.search, item.filename) : undefined if (normalized.search && !match) return [] - return [{ item, score: match?.score ?? 0 }] + return [{ item, score: match?.score ?? 0, identity: artifactIdentity(item) }] }) - if (normalized.search) { - ranked.sort( - (left, right) => - right.score - left.score || - right.item.sortAtMs - left.item.sortAtMs || - (`${left.item.source}:${left.item.sourceFileId}` < - `${right.item.source}:${right.item.sourceFileId}` - ? -1 - : 1) - ) - } + ranked.sort(compareRankedArtifacts) const queryKey = JSON.stringify({ projectId: context.projectId, @@ -278,18 +327,41 @@ class HostArtifactsService { afterMs: normalized.afterMs, beforeMs: normalized.beforeMs }) - const offset = normalized.cursor ? decodeCursor(normalized.cursor, queryKey).offset : 0 - if (offset > ranked.length) throw new Error('host.artifacts cursor is no longer valid.') - const page = ranked.slice(offset, offset + normalized.limit) - const nextOffset = offset + page.length - const truncated = nextOffset < ranked.length + const cursor = normalized.cursor ? decodeCursor(normalized.cursor, queryKey) : undefined + if (cursor && cursor.snapshotFingerprint !== snapshotFingerprint) { + throw new Error( + `${HOST_ARTIFACTS_CURSOR_SNAPSHOT_CHANGED}: host.artifacts catalog changed; restart from the first page.` + ) + } + const remaining = cursor + ? ranked.filter( + (candidate) => + compareRankedArtifacts(candidate, { + score: cursor.score, + identity: cursor.identity, + item: { sortAtMs: cursor.sortAtMs } + }) > 0 + ) + : ranked + const page = remaining.slice(0, normalized.limit) + const truncated = page.length < remaining.length const artifacts = page.map(({ item }) => toHostArtifact(item)) + const last = page.at(-1) return { count: ranked.length, projectId: context.projectId, truncated, - ...(truncated - ? { nextCursor: encodeCursor({ version: 1, queryKey, offset: nextOffset }) } + ...(truncated && last + ? { + nextCursor: encodeCursor({ + version: 2, + queryKey, + snapshotFingerprint, + score: last.score, + sortAtMs: last.item.sortAtMs, + identity: last.identity + }) + } : {}), artifacts } @@ -331,5 +403,5 @@ class HostArtifactsService { } } -export { HostArtifactsService } +export { HOST_ARTIFACTS_CURSOR_SNAPSHOT_CHANGED, HostArtifactsService } export type { HostArtifactCatalog, HostArtifactPathResolvers, HostArtifactReadContext } diff --git a/src/main/notebook/input-registry.test.ts b/src/main/notebook/input-registry.test.ts index e3d12119c..43fd7110c 100644 --- a/src/main/notebook/input-registry.test.ts +++ b/src/main/notebook/input-registry.test.ts @@ -118,6 +118,7 @@ const createArtifact = async (input: { checksum: checksum(input.content), evidenceJson: '{}', evidenceChecksum: checksum('{}'), + evidenceSchemaVersion: 1, createdAt: new Date('2026-07-27T10:05:00.000Z') } } diff --git a/src/main/project-files/host-artifact-catalog.test.ts b/src/main/project-files/host-artifact-catalog.test.ts index ea91c717b..8385b08fd 100644 --- a/src/main/project-files/host-artifact-catalog.test.ts +++ b/src/main/project-files/host-artifact-catalog.test.ts @@ -41,9 +41,14 @@ describe('ManagedFileIndexRepository host Artifact catalog', () => { artifactId: string, versionId: string, versionNumber = 1, - agentFrameId = 'agent-frame', - rootFrameId = `root-${versionId}` + options: { + managedVisibleAt?: Date | null + agentFrameId?: string + rootFrameId?: string + } = {} ): Promise => { + const agentFrameId = options.agentFrameId ?? 'agent-frame' + const rootFrameId = options.rootFrameId ?? `root-${versionId}` await client.artifactLineage.upsert({ where: { projectId_sessionId_normalizedFilename: { @@ -76,8 +81,13 @@ describe('ManagedFileIndexRepository host Artifact catalog', () => { runtimeSegmentId: 'runtime', promptMessageId: 'prompt', state: 'finalized', + managedVisibleAt: + options.managedVisibleAt === undefined + ? new Date(`2026-08-0${versionNumber}T00:00:01.000Z`) + : options.managedVisibleAt, contentStorageKey: `artifacts/${projectId}/${sessionId}/${versionId}/content`, evidenceStorageKey: `artifacts/${projectId}/${sessionId}/${versionId}/evidence.json`, + evidenceSchemaVersion: 1, contentType: 'text/csv', sizeBytes: 10n, checksum: checksum(versionId), @@ -86,6 +96,10 @@ describe('ManagedFileIndexRepository host Artifact catalog', () => { createdAt: new Date(`2026-08-0${versionNumber}T00:00:00.000Z`) } await client.artifactVersion.create({ data }) + await client.artifactLineage.update({ + where: { id: artifactId }, + data: { currentVersionId: versionId } + }) } const createUploadVersion = async ( @@ -120,8 +134,309 @@ describe('ManagedFileIndexRepository host Artifact catalog', () => { } } }) + await client.uploadFile.update({ + where: { id: uploadId }, + data: { currentVersionId: versionId } + }) } + it('resolves default catalog entries from DB heads when ManagedFile projections are stale', async () => { + await createArtifactVersion('project-a', 'session-a', 'artifact-a', 'artifact-v1', 1) + await createArtifactVersion('project-a', 'session-a', 'artifact-a', 'artifact-v2', 2) + await createUploadVersion('project-a', 'session-b', 'upload-a', 'upload-v1') + await client.uploadVersion.create({ + data: { + id: 'upload-v2', + uploadFileId: 'upload-a', + versionNumber: 2, + state: 'ready', + contentStorageKey: 'uploads/project-a/session-b/upload-v2', + filename: 'upload-a-v2.pdf', + originalFilename: 'upload-a-v2.pdf', + contentType: 'application/pdf', + sizeBytes: 22n, + checksum: checksum('upload-v2'), + createdAt: new Date('2026-08-04T00:00:00.000Z') + } + }) + await client.uploadFile.update({ + where: { id: 'upload-a' }, + data: { currentVersionId: 'upload-v2' } + }) + await client.managedFile.createMany({ + data: [ + { + source: 'artifact', + sourceFileId: 'artifact-a', + sourceVersionId: 'artifact-v1', + checksum: checksum('artifact-v1'), + projectId: 'project-a', + sessionId: 'session-a', + displayName: 'stale-artifact.csv', + storageKey: 'artifact-a', + mimeType: 'text/plain', + sizeBytes: 1n, + sortAtMs: 1n + }, + { + source: 'upload', + sourceFileId: 'upload-a', + sourceVersionId: 'upload-v1', + checksum: checksum('upload-v1'), + projectId: 'project-a', + sessionId: 'session-b', + displayName: 'stale-upload.pdf', + storageKey: 'upload-a', + mimeType: 'text/plain', + sizeBytes: 1n, + sortAtMs: 2n + } + ] + }) + + await expect(repository.readHostArtifactCatalog({ projectId: 'project-a' })).resolves.toEqual([ + expect.objectContaining({ + source: 'upload', + versionId: 'upload-v2', + filename: 'upload-a-v2.pdf', + checksum: checksum('upload-v2'), + sizeBytes: 22 + }), + expect.objectContaining({ + source: 'artifact', + versionId: 'artifact-v2', + filename: 'artifact-a.csv', + checksum: checksum('artifact-v2'), + rootFrameId: 'root-artifact-v2' + }) + ]) + }) + + it('lists current DB heads when ManagedFile projections are missing', async () => { + await createArtifactVersion('project-a', 'session-a', 'artifact-a', 'artifact-v1', 1) + await createUploadVersion('project-a', 'session-b', 'upload-a', 'upload-v1') + + await expect(repository.readHostArtifactCatalog({ projectId: 'project-a' })).resolves.toEqual([ + expect.objectContaining({ + source: 'upload', + sourceFileId: 'upload-a', + versionId: 'upload-v1' + }), + expect.objectContaining({ + source: 'artifact', + sourceFileId: 'artifact-a', + versionId: 'artifact-v1' + }) + ]) + }) + + it('hides compatibility-failed generated heads without hiding legacy or user edits', async () => { + await createArtifactVersion( + 'project-a', + 'session-a', + 'catalog-failed', + 'catalog-failed-v1', + 1, + { managedVisibleAt: null } + ) + await client.artifactLineage.create({ + data: { + id: 'catalog-legacy', + projectId: 'project-a', + sessionId: 'session-a', + normalizedFilename: 'catalog-legacy.csv', + filename: 'catalog-legacy.csv' + } + }) + await client.artifactVersion.create({ + data: { + id: 'catalog-legacy-v1', + artifactId: 'catalog-legacy', + versionNumber: 1, + filename: 'catalog-legacy.csv', + originKind: 'legacy', + state: 'finalized', + contentStorageKey: 'artifacts/project-a/session-a/catalog-legacy-v1/content', + contentType: 'text/csv', + sizeBytes: 11n, + checksum: checksum('catalog-legacy-v1'), + createdAt: new Date('2026-08-02T00:00:00.000Z') + } + }) + await client.artifactLineage.update({ + where: { id: 'catalog-legacy' }, + data: { currentVersionId: 'catalog-legacy-v1' } + }) + + await client.artifactLineage.create({ + data: { + id: 'catalog-edit', + projectId: 'project-a', + sessionId: 'session-a', + normalizedFilename: 'catalog-edit.csv', + filename: 'catalog-edit.csv' + } + }) + await client.artifactVersion.create({ + data: { + id: 'catalog-edit-v1', + artifactId: 'catalog-edit', + versionNumber: 1, + filename: 'catalog-edit.csv', + originKind: 'legacy', + state: 'finalized', + contentStorageKey: 'artifacts/project-a/session-a/catalog-edit-v1/content', + contentType: 'text/csv', + sizeBytes: 12n, + checksum: checksum('catalog-edit-v1'), + createdAt: new Date('2026-08-02T00:00:00.000Z') + } + }) + await client.artifactVersion.create({ + data: { + id: 'catalog-edit-v2', + artifactId: 'catalog-edit', + versionNumber: 2, + filename: 'catalog-edit.csv', + originKind: 'user_edit', + basedOnVersionId: 'catalog-edit-v1', + storageTag: 'edit1234', + storedFilename: '{edit1234}_catalog-edit.csv', + state: 'finalized', + contentStorageKey: 'artifacts/project-a/session-a/catalog-edit-v2/content', + contentType: 'text/csv', + sizeBytes: 13n, + checksum: checksum('catalog-edit-v2'), + createdAt: new Date('2026-08-03T00:00:00.000Z') + } + }) + await client.artifactLineage.update({ + where: { id: 'catalog-edit' }, + data: { currentVersionId: 'catalog-edit-v2' } + }) + + const files = await repository.listFiles({ + projectId: 'project-a', + collection: { kind: 'sessionArtifacts', sessionId: 'session-a' }, + limit: 10 + }) + expect(files.items.map((item) => item.sourceFileId).sort()).toEqual([ + 'catalog-edit', + 'catalog-legacy' + ]) + + const search = await repository.searchArtifacts({ + primaryProjectId: 'project-a', + otherProjectIds: [], + filenameContains: 'catalog-', + primaryLimit: 10, + otherLimit: 0 + }) + expect(search.primary.items.map((item) => item.sourceFileId).sort()).toEqual([ + 'catalog-edit', + 'catalog-legacy' + ]) + + const hostCatalog = await repository.readHostArtifactCatalog({ projectId: 'project-a' }) + expect(hostCatalog.map((item) => item.sourceFileId).sort()).toEqual([ + 'catalog-edit', + 'catalog-legacy' + ]) + }) + + it('does not resurrect native heads across authoritative deletion membership gates', async () => { + await createArtifactVersion('project-a', 'session-a', 'artifact-a', 'artifact-v1') + await createUploadVersion('project-a', 'session-b', 'upload-a', 'upload-v1') + await client.managedFile.create({ + data: { + source: 'artifact', + sourceFileId: 'artifact-a', + sourceVersionId: 'artifact-v1', + checksum: checksum('artifact-v1'), + projectId: 'project-a', + sessionId: 'session-a', + displayName: 'artifact-a.csv', + storageKey: 'artifact-a', + mimeType: 'text/csv', + sizeBytes: 10n, + sortAtMs: 1n, + deletedAt: new Date('2026-08-05T00:00:00.000Z'), + deleteOperationId: 'delete-artifact-a' + } + }) + await client.managedFileSessionSync.create({ + data: { + projectId: 'project-a', + sessionId: 'session-b', + filesRevision: 1, + groupSortAtMs: 1n, + uploadCount: 1, + deletedAt: new Date('2026-08-05T00:00:00.000Z'), + deleteOperationId: 'delete-session-b' + } + }) + + const expectCatalogIds = async (expected: string[]): Promise => { + const files = await repository.listFiles({ + projectId: 'project-a', + collection: { kind: 'all' }, + limit: 10 + }) + expect(files.items.map((item) => item.sourceFileId).sort()).toEqual(expected) + const search = await repository.searchArtifacts({ + primaryProjectId: 'project-a', + otherProjectIds: [], + filenameContains: 'artifact-a', + primaryLimit: 10, + otherLimit: 0 + }) + expect(search.primary.items.map((item) => item.sourceFileId).sort()).toEqual( + expected.filter((id) => id.startsWith('artifact-')) + ) + const hostCatalog = await repository.readHostArtifactCatalog({ projectId: 'project-a' }) + expect(hostCatalog.map((item) => item.sourceFileId).sort()).toEqual(expected) + } + + await expectCatalogIds([]) + + await client.managedFile.deleteMany({ where: { projectId: 'project-a' } }) + await client.managedFileSessionSync.deleteMany({ where: { projectId: 'project-a' } }) + await client.project.create({ + data: { + id: 'project-a', + name: 'Archived project', + archivedAt: new Date('2026-08-05T00:00:00.000Z') + } + }) + await expectCatalogIds([]) + + await client.project.update({ where: { id: 'project-a' }, data: { archivedAt: null } }) + await client.projectDeletionIntent.create({ data: { projectId: 'project-a' } }) + await expectCatalogIds([]) + + await client.projectDeletionIntent.delete({ where: { projectId: 'project-a' } }) + await client.fileOriginSession.updateMany({ + where: { projectId: 'project-a' }, + data: { + state: 'deleting', + deletionOperationId: 'delete-origin', + retainedReviewIdsJson: '[]' + } + }) + await expectCatalogIds([]) + + await client.fileOriginSession.updateMany({ + where: { projectId: 'project-a' }, + data: { + state: 'deleted', + deletedAt: new Date('2026-08-05T00:00:00.000Z'), + deletionOperationId: null, + retainedReviewIdsJson: null + } + }) + await expectCatalogIds(['artifact-a', 'upload-a']) + }) + it('projects latest generated Artifacts and Uploads from the current Project catalog', async () => { await createArtifactVersion('project-a', 'session-a', 'artifact-a', 'artifact-version-a') await createUploadVersion('project-a', 'session-b', 'upload-a', 'upload-version-a') @@ -243,24 +558,14 @@ describe('ManagedFileIndexRepository host Artifact catalog', () => { }) it('projects producer provenance from the latest generated Version only', async () => { - await createArtifactVersion( - 'project-a', - 'session-a', - 'artifact-a', - 'artifact-a-v1', - 1, - 'frame-a', - 'shared-root' - ) - await createArtifactVersion( - 'project-a', - 'session-a', - 'artifact-a', - 'artifact-a-v2', - 2, - 'frame-b', - 'shared-root' - ) + await createArtifactVersion('project-a', 'session-a', 'artifact-a', 'artifact-a-v1', 1, { + agentFrameId: 'frame-a', + rootFrameId: 'shared-root' + }) + await createArtifactVersion('project-a', 'session-a', 'artifact-a', 'artifact-a-v2', 2, { + agentFrameId: 'frame-b', + rootFrameId: 'shared-root' + }) await client.managedFile.create({ data: { source: 'artifact', diff --git a/src/main/project-files/mutation-owner.ts b/src/main/project-files/mutation-owner.ts index 1345a980b..a94e9486a 100644 --- a/src/main/project-files/mutation-owner.ts +++ b/src/main/project-files/mutation-owner.ts @@ -363,6 +363,7 @@ class ProjectFilesMutationOwner { client.artifactLineage.findMany({ where: { projectId, sessionId }, include: { + currentVersion: true, versions: { where: { state: 'finalized' }, orderBy: [{ versionNumber: 'desc' }, { id: 'desc' }], @@ -373,6 +374,7 @@ class ProjectFilesMutationOwner { client.uploadFile.findMany({ where: { projectId, sessionId }, include: { + currentVersion: true, versions: { where: { state: 'ready' }, orderBy: [{ versionNumber: 'desc' }, { id: 'desc' }], @@ -382,8 +384,8 @@ class ProjectFilesMutationOwner { }) ]) const artifactFiles: IndexedFileInput[] = lineages.flatMap((lineage) => { - const version = lineage.versions[0] - return version + const version = lineage.currentVersion ?? lineage.versions[0] + return version?.state === 'finalized' ? [ { source: 'artifact' as const, @@ -404,9 +406,9 @@ class ProjectFilesMutationOwner { : [] }) const uploadFiles: IndexedFileInput[] = uploads.flatMap((upload) => { - const version = upload.versions[0] + const version = upload.currentVersion ?? upload.versions[0] const createdAt = version?.createdAt ?? version?.registeredAt - return version && createdAt + return version?.state === 'ready' && createdAt ? [ { source: 'upload' as const, diff --git a/src/main/project-files/mutation-projection.ts b/src/main/project-files/mutation-projection.ts index 6b14d39ff..75ab211d4 100644 --- a/src/main/project-files/mutation-projection.ts +++ b/src/main/project-files/mutation-projection.ts @@ -128,10 +128,11 @@ const isFileProjectionCurrent = async ( projectId: string, sessionId: string ): Promise => { - const [lineages, rows] = await Promise.all([ + const [lineages, uploads, rows] = await Promise.all([ client.artifactLineage.findMany({ where: { projectId, sessionId }, include: { + currentVersion: true, versions: { where: { state: 'finalized' }, orderBy: [{ versionNumber: 'desc' }, { id: 'desc' }], @@ -139,6 +140,17 @@ const isFileProjectionCurrent = async ( } } }), + client.uploadFile.findMany({ + where: { projectId, sessionId }, + include: { + currentVersion: true, + versions: { + where: { state: 'ready' }, + orderBy: [{ versionNumber: 'desc' }, { id: 'desc' }], + take: 1 + } + } + }), client.managedFile.findMany({ where: { projectId, sessionId, deletedAt: null }, select: { @@ -152,8 +164,8 @@ const isFileProjectionCurrent = async ( ]) const expectedArtifacts = new Map( lineages.flatMap((lineage) => { - const version = lineage.versions[0] - return version ? [[lineage.id, version.id] as const] : [] + const version = lineage.currentVersion ?? lineage.versions[0] + return version?.state === 'finalized' ? [[lineage.id, version.id] as const] : [] }) ) const projectedArtifacts = rows.filter( @@ -175,6 +187,24 @@ const isFileProjectionCurrent = async ( }) if (hasMismatchedLegacyUploadOwner) return false + const expectedUploads = new Map( + uploads.flatMap((upload) => { + const version = upload.currentVersion ?? upload.versions[0] + return version?.state === 'ready' ? [[upload.id, version.id] as const] : [] + }) + ) + const projectedOwnedUploads = rows.filter( + (row) => row.source === 'upload' && row.sourceVersionId !== null && row.sessionId === sessionId + ) + if ( + projectedOwnedUploads.length !== expectedUploads.size || + projectedOwnedUploads.some( + (row) => expectedUploads.get(row.sourceFileId) !== row.sourceVersionId + ) + ) { + return false + } + // A native Upload row is owned by its source Session, even when another Session references it. // Detect old derived rows that copied the referencing Session so one startup sync repairs their // locator scope instead of repeatedly sending unauthorized preview requests. @@ -243,11 +273,13 @@ const extractSessionFiles = async ( versions: { where: { id: upload.versionId, state: 'ready' }, take: 1 - } + }, + currentVersion: true } }) - const version = file?.versions[0] - if (!file || !version) { + const referencedVersion = file?.versions[0] + const version = file?.currentVersion ?? referencedVersion + if (!file || !referencedVersion || !version || version.state !== 'ready') { throw new Error(`Upload Version is unavailable: ${upload.versionId}`) } files.push({ @@ -299,6 +331,7 @@ const extractSessionFiles = async ( const lineages = await client.artifactLineage.findMany({ where: { projectId: session.projectId, sessionId: session.id }, include: { + currentVersion: true, versions: { where: { state: 'finalized' }, orderBy: [{ versionNumber: 'desc' }, { id: 'desc' }], @@ -309,8 +342,8 @@ const extractSessionFiles = async ( for (const lineage of lineages) { authoritativeArtifactIds.add(lineage.id) - const version = lineage.versions[0] - if (!version) continue + const version = lineage.currentVersion ?? lineage.versions[0] + if (!version || version.state !== 'finalized') continue const createdAtMs = BigInt(version.createdAt.getTime()) files.push({ source: 'artifact', diff --git a/src/main/project-files/query-owner.ts b/src/main/project-files/query-owner.ts index a74653f6b..f25f62854 100644 --- a/src/main/project-files/query-owner.ts +++ b/src/main/project-files/query-owner.ts @@ -1,5 +1,3 @@ -import { Prisma, type ManagedFile } from '@prisma/client' - import type { ArtifactGroupPage, GetProjectFilesOverviewRequest, @@ -14,18 +12,17 @@ import type { } from '../../shared/project-files' import type { ProjectFilesClientProvider } from './mutation-projection' import { - countMatchingArtifacts, decodeFileCursor, decodeGroupCursor, decodeSearchArtifactCursor, encodeCursor, - getMatchingOverviewCounts, - listMatchingArtifactGroups, - listMatchingArtifacts, - listMatchingFiles, - listOtherProjectArtifacts, + getAuthoritativeOverviewCounts, + listAuthoritativeArtifactGroups, + listAuthoritativeFiles, + listAuthoritativeManagedFiles, normalizeLimit, normalizeSearch, + queryAuthoritativeFiles, requireIdentifier, toOriginProjection, toProjectFileItem, @@ -50,16 +47,8 @@ class ProjectFilesQueryOwner { requireIdentifier(projectId, 'projectId') const search = normalizeSearch(rawSearch) const client = await this.getClient() - const [totalCount, uploadCount, artifactCount, artifactGroupCount] = search - ? await getMatchingOverviewCounts(client, projectId, search) - : await Promise.all([ - client.managedFile.count({ where: { projectId, deletedAt: null } }), - client.managedFile.count({ where: { projectId, source: 'upload', deletedAt: null } }), - client.managedFile.count({ where: { projectId, source: 'artifact', deletedAt: null } }), - client.managedFileSessionSync.count({ - where: { projectId, deletedAt: null, artifactCount: { gt: 0 } } - }) - ]) + const [totalCount, uploadCount, artifactCount, artifactGroupCount] = + await getAuthoritativeOverviewCounts(client, projectId, search) return { totalCount, @@ -100,41 +89,14 @@ class ProjectFilesQueryOwner { return { items: [], totalCount: 0 } } const cursor = request.cursor ? decodeFileCursor(request.cursor, normalizedRequest) : undefined - const where: Prisma.ManagedFileWhereInput = { - projectId: request.projectId, - ...(source ? { source } : {}), - deletedAt: null, - ...(sessionId !== undefined - ? { sessionId } - : search?.excludedSessionIds.length - ? { sessionId: { notIn: search.excludedSessionIds } } - : {}), - ...(cursor - ? { - OR: [ - { sortAtMs: { lt: BigInt(cursor.sortAtMs) } }, - { sortAtMs: BigInt(cursor.sortAtMs), seq: { lt: cursor.seq } } - ] - } - : {}) - } - const [rows, totalCount] = search - ? await listMatchingFiles(client, request.projectId, source, sessionId, search, cursor, limit) - : await Promise.all([ - client.managedFile.findMany({ - where, - orderBy: [{ sortAtMs: 'desc' }, { seq: 'desc' }], - take: limit + 1 - }), - client.managedFile.count({ - where: { - projectId: request.projectId, - ...(source ? { source } : {}), - deletedAt: null, - ...(sessionId !== undefined ? { sessionId } : {}) - } - }) - ]) + const [rows, totalCount] = await listAuthoritativeFiles(client, { + projectIds: [request.projectId], + source, + sessionId, + search, + cursor, + limit: limit + 1 + }) const pageRows = rows.slice(0, limit) const lastRow = pageRows.at(-1) const origins = await client.fileOriginSession.findMany({ @@ -191,27 +153,24 @@ class ProjectFilesQueryOwner { ? decodeSearchArtifactCursor(request.primaryCursor, request.primaryProjectId, search) : undefined const client = await this.getClient() - const excludedSessionIds = search?.excludedSessionIds ?? [] - const [primaryRows, primaryTotalCount, otherRows] = await Promise.all([ - listMatchingArtifacts( - client, - request.primaryProjectId, + const [primaryResult, otherRows] = await Promise.all([ + listAuthoritativeFiles(client, { + projectIds: [request.primaryProjectId], + source: 'artifact', search, - excludedSessionIds, cursor, - primaryLimit - ), - countMatchingArtifacts(client, request.primaryProjectId, search, excludedSessionIds), - request.otherLimit > 0 && otherProjectIds.length > 0 - ? listOtherProjectArtifacts( - client, - otherProjectIds, + limit: primaryLimit + 1 + }), + request.otherLimit > 0 + ? queryAuthoritativeFiles(client, { + projectIds: otherProjectIds, + source: 'artifact', search, - excludedSessionIds, - request.otherLimit - ) + limit: request.otherLimit + }) : Promise.resolve([]) ]) + const [primaryRows, primaryTotalCount] = primaryResult const primaryPageRows = primaryRows.slice(0, primaryLimit) const lastPrimaryRow = primaryPageRows.at(-1) const rows = [...primaryPageRows, ...otherRows] @@ -228,7 +187,7 @@ class ProjectFilesQueryOwner { const originsBySession = new Map( origins.map((origin) => [`${origin.projectId}:${origin.sessionId}`, origin]) ) - const toItem = (row: ManagedFile): ProjectFileItem => + const toItem = (row: (typeof rows)[number]): ProjectFileItem => toProjectFileItem( row, this.dataRoot, @@ -340,17 +299,7 @@ class ProjectFilesQueryOwner { : [] } - // ponytail: fuzzy search needs the whole current Project catalog; move filtering into SQLite - // only if measured Project sizes make this bounded metadata scan material. - const rows = await client.managedFile.findMany({ - where: { - projectId: request.projectId, - deletedAt: null, - sourceVersionId: { not: null }, - source: { in: ['artifact', 'upload'] } - }, - orderBy: [{ sortAtMs: 'desc' }, { seq: 'desc' }] - }) + const rows = await listAuthoritativeManagedFiles(client, [request.projectId]) const artifactVersionIds = rows.flatMap((row) => row.source === 'artifact' && row.sourceVersionId ? [row.sourceVersionId] : [] ) @@ -454,40 +403,12 @@ class ProjectFilesQueryOwner { const limit = normalizeLimit(request.limit) const search = normalizeSearch(request.search) const cursor = request.cursor ? decodeGroupCursor(request.cursor, request) : undefined - const groupWhere: Prisma.ManagedFileSessionSyncWhereInput = { + const [rows, totalCount] = await listAuthoritativeArtifactGroups(client, { projectId: request.projectId, - deletedAt: null, - artifactCount: { gt: 0 }, - ...(search?.excludedSessionIds.length - ? { sessionId: { notIn: search.excludedSessionIds } } - : {}) - } - const where: Prisma.ManagedFileSessionSyncWhereInput = { - ...groupWhere, - ...(cursor - ? { - OR: [ - { groupSortAtMs: { lt: BigInt(cursor.groupSortAtMs) } }, - { - groupSortAtMs: BigInt(cursor.groupSortAtMs), - sessionId: { lt: cursor.sessionId } - } - ] - } - : {}) - } - const [rows, totalCount] = search - ? await listMatchingArtifactGroups(client, request.projectId, search, cursor, limit) - : await Promise.all([ - client.managedFileSessionSync.findMany({ - where, - orderBy: [{ groupSortAtMs: 'desc' }, { sessionId: 'desc' }], - take: limit + 1 - }), - client.managedFileSessionSync.count({ - where: groupWhere - }) - ]) + search, + cursor, + limit: limit + 1 + }) const pageRows = rows.slice(0, limit) const lastRow = pageRows.at(-1) const origins = await client.fileOriginSession.findMany({ @@ -501,7 +422,7 @@ class ProjectFilesQueryOwner { return { items: pageRows.map((row) => ({ sessionId: row.sessionId, - artifactCount: toSafeCount(row.artifactCount, 'artifact group count'), + artifactCount: toSafeCount(row.artifactCount, 'catalog artifact group size'), ...toOriginProjection(originsBySession.get(row.sessionId)) })), totalCount, diff --git a/src/main/project-files/query-support.ts b/src/main/project-files/query-support.ts index d99f768bc..5db2e6bb1 100644 --- a/src/main/project-files/query-support.ts +++ b/src/main/project-files/query-support.ts @@ -49,19 +49,30 @@ type NormalizedSearch = { queryKey: string } -type SearchArtifactGroupRow = { - sessionId: string - groupSortAtMs: bigint - artifactCount: bigint +type CatalogCursor = { sortAtMs: string; seq: number } + +type AuthoritativeCatalogQuery = { + projectIds: string[] + source?: ProjectFileSource + sessionId?: string + search?: NormalizedSearch + cursor?: CatalogCursor + limit?: number } -type SearchOverviewRow = { +type AuthoritativeOverviewCounts = { totalCount: bigint uploadCount: bigint artifactCount: bigint artifactGroupCount: bigint } +type AuthoritativeArtifactGroupRow = { + sessionId: string + groupSortAtMs: bigint + artifactCount: bigint +} + const normalizeLimit = (limit: number): number => { if (!Number.isInteger(limit) || limit < 1 || limit > MAX_PAGE_LIMIT) { throw new Error(`Project files page limit must be between 1 and ${MAX_PAGE_LIMIT}.`) @@ -101,266 +112,345 @@ const normalizeExcludedSessionIds = (value: unknown): string[] => { const foldAsciiCase = (value: string): string => value.replace(/[A-Z]/g, (character) => character.toLowerCase()) -const filenameContainsPredicate = ( - displayNameColumn: Prisma.Sql, - search: NormalizedSearch | undefined -): Prisma.Sql => - search?.filenameContains - ? Prisma.sql`AND instr(lower(${displayNameColumn}), lower(${search.filenameContains})) > 0` - : Prisma.empty - -const excludedSessionIdsPredicate = ( - sessionIdColumn: Prisma.Sql, - excludedSessionIds: string[] -): Prisma.Sql => - excludedSessionIds.length > 0 - ? Prisma.sql`AND ${sessionIdColumn} NOT IN (${Prisma.join(excludedSessionIds)})` - : Prisma.empty - const requireIdentifier = (value: string, field: string): void => { if (!value.trim()) throw new Error(`Project files ${field} is required.`) } -const getMatchingOverviewCounts = async ( - client: ProjectFilesClient, - projectId: string, - search: NormalizedSearch -): Promise<[number, number, number, number]> => { - const rows = await client.$queryRaw(Prisma.sql` - SELECT - COUNT(file."seq") AS "totalCount", - COALESCE(SUM(CASE WHEN file."source" = 'upload' THEN 1 ELSE 0 END), 0) AS "uploadCount", - COALESCE(SUM(CASE WHEN file."source" = 'artifact' THEN 1 ELSE 0 END), 0) AS "artifactCount", - COUNT(DISTINCT CASE - WHEN file."source" = 'artifact' AND sync."sessionId" IS NOT NULL THEN file."sessionId" - END) AS "artifactGroupCount" +// Native lineage/currentVersion rows are the catalog authority. ManagedFile remains a rebuildable +// compatibility projection, so legacy rows participate only when no native logical identity exists. +const authoritativeCatalogCte = (projectIds: string[]): Prisma.Sql => { + const projectScopeRows = Prisma.join(projectIds.map((projectId) => Prisma.sql`(${projectId})`)) + return Prisma.sql` + WITH "CatalogProjectScope"("projectId") AS ( + VALUES ${projectScopeRows} + ), + "BlockedCatalogProject" AS ( + SELECT intent."projectId" AS "projectId" + FROM "ProjectDeletionIntent" AS intent + WHERE intent."projectId" IN (SELECT scope."projectId" FROM "CatalogProjectScope" AS scope) + + UNION + + SELECT project."id" AS "projectId" + FROM "Project" AS project + WHERE project."id" IN (SELECT scope."projectId" FROM "CatalogProjectScope" AS scope) + AND project."archivedAt" IS NOT NULL + ), + "BlockedCatalogSession" AS ( + SELECT sync."projectId" AS "projectId", sync."sessionId" AS "sessionId" + FROM "ManagedFileSessionSync" AS sync + WHERE sync."projectId" IN (SELECT scope."projectId" FROM "CatalogProjectScope" AS scope) + AND (sync."deletedAt" IS NOT NULL OR sync."deleteOperationId" IS NOT NULL) + + UNION + + SELECT origin."projectId" AS "projectId", origin."sessionId" AS "sessionId" + FROM "FileOriginSession" AS origin + WHERE origin."projectId" IN (SELECT scope."projectId" FROM "CatalogProjectScope" AS scope) + AND (origin."state" = 'deleting' OR origin."deletionOperationId" IS NOT NULL) + ), + "BlockedCatalogFile" AS ( + SELECT file."projectId" AS "projectId", file."source" AS "source", + file."sourceFileId" AS "sourceFileId" FROM "ManagedFile" AS file - LEFT JOIN "ManagedFileSessionSync" AS sync - ON sync."projectId" = file."projectId" - AND sync."sessionId" = file."sessionId" - AND sync."deletedAt" IS NULL - WHERE file."projectId" = ${projectId} - AND file."deletedAt" IS NULL - ${filenameContainsPredicate(Prisma.sql`file."displayName"`, search)} - ${excludedSessionIdsPredicate(Prisma.sql`file."sessionId"`, search.excludedSessionIds)} - `) - const counts = rows[0] + WHERE file."projectId" IN (SELECT scope."projectId" FROM "CatalogProjectScope" AS scope) + AND (file."deletedAt" IS NOT NULL OR file."deleteOperationId" IS NOT NULL) + ), + "AuthoritativeFile" AS ( + SELECT + CAST(lineage.rowid * 2 AS INTEGER) AS "seq", + 'artifact' AS "source", + lineage."id" AS "sourceFileId", + version."id" AS "sourceVersionId", + version."checksum" AS "checksum", + lineage."projectId" AS "projectId", + lineage."sessionId" AS "sessionId", + version."messageId" AS "messageId", + lineage."filename" AS "displayName", + version."contentStorageKey" AS "storageKey", + version."contentType" AS "mimeType", + version."sizeBytes" AS "sizeBytes", + CAST(version."createdAt" AS INTEGER) AS "mtimeMs", + CAST(version."createdAt" AS INTEGER) AS "sortAtMs", + lineage."createdAt" AS "createdAt", + lineage."updatedAt" AS "updatedAt", + NULL AS "deletedAt", + NULL AS "deleteOperationId" + FROM "ArtifactLineage" AS lineage + INNER JOIN "ArtifactVersion" AS version + ON version."artifactId" = lineage."id" + AND version."id" = lineage."currentVersionId" + WHERE lineage."projectId" IN (SELECT scope."projectId" FROM "CatalogProjectScope" AS scope) + AND version."state" IN ('pending', 'finalized') + AND (version."originKind" <> 'agent_generated' OR version."managedVisibleAt" IS NOT NULL) + AND NOT EXISTS ( + SELECT 1 FROM "BlockedCatalogProject" AS blocked + WHERE blocked."projectId" = lineage."projectId" + ) + AND NOT EXISTS ( + SELECT 1 FROM "BlockedCatalogSession" AS blocked + WHERE blocked."projectId" = lineage."projectId" + AND blocked."sessionId" = lineage."sessionId" + ) + AND NOT EXISTS ( + SELECT 1 FROM "BlockedCatalogFile" AS blocked + WHERE blocked."projectId" = lineage."projectId" + AND blocked."source" = 'artifact' + AND blocked."sourceFileId" = lineage."id" + ) + + UNION ALL - return [ - toSafeCount(counts?.totalCount ?? 0n, 'search result count'), - toSafeCount(counts?.uploadCount ?? 0n, 'upload search result count'), - toSafeCount(counts?.artifactCount ?? 0n, 'artifact search result count'), - toSafeCount(counts?.artifactGroupCount ?? 0n, 'artifact group count') - ] -} + SELECT + CAST(upload.rowid * 2 + 1 AS INTEGER) AS "seq", + 'upload' AS "source", + upload."id" AS "sourceFileId", + version."id" AS "sourceVersionId", + version."checksum" AS "checksum", + upload."projectId" AS "projectId", + upload."sessionId" AS "sessionId", + NULL AS "messageId", + COALESCE(NULLIF(version."originalFilename", ''), version."filename") AS "displayName", + version."contentStorageKey" AS "storageKey", + version."contentType" AS "mimeType", + version."sizeBytes" AS "sizeBytes", + CAST(COALESCE(version."createdAt", version."registeredAt") AS INTEGER) AS "mtimeMs", + CAST(COALESCE(version."createdAt", version."registeredAt") AS INTEGER) AS "sortAtMs", + upload."createdAt" AS "createdAt", + upload."updatedAt" AS "updatedAt", + NULL AS "deletedAt", + NULL AS "deleteOperationId" + FROM "UploadFile" AS upload + INNER JOIN "UploadVersion" AS version + ON version."uploadFileId" = upload."id" + AND version."id" = upload."currentVersionId" + WHERE upload."projectId" IN (SELECT scope."projectId" FROM "CatalogProjectScope" AS scope) + AND version."state" = 'ready' + AND NOT EXISTS ( + SELECT 1 FROM "BlockedCatalogProject" AS blocked + WHERE blocked."projectId" = upload."projectId" + ) + AND NOT EXISTS ( + SELECT 1 FROM "BlockedCatalogSession" AS blocked + WHERE blocked."projectId" = upload."projectId" + AND blocked."sessionId" = upload."sessionId" + ) + AND NOT EXISTS ( + SELECT 1 FROM "BlockedCatalogFile" AS blocked + WHERE blocked."projectId" = upload."projectId" + AND blocked."source" = 'upload' + AND blocked."sourceFileId" = upload."id" + ) + + UNION ALL -const countMatchingFiles = async ( - client: ProjectFilesClient, - projectId: string, - search: NormalizedSearch, - source?: ProjectFileSource, - sessionId?: string -): Promise => { - const sourcePredicate = source === undefined ? Prisma.empty : Prisma.sql`AND "source" = ${source}` - const sessionPredicate = - sessionId === undefined ? Prisma.empty : Prisma.sql`AND "sessionId" = ${sessionId}` - const rows = await client.$queryRaw>(Prisma.sql` - SELECT COUNT(*) AS "count" - FROM "ManagedFile" - WHERE "projectId" = ${projectId} - AND "deletedAt" IS NULL - ${sourcePredicate} - ${sessionPredicate} - ${filenameContainsPredicate(Prisma.sql`"displayName"`, search)} - ${excludedSessionIdsPredicate(Prisma.sql`"sessionId"`, search.excludedSessionIds)} - `) - return toSafeCount(rows[0]?.count ?? 0n, 'search result count') + SELECT + -file."seq" AS "seq", + file."source", file."sourceFileId", file."sourceVersionId", file."checksum", + file."projectId", file."sessionId", file."messageId", file."displayName", + file."storageKey", file."mimeType", file."sizeBytes", file."mtimeMs", file."sortAtMs", + file."createdAt", file."updatedAt", file."deletedAt", file."deleteOperationId" + FROM "ManagedFile" AS file + WHERE file."projectId" IN (SELECT scope."projectId" FROM "CatalogProjectScope" AS scope) + AND file."deletedAt" IS NULL + AND file."deleteOperationId" IS NULL + AND NOT EXISTS ( + SELECT 1 FROM "BlockedCatalogProject" AS blocked + WHERE blocked."projectId" = file."projectId" + ) + AND NOT EXISTS ( + SELECT 1 FROM "BlockedCatalogSession" AS blocked + WHERE blocked."projectId" = file."projectId" + AND blocked."sessionId" = file."sessionId" + ) + AND ( + (file."source" = 'artifact' AND NOT EXISTS ( + SELECT 1 FROM "ArtifactLineage" AS lineage + WHERE lineage."projectId" = file."projectId" + AND lineage."id" = file."sourceFileId" + AND lineage."currentVersionId" IS NOT NULL + )) + OR + (file."source" = 'upload' AND NOT EXISTS ( + SELECT 1 FROM "UploadFile" AS upload + WHERE upload."projectId" = file."projectId" + AND upload."id" = file."sourceFileId" + AND upload."currentVersionId" IS NOT NULL + )) + ) + ) +` } -const listMatchingFiles = async ( - client: ProjectFilesClient, - projectId: string, - source: ProjectFileSource | undefined, - sessionId: string | undefined, - search: NormalizedSearch, - cursor: FileCursor | undefined, - limit: number -): Promise<[ManagedFile[], number]> => { - const sourcePredicate = source === undefined ? Prisma.empty : Prisma.sql`AND "source" = ${source}` - const sessionPredicate = - sessionId === undefined ? Prisma.empty : Prisma.sql`AND "sessionId" = ${sessionId}` - const exclusionPredicate = excludedSessionIdsPredicate( - Prisma.sql`"sessionId"`, - search.excludedSessionIds - ) - const cursorPredicate = cursor - ? Prisma.sql`AND ("sortAtMs" < ${BigInt(cursor.sortAtMs)} OR ("sortAtMs" = ${BigInt(cursor.sortAtMs)} AND "seq" < ${cursor.seq}))` +const authoritativeCatalogPredicates = ( + query: Omit +): Prisma.Sql => { + const sourcePredicate = query.source + ? Prisma.sql`AND file."source" = ${query.source}` : Prisma.empty - const [rows, totalCount] = await Promise.all([ - client.$queryRaw(Prisma.sql` - SELECT - "seq", "source", "sourceFileId", "sourceVersionId", "checksum", - "projectId", "sessionId", "messageId", - "displayName", "storageKey", "mimeType", "sizeBytes", "mtimeMs", "sortAtMs", - "createdAt", "updatedAt", "deletedAt", "deleteOperationId" - FROM "ManagedFile" - WHERE "projectId" = ${projectId} - ${sourcePredicate} - AND "deletedAt" IS NULL - ${sessionPredicate} - ${filenameContainsPredicate(Prisma.sql`"displayName"`, search)} - ${exclusionPredicate} - ${cursorPredicate} - ORDER BY "sortAtMs" DESC, "seq" DESC - LIMIT ${limit + 1} - `), - countMatchingFiles(client, projectId, search, source, sessionId) - ]) - return [rows, totalCount] + const sessionPredicate = query.sessionId + ? Prisma.sql`AND file."sessionId" = ${query.sessionId}` + : Prisma.empty + const filenamePredicate = query.search?.filenameContains + ? Prisma.sql`AND instr(lower(file."displayName"), lower(${query.search.filenameContains})) > 0` + : Prisma.empty + const excludedSessionsPredicate = query.search?.excludedSessionIds.length + ? Prisma.sql`AND file."sessionId" NOT IN (${Prisma.join(query.search.excludedSessionIds)})` + : Prisma.empty + const cursorPredicate = query.cursor + ? Prisma.sql`AND ( + file."sortAtMs" < ${BigInt(query.cursor.sortAtMs)} + OR (file."sortAtMs" = ${BigInt(query.cursor.sortAtMs)} AND file."seq" < ${query.cursor.seq}) + )` + : Prisma.empty + return Prisma.sql` + ${sourcePredicate} + ${sessionPredicate} + ${filenamePredicate} + ${excludedSessionsPredicate} + ${cursorPredicate} + ` } -const listMatchingArtifacts = async ( +const normalizeCatalogRows = (rows: Array): ManagedFile[] => + rows.map((row) => ({ + ...row, + seq: typeof row.seq === 'bigint' ? toSafeNumber(row.seq, 'catalog sequence') : row.seq + })) + +const queryAuthoritativeFiles = async ( client: ProjectFilesClient, - projectId: string, - search: NormalizedSearch | undefined, - excludedSessionIds: string[], - cursor: SearchArtifactCursor | undefined, - limit: number + query: AuthoritativeCatalogQuery ): Promise => { - const filenamePredicate = filenameContainsPredicate(Prisma.sql`"displayName"`, search) - const exclusionPredicate = excludedSessionIdsPredicate( - Prisma.sql`"sessionId"`, - excludedSessionIds - ) - const cursorPredicate = cursor - ? Prisma.sql`AND ("sortAtMs" < ${BigInt(cursor.sortAtMs)} OR ("sortAtMs" = ${BigInt(cursor.sortAtMs)} AND "seq" < ${cursor.seq}))` - : Prisma.empty - - return client.$queryRaw(Prisma.sql` - SELECT - "seq", "source", "sourceFileId", "sourceVersionId", "checksum", - "projectId", "sessionId", "messageId", - "displayName", "storageKey", "mimeType", "sizeBytes", "mtimeMs", "sortAtMs", - "createdAt", "updatedAt", "deletedAt", "deleteOperationId" - FROM "ManagedFile" - WHERE "projectId" = ${projectId} - AND "source" = 'artifact' - AND "deletedAt" IS NULL - ${filenamePredicate} - ${exclusionPredicate} - ${cursorPredicate} - ORDER BY "sortAtMs" DESC, "seq" DESC - LIMIT ${limit + 1} + if (query.projectIds.length === 0) return [] + const predicates = authoritativeCatalogPredicates(query) + const limit = query.limit === undefined ? Prisma.empty : Prisma.sql`LIMIT ${query.limit}` + const rows = await client.$queryRaw>(Prisma.sql` + ${authoritativeCatalogCte(query.projectIds)} + SELECT file.* + FROM "AuthoritativeFile" AS file + WHERE 1 = 1 + ${predicates} + ORDER BY file."sortAtMs" DESC, file."seq" DESC + ${limit} `) + return normalizeCatalogRows(rows) } -const countMatchingArtifacts = async ( +const countAuthoritativeFiles = async ( client: ProjectFilesClient, - projectId: string, - search: NormalizedSearch | undefined, - excludedSessionIds: string[] + query: Omit ): Promise => { - const filenamePredicate = filenameContainsPredicate(Prisma.sql`"displayName"`, search) - const exclusionPredicate = excludedSessionIdsPredicate( - Prisma.sql`"sessionId"`, - excludedSessionIds - ) + if (query.projectIds.length === 0) return 0 + const predicates = authoritativeCatalogPredicates(query) const rows = await client.$queryRaw>(Prisma.sql` + ${authoritativeCatalogCte(query.projectIds)} SELECT COUNT(*) AS "count" - FROM "ManagedFile" - WHERE "projectId" = ${projectId} - AND "source" = 'artifact' - AND "deletedAt" IS NULL - ${filenamePredicate} - ${exclusionPredicate} + FROM "AuthoritativeFile" AS file + WHERE 1 = 1 + ${predicates} `) - return toSafeCount(rows[0]?.count ?? 0n, 'artifact search result count') + return toSafeCount(rows[0]?.count ?? 0n, 'catalog count') } -const listOtherProjectArtifacts = async ( +const getAuthoritativeOverviewCounts = async ( client: ProjectFilesClient, - projectIds: string[], - search: NormalizedSearch | undefined, - excludedSessionIds: string[], - limit: number -): Promise => { - const filenamePredicate = filenameContainsPredicate(Prisma.sql`"displayName"`, search) - const exclusionPredicate = excludedSessionIdsPredicate( - Prisma.sql`"sessionId"`, - excludedSessionIds - ) - - return client.$queryRaw(Prisma.sql` + projectId: string, + search: NormalizedSearch | undefined +): Promise<[number, number, number, number]> => { + const predicates = authoritativeCatalogPredicates({ search }) + const rows = await client.$queryRaw(Prisma.sql` + ${authoritativeCatalogCte([projectId])} SELECT - "seq", "source", "sourceFileId", "sourceVersionId", "checksum", - "projectId", "sessionId", "messageId", - "displayName", "storageKey", "mimeType", "sizeBytes", "mtimeMs", "sortAtMs", - "createdAt", "updatedAt", "deletedAt", "deleteOperationId" - FROM "ManagedFile" - WHERE "projectId" IN (${Prisma.join(projectIds)}) - AND "source" = 'artifact' - AND "deletedAt" IS NULL - ${filenamePredicate} - ${exclusionPredicate} - ORDER BY "sortAtMs" DESC, "seq" DESC - LIMIT ${limit} + COUNT(*) AS "totalCount", + COALESCE(SUM(CASE WHEN file."source" = 'upload' THEN 1 ELSE 0 END), 0) AS "uploadCount", + COALESCE(SUM(CASE WHEN file."source" = 'artifact' THEN 1 ELSE 0 END), 0) AS "artifactCount", + COUNT(DISTINCT CASE WHEN file."source" = 'artifact' THEN file."sessionId" END) + AS "artifactGroupCount" + FROM "AuthoritativeFile" AS file + WHERE 1 = 1 + ${predicates} `) + const counts = rows[0] + return [ + toSafeCount(counts?.totalCount ?? 0n, 'catalog total count'), + toSafeCount(counts?.uploadCount ?? 0n, 'catalog upload count'), + toSafeCount(counts?.artifactCount ?? 0n, 'catalog artifact count'), + toSafeCount(counts?.artifactGroupCount ?? 0n, 'catalog artifact group count') + ] } -const countMatchingArtifactGroups = async ( +const listAuthoritativeFiles = async ( client: ProjectFilesClient, - projectId: string, - search: NormalizedSearch -): Promise => { - const rows = await client.$queryRaw>(Prisma.sql` - SELECT COUNT(DISTINCT sync."sessionId") AS "count" - FROM "ManagedFileSessionSync" AS sync - INNER JOIN "ManagedFile" AS file - ON file."projectId" = sync."projectId" AND file."sessionId" = sync."sessionId" - WHERE sync."projectId" = ${projectId} - AND sync."deletedAt" IS NULL - AND file."source" = 'artifact' - AND file."deletedAt" IS NULL - ${filenameContainsPredicate(Prisma.sql`file."displayName"`, search)} - ${excludedSessionIdsPredicate(Prisma.sql`sync."sessionId"`, search.excludedSessionIds)} - `) - return toSafeCount(rows[0]?.count ?? 0n, 'artifact group count') -} + query: AuthoritativeCatalogQuery & { limit: number } +): Promise<[ManagedFile[], number]> => + Promise.all([ + queryAuthoritativeFiles(client, query), + countAuthoritativeFiles(client, { + projectIds: query.projectIds, + source: query.source, + sessionId: query.sessionId, + search: query.search + }) + ]) -const listMatchingArtifactGroups = async ( +const listAuthoritativeArtifactGroups = async ( client: ProjectFilesClient, - projectId: string, - search: NormalizedSearch, - cursor: GroupCursor | undefined, - limit: number -): Promise<[SearchArtifactGroupRow[], number]> => { - const cursorPredicate = cursor - ? Prisma.sql`AND (sync."groupSortAtMs" < ${BigInt(cursor.groupSortAtMs)} OR (sync."groupSortAtMs" = ${BigInt(cursor.groupSortAtMs)} AND sync."sessionId" < ${cursor.sessionId}))` + input: { + projectId: string + search: NormalizedSearch | undefined + cursor: GroupCursor | undefined + limit: number + } +): Promise<[AuthoritativeArtifactGroupRow[], number]> => { + const predicates = authoritativeCatalogPredicates({ source: 'artifact', search: input.search }) + const cursorPredicate = input.cursor + ? Prisma.sql`WHERE ( + groups."groupSortAtMs" < ${BigInt(input.cursor.groupSortAtMs)} + OR ( + groups."groupSortAtMs" = ${BigInt(input.cursor.groupSortAtMs)} + AND groups."sessionId" < ${input.cursor.sessionId} + ) + )` : Prisma.empty - return Promise.all([ - client.$queryRaw(Prisma.sql` - SELECT - sync."sessionId" AS "sessionId", - sync."groupSortAtMs" AS "groupSortAtMs", - COUNT(file."seq") AS "artifactCount" - FROM "ManagedFileSessionSync" AS sync - INNER JOIN "ManagedFile" AS file - ON file."projectId" = sync."projectId" AND file."sessionId" = sync."sessionId" - WHERE sync."projectId" = ${projectId} - AND sync."deletedAt" IS NULL - AND file."source" = 'artifact' - AND file."deletedAt" IS NULL - ${filenameContainsPredicate(Prisma.sql`file."displayName"`, search)} - ${excludedSessionIdsPredicate(Prisma.sql`sync."sessionId"`, search.excludedSessionIds)} - ${cursorPredicate} - GROUP BY sync."sessionId", sync."groupSortAtMs" - ORDER BY sync."groupSortAtMs" DESC, sync."sessionId" DESC - LIMIT ${limit + 1} + const [rows, totalCount] = await Promise.all([ + client.$queryRaw(Prisma.sql` + ${authoritativeCatalogCte([input.projectId])}, + "ArtifactGroup" AS ( + SELECT + file."sessionId" AS "sessionId", + MAX(file."sortAtMs") AS "groupSortAtMs", + COUNT(*) AS "artifactCount" + FROM "AuthoritativeFile" AS file + WHERE 1 = 1 + ${predicates} + GROUP BY file."sessionId" + ) + SELECT groups.* + FROM "ArtifactGroup" AS groups + ${cursorPredicate} + ORDER BY groups."groupSortAtMs" DESC, groups."sessionId" DESC + LIMIT ${input.limit} `), - countMatchingArtifactGroups(client, projectId, search) + (async () => { + const countRows = await client.$queryRaw>(Prisma.sql` + ${authoritativeCatalogCte([input.projectId])} + SELECT COUNT(DISTINCT file."sessionId") AS "count" + FROM "AuthoritativeFile" AS file + WHERE 1 = 1 + ${predicates} + `) + return toSafeCount(countRows[0]?.count ?? 0n, 'catalog artifact group count') + })() ]) + return [rows, totalCount] } +const listAuthoritativeManagedFiles = async ( + client: ProjectFilesClient, + projectIds: string[] +): Promise => queryAuthoritativeFiles(client, { projectIds }) + const encodeCursor = (cursor: FileCursor | GroupCursor | SearchArtifactCursor): string => Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64url') @@ -511,19 +601,18 @@ const toProjectFileItem = ( }) export { - countMatchingArtifacts, decodeFileCursor, decodeGroupCursor, decodeSearchArtifactCursor, encodeCursor, - getMatchingOverviewCounts, - listMatchingArtifactGroups, - listMatchingArtifacts, - listMatchingFiles, - listOtherProjectArtifacts, + getAuthoritativeOverviewCounts, + listAuthoritativeArtifactGroups, + listAuthoritativeFiles, + listAuthoritativeManagedFiles, normalizeLimit, normalizeSearch, requireIdentifier, + queryAuthoritativeFiles, toOriginProjection, toProjectFileItem, toSafeCount diff --git a/src/main/project-files/repository.test.ts b/src/main/project-files/repository.test.ts index 37ee8376f..6140d926d 100644 --- a/src/main/project-files/repository.test.ts +++ b/src/main/project-files/repository.test.ts @@ -251,6 +251,7 @@ describe('ManagedFileIndexRepository', () => { state: 'finalized', contentStorageKey: storageKey(storageRoot, contentPath), evidenceStorageKey: `artifacts/${PROJECT_ID}/${SESSION_ID}/.provenance/${lineageId}/versions/${id}/evidence.json`, + evidenceSchemaVersion: 1, contentType: 'image/png', sizeBytes: BigInt(versionNumber === 1 ? 9 : 12), checksum: checksumCharacter.repeat(64), @@ -406,6 +407,336 @@ describe('ManagedFileIndexRepository', () => { }) }) + it('keeps the DB Upload head as the default projection when Session references an older Version', async () => { + const uploadId = 'upload-versioned' + const versions = await Promise.all( + [1, 2, 3].map(async (versionNumber) => { + const id = `upload-version-${versionNumber}` + const content = `version ${versionNumber}\n` + const path = join( + storageRoot, + 'uploads', + PROJECT_ID, + SESSION_ID, + uploadId, + 'versions', + id, + 'content' + ) + await writeManagedFile(path, content) + return { + id, + versionNumber, + state: 'ready', + contentStorageKey: storageKey(storageRoot, path), + filename: 'notes.md', + originalFilename: 'notes.md', + contentType: 'text/markdown', + sizeBytes: BigInt(Buffer.byteLength(content)), + checksum: createHash('sha256').update(content).digest('hex') + } + }) + ) + await client.fileOriginSession.create({ + data: { projectId: PROJECT_ID, sessionId: SESSION_ID } + }) + await client.uploadFile.create({ + data: { + id: uploadId, + projectId: PROJECT_ID, + sessionId: SESSION_ID, + filename: 'notes.md', + originalFilename: 'notes.md', + currentVersionId: null, + versions: { create: versions } + } + }) + await client.uploadFile.update({ + where: { id: uploadId }, + data: { currentVersionId: versions[1]!.id } + }) + const session = createSession({ + messages: [ + { + id: 'message-upload-v1', + role: 'user', + content: 'Use the original upload', + status: 'complete', + eventIds: [], + uploads: [ + { + id: uploadId, + versionId: versions[0]!.id, + versionNumber: 1, + sessionId: SESSION_ID, + name: 'notes.md', + originalName: 'notes.md', + size: Number(versions[0]!.sizeBytes) + } + ], + createdAt: 1_710_000_000_100, + updatedAt: 1_710_000_000_200 + } + ] + }) + await repository.syncSession(session) + await client.uploadFile.update({ + where: { id: uploadId }, + data: { currentVersionId: versions[2]!.id } + }) + + await repository.syncSession({ ...session, filesRevision: 2 }) + + await expect( + client.managedFile.findFirstOrThrow({ where: { source: 'upload', sourceFileId: uploadId } }) + ).resolves.toMatchObject({ + sourceVersionId: versions[2]!.id, + storageKey: versions[2]!.contentStorageKey, + checksum: versions[2]!.checksum + }) + }) + + it('invalidates the filesRevision fast path when the native Upload head advances', async () => { + const uploadId = 'upload-head-fast-path' + const versions = await Promise.all( + [1, 2].map(async (versionNumber) => { + const id = `upload-head-fast-path-v${versionNumber}` + const content = `version ${versionNumber}\n` + const path = join( + storageRoot, + 'uploads', + PROJECT_ID, + SESSION_ID, + uploadId, + 'versions', + id, + 'content' + ) + await writeManagedFile(path, content) + return { + id, + versionNumber, + state: 'ready', + contentStorageKey: storageKey(storageRoot, path), + filename: 'fast-path.md', + originalFilename: 'fast-path.md', + contentType: 'text/markdown', + sizeBytes: BigInt(Buffer.byteLength(content)), + checksum: createHash('sha256').update(content).digest('hex') + } + }) + ) + await client.fileOriginSession.create({ + data: { projectId: PROJECT_ID, sessionId: SESSION_ID } + }) + await client.uploadFile.create({ + data: { + id: uploadId, + projectId: PROJECT_ID, + sessionId: SESSION_ID, + filename: 'fast-path.md', + originalFilename: 'fast-path.md', + versions: { create: versions } + } + }) + await client.uploadFile.update({ + where: { id: uploadId }, + data: { currentVersionId: versions[0]!.id } + }) + const session = createSession({ + filesRevision: 7, + messages: [ + { + id: 'message-fast-path', + role: 'user', + content: 'upload', + status: 'complete', + eventIds: [], + uploads: [ + { + id: uploadId, + versionId: versions[0]!.id, + versionNumber: 1, + sessionId: SESSION_ID, + name: 'fast-path.md', + originalName: 'fast-path.md', + size: Number(versions[0]!.sizeBytes) + } + ], + createdAt: 1_710_000_000_100, + updatedAt: 1_710_000_000_200 + } + ] + }) + await repository.syncSession(session) + await client.uploadFile.update({ + where: { id: uploadId }, + data: { currentVersionId: versions[1]!.id } + }) + + await expect(repository.syncSession(session)).resolves.toEqual(['upload']) + await expect( + client.managedFile.findFirstOrThrow({ where: { sourceFileId: uploadId } }) + ).resolves.toMatchObject({ + sourceVersionId: versions[1]!.id, + storageKey: versions[1]!.contentStorageKey + }) + }) + + it('paginates native Upload heads by authoritative session and head sort metadata', async () => { + await client.fileOriginSession.createMany({ + data: [ + { projectId: PROJECT_ID, sessionId: 'session-old' }, + { projectId: PROJECT_ID, sessionId: 'session-new' } + ] + }) + for (const input of [ + { + fileId: 'upload-old', + sessionId: 'session-old', + versionId: 'upload-old-v2', + createdAt: new Date('2026-08-13T00:00:00.000Z') + }, + { + fileId: 'upload-new', + sessionId: 'session-new', + versionId: 'upload-new-v2', + createdAt: new Date('2026-08-14T00:00:00.000Z') + } + ]) { + await client.uploadFile.create({ + data: { + id: input.fileId, + projectId: PROJECT_ID, + sessionId: input.sessionId, + filename: `${input.fileId}.txt`, + originalFilename: `${input.fileId}.txt`, + versions: { + create: { + id: input.versionId, + versionNumber: 2, + state: 'ready', + contentStorageKey: `uploads/${PROJECT_ID}/${input.sessionId}/${input.versionId}`, + filename: `${input.fileId}.txt`, + originalFilename: `${input.fileId}.txt`, + contentType: 'text/plain', + sizeBytes: 2n, + checksum: input.fileId === 'upload-new' ? 'a'.repeat(64) : 'b'.repeat(64), + createdAt: input.createdAt + } + } + } + }) + await client.uploadFile.update({ + where: { id: input.fileId }, + data: { currentVersionId: input.versionId } + }) + } + await client.managedFile.createMany({ + data: [ + { + source: 'upload', + sourceFileId: 'upload-new', + sourceVersionId: 'stale-new-v1', + projectId: PROJECT_ID, + sessionId: 'stale-session', + displayName: 'stale-new.txt', + storageKey: 'stale-new', + sizeBytes: 1n, + sortAtMs: 1n + }, + { + source: 'upload', + sourceFileId: 'upload-old', + sourceVersionId: 'stale-old-v1', + projectId: PROJECT_ID, + sessionId: 'stale-session', + displayName: 'stale-old.txt', + storageKey: 'stale-old', + sizeBytes: 1n, + sortAtMs: 9_999_999_999_999n + } + ] + }) + + const first = await repository.listFiles({ + projectId: PROJECT_ID, + collection: { kind: 'uploads' }, + limit: 1 + }) + const second = await repository.listFiles({ + projectId: PROJECT_ID, + collection: { kind: 'uploads' }, + limit: 1, + cursor: first.nextCursor + }) + + expect(first).toMatchObject({ + items: [ + { + sourceFileId: 'upload-new', + sourceVersionId: 'upload-new-v2', + sessionId: 'session-new', + name: 'upload-new.txt' + } + ], + totalCount: 2 + }) + expect(second).toMatchObject({ + items: [ + { + sourceFileId: 'upload-old', + sourceVersionId: 'upload-old-v2', + sessionId: 'session-old' + } + ], + totalCount: 2 + }) + }) + + it('bounds large authoritative catalog pages inside SQLite on every cursor request', async () => { + await client.fileOriginSession.create({ + data: { projectId: PROJECT_ID, sessionId: 'large-session' } + }) + await client.managedFile.createMany({ + data: Array.from({ length: 125 }, (_, index) => ({ + source: 'artifact', + sourceFileId: `large-artifact-${index.toString().padStart(3, '0')}`, + projectId: PROJECT_ID, + sessionId: 'large-session', + displayName: `large-${index.toString().padStart(3, '0')}.txt`, + storageKey: `large/${index}`, + sizeBytes: 1n, + sortAtMs: BigInt(index) + })) + }) + const queryRaw = vi.spyOn(client, '$queryRaw') + + const first = await repository.listFiles({ + projectId: PROJECT_ID, + collection: { kind: 'sessionArtifacts', sessionId: 'large-session' }, + limit: 100 + }) + const second = await repository.listFiles({ + projectId: PROJECT_ID, + collection: { kind: 'sessionArtifacts', sessionId: 'large-session' }, + cursor: first.nextCursor, + limit: 100 + }) + + expect(first.items).toHaveLength(100) + expect(second.items).toHaveLength(25) + expect(new Set([...first.items, ...second.items].map((item) => item.sourceFileId)).size).toBe( + 125 + ) + const sqlCalls = queryRaw.mock.calls.map(([query]) => + 'strings' in (query as object) + ? (query as { strings: readonly string[] }).strings.join('?') + : String(query) + ) + expect(sqlCalls.filter((sql) => /ORDER BY[\s\S]+LIMIT/u.test(sql))).toHaveLength(2) + }) + it('repairs a native Upload projection that copied the referencing Session scope', async () => { const sourceSessionId = 'session-source' const uploadId = 'upload-cross-session' @@ -717,6 +1048,7 @@ describe('ManagedFileIndexRepository', () => { contentStorageKey: storageKey(storageRoot, artifactPath), evidenceStorageKey: 'artifacts/project-a/session-a/.provenance/artifact-lineage-inactive/versions/artifact-version-inactive/evidence.json', + evidenceSchemaVersion: 1, contentType: 'text/plain', sizeBytes: BigInt(Buffer.byteLength('inactive artifact')), checksum: createHash('sha256').update('inactive artifact').digest('hex'), @@ -2276,9 +2608,11 @@ describe('ManagedFileIndexRepository', () => { promptMessageId: 'prompt-1', messageId: 'message-1', state: 'finalized', + managedVisibleAt: new Date('2026-07-27T11:59:59.000Z'), contentStorageKey: storageKey(storageRoot, artifactPath), evidenceStorageKey: 'artifacts/project-a/session-a/.provenance/artifact-lineage-1/versions/artifact-version-1/evidence.json', + evidenceSchemaVersion: 1, contentType: 'text/plain', sizeBytes: 6n, checksum: 'a'.repeat(64), @@ -2286,10 +2620,56 @@ describe('ManagedFileIndexRepository', () => { evidenceChecksum: 'c'.repeat(64) } }) + await client.artifactLineage.update({ + where: { id: 'artifact-lineage-1' }, + data: { currentVersionId: 'artifact-version-1' } + }) // Session JSON is gone and the derived row is accidentally lost. SQLite Version authority must // be sufficient to recreate Project Files without reconstructing identity from a filename/path. await client.managedFile.deleteMany({ where: { projectId: PROJECT_ID } }) + await expect( + repository.listFiles({ + projectId: PROJECT_ID, + collection: { kind: 'sessionArtifacts', sessionId: SESSION_ID }, + limit: 20 + }) + ).resolves.toMatchObject({ + items: [ + { + sourceFileId: 'artifact-lineage-1', + sourceVersionId: 'artifact-version-1', + checksum: 'a'.repeat(64) + } + ], + totalCount: 1 + }) + await expect( + repository.searchArtifacts({ + primaryProjectId: PROJECT_ID, + otherProjectIds: [], + filenameContains: 'result', + primaryLimit: 10, + otherLimit: 0 + }) + ).resolves.toMatchObject({ + primary: { + items: [ + { + sourceFileId: 'artifact-lineage-1', + sourceVersionId: 'artifact-version-1' + } + ], + totalCount: 1 + } + }) + await expect( + repository.listArtifactGroups({ projectId: PROJECT_ID, limit: 20 }) + ).resolves.toMatchObject({ + items: [{ sessionId: SESSION_ID, artifactCount: 1 }], + totalCount: 1 + }) + await repository.reconcileActiveSessions([]) await expect(repository.getOverview(PROJECT_ID)).resolves.toMatchObject({ diff --git a/src/main/projects/project-owned-data.catalog.ts b/src/main/projects/project-owned-data.catalog.ts index cdfc5f081..dfc359c25 100644 --- a/src/main/projects/project-owned-data.catalog.ts +++ b/src/main/projects/project-owned-data.catalog.ts @@ -258,7 +258,8 @@ const PROJECT_OWNED_DATA_CATALOG: readonly ProjectOwnedDataCatalogEntry[] = [ 'ArtifactLineage', 'UploadFile', 'ArtifactMessageSnapshot', - 'ArtifactVersionInput' + 'ArtifactVersionInput', + 'ManagedFileVersionWriteOperation' ], prismaModels: [ { @@ -312,6 +313,10 @@ const PROJECT_OWNED_DATA_CATALOG: readonly ProjectOwnedDataCatalogEntry[] = [ onDelete: 'Restrict' } ] + }, + { + name: 'ManagedFileVersionWriteOperation', + ownerFields: [requiredOwner('projectId')] } ], policy: { diff --git a/src/main/session-persistence/artifact-finalization-recovery.integration.test.ts b/src/main/session-persistence/artifact-finalization-recovery.integration.test.ts index 37108e40f..5df368cfa 100644 --- a/src/main/session-persistence/artifact-finalization-recovery.integration.test.ts +++ b/src/main/session-persistence/artifact-finalization-recovery.integration.test.ts @@ -15,6 +15,7 @@ import type { PersistedChatSession } from '../../shared/session-persistence' import { createPngInlineSource } from '../artifacts/artifact-test-fixtures' import { ProvenanceMessageSnapshotRepository } from '../artifacts/provenance-message-snapshot' import { ArtifactProvenanceRepository } from '../artifacts/provenance-repository' +import { requireAgentArtifactVersion } from '../artifacts/provenance-version-kind' import { ArtifactRepository } from '../artifacts/repository' import { ManagedFileIndexRepository } from '../project-files/repository' import { createProjectDbClient, migrateApplicationDatabase } from '../projects/prisma-client' @@ -240,9 +241,9 @@ describe('artifact finalization startup recovery', () => { it('keeps pending bytes in place when producer evidence is available but its snapshot is missing', async () => { const compatibility = new ArtifactRepository(storageRoot) const { provenance, version } = await prepareRecovery(compatibility) - const persisted = await client.artifactVersion.findUniqueOrThrow({ - where: { id: version.versionId } - }) + const persisted = requireAgentArtifactVersion( + await client.artifactVersion.findUniqueOrThrow({ where: { id: version.versionId } }) + ) const evidence = JSON.stringify({ ...(JSON.parse(persisted.evidenceJson) as object), producer: { state: 'available' } @@ -379,6 +380,100 @@ describe('artifact finalization startup recovery', () => { ).resolves.toEqual([expect.objectContaining({ name: 'result.png' })]) }) + it('keeps an existing Files tile on its visible head until compatibility finalization succeeds', async () => { + let failDirectorySync = false + const compatibility = new ArtifactRepository(storageRoot, { + syncFile: async () => undefined, + syncDirectory: async () => { + if (failDirectorySync) throw new Error('compatibility storage is read-only') + } + }) + const { provenance, version } = await prepareRecovery(compatibility) + const visibleBytes = Buffer.from('previous visible artifact') + const visibleVersionId = 'artifact-visible-v0' + const visibleStorageKey = + 'artifacts/project-1/session-1/.provenance/artifact-visible/versions/artifact-visible-v0/content' + const visiblePath = join(storageRoot, ...visibleStorageKey.split('/')) + await mkdir(dirname(visiblePath), { recursive: true }) + await writeFile(visiblePath, visibleBytes) + await client.artifactVersion.create({ + data: { + id: visibleVersionId, + artifactId: version.artifactId, + versionNumber: 0, + filename: 'result.png', + originKind: 'legacy', + state: 'finalized', + contentStorageKey: visibleStorageKey, + sizeBytes: BigInt(visibleBytes.byteLength), + checksum: createHash('sha256').update(visibleBytes).digest('hex') + } + }) + await client.artifactLineage.update({ + where: { id: version.artifactId }, + data: { currentVersionId: visibleVersionId } + }) + await client.managedFile.create({ + data: { + source: 'artifact', + sourceFileId: version.artifactId, + sourceVersionId: visibleVersionId, + checksum: createHash('sha256').update(visibleBytes).digest('hex'), + projectId: PROJECT_ID, + sessionId: SESSION_ID, + displayName: 'result.png', + storageKey: visibleStorageKey, + sizeBytes: BigInt(visibleBytes.byteLength), + mtimeMs: BigInt(1), + sortAtMs: BigInt(1) + } + }) + failDirectorySync = true + const coordinator = new SessionPersistenceCoordinator( + sessions, + files, + undefined, + undefined, + undefined, + provenance + ) + + await coordinator.loadAll() + + await expect( + client.artifactLineage.findUniqueOrThrow({ where: { id: version.artifactId } }) + ).resolves.toMatchObject({ currentVersionId: visibleVersionId }) + await expect( + client.managedFile.findUniqueOrThrow({ + where: { + projectId_source_sourceFileId: { + projectId: PROJECT_ID, + source: 'artifact', + sourceFileId: version.artifactId + } + } + }) + ).resolves.toMatchObject({ sourceVersionId: visibleVersionId }) + + failDirectorySync = false + await coordinator.loadAll() + + await expect( + client.artifactLineage.findUniqueOrThrow({ where: { id: version.artifactId } }) + ).resolves.toMatchObject({ currentVersionId: version.versionId }) + await expect( + client.managedFile.findUniqueOrThrow({ + where: { + projectId_source_sourceFileId: { + projectId: PROJECT_ID, + source: 'artifact', + sourceFileId: version.artifactId + } + } + }) + ).resolves.toMatchObject({ sourceVersionId: version.versionId }) + }) + it('moves compatibility bytes for a finalized Version already linked to the active Message', async () => { const compatibility = new ArtifactRepository(storageRoot) const { provenance, version, context } = await prepareRecovery(compatibility) diff --git a/src/main/skills/skill-archive-sniffer.test.ts b/src/main/skills/skill-archive-sniffer.test.ts index 716607fa2..c1e2409d2 100644 --- a/src/main/skills/skill-archive-sniffer.test.ts +++ b/src/main/skills/skill-archive-sniffer.test.ts @@ -5,7 +5,11 @@ import { deflateRawSync } from 'node:zlib' import { describe, expect, it } from 'vitest' -import { inspectOuterArchive, isImportableSkillArchivePath } from './skill-archive-sniffer' +import { + inspectOuterArchive, + isImportableSkillArchive, + isImportableSkillArchivePath +} from './skill-archive-sniffer' import { UserSkillRepository } from './user-skill-repository' type ZipInput = { path: string; content: Buffer; method?: number } @@ -102,6 +106,22 @@ const incompressibleBytes = (size: number): Buffer => { } describe('isImportableSkillArchivePath', () => { + it('classifies an importable Skill through an anchored reader', async () => { + const archive = buildZip([ + { + path: 'reader-skill/SKILL.md', + content: Buffer.from('---\nname: Reader Skill\ndescription: From a lease.\n---\nRun it.') + } + ]) + + await expect( + isImportableSkillArchive({ + size: archive.byteLength, + read: async (position, length) => archive.subarray(position, position + length) + }) + ).resolves.toBe(true) + }) + it('finds a named Skill manifest without inflating unrelated large entries', async () => { const archive = buildZip([ { path: 'paper-finder/assets/model.bin', content: Buffer.alloc(2 * 1024 * 1024), method: 0 }, diff --git a/src/main/skills/skill-archive-sniffer.ts b/src/main/skills/skill-archive-sniffer.ts index 2cd6c8bed..b30e711c3 100644 --- a/src/main/skills/skill-archive-sniffer.ts +++ b/src/main/skills/skill-archive-sniffer.ts @@ -839,6 +839,15 @@ const inspectOuterArchive = async ( return false } +const isImportableSkillArchive = async (reader: ArchiveReader): Promise => { + try { + if (reader.size > SKILL_IMPORT_LIMITS.maxBundleBytes) return false + return await inspectOuterArchive(reader) + } catch { + return false + } +} + // Classifies a ZIP without loading the whole upload. Central records and entry validation are streamed; // validated bodies are discarded, while only selected frontmatter and one importer-supported nested // archive are retained under the same caps as full discovery. Any ambiguity fails closed to the ordinary @@ -848,8 +857,7 @@ const isImportableSkillArchivePath = async (filePath: string): Promise try { handle = await open(filePath, 'r') const { size } = await handle.stat() - if (size > SKILL_IMPORT_LIMITS.maxBundleBytes) return false - return await inspectOuterArchive(fileReader(handle, size)) + return await isImportableSkillArchive(fileReader(handle, size)) } catch { return false } finally { @@ -857,4 +865,5 @@ const isImportableSkillArchivePath = async (filePath: string): Promise } } -export { inspectOuterArchive, isImportableSkillArchivePath } +export { inspectOuterArchive, isImportableSkillArchive, isImportableSkillArchivePath } +export type { ArchiveReader } diff --git a/src/main/storage/provenance-migration-validation.test.ts b/src/main/storage/provenance-migration-validation.test.ts index bc07600d9..efea5a0eb 100644 --- a/src/main/storage/provenance-migration-validation.test.ts +++ b/src/main/storage/provenance-migration-validation.test.ts @@ -231,6 +231,7 @@ describe('validateProvenanceMigrationState', () => { state: 'pending', contentStorageKey: `${versionRoot}/content`, evidenceStorageKey: `${versionRoot}/evidence.json`, + evidenceSchemaVersion: 1, contentType: 'text/plain', sizeBytes: BigInt(Buffer.byteLength(artifactContent)), checksum: artifactChecksum, @@ -301,6 +302,7 @@ describe('validateProvenanceMigrationState', () => { 'artifacts/project-2/session-2/.provenance/artifact-1/versions/version-1/content', evidenceStorageKey: 'artifacts/project-1/session-1/.provenance/artifact-1/versions/version-1/evidence.json', + evidenceSchemaVersion: 1, contentType: 'text/plain', sizeBytes: 0, checksum: sha256(''), diff --git a/src/main/storage/provenance-migration-validation.ts b/src/main/storage/provenance-migration-validation.ts index 8b1789b41..d90dc0746 100644 --- a/src/main/storage/provenance-migration-validation.ts +++ b/src/main/storage/provenance-migration-validation.ts @@ -8,6 +8,7 @@ import { NOTEBOOK_RUN_FILE } from '../../shared/notebook' import { normalizeSessionFile } from '../../shared/session-persistence' import { operationJournalPath, RuntimeOperationJournal } from '../notebook/operation-journal' import { createProjectDbClient } from '../projects/prisma-client' +import { requireAgentArtifactVersion } from '../artifacts/provenance-version-kind' const SHA256_PATTERN = /^[a-f0-9]{64}$/ const storageKey = (...segments: string[]): string => segments.join('/') @@ -294,31 +295,29 @@ const validateSqliteStore = async (dataRoot: string, authorityRoot: string): Pro if (version.state === 'staging') { throw new Error(`Unfinished Artifact staging blocks migration: ${version.id}`) } - if ( - createHash('sha256').update(version.evidenceJson).digest('hex') !== - version.evidenceChecksum - ) { - throw new Error(`Artifact canonical evidence checksum mismatch: ${version.id}`) + const agentVersion = + version.originKind === 'agent_generated' + ? requireAgentArtifactVersion(version) + : undefined + const versionRoot = agentVersion + ? storageKey( + 'artifacts', + agentVersion.artifact.projectId, + agentVersion.artifact.sessionId, + '.provenance', + agentVersion.artifactId, + 'versions', + agentVersion.id + ) + : undefined + if (agentVersion && versionRoot) { + assertStorageKey( + agentVersion.contentStorageKey, + storageKey(versionRoot, 'content'), + `Artifact ${agentVersion.id} content` + ) } - const versionRoot = storageKey( - 'artifacts', - version.artifact.projectId, - version.artifact.sessionId, - '.provenance', - version.artifactId, - 'versions', - version.id - ) - assertStorageKey( - version.contentStorageKey, - storageKey(versionRoot, 'content'), - `Artifact ${version.id} content` - ) - assertStorageKey( - version.evidenceStorageKey, - storageKey(versionRoot, 'evidence.json'), - `Artifact ${version.id} evidence` - ) + const contentPath = resolveManagedStorageKey(dataRoot, version.contentStorageKey) if ( (await stat(contentPath)).size !== Number(version.sizeBytes) || @@ -326,27 +325,39 @@ const validateSqliteStore = async (dataRoot: string, authorityRoot: string): Pro ) { throw new Error(`Artifact SQLite content checksum mismatch: ${version.id}`) } - const evidencePath = resolveManagedStorageKey(dataRoot, version.evidenceStorageKey) - if ((await readFile(evidencePath, 'utf8')) !== version.evidenceJson) { - throw new Error(`Artifact evidence mirror mismatch: ${version.id}`) + if (!agentVersion || !versionRoot) continue + if ( + createHash('sha256').update(agentVersion.evidenceJson).digest('hex') !== + agentVersion.evidenceChecksum + ) { + throw new Error(`Artifact canonical evidence checksum mismatch: ${agentVersion.id}`) + } + assertStorageKey( + agentVersion.evidenceStorageKey, + storageKey(versionRoot, 'evidence.json'), + `Artifact ${agentVersion.id} evidence` + ) + const evidencePath = resolveManagedStorageKey(dataRoot, agentVersion.evidenceStorageKey) + if ((await readFile(evidencePath, 'utf8')) !== agentVersion.evidenceJson) { + throw new Error(`Artifact evidence mirror mismatch: ${agentVersion.id}`) } - if (version.executionSnapshotJson) { + if (agentVersion.executionSnapshotJson) { assertStorageKey( - version.executionSnapshotStorageKey ?? '', + agentVersion.executionSnapshotStorageKey ?? '', storageKey(versionRoot, 'execution.json'), - `Artifact ${version.id} execution` + `Artifact ${agentVersion.id} execution` ) if ( - !version.executionSnapshotChecksum || - !version.executionSnapshotStorageKey || - createHash('sha256').update(version.executionSnapshotJson).digest('hex') !== - version.executionSnapshotChecksum || + !agentVersion.executionSnapshotChecksum || + !agentVersion.executionSnapshotStorageKey || + createHash('sha256').update(agentVersion.executionSnapshotJson).digest('hex') !== + agentVersion.executionSnapshotChecksum || (await readFile( - resolveManagedStorageKey(dataRoot, version.executionSnapshotStorageKey), + resolveManagedStorageKey(dataRoot, agentVersion.executionSnapshotStorageKey), 'utf8' - )) !== version.executionSnapshotJson + )) !== agentVersion.executionSnapshotJson ) { - throw new Error(`Artifact execution mirror mismatch: ${version.id}`) + throw new Error(`Artifact execution mirror mismatch: ${agentVersion.id}`) } } } diff --git a/src/main/uploads/atomic-no-replace-publisher.capability.test.ts b/src/main/uploads/atomic-no-replace-publisher.capability.test.ts new file mode 100644 index 000000000..35b6edbf9 --- /dev/null +++ b/src/main/uploads/atomic-no-replace-publisher.capability.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest' + +import { managedFileVersionNativeCapability } from './atomic-no-replace-publisher' + +describe('managed file version native capability', () => { + it('reports NATIVE_WRITE_REQUIRED when the native binding cannot load', () => { + expect( + managedFileVersionNativeCapability(() => { + throw Object.assign(new Error('native module missing'), { code: 'MODULE_NOT_FOUND' }) + }) + ).toEqual({ + available: false, + reason: 'NATIVE_WRITE_REQUIRED', + readFallbackAvailable: false + }) + }) + + it('reports NATIVE_WRITE_REQUIRED when the native binding cannot publish anchored writes', () => { + const completeBinding = { + supportsAnchoredWrites: false, + publishNoReplace: () => undefined, + writeAndPublishNoReplace: () => undefined, + readFile: () => Buffer.alloc(0), + readFileBounded: () => Buffer.alloc(0), + publishVerifiedNoReplace: () => undefined, + verifyFile: () => true, + statFile: () => ({ sizeBytes: 0 }), + removeFile: () => false, + listDirectory: () => [] + } + + expect(managedFileVersionNativeCapability(() => completeBinding)).toEqual({ + available: false, + reason: 'NATIVE_WRITE_REQUIRED', + readFallbackAvailable: true + }) + }) + + it('rejects a corrupt binding shape instead of reporting partial support', () => { + expect(managedFileVersionNativeCapability(() => ({ readFile: () => Buffer.alloc(0) }))).toEqual( + { + available: false, + reason: 'NATIVE_WRITE_REQUIRED', + readFallbackAvailable: false + } + ) + }) + + it('reports the compiled binding capability for the current platform', () => { + expect(managedFileVersionNativeCapability()).toEqual( + process.platform === 'win32' + ? { + available: false, + reason: 'NATIVE_WRITE_REQUIRED', + readFallbackAvailable: true + } + : { available: true, readFallbackAvailable: false } + ) + }) +}) diff --git a/src/main/uploads/atomic-no-replace-publisher.test.ts b/src/main/uploads/atomic-no-replace-publisher.test.ts index 54f0f5776..64bbadff5 100644 --- a/src/main/uploads/atomic-no-replace-publisher.test.ts +++ b/src/main/uploads/atomic-no-replace-publisher.test.ts @@ -1,32 +1,55 @@ import { createRequire } from 'node:module' -import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises' +import { createHash } from 'node:crypto' +import { spawn } from 'node:child_process' +import { access, mkdir, mkdtemp, readFile, rename, rm, symlink, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { basename, join } from 'node:path' +import { basename, join, relative } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' import { publishNoReplace } from './atomic-no-replace-publisher' const require = createRequire(import.meta.url) -const nativeBindingAvailable = (() => { - try { - require('@aipoch/safe-file-publisher-native') - return true - } catch { - return false - } -})() - let cleanupRoot: string | undefined +const waitForPath = async (path: string, timeoutMs = 5_000): Promise => { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + try { + await access(path) + return + } catch { + await new Promise((resolvePoll) => setTimeout(resolvePoll, 5)) + } + } + throw new Error(`Timed out waiting for ${path}`) +} + afterEach(async () => { if (cleanupRoot) await rm(cleanupRoot, { recursive: true, force: true }) cleanupRoot = undefined }) -describe.skipIf(!nativeBindingAvailable)('atomic no-replace publisher', () => { +describe('atomic no-replace publisher', () => { + it('publishes from the already-open verified descriptor on every POSIX platform', async () => { + const source = await readFile( + new URL( + '../../../packages/safe-file-publisher-native/src/safe_file_publisher_native.cc', + import.meta.url + ), + 'utf8' + ) + + expect(source).toMatch( + /linkat\(\s*file_fd,\s*"",\s*parent_fd,\s*destination_name\.c_str\(\),\s*AT_EMPTY_PATH\s*\)/ + ) + expect( + source.match(/fclonefileat\(file_fd, parent_fd, destination_name\.c_str\(\), 0\)/g) + ).toHaveLength(2) + }) + it('reports publication capabilities for a local storage root', async () => { cleanupRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-')) - const binding = require('@aipoch/safe-file-publisher-native') as { + const binding = require('../../../packages/safe-file-publisher-native') as { inspectPath: (path: string) => { isRemote: boolean; supportsHardLinks: boolean } } @@ -68,4 +91,443 @@ describe.skipIf(!nativeBindingAvailable)('atomic no-replace publisher', () => { code: 'ENOENT' }) }) + + it('creates, reads, and removes a temporary file through anchored no-follow handles', async () => { + cleanupRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-')) + const binding = require('../../../packages/safe-file-publisher-native') as { + writeAndPublishNoReplace: ( + rootPath: string, + relativeParentPath: string, + temporaryName: string, + destinationName: string, + bytes: Buffer + ) => void + readFile: (rootPath: string, relativeParentPath: string, name: string) => Buffer + statFile: ( + rootPath: string, + relativeParentPath: string, + name: string + ) => { + sizeBytes: number + } + removeFile: (rootPath: string, relativeParentPath: string, name: string) => boolean + } + const parentPath = join(cleanupRoot, 'new', 'nested') + const relativeParentPath = relative(cleanupRoot, parentPath) + + binding.writeAndPublishNoReplace( + cleanupRoot, + relativeParentPath, + 'content.tmp', + 'content', + Buffer.from('safe') + ) + + expect(binding.readFile(cleanupRoot, relativeParentPath, 'content')).toEqual( + Buffer.from('safe') + ) + expect(binding.statFile(cleanupRoot, relativeParentPath, 'content')).toEqual({ sizeBytes: 4 }) + expect(binding.removeFile(cleanupRoot, relativeParentPath, 'content')).toBe(true) + expect(binding.removeFile(cleanupRoot, relativeParentPath, 'content')).toBe(false) + }) + + it('enforces bounded reads atomically and streams integrity verification', async () => { + cleanupRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-')) + const binding = require('../../../packages/safe-file-publisher-native') as { + readFileBounded: ( + rootPath: string, + relativeParentPath: string, + name: string, + maxBytes: number + ) => Buffer + verifyFile: ( + rootPath: string, + relativeParentPath: string, + name: string, + expectedSizeBytes: number, + expectedSha256: string + ) => boolean + } + const bytes = Buffer.alloc(3 * 64 * 1024 + 17, 0x5a) + await writeFile(join(cleanupRoot, 'large.bin'), bytes) + const sha256 = createHash('sha256').update(bytes).digest('hex') + + expect(binding.readFileBounded(cleanupRoot, '', 'large.bin', bytes.length)).toEqual(bytes) + expect(() => binding.readFileBounded(cleanupRoot!, '', 'large.bin', bytes.length - 1)).toThrow( + expect.objectContaining({ code: 'EFBIG' }) + ) + expect(binding.verifyFile(cleanupRoot, '', 'large.bin', bytes.length, sha256)).toBe(true) + expect(binding.verifyFile(cleanupRoot, '', 'large.bin', bytes.length, '0'.repeat(64))).toBe( + false + ) + expect(binding.verifyFile(cleanupRoot, '', 'large.bin', bytes.length + 1, sha256)).toBe(false) + + await symlink(join(cleanupRoot, 'large.bin'), join(cleanupRoot, 'linked.bin')) + expect(() => binding.readFileBounded(cleanupRoot!, '', 'linked.bin', bytes.length)).toThrow( + expect.objectContaining({ code: 'ELOOP' }) + ) + }) + + it('does not follow a bounded-read parent replaced outside the storage root', async () => { + cleanupRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-')) + const outsideRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-outside-')) + const binding = require('../../../packages/safe-file-publisher-native') as { + readFileBounded: ( + rootPath: string, + relativeParentPath: string, + name: string, + maxBytes: number + ) => Buffer + } + await mkdir(join(cleanupRoot, 'managed')) + await writeFile(join(cleanupRoot, 'managed', 'entry.txt'), 'inside') + await writeFile(join(outsideRoot, 'entry.txt'), 'outside') + expect(binding.readFileBounded(cleanupRoot, 'managed', 'entry.txt', 6)).toEqual( + Buffer.from('inside') + ) + + await rename(join(cleanupRoot, 'managed'), join(cleanupRoot, 'managed-real')) + await symlink(outsideRoot, join(cleanupRoot, 'managed')) + expect(() => binding.readFileBounded(cleanupRoot!, 'managed', 'entry.txt', 7)).toThrow( + expect.objectContaining({ code: 'ELOOP' }) + ) + await rm(outsideRoot, { recursive: true, force: true }) + }) + + it('rejects a file that grows after the bounded read descriptor is sized', async () => { + cleanupRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-')) + const filePath = join(cleanupRoot, 'growing.txt') + const sizedMarker = join(cleanupRoot, 'bounded-sized.marker') + const resumeMarker = join(cleanupRoot, 'bounded-resume.marker') + await writeFile(filePath, 'bounded') + const child = spawn( + process.execPath, + [ + '-e', + ` + const binding = require(process.argv[1]) + try { + binding.readFileBounded(process.argv[2], '', 'growing.txt', 7) + process.exit(2) + } catch (error) { + process.exit(error.code === 'EFBIG' ? 0 : 3) + } + `, + join(process.cwd(), 'packages/safe-file-publisher-native'), + cleanupRoot + ], + { + env: { + ...process.env, + NODE_ENV: 'test', + VITEST: 'true', + OPEN_SCIENCE_NATIVE_TEST_HOOKS: '1', + OPEN_SCIENCE_TEST_BOUNDED_READ_MARKER: sizedMarker, + OPEN_SCIENCE_TEST_BOUNDED_READ_RESUME: resumeMarker + }, + stdio: 'ignore' + } + ) + const childExit = new Promise((resolveExit, rejectExit) => { + const timeout = setTimeout(() => { + child.kill('SIGKILL') + rejectExit(new Error('bounded read child exit timed out')) + }, 5_000) + child.once('exit', (code) => { + clearTimeout(timeout) + resolveExit(code) + }) + child.once('error', rejectExit) + }) + + await Promise.race([ + waitForPath(sizedMarker), + childExit.then((code) => { + throw new Error(`bounded read child exited ${code} before size marker`) + }) + ]) + await writeFile(filePath, 'bounded!') + await writeFile(resumeMarker, 'resume') + expect(await childExit).toBe(0) + }) + + it('publishes only the verified recovery temp and never replaces a destination', async () => { + cleanupRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-')) + const binding = require('../../../packages/safe-file-publisher-native') as { + publishVerifiedNoReplace: ( + rootPath: string, + relativeParentPath: string, + temporaryName: string, + destinationName: string, + expectedBytes: Buffer + ) => void + } + const expected = Buffer.from('verified recovery') + await writeFile(join(cleanupRoot, 'valid.tmp'), expected) + + binding.publishVerifiedNoReplace(cleanupRoot, '', 'valid.tmp', 'recovered.txt', expected) + await expect(readFile(join(cleanupRoot, 'recovered.txt'))).resolves.toEqual(expected) + await expect(readFile(join(cleanupRoot, 'valid.tmp'))).rejects.toMatchObject({ code: 'ENOENT' }) + + await writeFile(join(cleanupRoot, 'invalid.tmp'), 'tampered') + expect(() => + binding.publishVerifiedNoReplace(cleanupRoot!, '', 'invalid.tmp', 'invalid.txt', expected) + ).toThrow() + await expect(readFile(join(cleanupRoot, 'invalid.tmp'), 'utf8')).resolves.toBe('tampered') + await expect(readFile(join(cleanupRoot, 'invalid.txt'))).rejects.toMatchObject({ + code: 'ENOENT' + }) + + await writeFile(join(cleanupRoot, 'next.tmp'), expected) + await writeFile(join(cleanupRoot, 'occupied.txt'), 'original') + expect(() => + binding.publishVerifiedNoReplace(cleanupRoot!, '', 'next.tmp', 'occupied.txt', expected) + ).toThrow(expect.objectContaining({ code: 'EEXIST' })) + await expect(readFile(join(cleanupRoot, 'occupied.txt'), 'utf8')).resolves.toBe('original') + await expect(readFile(join(cleanupRoot, 'next.tmp'))).resolves.toEqual(expected) + }) + + it('never publishes or removes a replacement raced against an already-open recovery temp', async () => { + cleanupRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-')) + const expected = Buffer.alloc(4 * 1024, 0x5a) + const attacker = Buffer.from('attacker replacement') + const tempPath = join(cleanupRoot, 'raced.tmp') + const attackerPath = join(cleanupRoot, 'attacker.tmp') + const destinationPath = join(cleanupRoot, 'raced.txt') + const verifiedMarker = join(cleanupRoot, 'verified.marker') + const resumeMarker = join(cleanupRoot, 'resume.marker') + await writeFile(tempPath, expected) + await writeFile(attackerPath, attacker) + const child = spawn( + process.execPath, + [ + '-e', + ` + const binding = require(process.argv[1]) + binding.publishVerifiedNoReplace( + process.argv[2], + '', + 'raced.tmp', + 'raced.txt', + Buffer.alloc(4 * 1024, 0x5a) + ) + `, + join(process.cwd(), 'packages/safe-file-publisher-native'), + cleanupRoot + ], + { + env: { + ...process.env, + NODE_ENV: 'test', + VITEST: 'true', + OPEN_SCIENCE_NATIVE_TEST_HOOKS: '1', + OPEN_SCIENCE_TEST_VERIFIED_TEMP_MARKER: verifiedMarker, + OPEN_SCIENCE_TEST_VERIFIED_TEMP_RESUME: resumeMarker + }, + stdio: ['ignore', 'ignore', 'pipe'] + } + ) + let childStderr = '' + child.stderr.on('data', (chunk) => { + childStderr += String(chunk) + }) + const childExit = new Promise((resolveExit, rejectExit) => { + const timeout = setTimeout(() => { + child.kill('SIGKILL') + rejectExit(new Error('replacement child exit timed out')) + }, 5_000) + child.once('exit', (code) => { + clearTimeout(timeout) + resolveExit(code) + }) + child.once('error', rejectExit) + }) + + await Promise.race([ + waitForPath(verifiedMarker), + childExit.then((code) => { + throw new Error(`publisher exited ${code} before verification marker: ${childStderr}`) + }) + ]) + await rename(attackerPath, tempPath) + await writeFile(resumeMarker, 'resume') + expect(await childExit).toBe(0) + + await expect(readFile(tempPath)).resolves.toEqual(attacker) + const destination = await readFile(destinationPath) + expect(destination.byteLength).toBe(expected.byteLength) + expect(createHash('sha256').update(destination).digest('hex')).toBe( + createHash('sha256').update(expected).digest('hex') + ) + }) + + it('publishes ordinary writes from their open descriptor without deleting a raced replacement', async () => { + cleanupRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-')) + const expected = Buffer.from('ordinary publication') + const attacker = Buffer.from('ordinary attacker replacement') + const tempPath = join(cleanupRoot, 'ordinary.tmp') + const attackerPath = join(cleanupRoot, 'ordinary-attacker.tmp') + const destinationPath = join(cleanupRoot, 'ordinary.txt') + const verifiedMarker = join(cleanupRoot, 'ordinary-verified.marker') + const resumeMarker = join(cleanupRoot, 'ordinary-resume.marker') + await writeFile(attackerPath, attacker) + const child = spawn( + process.execPath, + [ + '-e', + ` + const binding = require(process.argv[1]) + binding.writeAndPublishNoReplace( + process.argv[2], + '', + 'ordinary.tmp', + 'ordinary.txt', + Buffer.from('ordinary publication') + ) + `, + join(process.cwd(), 'packages/safe-file-publisher-native'), + cleanupRoot + ], + { + env: { + ...process.env, + NODE_ENV: 'test', + VITEST: 'true', + OPEN_SCIENCE_NATIVE_TEST_HOOKS: '1', + OPEN_SCIENCE_TEST_VERIFIED_TEMP_MARKER: verifiedMarker, + OPEN_SCIENCE_TEST_VERIFIED_TEMP_RESUME: resumeMarker + }, + stdio: ['ignore', 'ignore', 'pipe'] + } + ) + let childStderr = '' + child.stderr.on('data', (chunk) => { + childStderr += String(chunk) + }) + const childExit = new Promise((resolveExit, rejectExit) => { + const timeout = setTimeout(() => { + child.kill('SIGKILL') + rejectExit(new Error('ordinary publisher child exit timed out')) + }, 5_000) + child.once('exit', (code) => { + clearTimeout(timeout) + resolveExit(code) + }) + child.once('error', rejectExit) + }) + + await Promise.race([ + waitForPath(verifiedMarker), + childExit.then((code) => { + throw new Error(`ordinary publisher exited ${code} before marker: ${childStderr}`) + }) + ]) + await rename(attackerPath, tempPath) + await writeFile(resumeMarker, 'resume') + expect(await childExit).toBe(0) + + await expect(readFile(tempPath)).resolves.toEqual(attacker) + await expect(readFile(destinationPath)).resolves.toEqual(expected) + }) + + it('rejects a stat after an ancestor is replaced with a symlink', async () => { + cleanupRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-')) + const outsideRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-outside-')) + const binding = require('../../../packages/safe-file-publisher-native') as { + writeAndPublishNoReplace: ( + rootPath: string, + relativeParentPath: string, + temporaryName: string, + destinationName: string, + bytes: Buffer + ) => void + statFile: ( + rootPath: string, + relativeParentPath: string, + name: string + ) => { + sizeBytes: number + } + } + binding.writeAndPublishNoReplace( + cleanupRoot, + 'managed/stat', + '.entry.tmp', + 'entry.txt', + Buffer.from('entry') + ) + expect(binding.statFile(cleanupRoot, 'managed/stat', 'entry.txt')).toEqual({ sizeBytes: 5 }) + + await rename(join(cleanupRoot, 'managed'), join(cleanupRoot, 'managed-real')) + await symlink(outsideRoot, join(cleanupRoot, 'managed')) + expect(() => binding.statFile(cleanupRoot!, 'managed/stat', 'entry.txt')).toThrow( + expect.objectContaining({ code: 'ELOOP' }) + ) + await rm(outsideRoot, { recursive: true, force: true }) + }) + + it('never creates temporary bytes through a symlinked ancestor', async () => { + cleanupRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-')) + const outsideRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-outside-')) + const linkedParent = join(cleanupRoot, 'linked') + await symlink(outsideRoot, linkedParent, process.platform === 'win32' ? 'junction' : 'dir') + const binding = require('../../../packages/safe-file-publisher-native') as { + writeAndPublishNoReplace: ( + rootPath: string, + relativeParentPath: string, + temporaryName: string, + destinationName: string, + bytes: Buffer + ) => void + } + + expect(() => + binding.writeAndPublishNoReplace( + cleanupRoot!, + 'linked/nested', + 'content.tmp', + 'content', + Buffer.from('escape') + ) + ).toThrow(expect.objectContaining({ code: expect.stringMatching(/ELOOP|EIO|ENOENT/) })) + await expect(readFile(join(outsideRoot, 'nested', 'content.tmp'))).rejects.toMatchObject({ + code: 'ENOENT' + }) + await rm(outsideRoot, { recursive: true, force: true }) + }) + + it('lists metadata through an anchored directory without following a replaced parent', async () => { + cleanupRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-')) + const outsideRoot = await mkdtemp(join(tmpdir(), 'safe-file-publisher-outside-')) + const binding = require('../../../packages/safe-file-publisher-native') as { + writeAndPublishNoReplace: ( + rootPath: string, + relativeParentPath: string, + temporaryName: string, + destinationName: string, + bytes: Buffer + ) => void + listDirectory: ( + rootPath: string, + relativeParentPath: string + ) => Array<{ name: string; isFile: boolean; mtimeMs: number }> + } + binding.writeAndPublishNoReplace( + cleanupRoot, + 'managed/list', + '.entry.tmp', + 'entry.txt', + Buffer.from('entry') + ) + + expect(binding.listDirectory(cleanupRoot, 'managed/list')).toEqual([ + expect.objectContaining({ name: 'entry.txt', isFile: true }) + ]) + await rename(join(cleanupRoot, 'managed'), join(cleanupRoot, 'managed-real')) + await symlink(outsideRoot, join(cleanupRoot, 'managed')) + expect(() => binding.listDirectory(cleanupRoot!, 'managed/list')).toThrow( + expect.objectContaining({ code: 'ELOOP' }) + ) + await rm(outsideRoot, { recursive: true, force: true }) + }) }) diff --git a/src/main/uploads/atomic-no-replace-publisher.ts b/src/main/uploads/atomic-no-replace-publisher.ts index 3b217319b..61fe16efd 100644 --- a/src/main/uploads/atomic-no-replace-publisher.ts +++ b/src/main/uploads/atomic-no-replace-publisher.ts @@ -2,28 +2,132 @@ import { createRequire } from 'node:module' import { isAbsolute, relative, sep } from 'node:path' type NativePublisherBinding = { + supportsAnchoredWrites: boolean publishNoReplace: ( rootPath: string, relativeParentPath: string, sourceName: string, destinationName: string ) => void + writeAndPublishNoReplace: ( + rootPath: string, + relativeParentPath: string, + temporaryName: string, + destinationName: string, + bytes: Buffer + ) => void + readFile: (rootPath: string, relativeParentPath: string, name: string) => Buffer + readFileBounded: ( + rootPath: string, + relativeParentPath: string, + name: string, + maxBytes: number + ) => Buffer + publishVerifiedNoReplace: ( + rootPath: string, + relativeParentPath: string, + temporaryName: string, + destinationName: string, + expectedBytes: Buffer + ) => void + verifyFile: ( + rootPath: string, + relativeParentPath: string, + name: string, + expectedSizeBytes: number, + expectedSha256: string + ) => boolean + statFile: (rootPath: string, relativeParentPath: string, name: string) => { sizeBytes: number } + removeFile: (rootPath: string, relativeParentPath: string, name: string) => boolean + listDirectory: ( + rootPath: string, + relativeParentPath: string + ) => Array<{ name: string; isFile: boolean; mtimeMs: number }> } const require = createRequire(import.meta.url) let binding: NativePublisherBinding | undefined const loadBinding = (): NativePublisherBinding => { - binding ??= require('@aipoch/safe-file-publisher-native') as NativePublisherBinding + if (!binding) { + try { + binding = assertNativePublisherBinding(require('@aipoch/safe-file-publisher-native')) + } catch (error) { + if ( + typeof error !== 'object' || + error === null || + !('code' in error) || + error.code !== 'MODULE_NOT_FOUND' + ) { + throw error + } + // Worktrees share the main checkout's dependencies, where local file packages are not linked + // back into this checkout. Production installs always resolve the package name above. + binding = assertNativePublisherBinding( + require('../../../packages/safe-file-publisher-native') + ) + } + } return binding } -export const publishNoReplace = ( - rootPath: string, - parentPath: string, - sourceName: string, - destinationName: string -): void => { +type NativeBindingLoader = () => unknown + +const isNativePublisherBinding = (candidate: unknown): candidate is NativePublisherBinding => { + if (!candidate || typeof candidate !== 'object') return false + const value = candidate as Record + return ( + typeof value.supportsAnchoredWrites === 'boolean' && + [ + 'publishNoReplace', + 'writeAndPublishNoReplace', + 'readFile', + 'readFileBounded', + 'publishVerifiedNoReplace', + 'verifyFile', + 'statFile', + 'removeFile', + 'listDirectory' + ].every((name) => typeof value[name] === 'function') + ) +} + +const assertNativePublisherBinding = (candidate: unknown): NativePublisherBinding => { + if (!isNativePublisherBinding(candidate)) { + const error = new Error('Native managed-file publisher binding is incomplete.') + Object.assign(error, { code: 'ENOTSUP' }) + throw error + } + return candidate +} + +export const managedFileVersionNativeCapability = ( + loader: NativeBindingLoader = loadBinding +): + | { available: true; readFallbackAvailable: false } + | { + available: false + reason: 'NATIVE_WRITE_REQUIRED' + readFallbackAvailable: boolean + } => { + try { + return assertNativePublisherBinding(loader()).supportsAnchoredWrites + ? { available: true, readFallbackAvailable: false } + : { + available: false, + reason: 'NATIVE_WRITE_REQUIRED', + readFallbackAvailable: true + } + } catch { + return { + available: false, + reason: 'NATIVE_WRITE_REQUIRED', + readFallbackAvailable: false + } + } +} + +const relativeParent = (rootPath: string, parentPath: string): string => { const relativeParentPath = relative(rootPath, parentPath) if ( isAbsolute(relativeParentPath) || @@ -34,5 +138,92 @@ export const publishNoReplace = ( Object.assign(error, { code: 'EINVAL' }) throw error } - loadBinding().publishNoReplace(rootPath, relativeParentPath, sourceName, destinationName) + return relativeParentPath } + +export const publishNoReplace = ( + rootPath: string, + parentPath: string, + sourceName: string, + destinationName: string +): void => { + loadBinding().publishNoReplace( + rootPath, + relativeParent(rootPath, parentPath), + sourceName, + destinationName + ) +} + +export const writeAndPublishNoReplace = ( + rootPath: string, + parentPath: string, + temporaryName: string, + destinationName: string, + bytes: Buffer +): void => { + loadBinding().writeAndPublishNoReplace( + rootPath, + relativeParent(rootPath, parentPath), + temporaryName, + destinationName, + bytes + ) +} + +export const readAnchoredFile = (rootPath: string, parentPath: string, name: string): Buffer => + loadBinding().readFile(rootPath, relativeParent(rootPath, parentPath), name) + +export const readAnchoredFileBounded = ( + rootPath: string, + parentPath: string, + name: string, + maxBytes: number +): Buffer => + loadBinding().readFileBounded(rootPath, relativeParent(rootPath, parentPath), name, maxBytes) + +export const publishVerifiedAnchoredFileNoReplace = ( + rootPath: string, + parentPath: string, + temporaryName: string, + destinationName: string, + expectedBytes: Buffer +): void => + loadBinding().publishVerifiedNoReplace( + rootPath, + relativeParent(rootPath, parentPath), + temporaryName, + destinationName, + expectedBytes + ) + +export const verifyAnchoredFile = ( + rootPath: string, + parentPath: string, + name: string, + expectedSizeBytes: number, + expectedSha256: string +): boolean => + loadBinding().verifyFile( + rootPath, + relativeParent(rootPath, parentPath), + name, + expectedSizeBytes, + expectedSha256 + ) + +export const statAnchoredFile = ( + rootPath: string, + parentPath: string, + name: string +): { sizeBytes: number } => + loadBinding().statFile(rootPath, relativeParent(rootPath, parentPath), name) + +export const removeAnchoredFile = (rootPath: string, parentPath: string, name: string): boolean => + loadBinding().removeFile(rootPath, relativeParent(rootPath, parentPath), name) + +export const listAnchoredDirectory = ( + rootPath: string, + parentPath: string +): Array<{ name: string; isFile: boolean; mtimeMs: number }> => + loadBinding().listDirectory(rootPath, relativeParent(rootPath, parentPath)) diff --git a/src/main/uploads/attachment-media.test.ts b/src/main/uploads/attachment-media.test.ts index d19f917b5..5ee379096 100644 --- a/src/main/uploads/attachment-media.test.ts +++ b/src/main/uploads/attachment-media.test.ts @@ -476,9 +476,24 @@ describe('buildImageContentData', () => { expect(sharpFactory).not.toHaveBeenCalled() }) + it('reads small managed images from the trusted byte source instead of the path', async () => { + const bytes = Buffer.from('trusted-image-bytes') + const readBytes = vi.fn(async () => bytes) + + const result = await buildImageContentData( + join(root, 'missing.png'), + 'image/png', + bytes.byteLength, + readBytes + ) + + expect(result).toEqual({ data: bytes.toString('base64'), mimeType: 'image/png' }) + expect(readBytes).toHaveBeenCalledOnce() + }) + it('downscales large images to the long-edge cap and re-encodes to JPEG', async () => { const filePath = join(root, 'large.jpg') - await writeFile(filePath, Buffer.from('ignored-because-nativeimage-is-mocked')) + await writeFile(filePath, Buffer.from('ignored-because-sharp-is-mocked')) const result = await buildImageContentData(filePath, 'image/jpeg', 3 * 1024 * 1024) @@ -490,6 +505,23 @@ describe('buildImageContentData', () => { expect(result.data).toBe(Buffer.from('jpeg-80').toString('base64')) }) + it('decodes large managed images from trusted bytes instead of reopening the path', async () => { + const bytes = Buffer.from('trusted-large-image') + const readBytes = vi.fn(async () => bytes) + + const result = await buildImageContentData( + join(root, 'missing-large.jpg'), + 'image/jpeg', + 3 * 1024 * 1024, + readBytes + ) + + expect(sharpFactory).toHaveBeenCalledOnce() + expect(sharpFactory).toHaveBeenCalledWith(bytes) + expect(readBytes).toHaveBeenCalledOnce() + expect(result.data).toBe(Buffer.from('jpeg-80').toString('base64')) + }) + it('keeps PNG encoding for large PNGs to preserve transparency', async () => { fakeImage.hasAlpha = true const filePath = join(root, 'large.png') @@ -617,6 +649,19 @@ describe('extractPdfText', () => { expect(result.text).toBe('--- Page 1 ---\nHello world\n\n--- Page 2 ---\nSecond page') }) + it('extracts managed PDFs from the trusted byte source instead of the path', async () => { + const bytes = Buffer.from('%PDF-1.4 trusted') + const readBytes = vi.fn(async () => bytes) + + const result = await extractPdfText(join(root, 'missing.pdf'), { + size: bytes.byteLength, + readBytes + }) + + expect(readBytes).toHaveBeenCalledOnce() + expect(result.text).toContain('Hello world') + }) + it('returns empty text for a PDF with no extractable content', async () => { fakePdf = { numPages: 1, pages: [[]] } const filePath = join(root, 'scanned.pdf') diff --git a/src/main/uploads/attachment-media.ts b/src/main/uploads/attachment-media.ts index ea88a6d04..6a08a5bef 100644 --- a/src/main/uploads/attachment-media.ts +++ b/src/main/uploads/attachment-media.ts @@ -125,6 +125,11 @@ export type PdfTextResult = { truncated: boolean } +export type AttachmentByteSource = { + size: number + readBytes: () => Promise +} + // Accounts for the bytes that will actually be inserted into JSON rather than the decoded image // size. Callers can fold this over prepared image blocks before dispatching a multimodal prompt. export const consumeInlineImageBudget = ( @@ -491,7 +496,8 @@ export const prepareImageContentData = async ( export const buildImageContentData = async ( filePath: string, mimeType: string | undefined, - size: number + size: number, + readBytes?: () => Promise ): Promise => { const fallbackMimeType = mimeType ?? 'application/octet-stream' @@ -504,11 +510,13 @@ export const buildImageContentData = async ( } if (size <= MAX_INLINE_IMAGE_BYTES) { - return { data: (await readFile(filePath)).toString('base64'), mimeType: fallbackMimeType } + const source = readBytes ? Buffer.from(await readBytes()) : await readFile(filePath) + return { data: source.toString('base64'), mimeType: fallbackMimeType } } try { - const bytes = await readFile(filePath) + // Managed versions supply integrity-checked bytes; legacy callers still read the resolved path. + const bytes = readBytes ? Buffer.from(await readBytes()) : await readFile(filePath) if (bytes.byteLength > MAX_AUTO_PROCESS_IMAGE_BYTES) { throw new ImageContentError( 'IMAGE_SOURCE_TOO_LARGE', @@ -551,16 +559,17 @@ const resolvePdfjsAssetUrls = (): { cMapUrl: string; standardFontDataUrl: string // Extracts selectable text from a PDF so the model receives readable content instead of the raw // (base64) file, which would otherwise overflow the request size limit. -export const extractPdfText = async (filePath: string): Promise => { - const fileInfo = await stat(filePath) - if (fileInfo.size > MAX_AUTO_EXTRACT_PDF_BYTES) { - throw new Error( - `PDF source is ${fileInfo.size} bytes, exceeding the automatic extraction limit.` - ) +export const extractPdfText = async ( + filePath: string, + source?: AttachmentByteSource +): Promise => { + const size = source?.size ?? (await stat(filePath)).size + if (size > MAX_AUTO_EXTRACT_PDF_BYTES) { + throw new Error(`PDF source is ${size} bytes, exceeding the automatic extraction limit.`) } const pdfjs = (await import('pdfjs-dist/legacy/build/pdf.mjs')) as typeof import('pdfjs-dist') const { cMapUrl, standardFontDataUrl } = resolvePdfjsAssetUrls() - const fileData = await readFile(filePath) + const fileData = source ? Buffer.from(await source.readBytes()) : await readFile(filePath) const loadingTask = pdfjs.getDocument({ data: new Uint8Array(fileData), diff --git a/src/main/uploads/command-owner.test.ts b/src/main/uploads/command-owner.test.ts index 6c710e396..46c37495f 100644 --- a/src/main/uploads/command-owner.test.ts +++ b/src/main/uploads/command-owner.test.ts @@ -455,4 +455,72 @@ describe('upload command owner', () => { expect(stagedOwner).toBe(owner) }) + + it('resolves a logical Upload preview at read time and preserves an explicit Version', async () => { + const root = await mkdtemp(join(tmpdir(), 'open-science-upload-preview-owner-')) + temporaryRoots.push(root) + const currentPath = join(root, 'current.txt') + await writeFile(currentPath, 'current upload head') + const resolveManagedFilePath = vi.fn().mockResolvedValue(currentPath) + const readManagedUploadPreview = vi.fn() + const owner = createUploadCommandOwner( + { readManagedUploadPreview } as unknown as UploadRepository, + { resolveManagedFilePath } + ) + const leases = new ApplicationCallerLeaseRegistry() + const caller = createCaller(leases, 16) + const request = { + path: '/stale/upload.txt', + projectId: 'project-1', + fileId: 'upload-1', + versionId: 'upload-v1', + maxBytes: 1024 + } + + await expect( + owner.readPreview(invocationFor(caller, [request] as const)) + ).resolves.toMatchObject({ content: 'current upload head' }) + expect(resolveManagedFilePath).toHaveBeenCalledWith(request) + expect(readManagedUploadPreview).not.toHaveBeenCalled() + }) + + it('reads a logical Upload preview through the verified lease and always closes it', async () => { + const bytes = Buffer.from('verified upload bytes') + const close = vi.fn().mockResolvedValue(undefined) + const verifyUnchanged = vi.fn().mockResolvedValue(undefined) + const openManagedFileVersion = vi.fn().mockResolvedValue({ + size: bytes.byteLength, + read: vi.fn(async (buffer: Uint8Array, offset: number, length: number, position: number) => { + const chunk = bytes.subarray(position, position + length) + buffer.set(chunk, offset) + return { bytesRead: chunk.byteLength } + }), + verifyUnchanged, + close + }) + const resolveManagedFilePath = vi.fn().mockRejectedValue(new Error('must not resolve a path')) + const readManagedUploadPreview = vi.fn() + const owner = createUploadCommandOwner( + { readManagedUploadPreview } as unknown as UploadRepository, + { openManagedFileVersion, resolveManagedFilePath } + ) + const leases = new ApplicationCallerLeaseRegistry() + const caller = createCaller(leases, 17) + const request = { + path: '/replaceable/upload.txt', + projectId: 'project-1', + fileId: 'upload-1', + versionId: 'upload-v1', + maxBytes: 1024 + } + + await expect( + owner.readPreview(invocationFor(caller, [request] as const)) + ).resolves.toMatchObject({ content: 'verified upload bytes' }) + expect(openManagedFileVersion).toHaveBeenCalledWith(request) + expect(resolveManagedFilePath).not.toHaveBeenCalled() + expect(readManagedUploadPreview).not.toHaveBeenCalled() + expect(verifyUnchanged).toHaveBeenCalledOnce() + expect(close).toHaveBeenCalledOnce() + }) }) diff --git a/src/main/uploads/command-owner.ts b/src/main/uploads/command-owner.ts index cdedaadb9..1f8bf2ae6 100644 --- a/src/main/uploads/command-owner.ts +++ b/src/main/uploads/command-owner.ts @@ -2,6 +2,11 @@ import { stat } from 'node:fs/promises' import type { ApplicationCallerLease, ApplicationInvocation } from '../application-command-router' import { acquireDataRootWriter, withDataRootWrite } from '../storage/migration-state' +import { + readBoundedManagedFilePreview, + readBoundedManagedFilePreviewLease, + type ManagedFilePreviewReadLease +} from '../managed-file-preview' import type { ArtifactPreviewResult, ReadArtifactPreviewRequest } from '../../shared/artifacts' import type { @@ -52,6 +57,10 @@ type UploadProgressTarget = Readonly<{ }> type UploadCommandOwnerOptions = Readonly<{ + resolveManagedFilePath?: (request: ReadArtifactPreviewRequest) => Promise + openManagedFileVersion?: ( + request: ReadArtifactPreviewRequest + ) => Promise withSessionMutation?: ( projectId: string, sessionId: string, @@ -392,7 +401,25 @@ const createUploadCommandOwner = ( ? options.withSessionMutation(request.projectId, request.sessionId, finalize) : finalize() }), - readPreview: ({ args: [request] }) => repository.readManagedUploadPreview(request) + readPreview: async ({ args: [request] }) => { + if (request.projectId && request.fileId && options.openManagedFileVersion) { + const lease = await options.openManagedFileVersion(request) + try { + return await readBoundedManagedFilePreviewLease( + lease, + request, + 'Invalid upload preview encoding.' + ) + } finally { + await lease.close() + } + } + if (!request.projectId || !request.fileId || !options.resolveManagedFilePath) { + return repository.readManagedUploadPreview(request) + } + const path = await options.resolveManagedFilePath(request) + return readBoundedManagedFilePreview(path, request, 'Invalid upload preview encoding.') + } }) } diff --git a/src/main/uploads/legacy-recovery-owner.ts b/src/main/uploads/legacy-recovery-owner.ts index 7d1750c20..886373beb 100644 --- a/src/main/uploads/legacy-recovery-owner.ts +++ b/src/main/uploads/legacy-recovery-owner.ts @@ -418,7 +418,20 @@ class LegacyRecoveryOwner { version.state === 'ready' ? version : await tx.uploadVersion.update({ where: { id: version.id }, data: { state: 'ready' } }) - const timestamp = updated.createdAt ?? new Date() + const file = await tx.uploadFile.findUniqueOrThrow({ + where: { id: version.uploadFileId }, + include: { currentVersion: true } + }) + const shouldAdvanceHead = + !file.currentVersion || file.currentVersion.versionNumber < updated.versionNumber + if (shouldAdvanceHead) { + await tx.uploadFile.update({ + where: { id: file.id }, + data: { currentVersionId: updated.id } + }) + } + const head = shouldAdvanceHead ? updated : file.currentVersion! + const timestamp = head.createdAt ?? new Date() await tx.managedFile.upsert({ where: { projectId_source_sourceFileId: { @@ -430,25 +443,25 @@ class LegacyRecoveryOwner { create: { source: 'upload', sourceFileId: version.uploadFileId, - sourceVersionId: version.id, - checksum: version.checksum, + sourceVersionId: head.id, + checksum: head.checksum, projectId, sessionId, - displayName: version.originalFilename || version.filename, - storageKey: version.contentStorageKey, - mimeType: version.contentType, - sizeBytes: version.sizeBytes, + displayName: head.originalFilename || head.filename, + storageKey: head.contentStorageKey, + mimeType: head.contentType, + sizeBytes: head.sizeBytes, mtimeMs: BigInt(timestamp.getTime()), sortAtMs: BigInt(timestamp.getTime()) }, update: { - sourceVersionId: version.id, - checksum: version.checksum, + sourceVersionId: head.id, + checksum: head.checksum, sessionId, - displayName: version.originalFilename || version.filename, - storageKey: version.contentStorageKey, - mimeType: version.contentType, - sizeBytes: version.sizeBytes, + displayName: head.originalFilename || head.filename, + storageKey: head.contentStorageKey, + mimeType: head.contentType, + sizeBytes: head.sizeBytes, mtimeMs: BigInt(timestamp.getTime()), sortAtMs: BigInt(timestamp.getTime()), deletedAt: null, diff --git a/src/main/uploads/repository.test.ts b/src/main/uploads/repository.test.ts index 8f99ad41c..e83483018 100644 --- a/src/main/uploads/repository.test.ts +++ b/src/main/uploads/repository.test.ts @@ -443,6 +443,7 @@ describe('upload repository', () => { expect(files).toHaveLength(2) expect(files.every((file) => file.versions[0]?.state === 'ready')).toBe(true) expect(files.every((file) => file.versions[0]?.versionNumber === 1)).toBe(true) + expect(files.every((file) => file.currentVersionId === file.versions[0]?.id)).toBe(true) const [again] = await repository.finalizePendingSessionUploads( 'session-1', @@ -553,6 +554,9 @@ describe('upload repository', () => { await expect( client.uploadVersion.findUniqueOrThrow({ where: { id: versionId } }) ).resolves.toMatchObject({ state: 'ready' }) + await expect( + client.uploadFile.findUniqueOrThrow({ where: { id: pending.id } }) + ).resolves.toMatchObject({ currentVersionId: versionId }) await expect( client.managedFile.findUniqueOrThrow({ where: { @@ -615,6 +619,39 @@ describe('upload repository', () => { } } }) + const newerContent = Buffer.from('newer ready version') + const newerVersionId = 'upload-version-newer-ready' + const newerStorageKey = [ + 'uploads', + 'project-1', + 'session-1', + 'upload-post-rename', + 'versions', + newerVersionId, + 'content' + ].join('/') + const newerPath = join(root, ...newerStorageKey.split('/')) + await mkdir(dirname(newerPath), { recursive: true }) + await writeFile(newerPath, newerContent) + await client.uploadVersion.create({ + data: { + id: newerVersionId, + uploadFileId: 'upload-post-rename', + versionNumber: 2, + state: 'ready', + originKind: 'legacy', + contentStorageKey: newerStorageKey, + filename: 'renamed.txt', + originalFilename: 'renamed.txt', + contentType: 'text/plain', + sizeBytes: BigInt(newerContent.byteLength), + checksum: createHash('sha256').update(newerContent).digest('hex') + } + }) + await client.uploadFile.update({ + where: { id: 'upload-post-rename' }, + data: { currentVersionId: newerVersionId } + }) await repository.recoverStagingUploads() @@ -633,7 +670,10 @@ describe('upload repository', () => { } } }) - ).resolves.toMatchObject({ sourceVersionId: versionId }) + ).resolves.toMatchObject({ sourceVersionId: newerVersionId, storageKey: newerStorageKey }) + await expect( + client.uploadFile.findUniqueOrThrow({ where: { id: 'upload-post-rename' } }) + ).resolves.toMatchObject({ currentVersionId: newerVersionId }) }) it('recovers and removes a deterministic live-copy temp left before its final rename', async () => { diff --git a/src/preload/electron-renderer-contract-adapter.test.ts b/src/preload/electron-renderer-contract-adapter.test.ts index 910fdf237..42fa46a3c 100644 --- a/src/preload/electron-renderer-contract-adapter.test.ts +++ b/src/preload/electron-renderer-contract-adapter.test.ts @@ -118,6 +118,29 @@ describe('electron renderer contract adapter', () => { expect(port.invoke).toHaveBeenCalledWith('preview:save', request) }) + it('forwards Project ZIP logical file identities without narrowing the request', async () => { + const port = createPort() + const adapter = createElectronRendererContractAdapter(port) + const request = { + projectId: 'project-1', + projectName: 'Research', + files: [ + { + source: 'artifact', + sessionId: 'session-1', + path: '/stale/report.md', + fileId: 'artifact-file-1', + versionId: 'artifact-version-2', + suggestedName: 'report.md' + } + ] + } + + await adapter.invoke('saveProjectArtifacts', request) + + expect(port.invoke).toHaveBeenCalledWith('file:save-project-artifacts', request) + }) + it('propagates request failures unchanged', async () => { const failure = new Error('main process unavailable') const port = createPort() diff --git a/src/preload/index.test.ts b/src/preload/index.test.ts index 11cbcd1b4..ea6919694 100644 --- a/src/preload/index.test.ts +++ b/src/preload/index.test.ts @@ -336,6 +336,11 @@ describe('preload bridge — public surface inventory', () => { 'logs.getPath', 'logs.openFile', 'logs.revealInFolder', + 'managedFileVersions.cancelDiff', + 'managedFileVersions.diffText', + 'managedFileVersions.getCapability', + 'managedFileVersions.inspect', + 'managedFileVersions.saveTextEdit', 'network.checkConnectivity', 'network.getInfo', 'notebook.appendCodeCell', @@ -723,6 +728,7 @@ describe('preload bridge — core renderer contract catalog', () => { 'locale', 'local-fs', 'logs', + 'managed-file-versions', 'network', 'notifications', 'office-preview', diff --git a/src/preload/index.ts b/src/preload/index.ts index abb6dedcc..5e1d4f95a 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -551,6 +551,16 @@ const api: OpenScienceAPI = { repairIndex: (request) => electronRendererContracts.invoke('projectFiles.repairIndex', request), onChanged: (listener) => electronRendererContracts.subscribe('projectFiles.onChanged', listener) }, + managedFileVersions: { + getCapability: () => electronRendererContracts.invoke('managedFileVersions.getCapability'), + inspect: (request) => electronRendererContracts.invoke('managedFileVersions.inspect', request), + diffText: (request) => + electronRendererContracts.invoke('managedFileVersions.diffText', request), + cancelDiff: (request) => + electronRendererContracts.invoke('managedFileVersions.cancelDiff', request), + saveTextEdit: (request) => + electronRendererContracts.invoke('managedFileVersions.saveTextEdit', request) + }, compute: { // SSH compute host record CRUD, backed by the same SQLite/Prisma layer as projects. list: () => electronRendererContracts.invoke('compute.list'), diff --git a/src/preload/renderer-api.d.ts b/src/preload/renderer-api.d.ts index 322e82686..7125754fb 100644 --- a/src/preload/renderer-api.d.ts +++ b/src/preload/renderer-api.d.ts @@ -55,6 +55,17 @@ import type { GenerateArtifactCodeReconstructionRequest, GetArtifactCodeReconstructionRequest } from '../shared/artifact-code-reconstruction' +import type { + ManagedFileVersionInspectRequest, + ManagedFileVersionInspectResult, + ManagedFileVersionHostCapability, + ManagedFileVersionIpcResult, + ManagedFileVersionDiffRequest, + ManagedFileVersionDiffResult, + ManagedFileVersionCancelDiffRequest, + ManagedFileVersionSaveTextEditRequest, + SaveTextEditResult +} from '../shared/managed-file-versions' import type { SaveBlobFileRequest, SaveBlobFileResult, @@ -738,6 +749,21 @@ export interface OpenScienceAPI { repairIndex(request: { projectId: string }): Promise onChanged(listener: AcpListener): RemoveListener } + managedFileVersions: { + getCapability(): Promise + inspect( + request: ManagedFileVersionInspectRequest + ): Promise> + diffText( + request: ManagedFileVersionDiffRequest + ): Promise> + cancelDiff( + request: ManagedFileVersionCancelDiffRequest + ): Promise> + saveTextEdit( + request: ManagedFileVersionSaveTextEditRequest + ): Promise> + } compute: { // SSH compute host record CRUD (Compute settings tab). No credentials cross this boundary. list(): Promise diff --git a/src/renderer/src/assets/diff-colors.test.ts b/src/renderer/src/assets/diff-colors.test.ts new file mode 100644 index 000000000..eae9727d5 --- /dev/null +++ b/src/renderer/src/assets/diff-colors.test.ts @@ -0,0 +1,89 @@ +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' + +import { describe, expect, it } from 'vitest' + +const css = readFileSync(resolve(__dirname, 'main.css'), 'utf8') + +type Rgb = [number, number, number] + +const themeBlock = (selector: ':root' | '.dark'): string => { + const escapedSelector = selector.replace('.', '\\.') + const match = css.match(new RegExp(`${escapedSelector}\\s*\\{([\\s\\S]*?)\\n\\}`)) + if (!match?.[1]) throw new Error(`Missing ${selector} theme block`) + return match[1] +} + +const hslToken = (block: string, name: string): Rgb => { + const match = block.match( + new RegExp(`--${name}:\\s*hsl\\(([\\d.]+)\\s+([\\d.]+)%\\s+([\\d.]+)%\\);`) + ) + if (!match) throw new Error(`Missing HSL token: ${name}`) + const hue = Number(match[1]) + const saturation = Number(match[2]) / 100 + const lightness = Number(match[3]) / 100 + const chroma = (1 - Math.abs(2 * lightness - 1)) * saturation + const second = chroma * (1 - Math.abs(((hue / 60) % 2) - 1)) + const offset = lightness - chroma / 2 + const [red, green, blue] = + hue < 60 + ? [chroma, second, 0] + : hue < 120 + ? [second, chroma, 0] + : hue < 180 + ? [0, chroma, second] + : hue < 240 + ? [0, second, chroma] + : hue < 300 + ? [second, 0, chroma] + : [chroma, 0, second] + return [red + offset, green + offset, blue + offset] +} + +const luminance = ([red, green, blue]: Rgb): number => + [red, green, blue] + .map((channel) => (channel <= 0.04045 ? channel / 12.92 : ((channel + 0.055) / 1.055) ** 2.4)) + .reduce((sum, channel, index) => sum + channel * [0.2126, 0.7152, 0.0722][index]!, 0) + +const contrast = (left: Rgb, right: Rgb): number => { + const [lighter, darker] = [luminance(left), luminance(right)].sort( + (first, second) => second - first + ) + return (lighter! + 0.05) / (darker! + 0.05) +} + +describe('diff color tokens', () => { + it.each([ + ['light', themeBlock(':root')], + ['dark', themeBlock('.dark')] + ])('keeps %s markers and text at WCAG AA contrast', (_theme, block) => { + const text = hslToken(block, 'text-000') + for (const kind of ['added', 'removed']) { + const surface = hslToken(block, `diff-${kind}-surface`) + const highlight = hslToken(block, `diff-${kind}-highlight`) + const foreground = hslToken(block, `diff-${kind}-foreground`) + + expect(contrast(foreground, surface)).toBeGreaterThanOrEqual(4.5) + expect(contrast(text, surface)).toBeGreaterThanOrEqual(4.5) + expect(contrast(text, highlight)).toBeGreaterThanOrEqual(4.5) + } + }) + + it('limits semantic change markers to their direct list or table carrier', () => { + for (const kind of ['added', 'removed']) { + const marker = `[data-managed-diff-marker='${kind}']` + expect(css).toContain(`:is(li, td, th):has(> ${marker})`) + expect(css).toContain(`:is(li, td, th):has(> p > ${marker})`) + expect(css).not.toContain(`:is(li, td, th):has(${marker})`) + } + }) + + it('marks semantic carrier text without painting the carrier rectangle', () => { + const semanticRules = css.match(/\.managed-version-diff-markdown[^{}]+\{[^{}]+\}/gu)?.join('\n') + + expect(semanticRules).toBeDefined() + expect(semanticRules).not.toContain('background-color') + expect(semanticRules).toContain('color: var(--diff-added-foreground)') + expect(semanticRules).toContain('color: var(--diff-removed-foreground)') + }) +}) diff --git a/src/renderer/src/assets/main.css b/src/renderer/src/assets/main.css index 88503485c..08577c2fb 100644 --- a/src/renderer/src/assets/main.css +++ b/src/renderer/src/assets/main.css @@ -85,6 +85,12 @@ --color-status-warning-dark-surface: var(--color-amber-950); --color-status-warning-dark-foreground: var(--color-amber-400); --color-success-000: var(--success-000); + --color-diff-added-surface: var(--diff-added-surface); + --color-diff-added-highlight: var(--diff-added-highlight); + --color-diff-added-foreground: var(--diff-added-foreground); + --color-diff-removed-surface: var(--diff-removed-surface); + --color-diff-removed-highlight: var(--diff-removed-highlight); + --color-diff-removed-foreground: var(--diff-removed-foreground); --color-syntax-keyword: var(--syntax-keyword); --color-syntax-string: var(--syntax-string); --color-syntax-number: var(--syntax-number); @@ -137,6 +143,14 @@ --warning-100: hsl(45 80% 90%); --warning-900: hsl(32 65% 28%); --success-000: hsl(145 45% 38%); + /* Version diffs keep semantic markers and body text above WCAG AA contrast on both row and + inline-highlight surfaces. */ + --diff-added-surface: hsl(145 45% 92%); + --diff-added-highlight: hsl(145 45% 84%); + --diff-added-foreground: hsl(145 60% 24%); + --diff-removed-surface: hsl(0 55% 95%); + --diff-removed-highlight: hsl(0 50% 88%); + --diff-removed-foreground: hsl(0 55% 32%); /* Notebook code highlighting: distinct blue/green/purple hues so keywords, strings, and numbers read apart from plain text on the warm paper surface. */ --syntax-keyword: hsl(222 60% 46%); @@ -211,6 +225,12 @@ --warning-100: hsl(45 45% 18%); --warning-900: hsl(45 90% 62%); --success-000: hsl(145 70% 72%); + --diff-added-surface: hsl(145 30% 19%); + --diff-added-highlight: hsl(145 28% 26%); + --diff-added-foreground: hsl(145 65% 78%); + --diff-removed-surface: hsl(0 30% 20%); + --diff-removed-highlight: hsl(0 28% 27%); + --diff-removed-foreground: hsl(0 65% 80%); /* Brighter code-highlight hues so keywords/strings/numbers keep enough contrast on the dark surface. */ --syntax-keyword: hsl(215 75% 72%); --syntax-string: hsl(140 45% 62%); @@ -303,6 +323,19 @@ } @layer components { + .managed-version-diff-markdown :is(li, td, th):has(> [data-managed-diff-marker='added']), + .managed-version-diff-markdown :is(li, td, th):has(> p > [data-managed-diff-marker='added']) { + color: var(--diff-added-foreground); + font-weight: 500; + text-decoration: none; + } + + .managed-version-diff-markdown :is(li, td, th):has(> [data-managed-diff-marker='removed']), + .managed-version-diff-markdown :is(li, td, th):has(> p > [data-managed-diff-marker='removed']) { + color: var(--diff-removed-foreground); + text-decoration: line-through; + } + @keyframes composer-specialist-color-in { from { opacity: 0; diff --git a/src/renderer/src/components/global-search/GlobalSearchDialog.test.tsx b/src/renderer/src/components/global-search/GlobalSearchDialog.test.tsx index 145cfcfa5..07c08e0ec 100644 --- a/src/renderer/src/components/global-search/GlobalSearchDialog.test.tsx +++ b/src/renderer/src/components/global-search/GlobalSearchDialog.test.tsx @@ -3,6 +3,7 @@ import { act } from 'react' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { i18next } from '@/i18n' import type { ChatSession } from '@/stores/session-store' import { createInitialPreviewWorkbenchState, @@ -11,6 +12,7 @@ import { import { createInitialProjectState, useProjectStore } from '@/stores/project-store' import { createInitialSessionState, useSessionStore } from '@/stores/session-store' import { useNavigationStore } from '@/stores/navigation-store' +import { previewLeaveGuards } from '@/stores/preview-leave-guard' import { GlobalSearchDialog } from './GlobalSearchDialog' @@ -38,6 +40,7 @@ beforeEach(() => { configurable: true, value: scrollIntoView }) + previewLeaveGuards.clear() container = document.createElement('div') document.body.appendChild(container) root = createRoot(container) @@ -113,6 +116,37 @@ beforeEach(() => { isIndexComplete: true }) }, + managedFileVersions: { + inspect: vi.fn().mockResolvedValue({ + ok: true, + value: { + source: 'artifact', + projectId: 'project-a', + fileId: 'artifact-1', + sessionId: 'session-a', + displayName: 'sin-head.png', + headVersionId: 'version-2', + selectedVersionId: 'version-2', + versions: [ + { + id: 'version-2', + source: 'artifact', + fileId: 'artifact-1', + versionNumber: 2, + displayName: 'sin-head.png', + originKind: 'user_edit', + basedOnVersionId: 'version-1', + contentType: 'image/png', + sizeBytes: 14, + checksum: '2'.repeat(64), + createdAt: '2026-08-14T00:00:00.000Z' + } + ], + canEdit: false, + canDiff: false + } + }) + }, previewResources: { acquire: vi.fn().mockResolvedValue({ id: 'preview-resource-1', @@ -130,6 +164,7 @@ afterEach(() => { container.remove() Reflect.deleteProperty(window.HTMLElement.prototype, 'scrollIntoView') vi.restoreAllMocks() + void i18next.changeLanguage('en') }) describe('GlobalSearchDialog', () => { @@ -161,9 +196,45 @@ describe('GlobalSearchDialog', () => { act(() => artifactRow.dispatchEvent(new MouseEvent('mouseenter', { bubbles: true }))) const mention = document.body.querySelector('[aria-label="Mention sin.png"]') expect(mention).not.toBeNull() - act(() => mention?.dispatchEvent(new MouseEvent('click', { bubbles: true }))) + await act(async () => { + mention?.dispatchEvent(new MouseEvent('click', { bubbles: true })) + await Promise.resolve() + }) - expect(useNavigationStore.getState().pendingArtifactMention).toMatchObject({ id: 'artifact-1' }) + expect(useNavigationStore.getState().pendingArtifactMention).toMatchObject({ + id: 'artifact-1', + sourceFileId: 'artifact-1', + sourceVersionId: 'version-2', + name: 'sin-head.png' + }) + }) + + it('keeps Global Search open and inserts nothing when mention head resolution fails', async () => { + window.api.managedFileVersions.inspect = vi.fn().mockResolvedValue({ + ok: false, + error: { code: 'VERSION_NOT_FOUND', message: 'Current file head is unavailable.' } + }) + const onOpenChange = vi.fn() + await act(async () => { + root.render() + await new Promise((resolve) => window.setTimeout(resolve, 20)) + }) + const artifactRow = [...document.body.querySelectorAll('[role="option"]')].find((element) => + element.textContent?.includes('sin.png') + ) as HTMLElement + act(() => artifactRow.dispatchEvent(new MouseEvent('mouseenter', { bubbles: true }))) + const mention = document.body.querySelector('[aria-label="Mention sin.png"]') + await act(async () => i18next.changeLanguage('zh-Hans')) + + await act(async () => { + mention?.dispatchEvent(new MouseEvent('click', { bubbles: true })) + await Promise.resolve() + }) + + expect(useNavigationStore.getState().pendingArtifactMention).toBeUndefined() + expect(onOpenChange).not.toHaveBeenCalledWith(false) + expect(document.body.textContent).toContain('无法解析文件版本。') + expect(document.body.textContent).not.toContain('Current file head is unavailable.') }) it('prioritizes Artifacts and selects the first Artifact for a keyword search', async () => { @@ -203,6 +274,7 @@ describe('GlobalSearchDialog', () => { artifactId: 'artifact-1', projectId: 'project-a' }) + expect(usePreviewWorkbenchStore.getState().fileDialogItem?.selectedVersionId).toBeUndefined() }) it('waits for Artifact search before showing a complete keyword result set', async () => { @@ -418,6 +490,97 @@ describe('GlobalSearchDialog', () => { expect(onOpenChange).toHaveBeenCalledWith(false) }) + it.each(['closed', 'escape', 'unmounted'] as const)( + 'ignores a late mention inspection after the dialog is %s', + async (lifecycle) => { + let resolveInspect!: ( + result: Awaited> + ) => void + const inspection = new Promise< + Awaited> + >((resolve) => { + resolveInspect = resolve + }) + vi.mocked(window.api.managedFileVersions.inspect).mockReturnValueOnce(inspection) + const onOpenChange = vi.fn() + await act(async () => { + root.render( + + ) + await new Promise((resolve) => window.setTimeout(resolve, 20)) + }) + + const input = document.body.querySelector('input[role="combobox"]') + await act(async () => { + input?.dispatchEvent( + new KeyboardEvent('keydown', { + key: 'Enter', + shiftKey: true, + bubbles: true, + cancelable: true + }) + ) + }) + + if (lifecycle === 'closed') { + await act(async () => { + root.render( + + ) + }) + } else if (lifecycle === 'escape') { + await act(async () => { + input?.dispatchEvent( + new KeyboardEvent('keydown', { key: 'Escape', bubbles: true, cancelable: true }) + ) + }) + expect(onOpenChange).toHaveBeenCalledTimes(1) + } else { + act(() => root.unmount()) + } + + await act(async () => { + resolveInspect({ + ok: true, + value: { + source: 'artifact', + projectId: 'project-a', + fileId: 'artifact-1', + sessionId: 'session-a', + displayName: 'late-head.png', + headVersionId: 'version-late', + selectedVersionId: 'version-late', + versions: [ + { + id: 'version-late', + source: 'artifact', + fileId: 'artifact-1', + versionNumber: 3, + displayName: 'late-head.png', + originKind: 'user_edit', + basedOnVersionId: 'version-2', + contentType: 'image/png', + sizeBytes: 15, + checksum: '3'.repeat(64), + createdAt: '2026-08-14T01:00:00.000Z' + } + ], + canEdit: false, + canDiff: false + } + }) + await Promise.resolve() + }) + + expect(useNavigationStore.getState().pendingArtifactMention).toBeUndefined() + expect(onOpenChange).toHaveBeenCalledTimes(lifecycle === 'escape' ? 1 : 0) + } + ) + it('opens the active Artifact on Shift+Enter when the current Session cannot accept a mention', async () => { useNavigationStore.setState({ artifactMentionAvailability: { projectId: 'project-a', canMention: false } @@ -504,6 +667,60 @@ describe('GlobalSearchDialog', () => { expect(onOpenChange).toHaveBeenCalledWith(false) }) + it('keeps search open and does not open a cross-project preview when leaving is rejected', async () => { + vi.mocked(window.api.projectFiles.searchArtifacts).mockResolvedValue({ + primary: { items: [], totalCount: 0 }, + other: [ + { + ...artifact, + id: 'artifact-2', + sourceFileId: 'artifact-2', + sourceVersionId: 'version-2', + projectId: 'project-b', + sessionId: 'session-b', + name: 'other.png', + path: 'artifact-version:project-b/session-b/artifact-2/version-2' + } + ], + isIndexComplete: true + }) + usePreviewWorkbenchStore.setState({ + activeProjectId: 'project-a', + activeItemId: 'dirty-file' + }) + previewLeaveGuards.register('workbench:project-a:dirty-file', () => false) + const onOpenChange = vi.fn() + await act(async () => { + root.render() + await new Promise((resolve) => window.setTimeout(resolve, 20)) + }) + const input = document.body.querySelector('input[role="combobox"]') + const valueSetter = Object.getOwnPropertyDescriptor( + window.HTMLInputElement.prototype, + 'value' + )?.set + await act(async () => { + valueSetter?.call(input, 'other.png') + input?.dispatchEvent(new Event('input', { bubbles: true })) + await new Promise((resolve) => window.setTimeout(resolve, 180)) + }) + onOpenChange.mockClear() + await act(async () => { + input?.dispatchEvent( + new KeyboardEvent('keydown', { + key: 'Enter', + shiftKey: true, + bubbles: true, + cancelable: true + }) + ) + }) + + expect(useNavigationStore.getState().activeProjectId).toBe('project-a') + expect(usePreviewWorkbenchStore.getState().fileDialogItem).toBeUndefined() + expect(onOpenChange).not.toHaveBeenCalledWith(false) + }) + it('uses the source message creation time for a legacy artifact', async () => { const createdAt = Date.now() - 4 * 24 * 60 * 60 * 1_000 useSessionStore.setState((state) => ({ diff --git a/src/renderer/src/components/global-search/GlobalSearchDialog.tsx b/src/renderer/src/components/global-search/GlobalSearchDialog.tsx index cc74413aa..c2a90b329 100644 --- a/src/renderer/src/components/global-search/GlobalSearchDialog.tsx +++ b/src/renderer/src/components/global-search/GlobalSearchDialog.tsx @@ -4,7 +4,7 @@ * states: default · hover · focus · active · disabled · loading · error · success * contrast: inherited from the app's verified semantic tokens · slop: pass */ -import { useCallback, useEffect, useId, useMemo, useRef, useState } from 'react' +import { useCallback, useEffect, useId, useLayoutEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { ArrowUpRight, AtSign, Hash, LoaderCircle, MessageCircle, Search, Zap } from 'lucide-react' import { Dialog } from 'radix-ui' @@ -85,7 +85,7 @@ const artifactToPreviewItem = ( size: artifact.size, mtimeMs: artifact.mtimeMs, artifactId: artifact.source === 'artifact' ? artifact.sourceFileId : undefined, - selectedVersionId: artifact.source === 'artifact' ? artifact.sourceVersionId : undefined, + managedFileId: artifact.sourceFileId, originSession: artifact.originSession }) @@ -137,6 +137,7 @@ export const GlobalSearchDialog = ({ const inputRef = useRef(null) const requestVersionRef = useRef(0) const keyboardNavigationRef = useRef(false) + const mentionVersionRef = useRef(0) const listboxId = useId() const [query, setQuery] = useState('') const [visibleSessionCount, setVisibleSessionCount] = useState(GLOBAL_SEARCH_PAGE_SIZE) @@ -147,6 +148,17 @@ export const GlobalSearchDialog = ({ const [actionError, setActionError] = useState() const [activeIndex, setActiveIndex] = useState(0) + useLayoutEffect(() => { + if (!open) mentionVersionRef.current += 1 + }, [open]) + + useLayoutEffect( + () => () => { + mentionVersionRef.current += 1 + }, + [] + ) + const allProjects = useProjectStore((state) => state.projects) const allSessions = useSessionStore((state) => state.sessions) const archivedSessionIds = useSessionStore( @@ -447,7 +459,14 @@ export const GlobalSearchDialog = ({ document.getElementById(activeRowId)?.scrollIntoView?.({ block: 'nearest' }) }, [activeRowId, open, selectableRows.length]) - const close = useCallback(() => onOpenChange(false), [onOpenChange]) + const handleOpenChange = useCallback( + (nextOpen: boolean): void => { + if (!nextOpen) mentionVersionRef.current += 1 + onOpenChange(nextOpen) + }, + [onOpenChange] + ) + const close = useCallback(() => handleOpenChange(false), [handleOpenChange]) const isArtifactMentionTarget = useCallback( (artifact: ProjectFileItem): boolean => view === 'workspace' && @@ -467,7 +486,7 @@ export const GlobalSearchDialog = ({ const previewArtifact = useCallback( (artifact: ProjectFileItem): void => { if (activeProjectId !== artifact.projectId || view !== 'workspace') { - openProject(artifact.projectId, 'user') + if (!openProject(artifact.projectId, 'user')) return } openFileDialog(artifactToPreviewItem(artifact)) close() @@ -475,12 +494,50 @@ export const GlobalSearchDialog = ({ [activeProjectId, close, openFileDialog, openProject, view] ) const mentionArtifact = useCallback( - (artifact: ProjectFileItem): void => { + async (artifact: ProjectFileItem): Promise => { if (!canMentionArtifact(artifact)) return - requestArtifactMention(artifact) - close() + const requestVersion = ++mentionVersionRef.current + setActionError(undefined) + const inspect = window.api.managedFileVersions?.inspect + if (!inspect) { + setActionError(t('File version resolution is unavailable.')) + return + } + try { + const result = await inspect({ + source: artifact.source, + projectId: artifact.projectId, + fileId: artifact.sourceFileId + }) + if (requestVersion !== mentionVersionRef.current) return + if (!result.ok) { + setActionError(t('Could not resolve file version.')) + return + } + const head = result.value.versions.find( + (version) => version.id === result.value.headVersionId + ) + if (!head) { + setActionError(t('The current file version is unavailable.')) + return + } + requestArtifactMention({ + ...artifact, + sourceVersionId: head.id, + checksum: head.checksum, + sessionId: result.value.sessionId, + name: result.value.displayName, + mimeType: head.contentType ?? artifact.mimeType, + size: head.sizeBytes, + sortAtMs: Date.parse(head.createdAt) + }) + close() + } catch { + if (requestVersion !== mentionVersionRef.current) return + setActionError(t('Could not resolve file version.')) + } }, - [canMentionArtifact, close, requestArtifactMention] + [canMentionArtifact, close, requestArtifactMention, t] ) const activate = useCallback( (row: SelectableRow | undefined, action?: 'mention' | 'preview'): void => { @@ -502,7 +559,7 @@ export const GlobalSearchDialog = ({ } if (row.kind === 'artifact') { if (action === 'mention' && canMentionArtifact(row.artifact)) { - mentionArtifact(row.artifact) + void mentionArtifact(row.artifact) } else previewArtifact(row.artifact) return } @@ -669,6 +726,7 @@ export const GlobalSearchDialog = ({ source={artifact.source} projectId={artifact.projectId} sessionId={artifact.sessionId} + managedFileId={artifact.sourceFileId} /> @@ -702,7 +760,7 @@ export const GlobalSearchDialog = ({ disabled={!canMention} onClick={(event) => { event.stopPropagation() - mentionArtifact(artifact) + void mentionArtifact(artifact) }} >