diff --git a/.bun-version b/.bun-version new file mode 100644 index 000000000..88c5fb891 --- /dev/null +++ b/.bun-version @@ -0,0 +1 @@ +1.4.0 diff --git a/.github/workflows/build-binary.yml b/.github/workflows/build-binary.yml new file mode 100644 index 000000000..17c4de5bc --- /dev/null +++ b/.github/workflows/build-binary.yml @@ -0,0 +1,247 @@ +name: Build Binaries + +# Builds the Bun single-file executables (`npm run build:binary`, six targets) +# and smoke-tests every one of them on real hardware: +# +# build (Linux x64) -> smoke linux-arm64 (Blacksmith Ubuntu ARM) +# -> smoke windows-x64 (Blacksmith Windows Server 2025) +# -> smoke windows-arm64 (GitHub-hosted windows-11-arm) +# -> smoke darwin-arm64 (Blacksmith macOS 15, Apple Silicon) +# -> smoke darwin-x64 (GitHub-hosted macos-15-intel) +# +# All six targets are cross-compiled on ONE Linux runner: Bun downloads a +# ~80 MB runtime per target and compiling is a matter of seconds, whereas +# cross-compiling ON a Windows runner is broken upstream (oven-sh/bun#11198, +# the workspace and the Bun cache sit on different drives). The Linux job also +# smoke-tests its own linux-x64 output twice, once against SQLite and once +# against the PostgreSQL service container (service containers are Linux-only, +# which is one reason the smoke matrix is a separate job). +# +# The smoke matrix mixes runner vendors because Blacksmith offers no Windows +# ARM64 and no macOS Intel runners; those two targets run on GitHub-hosted +# runners, which are free for public repositories. Every smoke job only +# downloads its artifact and runs `node scripts/smoke-binary.ts`, so it needs +# neither `npm ci` nor Bun. +# +# The build job's `npm run build:binary` runs `npm run build:web`, which fetches +# https://oomol.com/en/apps/catalog.json (the provider icon map) at build time, +# so an outage there fails the build job. This is the first PR-gated path +# through the web build. +# +# darwin binaries built on Linux carry an invalid ad-hoc signature (Bun 1.4.0 +# writes a wrong page hash, which macOS 27 rejects), so the macOS smoke jobs +# re-sign the binary with `codesign` and verify the signature before running it. +# +# A `runs-on` label that no runner serves queues forever instead of failing, so +# `workflow_dispatch` is here to verify the labels before relying on them. +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +# Cancel superseded runs for the same ref (e.g. new push to an open PR). +concurrency: + group: build-binary-${{ github.ref }} + cancel-in-progress: true + +# Least privilege: these jobs only read the repository; the binaries are +# workflow artifacts, nothing is published. +permissions: + contents: read + +env: + # This project runs TypeScript directly with `node` (native type stripping) and + # forbids `tsx` / `--experimental-strip-types` (see AGENTS.md). Type stripping is + # on by default only on Node >= 22.18 / >= 23.6, so pin an LTS that supports it. + # Do NOT downgrade below 22.18 — `npm ci` (postinstall codegen) and typecheck fail. + NODE_VERSION: "24" + +jobs: + build: + name: Build all targets + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + services: + postgres: + image: postgres:15-alpine + env: + POSTGRES_DB: open_connector_test + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + # No step needs the persisted GITHUB_TOKEN in git config; dropping it + # keeps `npm ci` postinstall scripts from reading it (zizmor hardening). + persist-credentials: false + + - name: Setup Node.js ${{ env.NODE_VERSION }} + uses: actions/setup-node@v7 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + + # Pinned to an immutable SHA rather than the mutable v2 tag: this + # third-party action installs the toolchain that produces the shipped + # binaries, so its supply chain must not be silently updatable. + # `.bun-version` is the single owner of the pinned Bun version; the build + # script refuses to run under any other version. + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: .bun-version + + # Bun downloads one runtime per cross-compile target (~80 MB each) from + # registry.npmjs.org into ~/.bun/install/cache; setup-bun caches only the + # host executable. The key changes with the pinned Bun version. + - name: Cache Bun cross-compile runtimes + uses: actions/cache@v6 + with: + path: ~/.bun/install/cache/bun-*-v* + key: bun-runtimes-${{ runner.os }}-${{ hashFiles('.bun-version') }} + + # Runs postinstall codegen (provider registry + catalog). + - name: Install dependencies + run: npm ci + + # Regenerates the catalog, builds the web console, then compiles all six targets. + - name: Build binaries + run: npm run build:binary + + # SQLite mode: the embedded catalog, migrations and console all come from the binary. + - name: Smoke test linux-x64 (SQLite) + run: node scripts/smoke-binary.ts dist/open-connector-linux-x64 + + # PostgreSQL mode: the binary's own `migrate` subcommand applies the embedded + # postgresql/ migrations, then the server must pass its schema check. + - name: Migrate PostgreSQL with linux-x64 + env: + OOMOL_CONNECT_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/open_connector_test + run: dist/open-connector-linux-x64 migrate + + - name: Smoke test linux-x64 (PostgreSQL) + env: + OOMOL_CONNECT_DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/open_connector_test + run: node scripts/smoke-binary.ts dist/open-connector-linux-x64 + + # One artifact per target so each smoke job downloads only its own ~35 MB + # zip. upload-artifact drops the exec bit; the smoke jobs chmod after download. + - name: Upload linux-x64 + uses: actions/upload-artifact@v7 + with: + name: binary-linux-x64 + path: dist/open-connector-linux-x64 + if-no-files-found: error + retention-days: 1 + + - name: Upload linux-arm64 + uses: actions/upload-artifact@v7 + with: + name: binary-linux-arm64 + path: dist/open-connector-linux-arm64 + if-no-files-found: error + retention-days: 1 + + - name: Upload darwin-x64 + uses: actions/upload-artifact@v7 + with: + name: binary-darwin-x64 + path: dist/open-connector-darwin-x64 + if-no-files-found: error + retention-days: 1 + + - name: Upload darwin-arm64 + uses: actions/upload-artifact@v7 + with: + name: binary-darwin-arm64 + path: dist/open-connector-darwin-arm64 + if-no-files-found: error + retention-days: 1 + + - name: Upload windows-x64 + uses: actions/upload-artifact@v7 + with: + name: binary-windows-x64 + path: dist/open-connector-windows-x64.exe + if-no-files-found: error + retention-days: 1 + + - name: Upload windows-arm64 + uses: actions/upload-artifact@v7 + with: + name: binary-windows-arm64 + path: dist/open-connector-windows-arm64.exe + if-no-files-found: error + retention-days: 1 + + # Run each remaining target on its native OS and architecture. linux-x64 was + # already exercised by the build job. + smoke: + name: Smoke ${{ matrix.target }} + needs: build + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - target: linux-arm64 + runner: blacksmith-2vcpu-ubuntu-2404-arm + file: open-connector-linux-arm64 + - target: windows-x64 + runner: blacksmith-2vcpu-windows-2025 + file: open-connector-windows-x64.exe + # Blacksmith has no Windows ARM64 runners. + - target: windows-arm64 + runner: windows-11-arm + file: open-connector-windows-arm64.exe + - target: darwin-arm64 + runner: blacksmith-6vcpu-macos-15 + file: open-connector-darwin-arm64 + # Blacksmith has no macOS Intel runners. + - target: darwin-x64 + runner: macos-15-intel + file: open-connector-darwin-x64 + runs-on: ${{ matrix.runner }} + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + persist-credentials: false + + # The smoke script uses only node: builtins, so no `npm ci` is needed. + - name: Setup Node.js ${{ env.NODE_VERSION }} + uses: actions/setup-node@v7 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Download ${{ matrix.target }} binary + uses: actions/download-artifact@v8 + with: + name: binary-${{ matrix.target }} + path: dist + + # Artifact zips do not preserve file modes. + - name: Restore executable bit + if: runner.os != 'Windows' + run: chmod +x dist/${{ matrix.file }} + + # Linux-built darwin binaries carry an invalid ad-hoc signature (Bun 1.4.0 + # writes a wrong page hash); re-sign them and assert the signature is valid, + # since macOS 15 would tolerate an invalid one silently. + - name: Ad-hoc sign and verify + if: runner.os == 'macOS' + run: | + codesign --force --sign - dist/${{ matrix.file }} + codesign --verify --verbose=2 dist/${{ matrix.file }} + + - name: Smoke test ${{ matrix.target }} + run: node scripts/smoke-binary.ts dist/${{ matrix.file }} diff --git a/README.md b/README.md index 847ff8a91..3a179f2cf 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,7 @@ Issues and pull requests are welcome. - [Fly.io deployment](docs/fly-io.md) - [Cloudflare deployment](docs/cloudflare.md) - [Docker image (GHCR)](docs/docker-ghcr.md) +- [Single binary](docs/single-binary.md) - [Configuration](docs/configuration.md) - [Credentials and OAuth](docs/credentials.md) - [Catalog format](docs/catalog-format.md) diff --git a/docs/single-binary.md b/docs/single-binary.md new file mode 100644 index 000000000..5126a5393 --- /dev/null +++ b/docs/single-binary.md @@ -0,0 +1,109 @@ +# Single Binary + +OpenConnector can be compiled into one self-contained executable per platform with +[Bun](https://bun.com/docs/bundler/executables). The binary embeds the runtime, the generated +provider catalog, the database migrations, and the built web console, so it runs from any directory +without a Node.js installation, a checkout, or `node_modules`. Nothing is extracted to disk at +runtime. + +## Build + +Building requires Node.js for the npm scripts and the Bun version pinned in `.bun-version`; the +build script exits with an error under any other Bun version. Compile on Linux or macOS. +Cross-compiling on a Windows host fails upstream +([oven-sh/bun#11198](https://github.com/oven-sh/bun/issues/11198)). + +```bash +npm install +npm run build:binary +``` + +`npm run build:binary` regenerates the catalog, builds the web console, and then writes six files +under `dist/`: + +```text +dist/open-connector-linux-x64 +dist/open-connector-linux-arm64 +dist/open-connector-darwin-x64 +dist/open-connector-darwin-arm64 +dist/open-connector-windows-x64.exe +dist/open-connector-windows-arm64.exe +``` + +Each file is roughly 145 to 170 MiB. To build a subset, pass one or more target names after `--`: + +```bash +npm run build:binary -- linux-x64 darwin-arm64 +``` + +Bun downloads a runtime for every target that differs from the host (about 80 MB each, from +`registry.npmjs.org`) into `~/.bun/install/cache` on first use. These downloads are not integrity +checked by Bun; TLS is the only protection. + +`npm run build:web` also fetches the provider icon map from https://oomol.com/en/apps/catalog.json +and fails when it is unreachable. + +`.bun-version` and the `@types/bun` devDependency pin in `package.json` are bumped together. + +## Run + +The binary takes the same environment variables as `npm start`; see +[configuration.md](configuration.md) for the full reference. The ones you will usually set: + +| Variable | Default | Meaning | +| ---------------------------- | ----------- | ----------------------------------------------------------------- | +| `OOMOL_CONNECT_DATA_DIR` | `./data` | SQLite database, transit files, and upload staging. | +| `PORT` | `3000` | HTTP port. | +| `HOST` | `127.0.0.1` | Bind address. | +| `OOMOL_CONNECT_DATABASE_URL` | unset | PostgreSQL connection URL. When unset, SQLite under the data dir. | + +```bash +OOMOL_CONNECT_DATA_DIR="$HOME/open-connector-data" \ +PORT=3000 \ +./dist/open-connector-linux-x64 +``` + +With SQLite, migrations are applied automatically when the database opens, exactly as with +`npm start`. + +### PostgreSQL Migrations + +PostgreSQL migrations are explicit. The binary has a `migrate` subcommand that applies the embedded +migrations and exits without starting the server. Run it before the first start and before starting +a newer binary that contains pending migrations: + +```bash +OOMOL_CONNECT_DATABASE_URL="postgresql://open_connector:password@db.example.com:5432/open_connector?sslmode=verify-full" \ +./dist/open-connector-linux-x64 migrate + +OOMOL_CONNECT_DATABASE_URL="postgresql://open_connector:password@db.example.com:5432/open_connector?sslmode=verify-full" \ +./dist/open-connector-linux-x64 +``` + +The server only checks schema readiness at startup and refuses to start when migrations are +missing; it never applies PostgreSQL DDL itself. Without `OOMOL_CONNECT_DATABASE_URL`, `migrate` +prints a notice that SQLite migrations are applied automatically and exits. + +## Differences From `npm start` + +- `NODE_ENV` is fixed to `production` inside the binary, so logs are always JSON (no pretty + printing). `OOMOL_CONNECT_LOG_LEVEL` and every other environment variable are read at runtime as + usual. +- macOS: binaries built on macOS are ad-hoc signed by the build script. Binaries built on another + operating system carry an invalid ad-hoc signature, and macOS 27 and newer refuses to run them + until you re-sign them: + + ```bash + codesign --force --sign - dist/open-connector-darwin-arm64 + ``` + + `codesign --verify --verbose=2 ` prints "valid on disk" for a usable binary and "invalid + signature" for one that still needs re-signing. + +## Notes + +- Like `npm start`, the binary does not load `.env` files (Bun's automatic loading is disabled at + build time). +- On Windows, stopping the process from a process manager or `taskkill` terminates it immediately; + the graceful shutdown hook that closes the HTTP server and the database on Linux and macOS does + not run. This matches `node src/server/index.ts` on Windows. diff --git a/package-lock.json b/package-lock.json index c04c812ea..9dee8eff4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -45,6 +45,7 @@ "@electric-sql/pglite-socket": "^0.2.8", "@hyrious/configs": "^0.1.6", "@types/ali-oss": "^6.23.3", + "@types/bun": "1.4.0", "@types/busboy": "^1.5.4", "@types/mailparser": "^3.4.6", "@types/mdast": "^4.0.4", @@ -4125,6 +4126,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/bun": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@types/bun/-/bun-1.4.0.tgz", + "integrity": "sha512-K+lZULY23vRgK/CfTjFIV+tyifaNdSMlPh9j+6mQ/cLfpOznLyAuzgV/JQysyECpkBQLVMSyvjlr2fBUSA9wFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bun-types": "1.4.0" + } + }, "node_modules/@types/busboy": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-1.5.4.tgz", @@ -5115,6 +5126,16 @@ "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==", "license": "MIT" }, + "node_modules/bun-types": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/bun-types/-/bun-types-1.4.0.tgz", + "integrity": "sha512-iIKw23BspnQQYd3prITOBxeUsxBHnwzX6YJfGMuNOZzeNcMmVqzIIVGRm1l69ogaPQmb4wB6BN8mA5bE9YuC5Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", diff --git a/package.json b/package.json index a144779cf..3c5fc22bb 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "dev": "node scripts/dev-local.ts", "build": "npm run typecheck", "build:web": "npm run build --workspace web", + "build:binary": "npm run generate:catalog && npm run build:web && bun scripts/build-binary.ts", "dev:cloudflare": "npm run generate:catalog && npm run build:web && node scripts/copy-catalog-assets.ts && wrangler dev --config wrangler.local.jsonc", "deploy:cloudflare": "npm run generate:catalog && npm run build:web && node scripts/copy-catalog-assets.ts && wrangler deploy --config wrangler.local.jsonc --minify", "start": "node scripts/ensure-generated.ts && node src/server/index.ts", @@ -61,6 +62,7 @@ "@electric-sql/pglite-socket": "^0.2.8", "@hyrious/configs": "^0.1.6", "@types/ali-oss": "^6.23.3", + "@types/bun": "1.4.0", "@types/busboy": "^1.5.4", "@types/mailparser": "^3.4.6", "@types/mdast": "^4.0.4", diff --git a/scripts/build-binary.ts b/scripts/build-binary.ts new file mode 100644 index 000000000..b9e835107 --- /dev/null +++ b/scripts/build-binary.ts @@ -0,0 +1,194 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs"; +import { join, relative, resolve } from "node:path"; + +// Build the single-file server executables with Bun. +// +// Usage: `bun scripts/build-binary.ts [target-name ...]` (normally through `npm run build:binary`, which +// regenerates the catalog and builds the web console first). No arguments builds every target. + +interface BinaryTarget { + /** Suffix of the output file: `dist/open-connector-` (Bun appends `.exe` for Windows targets). */ + name: string; + target: Bun.Build.CompileTarget; +} + +const rootDir = resolve(import.meta.dirname, ".."); +const bunVersionFile = join(rootDir, ".bun-version"); +const targets: BinaryTarget[] = [ + { name: "linux-x64", target: "bun-linux-x64" }, + { name: "linux-arm64", target: "bun-linux-arm64" }, + { name: "darwin-x64", target: "bun-darwin-x64" }, + { name: "darwin-arm64", target: "bun-darwin-arm64" }, + { name: "windows-x64", target: "bun-windows-x64" }, + { name: "windows-arm64", target: "bun-windows-arm64" }, +]; + +assertPinnedBunVersion(); +assertBuildInputs(); +for (const target of selectTargets(process.argv.slice(2))) { + await buildTarget(target); +} + +/** `.bun-version` is the single owner of the pinned Bun version; CI installs it and local builds must match it. */ +function assertPinnedBunVersion(): void { + const pinnedVersion = readFileSync(bunVersionFile, "utf8").trim(); + if (Bun.version !== pinnedVersion) { + fail( + `Bun ${Bun.version} is running but .bun-version pins ${pinnedVersion}. Install it with: curl -fsSL https://bun.sh/install | bash -s "bun-v${pinnedVersion}"`, + ); + } +} + +/** The embedded directories are generated; refuse to build a binary that would ship an empty catalog or console. */ +function assertBuildInputs(): void { + const problems: string[] = []; + if (listFiles(join(rootDir, "catalog/apps"), ".json").length === 0) { + problems.push("catalog/apps contains no .json files"); + } + if (listFiles(join(rootDir, "migrations"), ".sql").length === 0) { + problems.push("migrations contains no .sql files"); + } + if (listFiles(join(rootDir, "migrations/postgresql"), ".sql").length === 0) { + problems.push("migrations/postgresql contains no .sql files"); + } + if (!existsSync(join(rootDir, "dist/web/index.html"))) { + problems.push("dist/web/index.html is missing"); + } + if (problems.length > 0) { + fail( + `Build inputs are missing:\n - ${problems.join("\n - ")}\nRun "npm run build:binary" so the catalog and the web console are generated before the binary is built.`, + ); + } +} + +function listFiles(directory: string, extension: string): string[] { + try { + return readdirSync(directory).filter((name) => name.endsWith(extension)); + } catch { + return []; + } +} + +function selectTargets(names: string[]): BinaryTarget[] { + if (names.length === 0) { + return targets; + } + + return names.map((name) => { + const target = targets.find((candidate) => candidate.name === name); + if (!target) { + fail(`Unknown target "${name}". Valid targets: ${targets.map((candidate) => candidate.name).join(", ")}.`); + } + + return target; + }); +} + +async function buildTarget(binaryTarget: BinaryTarget): Promise { + const outfile = join(rootDir, "dist", `open-connector-${binaryTarget.name}`); + // Bun appends .exe to Windows outputs; clear both spellings so an earlier build can never be mistaken for this one. + rmSync(outfile, { force: true }); + rmSync(`${outfile}.exe`, { force: true }); + + const result = await runBunBuild(binaryTarget.target, outfile); + const [artifact] = result.outputs; + if (!artifact) { + fail(`Bun.build produced no output for ${binaryTarget.name}.`); + } + + if (binaryTarget.target.startsWith("bun-darwin-")) { + signDarwinBinary(artifact.path); + } + + const sizeMiB = statSync(artifact.path).size / (1024 * 1024); + console.log(`built ${relative(rootDir, artifact.path)} (${sizeMiB.toFixed(1)} MiB)`); +} + +async function runBunBuild(target: Bun.Build.CompileTarget, outfile: string): Promise { + let result: Bun.BuildOutput; + try { + result = await Bun.build({ + entrypoints: [join(rootDir, "src/server/index.ts")], + target: "bun", + format: "esm", + // ali-oss depends on urllib, which lazily `require("proxy-agent")`, an optional peer dependency that is not + // installed here. The bundler cannot resolve it, so it stays a runtime require that is never reached. + external: ["proxy-agent"], + // Bun inlines `process.env.NODE_ENV` at compile time ("development" unless defined). src/server/logger.ts then + // loads the pino-pretty worker transport, which cannot run inside a standalone executable. Every other + // environment variable is still read at runtime. + define: { "process.env.NODE_ENV": JSON.stringify("production") }, + compile: { + target, + outfile, + // Each directory is embedded under its basename next to the bundle: migrations/, apps/ and web/. + // catalog/apps rather than catalog/: an interrupted `npm run generate:catalog` leaves catalog/.apps-- + // temp directories behind, and they must never end up inside a release. + assets: [join(rootDir, "migrations"), join(rootDir, "catalog/apps"), join(rootDir, "dist/web")], + // `node src/server/index.ts` reads neither .env nor bunfig.toml; keep the binary's configuration surface the same. + autoloadDotenv: false, + autoloadBunfig: false, + }, + }); + } catch (error) { + // Bun.build rejects with an AggregateError whose `errors` hold the BuildMessage / ResolveMessage entries. + if (error instanceof AggregateError) { + for (const message of error.errors) { + console.error(describeBuildMessage(message)); + } + fail(`Bundling ${target} failed.`); + } + + throw error; + } + + for (const message of result.logs) { + console.error(describeBuildMessage(message)); + } + if (!result.success) { + fail(`Bundling ${target} failed.`); + } + + return result; +} + +function describeBuildMessage(message: unknown): string { + if (message instanceof BuildMessage || message instanceof ResolveMessage) { + const position = message.position; + const location = position ? ` (${position.file}:${position.line}:${position.column})` : ""; + return `${message.level}: ${message.message}${location}`; + } + + return String(message); +} + +/** + * Bun 1.4.0 writes an ad-hoc signature whose last page hash is wrong (oven-sh/bun#39837, fixed upstream but + * unreleased). macOS 27 refuses to start such a binary (SIGKILL at exec), so re-sign it in place. codesign only + * exists on macOS; darwin outputs built elsewhere are re-signed on the macOS smoke runner instead. + */ +function signDarwinBinary(output: string): void { + const displayPath = relative(rootDir, output); + if (process.platform !== "darwin") { + console.warn( + `warning: ${displayPath} still carries Bun's invalid ad-hoc signature because codesign is only available on macOS; run "codesign --force --sign - ${displayPath}" on macOS 27 or later before executing it.`, + ); + return; + } + + const result = spawnSync("codesign", ["--force", "--sign", "-", output], { stdio: "inherit" }); + if (result.error) { + throw result.error; + } + if (result.status !== 0) { + fail( + `codesign failed for ${displayPath} (${result.signal ? `signal ${result.signal}` : `exit code ${result.status}`}).`, + ); + } +} + +function fail(message: string): never { + console.error(message); + process.exit(1); +} diff --git a/scripts/runtime-data.ts b/scripts/runtime-data.ts index 2bccb495a..e9b59d50b 100644 --- a/scripts/runtime-data.ts +++ b/scripts/runtime-data.ts @@ -1,11 +1,13 @@ import { mkdir } from "node:fs/promises"; import { join, resolve } from "node:path"; import { parseArgs } from "node:util"; -import { Pool } from "pg"; import { logger } from "../src/server/logger.ts"; import { createSecretCodec } from "../src/server/secrets/secret-codec.ts"; -import { assertPostgresDatabaseUrl, createNodeRuntimeDatabase } from "../src/server/storage/node-runtime-database.ts"; -import { migratePostgresDatabase } from "../src/server/storage/postgres-migrations.ts"; +import { + createNodeRuntimeDatabase, + migratePostgresRuntimeDatabase, + sqliteMigrationsNotice, +} from "../src/server/storage/node-runtime-database.ts"; const { positionals, values: options } = parseArgs({ args: process.argv.slice(2), @@ -49,20 +51,13 @@ const dataDir = resolve(options["data-dir"] ?? process.env.OOMOL_CONNECT_DATA_DI const databasePath = join(dataDir, "connect.sqlite"); if (command === "migrate") { if (!databaseUrl) { - console.log("SQLite migrations are applied automatically when the local runtime database opens."); + console.log(sqliteMigrationsNotice); } else { - assertPostgresDatabaseUrl(databaseUrl); - const pool = new Pool({ - application_name: "open-connector-migrate", + await migratePostgresRuntimeDatabase({ connectionString: databaseUrl, - connectionTimeoutMillis: readPositiveIntegerEnv("OOMOL_CONNECT_DATABASE_CONNECT_TIMEOUT_MS", 10_000), - max: 1, + connectionTimeoutMs: readPositiveIntegerEnv("OOMOL_CONNECT_DATABASE_CONNECT_TIMEOUT_MS", 10_000), + logger, }); - try { - await migratePostgresDatabase({ pool, logger }); - } finally { - await pool.end(); - } } } else { const secretCodec = createSecretCodec(process.env.OOMOL_CONNECT_ENCRYPTION_KEY); diff --git a/scripts/smoke-binary.ts b/scripts/smoke-binary.ts new file mode 100644 index 000000000..2e5d26180 --- /dev/null +++ b/scripts/smoke-binary.ts @@ -0,0 +1,349 @@ +import type { ChildProcess } from "node:child_process"; + +import { strict as assert } from "node:assert"; +import { spawn } from "node:child_process"; +import { access, mkdtemp, rm } from "node:fs/promises"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; + +// Start a built single-file executable against a fresh data directory and check that its embedded catalog, +// web console, migrations and shutdown path work. +// +// Usage: `node scripts/smoke-binary.ts `. Set OOMOL_CONNECT_DATABASE_URL to run the PostgreSQL +// mode; every other OOMOL_CONNECT_* variable is removed from the server's environment. Uses only Node built-ins so +// the smoke runners need no `npm ci`. + +interface ProcessExit { + code: number | null; + signal: NodeJS.Signals | null; +} + +interface ServerProcessOptions { + binaryPath: string; + dataDir: string; + port: number; + databaseUrl: string | undefined; +} + +const healthTimeoutMs = 60_000; +const healthPollIntervalMs = 500; +const requestTimeoutMs = 10_000; +const shutdownTimeoutMs = 10_000; + +/** The server binary with its captured output and exit state. */ +class ServerProcess { + private readonly child: ChildProcess; + private readonly stdoutChunks: Buffer[] = []; + private readonly stderrChunks: Buffer[] = []; + private readonly closed: Promise; + private exitState: ProcessExit | undefined; + private spawnError: Error | undefined; + + constructor(options: ServerProcessOptions) { + this.child = spawn(options.binaryPath, [], { + cwd: options.dataDir, + env: buildServerEnvironment(options), + stdio: ["ignore", "pipe", "pipe"], + }); + this.child.stdout?.on("data", (chunk: Buffer) => this.stdoutChunks.push(chunk)); + this.child.stderr?.on("data", (chunk: Buffer) => this.stderrChunks.push(chunk)); + this.closed = new Promise((resolveClosed) => { + this.child.once("error", (error) => { + this.spawnError = error; + this.exitState ??= { code: null, signal: null }; + resolveClosed(); + }); + this.child.once("close", (code, signal) => { + this.exitState = { code, signal }; + resolveClosed(); + }); + }); + } + + get exit(): ProcessExit | undefined { + return this.exitState; + } + + get stdout(): string { + return Buffer.concat(this.stdoutChunks).toString("utf8"); + } + + get stderr(): string { + return Buffer.concat(this.stderrChunks).toString("utf8"); + } + + describeExit(): string { + if (this.spawnError) { + return `spawn failed: ${this.spawnError.message}`; + } + if (!this.exitState) { + return "still running"; + } + + return this.exitState.signal ? `signal ${this.exitState.signal}` : `exit code ${this.exitState.code}`; + } + + kill(signal: NodeJS.Signals): void { + this.child.kill(signal); + } + + /** Resolve with the exit state, or undefined when the process is still running after the timeout. */ + async waitForExit(timeoutMs: number): Promise { + await Promise.race([this.closed, sleep(timeoutMs, undefined, { ref: false })]); + return this.exitState; + } + + /** Terminate a process that is still running; no-op after it exited. */ + async forceStop(): Promise { + if (this.exitState) { + return; + } + + this.kill("SIGKILL"); + await this.waitForExit(shutdownTimeoutMs); + } +} + +const binaryPath = await resolveBinaryPath(process.argv[2]); +const databaseUrl = process.env.OOMOL_CONNECT_DATABASE_URL?.trim() || undefined; +const mode = databaseUrl ? "postgresql" : "sqlite"; +const startedAt = performance.now(); +// Probe the port before creating the data directory so a probe failure cannot leak the temp directory. +const port = await findFreePort(); +const dataDir = await mkdtemp(join(tmpdir(), "open-connector-smoke-")); +const baseUrl = `http://127.0.0.1:${port}`; +const server = new ServerProcess({ binaryPath, dataDir, port, databaseUrl }); + +try { + await waitForHealth(server, baseUrl); + const readyAt = performance.now(); + const indexHtml = await checkConsoleIndex(baseUrl); + await checkConsoleAssets(baseUrl, indexHtml); + await checkProviders(baseUrl); + await checkApps(baseUrl); + await checkDatabaseBackend(server, dataDir, databaseUrl); + await checkGracefulShutdown(server); + await removeDataDir(dataDir); + console.log( + `PASS ${binaryPath} mode=${mode} startup=${formatMs(readyAt - startedAt)} total=${formatMs(performance.now() - startedAt)}`, + ); +} catch (error) { + console.error(`FAIL ${binaryPath}: ${error instanceof Error ? error.message : String(error)}`); + printServerLogs(server); + process.exitCode = 1; +} finally { + await server.forceStop(); + await removeDataDir(dataDir).catch(() => undefined); +} + +/** Resolve the binary before spawning: spawn resolves relative paths against the child's cwd, which is the data dir. */ +async function resolveBinaryPath(argument: string | undefined): Promise { + if (!argument) { + console.error("Usage: node scripts/smoke-binary.ts "); + process.exit(1); + } + + const resolved = resolve(argument); + try { + await access(resolved); + } catch { + console.error(`Binary not found: ${resolved}`); + process.exit(1); + } + + return resolved; +} + +/** process.env without OOMOL_CONNECT_* so the caller's shell cannot leak configuration into the server under test. */ +function buildServerEnvironment(options: ServerProcessOptions): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const [key, value] of Object.entries(process.env)) { + if (!key.toUpperCase().startsWith("OOMOL_CONNECT_")) { + env[key] = value; + } + } + + env.PORT = String(options.port); + env.HOST = "127.0.0.1"; + env.OOMOL_CONNECT_DATA_DIR = options.dataDir; + if (options.databaseUrl) { + env.OOMOL_CONNECT_DATABASE_URL = options.databaseUrl; + } + + return env; +} + +function findFreePort(): Promise { + return new Promise((resolvePort, reject) => { + const probe = createServer(); + probe.once("error", reject); + probe.listen(0, "127.0.0.1", () => { + const address = probe.address(); + if (!address || typeof address === "string") { + probe.close(); + reject(new Error("Could not determine a free port.")); + return; + } + + const { port: freePort } = address; + probe.close((error) => (error ? reject(error) : resolvePort(freePort))); + }); + }); +} + +async function waitForHealth(server: ServerProcess, baseUrl: string): Promise { + const deadline = Date.now() + healthTimeoutMs; + while (Date.now() < deadline) { + if (server.exit) { + throw new Error(`server exited before /health responded (${server.describeExit()})`); + } + if (await isHealthy(baseUrl)) { + return; + } + + await sleep(healthPollIntervalMs); + } + + throw new Error(`/health did not respond within ${healthTimeoutMs} ms`); +} + +async function isHealthy(baseUrl: string): Promise { + try { + const response = await fetch(`${baseUrl}/health`, { signal: AbortSignal.timeout(healthPollIntervalMs * 4) }); + await response.text(); + return response.status === 200; + } catch { + return false; + } +} + +async function checkConsoleIndex(baseUrl: string): Promise { + const response = await fetch(`${baseUrl}/`, { signal: AbortSignal.timeout(requestTimeoutMs) }); + const body = await response.text(); + assert(response.status === 200, `GET / returned ${response.status}`); + const contentType = response.headers.get("content-type") ?? ""; + assert(contentType.startsWith("text/html"), `GET / content-type is ${contentType || "missing"}`); + assert(body.includes(" { + const assetPaths = collectAssetPaths(indexHtml); + assert(assetPaths.length > 0, 'GET / references no src="/..." or href="/..." assets'); + for (const assetPath of assetPaths) { + const response = await fetch(`${baseUrl}${assetPath}`, { signal: AbortSignal.timeout(requestTimeoutMs) }); + const body = await response.arrayBuffer(); + assert(response.status === 200, `GET ${assetPath} returned ${response.status}`); + const contentType = response.headers.get("content-type") ?? ""; + assert( + !contentType.startsWith("text/html"), + `GET ${assetPath} was answered by the SPA fallback (content-type ${contentType || "missing"})`, + ); + const contentLength = response.headers.get("content-length"); + assert( + Number(contentLength) > 0 && Number(contentLength) === body.byteLength, + `GET ${assetPath} Content-Length ${contentLength ?? "missing"} does not match the ${body.byteLength}-byte body`, + ); + } +} + +/** Root-relative `src` and `href` values; protocol-relative `//host/...` URLs are not served by the binary. */ +function collectAssetPaths(html: string): string[] { + const paths = new Set(); + for (const match of html.matchAll(/\b(?:src|href)="(\/[^"]*)"/g)) { + if (!match[1].startsWith("//")) { + paths.add(match[1]); + } + } + + return [...paths]; +} + +async function checkProviders(baseUrl: string): Promise { + const data = await fetchEnvelopeData(baseUrl, "/v1/providers"); + assert(Array.isArray(data), "/v1/providers data is not an array"); + assert(data.length > 1000, `/v1/providers returned ${data.length} providers; expected more than 1000`); + assert( + data.some((entry) => isRecord(entry) && entry.service === "slack"), + "/v1/providers does not include slack", + ); +} + +async function checkApps(baseUrl: string): Promise { + const data = await fetchEnvelopeData(baseUrl, "/v1/apps"); + assert(Array.isArray(data), "/v1/apps data is not an array"); +} + +/** Return `data` of a `{ success: true, data }` envelope. */ +async function fetchEnvelopeData(baseUrl: string, path: string): Promise { + const response = await fetch(`${baseUrl}${path}`, { signal: AbortSignal.timeout(requestTimeoutMs) }); + const body = await response.text(); + assert(response.status === 200, `GET ${path} returned ${response.status}`); + const payload: unknown = JSON.parse(body); + assert(isRecord(payload), `GET ${path} did not return a JSON object`); + assert(payload.success === true, `GET ${path} envelope success is ${JSON.stringify(payload.success)}`); + return payload.data; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function checkDatabaseBackend( + server: ServerProcess, + dataDir: string, + databaseUrl: string | undefined, +): Promise { + if (databaseUrl) { + assert(server.stdout.includes('"backend":"postgresql"'), 'server log has no "backend":"postgresql" line'); + return; + } + + try { + await access(join(dataDir, "connect.sqlite")); + } catch { + throw new Error(`connect.sqlite was not created in ${dataDir}`); + } +} + +async function checkGracefulShutdown(server: ServerProcess): Promise { + server.kill("SIGTERM"); + const exit = await server.waitForExit(shutdownTimeoutMs); + if (!exit) { + await server.forceStop(); + throw new Error(`server did not exit within ${shutdownTimeoutMs} ms after SIGTERM`); + } + if (exit.code === 0) { + return; + } + // Node's kill() is TerminateProcess on Windows: the SIGTERM handler never runs, so graceful shutdown is not + // exercised there and the forced termination is the expected outcome. + if (process.platform === "win32" && exit.code === null && exit.signal === "SIGTERM") { + return; + } + + throw new Error(`server exited with ${server.describeExit()} after SIGTERM; expected exit code 0`); +} + +/** Windows can report EBUSY for a short while after the process exits; retry instead of failing. */ +function removeDataDir(directory: string): Promise { + return rm(directory, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); +} + +function printServerLogs(server: ServerProcess): void { + console.error(`server ${server.describeExit()}`); + console.error("--- server stdout ---"); + console.error(server.stdout); + console.error("--- server stderr ---"); + console.error(server.stderr); +} + +function formatMs(value: number): string { + return `${Math.round(value)}ms`; +} diff --git a/src/catalog-store.ts b/src/catalog-store.ts index c84eccabc..895985f39 100644 --- a/src/catalog-store.ts +++ b/src/catalog-store.ts @@ -143,10 +143,7 @@ function toProviderSummary(provider: RuntimeProviderDefinition): ProviderSummary /** * Load generated provider catalog files from disk. */ -export async function loadCatalog( - catalogDir: string = join(process.cwd(), "catalog/apps"), - options: LoadCatalogOptions = {}, -): Promise { +export async function loadCatalog(catalogDir: string, options: LoadCatalogOptions = {}): Promise { const entries = await readdir(catalogDir, { withFileTypes: true }); const providers = await Promise.all( entries diff --git a/src/server/api/static-routes.test.ts b/src/server/api/static-routes.test.ts new file mode 100644 index 000000000..5466729c7 --- /dev/null +++ b/src/server/api/static-routes.test.ts @@ -0,0 +1,147 @@ +import { Hono } from "hono"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { registerStaticRoutes } from "./static-routes.ts"; + +const indexHtml = '
'; +const consoleScript = "console.log('ok');"; +const headersFile = "/*\n X-Frame-Options: DENY\n"; +const tempDirs: string[] = []; + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +describe("registerStaticRoutes in embedded mode", () => { + it("serves index.html for the root path", async () => { + const app = await createEmbeddedApp(); + + const response = await app.request("/"); + + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toContain("text/html"); + expect(response.headers.get("content-length")).toBe(String(Buffer.byteLength(indexHtml))); + await expect(response.text()).resolves.toBe(indexHtml); + }); + + it("serves nested assets with their content type", async () => { + const app = await createEmbeddedApp(); + + const response = await app.request("/assets/x.js"); + + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toMatch(/^text\/javascript/); + expect(response.headers.get("content-length")).toBe(String(Buffer.byteLength(consoleScript))); + await expect(response.text()).resolves.toBe(consoleScript); + }); + + it("falls back to application/octet-stream for files without an extension", async () => { + const app = await createEmbeddedApp(); + + const response = await app.request("/_headers"); + + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toBe("application/octet-stream"); + await expect(response.text()).resolves.toBe(headersFile); + }); + + it("answers HEAD with headers only", async () => { + const app = await createEmbeddedApp(); + + const response = await app.request("/assets/x.js", { method: "HEAD" }); + + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toMatch(/^text\/javascript/); + expect(response.headers.get("content-length")).toBe(String(Buffer.byteLength(consoleScript))); + await expect(response.text()).resolves.toBe(""); + }); + + it("serves the console shell for unknown console paths", async () => { + const app = await createEmbeddedApp(); + + const response = await app.request("/providers/github"); + + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toContain("text/html"); + await expect(response.text()).resolves.toBe(indexHtml); + }); + + it("keeps JSON 404 responses for API paths", async () => { + const app = await createEmbeddedApp(); + + const response = await app.request("/v1/nope"); + + expect(response.status).toBe(404); + expect(response.headers.get("content-type")).toContain("application/json"); + await expect(response.json()).resolves.toEqual({ + error: { code: "not_found", message: "Not found." }, + }); + }); + + it("does not serve assets for POST requests", async () => { + const app = await createEmbeddedApp(); + + const response = await app.request("/assets/x.js", { method: "POST" }); + + expect(response.status).toBe(404); + await expect(response.json()).resolves.toEqual({ + error: { code: "not_found", message: "Not found." }, + }); + }); + + it("marks content-hashed assets immutable and leaves index.html uncached", async () => { + const app = await createEmbeddedApp(); + + const asset = await app.request("/assets/x.js"); + const index = await app.request("/index.html"); + + expect(asset.status).toBe(200); + expect(asset.headers.get("cache-control")).toBe("public, max-age=31536000, immutable"); + expect(index.status).toBe(200); + expect(index.headers.get("cache-control")).toBeNull(); + }); + + it("loads the tree into memory at registration", async () => { + const root = await createConsoleRoot(); + const app = new Hono(); + registerStaticRoutes(app, { root, embedded: true }); + await rm(root, { recursive: true, force: true }); + + const response = await app.request("/assets/x.js"); + + expect(response.status).toBe(200); + await expect(response.text()).resolves.toBe(consoleScript); + }); +}); + +describe("registerStaticRoutes in Node mode", () => { + it("streams assets from disk", async () => { + const root = await createConsoleRoot(); + const app = new Hono(); + registerStaticRoutes(app, { root }); + + const response = await app.request("/assets/x.js"); + + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toMatch(/^text\/javascript/); + await expect(response.text()).resolves.toBe(consoleScript); + }); +}); + +async function createEmbeddedApp(): Promise { + const app = new Hono(); + registerStaticRoutes(app, { root: await createConsoleRoot(), embedded: true }); + return app; +} + +async function createConsoleRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), "oomol-connect-static-")); + tempDirs.push(root); + await mkdir(join(root, "assets"), { recursive: true }); + await writeFile(join(root, "index.html"), indexHtml); + await writeFile(join(root, "assets", "x.js"), consoleScript); + await writeFile(join(root, "_headers"), headersFile); + return root; +} diff --git a/src/server/api/static-routes.ts b/src/server/api/static-routes.ts index 017fe9b38..eda8799cf 100644 --- a/src/server/api/static-routes.ts +++ b/src/server/api/static-routes.ts @@ -1,27 +1,57 @@ import type { Hono } from "hono"; import { serveStatic } from "@hono/node-server/serve-static"; +import { getMimeType } from "hono/utils/mime"; +import { readdirSync, readFileSync } from "node:fs"; import { readFile } from "node:fs/promises"; -import { join } from "node:path"; +import { join, relative, sep } from "node:path"; import { isConsoleShellRequest } from "./console-paths.ts"; import { notFound } from "./http-utils.ts"; +export interface StaticRoutesOptions { + /** Built console directory; undefined when the console is not built. */ + root?: string; + /** Serve from an embedded (non-streamable, epoch-mtime) tree: files are loaded into memory once at registration. */ + embedded?: boolean; +} + +interface EmbeddedFile { + /** Backed by a plain ArrayBuffer, which is what Hono's response body type requires. */ + body: Uint8Array; + contentType: string; +} + +/** The console entry file is owned here: `/` resolves to `/index.html` in both serving modes. */ +function rewriteRequestPath(path: string): string { + return path === "/" ? "/index.html" : path; +} + /** * Register static web-console routes for the local server. * * The web console is intentionally outside `src` and may be absent during * backend development. Hono's static middleware handles real files; this * wrapper owns only the fallback behavior for API and browser requests. + * + * Inside a Bun standalone executable the console lives in the embedded tree, + * where `createReadStream` does not work and every mtime is the Unix epoch, so + * the streaming middleware cannot be used. Embedded mode instead reads the + * whole tree into memory once and answers from that map. */ -export function registerStaticRoutes(app: Hono, root?: string): void { +export function registerStaticRoutes(app: Hono, options: StaticRoutesOptions = {}): void { + const root = options.root; if (root) { - app.use( - "*", - serveStatic({ - root, - rewriteRequestPath: (path) => (path === "/" ? "/index.html" : path), - }), - ); + if (options.embedded) { + registerEmbeddedFiles(app, readEmbeddedFiles(root)); + } else { + app.use( + "*", + serveStatic({ + root, + rewriteRequestPath, + }), + ); + } } app.notFound(async (context) => { @@ -48,3 +78,60 @@ export function registerStaticRoutes(app: Hono, root?: string): void { } }); } + +/** + * Walk the console tree synchronously: `createApp()` registers routes + * synchronously, so an asynchronous walk would race the first request. + */ +function readEmbeddedFiles(root: string): Map { + const files = new Map(); + for (const entry of readdirSync(root, { withFileTypes: true, recursive: true })) { + if (!entry.isFile()) { + continue; + } + const absolute = join(entry.parentPath, entry.name); + // The recursive walk yields backslash-separated names on Windows; request + // paths always use forward slashes, so normalise the key or nothing matches. + const key = `/${relative(root, absolute).split(sep).join("/")}`; + files.set(key, { + body: readFileSync(absolute), + contentType: getMimeType(entry.name) ?? "application/octet-stream", + }); + } + return files; +} + +/** + * Serve the in-memory console files. No Last-Modified or ETag headers: embedded + * mtimes are the epoch, so unlike the streaming middleware there is nothing + * truthful to validate against. Content-hashed assets are marked immutable + * instead; everything else is served uncached. + */ +function registerEmbeddedFiles(app: Hono, files: Map): void { + app.use("*", async (context, next) => { + const method = context.req.method; + if (method !== "GET" && method !== "HEAD") { + return next(); + } + + const path = rewriteRequestPath(context.req.path); + const file = files.get(path); + if (!file) { + return next(); + } + + const headers: Record = { + "Content-Type": file.contentType, + "Content-Length": String(file.body.byteLength), + }; + // Vite names everything under assets/ by content hash, so a URL never changes + // meaning and a year-long immutable cache is safe; index.html and the other + // root files keep their names between releases and stay uncached. Embedded + // mode cannot offer Last-Modified as the alternative: every mtime in the + // embedded tree is the Unix epoch, so a validator would never see a change. + if (path.startsWith("/assets/")) { + headers["Cache-Control"] = "public, max-age=31536000, immutable"; + } + return method === "HEAD" ? context.body(null, 200, headers) : context.body(file.body, 200, headers); + }); +} diff --git a/src/server/connect-server.test.ts b/src/server/connect-server.test.ts index 45d839f41..2ac05a65c 100644 --- a/src/server/connect-server.test.ts +++ b/src/server/connect-server.test.ts @@ -3728,7 +3728,7 @@ function createTestServer(providers: ProviderDefinition[], options: CreateTestSe uploadTransitFile: options.uploadTransitFile, runtimeTokens, runtimePolicyStore: options.runtimePolicyStore ?? new MemoryRuntimePolicyStore(), - registerStaticRoutes: staticRoot ? (app) => registerStaticRoutes(app, staticRoot) : undefined, + registerStaticRoutes: staticRoot ? (app) => registerStaticRoutes(app, { root: staticRoot }) : undefined, auth: { ...options.auth, hasRuntimeTokens: async () => (await runtimeTokens.listTokens()).length > 0, diff --git a/src/server/index.ts b/src/server/index.ts index 9afc1c4c8..0bc2337c2 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -3,7 +3,7 @@ import type { ServerType } from "@hono/node-server"; import { S3Client } from "@aws-sdk/client-s3"; import { serve } from "@hono/node-server"; -import { access, mkdir } from "node:fs/promises"; +import { mkdir } from "node:fs/promises"; import { join } from "node:path"; import { loadCatalog } from "../catalog-store.ts"; import { ActionPolicyService, parseActionPolicyList } from "../core/action-policy.ts"; @@ -23,7 +23,12 @@ import { S3TransitFileService } from "./files/s3-transit-files.ts"; import { TransitFileService } from "./files/transit-files.ts"; import { logger } from "./logger.ts"; import { createSecretCodec } from "./secrets/secret-codec.ts"; -import { createNodeRuntimeDatabase } from "./storage/node-runtime-database.ts"; +import { resolveServerAssets } from "./server-assets.ts"; +import { + createNodeRuntimeDatabase, + migratePostgresRuntimeDatabase, + sqliteMigrationsNotice, +} from "./storage/node-runtime-database.ts"; import { DEFAULT_RUN_LIMIT } from "./storage/runtime-store.ts"; const port = Number(process.env.PORT ?? 3000); @@ -37,10 +42,22 @@ const databaseUrl = optionalEnv("OOMOL_CONNECT_DATABASE_URL"); const databasePoolMax = readPositiveIntegerEnv("OOMOL_CONNECT_DATABASE_POOL_MAX", 10); const databaseConnectTimeoutMs = readPositiveIntegerEnv("OOMOL_CONNECT_DATABASE_CONNECT_TIMEOUT_MS", 10_000); +// The standalone binary embeds migrations/postgresql, but the PostgreSQL startup validator refuses to serve until +// they are applied and its error text points at `npm run runtime:migrate`, which a binary user does not have. +// `migrate` applies them from the same source the validator reads, so validation and execution cannot diverge. +const [command, ...rest] = process.argv.slice(2); + try { - await main(); + if (command === undefined) { + await main(); + } else if (command === "migrate" && rest.length === 0) { + await runMigrateCommand(); + } else { + console.error("Usage: open-connector [migrate]"); + process.exitCode = 1; + } } catch (error) { - logger.error({ err: error }, "connect server failed"); + logger.error({ err: error }, command === "migrate" ? "migrate failed" : "connect server failed"); process.exitCode = 1; } @@ -65,8 +82,8 @@ async function main(): Promise { const allowedCustomOAuth = parseActionPolicyList(process.env.OOMOL_CONNECT_ALLOWED_CUSTOM_OAUTH); await mkdir(dataDir, { recursive: true }); - const staticRoot = await resolveStaticRoot(join(process.cwd(), "dist/web")); - const catalog = await loadCatalog(undefined, { + const assets = await resolveServerAssets(); + const catalog = await loadCatalog(assets.catalogDir, { executableServices: Object.keys(executorModules), }); const runtimeDatabase = databaseUrl @@ -78,6 +95,7 @@ async function main(): Promise { runLimit, poolMax: databasePoolMax, connectionTimeoutMs: databaseConnectTimeoutMs, + migrations: assets.migrations, }) : await createNodeRuntimeDatabase({ backend: "sqlite", @@ -85,6 +103,7 @@ async function main(): Promise { logger, secretCodec, runLimit, + migrations: assets.migrations, }); try { @@ -106,7 +125,7 @@ async function main(): Promise { verifyRuntimeJwt, actionPolicy, allowedCustomOAuth, - registerStaticRoutes: (app) => registerStaticRoutes(app, staticRoot), + registerStaticRoutes: (app) => registerStaticRoutes(app, { root: assets.staticRoot, embedded: assets.embedded }), logger, }); @@ -133,7 +152,7 @@ async function main(): Promise { "runtime data encryption is disabled; set OOMOL_CONNECT_ENCRYPTION_KEY to encrypt stored credentials, Marketplace API keys, OAuth client configuration, pending OAuth state, and completed idempotent action responses", ); } - if (!staticRoot) { + if (!assets.staticRoot) { logger.warn("web console assets are not built; use http://localhost:5173 for local console development"); } }, @@ -145,6 +164,21 @@ async function main(): Promise { } } +async function runMigrateCommand(): Promise { + if (!databaseUrl) { + logger.info(sqliteMigrationsNotice); + return; + } + + const assets = await resolveServerAssets(); + await migratePostgresRuntimeDatabase({ + connectionString: databaseUrl, + connectionTimeoutMs: databaseConnectTimeoutMs, + logger, + migrations: assets.migrations, + }); +} + function waitForShutdown(server: ServerType): Promise { return new Promise((resolve, reject) => { let closing = false; @@ -169,15 +203,6 @@ function waitForShutdown(server: ServerType): Promise { }); } -async function resolveStaticRoot(root: string): Promise { - try { - await access(join(root, "index.html")); - return root; - } catch { - return undefined; - } -} - function readPositiveIntegerEnv(name: string, fallback: number): number { const value = process.env[name]; if (value === undefined) { diff --git a/src/server/server-assets.test.ts b/src/server/server-assets.test.ts new file mode 100644 index 000000000..41f5608af --- /dev/null +++ b/src/server/server-assets.test.ts @@ -0,0 +1,63 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { resolveServerAssets } from "./server-assets.ts"; +import { defaultMigrationSource } from "./storage/migration-source.ts"; + +const tempDirs: string[] = []; + +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + await Promise.all(tempDirs.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +describe("resolveServerAssets", () => { + it("uses the working directory layout of npm start under Node", async () => { + const cwd = await createTempCwd(); + await mkdir(join(cwd, "dist", "web"), { recursive: true }); + await writeFile(join(cwd, "dist", "web", "index.html"), ""); + + const assets = await resolveServerAssets(); + + expect(assets.catalogDir).toBe(join(cwd, "catalog/apps")); + expect(assets.migrations).toBe(defaultMigrationSource); + expect(assets.staticRoot).toBe(join(cwd, "dist/web")); + expect(assets.embedded).toBe(false); + }); + + it("reports the console as not built when dist/web/index.html is missing", async () => { + const cwd = await createTempCwd(); + await mkdir(join(cwd, "dist", "web"), { recursive: true }); + + const assets = await resolveServerAssets(); + + expect(assets.catalogDir).toBe(join(cwd, "catalog/apps")); + expect(assets.migrations).toBe(defaultMigrationSource); + expect(assets.staticRoot).toBeUndefined(); + expect(assets.embedded).toBe(false); + }); + + it("reads the tree embedded next to the module inside a Bun standalone executable", async () => { + vi.stubGlobal("Bun", { isStandaloneExecutable: true }); + const root = import.meta.dirname; + + const assets = await resolveServerAssets(); + + expect(assets.embedded).toBe(true); + expect(assets.catalogDir).toBe(join(root, "apps")); + // The directory source touches the filesystem lazily; the ENOENT it raises names the embedded directory. + expect(assets.migrations).not.toBe(defaultMigrationSource); + expect(() => assets.migrations.readMigrations("sqlite")).toThrow(join(root, "migrations")); + // No web/index.html lives beside this module, so the console is reported as not built. + expect(assets.staticRoot).toBeUndefined(); + }); +}); + +async function createTempCwd(): Promise { + const cwd = await mkdtemp(join(tmpdir(), "oomol-connect-assets-")); + tempDirs.push(cwd); + vi.spyOn(process, "cwd").mockReturnValue(cwd); + return cwd; +} diff --git a/src/server/server-assets.ts b/src/server/server-assets.ts new file mode 100644 index 000000000..7c1e31eb0 --- /dev/null +++ b/src/server/server-assets.ts @@ -0,0 +1,66 @@ +import type { MigrationSource } from "./storage/migration-source.ts"; + +import { access } from "node:fs/promises"; +import { join } from "node:path"; +import { createDirectoryMigrationSource, defaultMigrationSource } from "./storage/migration-source.ts"; + +/** + * Locations of the assets that are generated or built outside `src` and read + * by the server at startup. + */ +export interface ServerAssets { + /** Directory of generated provider catalog JSON files. */ + catalogDir: string; + migrations: MigrationSource; + /** Built web console directory, or undefined when the console is not built (index.html missing). */ + staticRoot: string | undefined; + /** + * True when the directories above live in the read-only tree embedded in a + * Bun standalone executable. Embedded files cannot be streamed and report + * epoch mtimes. + */ + embedded: boolean; +} + +/** + * Resolve where catalog, migrations and web console live for this process: the + * embedded tree in a standalone executable, otherwise the working directory and + * repository layout used by `npm start`. + * + * The build script embeds `migrations/`, `catalog/apps/` and `dist/web/` next + * to the bundled entry point, so inside the executable they appear under + * `import.meta.dirname` as `migrations/`, `apps/` and `web/`. + */ +export async function resolveServerAssets(): Promise { + if (isStandaloneExecutable()) { + const root = import.meta.dirname; + return { + catalogDir: join(root, "apps"), + migrations: createDirectoryMigrationSource(join(root, "migrations")), + staticRoot: await resolveStaticRoot(join(root, "web")), + embedded: true, + }; + } + + const cwd = process.cwd(); + return { + catalogDir: join(cwd, "catalog/apps"), + migrations: defaultMigrationSource, + staticRoot: await resolveStaticRoot(join(cwd, "dist/web")), + embedded: false, + }; +} + +/** Only Bun defines the `Bun` global; Node and workerd never do, so the read is safe everywhere. */ +function isStandaloneExecutable(): boolean { + return (globalThis as { Bun?: { isStandaloneExecutable?: boolean } }).Bun?.isStandaloneExecutable === true; +} + +async function resolveStaticRoot(root: string): Promise { + try { + await access(join(root, "index.html")); + return root; + } catch { + return undefined; + } +} diff --git a/src/server/storage/d1-runtime-store.test.ts b/src/server/storage/d1-runtime-store.test.ts index 25a7cb6a4..38924e9af 100644 --- a/src/server/storage/d1-runtime-store.test.ts +++ b/src/server/storage/d1-runtime-store.test.ts @@ -1,15 +1,13 @@ import type { RuntimeActionHttpResult } from "../api/runtime-api.ts"; import type { D1DatabaseBinding, D1PreparedStatementBinding } from "../cloudflare/cloudflare-bindings.ts"; -import { readFileSync, readdirSync } from "node:fs"; import { DatabaseSync } from "node:sqlite"; import { describe, expect, it } from "vitest"; import { AesGcmSecretCodec } from "../secrets/secret-codec.ts"; import { D1RuntimeDatabase } from "./d1-runtime-store.ts"; +import { defaultMigrationSource } from "./migration-source.ts"; import { RuntimeTokenService } from "./runtime-token-service.ts"; -const migrationDirectory = new URL("../../../migrations/", import.meta.url); - const githubProfile = { accountId: "github:octocat", displayName: "octocat", @@ -607,14 +605,8 @@ class SqliteD1Database implements D1DatabaseBinding { private readonly database = new DatabaseSync(":memory:"); constructor() { - // Derive the migration list the same way runSqliteMigrations does, so a new migration reaches the - // D1 harness on its own. The three lines stay duplicated here because sharing an owner would mean - // exporting from production code, which this test-only fix deliberately leaves alone. - const migrationFiles = readdirSync(migrationDirectory) - .filter((name) => /^\d+_.*\.sql$/.test(name)) - .sort(); - for (const file of migrationFiles) { - this.database.exec(readFileSync(new URL(file, migrationDirectory), "utf8")); + for (const migration of defaultMigrationSource.readMigrations("sqlite")) { + this.database.exec(migration.sql); } } diff --git a/src/server/storage/migration-source.test.ts b/src/server/storage/migration-source.test.ts new file mode 100644 index 000000000..e44ddbc70 --- /dev/null +++ b/src/server/storage/migration-source.test.ts @@ -0,0 +1,98 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { createDirectoryMigrationSource, defaultMigrationSource } from "./migration-source.ts"; + +const tempDirs: string[] = []; + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +describe("createDirectoryMigrationSource", () => { + it("reads sqlite migrations from the directory and postgresql migrations from its subdirectory", async () => { + const directory = await createMigrationDirectory(); + await writeFile(join(directory, "0001_sqlite.sql"), "create table sqlite_only (id integer);"); + await writeFile(join(directory, "postgresql", "0001_postgres.sql"), "create table postgres_only (id integer);"); + + const source = createDirectoryMigrationSource(directory); + expect(source.readMigrations("sqlite")).toEqual([ + { name: "0001_sqlite.sql", sql: "create table sqlite_only (id integer);" }, + ]); + expect(source.readMigrations("postgresql")).toEqual([ + { name: "0001_postgres.sql", sql: "create table postgres_only (id integer);" }, + ]); + }); + + it("keeps only numbered .sql files and ignores other entries and directories", async () => { + const directory = await createMigrationDirectory(); + await writeFile(join(directory, "0001_runtime.sql"), "select 1;"); + await writeFile(join(directory, "README.md"), "# migrations"); + await writeFile(join(directory, "notes.sql"), "select 2;"); + await writeFile(join(directory, "0002_backup.sql.bak"), "select 3;"); + await mkdir(join(directory, "archive")); + await mkdir(join(directory, "0002_archive.sql")); + + expect(createDirectoryMigrationSource(directory).readMigrations("sqlite")).toEqual([ + { name: "0001_runtime.sql", sql: "select 1;" }, + ]); + }); + + it("sorts migrations by file name using string order", async () => { + const directory = await createMigrationDirectory(); + await writeFile(join(directory, "0010_second.sql"), "select 10;"); + await writeFile(join(directory, "0002_first.sql"), "select 2;"); + await writeFile(join(directory, "10_last.sql"), "select 3;"); + + expect( + createDirectoryMigrationSource(directory) + .readMigrations("sqlite") + .map((migration) => migration.name), + ).toEqual(["0002_first.sql", "0010_second.sql", "10_last.sql"]); + }); + + it("reads migration bodies as utf8", async () => { + const directory = await createMigrationDirectory(); + const sql = "-- 运行时表\ncreate table runtime (name text);\n"; + await writeFile(join(directory, "0001_runtime.sql"), sql, "utf8"); + + expect(createDirectoryMigrationSource(directory).readMigrations("sqlite")).toEqual([ + { name: "0001_runtime.sql", sql }, + ]); + }); + + it("defers filesystem errors to readMigrations", async () => { + const directory = await createMigrationDirectory(); + await writeFile(join(directory, "0001_runtime.sql"), "select 1;"); + await rm(join(directory, "postgresql"), { recursive: true }); + + const missing = createDirectoryMigrationSource(join(directory, "missing")); + expect(() => missing.readMigrations("sqlite")).toThrow(/ENOENT/); + expect(() => missing.readMigrations("postgresql")).toThrow(/ENOENT/); + + const withoutPostgres = createDirectoryMigrationSource(directory); + expect(withoutPostgres.readMigrations("sqlite")).toEqual([{ name: "0001_runtime.sql", sql: "select 1;" }]); + expect(() => withoutPostgres.readMigrations("postgresql")).toThrow(/ENOENT/); + }); +}); + +describe("defaultMigrationSource", () => { + it("resolves the repository migrations directory", () => { + const sqlite = defaultMigrationSource.readMigrations("sqlite"); + expect(sqlite[0]).toMatchObject({ name: "0001_runtime.sql" }); + expect(sqlite[0]?.sql).toContain("create table"); + expect(sqlite.every((migration) => migration.name.endsWith(".sql"))).toBe(true); + + const postgresql = defaultMigrationSource.readMigrations("postgresql"); + expect(postgresql[0]).toMatchObject({ name: "0010_runtime.sql" }); + expect(postgresql.map((migration) => migration.name)).not.toContain("0001_runtime.sql"); + }); +}); + +async function createMigrationDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), "oomol-connect-migrations-")); + tempDirs.push(directory); + await mkdir(join(directory, "postgresql")); + return directory; +} diff --git a/src/server/storage/migration-source.ts b/src/server/storage/migration-source.ts new file mode 100644 index 000000000..a80e65f57 --- /dev/null +++ b/src/server/storage/migration-source.ts @@ -0,0 +1,36 @@ +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** + * Where runtime migrations come from. Validation and execution must read the same source so that + * a migration cannot be required by one and invisible to the other. + */ +export interface MigrationSource { + /** Migrations for one dialect, sorted by file name. Synchronous because SqliteRuntimeDatabase migrates inside its constructor. */ + readMigrations(dialect: "sqlite" | "postgresql"): { name: string; sql: string }[]; +} + +/** + * Directory layout: /*.sql = sqlite, /postgresql/*.sql = postgresql. + * Construction performs no filesystem access; errors (ENOENT etc.) surface from readMigrations(). + * Accepts a directory path string only (no URL): a file: URL base without a trailing slash silently resolves against the + * parent directory; not accepting URLs removes the trap by construction. + */ +export function createDirectoryMigrationSource(directory: string): MigrationSource { + return { + readMigrations(dialect) { + const dir = dialect === "sqlite" ? directory : join(directory, "postgresql"); + return readdirSync(dir, { withFileTypes: true }) + .filter((entry) => entry.isFile() && /^\d+_.*\.sql$/.test(entry.name)) + .map((entry) => entry.name) + .sort() + .map((name) => ({ name, sql: readFileSync(join(dir, name), "utf8") })); + }, + }; +} + +/** The repository's migrations/ directory resolved relative to this module. Used by every Node (non-binary) entry point. */ +export const defaultMigrationSource: MigrationSource = createDirectoryMigrationSource( + fileURLToPath(new URL("../../../migrations/", import.meta.url)), +); diff --git a/src/server/storage/node-runtime-database.ts b/src/server/storage/node-runtime-database.ts index 10a6dc58a..647441863 100644 --- a/src/server/storage/node-runtime-database.ts +++ b/src/server/storage/node-runtime-database.ts @@ -1,7 +1,10 @@ import type { RuntimeLogger } from "../../core/types.ts"; import type { ISecretCodec } from "../secrets/secret-codec-core.ts"; +import type { MigrationSource } from "./migration-source.ts"; import type { RuntimeDatabase } from "./runtime-database.ts"; +import { Pool } from "pg"; +import { migratePostgresDatabase } from "./postgres-migrations.ts"; import { PostgresRuntimeDatabase } from "./postgres-runtime-store.ts"; import { SqliteRuntimeDatabase } from "./sqlite-runtime-store.ts"; @@ -15,6 +18,7 @@ interface CommonOptions { logger?: RuntimeLogger; runLimit?: number; secretCodec?: ISecretCodec; + migrations?: MigrationSource; } interface SqliteOptions extends CommonOptions { @@ -31,6 +35,13 @@ interface PostgresOptions extends CommonOptions { export type NodeRuntimeDatabaseOptions = SqliteOptions | PostgresOptions; +export interface MigratePostgresRuntimeDatabaseOptions { + connectionString: string; + connectionTimeoutMs: number; + logger?: RuntimeLogger; + migrations?: MigrationSource; +} + export async function createNodeRuntimeDatabase(options: NodeRuntimeDatabaseOptions): Promise { if (options.backend === "sqlite") { return new SqliteRuntimeDatabase(options.path, options); @@ -41,7 +52,27 @@ export async function createNodeRuntimeDatabase(options: NodeRuntimeDatabaseOpti return await PostgresRuntimeDatabase.open(connectionString, options); } -export function assertPostgresDatabaseUrl(value: string): void { +/** What `migrate` entry points print without OOMOL_CONNECT_DATABASE_URL: SQLite has no explicit migrate step. */ +export const sqliteMigrationsNotice = + "SQLite migrations are applied automatically when the local runtime database opens."; + +/** Validate the URL, open a single-connection pool named open-connector-migrate, apply pending migrations, and close the pool. */ +export async function migratePostgresRuntimeDatabase(options: MigratePostgresRuntimeDatabaseOptions): Promise { + assertPostgresDatabaseUrl(options.connectionString); + const pool = new Pool({ + application_name: "open-connector-migrate", + connectionString: options.connectionString, + connectionTimeoutMillis: options.connectionTimeoutMs, + max: 1, + }); + try { + await migratePostgresDatabase({ pool, logger: options.logger, migrations: options.migrations }); + } finally { + await pool.end(); + } +} + +function assertPostgresDatabaseUrl(value: string): void { let protocol: string; try { protocol = new URL(value).protocol; diff --git a/src/server/storage/postgres-migrations.ts b/src/server/storage/postgres-migrations.ts index 2e08d0fea..6c2f33ca2 100644 --- a/src/server/storage/postgres-migrations.ts +++ b/src/server/storage/postgres-migrations.ts @@ -1,20 +1,16 @@ import type { RuntimeLogger } from "../../core/types.ts"; +import type { MigrationSource } from "./migration-source.ts"; import type { Pool, PoolClient } from "pg"; -import { readFileSync, readdirSync } from "node:fs"; +import { defaultMigrationSource } from "./migration-source.ts"; -const migrationDirectory = new URL("../../../migrations/postgresql/", import.meta.url); const migrationLockNamespace = 1_326_382_671; const migrationLockId = 1; -interface PostgresMigration { - name: string; - sql: string; -} - export interface PostgresMigrationOptions { pool: Pool; logger?: RuntimeLogger; + migrations?: MigrationSource; } export async function migratePostgresDatabase(options: PostgresMigrationOptions): Promise { @@ -31,7 +27,7 @@ export async function migratePostgresDatabase(options: PostgresMigrationOptions) `); const startedAt = Date.now(); - const migrations = readPostgresMigrations(); + const migrations = (options.migrations ?? defaultMigrationSource).readMigrations("postgresql"); const applied = await readAppliedMigrations(client); let newlyAppliedCount = 0; @@ -86,8 +82,11 @@ export async function migratePostgresDatabase(options: PostgresMigrationOptions) } } -export async function assertPostgresSchemaReady(pool: Pool): Promise { - const migrations = readPostgresMigrations(); +export async function assertPostgresSchemaReady( + pool: Pool, + migrations: MigrationSource = defaultMigrationSource, +): Promise { + const required = migrations.readMigrations("postgresql"); const relation = await pool.query<{ name: string | null }>("select to_regclass($1) as name", ["runtime_migrations"]); if (!relation.rows[0]?.name) { throw new Error( @@ -96,7 +95,7 @@ export async function assertPostgresSchemaReady(pool: Pool): Promise { } const applied = await readAppliedMigrations(pool); - const missing = migrations.filter((migration) => !applied.has(migration.name)).map((migration) => migration.name); + const missing = required.filter((migration) => !applied.has(migration.name)).map((migration) => migration.name); if (missing.length > 0) { throw new Error( `PostgreSQL runtime schema is not ready. Missing migrations: ${missing.join(", ")}. Run \`npm run runtime:migrate\` before starting the server.`, @@ -104,16 +103,6 @@ export async function assertPostgresSchemaReady(pool: Pool): Promise { } } -function readPostgresMigrations(): PostgresMigration[] { - return readdirSync(migrationDirectory) - .filter((name) => /^\d+_.*\.sql$/.test(name)) - .sort() - .map((name) => ({ - name, - sql: readFileSync(new URL(name, migrationDirectory), "utf8"), - })); -} - async function readAppliedMigrations(queryable: Pool | PoolClient): Promise> { const result = await queryable.query<{ name: string }>("select name from runtime_migrations"); return new Set(result.rows.map((row) => row.name)); diff --git a/src/server/storage/postgres-runtime-store.test.ts b/src/server/storage/postgres-runtime-store.test.ts index 9755ac6a8..ab3878949 100644 --- a/src/server/storage/postgres-runtime-store.test.ts +++ b/src/server/storage/postgres-runtime-store.test.ts @@ -1,10 +1,12 @@ import type { RuntimeActionHttpResult } from "../api/runtime-api.ts"; +import type { MigrationSource } from "./migration-source.ts"; import { PGlite } from "@electric-sql/pglite"; import { PGLiteSocketServer } from "@electric-sql/pglite-socket"; import { Pool } from "pg"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { AesGcmSecretCodec } from "../secrets/secret-codec.ts"; +import { defaultMigrationSource } from "./migration-source.ts"; import { assertPostgresSchemaReady, migratePostgresDatabase } from "./postgres-migrations.ts"; import { PostgresRuntimeDatabase } from "./postgres-runtime-store.ts"; import { RuntimeTokenService } from "./runtime-token-service.ts"; @@ -68,6 +70,62 @@ describe("PostgreSQL migrations with PGlite", () => { }); }); +describe("PostgreSQL migrations with a custom migration source", () => { + let testServer: PGliteTestServer; + + beforeAll(async () => { + testServer = await startPGliteTestServer(); + }); + + afterAll(async () => { + await testServer.server.stop(); + await testServer.database.close(); + }); + + it("validates and executes migrations through the same source", async () => { + const migrations: MigrationSource = { + readMigrations(dialect) { + return [ + ...defaultMigrationSource.readMigrations(dialect), + { name: "9998_custom.sql", sql: "create table custom_records (id integer primary key)" }, + ]; + }, + }; + const unapplied: MigrationSource = { + readMigrations: () => [{ name: "9999_unapplied.sql", sql: "select 1" }], + }; + + const pool = new Pool({ connectionString: testServer.url, max: 1 }); + try { + await migratePostgresDatabase({ pool, migrations }); + await expect(assertPostgresSchemaReady(pool, migrations)).resolves.toBeUndefined(); + await expect(assertPostgresSchemaReady(pool)).resolves.toBeUndefined(); + await expect(assertPostgresSchemaReady(pool, unapplied)).rejects.toThrow( + "Missing migrations: 9999_unapplied.sql", + ); + await expect(pool.query("select name from runtime_migrations order by name")).resolves.toMatchObject({ + rows: [ + { name: "0010_runtime.sql" }, + { name: "0011_runtime_token_connection_scope.sql" }, + { name: "0012_marketplace.sql" }, + { name: "9998_custom.sql" }, + ], + }); + await expect(pool.query("select to_regclass($1) as name", ["custom_records"])).resolves.toMatchObject({ + rows: [{ name: "custom_records" }], + }); + } finally { + await pool.end(); + } + + await expect(PostgresRuntimeDatabase.open(testServer.url, { migrations: unapplied })).rejects.toThrow( + "Missing migrations: 9999_unapplied.sql", + ); + const database = await PostgresRuntimeDatabase.open(testServer.url, { migrations }); + await database.close(); + }); +}); + describe("PostgresRuntimeDatabase with PGlite", () => { let testServer: PGliteTestServer; let database: PostgresRuntimeDatabase; diff --git a/src/server/storage/postgres-runtime-store.ts b/src/server/storage/postgres-runtime-store.ts index 172b55d02..b61b109ca 100644 --- a/src/server/storage/postgres-runtime-store.ts +++ b/src/server/storage/postgres-runtime-store.ts @@ -15,6 +15,7 @@ import type { IdempotencyClaimResult, IIdempotencyStore, } from "./idempotency-store.ts"; +import type { MigrationSource } from "./migration-source.ts"; import type { RuntimeDatabase } from "./runtime-database.ts"; import type { IRuntimePolicyStore, RuntimePolicyRecord } from "./runtime-policy-store.ts"; import type { RuntimeRow } from "./runtime-sql.ts"; @@ -43,6 +44,7 @@ export interface PostgresRuntimeDatabaseOptions { secretCodec?: ISecretCodec; poolMax?: number; connectionTimeoutMs?: number; + migrations?: MigrationSource; } export class PostgresRuntimeDatabase implements RuntimeDatabase { @@ -86,7 +88,7 @@ export class PostgresRuntimeDatabase implements RuntimeDatabase { }); try { - await assertPostgresSchemaReady(pool); + await assertPostgresSchemaReady(pool, options.migrations); return new PostgresRuntimeDatabase(pool, options); } catch (error) { await pool.end(); diff --git a/src/server/storage/sqlite-runtime-store.test.ts b/src/server/storage/sqlite-runtime-store.test.ts index e573566df..ccf5a699d 100644 --- a/src/server/storage/sqlite-runtime-store.test.ts +++ b/src/server/storage/sqlite-runtime-store.test.ts @@ -1,12 +1,13 @@ import type { RuntimeActionHttpResult } from "../api/runtime-api.ts"; import { readFileSync } from "node:fs"; -import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { afterEach, describe, expect, it } from "vitest"; import { AesGcmSecretCodec } from "../secrets/secret-codec.ts"; +import { createDirectoryMigrationSource } from "./migration-source.ts"; import { RuntimeTokenService } from "./runtime-token-service.ts"; import { SqliteRunLogStore, SqliteRuntimeDatabase } from "./sqlite-runtime-store.ts"; @@ -459,6 +460,31 @@ describe("SqliteRuntimeDatabase", () => { database.close(); }); + it("applies migrations from a custom migration source", async () => { + const databasePath = await createDatabasePath(); + const migrationDirectory = join(dirname(databasePath), "migrations"); + await mkdir(migrationDirectory); + await writeFile( + join(migrationDirectory, "0001_custom.sql"), + "create table custom_records (id integer primary key);", + ); + + const database = new SqliteRuntimeDatabase(databasePath, { + migrations: createDirectoryMigrationSource(migrationDirectory), + }); + database.close(); + + const inspected = new DatabaseSync(databasePath); + expect(inspected.prepare("select name from runtime_migrations order by name").all()).toEqual([ + { name: "0001_custom.sql" }, + ]); + expect( + inspected.prepare("select name from sqlite_master where type = 'table' and name = 'custom_records'").get(), + ).toEqual({ name: "custom_records" }); + expect(inspected.prepare("select name from sqlite_master where name = 'connections'").get()).toBeUndefined(); + inspected.close(); + }); + it("keeps an inserted run when retention cleanup fails", async () => { const raw = new DatabaseSync(":memory:"); for (const migration of [ diff --git a/src/server/storage/sqlite-runtime-store.ts b/src/server/storage/sqlite-runtime-store.ts index 24298ea1f..2c23aaaca 100644 --- a/src/server/storage/sqlite-runtime-store.ts +++ b/src/server/storage/sqlite-runtime-store.ts @@ -15,15 +15,16 @@ import type { IdempotencyClaimResult, IIdempotencyStore, } from "./idempotency-store.ts"; +import type { MigrationSource } from "./migration-source.ts"; import type { RuntimeDatabase } from "./runtime-database.ts"; import type { IRuntimePolicyStore, RuntimePolicyRecord } from "./runtime-policy-store.ts"; import type { IRunLogStore, RunLog, RunLogListInput, RunLogPage, RunLogWriteResult } from "./runtime-store.ts"; import type { IRuntimeTokenStore, RuntimeTokenRecord } from "./runtime-token-service.ts"; -import { readFileSync, readdirSync } from "node:fs"; import { DatabaseSync } from "node:sqlite"; import { parseRuntimeActionHttpResult } from "../api/runtime-api.ts"; import { PlainTextSecretCodec } from "../secrets/secret-codec-core.ts"; +import { defaultMigrationSource } from "./migration-source.ts"; import { listRunLogs, parseJson, @@ -36,12 +37,12 @@ import { import { DEFAULT_RUN_LIMIT } from "./runtime-store.ts"; type SecretJsonTable = "oauth_client_configs"; -const migrationDirectory = new URL("../../../migrations/", import.meta.url); export interface SqliteRuntimeDatabaseOptions { logger?: RuntimeLogger; runLimit?: number; secretCodec?: ISecretCodec; + migrations?: MigrationSource; } interface SecretJsonInput { @@ -95,7 +96,7 @@ export class SqliteRuntimeDatabase implements RuntimeDatabase { constructor(filename: string, options: SqliteRuntimeDatabaseOptions = {}) { this.database = new DatabaseSync(filename); this.secretCodec = options.secretCodec ?? new PlainTextSecretCodec(); - this.initialize(options.logger); + this.initialize(options.migrations ?? defaultMigrationSource, options.logger); this.connectionStore = new SqliteConnectionStore(this.database, this.secretCodec); this.oauthClientConfigStore = new SqliteOAuthClientConfigStore(this.database, this.secretCodec); this.oauthStateStore = new SqliteOAuthStateStore(this.database, this.secretCodec); @@ -156,9 +157,9 @@ export class SqliteRuntimeDatabase implements RuntimeDatabase { `); } - private initialize(logger?: RuntimeLogger): void { + private initialize(migrations: MigrationSource, logger?: RuntimeLogger): void { this.database.exec("pragma journal_mode = wal;"); - runSqliteMigrations(this.database, logger); + runSqliteMigrations(this.database, migrations, logger); } } @@ -648,7 +649,7 @@ function insertRun(database: DatabaseSync, run: RunLog): void { ); } -function runSqliteMigrations(database: DatabaseSync, logger?: RuntimeLogger): void { +function runSqliteMigrations(database: DatabaseSync, migrations: MigrationSource, logger?: RuntimeLogger): void { const startedAt = Date.now(); database.exec(` create table if not exists runtime_migrations ( @@ -662,42 +663,42 @@ function runSqliteMigrations(database: DatabaseSync, logger?: RuntimeLogger): vo .all() .map((row) => readString(row, "name")), ); - const migrationFiles = readdirSync(migrationDirectory) - .filter((name) => /^\d+_.*\.sql$/.test(name)) - .sort(); + const migrationFiles = migrations.readMigrations("sqlite"); let newlyAppliedCount = 0; - for (const file of migrationFiles) { - if (applied.has(file)) { + for (const migration of migrationFiles) { + if (applied.has(migration.name)) { continue; } const migrationStartedAt = Date.now(); - logger?.info({ migration: file }, "sqlite migration started"); + logger?.info({ migration: migration.name }, "sqlite migration started"); try { - const sql = readFileSync(new URL(file, migrationDirectory), "utf8"); runInTransaction(database, () => { - database.exec(sql); + database.exec(migration.sql); database .prepare("insert into runtime_migrations (name, applied_at) values (?, ?)") - .run(file, new Date().toISOString()); + .run(migration.name, new Date().toISOString()); }); } catch (error) { logger?.error( - { migration: file, durationMs: Date.now() - migrationStartedAt, err: error }, + { migration: migration.name, durationMs: Date.now() - migrationStartedAt, err: error }, "sqlite migration failed", ); throw error; } - applied.add(file); + applied.add(migration.name); newlyAppliedCount += 1; - logger?.info({ migration: file, durationMs: Date.now() - migrationStartedAt }, "sqlite migration completed"); + logger?.info( + { migration: migration.name, durationMs: Date.now() - migrationStartedAt }, + "sqlite migration completed", + ); } logger?.info( { migrationCount: migrationFiles.length, - appliedCount: migrationFiles.filter((file) => applied.has(file)).length, + appliedCount: migrationFiles.filter((migration) => applied.has(migration.name)).length, newlyAppliedCount, durationMs: Date.now() - startedAt, },