diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b106448b..34688684 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,24 @@ jobs: - name: Test create-bench run: pnpm --filter create-bench run test + playwright-native: + name: Test Playwright-native benchmarks + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'pnpm' + - run: pnpm install --frozen-lockfile --ignore-scripts + - run: pnpm -r --filter './packages/**' run build + - name: Install fixture browser + run: node node_modules/playwright-native-core/cli.js install --with-deps chromium + - name: Exercise native suites against local fixtures + run: pnpm test:playwright-native + actionlint: name: Lint workflows runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index f5d264c7..3f6e6e45 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ .env node_modules/ +.pnpm-store/ .DS_Store dist/ scale-run.log scale.mjs -fetch-raw-json.ts \ No newline at end of file +fetch-raw-json.ts +results/playwright-native/ +playwright-native-urls.json +playwright-native-environment.json diff --git a/PLAYWRIGHT_NATIVE_BENCHMARKS.md b/PLAYWRIGHT_NATIVE_BENCHMARKS.md new file mode 100644 index 00000000..ec06bbd6 --- /dev/null +++ b/PLAYWRIGHT_NATIVE_BENCHMARKS.md @@ -0,0 +1,365 @@ +# Proposal: Playwright-native browser benchmarks + +This proposal requests ComputeSDK maintainer feedback, following Noah's request +for a proposal PR. It includes runnable implementations of two separate +benchmarks over `chromium.connect()`: **browser readiness** and **action +throughput**. It contains no provider benchmark results or performance claims. + +## Motivation and scope + +Providers exposing Playwright-native endpoints should be measurable alongside +the providers served by the existing CDP benchmarks. Keep the CDP suites, +identities, methodology, and historical results intact; give the new suites +separate identities and result series. This is a provider comparison under a +specified workload, not evidence that either protocol is inherently faster. + +[CONTRIBUTING.md](./CONTRIBUTING.md) asks contributors to discuss methodology +and wait for maintainer feedback before implementing changes, because historical +comparability matters. This PR supplies that discussion for approval or revision, +alongside isolated native suites, local fixture validation, and run instructions. +The existing CDP suites and their dependency version remain unchanged. Approval +of the methodology and environment prerequisites is still required before a +published provider comparison. + +## Initial participants and disclosure + +Propose **Momentic** and **Azure Playwright Workspaces** initially. Other providers +may participate if they expose a Playwright-native endpoint compatible with +`chromium.connect()` and can meet the same workload, settings, and lifecycle +requirements. A CDP endpoint alone does not qualify for these suites. + +The proposer is affiliated with Momentic. Momentic offers to fund Azure usage for +the comparison; the billing arrangement remains to be agreed before execution. +Funding does not determine inclusion, methodology, or interpretation of results. +Provider participation and sponsorship remain separate, as required by the +contribution guidance. + +## Execution and environment + +- Run **100 fresh sessions per provider per suite**, with `iterations: 100`, + `concurrency: 1`, and `groupBy: 'round'`. One runner executes one session at a + time: Momentic round 1, Azure round 1, Momentic round 2, Azure round 2, and so on. + Record provider order. Do not substitute parallel provider matrix jobs for + this initial interleaved comparison. +- Use separate benchmark slugs, run IDs/keys, and result series for readiness + and throughput. Do not reuse sessions between suites or rounds. Fresh means a + new logical provider session, context, and page, not a guarantee of a cold + physical host or empty provider cache. +- Require both providers' browsers to execute in the **same Azure region** and + keep the runner location fixed for the entire initial comparison. Agree the + exact region and runner location before a provider comparison. Record the actual + browser regions, runner region/location, routing configuration, and evidence + of effective placement; a workspace or requested region alone is insufficient + if routing can select another region. +- Use Linux Chromium in headless mode and an explicitly configured + **1920 × 1080 viewport** in every fresh context. Fix other context settings + equally across participants and record them. Disable optional stealth, proxy, + video, trace, HAR, and provider recording features. Screenshots remain part of + the measured workload. +- Pin a common compatible Playwright client version and compatible provider + server versions. [Playwright's connection contract](https://playwright.dev/docs/api/class-browsertype#browser-type-connect) + requires matching client/server major and minor versions. Record the exact + client, server Playwright, and Chromium versions, plus runner OS and Node.js + version. Confirm provider support and version reporting before a provider comparison; + do not silently change versions during a run. + +## Suite 1: browser readiness + +Prepare reusable credentials and provider clients before measurement. Workspace +creation, account setup, reusable token acquisition, and other one-time setup +are prerequisites, not readiness samples. Any authentication performed for each +individual session belongs inside that session's timed boundary. + +Start a monotonic timer **immediately before the first per-session provisioning +or connection request**, whichever comes first for that provider. Include all +per-session authentication, session provisioning and polling where required, +endpoint acquisition, `chromium.connect()`, `browser.newContext()` with the fixed +viewport, and `context.newPage()`. Stop immediately after +`page.goto('https://www.example.com', { waitUntil: 'load' })` completes +successfully. For a provider that provisions on connection, start immediately +before that connection request; do not omit provisioning hidden in the handshake. + +Report this elapsed interval as `readinessMs`, with component timings where +available. The end condition is the completed page load, not merely an open +socket. Each attempt uses a fresh context/page rather than a provider's default +page. Measure cleanup separately as `cleanupMs`, including context/browser close +and any provider release or termination; exclude it from `readinessMs`. + +## Suite 2: action throughput + +Provision and connect a fresh session, then create a fresh context/page using the +same settings. Record setup and cleanup separately; exclude both from the action +throughput denominator. Reuse the ten-action Wikipedia workload from +[browser-throughput.bench.ts](./benchmarks/browser/browser-throughput.bench.ts), +with **five repetitions per session: 50 sequential actions**. + +Before measured sessions begin, resolve and validate one concrete article URL +per round, using the same heading and article-body link requirements as the +workload. Freeze and publish the ordered URL list; both providers use the same +round's URL for all five repetitions. Require 100 validated inputs rather than +silently wrapping a short list or falling back to `Special:Random` during +measurement. URL preparation is outside session timing. Shared inputs reduce +page-selection variance; live Wikipedia content and network conditions can still +vary and should not be treated as isolated protocol overhead. + +| Action in each loop | Operation | +| --- | --- | +| 1 | `page.goto(sharedArticleUrl, { waitUntil: 'load' })` | +| 2 | `page.waitForSelector('#firstHeading')` | +| 3 | `page.screenshot()` with the fixed viewport | +| 4 | `page.textContent('#firstHeading')` | +| 5 | Wait for `#mw-content-text a[href*="/wiki/"]`, inspect links in document order, and click the first article link whose path after `/wiki/` has no namespace colon. Accept relative, protocol-relative, and absolute URL forms. | +| 6 | `page.waitForSelector('#firstHeading')` on the linked article | +| 7 | `page.screenshot()` | +| 8 | `page.textContent('#firstHeading')` | +| 9 | `page.goBack({ waitUntil: 'commit' })` | +| 10 | `page.waitForSelector('#firstHeading')` on the original article | + +The click's link discovery and selection remain inside that action's timing. +Keep the existing 30-second bound per action and the click's 10-second internal +selector wait. Preserve dependency skipping: if action 5 fails, record actions +6–10 as unsuccessful, skipped actions with zero duration, then continue with +the next repetition. Other action failures are recorded without aborting the +loop. Back navigation resolves at `commit`; the following selector wait confirms +the heading is present. + +Use the existing metric definitions: `taskMs` is the **sum of the individual +action durations**, and `actionsPerSecond = actionsCompleted / (taskMs / 1000)`. +A complete successful session has `actionsCompleted: 50`. Record the enclosing +`actions` step's elapsed time too, but do not substitute it for `taskMs` without +maintainer agreement. Each complete session contains ten screenshots; report +their latency separately from actions per second. + +## Integration and failure handling + +Follow the existing declarative +[config + task conventions](./WRITING_BENCHMARKS.md): each suite exports `config` +and `task`; the `bench run` runner owns scheduling and reporting. Participants +supply endpoint/authentication information, per-session provisioning where +required, and lifecycle cleanup. Provider-specific setup surrounds a shared +measured workload; it must not change the action sequence or timing definitions. +Do not assume the current CDP session interface already satisfies these needs. + +Use `step()` for setup/provisioning, connection and fresh context/page creation, +navigation or actions, and cleanup. Use `measure()` for readiness elapsed time, +action counts, rates, and screenshot metrics, and retain ordered action records +in task data. Capture readiness's start and stop explicitly across steps; the +runner's overall task duration includes cleanup and is not `readinessMs`. +Use the existing task error/data conventions to retain diagnostics and partial +records when a session-level operation fails. + +Bound provisioning/polling, per-session authentication, connection, context/page +creation, navigation, and every cleanup operation. Agree and record limits +before a provider comparison. `chromium.connect()` needs an explicit timeout because +its default is unbounded. Attempt cleanup in `finally` after success or failure, +including partial setup. Closing a connection must not be assumed to release +provider resources: verify each provider's lifecycle semantics, retain session +identifiers as soon as available, and define how timed-out provisioning or +late-created sessions are reconciled. Record cleanup failures without masking +the original failure; prevent uncertain leaked resources from accumulating. + +Every started attempt remains in the reliability denominator, including +provisioning, authentication, connection, navigation, action, and cleanup +failures. Do not replace failed samples with successful retries. Report workload +success and cleanup success separately, as well as overall session success +(both succeeded). Readiness workload success requires the completed target load; +throughput workload success requires all 50 actions and no setup/session error. +Incomplete sessions never enter successful-session throughput summaries. +Raw failed/partial action records, elapsed failure times, timeout reasons, and +skipped dependencies remain available. A participant skipped for missing +credentials is unavailable, not a measured success; the planned comparison +requires both initial participants to be ready before launch. + +Separate SDK integration and package publication are follow-up work. The +contribution guidance describes provider integration in +[`computesdk/computesdk`](https://github.com/computesdk/computesdk), package +publication, and credentials for ongoing tests. Maintainers should confirm how +Playwright-native endpoint/authentication and cleanup requirements fit that +process before a provider comparison; this PR proposes no new SDK API or package. + +## Results and methodology review + +Publish raw session and action records for all attempts, the shared URL list, +settings/version/region metadata, sample counts, and success rates. Remove +credentials and credential-bearing endpoints from public artifacts. Publish +**untrimmed median/P95/P99** timings for successful workload sessions, with +failures and cleanup outcomes visible alongside them. Do not turn a timeout into +a successful latency observation or report missing samples as zero latency. + +Report `readinessMs`, `taskMs`, per-action-type latency, and `cleanupMs` separately. +For throughput, summarize per-session actions per second and publish a separate +screenshot latency distribution over successful screenshots in complete +sessions. Retain the existing `screenshotMs` metric as each session's median +successful screenshot latency, clearly distinguishing its across-session +summary from the pooled screenshot distribution. Define median as the middle +value (mean of the middle two for an even count); use nearest-rank P95/P99 on +sorted samples and disclose sample sizes. With 100 attempted sessions, tail +estimates may rest on very few observations, especially after failures. + +Defer composite scores, ceilings, and weights to maintainer review. Initially +report distributions without a composite ranking. The runner supports omitting +`scoring`; use metric-based display and retain explicit workload-success flags. +Do not copy the existing 5% tail trimming or legacy result summaries into these +suites. If scoring is approved later, use `trim: 0` for the agreed untrimmed +metrics and preserve the explicit complete-session success criterion. + +## Correspondence with current executable behavior + +The proposal uses current code and workflows to resolve documentation drift: + +- [THROUGHPUT.md](./THROUGHPUT.md) describes five repetitions, but + [throughput-types.ts](./benchmarks/browser/throughput-types.ts) currently sets + `LOOPS_PER_SESSION = 1`: the executable workload is ten actions. This proposal + adopts the documented 50-action design explicitly; it does not claim the + current executable suite already runs 50 or change the existing CDP constant. +- The executable action loop uses shared URLs when supplied, supports three + article-link URL forms, and skips dependent actions after click failure. + Those behaviors take precedence over the older random-page selector and + failure descriptions in `THROUGHPUT.md`. +- [browser.bench.ts](./benchmarks/browser/browser.bench.ts) measures a CDP + lifecycle whose `totalMs` includes browser close and session release. The new + readiness boundary excludes cleanup and includes fresh context/page creation. +- Both browser configs default to two iterations; the + [throughput workflow](./.github/workflows/browser-throughput-benchmarks.yml) + supplies 100 for scheduled/manual runs and three for browser-code pushes. It + currently runs weekly, with providers in separate matrix jobs. Those settings + do not establish the proposed single-runner interleaving or runner location. +- [throughput-benchmark.ts](./benchmarks/browser/throughput-benchmark.ts) trims + tails and its legacy summaries can include incomplete sessions. The declarative + throughput scoring config requires complete action counts. The new suites + must explicitly filter workload success and publish untrimmed distributions, + rather than assuming both reporting paths have the same semantics. + +## Prerequisites and follow-up work + +Before launching a provider comparison, obtain maintainer agreement on these timing +boundaries, the 50-action workload, reliability and summary definitions, and +provider integration requirements. Select and verify the shared Azure region +and fixed runner location, compatible pinned versions, credential setup, Azure +funding arrangement, lifecycle timeouts, and cleanup behavior for both providers. +These are outstanding prerequisites, not provider capabilities validated by the +local fixture tests. + +Validate the eventual implementation with repeatable checks of timing boundaries, +identical round inputs, exact action counts, timeout/dependency behavior, partial +result retention, actual placement/settings, and resource cleanup after failures. +The included test exercises the real native protocol and existing declarative +runner against local HTTP/TLS fixtures. It validates the 50-action workload, +round interleaving, load boundary, authentication headers, partial results, +lost-response reconciliation, cleanup failures, credential redaction, and +untrimmed summaries. This does not validate the deployed providers, their +placement, or their resource-release semantics. + +Concurrency testing, multiple regions, composite scoring, provider/workspace +provisioning, SDK/package publication, and benchmark publication are follow-up +work after methodology review. + +## Running the implementations + +The runnable entrypoints are +[playwright-readiness.bench.ts](./benchmarks/browser/playwright-readiness.bench.ts) +and [playwright-throughput.bench.ts](./benchmarks/browser/playwright-throughput.bench.ts). +They use a separate, exact `playwright-core@1.60.0` dependency alias; the existing +CDP dependency remains unchanged. Compatibility with deployed providers must be +verified before running a comparison. + +```bash +pnpm install --frozen-lockfile --ignore-scripts +pnpm -r --filter './packages/**' run build + +# Install the local browser used only to prepare inputs and run fixture tests. +node node_modules/playwright-native-core/cli.js install chromium +pnpm test:playwright-native + +# Resolve and DOM-validate the shared inputs before provider measurement. +pnpm bench:playwright-prepare-urls 100 playwright-native-urls.json + +# Run separately; --no-ingest writes local records without platform publication. +pnpm bench:playwright-readiness --no-ingest +pnpm bench:playwright-throughput --no-ingest +``` + +The default commands require credentials for both providers and fail before any +provider request if either is unavailable. For an explicit single-provider +diagnostic, pass `--provider momentic` or `--provider azure`. Raw diagnostic +outputs are labeled `provider-diagnostic` and list unavailable participants +separately from attempted-session denominators. + +Store credentials in your shell or the ignored `benchmarks/.env`, never in Git: + +- `MOMENTIC_BROWSER_FLEET_URL`: the Browser Fleet API base URL. +- `MOMENTIC_BROWSER_FLEET_API_KEY`: a credential authorized to create, read, and + delete benchmark sessions. A normal Momentic app API key is not interchangeable. +- `AZURE_PLAYWRIGHT_SERVICE_URL`: the workspace browser endpoint ending in + `/playwrightworkspaces//browsers`. The adapter resolves the + [documented browser redirect](https://learn.microsoft.com/en-us/rest/api/playwright/dataplane/workspaces/get-browsers?view=rest-playwright-dataplane-2025-09-01) + inside measurement, then connects with bearer authentication and `os=Linux`. +- `AZURE_PLAYWRIGHT_ACCESS_TOKEN`: a reusable Entra or workspace access token + prepared before measurement; its authentication mode must already be enabled. +- `PLAYWRIGHT_NATIVE_URLS_FILE`: the prepared `playwright-native-urls.json` path. +- `PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE`: an operator-verified environment manifest + path. The manifest validates the shared region, compatible versions, and fixed + settings and records evidence; it does not discover actual placement itself. +- `PLAYWRIGHT_NATIVE_RESULTS_DIR`: optional local output directory (default + `results/playwright-native`). Each suite writes a fresh JSON file with all task + and action records, reliability denominators, and untrimmed summaries. + +Create an ignored `playwright-native-environment.json` only after verifying the +environment. Use this shape for **both** entries in `providers` (`momentic` and +`azure`), replacing the evidence placeholders with non-secret verification +references: + +```json +{ + "region": "", + "runnerLocation": "", + "providers": { + "momentic": { + "region": "", + "playwrightVersion": "1.60.0", + "os": "linux", + "headless": true, + "stealth": false, + "proxy": false, + "recording": false, + "placementEvidence": "", + "versionEvidence": "", + "cleanupEvidence": "" + }, + "azure": { + "region": "", + "playwrightVersion": "1.60.0", + "os": "linux", + "headless": true, + "stealth": false, + "proxy": false, + "recording": false, + "placementEvidence": "", + "versionEvidence": "", + "cleanupEvidence": "" + } + } +} +``` + +Momentic provisioning creates a session with an idempotency key and polls READY; +cleanup requests deletion and polls a terminal state. A lost creation response +is reconciled with the same key during cleanup, without replacing the failed +sample. Azure teardown closes the context and connection and relies on the +service's verified release semantics. +If Azure provisioning/connection fails without an owned connection to close, +cleanup is recorded as uncertain and further allocations stop for that +participant; the implementation does not claim it released an unknown resource. +Provisioning has a 120-second bound; +connection, context/page creation, and navigation have 30-second bounds; each +cleanup operation has a 15-second bound. Timed-out context creation is retained +and reconciled within that cleanup bound; late contexts are explicitly closed. +Unresolved context ownership makes cleanup uncertain. After uncertain cleanup, +the process refuses further allocations for that participant and records unstarted tasks +separately from attempted-session reliability denominators. + +No real credentials are used by fixture tests. Local fixture credentials and +certificates are disposable, and the CI fixture job runs without provider +credentials. Shared input/setup files and generated result files are ignored by +Git. Actual Chromium versions are collected from each connected browser; +server versions and placement evidence are labeled by their source in raw data. diff --git a/benchmarks/.env.example b/benchmarks/.env.example index 04e66ca1..a464f46f 100644 --- a/benchmarks/.env.example +++ b/benchmarks/.env.example @@ -185,3 +185,11 @@ NEON_AI_GATEWAY_TOKEN=your_neon_ai_gateway_token NGROK_AI_GATEWAY_API_KEY=your_ngrok_ai_gateway_api_key LLMAPI_API_KEY=your_llmapi_api_key BLAZERAIL_API_KEY=your_blazerail_api_key +# Playwright-native browser benchmarks (keep real values in the ignored .env). +# MOMENTIC_BROWSER_FLEET_URL= +# MOMENTIC_BROWSER_FLEET_API_KEY= +# AZURE_PLAYWRIGHT_SERVICE_URL= +# AZURE_PLAYWRIGHT_ACCESS_TOKEN= +# PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE=playwright-native-environment.json +# PLAYWRIGHT_NATIVE_URLS_FILE=playwright-native-urls.json +# PLAYWRIGHT_NATIVE_RESULTS_DIR=results/playwright-native diff --git a/benchmarks/browser/playwright-native.test.ts b/benchmarks/browser/playwright-native.test.ts new file mode 100644 index 00000000..2e9d6982 --- /dev/null +++ b/benchmarks/browser/playwright-native.test.ts @@ -0,0 +1,289 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; +import https from 'node:https'; +import net from 'node:net'; +import { execFileSync, execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { mkdtempSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { chromium, type BrowserContext, type BrowserContextOptions } from 'playwright-native-core'; +import { withTimeout } from '../src/util/timeout.js'; +import { runBenchmark, type BenchmarkRunOutcome } from '@benchsdk/runner'; +import { distribution } from './playwright-results.js'; +import { articleUrl } from './playwright-workload.js'; + +test('native suites use real Playwright connections and preserve reliability records', { timeout: 180_000 }, async t => { + const directory = mkdtempSync(path.join(tmpdir(), 'native-bench-')); + const envNames = ['MOMENTIC_BROWSER_FLEET_URL', 'MOMENTIC_BROWSER_FLEET_API_KEY', 'AZURE_PLAYWRIGHT_SERVICE_URL', 'AZURE_PLAYWRIGHT_ACCESS_TOKEN', 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE', 'PLAYWRIGHT_NATIVE_URLS_FILE', 'PLAYWRIGHT_NATIVE_RESULTS_DIR']; + const previous = new Map(envNames.map(key => [key, process.env[key]])); + execFileSync('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', path.join(directory, 'key.pem'), '-out', path.join(directory, 'cert.pem'), '-days', '1', '-subj', '/CN=localhost'], { stdio: 'ignore' }); + const listen = (server: http.Server | https.Server) => new Promise(resolve => { + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + resolve(address.port); + }); + }); + const fixtures = https.createServer({ key: readFileSync(path.join(directory, 'key.pem')), cert: readFileSync(path.join(directory, 'cert.pem')) }, (req, res) => { + if (req.url === '/slow.gif') { + setTimeout(() => { res.setHeader('Content-Type', 'image/gif'); res.end(Buffer.from('47494638396101000100800000000000ffffff21f90401000000002c00000000010001000002024401003b', 'hex')); }, 150); + return; + } + res.setHeader('Content-Type', 'text/html'); + if (req.headers.host?.includes('example.com')) { + res.end('Ready'); + return; + } + const link = req.url?.includes('Native_no_links') ? '' : 'Article'; + res.end(`

Fixture

Help${link}
`); + }); + const fixturePort = await listen(fixtures); + fixtures.on('connection', socket => socket.on('error', () => {})); + const sessions = new Map(); + const terminated = new Set(); + const order: string[] = []; + const websocketAuth: string[] = []; + const azureRunIds = new Set(); + let mode = ''; + let lost = false; + let endpoint = ''; + const api = http.createServer(async (req, res) => { + const url = new URL(req.url ?? '/', 'http://localhost'); + res.setHeader('Content-Type', 'application/json'); + if (url.pathname === '/browsers') { + assert.equal(req.headers.authorization, 'Bearer local-azure-credential'); + assert.equal(url.searchParams.get('os'), 'Linux'); + assert.equal(url.searchParams.get('api-version'), '2025-09-01'); + const runId = url.searchParams.get('runId'); + assert.ok(runId); + azureRunIds.add(runId); + order.push('azure'); + res.writeHead(302, { Location: mode === 'refused-connection' ? 'ws://127.0.0.1:1/refused?token=private-query' : `ws://127.0.0.1:${apiPort}/connect/azure/${url.searchParams.get('runId')}` }); + res.end(); + return; + } + // Chromium may make unrelated startup requests through the fixture proxy. + if (!url.pathname.startsWith('/v1/sessions')) { res.writeHead(404); res.end('{}'); return; } + assert.equal(req.headers.authorization, 'Bearer local-momentic-credential'); + const id = url.pathname.split('/').at(-1); + if (req.method === 'POST') { + let body = ''; + for await (const chunk of req) body += chunk; + assert.equal(JSON.parse(body).playwrightVersion, '1.60.0'); + const key = String(req.headers['idempotency-key']); + if (!sessions.has(key)) { sessions.set(key, `session-${sessions.size}`); order.push('momentic'); } + const sessionId = sessions.get(key); + if (mode === 'lost-response' && !lost) { lost = true; req.socket.destroy(); return; } + if (mode === 'malformed-json') { res.end('{"connectToken":"local-unpublished-credential'); return; } + res.end(JSON.stringify({ id: sessionId, status: 'READY', playwrightVersion: mode === 'compatible-patch' ? '1.60.1' : mode === 'incompatible-version' ? '1.61.0' : '1.60.0', wsEndpoint: mode === 'refused-connection' ? 'ws://127.0.0.1:1/refused?token=private-query' : `ws://127.0.0.1:${apiPort}/connect/momentic/${sessionId}`, ...(mode !== 'malformed-ready' ? { connectToken: 'local-connect-credential' } : {}) })); + } else if (req.method === 'DELETE') { + if (mode === 'cleanup-failure') { res.writeHead(500); res.end('{}'); return; } + if (id) terminated.add(id); + res.writeHead(204); res.end(); + } else { + res.end(JSON.stringify({ id, status: terminated.has(id ?? '') ? 'TERMINATED' : 'READY' })); + } + }); + const sockets = new Set(); + api.on('connection', socket => { sockets.add(socket); socket.on('error', () => {}); socket.on('close', () => sockets.delete(socket)); }); + api.on('connect', (_req, socket, head) => { + const upstream = net.connect(fixturePort, '127.0.0.1', () => { + socket.write('HTTP/1.1 200 Connection Established\r\n\r\n'); + if (head.length) upstream.write(head); + socket.pipe(upstream).pipe(socket); + }); + upstream.on('error', () => socket.destroy()); + socket.on('close', () => upstream.destroy()); + }); + api.on('upgrade', (req, socket, head) => { + const provider = req.url?.includes('/momentic/') ? 'momentic' : 'azure'; + const expected = provider === 'momentic' ? 'Bearer local-connect-credential' : 'Bearer local-azure-credential'; + if (req.headers.authorization !== expected) { socket.end('HTTP/1.1 401 Unauthorized\r\n\r\n'); return; } + websocketAuth.push(provider); + const target = new URL(endpoint); + const upstream = net.connect(Number(target.port), '127.0.0.1', () => { + const headers = Object.entries(req.headers).filter(([key]) => key !== 'host').map(([key, value]) => `${key}: ${value}`).join('\r\n'); + upstream.write(`GET ${target.pathname} HTTP/1.1\r\nHost: ${target.host}\r\n${headers}\r\n\r\n`); + if (head.length) upstream.write(head); + socket.pipe(upstream).pipe(socket); + }); + upstream.on('error', () => socket.destroy()); + socket.on('close', () => upstream.destroy()); + }); + const apiPort = await listen(api); + const server = await chromium.launchServer({ host: '127.0.0.1', headless: true, ...(process.env.PLAYWRIGHT_NATIVE_TEST_EXECUTABLE ? { executablePath: process.env.PLAYWRIGHT_NATIVE_TEST_EXECUTABLE } : {}), args: ['--ignore-certificate-errors'], proxy: { server: `http://127.0.0.1:${apiPort}` } }); + endpoint = server.wsEndpoint(); + try { + const environment = { region: 'local-fixture', runnerLocation: 'local-fixture', providers: Object.fromEntries(['momentic', 'azure'].map(name => [name, { region: 'local-fixture', playwrightVersion: '1.60.0', os: 'linux', headless: true, stealth: false, proxy: false, recording: false, placementEvidence: 'local fixture only', versionEvidence: 'local launchServer version', cleanupEvidence: 'fixture lifecycle verification' }])) }; + const urls = Array.from({ length: 100 }, (_, i) => `https://en.wikipedia.org/wiki/${i === 2 ? 'Native_no_links' : `Native_fixture_${i}`}`); + const environmentFile = path.join(directory, 'environment.json'); + const urlsFile = path.join(directory, 'urls.json'); + writeFileSync(environmentFile, JSON.stringify(environment)); + writeFileSync(urlsFile, JSON.stringify(urls)); + Object.assign(process.env, { MOMENTIC_BROWSER_FLEET_URL: `http://127.0.0.1:${apiPort}`, MOMENTIC_BROWSER_FLEET_API_KEY: 'local-momentic-credential', AZURE_PLAYWRIGHT_SERVICE_URL: `http://127.0.0.1:${apiPort}/browsers`, AZURE_PLAYWRIGHT_ACCESS_TOKEN: 'local-azure-credential', PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE: environmentFile, PLAYWRIGHT_NATIVE_URLS_FILE: urlsFile, PLAYWRIGHT_NATIVE_RESULTS_DIR: directory }); + const readiness = await import('./playwright-readiness.bench.js'); + const throughput = await import('./playwright-throughput.bench.js'); + const records = (outcome: BenchmarkRunOutcome) => outcome.participants.flatMap(p => p.records); + const ready = await runBenchmark({ ...readiness.config, iterations: 2 }, readiness.task, ['--no-ingest']); + assert.deepEqual(order, ['momentic', 'azure', 'momentic', 'azure']); + assert.deepEqual(websocketAuth, order); + for (const { participant, records } of ready.participants) { + if (participant !== 'azure') continue; + assert.deepEqual(records.map(record => record.data?.runId), [...azureRunIds]); + assert.ok(records.every(record => record.data?.sessionId === undefined), 'run IDs must not be reported as service session IDs'); + } + for (const record of records(ready)) { + assert.equal(record.status, 'success'); + assert.equal(record.data?.sessionSuccess, true); + assert.ok(Number(record.data?.readinessMs) >= 150, 'readiness waits for the delayed subresource load'); + assert.ok(Number(record.latencyMs) >= Number(record.data?.readinessMs)); + } + order.length = 0; + const through = await runBenchmark({ ...throughput.config, iterations: 3 }, throughput.task, ['--no-ingest']); + assert.deepEqual(order, ['momentic', 'azure', 'momentic', 'azure', 'momentic', 'azure']); + for (const record of records(through)) { + const actions = record.data?.actions; + assert.ok(Array.isArray(actions)); + assert.equal(actions.length, 50); + const screenshots = actions.filter(a => a && typeof a === 'object' && !Array.isArray(a) && a.type === 'screenshot'); + if (record.taskIndex < 2) { + assert.equal(record.status, 'success'); + assert.equal(record.data?.actionsCompleted, 50); + assert.equal(screenshots.length, 10); + assert.equal(record.data?.articleUrl, urls[record.taskIndex]); + } else { + assert.equal(record.status, 'error'); + assert.equal(record.data?.actionsCompleted, 20); + assert.equal(actions.filter(a => a && typeof a === 'object' && !Array.isArray(a) && a.error === 'skipped: click failed').length, 25); + } + const sum = actions.reduce((total, a) => total + (a && typeof a === 'object' && !Array.isArray(a) ? Number(a.durationMs) : 0), 0); + assert.equal(record.data?.taskMs, sum); + assert.equal(record.data?.actionsPerSecond, Number(record.data?.actionsCompleted) / (sum / 1000)); + assert.equal(record.steps?.find(step => step.name === 'actions')?.data?.actionsPerSecond, record.data?.actionsPerSecond); + } + const outputFile = path.join(directory, 'throughput', readdirSync(path.join(directory, 'throughput'))[0]); + const raw = readFileSync(outputFile, 'utf8'); + for (const secret of ['local-momentic-credential', 'local-connect-credential', 'local-azure-credential']) assert.ok(!raw.includes(secret)); + const output = JSON.parse(raw); + assert.deepEqual(output.articleUrls, urls); + for (const summary of output.summaries) { + assert.equal(summary.attempts, 3); + assert.equal(summary.successRate, 2 / 3); + assert.equal(summary.metrics.actionsPerSecond.count, 2); + assert.equal(summary.perActionType.screenshot.count, 20); + } + for (const failureMode of ['lost-response', 'malformed-ready', 'refused-connection', 'incompatible-version', 'cleanup-failure']) { + mode = failureMode; + order.length = 0; + const participants = readiness.config.participants.filter(p => p.name === 'momentic').map(p => ({ ...p })); + const failed = await runBenchmark({ ...readiness.config, participants, iterations: failureMode === 'cleanup-failure' ? 2 : 1 }, readiness.task, ['--no-ingest']); + assert.equal(records(failed)[0].status, 'error'); + assert.equal(records(failed)[0].data?.attemptStarted, true); + assert.equal(records(failed)[0].data?.cleanupSuccess, failureMode !== 'cleanup-failure'); + assert.ok(!JSON.stringify(failed).includes('private-query')); + if (failureMode === 'cleanup-failure') { + assert.equal(records(failed)[1].data?.attemptStarted, false); + assert.deepEqual(order, ['momentic'], 'uncertain cleanup stops new allocations'); + } else assert.equal(terminated.size, sessions.size); + } + mode = 'compatible-patch'; + const compatible = await runBenchmark({ ...readiness.config, participants: readiness.config.participants.filter(p => p.name === 'momentic').map(p => ({ ...p })), iterations: 1 }, readiness.task, ['--no-ingest']); + assert.equal(records(compatible)[0].status, 'success'); + assert.equal(records(compatible)[0].data?.serverPlaywrightVersion, '1.60.1'); + mode = 'refused-connection'; + const azureFailure = await runBenchmark({ ...readiness.config, participants: readiness.config.participants.filter(p => p.name === 'azure').map(p => ({ ...p })), iterations: 2 }, readiness.task, ['--no-ingest']); + assert.equal(records(azureFailure)[0].data?.cleanupSuccess, false); + assert.equal(records(azureFailure)[1].data?.attemptStarted, false); + assert.ok(!JSON.stringify(azureFailure).includes('private-query')); + mode = 'malformed-json'; + const malformed = await runBenchmark({ ...readiness.config, participants: readiness.config.participants.filter(p => p.name === 'momentic').map(p => ({ ...p })), iterations: 1 }, readiness.task, ['--no-ingest']); + assert.equal(records(malformed)[0].data?.cleanupSuccess, false); + assert.ok(!JSON.stringify(malformed).includes('local-unpublished-credential')); + assert.throws(() => articleUrl('https://local-user:local-password@en.wikipedia.org/wiki/Fixture')); + mode = ''; + order.length = 0; + const missing = await runBenchmark({ ...throughput.config, iterations: 1 }, ctx => throughput.task({ ...ctx, taskIndex: 100 }), ['--no-ingest']); + assert.deepEqual(order, []); + assert.ok(records(missing).every(record => record.status === 'error' && record.data?.attemptStarted === false)); + assert.deepEqual(distribution(Array.from({ length: 100 }, (_, i) => i + 1)), { count: 100, median: 50.5, p95: 95, p99: 99 }); + assert.equal(distribution([]).median, null); + writeFileSync(urlsFile, JSON.stringify(urls.slice(0, 1))); + order.length = 0; + await assert.rejects(promisify(execFile)(process.execPath, ['node_modules/tsx/dist/cli.mjs', 'packages/benchsdk-runner/dist/bin.js', 'run', 'benchmarks/browser/playwright-throughput.bench.ts', '--no-ingest', '--iterations', '2'], { env: process.env }), /Prepare a validated article URL/); + assert.deepEqual(order, [], 'short input lists fail before any provider request'); + writeFileSync(urlsFile, JSON.stringify(urls)); + // Exercise the published CLI entrypoints, not only runBenchmark imports. + for (const entry of ['playwright-readiness', 'playwright-throughput']) { + const { stdout } = await promisify(execFile)(process.execPath, ['node_modules/tsx/dist/cli.mjs', 'packages/benchsdk-runner/dist/bin.js', 'run', `benchmarks/browser/${entry}.bench.ts`, '--no-ingest', '--iterations', '1'], { env: process.env }); + assert.ok(stdout.includes('All done. No platform run created.')); + assert.ok(!stdout.includes('FAILED')); + } + // Missing credentials must fail the comparison before any allocation. + const diagnosticEnv = { ...process.env }; + delete diagnosticEnv.AZURE_PLAYWRIGHT_ACCESS_TOKEN; + for (const entry of ['playwright-readiness', 'playwright-throughput']) { + const command = ['node_modules/tsx/dist/cli.mjs', 'packages/benchsdk-runner/dist/bin.js', 'run', `benchmarks/browser/${entry}.bench.ts`, '--no-ingest', '--iterations=1']; + order.length = 0; + await assert.rejects(promisify(execFile)(process.execPath, command, { env: diagnosticEnv }), /Unavailable Playwright participant\(s\): azure: AZURE_PLAYWRIGHT_ACCESS_TOKEN/); + assert.deepEqual(order, []); + const { stdout } = await promisify(execFile)(process.execPath, [...command, '--provider=momentic'], { env: diagnosticEnv }); + assert.deepEqual(order, ['momentic']); + const match = stdout.match(/Raw Playwright-native records and untrimmed summaries: (.+)/); + assert.ok(match); + const diagnostic = JSON.parse(readFileSync(match[1].trim(), 'utf8')); + assert.equal(diagnostic.mode, 'provider-diagnostic'); + assert.deepEqual(diagnostic.unavailableParticipants, [{ name: 'azure', missing: ['AZURE_PLAYWRIGHT_ACCESS_TOKEN'] }]); + } + // Delay returning an actual remote context beyond the 30-second deadline. + // Cover reconciliation within cleanup, and ownership still unresolved at its bound. + for (const delayMs of [31_000, 46_000]) { + const client = await chromium.connect(endpoint); + const newContext = client.newContext.bind(client); + let pending: Promise | undefined; + let closed: Promise | undefined; + const contextMock = t.mock.method(client, 'newContext', (options: BrowserContextOptions) => { + pending = newContext(options).then(async created => { + closed = new Promise(resolve => created.once('close', () => resolve())); + await new Promise(resolve => setTimeout(resolve, delayMs)); + return created; + }); + return pending; + }); + const connectMock = t.mock.method(chromium, 'connect', async () => client); + const closeMock = delayMs === 46_000 ? t.mock.method(client, 'close', async () => { throw new Error('fixture disconnect failed'); }) : undefined; + try { + order.length = 0; + const participants = readiness.config.participants.filter(p => p.name === 'momentic').map(p => ({ ...p })); + const late = await runBenchmark({ ...readiness.config, participants, iterations: delayMs === 46_000 ? 2 : 1 }, readiness.task, ['--no-ingest']); + const [first, second] = records(late); + assert.equal(first.status, 'error'); + assert.match(String(first.data?.errorMessage), /Context creation timed out/); + assert.equal(first.data?.cleanupSuccess, delayMs === 31_000); + if (delayMs === 46_000) { + assert.equal(second.data?.attemptStarted, false); + assert.deepEqual(order, ['momentic'], 'unresolved context ownership stops allocations'); + } + assert.ok(pending && closed); + await pending; + await withTimeout(closed, 5_000, 'Late context was not closed'); + assert.equal(client.contexts().length, 0); + } finally { + contextMock.mock.restore(); connectMock.mock.restore(); closeMock?.mock.restore(); + await client.close(); + } + } + const audit = await chromium.connect(endpoint); + assert.equal(audit.contexts().length, 0, 'all benchmark contexts were closed'); + await audit.close(); + console.log('Native benchmark end-to-end checks passed.'); + } finally { + await server.close(); + for (const socket of sockets) socket.destroy(); + api.closeAllConnections(); fixtures.closeAllConnections(); + await Promise.all([new Promise(resolve => api.close(() => resolve())), new Promise(resolve => fixtures.close(() => resolve()))]); + for (const [key, value] of previous) { if (value === undefined) delete process.env[key]; else process.env[key] = value; } + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/benchmarks/browser/playwright-prepare-urls.ts b/benchmarks/browser/playwright-prepare-urls.ts new file mode 100644 index 00000000..35899986 --- /dev/null +++ b/benchmarks/browser/playwright-prepare-urls.ts @@ -0,0 +1,27 @@ +/** Resolve and DOM-validate shared Wikipedia inputs before measured sessions. */ +import { chromium } from 'playwright-native-core'; +import { writeFileSync } from 'node:fs'; +import { articleUrl, firstArticleLink } from './playwright-workload.js'; +import { withTimeout } from '../src/util/timeout.js'; +import { VIEWPORT } from './playwright-providers.js'; + +const count = Number(process.argv[2] ?? '100'); +const filename = process.argv[3] ?? 'playwright-native-urls.json'; +if (!Number.isInteger(count) || count < 1) throw new Error('Count must be a positive integer'); +const browser = await chromium.launch({ headless: true, timeout: 30_000 }); +try { + const page = await browser.newPage({ viewport: VIEWPORT }); + page.setDefaultTimeout(30_000); + page.setDefaultNavigationTimeout(30_000); + const urls: string[] = []; + for (let i = 0; i < count; i++) { + await page.goto('https://en.wikipedia.org/wiki/Special:Random', { waitUntil: 'load' }); + const url = articleUrl(page.url()); + await page.waitForSelector('#firstHeading'); + await withTimeout((await firstArticleLink(page)).click(), 30_000, 'Article link validation timed out'); + await page.waitForSelector('#firstHeading'); + urls.push(url); + } + writeFileSync(filename, JSON.stringify(urls, null, 2)); + console.log(`Wrote ${urls.length} validated shared inputs to ${filename}`); +} finally { await withTimeout(browser.close(), 15_000, 'URL preparation cleanup timed out'); } diff --git a/benchmarks/browser/playwright-providers.ts b/benchmarks/browser/playwright-providers.ts new file mode 100644 index 00000000..2a3660f1 --- /dev/null +++ b/benchmarks/browser/playwright-providers.ts @@ -0,0 +1,191 @@ +import { setTimeout as delay } from 'node:timers/promises'; +import { readFileSync } from 'node:fs'; +import type { JsonObject } from '@benchsdk/api'; +import { parseCliArgs, selectParticipants, filterParticipantsByEnv } from '@benchsdk/runner'; +import type { PlaywrightProviderConfig } from './playwright-types.js'; + +export const PLAYWRIGHT_VERSION = '1.60.0'; +export const PROVISION_TIMEOUT_MS = 120_000; +export const CONNECT_TIMEOUT_MS = 30_000; +export const CLEANUP_TIMEOUT_MS = 15_000; +export const VIEWPORT = { width: 1920, height: 1080 }; + +// Parse only flags owned by the suites; the runner handles the rest of the CLI. +export const playwrightArgs = parseCliArgs(process.argv.flatMap((arg, index) => { + if (/^--(?:provider|iterations)=/.test(arg)) return [arg]; + if (arg === '--provider' || arg === '--iterations') return [arg, process.argv[index + 1] ?? '']; + return []; +})); + +export function requireProviderCredentials(): void { + const selected = selectParticipants(playwrightProviders, playwrightArgs.providers); + const { skipped } = filterParticipantsByEnv(selected); + if (skipped.length) { + const missing = skipped.map(({ name, missing }) => `${name}: ${missing.join(', ')}`).join('; '); + throw new Error(`Unavailable Playwright participant(s): ${missing}. Supply credentials for every selected provider; use --provider for an explicit diagnostic run.`); + } +} + +export function requiredEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`Missing ${name}`); + return value; +} + +function object(value: unknown): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('Invalid provider response'); + return value as Record; +} + +function string(value: unknown): string { + if (typeof value !== 'string' || !value.trim()) throw new Error('Missing provider response field'); + return value; +} + +function environment(provider: string): JsonObject { + const filename = requiredEnv('PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE'); + let parsed: unknown; + try { parsed = JSON.parse(readFileSync(filename, 'utf8')); } + catch { throw new Error('Cannot read a valid Playwright-native environment manifest'); } + const manifest = object(parsed); + const entry = object(object(manifest.providers)[provider]); + const region = string(manifest.region); + const serverVersion = string(entry.playwrightVersion); + if (entry.region !== region || !/^1\.60\.\d+$/.test(serverVersion)) { + throw new Error('Both providers must attest to the shared region and Playwright 1.60.x'); + } + if (entry.os !== 'linux' || entry.headless !== true || entry.stealth !== false || entry.proxy !== false || entry.recording !== false) { + throw new Error('Environment must attest to headless Linux Chromium without optional features'); + } + return { + region, + runnerLocation: string(manifest.runnerLocation), + placementEvidence: string(entry.placementEvidence), + versionEvidence: string(entry.versionEvidence), + cleanupEvidence: string(entry.cleanupEvidence), + serverPlaywrightVersion: serverVersion, + serverPlaywrightVersionSource: 'operator-verified environment manifest', + clientPlaywrightVersion: PLAYWRIGHT_VERSION, + browserOs: 'linux', headless: true, viewport: VIEWPORT, + stealth: false, proxy: false, recording: false, + metadataSource: 'operator-verified environment manifest', + }; +} + +export function safeError(error: unknown, secrets: string[] = []): string { + let message = error instanceof Error ? error.message : 'Operation failed'; + const values = [...secrets, ...Object.entries(process.env) + .filter(([key]) => /token|key|secret|password/i.test(key)) + .map(([, value]) => value ?? '')]; + for (const value of values) if (value) message = message.split(value).join('[redacted]'); + return message.replace(/(?:https?|wss?):\/\/[^\s<>]+/g, '[endpoint]') + .replace(/Bearer\s+[^\s]+/gi, 'Bearer [redacted]'); +} + +async function request(url: URL, init: RequestInit, signal: AbortSignal): Promise { + const response = await fetch(url, { ...init, signal, redirect: 'error' }); + if (!response.ok) throw new Error(`Provider request failed (HTTP ${response.status})`); + return response; +} + +async function sessionResponse(response: Response): Promise> { + // JSON parser diagnostics may include credential-bearing response snippets. + try { return object(await response.json()); } + catch { throw new Error('Provider returned invalid session JSON'); } +} + +export const playwrightProviders: PlaywrightProviderConfig[] = [ + { + name: 'momentic', + requiredEnvVars: ['MOMENTIC_BROWSER_FLEET_URL', 'MOMENTIC_BROWSER_FLEET_API_KEY', 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE'], + session(identity) { + const base = new URL(requiredEnv('MOMENTIC_BROWSER_FLEET_URL')); + const headers = { Authorization: `Bearer ${requiredEnv('MOMENTIC_BROWSER_FLEET_API_KEY')}`, 'Content-Type': 'application/json', 'Idempotency-Key': identity }; + const metadata = environment('momentic'); + let sessionId: string | undefined; + let creationUncertain = false; + const create = async (signal: AbortSignal) => { + creationUncertain = true; + const response = await fetch(new URL('/v1/sessions', base), { + method: 'POST', headers, signal, redirect: 'error', + body: JSON.stringify({ browser: 'chromium', playwrightVersion: PLAYWRIGHT_VERSION, network: { mode: 'playwright' }, ttlSeconds: 3600, waitTimeoutMs: 0 }), + }); + if (!response.ok) { + // A definitive rejection did not allocate a session. A 5xx may have. + creationUncertain = response.status >= 500; + throw new Error(`Session creation failed (HTTP ${response.status})`); + } + const data = await sessionResponse(response); + sessionId = string(data.id); // Retain the ID before validating other fields. + metadata.sessionId = sessionId; + creationUncertain = false; + return data; + }; + const retrieve = async (signal: AbortSignal) => { + if (!sessionId) throw new Error('Session ID unavailable'); + return sessionResponse(await request(new URL(`/v1/sessions/${encodeURIComponent(sessionId)}`, base), { headers }, signal)); + }; + return { + metadata, + async provision() { + const signal = AbortSignal.timeout(PROVISION_TIMEOUT_MS); + let data = await create(signal); + while (data.status === 'QUEUED' || data.status === 'PROVISIONING') { + await delay(500, undefined, { signal }); + data = await retrieve(signal); + } + if (data.status !== 'READY') throw new Error('Momentic session did not become READY'); + if (typeof data.playwrightVersion !== 'string' || !/^1\.60\.\d+$/.test(data.playwrightVersion)) throw new Error('Momentic returned an incompatible Playwright version'); + metadata.serverPlaywrightVersion = string(data.playwrightVersion); + metadata.serverPlaywrightVersionSource = 'Momentic session response'; + return { endpoint: string(data.wsEndpoint), headers: { Authorization: `Bearer ${string(data.connectToken)}` } }; + }, + async cleanup() { + const signal = AbortSignal.timeout(CLEANUP_TIMEOUT_MS); + // Reconcile a lost POST response using the same key, never a new attempt. + if (!sessionId && creationUncertain) await create(signal); + if (!sessionId) return; + await request(new URL(`/v1/sessions/${encodeURIComponent(sessionId)}`, base), { method: 'DELETE', headers }, signal); + for (;;) { + const data = await retrieve(signal); + if (data.status === 'TERMINATED' || data.status === 'EXPIRED' || data.status === 'FAILED') return; + await delay(500, undefined, { signal }); + } + }, + }; + }, + }, + { + name: 'azure', + requiredEnvVars: ['AZURE_PLAYWRIGHT_SERVICE_URL', 'AZURE_PLAYWRIGHT_ACCESS_TOKEN', 'PLAYWRIGHT_NATIVE_ENVIRONMENT_FILE'], + session(identity) { + const url = new URL(requiredEnv('AZURE_PLAYWRIGHT_SERVICE_URL')); + const headers = { Authorization: `Bearer ${requiredEnv('AZURE_PLAYWRIGHT_ACCESS_TOKEN')}` }; + const metadata = { ...environment('azure'), runId: identity }; + url.protocol = url.protocol === 'wss:' ? 'https:' : url.protocol === 'ws:' ? 'http:' : url.protocol; + url.searchParams.set('api-version', '2025-09-01'); + url.searchParams.set('os', 'Linux'); + url.searchParams.set('runId', identity); + let allocationUncertain = false; + return { + metadata, + async provision() { + // Resolve the documented redirect inside the measured interval. + allocationUncertain = true; + const response = await fetch(url, { headers, redirect: 'manual', signal: AbortSignal.timeout(PROVISION_TIMEOUT_MS) }); + if (response.status >= 400 && response.status < 500) allocationUncertain = false; + if (response.status !== 302 && response.status !== 307) throw new Error(`Azure browser redirect failed (HTTP ${response.status})`); + const endpoint = new URL(string(response.headers.get('location')), url); + if (endpoint.protocol !== 'wss:' && endpoint.protocol !== 'ws:') throw new Error('Azure did not return a Playwright WebSocket endpoint'); + return { endpoint: endpoint.href, headers }; + }, + // Azure owns termination on connection close; verify this before a comparison. + async cleanup({ connected }) { + if (allocationUncertain && !connected) { + throw new Error('Azure allocation cleanup is uncertain after failed provisioning/connection; verify resources before rerunning'); + } + }, + }; + }, + }, +]; diff --git a/benchmarks/browser/playwright-readiness.bench.ts b/benchmarks/browser/playwright-readiness.bench.ts new file mode 100644 index 00000000..feea7f00 --- /dev/null +++ b/benchmarks/browser/playwright-readiness.bench.ts @@ -0,0 +1,44 @@ +/** First per-session request → example.com load; cleanup is measured separately. */ +import '../src/env.js'; +import { defineBenchmarkConfig, defineTask } from '@benchsdk/runner'; +import { runSession } from './playwright-session.js'; +import { playwrightProviders, requireProviderCredentials } from './playwright-providers.js'; +import type { PlaywrightProviderConfig } from './playwright-types.js'; +import { writeNativeResults } from './playwright-results.js'; + +requireProviderCredentials(); + +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'playwright-native-readiness', + benchmarkName: 'Playwright-native Browser Readiness', + iterations: 100, + concurrency: 1, + groupBy: 'round', + participants: playwrightProviders, + display: { + metrics: [ + { key: 'readinessMs', label: 'Browser readiness', unit: 'ms', direction: 'lower-better' }, + { key: 'cleanupMs', label: 'Cleanup', unit: 'ms', direction: 'lower-better' }, + ], + steps: [ + { key: 'provision', label: 'Provision session' }, + { key: 'connect', label: 'Connect Playwright' }, + { key: 'page', label: 'Create context and page' }, + { key: 'navigate', label: 'Navigate page' }, + { key: 'reconcile-context', label: 'Reconcile pending context' }, + { key: 'close-context', label: 'Close context' }, + { key: 'close-browser', label: 'Close browser' }, + { key: 'release', label: 'Release session' }, + ], + overview: { defaultMetric: 'readinessMs', defaultLayout: 'chart' }, + }, + onComplete: outcome => writeNativeResults(outcome, 'readiness'), +}); + +export const task = defineTask(ctx => runSession(ctx, async (page, startedAt, data) => { + await ctx.step('navigate', async () => { + await page.goto('https://www.example.com', { waitUntil: 'load' }); + data.readinessMs = performance.now() - startedAt; + ctx.measure({ readinessMs: data.readinessMs }); + }); +})); diff --git a/benchmarks/browser/playwright-results.ts b/benchmarks/browser/playwright-results.ts new file mode 100644 index 00000000..b64200fd --- /dev/null +++ b/benchmarks/browser/playwright-results.ts @@ -0,0 +1,54 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { filterParticipantsByEnv, type BenchmarkRunOutcome } from '@benchsdk/runner'; +import { playwrightProviders } from './playwright-providers.js'; +import { ACTION_TYPES } from './throughput-types.js'; + +/** Untrimmed nearest-rank percentiles; missing observations are not zeroes. */ +export function distribution(values: number[]) { + const sorted = [...values].sort((a, b) => a - b); + const n = sorted.length; + if (!n) return { count: 0, median: null, p95: null, p99: null }; + const mid = Math.floor(n / 2); + return { + count: n, + median: n % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2, + p95: sorted[Math.ceil(n * 0.95) - 1], + p99: sorted[Math.ceil(n * 0.99) - 1], + }; +} + +export function writeNativeResults(outcome: BenchmarkRunOutcome, suite: 'readiness' | 'throughput', articleUrls?: string[]): void { + const comparison = playwrightProviders.every(provider => outcome.participants.some(p => p.participant === provider.name)); + const { skipped: unavailableParticipants } = filterParticipantsByEnv(playwrightProviders); + const summaries = outcome.participants.map(({ participant, records }) => { + const attempted = records.filter(record => record.data?.attemptStarted === true); + const complete = attempted.filter(record => record.data?.workloadSuccess === true); + const metrics = Object.fromEntries(['readinessMs', 'taskMs', 'actionsPerSecond', 'screenshotMs', 'cleanupMs'].map(key => [key, + distribution((key === 'cleanupMs' ? attempted : complete).flatMap(record => { + const value = record.data?.[key]; + return typeof value === 'number' && Number.isFinite(value) ? [value] : []; + })), + ])); + const actions = complete.flatMap(record => { + const actions = record.data?.actions; + if (!Array.isArray(actions)) return []; + return actions.flatMap(action => { + if (!action || typeof action !== 'object' || Array.isArray(action)) return []; + if (action.success !== true || typeof action.durationMs !== 'number') return []; + return [{ type: action.type, durationMs: action.durationMs }]; + }); + }); + const perActionType = Object.fromEntries(ACTION_TYPES.map(type => [type, + distribution(actions.filter(action => action.type === type).map(action => action.durationMs)), + ])); + const rate = (key: string) => attempted.length ? attempted.filter(record => record.data?.[key] === true).length / attempted.length : null; + return { participant, scheduled: records.length, attempts: attempted.length, workloadSuccessRate: rate('workloadSuccess'), cleanupSuccessRate: rate('cleanupSuccess'), successRate: rate('sessionSuccess'), metrics, perActionType }; + }); + const directory = path.resolve(process.env.PLAYWRIGHT_NATIVE_RESULTS_DIR ?? 'results/playwright-native', suite); + mkdirSync(directory, { recursive: true }); + const filename = path.join(directory, `${randomUUID()}.json`); + writeFileSync(filename, JSON.stringify({ suite, mode: comparison ? 'comparison' : 'provider-diagnostic', unavailableParticipants, runId: outcome.runId, createdAt: new Date().toISOString(), config: outcome.config, ...(articleUrls ? { articleUrls } : {}), summaries, participants: outcome.participants }, null, 2)); + console.log(`Raw Playwright-native records and untrimmed summaries: ${filename}`); +} diff --git a/benchmarks/browser/playwright-session.ts b/benchmarks/browser/playwright-session.ts new file mode 100644 index 00000000..48d3eaf8 --- /dev/null +++ b/benchmarks/browser/playwright-session.ts @@ -0,0 +1,95 @@ +import { chromium, type Page, type Browser, type BrowserContext } from 'playwright-native-core'; +import { randomUUID } from 'node:crypto'; +import { TaskError, type TaskContext, type TaskStepOptions } from '@benchsdk/runner'; +import type { JsonObject } from '@benchsdk/api'; +import { withTimeout } from '../src/util/timeout.js'; +import { CLEANUP_TIMEOUT_MS, CONNECT_TIMEOUT_MS, VIEWPORT, safeError } from './playwright-providers.js'; +import type { PlaywrightProviderConfig } from './playwright-types.js'; + +const unsafeParticipants = new WeakSet(); + +/** Timer spans the first provider request through workload completion only. */ +export async function runSession( + ctx: TaskContext, + workload: (page: Page, startedAt: number, data: JsonObject) => Promise, +): Promise<{ data: JsonObject }> { + if (unsafeParticipants.has(ctx.participant)) { + throw new TaskError('Participant stopped after uncertain cleanup; verify resources before rerunning', { + code: 'CLEANUP_UNCERTAIN', data: { attemptStarted: false }, + }); + } + // Read credentials and validate settings before starting the timer. + const session = ctx.participant.session(randomUUID()); + const data: JsonObject = { ...session.metadata, attemptStarted: true, workloadSuccess: false, cleanupSuccess: false }; + let browser: Browser | undefined; + let context: BrowserContext | undefined; + let pendingContext: Promise | undefined; + let closeLateContext = false; + let secrets: string[] = []; + let failure: string | undefined; + const cleanupErrors: string[] = []; + // Redact errors before the runner records step diagnostics. + const step = (name: string, fn: () => Promise, options?: Pick) => + ctx.step(name, async () => { + try { return await fn(); } + catch (error) { throw new Error(safeError(error, secrets)); } + }, options); + const startedAt = performance.now(); + try { + const connection = await step('provision', () => session.provision()); + secrets = Object.values(connection.headers); + browser = await step('connect', () => chromium.connect(connection.endpoint, { headers: connection.headers, timeout: CONNECT_TIMEOUT_MS })); + data.chromiumVersion = browser.version(); + data.runnerOs = process.platform; + data.nodeVersion = process.version; + const connected = browser; + const page = await step('page', async () => { + // Keep ownership even when the caller's timeout wins the race. + pendingContext = connected.newContext({ viewport: VIEWPORT }).then(async created => { + context = created; + if (closeLateContext) await withTimeout(created.close(), CLEANUP_TIMEOUT_MS, 'Late context cleanup timed out'); + return created; + }); + context = await withTimeout(pendingContext, CONNECT_TIMEOUT_MS, 'Context creation timed out'); + context.setDefaultTimeout(30_000); + context.setDefaultNavigationTimeout(30_000); + return withTimeout(context.newPage(), CONNECT_TIMEOUT_MS, 'Page creation timed out'); + }); + await workload(page, startedAt, data); + data.workloadSuccess = true; + } catch (error) { + failure = safeError(error, secrets); + data.failedAfterMs = performance.now() - startedAt; + } finally { + const cleanupStart = performance.now(); + const cleanup = async (name: string, fn: () => Promise) => { + try { + await step(name, () => withTimeout(fn(), CLEANUP_TIMEOUT_MS, `${name} timed out`), { reportConcurrency: false }); + } catch (error) { cleanupErrors.push(safeError(error, secrets)); } + }; + const creation = pendingContext; + if (creation && !context) { + // A rejected creation owns no context; an unresolved one makes cleanup uncertain. + await cleanup('reconcile-context', () => creation.catch(() => undefined)); + } + closeLateContext = true; + for (const [name, resource] of [['close-context', context], ['close-browser', browser]] as const) { + if (resource) await cleanup(name, () => resource.close()); + } + await cleanup('release', () => session.cleanup({ connected: browser !== undefined })); + data.cleanupMs = performance.now() - cleanupStart; + data.cleanupSuccess = cleanupErrors.length === 0; + Object.assign(data, session.metadata); + if (cleanupErrors.length) { + data.cleanupErrors = cleanupErrors; + unsafeParticipants.add(ctx.participant); + } + } + data.sessionSuccess = data.workloadSuccess === true && data.cleanupSuccess === true; + if (failure) data.errorMessage = failure; + ctx.measure(data); + if (failure || cleanupErrors.length) { + throw new TaskError(failure ?? 'Session cleanup failed', { code: 'PLAYWRIGHT_NATIVE_ERROR', data }); + } + return { data }; +} diff --git a/benchmarks/browser/playwright-throughput.bench.ts b/benchmarks/browser/playwright-throughput.bench.ts new file mode 100644 index 00000000..68227eb5 --- /dev/null +++ b/benchmarks/browser/playwright-throughput.bench.ts @@ -0,0 +1,69 @@ +/** Five ten-action loops; setup and cleanup excluded from action throughput. */ +import '../src/env.js'; +import { defineBenchmarkConfig, defineTask, TaskError, mergeConfig } from '@benchsdk/runner'; +import { runSession } from './playwright-session.js'; +import { playwrightProviders, requireProviderCredentials, playwrightArgs } from './playwright-providers.js'; +import type { PlaywrightProviderConfig } from './playwright-types.js'; +import { distribution, writeNativeResults } from './playwright-results.js'; +import { ACTIONS_PER_SESSION, loadArticleUrls, runActionLoop } from './playwright-workload.js'; +import type { ActionResult } from './throughput-types.js'; + +// Load the frozen validated inputs before any measured provider request. +const urls = loadArticleUrls(); + +requireProviderCredentials(); + +export const config = defineBenchmarkConfig({ + benchmarkSlug: 'playwright-native-throughput', + benchmarkName: 'Playwright-native Browser Throughput', + iterations: 100, + concurrency: 1, + groupBy: 'round', + participants: playwrightProviders, + display: { + metrics: [ + { key: 'actionsPerSecond', label: 'Actions/s', unit: '/s', direction: 'higher-better' }, + { key: 'actionsCompleted', label: 'Actions completed', direction: 'higher-better', decimals: 0 }, + { key: 'taskMs', label: 'Sum of action durations', unit: 'ms', direction: 'lower-better' }, + { key: 'screenshotMs', label: 'Session median screenshot', unit: 'ms', direction: 'lower-better' }, + { key: 'cleanupMs', label: 'Cleanup', unit: 'ms', direction: 'lower-better' }, + ], + steps: [ + { key: 'provision', label: 'Provision session' }, + { key: 'connect', label: 'Connect Playwright' }, + { key: 'page', label: 'Create context and page' }, + { key: 'actions', label: 'Actions' }, + { key: 'reconcile-context', label: 'Reconcile pending context' }, + { key: 'close-context', label: 'Close context' }, + { key: 'close-browser', label: 'Close browser' }, + { key: 'release', label: 'Release session' }, + ], + overview: { defaultMetric: 'actionsPerSecond', defaultLayout: 'chart' }, + }, + onComplete: outcome => writeNativeResults(outcome, 'throughput', urls), +}); + +const iterations = mergeConfig(config, playwrightArgs).iterations; +if (urls.length < iterations) throw new Error('Prepare a validated article URL for every requested iteration before measurement'); + +export const task = defineTask(ctx => { + const url = urls[ctx.taskIndex]; + if (!url) throw new TaskError('Validated URL list is shorter than the requested run; prepare all inputs first', { code: 'MISSING_INPUT', data: { attemptStarted: false } }); + const actions: ActionResult[] = []; + return runSession(ctx, async (page, _startedAt, data) => { + data.articleUrl = url; + await ctx.step('actions', async () => { + try { + await runActionLoop(page, url, actions); + } finally { + const taskMs = actions.reduce((sum, action) => sum + action.durationMs, 0); + const actionsCompleted = actions.filter(action => action.success).length; + const screenshotMs = distribution(actions.filter(action => action.type === 'screenshot' && action.success).map(action => action.durationMs)).median; + const metrics = { taskMs, actionsCompleted, actionsPerSecond: taskMs > 0 ? actionsCompleted / (taskMs / 1000) : 0, ...(screenshotMs !== null ? { screenshotMs } : {}) }; + Object.assign(data, metrics, { actions }); + ctx.measure(metrics); + } + }); + if (data.actionsCompleted !== ACTIONS_PER_SESSION) throw new Error('Incomplete action session'); + }); +}); diff --git a/benchmarks/browser/playwright-types.ts b/benchmarks/browser/playwright-types.ts new file mode 100644 index 00000000..d917ae20 --- /dev/null +++ b/benchmarks/browser/playwright-types.ts @@ -0,0 +1,13 @@ +import type { JsonObject } from '@benchsdk/api'; + +export interface PlaywrightSession { + metadata: JsonObject; + provision(): Promise<{ endpoint: string; headers: Record }>; + cleanup(options: { connected: boolean }): Promise; +} + +export interface PlaywrightProviderConfig { + name: string; + requiredEnvVars: string[]; + session(identity: string): PlaywrightSession; +} diff --git a/benchmarks/browser/playwright-workload.ts b/benchmarks/browser/playwright-workload.ts new file mode 100644 index 00000000..9bd2a3b2 --- /dev/null +++ b/benchmarks/browser/playwright-workload.ts @@ -0,0 +1,70 @@ +import { readFileSync } from 'node:fs'; +import type { Page, ElementHandle } from 'playwright-native-core'; +import { withTimeout } from '../src/util/timeout.js'; +import type { ActionResult, ActionType } from './throughput-types.js'; +import { requiredEnv, safeError } from './playwright-providers.js'; + +const LOOPS_PER_SESSION = 5; +export const ACTIONS_PER_SESSION = LOOPS_PER_SESSION * 10; +export const ACTION_TIMEOUT_MS = 30_000; +export const ARTICLE_LINK_SELECTOR = '#mw-content-text a[href*="/wiki/"]'; + +export function articleUrl(value: unknown): string { + if (typeof value !== 'string') throw new Error('Article URL must be a string'); + const url = new URL(value); + const article = url.pathname.slice('/wiki/'.length); + if (url.origin !== 'https://en.wikipedia.org' || url.username || url.password || !url.pathname.startsWith('/wiki/') || !article || decodeURIComponent(article).includes(':') || url.search || url.hash) { + throw new Error('Expected a concrete Wikipedia article URL'); + } + return url.href; +} + +export function loadArticleUrls(): string[] { + const parsed: unknown = JSON.parse(readFileSync(requiredEnv('PLAYWRIGHT_NATIVE_URLS_FILE'), 'utf8')); + if (!Array.isArray(parsed) || parsed.length === 0) throw new Error('Expected a nonempty array of validated article URLs'); + return parsed.map(articleUrl); +} + +export async function firstArticleLink(page: Page): Promise { + await page.waitForSelector(ARTICLE_LINK_SELECTOR, { timeout: 10_000 }); + const links = await page.$$(ARTICLE_LINK_SELECTOR); + for (const link of links) { + const href = await link.getAttribute('href'); + const match = href?.match(/\/wiki\/([^#]*)/); + if (match && !match[1].includes(':')) return link; + } + throw new Error('No article body link found on page'); +} + +export async function runActionLoop(page: Page, url: string, actions: ActionResult[]): Promise { + const operations: [ActionType, () => Promise][] = [ + ['navigate', () => page.goto(url, { waitUntil: 'load' })], + ['waitForSelector', () => page.waitForSelector('#firstHeading')], + ['screenshot', () => page.screenshot()], + ['textContent', () => page.textContent('#firstHeading')], + ['click', async () => { await (await firstArticleLink(page)).click(); }], + ['waitForSelector', () => page.waitForSelector('#firstHeading')], + ['screenshot', () => page.screenshot()], + ['textContent', () => page.textContent('#firstHeading')], + ['goBack', () => page.goBack({ waitUntil: 'commit' })], + ['waitForSelector', () => page.waitForSelector('#firstHeading')], + ]; + for (let loop = 0; loop < LOOPS_PER_SESSION; loop++) { + let clickFailed = false; + for (const [offset, [type, operation]] of operations.entries()) { + const index = loop * operations.length + offset + 1; + if (clickFailed && offset >= 5) { + actions.push({ index, type, durationMs: 0, success: false, error: 'skipped: click failed' }); + continue; + } + const start = performance.now(); + try { + await withTimeout(operation(), ACTION_TIMEOUT_MS, 'Action timed out'); + actions.push({ index, type, durationMs: performance.now() - start, success: true }); + } catch (error) { + actions.push({ index, type, durationMs: performance.now() - start, success: false, error: safeError(error) }); + if (offset === 4) clickFailed = true; + } + } + } +} diff --git a/package.json b/package.json index ec168cd1..7b0d3703 100644 --- a/package.json +++ b/package.json @@ -147,7 +147,11 @@ "bench:ai-gateway-model-index:ngrok": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-model-index.bench.ts --provider ngrok", "bench:ai-gateway-model-index:llmapi": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-model-index.bench.ts --provider llmapi", "bench:ai-gateway-model-index:novita": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-model-index.bench.ts --provider novita", - "bench:ai-gateway-model-index:github-copilot": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-model-index.bench.ts --provider github-copilot" + "bench:ai-gateway-model-index:github-copilot": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/ai-gateway/ai-gateway-model-index.bench.ts --provider github-copilot", + "bench:playwright-readiness": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/playwright-readiness.bench.ts", + "bench:playwright-throughput": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/playwright-throughput.bench.ts", + "bench:playwright-prepare-urls": "tsx benchmarks/browser/playwright-prepare-urls.ts", + "test:playwright-native": "tsx --test benchmarks/browser/playwright-native.test.ts" }, "dependencies": { "@aws-sdk/client-s3": "^3.1152.0", @@ -223,6 +227,7 @@ "p-limit": "^7.3.3", "pg": "^8.23.1", "playwright-core": "^1.64.1", + "playwright-native-core": "npm:playwright-core@1.60.0", "sql.js": "^1.15.0", "tensorlake": "0.5.148" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 21e87d30..585cb71a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -227,6 +227,9 @@ importers: playwright-core: specifier: ^1.64.1 version: 1.64.1 + playwright-native-core: + specifier: npm:playwright-core@1.60.0 + version: playwright-core@1.60.0 sql.js: specifier: ^1.15.0 version: 1.15.0 @@ -4797,6 +4800,11 @@ packages: platform@1.3.6: resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + playwright-core@1.60.0: + resolution: {integrity: sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==} + engines: {node: '>=18'} + hasBin: true + playwright-core@1.64.1: resolution: {integrity: sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ==} engines: {node: '>=20'} @@ -10941,6 +10949,8 @@ snapshots: platform@1.3.6: {} + playwright-core@1.60.0: {} + playwright-core@1.64.1: {} playwright@1.64.1: