diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cb87553..6fdb1f7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,14 +11,14 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 with: - node-version: 14 + node-version: 20 - name: Get yarn cache directory path id: yarn-cache-dir-path run: echo "::set-output name=dir::$(yarn cache dir)" - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: v1/${{ runner.os }}/node-14/${{ hashFiles('**/yarn.lock') }} - restore-keys: v1/${{ runner.os }}/node-14/ + key: v1/${{ runner.os }}/node-20/${{ hashFiles('**/yarn.lock') }} + restore-keys: v1/${{ runner.os }}/node-20/ - run: yarn - run: yarn lint diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3b6f3a0..b2be6fe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - node: [14] + node: [20, 22] runs-on: ${{ matrix.os }} steps: - uses: actions-ecosystem/action-regex-match@9e6c4fb3d5e898f505be7a1fb6e7b0a278f6665b # v2.0.2 @@ -41,6 +41,9 @@ jobs: key: v1/${{ runner.os }}/node-${{ matrix.node }}/${{ hashFiles('**/yarn.lock') }} restore-keys: v1/${{ runner.os }}/node-${{ matrix.node }}/ - run: yarn + # Playwright >=1.39 no longer downloads browsers on package install — fetch the one the + # suite launches explicitly (system deps included for the runner image). + - run: npx playwright install --with-deps chromium - name: Set up @percy/cli from git if: ${{ github.event_name == 'workflow_dispatch' }} env: diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index c65e28d..02bfb54 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -11,14 +11,14 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 with: - node-version: 14 + node-version: 20 - name: Get yarn cache directory path id: yarn-cache-dir-path run: echo "::set-output name=dir::$(yarn cache dir)" - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: v1/${{ runner.os }}/node-14/${{ hashFiles('**/yarn.lock') }} - restore-keys: v1/${{ runner.os }}/node-14/ + key: v1/${{ runner.os }}/node-20/${{ hashFiles('**/yarn.lock') }} + restore-keys: v1/${{ runner.os }}/node-20/ - run: yarn - run: yarn test:types diff --git a/.nycrc b/.nycrc index 86953b6..9d1d3b0 100644 --- a/.nycrc +++ b/.nycrc @@ -1,5 +1,5 @@ { - "exclude": ["test"], + "exclude": ["test", "dropin", "bin"], "check-coverage": true, "branches": 100, "lines": 100, diff --git a/README.md b/README.md index 9ea9ae3..d043cc7 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,59 @@ $ percy exec -- node script.js - `options` - [See per-snapshot configuration options](https://www.browserstack.com/docs/percy/take-percy-snapshots/overview#per-snapshot-configuration) +## toHaveScreenshot drop-in + +Route your existing Playwright `expect(...).toHaveScreenshot()` assertions through Percy with +**one config line and no test changes**: + +```js +// playwright.config.js +require('@percy/playwright/dropin'); // registers the toHaveScreenshot override + +module.exports = defineConfig({ /* your config */ }); +``` + +```bash +PERCY_TOKEN= npx percy-playwright exec -- npx playwright test +``` + +The bundled `percy-playwright` wrapper tags the build (`PERCY_BUILD_SOURCE=playwright-dropin`) and +marks it as a first-build baseline candidate (`PERCY_DROPIN_BASELINE_CANDIDATE=true`); the Percy +API decides first-ness server-side. Every `toHaveScreenshot()` is captured and uploaded to Percy; +the assertion **always passes locally** — the visual verdict moves to Percy's review UI, and a +missing/invalid token or any Percy error **never fails your suite** (the whole run falls back to +native `toHaveScreenshot`). + +### First build from your committed baselines + +Add the drop-in's `globalSetup` and the project's **first** build is seeded from the Playwright +baseline PNGs already committed in your repo — the baselines you've already blessed — and +auto-approved server-side (flag-gated), so diffs start on your very next run: + +```js +module.exports = defineConfig({ + globalSetup: require.resolve('@percy/playwright/dropin/global-setup'), + /* your config */ +}); +``` + +### Capture modes + +Zero-config uses screenshot mode (raw-PNG upload — generic/app Percy projects). For a **web** +Percy project, switch to DOM capture in `.percy-playwright-dropin.json`: + +```json +{ "captureMode": "snapshot" } +``` + +Snapshot mode serializes the live page with the same capture `percySnapshot()` uses (readiness +gate, responsive capture, cross-origin iframes) and Percy renders it server-side. Locator +subjects become element-scoped snapshots. An optional CI gate is available via +`reporter: [['@percy/playwright/dropin/reporter']]` with `{ "gate": "fail-on-changes" }`. + +Requires `@playwright/test` >= 1.49 (the override hooks Playwright's expect internals; on +unsupported versions it degrades to a no-op **with a loud warning** — never silently). + ## Percy on Automate ## Usage diff --git a/bin/percy-playwright.js b/bin/percy-playwright.js new file mode 100755 index 0000000..09c187a --- /dev/null +++ b/bin/percy-playwright.js @@ -0,0 +1,91 @@ +#!/usr/bin/env node +'use strict'; + +// percy-playwright — zero-config head-build tagging wrapper. +// +// The head build is tagged via the PERCY_BUILD_SOURCE env var, which @percy/cli reads when it +// CREATES the build at `percy exec` startup — in the PARENT process, before any test runs. The SDK +// (which runs inside the test process) can't set it after the fact. This thin wrapper closes that +// gap: it sets PERCY_BUILD_SOURCE for you (when unset) and then execs `percy` with your args, so +// `npx percy-playwright exec -- npx playwright test` needs no manual env var. +// +// It is intentionally minimal — it only injects the env var and delegates everything else to the +// real `percy` binary (stdio inherited, exit code forwarded). +// NOTE: reference child_process as a module object (not a destructured `spawn`) so the spawn call +// site stays stubbable from tests (destructuring would bind the reference at import time). +const childProcess = require('child_process'); +const path = require('path'); +const fs = require('fs'); + +const BUILD_SOURCE = 'playwright-dropin'; + +// Resolve the `percy` executable from the locally-installed @percy/cli when possible (the version +// this drop-in was tested against), else fall back to PATH so a globally-installed `percy` works. +function resolvePercyBin() { + try { + // @percy/cli ships an `exports` map that does NOT expose ./package.json, so we resolve its main + // entry and walk up to the package root (the dir containing package.json). + const mainEntry = require.resolve('@percy/cli'); + let dir = path.dirname(mainEntry); + while (dir !== path.dirname(dir)) { + const pkgFile = path.join(dir, 'package.json'); + if (fs.existsSync(pkgFile)) { + const pkg = JSON.parse(fs.readFileSync(pkgFile, 'utf8')); + if (pkg.name === '@percy/cli') { + const binRel = typeof pkg.bin === 'string' ? pkg.bin : pkg.bin && pkg.bin.percy; + if (binRel) return path.resolve(dir, binRel); + } + } + dir = path.dirname(dir); + } + } catch { + // @percy/cli not resolvable from here — fall back to PATH. + } + return 'percy'; +} + +function main(argv = process.argv.slice(2)) { + // Zero-config tagging: only set PERCY_BUILD_SOURCE when the user hasn't already chosen a value. + const env = { ...process.env }; + if (!env.PERCY_BUILD_SOURCE) env.PERCY_BUILD_SOURCE = BUILD_SOURCE; + // First-build-as-baseline candidate flag: rides createBuild via @percy/client; the SERVER + // decides first-ness (a no-op on established projects), so it is always safe to send. + if (!env.PERCY_DROPIN_BASELINE_CANDIDATE) env.PERCY_DROPIN_BASELINE_CANDIDATE = 'true'; + + const percyBin = resolvePercyBin(); + // When percyBin is a resolved .cjs/.js path, run it through the current node. When it's the bare + // "percy" PATH fallback, spawn it directly (shell PATH resolution). + const isScript = percyBin !== 'percy'; + const command = isScript ? process.execPath : percyBin; + const args = isScript ? [percyBin, ...argv] : argv; + + const child = childProcess.spawn(command, args, { stdio: 'inherit', env }); + + child.on('error', (err) => { + if (err && err.code === 'ENOENT') { + process.stderr.write( + 'percy-playwright: could not find the `percy` executable. Install @percy/cli ' + + '(npm i -D @percy/cli) or ensure `percy` is on your PATH.\n' + ); + } else { + process.stderr.write(`percy-playwright: failed to launch percy — ${err && err.message}\n`); + } + process.exit(1); + }); + + child.on('exit', (code, signal) => { + if (signal) { + // Re-raise the signal so the parent's exit status reflects it (CI signal handling). + process.kill(process.pid, signal); + return; + } + process.exit(code == null ? 1 : code); + }); + + return child; +} + +module.exports = { main, resolvePercyBin, BUILD_SOURCE }; + +// Run when invoked as a CLI (not when required by the unit test). +if (require.main === module) main(); diff --git a/dropin/baseline/base-branch.js b/dropin/baseline/base-branch.js new file mode 100644 index 0000000..0bba38f --- /dev/null +++ b/dropin/baseline/base-branch.js @@ -0,0 +1,23 @@ +'use strict'; + +// KD2 — the seeded baseline must live on the branch the HEAD actually resolves its base against, +// or the `latest_commit` base-selection (which matches by branch) will never pick it. +// +// percy-api's `LatestCommit` strategy walks: PR-base → target-branch → default_base_branch → head +// branch. We do NOT replicate that whole chain here (the server owns it). Instead we exploit a +// structural fact: the baseline build is created with the SAME git env as the head build, so +// percy-api derives an identical `branch` for both — they automatically share a branch and the +// `id <` ordering does the rest. This module exists to make that attribution EXPLICIT and to give +// the discover/seed path the branch value for logging + the KD2 same-branch (never default-branch) +// guard: we refuse to retarget the baseline onto the default branch (irreversible mainline +// contamination — KD2 / R-lineage-permanence). +// +// `env` is a @percy/env PercyEnv instance. +function resolveBaseBranch(env) { + // The head build's branch is what percy-api keys the baseline match on. Reusing it guarantees + // the baseline and head share a branch without us guessing the PR/target/default fallbacks. + const branch = env?.git?.branch || null; + return branch; +} + +module.exports = { resolveBaseBranch }; diff --git a/dropin/baseline/discover.js b/dropin/baseline/discover.js new file mode 100644 index 0000000..7c6c0a0 --- /dev/null +++ b/dropin/baseline/discover.js @@ -0,0 +1,256 @@ +'use strict'; + +// Unit 3 / R6 — discover the repo's committed Playwright baseline PNGs and reconstruct their Percy +// identity `(name, browser_family, width)` so they pair with the head capture into ONE comparison. +// +// FORWARD reconstruction (not backward filename parsing). Playwright's default screenshot path is +// {snapshotDir}/{testFileDir}/{testFileName}-snapshots/{arg}{-projectName}{-snapshotSuffix}{ext} +// with `snapshotSuffix` defaulting to `process.platform` (darwin|win32|linux). From the filename we +// can recover ONLY `name` (the `{arg}` stem, lossily — sanitization is irreversible). `browser_family` +// and `width` are NOT in the path: `browser_family` comes from `projectName → use.browserName` and +// `width` from `use.viewport.width`. So we enumerate the RESOLVED config's projects, derive +// `{browser_family, width}` per project, and match PNGs by peeling the known `-{projectName}` and +// `-{platform}` tail off the stem — the remainder (including any auto `-N` index) is the `name`, +// derived identically by the head capture (src/identity.js). +// +// When anything cannot be mapped cleanly we DEGRADE to baseline-only (a clear reason, never a guess) +// rather than seed a mismatched identity that would surface as a phantom "new" snapshot. +const fs = require('fs'); +const path = require('path'); +const { sanitizeForFilePath } = require('../identity'); +const { sanitizePath, sanitizeDirentName } = require('../paths'); + +// Node's process.platform values Playwright bakes into the `{snapshotSuffix}` segment. +const PLATFORM_SUFFIXES = Object.freeze(['darwin', 'linux', 'win32']); + +// Degrade reasons (surfaced by the seed/global-setup copy). +const DEGRADE = Object.freeze({ + CUSTOM_TEMPLATE: 'custom_path_template', + PROJECT_MISSING_FIELDS: 'project_missing_browser_or_viewport', + AMBIGUOUS_TAIL: 'ambiguous_project_platform_tail', + PATH_ARG: 'path_array_arg', + NO_PROJECTS: 'no_resolvable_projects' +}); + +// A custom snapshot path template means the default `{arg}{-projectName}{-snapshotSuffix}` layout no +// longer holds, so forward reconstruction can't trust the tail. Detect by PRESENCE (unset-vs-set), +// not by string-matching the default — Playwright resolves the default internally, it never appears +// on the user's config object. Checked at both the config root and per-project. +function hasCustomTemplate(config = {}, project = {}) { + const screenshotTpl = obj => obj && obj.expect && obj.expect.toHaveScreenshot && obj.expect.toHaveScreenshot.pathTemplate; + return Boolean( + config.snapshotPathTemplate || + project.snapshotPathTemplate || + screenshotTpl(config) || + screenshotTpl(project) + ); +} + +// Pull `(name, browserFamily, width)` inputs from a resolved Playwright project. The project's +// `use.browserName` → browser_family; `use.viewport.width` → width. A project missing either cannot +// be mapped (we refuse to guess a default browser/width for an EXISTING project) → degrade. +function projectIdentity(project = {}) { + const use = project.use || {}; + const browserFamily = use.browserName; + const width = use.viewport && use.viewport.width; + if (!browserFamily || !width) return null; + // The sanitized project-name segment as Playwright writes it into the path (`-{projectName}`). + const segment = project.name ? sanitizeForFilePath(project.name) : ''; + return { browserFamily, width, projectSegment: segment, projectName: project.name || '' }; +} + +// Find the directory holding committed baseline PNGs. An explicit `snapshotDir` always wins; +// otherwise probe for Playwright's `-snapshots` convention under the root. +function findSnapshotDirs(rootDir, snapshotDir) { + if (snapshotDir) { + const dir = sanitizePath(snapshotDir); + const abs = path.isAbsolute(dir) ? dir : path.join(sanitizePath(rootDir), dir); + return fs.existsSync(abs) ? [abs] : []; + } + return collectSnapshotDirs(rootDir); +} + +// Recursively collect every `*-snapshots` directory under `rootDir` (Playwright nests them next to +// each test file, which may itself sit in subdirs). +function collectSnapshotDirs(rootDir) { + const out = []; + const walk = current => { + for (const entry of safeReaddir(current, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const name = sanitizeDirentName(entry.name); + if (name === null) continue; + const full = path.join(sanitizePath(current), name); + if (/-snapshots$/.test(name)) out.push(full); + else if (name !== 'node_modules' && !name.startsWith('.')) walk(full); + } + }; + walk(rootDir); + return out; +} + +// Recursively collect every `.png` under `dir`, returning the path relative to `dir` (so nested +// `{arg}` subdir names are preserved for the path-array degrade check). +function listPngs(dir) { + const out = []; + const walk = current => { + for (const entry of safeReaddir(current, { withFileTypes: true })) { + const name = sanitizeDirentName(entry.name); + if (name === null) continue; + const full = path.join(sanitizePath(current), name); + if (entry.isDirectory()) walk(full); + else if (entry.isFile() && /\.png$/i.test(name)) out.push(path.relative(dir, full)); + } + }; + walk(dir); + return out; +} + +// Peel the known `-{platform}` then `-{projectName}` tail off a PNG stem (filename minus `.png`), +// reconstructing `name`. Returns the matched project + name, or a degrade marker. +// +// `projects` is the list of resolved project identities; `platformSuffixes` the configured/observed +// suffix set. We try every (project, platform) combination and require EXACTLY ONE to match so a +// hyphenated project/platform name can't ambiguously peel two different ways. +function reconstructName(stem, projects, platformSuffixes) { + // Path-array `{arg}` (nested subdirs) → the stem contains a path separator. We can't reliably map + // it back to a single `name` token, so degrade rather than guess. + if (stem.includes('/') || stem.includes(path.sep)) { + return { degrade: DEGRADE.PATH_ARG }; + } + + // Strip the platform suffix (single-OS CI — do NOT fork identity by OS). Prefer a real platform + // match; only when NO known platform suffix is present do we fall back to the suffix-less peel. + // This keeps an unnamed/empty-segment project from spuriously double-matching (with and without a + // platform) when the stem genuinely carries a suffix. + const withPlatform = peelMatches(stem, projects, platformSuffixes); + const matches = withPlatform.length ? withPlatform : peelMatches(stem, projects, [null]); + + if (!matches.length) return { degrade: null, unmatched: true }; + + // Two genuinely different (name, project) resolutions of the same tail → ambiguous; don't guess. + const distinct = dedupeMatches(matches); + if (distinct.length > 1) return { degrade: DEGRADE.AMBIGUOUS_TAIL }; + + return { name: distinct[0].name, project: distinct[0].project }; +} + +// Try peeling each (project, platform) combination off the stem. `platforms` may include `null` to +// mean "no platform suffix". Returns every combination that cleanly resolves to a non-empty name. +function peelMatches(stem, projects, platforms) { + const out = []; + for (const project of projects) { + for (const platform of platforms) { + let rest = stem; + if (platform) { + if (!rest.endsWith(`-${platform}`)) continue; + rest = rest.slice(0, -(platform.length + 1)); + } + if (project.projectSegment) { + if (!rest.endsWith(`-${project.projectSegment}`)) continue; + rest = rest.slice(0, -(project.projectSegment.length + 1)); + } + if (!rest) continue; // nothing left to be the name → not a real match + out.push({ name: rest, project, platform }); + } + } + return out; +} + +// Two matches are equivalent when they resolve to the same name + project. A PNG with no platform +// suffix matches every project's `null`-platform peel identically; only genuinely different +// (name, project) pairs make the tail ambiguous. +function dedupeMatches(matches) { + const seen = new Map(); + for (const m of matches) { + const key = `${m.name}${m.project.projectName}`; + if (!seen.has(key)) seen.set(key, m); + } + return [...seen.values()]; +} + +// Discover the committed baseline PNGs and reconstruct each one's Percy identity. +// +// Inputs: +// rootDir — repo root (where `*-snapshots` dirs live) +// snapshotDir — explicit Playwright `snapshotDir`, if configured +// baseBranch — passed through (KD2) +// config — the RESOLVED Playwright config: { projects: [{ name, use: { browserName, +// viewport: { width } } }], snapshotPathTemplate?, expect?: { toHaveScreenshot?: +// { pathTemplate? } } } +// +// Returns { baselines: [{ filepath, name, browserFamily, width }], snapshotDir, baseBranch, degraded?, +// reason? }. On any unmappable condition `baselines` is empty and `{ degraded: true, reason }` is +// set so the caller seeds nothing and surfaces a clear baseline-only message. +function discoverBaselines({ rootDir = process.cwd(), snapshotDir = null, baseBranch = null, config = {} } = {}) { + const projectsConfig = Array.isArray(config.projects) && config.projects.length + ? config.projects + : (config.use ? [{ name: '', use: config.use }] : []); + + // Degrade: a custom template anywhere means the default tail layout is gone — never guess. + if (projectsConfig.some(p => hasCustomTemplate(config, p)) || hasCustomTemplate(config)) { + return degraded(DEGRADE.CUSTOM_TEMPLATE, { baseBranch }); + } + + // Build the resolved project identities. ANY project missing browserName/viewport.width → degrade + // (a real configured project we can't map would otherwise be silently dropped). + const projects = []; + for (const p of projectsConfig) { + const id = projectIdentity(p); + if (!id) return degraded(DEGRADE.PROJECT_MISSING_FIELDS, { baseBranch }); + projects.push(id); + } + if (!projects.length) return degraded(DEGRADE.NO_PROJECTS, { baseBranch }); + + const dirs = findSnapshotDirs(rootDir, snapshotDir); + if (!dirs.length) { + return { baselines: [], snapshotDir: null, baseBranch }; + } + + const baselines = []; + for (const dir of dirs) { + for (const rel of listPngs(dir)) { + const stem = rel.replace(/\.png$/i, ''); + const result = reconstructName(stem, projects, PLATFORM_SUFFIXES); + + if (result.degrade) { + return degraded(result.degrade, { baseBranch }); + } + if (result.unmatched) { + // A PNG whose tail matches no configured project/platform — likely a stray file. Skip it + // (do NOT seed an identity we can't trust), but don't fail the whole discovery. + continue; + } + + baselines.push({ + filepath: path.join(dir, rel), + name: result.name, + browserFamily: result.project.browserFamily, + width: result.project.width + }); + } + } + + return { baselines, snapshotDir: dirs.length === 1 ? dirs[0] : dirs, baseBranch }; +} + +function degraded(reason, extra = {}) { + return { baselines: [], snapshotDir: null, degraded: true, reason, ...extra }; +} + +function safeReaddir(dir, opts) { + try { + return fs.readdirSync(dir, opts); + } catch { + return []; + } +} + +module.exports = { + discoverBaselines, + findSnapshotDirs, + reconstructName, + projectIdentity, + hasCustomTemplate, + PLATFORM_SUFFIXES, + DEGRADE +}; diff --git a/dropin/baseline/first-build.js b/dropin/baseline/first-build.js new file mode 100644 index 0000000..ebfca67 --- /dev/null +++ b/dropin/baseline/first-build.js @@ -0,0 +1,107 @@ +'use strict'; + +// First-build-as-baseline (flag-passing model — supersedes the two-build parallel-nonce seed). +// +// Chain: the `percy-playwright` wrapper (or the customer) sets PERCY_DROPIN_BASELINE_CANDIDATE=true +// → @percy/client sends `dropin-baseline-candidate` on createBuild → percy-api rewrites the build's +// source to 'playwright-dropin-baseline' IFF this is the project's FIRST visible build (server +// decides first-ness; the flag can never rebaseline an established project) → the CLI exposes the +// decided source through /percy/healthcheck (`percy.build.source`). +// +// When the server says "this build IS the baseline": +// • globalSetup fills the run's ONE build with the repo's COMMITTED snapshot PNGs — the baselines +// the user has already blessed (TB) — via the normal CLI comparison path (they belong to this +// very build, so no client-direct ingest, no nonce, no poll, no defer-uploads dance). +// • The toHaveScreenshot override SKIPS posting live captures (PERCY_DROPIN_SEEDED_BASELINE=1) so +// the baseline contains exactly the blessed PNGs; assertions still pass (always-pass). +// • percy-api auto-approves the build at finish (KD13: source-keyed + first-baseline bound + +// the `playwright-dropin-baseline-ingest` rollout kill-switch). Diffs start on the next run. +// +// With no committed snapshots the live captures become the first build's content instead (the +// override posts normally) — still auto-approved server-side, matching native Playwright's own +// "first run writes the baselines" behavior. +const fs = require('fs'); +const utils = require('@percy/sdk-utils'); +const { discoverBaselines } = require('./discover'); +const { pngDimensions } = require('../png'); + +const log = utils.logger('playwright-dropin'); + +const BASELINE_SOURCE = 'playwright-dropin-baseline'; + +// Parallel seed-upload cap: high enough to keep globalSetup fast on large baseline sets, low +// enough not to stampede the local CLI server's request queue. +const SEED_CONCURRENCY = 8; + +const OUTCOME = Object.freeze({ + NOT_FIRST_BUILD: 'not_first_build', + SEEDED: 'seeded', + NO_BASELINES: 'no_baselines', + UNMAPPABLE: 'unmappable' +}); + +// Runs inside globalSetup, after isPercyEnabled() has populated `utils.percy`. Returns +// { firstBuild, seeded, outcome } and never throws (a Percy problem must not abort the suite). +async function firstBuildBaseline( + { rootDir, snapshotDir, playwrightConfig, clientInfo, environmentInfo } = {}, + deps = {} +) { + const discover = deps.discoverBaselines || discoverBaselines; + const post = deps.postComparison || (options => utils.postComparison(options)); + const readFile = deps.readFile || fs.promises.readFile; + const build = deps.build !== undefined ? deps.build : (utils.percy && utils.percy.build); + + // The server decided this is NOT the project's first build (or the CLI predates the candidate + // flag and never sent it) — the run's build is a normal head; nothing to seed. + if (!build || build.source !== BASELINE_SOURCE) { + return { firstBuild: false, seeded: 0, outcome: OUTCOME.NOT_FIRST_BUILD }; + } + + const { baselines, degraded, reason } = discover({ + rootDir, snapshotDir, config: playwrightConfig + }); + + if (degraded) { + return { firstBuild: true, seeded: 0, outcome: OUTCOME.UNMAPPABLE, degradeReason: reason }; + } + if (!baselines || !baselines.length) { + return { firstBuild: true, seeded: 0, outcome: OUTCOME.NO_BASELINES }; + } + + // Bounded-concurrency ingest (TB §11 scale): a repo can carry hundreds of committed baselines + // and globalSetup blocks the suite start — post in parallel, but capped so we never stampede + // the local CLI server. Per-file failures are skipped (partial baseline beats none). + const concurrency = deps.concurrency || SEED_CONCURRENCY; + let seeded = 0; + const queue = [...baselines]; + const worker = async () => { + for (let b = queue.shift(); b; b = queue.shift()) { + try { + // The buffer is both the tile content and the source of truth for tag height (percy-api + // validates height presence on screenshot records — see src/png.js). + const buf = await readFile(b.filepath); + const dims = pngDimensions(buf); + await post({ + name: b.name, + clientInfo, + environmentInfo, + tag: { + name: b.browserFamily, + browserName: b.browserFamily, + width: b.width || (dims && dims.width) || undefined, + height: (dims && dims.height) || undefined + }, + tiles: [{ content: buf.toString('base64') }] + }); + seeded += 1; + } catch (err) { + log.debug(`Percy: skipped committed baseline "${b.name}" — ${err.message}`); + } + } + }; + await Promise.all(Array.from({ length: Math.min(concurrency, baselines.length) }, worker)); + + return { firstBuild: true, seeded, outcome: seeded > 0 ? OUTCOME.SEEDED : OUTCOME.NO_BASELINES }; +} + +module.exports = { firstBuildBaseline, BASELINE_SOURCE, OUTCOME }; diff --git a/dropin/capture.js b/dropin/capture.js new file mode 100644 index 0000000..2bdf4c0 --- /dev/null +++ b/dropin/capture.js @@ -0,0 +1,20 @@ +'use strict'; + +// KD4 (spike-decided): full-override capture — own the screenshot, return a pure PNG buffer with +// NO assertion side-effects. Capture options mirror Playwright's toHaveScreenshot defaults so the +// bytes match how repo baselines were generated (D2 — keeps first-build noise low). +const PASS_THROUGH_OPTS = ['clip', 'fullPage', 'mask', 'maskColor', 'omitBackground', 'scale', 'animations', 'caret', 'style', 'stylePath']; + +async function captureFullOverride(pageOrLocator, options = {}) { + const target = pageOrLocator && typeof pageOrLocator.screenshot === 'function' + ? pageOrLocator // Page or Locator both expose screenshot() + : pageOrLocator.page(); + + const shotOpts = { animations: 'disabled', caret: 'hide', scale: 'css' }; + for (const k of PASS_THROUGH_OPTS) { + if (options && options[k] !== undefined) shotOpts[k] = options[k]; + } + return target.screenshot(shotOpts); +} + +module.exports = { captureFullOverride }; diff --git a/dropin/config.js b/dropin/config.js new file mode 100644 index 0000000..1ef567e --- /dev/null +++ b/dropin/config.js @@ -0,0 +1,201 @@ +'use strict'; + +// Unit 7 — optional in-package drop-in config (D8) + the CENTRAL throw-policy / footgun-rejection +// point (KD4/KD14). +// +// "One config line" stays the playwright.config registration; THIS file is an optional escape +// hatch. Zero-config works WITHOUT a file — the file only overrides defaults. +// +// Fields (drop-in only — NOT sync): +// • captureMode: 'screenshot' (default — BYOS raw-PNG upload, generic/app projects) or +// 'snapshot' (V2 — serialized-DOM web snapshot, Percy renders server-side, WEB projects). +// • gate: 'informational' (default) | 'fail-on-changes' (Unit 5) +// • compat: boolean — preserve native throw semantics (Unit 6 / D6) +// • fallback: boolean (default true) — native fallback when Percy is disabled at run start (D7) +// • alwaysPass: boolean (default true) — the D6 async always-pass posture +// • passIfApproved: boolean — gate carve-out +// +// SYNC IS NOT A DROP-IN FIELD. It is read from the GLOBAL `.percy.yml snapshot.sync` via the +// healthcheck `percy.config` (utils.percy.config.snapshot.sync), populated by isPercyEnabled(). +// +// THREE-WAY MUTUAL EXCLUSION (resolved here): sync ⊕ always-pass ⊕ compat. Each changes the throw +// decision differently, so at most ONE may be active. validateConfig() rejects any pair. +const fs = require('fs'); +const path = require('path'); +const utils = require('@percy/sdk-utils'); +const { preflightTokenScope } = require('./sync'); +const { sanitizePath } = require('./paths'); + +const log = utils.logger('playwright-dropin'); + +const CONFIG_FILENAMES = ['.percy-playwright-dropin.js', '.percy-playwright-dropin.json', 'percy-playwright-dropin.config.js']; +const VALID_CAPTURE_MODES = Object.freeze(['screenshot', 'snapshot']); + +const DEFAULTS = Object.freeze({ + captureMode: 'screenshot', + gate: 'informational', + compat: false, + fallback: true, + alwaysPass: true, + passIfApproved: false +}); + +// Cache the loaded config for the process (re-read only via _reset in tests). +let _cache = null; + +// Read the in-package config file from `rootDir` if present. Returns {} when absent. +function readConfigFile(rootDir) { + const base = sanitizePath(rootDir); + for (const name of CONFIG_FILENAMES) { + const file = path.join(base, name); + if (!fs.existsSync(file)) continue; + try { + if (file.endsWith('.json')) return JSON.parse(fs.readFileSync(file, 'utf8')); + return require(file); + } catch (err) { + throw new Error(`Percy drop-in: failed to load config file ${name} — ${err.message}`); + } + } + return {}; +} + +// Read the GLOBAL sync flag from the healthcheck percy.config (populated by isPercyEnabled()). +function readSyncFromHealthcheck() { + return Boolean(utils.percy && utils.percy.config && utils.percy.config.snapshot && utils.percy.config.snapshot.sync); +} + +// Detect a deferred/skip/delay upload setting in the global percy.config — sync silently no-ops +// under any of these (percy.js syncMode()), so we must REJECT the combination (R-sync-silentdisable). +function deferredUploadSet() { + const p = (utils.percy && utils.percy.config && utils.percy.config.percy) || {}; + return Boolean(p.deferUploads || p.delayUploads || p.skipUploads); +} + +// Merge file overrides onto defaults + the (healthcheck-sourced) sync flag. SYNCHRONOUS — callable +// from inside the matcher (isPercyEnabled() has already run by then, so percy.config is cached). +// Footgun validation is async (token preflight) and lives in validateConfig(). +function loadConfig({ rootDir = process.cwd(), force = false } = {}) { + if (_cache && !force) return _cache; + + const file = readConfigFile(rootDir); + const merged = { ...DEFAULTS }; + const explicit = {}; + for (const key of Object.keys(DEFAULTS)) { + if (file[key] !== undefined) { merged[key] = file[key]; explicit[key] = true; } + } + // Track which throw-mode fields the user set explicitly (vs the default). Used by validateConfig + // to distinguish a deliberate conflict from the implicit always-pass default. + merged._explicit = explicit; + + // captureMode validation: 'screenshot' (raw PNG / generic+app projects) or 'snapshot' + // (serialized-DOM web snapshot / web projects). + if (!VALID_CAPTURE_MODES.includes(merged.captureMode)) { + throw new Error( + `Percy drop-in: captureMode "${merged.captureMode}" is not supported — ` + + `use ${VALID_CAPTURE_MODES.map(m => `"${m}"`).join(' or ')}.` + ); + } + + // sync comes from the global .percy.yml (never the file). When sync is on, the always-pass posture + // is implicitly off (sync owns the throw decision) — but we DON'T silently flip the user's + // explicit always-pass; validateConfig() rejects the conflicting pair instead. + merged.sync = readSyncFromHealthcheck(); + + _cache = merged; + return _cache; +} + +// Async footgun validation + pre-flight checks. Call once at run start (index.js resolveRunMode / +// globalSetup). Throws a clear error on a rejected combination so the user fixes their config rather +// than silently getting the wrong behaviour. Returns the validated config. +async function validateConfig(config = loadConfig(), { token = process.env.PERCY_TOKEN, probe } = {}) { + // Determine which throw-modes are active. always-pass is the DEFAULT posture; sync (global) and + // compat (file) are deliberate overrides that implicitly supersede the default always-pass. + // A conflict is rejected when MORE THAN ONE mode is deliberately chosen — i.e. sync+compat, or an + // EXPLICIT alwaysPass:true alongside sync/compat. The implicit default never conflicts (otherwise + // turning on snapshot.sync would always require also editing the drop-in config). + const explicit = (config._explicit) || {}; + const deliberate = []; + if (config.sync) deliberate.push('sync (.percy.yml snapshot.sync)'); + if (config.compat) deliberate.push('compat-mode'); + if (config.alwaysPass && explicit.alwaysPass) deliberate.push('always-pass (explicit)'); + + // THREE-WAY MUTUAL EXCLUSION: at most one throw-mode may be deliberately active. + if (deliberate.length > 1) { + throw new Error( + `Percy drop-in: ${deliberate.join(' + ')} are mutually exclusive — pick one. ` + + 'always-pass (default), compat-mode, and sync (.percy.yml snapshot.sync) each define a ' + + 'different throw policy. To use sync or compat, do not also set alwaysPass:true.' + ); + } + + // Normalise: a deliberate sync/compat override turns off the implicit always-pass so downstream + // (index.js) doesn't run both the sync/compat path AND the always-pass return. + if ((config.sync || config.compat) && !explicit.alwaysPass) config.alwaysPass = false; + + if (config.sync) { + // sync + any deferred/skip/delay upload → rejected (else syncMode() silently no-ops sync). + if (deferredUploadSet()) { + throw new Error( + 'Percy drop-in: sync mode is incompatible with deferred/skip/delayed uploads — the CLI ' + + 'silently disables sync under those settings. Remove deferUploads/delayUploads/skipUploads ' + + 'or remove snapshot.sync from .percy.yml.' + ); + } + + // Pre-flight token-scope check: refuse to enable sync on a write-only token (don't wait for the + // first 403, which would bucket every assertion as no-verdict AND break the Gate-A backstop). + const scope = await preflightTokenScope({ token, probe }); + if (!scope.ok) { + throw new Error( + 'Percy: sync mode is disabled — it needs a read-capable token, but the configured token ' + + `is write-only (${scope.reason}). Use a full/read token, or remove snapshot.sync. ` + + 'See https://percy.io/docs for token scopes.' + ); + } + + // Blast-radius warning (a full token leak is org-wide — plan §User-Facing States). + log.warn('Percy: sync mode is using a full-access token. If it leaks from CI it grants ' + + 'org-wide read/approve/delete across all projects. Prefer a dedicated read-only service ' + + 'account; never log it.'); + } + + return config; +} + +// Runtime assertion that sync actually engaged (don't silently degrade). Call after the first +// comparison post in sync mode: if the CLI disabled sync (deferred uploads slipped in at runtime), +// surface it loudly rather than producing false-greens. +function assertSyncEngaged(config = loadConfig()) { + if (!config.sync) return true; + if (deferredUploadSet()) { + log.error('Percy: sync mode did NOT engage — a deferred/skip/delayed upload setting disabled ' + + 'it at runtime. Inline verdicts are unavailable; rely on the post-run gate.'); + return false; + } + return true; +} + +// Status line (plan §User-Facing States "Mode status line"). +function modeStatusLine(config = loadConfig()) { + let mode = 'async-always-pass'; + if (config.sync) mode = 'sync'; + else if (config.compat) mode = 'compat'; + const gate = config.sync ? 'fail-on-changes' : config.gate; + return `Percy drop-in: mode=${mode} | capture=${config.captureMode} | gate=${gate}`; +} + +function _reset() { _cache = null; } + +module.exports = { + loadConfig, + validateConfig, + assertSyncEngaged, + modeStatusLine, + readConfigFile, + deferredUploadSet, + DEFAULTS, + VALID_CAPTURE_MODES, + CONFIG_FILENAMES, + _reset +}; diff --git a/dropin/dom.js b/dropin/dom.js new file mode 100644 index 0000000..38d26d6 --- /dev/null +++ b/dropin/dom.js @@ -0,0 +1,95 @@ +'use strict'; + +// Snapshot/DOM capture seam for the toHaveScreenshot drop-in (`captureMode: 'snapshot'`). +// +// Unlike the standalone drop-in package, THIS port delegates the heavy lifting to this repo's own +// `captureDOM` (index.js) — the exact capture `percySnapshot()` uses — so the drop-in inherits the +// readiness gate, responsive DOM capture, and cross-origin iframe serialization for free and the +// two entry points can never drift apart. +// +// What this seam adds on top: +// • Locator subjects → SCOPED snapshots: the element is marked with a data attribute that +// survives serialization and the snapshot is posted with `scope`, so Percy's server-side +// render clips to the element (you cannot screenshot "part of a DOM" any other way). +// • toHaveScreenshot-only options (clip/mask/animations/…) are surfaced at debug as ignored — +// they apply to raw pixels, not a server-side render. +// +// SEMANTICS (vs screenshot mode): +// • width: the test's viewport width is sent as `widths: [width]` so Percy renders at the same +// width the assertion ran at — keeping snapshot identity aligned with the committed-baseline +// naming. Percy's project-level width config does not multiply drop-in snapshots. +// • browser: Percy renders web snapshots in ITS OWN browsers (project settings), not the +// Playwright browser the test ran in — `browser_family` identity is server-controlled here. +const utils = require('@percy/sdk-utils'); + +const log = utils.logger('playwright-dropin'); + +// Lazy-required at capture time: the root module may itself be mid-load when the drop-in entry is +// evaluated (specs import both as ESM), and a top-level require here trips Node's CJS↔ESM +// interop on a partially-initialized module. +function rootCaptureDOM(...args) { + return require('../index.js').captureDOM(...args); +} + +const SCOPE_ATTR = 'data-percy-dropin-scope'; + +// toHaveScreenshot options that only make sense for the raw-pixel screenshot flow. +const SCREENSHOT_ONLY_OPTS = Object.freeze([ + 'clip', 'fullPage', 'mask', 'maskColor', 'omitBackground', 'scale', 'animations', 'caret', 'style', 'stylePath' +]); + +// A Locator exposes `.page()`; a Page does not. +function resolvePageAndLocator(pageOrLocator) { + const isLocator = pageOrLocator && typeof pageOrLocator.page === 'function'; + return isLocator + ? { page: pageOrLocator.page(), locator: pageOrLocator } + : { page: pageOrLocator, locator: null }; +} + +// Capture the serialized DOM for a Page or Locator subject. Returns +// { domSnapshot, url, scope, viewport } +// — everything dropin/index.js needs to build the postSnapshot options. Throws on capture failure; +// the caller's never-fail-the-suite catch owns the policy. `deps` is injectable for tests. +async function captureDomSnapshot(pageOrLocator, options = {}, deps = {}) { + const fetchPercyDOM = deps.fetchPercyDOM || (() => utils.fetchPercyDOM()); + const capture = deps.captureDOM || rootCaptureDOM; + + const { page, locator } = resolvePageAndLocator(pageOrLocator); + + const ignored = SCREENSHOT_ONLY_OPTS.filter(k => options && options[k] !== undefined); + if (ignored.length) { + log.debug(`Percy: snapshot mode ignores screenshot-only option(s): ${ignored.join(', ')}`); + } + + // Inject the DOM serialization script, exactly as percySnapshot() does. + const percyDOM = await fetchPercyDOM(); + await page.evaluate(percyDOM); + + // Locator subject → mark the element so the server-side render can be scoped to it. The marker + // attribute survives serialization (no fragile CSS-selector reconstruction) and is removed right + // after capture so the live page is left untouched. + let scope = null; + if (locator) { + // istanbul ignore next - browser-executed function (instrumentation counters don't exist there) + await locator.evaluate((el, attr) => el.setAttribute(attr, ''), SCOPE_ATTR); + scope = `[${SCOPE_ATTR}]`; + } + + let domSnapshot; + try { + // Reuse the repo's full capture: readiness gate, responsive capture, CORS iframes, cookies. + // toHaveScreenshot options are NOT forwarded — they are pixel-flow options (logged above); + // Percy-level snapshot options are not part of the toHaveScreenshot signature. + domSnapshot = await capture(page, {}, percyDOM); + } finally { + if (locator) { + // istanbul ignore next - browser-executed function (instrumentation counters don't exist there) + await locator.evaluate((el, attr) => el.removeAttribute(attr), SCOPE_ATTR).catch(() => {}); + } + } + + const viewport = (page && typeof page.viewportSize === 'function' && page.viewportSize()) || null; + return { domSnapshot, url: page.url(), scope, viewport }; +} + +module.exports = { captureDomSnapshot, resolvePageAndLocator, SCOPE_ATTR, SCREENSHOT_ONLY_OPTS }; diff --git a/dropin/fallback.js b/dropin/fallback.js new file mode 100644 index 0000000..f96ad83 --- /dev/null +++ b/dropin/fallback.js @@ -0,0 +1,175 @@ +'use strict'; + +// Unit 6 — Native fallback (D7) + compat-mode native throw (D6/KD5). +// +// Two distinct jobs, both centred on Playwright's ORIGINAL `toHaveScreenshot`: +// +// 1. Native fallback (D7/KD6): if Percy is NOT enabled at the START of the run (no token, +// CLI down, healthcheck fail), the WHOLE run routes through the native matcher so the suite +// behaves EXACTLY as it did pre-install (pixel diff against committed baselines, native +// throw). This is a run-level decision, latched once — never partial-native inside a live +// Percy run (mid-run blips are retried/queued instead, see retryablePost). +// +// 2. Compat mode (D6/KD5): the user opts in to keep native THROW semantics even with Percy on, +// but we SUPPRESS the missing-baseline first-run throw so installing the drop-in (which means +// a repo may have no committed baseline yet) never reds a first run just because no baseline +// exists. +// +// To invoke the native matcher we MUST hold the original `toHaveScreenshot` captured BEFORE +// `baseExpect.extend({ toHaveScreenshot })` replaced it (index.js does the capture at module load +// and hands it here). Playwright keeps the default matchers on a prototype in the matcher object's +// chain; `captureNativeMatcher` walks that chain and grabs the slot's value. +const utils = require('@percy/sdk-utils'); + +const log = utils.logger('playwright-dropin'); + +// Playwright signals "no committed baseline yet" with a matcher error whose message mentions a +// missing snapshot / "writing actual". We match conservatively so a genuine pixel diff still throws. +const MISSING_BASELINE_RE = /(snapshot|screenshot).*(doesn't exist|does not exist|is missing|not found)|writing actual|to update snapshots/i; + +// Capture the native `toHaveScreenshot` from the live matcher prototype chain. MUST be called +// before `baseExpect.extend` overrides the slot. Returns null if it can't be found (older/newer +// Playwright) — callers then degrade gracefully (treat as no-native-available). +function captureNativeMatcher(baseExpect) { + try { + let obj = baseExpect(undefined); + while (obj) { + if (Object.prototype.hasOwnProperty.call(obj, 'toHaveScreenshot')) { + const desc = Object.getOwnPropertyDescriptor(obj, 'toHaveScreenshot'); + return (desc && typeof desc.value === 'function') ? desc.value : null; + } + obj = Object.getPrototypeOf(obj); + } + } catch (err) { + log.debug(`Percy: could not capture native toHaveScreenshot — ${err.message}`); + } + return null; +} + +// Detect whether a thrown native error / failing matcher result is the "no committed baseline" +// first-run case (which compat mode must NOT surface as a failure). +function isMissingBaselineFailure(errOrResult) { + if (!errOrResult) return false; + let msg = errOrResult.message; + // Matcher results carry message as a FUNCTION — call it (String(fn) would test the source code). + if (typeof msg === 'function') { try { msg = msg(); } catch { msg = ''; } } + return Boolean(msg && MISSING_BASELINE_RE.test(String(msg))); +} + +// Invoke the captured native matcher with the same `this` (matcher state) and args Playwright would +// have used. Playwright's matchers may either THROW (hard assertions) or RETURN `{ pass:false }` +// (soft path) on a diff; we normalise both. When `suppressMissingBaseline` is set (compat mode), +// a missing-baseline outcome is converted to a PASS so a first run never reds on an absent baseline. +async function runNativeMatcher(nativeMatcher, matcherState, args, { suppressMissingBaseline = false } = {}) { + if (typeof nativeMatcher !== 'function') { + // No native matcher available → we cannot do a native compare. Pass so we never fail worse + // than pre-install would on an unsupported Playwright (D3 spirit). + return { pass: true, message: () => 'Percy: native screenshot matcher unavailable — skipped' }; + } + + try { + const result = await nativeMatcher.apply(matcherState, args); + if (result && result.pass === false && suppressMissingBaseline && isMissingBaselineFailure(result)) { + return { pass: true, message: () => 'Percy: first run — no committed baseline yet (compat-mode suppressed)' }; + } + return result; + } catch (err) { + if (suppressMissingBaseline && isMissingBaselineFailure(err)) { + return { pass: true, message: () => 'Percy: first run — no committed baseline yet (compat-mode suppressed)' }; + } + throw err; + } +} + +// One-time native-fallback notice (plan §User-Facing States): printed once on entering native so a +// green CI isn't mistaken for "Percy passed". +let noticeShown = false; +function noteNativeFallback(reason) { + if (noticeShown) return; + noticeShown = true; + log.warn(`Percy unavailable (${reason}) — running native screenshot comparison; no Percy build created`); +} + +// Reset hook for tests (the one-time notice latch). +function _resetNotice() { noticeShown = false; } + +// Mid-run upload resilience (D7/KD6): a transient post failure inside a LIVE Percy run must NOT +// drop to native (that would mix native + Percy in one build). Instead retry the post a few times +// with backoff; if it still fails, swallow (D3 — never fail the suite on a Percy error) at +// debug-level (plan: mid-run blip is debug-only, no user-facing alarm). +async function retryablePost(postFn, { retries = 3, backoff = 200, sleep = ms => new Promise(r => setTimeout(r, ms)) } = {}) { + let lastErr; + for (let attempt = 0; attempt <= retries; attempt++) { + try { + return await postFn(); + } catch (err) { + lastErr = err; + log.debug(`Percy: upload blip (attempt ${attempt + 1}/${retries + 1}) — ${err.message}`); + if (attempt < retries) await sleep(backoff * (attempt + 1)); + } + } + // Exhausted retries: swallow per D3 — the run stays green; the post-matrix gate (Unit 5) is the + // backstop for anything that genuinely didn't land. + log.debug(`Percy: upload failed after retries — ${lastErr && lastErr.message}`); + return undefined; +} + +// With --update-snapshots=missing (Playwright's default), the built-in records the "snapshot +// doesn't exist, writing actual" failure as a SOFT ERROR: `handleMissing` returns `pass: true` plus +// a `softError` the step machinery feeds to `testInfo._failWithError` — pushing to testInfo.errors +// AND flipping testInfo.status to 'failed', entirely OUTSIDE the matcher's return value. Converting +// our result can't suppress it, so we remove the recorded error and restore status — but only when +// ours was the only failure on the test. +function scrubMissingBaselineSoftError() { + try { + const testInfo = require('@playwright/test').test.info(); + const errors = testInfo && testInfo.errors; + if (!Array.isArray(errors)) return; + let removed = false; + for (let i = errors.length - 1; i >= 0; i--) { + if (isMissingBaselineFailure(errors[i])) { errors.splice(i, 1); removed = true; } + } + if (removed && errors.length === 0 && testInfo.status === 'failed') { + testInfo.status = testInfo.expectedStatus; + } + } catch { /* no live test — nothing to scrub */ } +} + +// Delegate to the BUILT-IN toHaveScreenshot through a pristine expect chain (snapshotted in +// index.js BEFORE the override was injected). Playwright handles subject binding, matcher state and +// step reporting; a failing native compare THROWS out of the chain, so missing-baseline detection +// sees a real Error message (not a message-function). +async function runNativeViaExpect(nativeExpect, matcherState, args, { suppressMissingBaseline = false } = {}) { + if (typeof nativeExpect !== 'function') { + // No pristine chain available → we cannot do a native compare. Pass so we never fail worse + // than pre-install would on an unsupported Playwright (D3 spirit). + return { pass: true, message: () => 'Percy: native screenshot matcher unavailable — skipped' }; + } + const [subject, ...rest] = args; + while (rest.length && rest[rest.length - 1] === undefined) rest.pop(); + try { + let chain = nativeExpect(subject); + if (matcherState && matcherState.isNot) chain = chain.not; + await chain.toHaveScreenshot(...rest); + // A first run with no committed baseline "passes" the chain but soft-fails the test (see above). + if (suppressMissingBaseline) scrubMissingBaselineSoftError(); + return { pass: !(matcherState && matcherState.isNot), message: () => '' }; + } catch (err) { + if (suppressMissingBaseline && isMissingBaselineFailure(err)) { + return { pass: true, message: () => 'Percy: first run — no committed baseline yet (suppressed)' }; + } + throw err; + } +} + +module.exports = { + captureNativeMatcher, + runNativeMatcher, + runNativeViaExpect, + scrubMissingBaselineSoftError, + isMissingBaselineFailure, + noteNativeFallback, + retryablePost, + MISSING_BASELINE_RE, + _resetNotice +}; diff --git a/dropin/global-setup.js b/dropin/global-setup.js new file mode 100644 index 0000000..7d78217 --- /dev/null +++ b/dropin/global-setup.js @@ -0,0 +1,140 @@ +'use strict'; + +// Unit 4b — first-build baseline, wired as a Playwright `globalSetup`. +// +// Flag-passing model (supersedes the two-build parallel-nonce seed): the run's ONE build was +// already created by `percy exec` with the `dropin-baseline-candidate` flag (via the +// `percy-playwright` wrapper's PERCY_DROPIN_BASELINE_CANDIDATE env, mirroring PERCY_BUILD_SOURCE). +// percy-api decided first-ness at create time; the CLI healthcheck exposes the decision as +// `percy.build.source`. When this build IS the baseline, globalSetup fills it with the repo's +// committed snapshot PNGs (the baselines the user already blessed — TB) and the override skips +// live captures; percy-api auto-approves the build at finish (KD13). +// +// One-line wiring (consumer's playwright.config.js): +// const { baselineGlobalSetup } = require('@percy/playwright-dropin'); +// module.exports = defineConfig({ globalSetup: require.resolve('@percy/playwright-dropin/global-setup'), ... }); +// or, if the consumer already has a globalSetup, call `baselineGlobalSetup()` from inside it. +const utils = require('@percy/sdk-utils'); +const { firstBuildBaseline, OUTCOME } = require('./baseline/first-build'); +const { loadConfig } = require('./config'); + +const { CLIENT_INFO, ENV_INFO } = require('./version-info'); +const log = utils.logger('playwright-dropin'); + +// D9 (Unit 8 drop-in part) — when a run SEEDS the project's first baseline, the acting user is +// establishing what "correct" looks like; warn them explicitly (CLI stderr once). The percy-api +// telemetry/flag side is handled separately — this is the drop-in console warning only. +function warnBaselineSeeding(result) { + if (!result || !result.firstBuild) return; + const who = process.env.PERCY_GIT_AUTHOR || process.env.GIT_AUTHOR_NAME || process.env.USER || 'you'; + log.warn('Percy: this run establishes the project\'s first baseline — ' + + `${who} is establishing the baseline these snapshots will be reviewed against. ` + + 'Subsequent runs diff against it.'); +} + +// User-facing first-build copy (plan §User-Facing States). Distinguish outcomes. +function reportOutcome(result) { + if (!result || !result.firstBuild) { + log.info('Percy: using your project\'s existing baseline — this build diffs against it as usual'); + return; + } + switch (result.outcome) { + case OUTCOME.SEEDED: + log.info(`Percy: first build — seeded ${result.seeded} committed snapshot(s) as the baseline ` + + '(auto-approved; diffs start on your next run)'); + break; + case OUTCOME.UNMAPPABLE: + log.info('Percy: your Playwright snapshot naming could not be mapped automatically — ' + + 'this run\'s captures become the baseline; diffs start on your next run'); + break; + case OUTCOME.NO_BASELINES: + default: + log.info('Percy: first build — no committed snapshots found; this run\'s captures become ' + + 'the baseline (auto-approved; diffs start on your next run)'); + } +} + +// Normalize a Playwright FullConfig (as passed to globalSetup) into the minimal shape discover.js +// reads. FullConfig exposes `.projects` (each a FullProject with `.name`, `.use`, and the resolved +// `.snapshotPathTemplate`) and may carry top-level template fields. We pass through only what +// forward identity reconstruction needs; missing fields make discovery degrade, not crash. +function normalizePlaywrightConfig(config) { + if (!config || typeof config !== 'object') return {}; + const projects = Array.isArray(config.projects) + ? config.projects.map(p => ({ + name: p.name || '', + use: p.use || {}, + snapshotPathTemplate: p.snapshotPathTemplate, + expect: p.expect + })) + : []; + return { + projects, + use: config.use, + snapshotPathTemplate: config.snapshotPathTemplate, + expect: config.expect + }; +} + +// The globalSetup entry. Never throws — a Percy seed failure must not block the test run (D3). +async function baselineGlobalSetup(config) { + try { + if (!(await utils.isPercyEnabled())) { + log.debug('Percy is disabled — skipping first-build baseline seed'); + return; + } + + // The committed-baseline seed uploads Playwright's PNGs through the raw-image (screenshot) + // ingest — meaningless for a web project, whose baselines are server-side renders. In snapshot + // mode the build diffs against the project's existing web baseline as usual. + if (loadConfig().captureMode === 'snapshot') { + log.info('Percy: the committed-baseline seed is screenshot-mode only — skipped ' + + '(captureMode: snapshot; this build diffs against your project\'s existing baseline as usual)'); + return { firstBuild: false, seeded: 0, outcome: 'snapshot-mode' }; + } + + // Map Playwright config → discover inputs. `config.rootDir`/`configFile` are best-effort: the + // discover module falls back to conventional `*-snapshots` locations when they're absent. + const rootDir = (config && (config.rootDir || (config.configFile && require('path').dirname(config.configFile)))) || process.cwd(); + + // Normalize the resolved Playwright FullConfig into the shape discover expects for forward + // identity reconstruction (Unit 3 / R6): per-project { name, use, snapshotPathTemplate, expect } + // plus the top-level template fields. Discovery degrades to live-capture baseline on anything + // it can't map (custom template, missing browserName/viewport, ambiguous tail, path-array {arg}). + const playwrightConfig = normalizePlaywrightConfig(config); + + const result = await firstBuildBaseline({ + rootDir, + playwrightConfig, + clientInfo: CLIENT_INFO, + environmentInfo: ENV_INFO + }); + + if (result.firstBuild) { + // KD7: this run IS build #1 — review-only for the gate/sync classifier. Workers read these + // via the env (globalSetup runs in the main process; workers fork after it). + process.env.PERCY_DROPIN_FIRST_BUILD = '1'; + + // Committed snapshots were seeded as this build's content — the override must NOT post live + // captures on top (the baseline is exactly the blessed PNGs). Without a seed, live captures + // become the baseline instead and the override posts normally. + if (result.outcome === OUTCOME.SEEDED) { + process.env.PERCY_DROPIN_SEEDED_BASELINE = '1'; + } + } + + warnBaselineSeeding(result); + reportOutcome(result); + return result; + } catch (err) { + // Belt-and-suspenders: firstBuildBaseline already swallows its own errors, but globalSetup must + // be bulletproof — a throw here would abort the whole suite. + log.debug(`Percy: first-build baseline seed skipped — ${err.message}`); + } +} + +module.exports = baselineGlobalSetup; +module.exports.baselineGlobalSetup = baselineGlobalSetup; +module.exports.reportOutcome = reportOutcome; +module.exports.warnBaselineSeeding = warnBaselineSeeding; +module.exports.CLIENT_INFO = CLIENT_INFO; diff --git a/dropin/identity.js b/dropin/identity.js new file mode 100644 index 0000000..895277d --- /dev/null +++ b/dropin/identity.js @@ -0,0 +1,112 @@ +'use strict'; + +// Unit 3 / R6 — the SINGLE source of snapshot-identity truth, shared by the head-capture override +// (index.js) and the committed-baseline discovery (baseline/discover.js). +// +// Percy pairs a comparison by its identity tuple `(name, browser_family, width)`. For a committed +// Playwright baseline PNG and its head capture to land on the SAME comparison (a real diff, not two +// "new" snapshots), both sides MUST derive that tuple the same way. +// +// `browser_family` and `width` are NOT recoverable from a PNG filename — `browser_family` comes from +// `projectName → use.browserName` and `width` from `use.viewport.width` (config-only; the width +// never appears in the path). So discovery reconstructs them FORWARD from the resolved config and we +// only ever derive `name` from a Playwright artifact. The `name` derivation below mirrors +// Playwright's own snapshot-path logic (workerProcessEntry `_resolveSnapshotPaths`) byte-for-byte so +// the capture-time name equals the on-disk `{arg}` stem. + +// Playwright's filename sanitizer (playwright-core `sanitizeForFilePath`): collapse every run of +// "special" chars into a single `-`. Letters, digits, `-` and `_` survive. Kept in lock-step with +// Playwright; if upstream changes this regex, baseline↔head pairing would silently break. +// eslint-disable-next-line no-control-regex +const SANITIZE_RE = /[\x00-\x2C\x2E-\x2F\x3A-\x40\x5B-\x60\x7B-\x7F]+/g; + +function sanitizeForFilePath(s) { + return String(s).replace(SANITIZE_RE, '-'); +} + +// Per-test anonymous/named snapshot counters, mirroring Playwright's `lastAnonymousSnapshotIndex` / +// `lastNamedSnapshotIndex`. Playwright increments these per test; we key by the live testInfo object +// so repeated `toHaveScreenshot()` calls in one test get `-1`, `-2`, … exactly as the committed +// filenames would. A WeakMap lets finished tests be GC'd. When no testInfo is available (shouldn't +// happen inside a Playwright test) we fall back to a per-process counter map keyed by stem. +const COUNTERS = new WeakMap(); +const FALLBACK_COUNTERS = { anon: 0, named: new Map() }; + +function countersFor(testInfo) { + if (!testInfo) return FALLBACK_COUNTERS; + let c = COUNTERS.get(testInfo); + if (!c) { + c = { anon: 0, named: new Map() }; + COUNTERS.set(testInfo, c); + } + return c; +} + +// Build the sanitized stem Playwright would write for an ANONYMOUS (no name arg) screenshot: +// sanitize([...titlePath.slice(1), index].join(' ')) +// where `index` is the 1-based per-test anonymous counter (1 produces no numeric suffix only in the +// sense that the stem is `… 1`; Playwright always appends the index for anonymous snapshots). +function anonymousStem(titlePath, index) { + const parts = (Array.isArray(titlePath) ? titlePath.slice(1) : []).filter(Boolean); + return sanitizeForFilePath([...parts, index].join(' ')); +} + +// Build the sanitized stem for a NAMED screenshot. Playwright sanitizes the name (sans `.png`) and, +// for the 2nd+ call with the SAME name in one test, appends `-{index-1}` (so `shot`, `shot-1`, …). +// A path-array name (nested subdirs) is joined with `/` then sanitized per-segment by Playwright; +// we join with `/` and sanitize the whole thing, which collapses `/` to `-` — see `degrade` note in +// discover.js (a path-array baseline is treated as unmappable there, so capture-side parity for the +// array case is best-effort only). +function namedStem(rawName, testInfo) { + const joined = Array.isArray(rawName) ? rawName.join('/') : String(rawName); + const withoutExt = joined.replace(/\.png$/i, ''); + const sanitized = sanitizeForFilePath(withoutExt); + + const counters = countersFor(testInfo); + const index = (counters.named.get(sanitized) || 0) + 1; + counters.named.set(sanitized, index); + return index > 1 ? `${sanitized}-${index - 1}` : sanitized; +} + +// Reconstruct the snapshot `name` for a head capture exactly as Playwright would name the committed +// baseline file. `nameArg` is the explicit `toHaveScreenshot(name)` argument (string or path-array) +// or undefined for an anonymous call. +function deriveName(nameArg, testInfo) { + if (nameArg !== undefined && nameArg !== null && nameArg !== '') { + return namedStem(nameArg, testInfo); + } + const titlePath = testInfo && Array.isArray(testInfo.titlePath) ? testInfo.titlePath : []; + const counters = countersFor(testInfo); + counters.anon += 1; + return anonymousStem(titlePath, counters.anon); +} + +// D2 — map a toHaveScreenshot call onto Percy's (name, browser_family, width). +// - name: the sanitized stem Playwright would write on disk (so it pairs with the committed PNG). +// - browserFamily: the Playwright project name (chromium/firefox/webkit) — the comparison-tag identity. +// - width: the page viewport width. +function deriveIdentity(pageOrLocator, nameArg, testInfo) { + const page = pageOrLocator && typeof pageOrLocator.page === 'function' + ? pageOrLocator.page() + : pageOrLocator; + const viewport = (page && typeof page.viewportSize === 'function' && page.viewportSize()) || { width: 1280 }; + + const name = deriveName(nameArg, testInfo); + const browserFamily = (testInfo && testInfo.project && testInfo.project.name) || 'chromium'; + return { name, browserFamily, width: viewport.width }; +} + +// Test-only: reset the fallback counters (the per-testInfo WeakMap clears itself as tests are GC'd, +// but the process-wide fallback map persists across in-process unit cases). +function _resetCounters() { + FALLBACK_COUNTERS.anon = 0; + FALLBACK_COUNTERS.named.clear(); +} + +module.exports = { + deriveIdentity, + deriveName, + sanitizeForFilePath, + anonymousStem, + _resetCounters +}; diff --git a/dropin/index.js b/dropin/index.js new file mode 100644 index 0000000..0e079a2 --- /dev/null +++ b/dropin/index.js @@ -0,0 +1,264 @@ +'use strict'; + +// @percy/playwright-dropin — overrides Playwright's toHaveScreenshot() so existing visual tests +// route their captured PNGs into Percy (screenshot/BYOS mode), with one config line and no test +// rewrites. Requiring this module registers the override globally (Q3-proven: applies to tests +// importing `expect` straight from @playwright/test). +// +// V1 behaviour (per plan): +// • Capture: full-override (KD4) via a side-effect-free seam (src/capture.js). +// • Post: the EXISTING postComparison path (KD3 — pure reuse, web-shaped tag + base64 content tile). +// • Throw policy is CENTRALIZED HERE, ABOVE the capture seam (KD4): the strategy returns data; +// index.js decides whether to pass/throw based on the active mode: +// - async always-pass (D6, default): never throw; verdict deferred to Percy review. +// - compat (D6/KD5): run the NATIVE matcher's throw semantics (missing-baseline suppressed). +// - sync (D10/KD14): await the per-comparison verdict; throw inline ONLY on verdict+diff. +// • D3: a Percy *error* NEVER fails the suite (try/catch + log.debug) — in every mode. +// • Native fallback (D7/Unit 6): if Percy is disabled at the START of the run, the WHOLE run goes +// native (latched once) so the suite behaves exactly as pre-install. +// +// The three modes are MUTUALLY EXCLUSIVE and resolved at config-load (Unit 7 / src/config.js). +const { expect: baseExpect, test } = require('@playwright/test'); +const utils = require('@percy/sdk-utils'); +const { captureFullOverride } = require('./capture'); +const { captureDomSnapshot } = require('./dom'); +const { deriveIdentity } = require('./identity'); +const fallback = require('./fallback'); +const { classifySyncResult } = require('./sync'); +const { loadConfig, validateConfig, assertSyncEngaged, modeStatusLine } = require('./config'); + +const { CLIENT_INFO, ENV_INFO } = require('./version-info'); +const { pngDimensions } = require('./png'); +const log = utils.logger('playwright-dropin'); + +// Capture Playwright's ORIGINAL toHaveScreenshot BEFORE we override the slot — the native-fallback +// (D7) and compat-mode (D6) paths invoke it. Must happen prior to the override registration below. +const nativeMatcher = fallback.captureNativeMatcher(baseExpect); + +// Pristine expect snapshot for native delegation. `extend()` COPIES userMatchers at call time, so a +// chain created from this instance keeps dispatching to the BUILT-IN toHaveScreenshot even after we +// inject our override into the shared instance below. Subject binding, matcher state and step +// reporting all come from Playwright itself. (`captureNativeMatcher`'s raw-slot grab returns a +// closure with the subject already bound to `undefined` on Playwright >=1.49's expect, so it cannot +// be applied to a real page — kept only as a shape probe / legacy export.) +const nativeExpect = baseExpect.extend({}); + +function currentTestInfo() { + try { return test.info(); } catch { return null; } +} + +// First-build detection for the sync classifier (KD7). The globalSetup seed (Unit 4b) sets +// PERCY_DROPIN_FIRST_BUILD when it establishes the project's first baseline → the head this run is +// build #1, whose diffs are baseline-establishment noise. The reporter (Gate A) does the +// authoritative post-finish detection via the build's base-build relationship. +function isFirstBuildRun() { + return process.env.PERCY_DROPIN_FIRST_BUILD === '1'; +} + +// Run-level native-fallback latch (D7/KD6). isPercyEnabled() is checked once at the FIRST assertion; +// its verdict is latched for the whole run so we never go partial-native mid-run (mid-run blips are +// retried instead). null = not yet decided. We also run the one-time config validation + footgun +// rejections (Unit 7) + the mode status line here. +let _runMode = null; // 'percy' | 'native' +let _validated = false; +async function resolveRunMode(config) { + if (_runMode) return _runMode; + + // Fallback can be disabled by config (then we stay in Percy mode and simply no-op when disabled). + const enabled = await utils.isPercyEnabled().catch(() => false); + + // One-time footgun validation + pre-flight checks (mutual exclusion, sync+deferred, token scope). + // A rejected combination is a CONFIGURATION error the user must fix — it is allowed to throw out + // of the matcher (unlike a Percy *runtime* error, which D3 swallows). We only validate when Percy + // is live (native fallback means none of the modes are in play). + if (enabled && !_validated) { + _validated = true; + await validateConfig(config); + log.info(modeStatusLine(config)); + } + + if (enabled) { + _runMode = 'percy'; + } else if (config.fallback) { + fallback.noteNativeFallback('Percy not enabled at run start'); + _runMode = 'native'; + } else { + // Fallback disabled → behave as the old skip-silently path (D6 always-pass, no native compare). + _runMode = 'percy'; + } + return _runMode; +} + +// Test-only reset of the latch + native-notice (the harness re-requires a fresh process in CI, but +// unit tests in-process need to flip run state between cases). +function _resetRunState() { _runMode = null; _validated = false; fallback._resetNotice(); } + +// Build the postComparison options for a captured tile. `sync` is added only in sync mode so the +// CLI awaits the per-comparison verdict (it also honours percy.config.snapshot.sync server-side). +function comparisonOptions({ name, browserFamily, width, pngBuffer, sync }) { + // percy-api requires tag height (screenshot records validate presence); width stays the + // IDENTITY width (viewport) so baseline↔head pairing is stable, height comes from the actual + // PNG bytes (accurate for both page and element captures). + const dims = pngDimensions(pngBuffer); + const options = { + name, + clientInfo: CLIENT_INFO, + environmentInfo: ENV_INFO, + tag: { name: browserFamily, browserName: browserFamily, width, height: dims && dims.height }, + tiles: [{ content: pngBuffer.toString('base64') }] + }; + if (sync) options.sync = true; + return options; +} + +// Build the postSnapshot options for a serialized DOM (captureMode: 'snapshot'). The test's +// viewport pins the render width/minHeight so the server-side render matches the width the +// assertion ran at; `scope` (present for Locator subjects) clips the render to the element. +function snapshotPostOptions({ name, width, viewport, url, domSnapshot, scope, sync }) { + const options = { + name, + widths: [width], + clientInfo: CLIENT_INFO, + environmentInfo: ENV_INFO, + url, + domSnapshot + }; + if (viewport && viewport.height) options.minHeight = viewport.height; + if (scope) options.scope = scope; + if (sync) options.sync = true; + return options; +} + +const percyMatchers = { + async toHaveScreenshot(pageOrLocator, nameOrOptions, maybeOptions) { + const config = loadConfig(); + const matcherState = this; + const nativeArgs = [pageOrLocator, nameOrOptions, maybeOptions]; + + // (1) Run-level native fallback (D7): Percy disabled at run start → native compare for the + // WHOLE run. Native throws on real diffs (pre-install behaviour) but we suppress the + // missing-baseline first-run throw so installing the drop-in can't red a fresh repo. + const mode = await resolveRunMode(config); + if (mode === 'native') { + return fallback.runNativeViaExpect(nativeExpect, matcherState, nativeArgs, { suppressMissingBaseline: true }); + } + + // (2) Percy is live. Capture + post. D3: a Percy *error* must never fail the suite. + let syncResult; + try { + if (!(await utils.isPercyEnabled())) { + return { pass: true, message: () => 'Percy is disabled — snapshot skipped' }; + } + + // First-build-as-baseline: globalSetup seeded the committed snapshot PNGs as this build's + // content, so live captures must NOT be posted on top — the auto-approved baseline is + // exactly the blessed repo PNGs. The assertion still passes (always-pass posture). + if (process.env.PERCY_DROPIN_SEEDED_BASELINE === '1') { + return { + pass: true, + message: () => 'Percy: first build — baseline established from committed snapshots; live capture skipped' + }; + } + + const nameArg = typeof nameOrOptions === 'string' || Array.isArray(nameOrOptions) ? nameOrOptions : undefined; + const options = (typeof nameOrOptions === 'object' && !Array.isArray(nameOrOptions) ? nameOrOptions : maybeOptions) || {}; + + const { name, browserFamily, width } = deriveIdentity(pageOrLocator, nameArg, currentTestInfo()); + + if (config.captureMode === 'snapshot') { + // V2 — serialized-DOM web snapshot (dom.js seam): Percy renders server-side (web project). + // Same identity name, same throw policy; the render width is pinned to the test's viewport. + const { domSnapshot, url, scope, viewport } = await captureDomSnapshot(pageOrLocator, options); + const postOptions = snapshotPostOptions({ name, width, viewport, url, domSnapshot, scope, sync: config.sync }); + + if (config.sync) { + // Snapshot sync rides the same .percy.yml snapshot.sync; an unrecognised verdict shape + // classifies as no-verdict (never a false-green — the gate backstops). + const response = await utils.postSnapshot(postOptions); + assertSyncEngaged(config); + syncResult = (response && response.body && response.body.data) || response; + } else { + await fallback.retryablePost(() => utils.postSnapshot(postOptions)); + } + } else { + // V1 — screenshot/BYOS (capture.js seam): upload the pre-rendered PNG (generic/app project). + const pngBuffer = await captureFullOverride(pageOrLocator, options); + const postOptions = comparisonOptions({ name, browserFamily, width, pngBuffer, sync: config.sync }); + + // KD3 reuse: the existing /percy/comparison ingest accepts a web-shaped tag + inline content + // tile. In sync mode postComparison returns the per-comparison verdict; otherwise we + // retry-on-blip (D7 mid-run) and never go native inside a live run. + if (config.sync) { + // Sync mode: a missing verdict must still red CI via the Gate-A backstop, so the post here + // is NOT wrapped in retryablePost's swallow — the classifier owns the {error} bucket. + syncResult = await utils.postComparison(postOptions); + // Runtime guard: assert sync actually engaged (a deferred-upload that slipped in at runtime + // would silently turn sync into a no-op). Surfaces loudly; the gate still backstops. + assertSyncEngaged(config); + } else { + await fallback.retryablePost(() => utils.postComparison(postOptions)); + } + } + + // (3) Sync mode (D10/KD14): apply the 3-way classifier ABOVE the capture seam. First-build + // review-only (KD7) and the {error} no-verdict bucket are handled inside the classifier. + if (config.sync) { + const verdict = classifySyncResult(syncResult, { name, browserFamily, width }, { isFirstBuild: isFirstBuildRun() }); + if (verdict.throw) { + // Real regression on a non-first build → fail THIS assertion inline (dashboard URL in msg). + return { pass: false, message: () => verdict.message }; + } + return { pass: true, message: () => verdict.message || '' }; + } + } catch (err) { + // D3: any Percy error (capture, post, classify) is swallowed — never fail the functional + // suite on a Percy problem. The async/always-pass and sync paths both land here on error. + log.debug(`Percy: skipped toHaveScreenshot — ${err.message}`); + } + + // (4) Compat mode (D6/KD5): preserve native THROW semantics even with Percy on, but suppress + // the missing-baseline first-run throw. Runs AFTER the Percy post so the snapshot still uploads. + if (config.compat) { + return fallback.runNativeViaExpect(nativeExpect, matcherState, nativeArgs, { suppressMissingBaseline: true }); + } + + // (5) Default async always-pass (D6): verdict deferred to Percy's async review. + return { pass: true, message: () => '' }; + } +}; + +// Register the override on the SHARED expect instance tests import. Playwright's public +// `expect.extend()` SILENTLY SKIPS matcher names that collide with built-ins on the shared instance +// (1.60: `if (name in allBuiltinMatchers) continue`; 1.49: qualified-name shadowing) — the override +// only takes effect on the NEW instance extend() returns, which tests never import. To keep the +// zero-test-change promise we inject the matcher into the shared instance's userMatchers via its +// META_INFO symbol: call-time dispatch spreads `{...allBuiltinMatchers, ...userMatchers}`, so +// userMatchers win. Falls back to plain extend() (custom-name semantics) if the internal shape ever +// changes, and warns — a silent no-op here means NO snapshot ever reaches Percy while CI stays +// green, the worst failure mode this package has. +const metaSym = Object.getOwnPropertySymbols(baseExpect) + .find(s => baseExpect[s] && typeof baseExpect[s] === 'object' && baseExpect[s].userMatchers); +if (metaSym) { + baseExpect[metaSym].userMatchers.toHaveScreenshot = percyMatchers.toHaveScreenshot; +} else { + baseExpect.extend(percyMatchers); + log.warn('Percy: could not inject the toHaveScreenshot override into this Playwright version — ' + + 'falling back to expect.extend(), which may be ignored for built-in matcher names. ' + + 'If snapshots do not appear in Percy, this Playwright version is unsupported.'); +} + +// Unit 4b — the first-build baseline seed (the bet). Exposed so a consumer can either point +// `globalSetup` at the package's `/global-setup` entry, or call this from their own globalSetup. +const baselineGlobalSetup = require('./global-setup'); + +// Unit 5 — the opt-in gate reporter, exposed for one-line wiring in playwright.config `reporter`. +const PercyGateReporter = require('./reporter'); + +module.exports = { + CLIENT_INFO, + ENV_INFO, + baselineGlobalSetup, + PercyGateReporter, + _resetRunState, + nativeMatcher +}; diff --git a/dropin/paths.js b/dropin/paths.js new file mode 100644 index 0000000..755349b --- /dev/null +++ b/dropin/paths.js @@ -0,0 +1,20 @@ +'use strict'; + +// Path hygiene at the filesystem boundary. The drop-in only ever reads the developer's own +// working tree, but the values reaching `path.join` (config rootDir, configured snapshotDir, +// directory-walk entries) are still validated before use (CWE-22). + +// Strip NUL bytes — the one byte that can smuggle a truncated path past fs APIs. +function sanitizePath(p) { + return String(p).replace(/\0/g, ''); +} + +// A directory entry name must be a single path component: no separators, no `.`/`..`. +// fs.readdir can't actually return anything else — enforced anyway; violations return null. +function sanitizeDirentName(name) { + const clean = sanitizePath(name); + if (!clean || clean === '.' || clean === '..' || clean.includes('/') || clean.includes('\\')) return null; + return clean; +} + +module.exports = { sanitizePath, sanitizeDirentName }; diff --git a/dropin/png.js b/dropin/png.js new file mode 100644 index 0000000..bd08153 --- /dev/null +++ b/dropin/png.js @@ -0,0 +1,19 @@ +'use strict'; + +// Minimal PNG IHDR reader. percy-api's screenshot records validate height presence +// (`Validation failed: Height can't be blank` from AfterRenderJob), and neither the +// /percy/comparison CLI endpoint nor the client-direct seed path enriches tag dimensions the way +// the newer /percy/screenshot endpoint does — so the SDK must send accurate dims itself. The PNG +// bytes are the source of truth: IHDR is always the first chunk, width/height big-endian at +// offsets 16/20. +function pngDimensions(buf) { + if (!Buffer.isBuffer(buf) || buf.length < 24) return null; + // \x89PNG\r\n\x1a\n signature + "IHDR" chunk type at offset 12. + if (buf.readUInt32BE(12) !== 0x49484452) return null; + const width = buf.readUInt32BE(16); + const height = buf.readUInt32BE(20); + if (!width || !height) return null; + return { width, height }; +} + +module.exports = { pngDimensions }; diff --git a/dropin/reporter.js b/dropin/reporter.js new file mode 100644 index 0000000..cdd5f65 --- /dev/null +++ b/dropin/reporter.js @@ -0,0 +1,207 @@ +'use strict'; + +// Unit 5 — the opt-in gate reporter (R5/KD7). +// +// A Playwright reporter whose `onEnd` waits for the Percy build to finish and decides CI pass/fail: +// • Default = INFORMATIONAL (green + the Percy build link) — Percy never reds CI unless opted in. +// • Opt-in gate (config.gate === 'fail-on-changes') → reuse `build:wait --fail-on-changes` +// semantics (cli/packages/cli-build/src/wait.js `isFailing`): red when a FINISHED build has +// diffs, unless the build is approved (pass-if-approved). +// • KD7 — the FIRST build is REVIEW-ONLY: build #1 has no base build (its repo-PNG-base-vs-head +// diffs are baseline-establishment noise, not regressions) → surface, NEVER hard-fail. +// • Attribution: list changed comparisons (test title + snapshot + browser/width) so a red build +// is traceable to the exact assertions. +// +// This is ALSO the mandatory Gate-A backstop for sync mode (Unit 5b/KD14): it catches the +// no-verdict population (their diffs still land in build-level total-comparisons-diff). So when sync +// is on we keep the gate active even if the consumer left it informational — see resolveGateMode. +const utils = require('@percy/sdk-utils'); + +const log = utils.logger('playwright-dropin'); + +const TRUNCATE_AT = 20; + +// Lazily load the ESM @percy/client (the drop-in is CommonJS). +async function loadClient() { + const mod = await import('@percy/client'); + return mod.default || mod.PercyClient; +} + +// `build:wait` failing semantics (mirror of cli/packages/cli-build/src/wait.js `isFailing`), with +// the KD7 first-build carve-out folded in. `attrs` is the build's JSONAPI attributes. +function isFailing(attrs = {}, { failOnChanges, passIfApproved, isFirstBuild } = {}) { + const state = attrs.state; + const diffs = attrs['total-comparisons-diff']; + const reviewState = attrs['review-state']; + + // KD7: the first build is review-only — never hard-fail on its (noise-dominated) diffs. + if (isFirstBuild) return false; + + return state != null && state !== 'pending' && state !== 'processing' && + (state !== 'finished' || (failOnChanges && !!diffs && !(passIfApproved && reviewState === 'approved'))); +} + +// First-build detection: a build with no resolved base build is build #1 for its lineage (its +// diffs are baseline-establishment noise — KD7). We treat a missing `base-build` relationship OR +// build-number 1 as first-build; either signal is sufficient. +function isFirstBuildResponse(buildResponse) { + const data = buildResponse && buildResponse.data; + if (!data) return false; + const baseRel = data.relationships && data.relationships['base-build']; + const hasBase = Boolean(baseRel && baseRel.data); + const number = data.attributes && data.attributes['build-number']; + return !hasBase || number === 1; +} + +// Format the attribution list (plan §User-Facing States "Gate attribution list"). `changed` is a +// list of { title, snapshot, browserFamily, width, url, reason? } already mapped from comparisons. +function formatAttribution(changed, { webUrl } = {}) { + if (!changed.length) { + return [`Percy: no visual changes — ${webUrl || ''}`.trim()]; + } + const lines = [`Percy: ${changed.length} visual change${changed.length === 1 ? '' : 's'} need review`]; + const shown = changed.slice(0, TRUNCATE_AT); + for (const c of shown) { + const id = [c.title || c.snapshot, c.browserFamily, c.width && `${c.width}px`].filter(Boolean).join(' · '); + const reason = c.reason ? ` (${c.reason})` : ''; + lines.push(` • ${id}${reason}${c.url ? ` — ${c.url}` : ''}`); + } + if (changed.length > TRUNCATE_AT) { + lines.push(` • …and ${changed.length - TRUNCATE_AT} more`); + } + return lines; +} + +// Map a build's changed comparisons to attribution rows. The comparison's snapshot name carries the +// test-title path the override derived (identity.js: titlePath joined with " > "); browser/width +// come from the comparison tag. `comparisons` is a JSONAPI list (data + included tags). +function mapChangedComparisons(comparisons = []) { + return comparisons + .filter(c => { + const a = c.attributes || {}; + // A diff: any non-equal/unreviewed-with-diff comparison. The API marks diffs via + // `diff-ratio` > 0 or a `state`/`review-state` change; we treat a positive diff-ratio as the + // signal (screenshot/BYOS comparisons always carry it when they differ). + return Number(a['diff-ratio']) > 0; + }) + .map(c => { + const a = c.attributes || {}; + const tag = a.tag || {}; + return { + title: a['snapshot-name'] || a.name, + snapshot: a['snapshot-name'] || a.name, + browserFamily: tag.name || tag['browser-name'], + width: tag.width, + url: a['web-url'] || a.url + }; + }); +} + +// The reporter class. Playwright instantiates `new Reporter(options)` from the `reporter` config +// entry; we accept injectable deps for unit testing. +class PercyGateReporter { + // `deps` (test-only): { client, buildId, config, exit }. + constructor(options = {}, deps = {}) { + this._options = options || {}; + this._deps = deps || {}; + } + + // Resolve the gate mode. Default informational; opt-in via reporter option or config; sync mode + // forces the gate ON as its mandatory backstop (KD14) — sync-without-gate would false-green the + // no-verdict population. + _gateMode(config) { + if (config && config.sync) return 'fail-on-changes'; + const opt = this._options.gate || (config && config.gate); + return opt === 'fail-on-changes' ? 'fail-on-changes' : 'informational'; + } + + // Resolve the build id: explicit dep (tests) → PERCY_BUILD_ID (set by `percy exec`) → healthcheck. + _resolveBuildId() { + if (this._deps.buildId) return this._deps.buildId; + if (process.env.PERCY_BUILD_ID) return process.env.PERCY_BUILD_ID; + return utils.percy && utils.percy.build && utils.percy.build.id; + } + + async onEnd() { + try { + const config = this._deps.config || require('./config').loadConfig(); + const enabled = await utils.isPercyEnabled().catch(() => false); + if (!enabled) { + // Native fallback was active (Unit 6) — a green CI here is NOT "Percy passed". + log.info('Percy: gate skipped — Percy not active for this run'); + return; + } + + const buildId = this._resolveBuildId(); + if (!buildId) { + log.info('Percy: gate skipped — no Percy build id found for this run'); + return; + } + + const gateMode = this._gateMode(config); + const failOnChanges = gateMode === 'fail-on-changes'; + const passIfApproved = Boolean(this._options.passIfApproved ?? (config && config.passIfApproved)); + + const PercyClient = this._deps.client ? null : await loadClient(); + const client = this._deps.client || new PercyClient({ token: process.env.PERCY_TOKEN }); + + // Wait for the build to reach a terminal state (reuse the client's waitForBuild poller). + const buildResponse = await client.waitForBuild({ build: String(buildId) }); + const attrs = (buildResponse && buildResponse.data && buildResponse.data.attributes) || {}; + const webUrl = attrs['web-url']; + const isFirstBuild = isFirstBuildResponse(buildResponse); + + // Attribution: fetch + map changed comparisons (best-effort — never block the verdict on it). + let changed = []; + try { + const comparisons = await this._fetchChangedComparisons(client, buildId); + changed = mapChangedComparisons(comparisons).map(c => ({ ...c, url: c.url || webUrl })); + } catch (err) { + log.debug(`Percy: could not fetch comparison attribution — ${err.message}`); + } + + for (const line of formatAttribution(changed, { webUrl })) log.info(line); + + if (isFirstBuild) { + log.info('Percy: first build — diffs compare your committed baselines against this CI run ' + + 'and may reflect environment differences, not regressions. Reviewing them sets your baseline.'); + } + + const failing = isFailing(attrs, { failOnChanges, passIfApproved, isFirstBuild }); + if (failing) { + log.error(`Percy: visual changes detected — failing CI. ${webUrl || ''}`.trim()); + this._fail(); + } else if (!failOnChanges) { + log.info(`Percy: informational — review your build at ${webUrl || ''}`.trim()); + } + } catch (err) { + // D3: a Percy/gate error must never red the suite by accident. A gate that can't reach Percy + // stays green (the user's tests already passed/failed on their own merits). + log.debug(`Percy: gate skipped — ${err.message}`); + } + } + + // Fetch the build's comparisons for attribution. Tries the client's comparison-list accessor; + // returns [] if the client/version doesn't expose one (attribution is best-effort). + async _fetchChangedComparisons(client, buildId) { + if (typeof client.getComparisons === 'function') { + const res = await client.getComparisons(buildId); + return (res && res.data) || []; + } + return []; + } + + // Set the failing exit status. Playwright's reporter contract honours a non-zero process.exitCode + // (or an exit-status return); we set process.exitCode so the CI run reds. + _fail() { + if (this._deps.exit) return this._deps.exit(1); + process.exitCode = 1; + } +} + +module.exports = PercyGateReporter; +module.exports.PercyGateReporter = PercyGateReporter; +module.exports.isFailing = isFailing; +module.exports.isFirstBuildResponse = isFirstBuildResponse; +module.exports.formatAttribution = formatAttribution; +module.exports.mapChangedComparisons = mapChangedComparisons; diff --git a/dropin/sync.js b/dropin/sync.js new file mode 100644 index 0000000..d24bc0a --- /dev/null +++ b/dropin/sync.js @@ -0,0 +1,227 @@ +'use strict'; + +// Unit 5b — sync-assertion mode (Option C / D10 / KD14). +// +// Sync is read from the GLOBAL `.percy.yml snapshot.sync` (via the healthcheck `percy.config`, +// surfaced by config.js) — it is NOT a drop-in field. When on, the override posts the comparison +// with `sync: true`, awaits the per-comparison verdict, and applies the KD14 3-WAY CLASSIFIER here +// (above the capture seam — the strategy returns data; index.js decides the throw): +// +// 1. verdict + diff → throw inline (this test fails; message has the dashboard URL). +// 2. verdict + no diff → pass. +// 3. {error} (timeout/comparison-error/CLI-exit/403) → do NOT throw; count locally + emit +// `playwright_dropin_sync_no_verdict`. The mandatory Gate-A backstop (reporter.js) is the ONLY +// signal that catches this population (their diffs still land in build-level +// total-comparisons-diff), so a no-verdict is NEVER a false-green. +// +// FIRST-BUILD-REVIEW-ONLY (KD7) is checked INSIDE the classifier, BEFORE the diff branch — first +// build diffs are baseline-establishment noise, so we never throw on them. +// +// `handleSyncJob` (cli core) returns the comparison detail on success and `{ error }` on +// timeout/exception/403 — indistinguishable from a clean pass UNLESS we inspect `.error`. The +// classifier makes that distinction explicit. +const utils = require('@percy/sdk-utils'); + +const log = utils.logger('playwright-dropin'); + +// Per-assertion wait cap (KD14). The CLI's WaitForJob also caps at ~90s; we surface the same bound. +const SYNC_TIMEOUT = 90_000; + +const NO_VERDICT_EVENT = 'playwright_dropin_sync_no_verdict'; + +// Local no-verdict counter (observable for the run; the telemetry event is emitted per occurrence). +let _noVerdictCount = 0; +function noVerdictCount() { return _noVerdictCount; } +function _resetNoVerdict() { _noVerdictCount = 0; } + +// One-time latch for the token-capability message (FIX #3). When the authoritative read (the +// CLI-side getComparisonDetails) 403s for a write-only token, EVERY assertion would otherwise spam +// the same warning. We emit the distinct capability message exactly ONCE per run, then quietly route +// subsequent auth-failure errors into the no-verdict bucket. +let _tokenCapabilityNotified = false; +function _resetTokenCapabilityNotice() { _tokenCapabilityNotified = false; } + +// The distinct, one-time message surfaced when the sync read is refused for lack of read scope. +const TOKEN_CAPABILITY_MESSAGE = + 'Percy sync: token cannot read comparison results — sync needs a read-capable (full) token. ' + + 'Inline verdicts are disabled; the gate backstop still protects CI.'; + +// Detect an auth/permission failure from a sync result's `{error}`. This is the AUTHORITATIVE +// write-only signal (FIX #3): the CLI's handleSyncJob surfaces the percy-api 403/401 here. The error +// may be a string, an Error, or an object carrying a statusCode/status — match HTTP 403/401 or the +// words "forbidden"/"unauthorized" case-insensitively. +function isAuthFailure(error) { + if (error == null) return false; + const status = (typeof error === 'object' && + (error.statusCode ?? error.status ?? error.code)) || null; + if (status === 403 || status === 401 || status === '403' || status === '401') return true; + const text = typeof error === 'string' + ? error + : (error.message || String(error)); + return /\b(403|401|forbidden|unauthorized|unauthorised)\b/i.test(text); +} + +// Extract the diff-ratio from a sync-cli comparison detail. The result nests the diff under +// screenshots[].diff-info['diff-ratio'] (percy-api ComparisonSerializerService). Returns a number +// (0 when none) or null when the structure is absent (treated as no-verdict upstream). +function extractDiffRatio(detail) { + const screenshots = detail && detail.screenshots; + if (!Array.isArray(screenshots) || !screenshots.length) return null; + let max = 0; + let sawDiffInfo = false; + for (const s of screenshots) { + const info = s && s['diff-info']; + if (info && info['diff-ratio'] != null) { + sawDiffInfo = true; + max = Math.max(max, Number(info['diff-ratio']) || 0); + } + } + return sawDiffInfo ? max : null; +} + +// Pull the best dashboard URL for the failing assertion message. +function dashboardUrl(detail) { + const urls = detail && detail['dashboard-urls']; + if (!urls) return null; + return urls['current-snapshot'] || urls['current-build'] || null; +} + +// Emit the no-verdict telemetry event (best-effort; never throws — learnings: instrumentation must +// not crash the caller). Records to the build-event endpoint when available. +function emitNoVerdict(identity, reason) { + _noVerdictCount += 1; + try { + if (typeof utils.postBuildEvents === 'function') { + // Fire-and-forget; do not await — the assertion path must not block on telemetry. + Promise.resolve(utils.postBuildEvents({ + event: NO_VERDICT_EVENT, + name: identity && identity.name, + browserFamily: identity && identity.browserFamily, + width: identity && identity.width, + reason + })).catch(() => {}); + } + } catch { /* swallow — telemetry must never fail the suite */ } +} + +// The 3-way classifier. Returns { throw: boolean, message: string, outcome: string }. +// `identity` = { name, browserFamily, width }; `opts.isFirstBuild` applies KD7 suppression. +function classifySyncResult(result, identity = {}, opts = {}) { + const idStr = [identity.name, identity.browserFamily, identity.width && `${identity.width}px`] + .filter(Boolean).join(' · '); + + // (3) No-verdict bucket FIRST — {error} is indistinguishable from a clean pass otherwise. + if (!result || result.error) { + const reason = (result && result.error) || 'no result returned'; + + // (3a) AUTH/PERMISSION failure (FIX #3) — the AUTHORITATIVE write-only signal. The sync read + // (getComparisonDetails) 403s for a write-only token; percy-api gates reads to master||read_only + // (build_policy.rb). Emit the distinct token-capability message ONCE, then route this (and every + // subsequent assertion) into the no-verdict bucket — NEVER throw inline, NEVER false-green. The + // mandatory Gate-A backstop (reporter.js) is the signal that reds CI. + if (isAuthFailure(result && result.error)) { + emitNoVerdict(identity, 'token cannot read comparison results (auth failure)'); + if (!_tokenCapabilityNotified) { + _tokenCapabilityNotified = true; + log.warn(TOKEN_CAPABILITY_MESSAGE); + } + return { throw: false, message: TOKEN_CAPABILITY_MESSAGE, outcome: 'no_verdict_auth' }; + } + + emitNoVerdict(identity, reason); + log.warn(`Percy: no verdict for "${identity.name || idStr}" within ${SYNC_TIMEOUT / 1000}s — ` + + 'not failing inline; the post-run gate decides'); + return { throw: false, message: `Percy: no verdict (${reason}) — gate will decide`, outcome: 'no_verdict' }; + } + + // KD7 — first build is review-only. Checked BEFORE the diff branch: build #1 diffs are + // baseline-establishment noise, never a regression → never throw. + if (opts.isFirstBuild) { + return { throw: false, message: 'Percy: first build — review-only (diffs are baseline noise)', outcome: 'first_build' }; + } + + const diffRatio = extractDiffRatio(result); + if (diffRatio == null) { + // Verdict-shaped but no diff-info (e.g. still-initialising) → treat as no-verdict, not a pass. + emitNoVerdict(identity, 'comparison not finished'); + return { throw: false, message: 'Percy: no verdict (comparison not finished) — gate will decide', outcome: 'no_verdict' }; + } + + // (1) verdict + diff → throw inline. + if (diffRatio > 0) { + const url = dashboardUrl(result); + const msg = `Percy: visual change detected for "${identity.name || idStr}"` + + (url ? ` — review at ${url}` : ''); + return { throw: true, message: msg, outcome: 'diff' }; + } + + // (2) verdict + no diff → pass. + return { throw: false, message: 'Percy: no visual change', outcome: 'no_diff' }; +} + +// NON-AUTHORITATIVE prefix hint (FIX #3). A token's prefix (`web_`/`auto_`) encodes the project +// TYPE, not the permission ROLE — the role isn't in the token string at all. So this is only a weak +// HINT for diagnostics: a generic-project write-only token (`ss_…`) has no `web_`/`auto_` prefix and +// a read-capable `web_` token exists. It MUST NOT drive a refusal on its own — the authoritative +// signal is the runtime 403 from the sync read (see isAuthFailure / the classifier's no-verdict +// bucket). Kept only so callers can log a hint; never used to refuse. +function isWriteOnlyToken(token) { + if (!token) return false; + return /^web_/i.test(token) || /^auto_/i.test(token); +} + +// Best-effort pre-flight token-scope check (FIX #3). We NO LONGER refuse based on the brittle prefix +// heuristic (false negatives on `ss_…` write-only tokens, false positives on read-capable `web_` +// tokens). Refusal is only ever returned from an AUTHORITATIVE read: +// • an injected `probe` that actually asks the API whether reads are allowed, or +// • a real resource-backed read via @percy/client.getBuild when a build id is already known. +// When NO resource is available we SKIP the pre-flight refusal (return ok) rather than guess — the +// runtime 403 in the classifier is the real backstop, so we never block sync on a guess. +// Resolves { ok, reason }. +async function preflightTokenScope({ token = process.env.PERCY_TOKEN, probe, buildId, client } = {}) { + if (probe) { + try { + const readable = await probe(token); + return readable + ? { ok: true } + : { ok: false, reason: 'token cannot read comparison results' }; + } catch (err) { + return { ok: false, reason: `token scope check failed — ${err.message}` }; + } + } + + // Resource-backed pre-flight: only when a build id is already known. getBuild reads through the + // same percy-api read gate, so a 403/401 here is the authoritative write-only signal. + if (buildId && client && typeof client.getBuild === 'function') { + try { + await client.getBuild(buildId); + return { ok: true }; + } catch (err) { + if (isAuthFailure(err)) { + return { ok: false, reason: 'token cannot read comparison results' }; + } + // Non-auth error (network, 404, …) is NOT a scope verdict — do not refuse on a guess. + return { ok: true }; + } + } + + // No resource to read against → cannot authoritatively decide pre-flight. Allow; the classifier's + // 403 handling backstops at runtime. + return { ok: true }; +} + +module.exports = { + classifySyncResult, + extractDiffRatio, + dashboardUrl, + preflightTokenScope, + isWriteOnlyToken, + isAuthFailure, + noVerdictCount, + emitNoVerdict, + _resetNoVerdict, + _resetTokenCapabilityNotice, + SYNC_TIMEOUT, + NO_VERDICT_EVENT, + TOKEN_CAPABILITY_MESSAGE +}; diff --git a/dropin/version-info.js b/dropin/version-info.js new file mode 100644 index 0000000..95d3e45 --- /dev/null +++ b/dropin/version-info.js @@ -0,0 +1,17 @@ +'use strict'; + +// Shared client/environment-info strings sent on every Percy post (postComparison clientInfo / +// environmentInfo). Computed once and reused so index.js and global-setup.js don't each recompute +// the identical pkg-name/version + playwright-version lookup. +const pkg = require('../package.json'); + +// `@percy/playwright/` — identifies this SDK to Percy. Drop-in traffic is additionally +// attributed via the build source tag (`playwright-dropin`), not a separate client string. +const CLIENT_INFO = `${pkg.name}/${pkg.version}`; + +// `playwright/` — best-effort; degrades to a bare label if @playwright/test isn't present. +const ENV_INFO = (() => { + try { return `playwright/${require('@playwright/test/package.json').version}`; } catch { return 'playwright'; } +})(); + +module.exports = { CLIENT_INFO, ENV_INFO }; diff --git a/index.js b/index.js index 3215d99..8d280ed 100644 --- a/index.js +++ b/index.js @@ -48,9 +48,16 @@ function isUnsupportedIframeSrc(src) { // Collect client and environment information const sdkPkg = require('./package.json'); -const playwrightPkg = require('playwright/package.json'); const CLIENT_INFO = `${sdkPkg.name}/${sdkPkg.version}`; -const ENV_INFO = `${playwrightPkg.name}/${playwrightPkg.version}`; +// Best-effort: on newer Playwright the runner may be mid-load when this module is required +// (CJS↔ESM interop), and consumers may have only @playwright/test installed — degrade to a bare +// label rather than crashing at import time. +// istanbul ignore next - which fallback executes depends on the consumer's installed packages +const ENV_INFO = (() => { + try { return `playwright/${require('playwright/package.json').version}`; } catch {} + try { return `playwright/${require('@playwright/test/package.json').version}`; } catch {} + return 'playwright'; +})(); const log = utils.logger('playwright'); // Use CDP to discover closed shadow roots and expose them to PercyDOM.serialize(). @@ -590,6 +597,9 @@ module.exports.ENV_INFO = ENV_INFO; module.exports.frameDepth = frameDepth; module.exports.isCyclicFrame = isCyclicFrame; module.exports.captureSerializedDOM = captureSerializedDOM; +// Internal: full DOM capture (readiness gate + responsive + CORS iframes), reused by the +// toHaveScreenshot drop-in's snapshot mode (dropin/dom.js). Not public API. +module.exports.captureDOM = captureDOM; module.exports.resolveIgnoreSelectors = resolveIgnoreSelectors; module.exports.isUnsupportedIframeSrc = isUnsupportedIframeSrc; module.exports.resolveMaxFrameDepth = resolveMaxFrameDepth; diff --git a/package.json b/package.json index 63d5783..732d951 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,13 @@ "index.js", "utils.js", "cache.js", + "dropin", + "bin", "types/index.d.ts" ], + "bin": { + "percy-playwright": "./bin/percy-playwright.js" + }, "engines": { "node": ">=14" }, @@ -32,11 +37,17 @@ "tag": "beta" }, "peerDependencies": { - "playwright-core": ">=1" + "playwright-core": ">=1", + "@playwright/test": ">=1.40.0" + }, + "peerDependenciesMeta": { + "@playwright/test": { + "optional": true + } }, "devDependencies": { "@percy/cli": "^1.32.0-beta.4", - "@playwright/test": "^1.24.2", + "@playwright/test": "^1.60.0", "babel-eslint": "^10.1.0", "cross-env": "^7.0.2", "eslint": "^7.18.0", @@ -46,7 +57,7 @@ "eslint-plugin-promise": "^5.1.0", "eslint-plugin-standard": "^5.0.0", "nyc": "^15.1.0", - "playwright": "^1.24.2", + "playwright": "^1.60.0", "sinon": "^18.0.1", "tsd": "^0.25.0" } diff --git a/tests/dropin.spec.mjs b/tests/dropin.spec.mjs new file mode 100644 index 0000000..2cc8045 --- /dev/null +++ b/tests/dropin.spec.mjs @@ -0,0 +1,128 @@ +// toHaveScreenshot drop-in — dispatch + unit coverage. +// +// The dispatch tests import the drop-in entry (which registers the override on the shared +// Playwright `expect`) and then call the REAL `expect(page).toHaveScreenshot()` — hard-asserting +// that a comparison reached the CLI testing server. A silent registration no-op (the failure mode +// Playwright's extend() guard makes possible) leaves the suite green with zero Percy traffic, so +// these tests must fail loudly if nothing was posted. +import helpers from '@percy/sdk-utils/test/helpers'; +import utils from '@percy/sdk-utils'; +import { test, expect } from '@playwright/test'; +import '../dropin/index.js'; +import dropinDom from '../dropin/dom.js'; +import firstBuild from '../dropin/baseline/first-build.js'; +import identity from '../dropin/identity.js'; +import paths from '../dropin/paths.js'; + +const { captureDomSnapshot, SCOPE_ATTR } = dropinDom; +const { firstBuildBaseline, OUTCOME, BASELINE_SOURCE } = firstBuild; +const { deriveName, _resetCounters } = identity; + +async function recordedComparisons() { + const res = await utils.request('/test/requests'); + return (res.body.requests || []).filter(r => r.url.startsWith('/percy/comparison')); +} + +test.describe('toHaveScreenshot drop-in (dispatch)', () => { + test.beforeEach(async ({ page }) => { + await helpers.setupTest(); + await page.goto(helpers.testSnapshotURL); + }); + + test('routes toHaveScreenshot (named + anonymous) through Percy and always passes', async ({ page }) => { + // The CLI testing server is shared across parallel workers and reset by every test's + // setupTest — a read-back can race a concurrent reset. Retry the whole post+read block as a + // unit; snapshot-name counters increment across retries, so match name families, not indices. + await expect(async () => { + await expect(page).toHaveScreenshot('dropin-named.png'); + await expect(page).toHaveScreenshot(); + await expect(page).toHaveScreenshot(); + + const comparisons = await recordedComparisons(); + + const named = comparisons.find(c => /^dropin-named(-\d+)?$/.test(c.body.name)); + expect(named, 'override did not post a comparison — the toHaveScreenshot override is NOT registered').toBeTruthy(); + expect(named.body.tag.width).toBe(page.viewportSize().width); + // percy-api requires a tag height; the drop-in parses it from the PNG bytes. + expect(named.body.tag.height).toBeGreaterThan(0); + expect(named.body.tiles.length).toBe(1); + expect(named.body.tiles[0].content.length).toBeGreaterThan(0); + + // Anonymous calls get Playwright's on-disk stem naming (title + per-test counter); the + // exact derivation contract is pinned in the unit tests below. + const anonymous = comparisons.filter(c => /-\d+$/.test(c.body.name) && !/^dropin-named/.test(c.body.name)); + expect(anonymous.length).toBeGreaterThanOrEqual(2); + }).toPass({ timeout: 15000 }); + }); + + test('a Percy upload error never fails the assertion (warn-and-continue)', async ({ page }) => { + await helpers.test('error', '/percy/comparison'); + // Must still pass — a Percy problem can never red the functional suite. + await expect(page).toHaveScreenshot('dropin-error-path.png'); + }); +}); + +test.describe('drop-in units', () => { + test('deriveName mirrors Playwright anonymous/named stem rules', () => { + _resetCounters(); + const ti = { titlePath: ['spec.ts', 'suite', 'case'] }; + expect(deriveName(undefined, ti)).toBe('suite-case-1'); + expect(deriveName(undefined, ti)).toBe('suite-case-2'); + expect(deriveName('banner.png', ti)).toBe('banner'); + expect(deriveName('banner.png', ti)).toBe('banner-1'); + }); + + test('path hygiene strips NUL bytes and rejects multi-component dirent names', () => { + expect(paths.sanitizePath('/repo\0/x')).toBe('/repo/x'); + expect(paths.sanitizeDirentName('home-snapshots')).toBe('home-snapshots'); + expect(paths.sanitizeDirentName('..')).toBe(null); + expect(paths.sanitizeDirentName('a/b')).toBe(null); + expect(paths.sanitizeDirentName('a\\b')).toBe(null); + expect(paths.sanitizeDirentName('')).toBe(null); + }); + + test('firstBuildBaseline seeds committed PNGs only when the server marked the build as baseline', async () => { + // Server said normal head → nothing seeded, discovery never runs. + const notFirst = await firstBuildBaseline({}, { + build: { id: '1', source: 'playwright-dropin' }, + discoverBaselines: () => { throw new Error('must not discover'); } + }); + expect(notFirst.outcome).toBe(OUTCOME.NOT_FIRST_BUILD); + + // Server said baseline → committed PNGs are posted with PNG-derived tag dims. + const png = Buffer.alloc(24); + png.writeUInt32BE(0x49484452, 12); + png.writeUInt32BE(1280, 16); + png.writeUInt32BE(720, 20); + const posted = []; + const seeded = await firstBuildBaseline({ clientInfo: 'c', environmentInfo: 'e' }, { + build: { id: '9', source: BASELINE_SOURCE }, + discoverBaselines: () => ({ + baselines: [{ filepath: '/repo/a.png', name: 'home', browserFamily: 'chromium', width: 1280 }] + }), + readFile: async () => png, + postComparison: async options => posted.push(options) + }); + expect(seeded.outcome).toBe(OUTCOME.SEEDED); + expect(posted[0].tag).toEqual({ name: 'chromium', browserName: 'chromium', width: 1280, height: 720 }); + }); + + test('captureDomSnapshot reuses the repo captureDOM and scopes Locator subjects', async ({ page }) => { + await page.setContent('
A card
'); + let sawMarkerDuringCapture = false; + + const result = await captureDomSnapshot(page.locator('#card'), {}, { + fetchPercyDOM: async () => 'window.__percy_dom_injected = true;', + captureDOM: async p => { + sawMarkerDuringCapture = await p.locator(`[${SCOPE_ATTR}]`).count() === 1; + return { html: 'captured' }; + } + }); + + expect(result.scope).toBe(`[${SCOPE_ATTR}]`); + expect(sawMarkerDuringCapture, 'scope marker must be present during capture').toBe(true); + // …and removed from the live page afterwards. + await expect(page.locator(`[${SCOPE_ATTR}]`)).toHaveCount(0); + expect(result.domSnapshot.html).toBe('captured'); + }); +}); diff --git a/types/index.d.ts b/types/index.d.ts index 6f233c2..230b146 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -12,3 +12,19 @@ export default function percyScreenshot( name: string, options?: SnapshotOptions ): Promise; + +// --- toHaveScreenshot drop-in (require('@percy/playwright/dropin')) --------------------------- +// Requiring the dropin entry registers a global override of Playwright's `toHaveScreenshot()` +// matcher (no new matcher signature; the existing one is intercepted). Capture is selected by the +// drop-in config `captureMode`: 'screenshot' (default — raw-PNG upload, generic/app projects) or +// 'snapshot' (serialized-DOM web snapshot via this package's own captureDOM, web projects). + +// First-build baseline hook. Point `playwright.config` `globalSetup` at +// `@percy/playwright/dropin/global-setup`, or call this from your own globalSetup. Never throws. +export function baselineGlobalSetup(config?: unknown): Promise; + +// Opt-in CI gate reporter: reporter: [['@percy/playwright/dropin/reporter']]. +export class PercyGateReporter { + constructor(options?: { gate?: 'informational' | 'fail-on-changes'; passIfApproved?: boolean }, deps?: unknown); + onEnd(): Promise; +} diff --git a/yarn.lock b/yarn.lock index f76c348..4bba234 100644 --- a/yarn.lock +++ b/yarn.lock @@ -640,13 +640,12 @@ "@percy/config" "1.32.0-beta.8" "@percy/sdk-utils" "1.32.0-beta.8" -"@playwright/test@^1.24.2": - version "1.27.1" - resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.27.1.tgz#9364d1e02021261211c8ff586d903faa79ce95c4" - integrity sha512-mrL2q0an/7tVqniQQF6RBL2saskjljXzqNcCOVMUjRIgE6Y38nCNaP+Dc2FBW06bcpD3tqIws/HT9qiMHbNU0A== +"@playwright/test@^1.60.0": + version "1.61.1" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.61.1.tgz#48568dc22af7819e55fa5e8e3bc79b7e6a3e6675" + integrity sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig== dependencies: - "@types/node" "*" - playwright-core "1.27.1" + playwright "1.61.1" "@pnpm/crypto.base32-hash@1.0.1": version "1.0.1" @@ -2009,6 +2008,11 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= +fsevents@2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== + function-bind@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" @@ -3277,17 +3281,19 @@ pkg-dir@^4.1.0: dependencies: find-up "^4.0.0" -playwright-core@1.27.1: - version "1.27.1" - resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.27.1.tgz#840ef662e55a3ed759d8b5d3d00a5f885a7184f4" - integrity sha512-9EmeXDncC2Pmp/z+teoVYlvmPWUC6ejSSYZUln7YaP89Z6lpAaiaAnqroUt/BoLo8tn7WYShcfaCh+xofZa44Q== +playwright-core@1.61.1: + version "1.61.1" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.61.1.tgz#3c99841307efbbabc9d724c41a88c914705d15fc" + integrity sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg== -playwright@^1.24.2: - version "1.27.1" - resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.27.1.tgz#4eecac5899566c589d4220ca8acc16abe8a67450" - integrity sha512-xXYZ7m36yTtC+oFgqH0eTgullGztKSRMb4yuwLPl8IYSmgBM88QiB+3IWb1mRIC9/NNwcgbG0RwtFlg+EAFQHQ== +playwright@1.61.1, playwright@^1.60.0: + version "1.61.1" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.61.1.tgz#d8c0c06eb93c28981afc747bace453bdbd5018bc" + integrity sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ== dependencies: - playwright-core "1.27.1" + playwright-core "1.61.1" + optionalDependencies: + fsevents "2.3.2" plur@^4.0.0: version "4.0.0"