From f1b86d7dd0002cc67c67c4cad3f66568d40c8364 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Fri, 23 Oct 2026 20:05:01 -0700 Subject: [PATCH 001/699] test(interaction): real touch input for browser interaction proofs Adds internal/interaction-harness (@astryxdesign/interaction-harness) and playwright.interaction.config.ts: phone-sized touch projects in Chromium and WebKit for *.interaction.browser.spec.ts specs. Touch scripts are data (tap, drag with hold and rest, flick with an exact release velocity and velocity profiles, hold-then-drag, pinch, rotate, any path, a second finger mid-gesture), played as real engine input: CDP Input.dispatchTouchEvent with scripted timestamps in Chromium, WebKit's own input command in WebKit. The harness's self-test proves the timestamps, a native fling and multi-touch delivery; a pr-interaction CI job runs the proofs against each PR's Storybook build, and a container script runs the WebKit project where host libraries are missing. --- .github/scripts/ci-test-routing.test.mjs | 49 +++ .github/workflows/ci.yml | 88 ++++ .github/workflows/deploy.yml | 3 + internal/README.md | 13 +- internal/interaction-harness/README.md | 67 +++ internal/interaction-harness/package.json | 21 + .../scripts/webkit-container.mjs | 72 ++++ .../src/harness.interaction.browser.spec.ts | 168 ++++++++ internal/interaction-harness/src/index.ts | 17 + .../interaction-harness/src/scripts.test.ts | 142 +++++++ internal/interaction-harness/src/scripts.ts | 395 ++++++++++++++++++ internal/interaction-harness/src/story.ts | 103 +++++ internal/interaction-harness/src/touch.ts | 295 +++++++++++++ internal/interaction-harness/tsconfig.json | 4 + package.json | 1 + playwright.interaction.config.ts | 77 ++++ pnpm-lock.yaml | 15 + 17 files changed, 1524 insertions(+), 6 deletions(-) create mode 100644 internal/interaction-harness/README.md create mode 100644 internal/interaction-harness/package.json create mode 100644 internal/interaction-harness/scripts/webkit-container.mjs create mode 100644 internal/interaction-harness/src/harness.interaction.browser.spec.ts create mode 100644 internal/interaction-harness/src/index.ts create mode 100644 internal/interaction-harness/src/scripts.test.ts create mode 100644 internal/interaction-harness/src/scripts.ts create mode 100644 internal/interaction-harness/src/story.ts create mode 100644 internal/interaction-harness/src/touch.ts create mode 100644 internal/interaction-harness/tsconfig.json create mode 100644 playwright.interaction.config.ts diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjs index dcb44405376a1..1e419f13d18f6 100644 --- a/.github/scripts/ci-test-routing.test.mjs +++ b/.github/scripts/ci-test-routing.test.mjs @@ -517,3 +517,52 @@ describe('deploy.yml push gating', () => { } }); }); + +describe('ci.yml interaction proofs', () => { + const workflow = load('ci.yml'); + const job = workflow.jobs['pr-interaction']; + const step = name => job.steps.find(candidate => candidate.name === name); + + it('runs the interaction config in Chromium against the exact PR head', () => { + expect(KNOWN_RUNNERS.has(job['runs-on'])).toBe(true); + expect(job.needs).toEqual( + expect.arrayContaining(['check-scope', 'build-storybook']), + ); + const checkout = job.steps.find(candidate => + candidate.uses?.startsWith('actions/checkout@'), + ); + expect(checkout?.with?.ref).toBe( + '${{ github.event.pull_request.head.sha }}', + ); + const run = step('Run interaction proofs').run; + expect(run).toContain('--config playwright.interaction.config.ts'); + expect(run).toContain('--project chromium'); + const packageJson = JSON.parse(read('package.json')); + expect(packageJson.scripts['test:interaction']).toBe( + 'playwright test --config playwright.interaction.config.ts', + ); + }); + + it('runs whenever the harness, its config, a component or a story changes', () => { + const scope = step('Check interaction scope').run; + expect(scope).toContain('.github/scripts/pr-scope-touches.sh'); + for (const path of [ + 'internal/interaction-harness/', + 'playwright\\.interaction\\.config\\.ts', + 'packages/[^/]+/src/', + 'apps/storybook/', + ]) { + expect(scope).toContain(path); + } + // Unknown scope (exit 2) runs the proofs; only "out of scope" skips them. + expect(scope).toContain('if [ "$scope" = 1 ]; then'); + }); + + it('typechecks the harness in both workflows', () => { + for (const file of ['ci.yml', 'deploy.yml']) { + expect(runLines(load(file).jobs.typecheck), file).toContain( + '@astryxdesign/interaction-harness typecheck', + ); + } + }); +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d19b9322e0b0..28db4356c3fc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1171,6 +1171,12 @@ jobs: if: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true' run: pnpm -F @astryxdesign/native typecheck:types + # The interaction harness is consumed as TypeScript source by Vitest and + # Playwright, which do not typecheck; this is the only gate over it. + - name: Typecheck interaction harness + if: needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true' + run: pnpm -F @astryxdesign/interaction-harness typecheck + # Sandbox preview build — the single slowest piece of PR CI (~4min), run as # its own job so it overlaps build-storybook instead of serializing after it. # PR-only: merge_group runs never deploy previews, and docsite-only PRs skip @@ -3437,6 +3443,88 @@ jobs: retention-days: 30 delete-merged: true + # Touch interaction and animation proofs: the harness in + # internal/interaction-harness and every *.interaction.browser.spec.ts. + # Real touch input, composited frames, rendering traces and presented + # animation values only exist in a real engine, and the specs read stories + # from this PR's own Storybook build. The harness's self-test runs with + # them, so a change to an instrument is proven by its negative controls + # before a component spec relies on it. Chromium only on these runners; + # see the harness README for running WebKit. + pr-interaction: + needs: [check-scope, build-storybook] + if: github.event_name == 'pull_request' && needs.check-scope.outputs.docsite_only != 'true' && needs.check-scope.outputs.spec_only != 'true' && needs.check-scope.outputs.tooling_only != 'true' + # Frame evidence measures timing: four cores, one Playwright worker. + runs-on: 4-core-ubuntu + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Verify evidence checkout + env: + EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD_SHA" + + # The helper comes from the base, since an older PR head lacks it. + # Exit 1 means out of scope; 0 in scope; 2 unknown, so run. + - name: Check interaction scope + id: scope + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + scope=2 + helper="$(mktemp)" + if git fetch --quiet --no-tags --depth=1 origin "$BASE_SHA" \ + && git show "$BASE_SHA:.github/scripts/pr-scope-touches.sh" > "$helper"; then + scope=0 + bash "$helper" "$BASE_SHA" '^(internal/interaction-harness/|playwright\.interaction\.config\.ts$|packages/[^/]+/src/|apps/storybook/)' || scope=$? + fi + if [ "$scope" = 1 ]; then + echo "Neither the interaction harness nor a component changed; skipping the interaction proofs." + echo 'run=false' >> "$GITHUB_OUTPUT" + else + echo 'run=true' >> "$GITHUB_OUTPUT" + fi + + - name: Setup Node and pnpm + if: steps.scope.outputs.run == 'true' + uses: ./.github/actions/setup + with: + install: 'false' + + - name: Download Storybook artifact + if: steps.scope.outputs.run == 'true' + uses: actions/download-artifact@v8 + with: + name: storybook-${{ needs.build-storybook.outputs.short_hash }} + path: apps/storybook/dist + + - name: Install Playwright + if: steps.scope.outputs.run == 'true' + run: pnpm install --frozen-lockfile && npx playwright install chromium + + - name: Run interaction proofs + if: steps.scope.outputs.run == 'true' + run: | + pnpm exec playwright test \ + --config playwright.interaction.config.ts \ + --project chromium \ + --output interaction-proof-output + + # Storyboards (PNG contact sheets) and curve traces, for review. + - name: Upload interaction evidence + if: always() && steps.scope.outputs.run == 'true' + uses: actions/upload-artifact@v7 + with: + name: interaction-evidence + path: test-results/interaction-evidence + if-no-files-found: ignore + retention-days: 7 + # Build package test owner. The plugin's production CSS partition is a Build # integration contract: source inspection cannot prove which declaration # paints after Vite emits the final bundle. The `test-build` name makes that diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3eaee6914b595..f3b572c1dd629 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -132,6 +132,9 @@ jobs: - name: Typecheck accessibility spec-test contracts run: pnpm -F @astryxdesign/a11y-spec typecheck + - name: Typecheck interaction harness + run: pnpm -F @astryxdesign/interaction-harness typecheck + # Stable join for the three post-merge validation lanes. test: needs: [test-ui, test-node, typecheck] diff --git a/internal/README.md b/internal/README.md index 101fd8b07c28f..650fbd245d1a6 100644 --- a/internal/README.md +++ b/internal/README.md @@ -4,9 +4,10 @@ Internal tooling packages not published to npm. -| Directory | Role | Purpose | -| ------------- | ------- | ----------------------------------------------------------------------- | -| `a11y-spec/` | Testing | Reusable accessibility spec-test contracts and their evidence harnesses | -| `scripts/` | Tooling | One-off automation scripts (Crowdin screenshot upload, etc.) | -| `test-utils/` | Testing | Shared testing utilities and setup for Vitest | -| `vibe-tests/` | Testing | LLM vibeability test harness for component system | +| Directory | Role | Purpose | +| ---------------------- | ------- | --------------------------------------------------------------------------------- | +| `a11y-spec/` | Testing | Reusable accessibility spec-test contracts and their evidence harnesses | +| `interaction-harness/` | Testing | Playwright instruments for touch interaction and animation proofs and storyboards | +| `scripts/` | Tooling | One-off automation scripts (Crowdin screenshot upload, etc.) | +| `test-utils/` | Testing | Shared testing utilities and setup for Vitest | +| `vibe-tests/` | Testing | LLM vibeability test harness for component system | diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md new file mode 100644 index 0000000000000..a0b56d854c6b0 --- /dev/null +++ b/internal/interaction-harness/README.md @@ -0,0 +1,67 @@ +# @astryxdesign/interaction-harness + +Playwright instruments for touch interaction proofs: real touch input. +**Internal and unpublished.** Specs live next to their component as +`*.interaction.browser.spec.ts` and run under +[`playwright.interaction.config.ts`](../../playwright.interaction.config.ts) on +a phone-sized touch screen (402 × 714 CSS px page, 402 × 874 screen, 3 device +pixels per CSS px, touch, mobile viewport rules), in Chromium and WebKit. + +The harness proves its own instruments in +[`src/harness.interaction.browser.spec.ts`](src/harness.interaction.browser.spec.ts), +on fixture pages written inline. + +## Running + +```sh +pnpm storybook:build # component specs read stories from the built Storybook +pnpm exec playwright install chromium webkit +pnpm test:interaction --project chromium # everything, Chromium +pnpm test:interaction # one spec, both engines +``` + +On Linux distributions whose system libraries are older than Playwright's +WebKit build needs, run the WebKit project inside the official Playwright +image (podman or docker; the repository is mounted at the same path): + +```sh +node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] +``` + +## Instruments + +| Module | What it gives a spec | +| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `scripts.ts` | Touch scripts as data: `tap`, `drag` (hold, rest), `flick` (exact release velocity, velocity profiles), `holdThenDrag`, `pinch`, `rotate`, `path`, `merge` (a second finger mid-gesture), `releaseVelocity`, `toTouchEvents`. | +| `touch.ts` | `touchDriver(page).play(script)`: the script as real engine input. `installInputLog` / `readInputLog`: what the page received. | +| `story.ts` | `phone`, `withStorybook`, `openStory`, `frames`. | + +## Real touch + +Untrusted DOM events are not input: they start no native scroll, claim no +gesture, and read `isTrusted: false`. Chromium input goes through CDP +`Input.dispatchTouchEvent` with an explicit timestamp per event: the page's +`event.timeStamp` is the scripted time to the microsecond, whatever the +dispatch latency, and the events go through the compositor's gesture detection +(native scrolling, flings with velocity from those timestamps, `touch-action`, +`pointercancel`). WebKit input goes through the engine's own input command; see +the limits below. Read positions a frame after a move: Chromium delivers +`pointermove` aligned to animation frames. + +## Engine limits (measured with Playwright 1.61) + +| | Chromium 149 | WebKit 26.5 (Linux build) | +| ------------------------------------------------------ | ------------ | ----------------------------------------------------------------------------------- | +| Trusted touch + pointer events, multi-touch | yes | yes (a `touchstart` lists only the fingers that landed in it; pointer events exact) | +| Page sees the scripted timestamps | yes, exactly | no: real dispatch time; real-time pacing keeps velocities within ~1 ms | +| Sub-pixel touch positions | yes | no: integer CSS px | +| Native scroll, fling, pinch-zoom from injected touches | yes | no | +| Click synthesised from a touch | yes | only from `driver.tap` (WebKit's tap gesture), not from a raw touch sequence | +| Input log | yes | yes | + +What WebKit cannot show here — native scroll hand-off, flings, and anything +iOS-only — needs Mobile Safari itself, on a device or the iOS simulator. + +The WebKit input command is reached through Playwright's in-process server +object, so it needs a locally launched browser (the default), not `connect()`. +Firefox has no touch input path and no project. diff --git a/internal/interaction-harness/package.json b/internal/interaction-harness/package.json new file mode 100644 index 0000000000000..08b1d02f61634 --- /dev/null +++ b/internal/interaction-harness/package.json @@ -0,0 +1,21 @@ +{ + "name": "@astryxdesign/interaction-harness", + "version": "0.0.0", + "private": true, + "type": "module", + "description": "Internal: Playwright helpers for touch interaction proofs (real touch input). Never published.", + "main": "./src/index.ts", + "exports": { + ".": "./src/index.ts", + "./scripts": "./src/scripts.ts" + }, + "scripts": { + "typecheck": "tsc --project tsconfig.json --noEmit" + }, + "devDependencies": { + "@astryxdesign/a11y-spec": "workspace:*", + "@playwright/test": "^1.61.1", + "typescript": "^6.0.3", + "vitest": "^4.1.11" + } +} diff --git a/internal/interaction-harness/scripts/webkit-container.mjs b/internal/interaction-harness/scripts/webkit-container.mjs new file mode 100644 index 0000000000000..32cc0452396df --- /dev/null +++ b/internal/interaction-harness/scripts/webkit-container.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * Runs the interaction proofs' WebKit project inside the official Playwright + * container image, for hosts where WebKit cannot launch (Playwright's Linux + * WebKit build needs newer system libraries than some distributions ship). + * + * node internal/interaction-harness/scripts/webkit-container.mjs [playwright test args…] + * + * The repository is mounted at the same path, so node_modules, the built + * Storybook and the evidence directory are the host's own. The image tag + * matches the installed playwright, so its browsers match the client. + * Uses podman when present, else docker. `--network=host` lets the specs' + * local Storybook server and the browser reach each other. + */ +import {spawnSync} from 'node:child_process'; +import fs from 'node:fs'; +import {createRequire} from 'node:module'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; + +const root = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../../..', +); +const require = createRequire(path.join(root, 'package.json')); +const {version} = JSON.parse( + fs.readFileSync(require.resolve('playwright/package.json'), 'utf8'), +); +const image = + process.env.PLAYWRIGHT_IMAGE ?? + `mcr.microsoft.com/playwright:v${version}-noble`; +const engine = ['podman', 'docker'].find( + cmd => spawnSync(cmd, ['--version'], {stdio: 'ignore'}).status === 0, +); +if (engine == null) { + console.error('webkit-container: neither podman nor docker is installed'); + process.exit(1); +} +const args = process.argv.slice(2); +const env = [ + 'ASTRYX_STORYBOOK_DIR', + 'ASTRYX_INTERACTION_EVIDENCE', + 'INTERACTION_WORKERS', + 'CI', +] + .filter(name => process.env[name] != null) + .flatMap(name => ['-e', `${name}=${process.env[name]}`]); +const command = [ + 'run', + '--rm', + '--network=host', + '--ipc=host', + '-v', + `${root}:${root}`, + '-w', + root, + ...env, + image, + 'node', + path.join(root, 'node_modules/@playwright/test/cli.js'), + 'test', + '--config', + 'playwright.interaction.config.ts', + '--project', + 'webkit', + ...args, +]; +console.log(`${engine} ${command.join(' ')}`); +const result = spawnSync(engine, command, {stdio: 'inherit'}); +process.exit(result.status ?? 1); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts new file mode 100644 index 0000000000000..10d41e668c9bf --- /dev/null +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -0,0 +1,168 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file harness.interaction.browser.spec.ts + * @input Fixture pages written inline (no Storybook), real Chromium/WebKit + * @output Proof that each touch instrument delivers real engine input: + * scripted timestamps reach the page; a flick flings natively (Chromium); + * a second finger and a pinch arrive as distinct pointers. + * @position The harness's own test. A component spec trusts these + * instruments; this file is why it may. + */ + +import {expect, test, type Page} from '@playwright/test'; +import { + drag, + flick, + installInputLog, + merge, + pinch, + readInputLog, + tap, + touchDriver, + withFinger, +} from './index'; + +const FIXTURE = ` + +
+
+
${Array.from({length: 200}, (_, i) => `
row ${i}
`).join('')}
`; + +// A real navigation (not setContent, whose document.open() erases the window +// listeners the input log installs). +async function fixture(page: Page) { + await installInputLog(page); + await page.route('http://harness.test/', route => + route.fulfill({contentType: 'text/html', body: FIXTURE}), + ); + await page.goto('http://harness.test/'); +} + +test.describe('touch input', () => { + test('the page receives trusted touches and pointers at the scripted positions and times', async ({ + page, + browserName, + }) => { + await fixture(page); + const driver = await touchDriver(page); + const script = drag( + {x: 100, y: 100}, + {x: 100, y: 220}, + {durationMs: 200, hz: 60, restMs: 120}, + ); + await driver.play(script); + const log = await readInputLog(page); + const moves = log.filter(r => r.type === 'pointermove'); + const down = log.find(r => r.type === 'pointerdown'); + const up = log.find(r => r.type === 'pointerup'); + expect(down?.isTrusted).toBe(true); + expect(down?.pointerType).toBe('touch'); + expect(up, JSON.stringify(log.map(r => r.type))).toBeDefined(); + // (Not `cancelable`: with only passive listeners an engine may dispatch + // touchmove as non-blocking, which reads cancelable false.) + expect( + log.filter(r => r.type === 'touchmove').every(r => r.isTrusted), + ).toBe(true); + expect(moves.at(-1)?.y).toBe(220); + if (browserName === 'chromium') { + // The scripted 16.67 ms spacing arrives exactly, and the 120 ms rest + // before the lift is in the timestamps too. + const deltas = moves + .slice(1) + .map((m, i) => m.timeStamp - moves[i].timeStamp); + for (const delta of deltas) { + expect(delta).toBeCloseTo(1000 / 60, 0); + } + expect( + (up?.timeStamp ?? NaN) - (moves.at(-1)?.timeStamp ?? NaN), + ).toBeCloseTo(120, 0); + } else { + // WebKit stamps the real dispatch time: real-time pacing keeps it close. + const deltas = moves + .slice(1) + .map((m, i) => m.timeStamp - moves[i].timeStamp); + const mean = deltas.reduce((a, b) => a + b, 0) / deltas.length; + expect(mean).toBeGreaterThan(14); + expect(mean).toBeLessThan(20); + } + }); + + test('a flick flings a native scroller, farther for a faster finger', async ({ + page, + browserName, + }) => { + test.skip( + browserName !== 'chromium', + 'injected WebKit touches drive no native scrolling (README: engine limits)', + ); + await fixture(page); + const driver = await touchDriver(page); + const scroller = page.locator('#scroller'); + const distanceFor = async (velocity: number) => { + await scroller.evaluate(el => (el.scrollTop = 0)); + await driver.play( + flick( + {x: 200, y: 680}, + {velocity: {x: 0, y: -velocity}, durationMs: 60, hz: 120}, + ), + ); + await expect + .poll( + async () => { + const a = await scroller.evaluate(el => el.scrollTop); + await page.waitForTimeout(150); + return a === (await scroller.evaluate(el => el.scrollTop)); + }, + {timeout: 8_000}, + ) + .toBe(true); + return scroller.evaluate(el => el.scrollTop); + }; + const slow = await distanceFor(800); + const fast = await distanceFor(2400); + expect(slow).toBeGreaterThan(48); // it coasted past the finger's own travel + expect(fast).toBeGreaterThan(slow * 1.8); + }); + + test('a second finger and a pinch arrive as distinct touch pointers', async ({ + page, + }) => { + await fixture(page); + const driver = await touchDriver(page); + const first = drag( + {x: 80, y: 80}, + {x: 80, y: 200}, + {durationMs: 300, hz: 60}, + ); + const second = withFinger(tap({x: 300, y: 120}, {holdMs: 100}), 1, 120); + await driver.play(merge(first, second)); + const downs = (await readInputLog(page, {clear: true})).filter( + r => r.type === 'pointerdown', + ); + expect(downs).toHaveLength(2); + expect(new Set(downs.map(d => d.pointerId)).size).toBe(2); + + await driver.play( + pinch({ + center: {x: 200, y: 150}, + fromDistance: 60, + toDistance: 200, + durationMs: 200, + }), + ); + const log = await readInputLog(page); + expect(log.filter(r => r.type === 'pointerdown')).toHaveLength(2); + expect( + Math.max( + ...log.filter(r => r.type === 'touchmove').map(r => r.touches ?? 0), + ), + ).toBe(2); + }); +}); diff --git a/internal/interaction-harness/src/index.ts b/internal/interaction-harness/src/index.ts new file mode 100644 index 0000000000000..3216b3c084322 --- /dev/null +++ b/internal/interaction-harness/src/index.ts @@ -0,0 +1,17 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file index.ts + * @input Re-exports the harness modules + * @output The `@astryxdesign/interaction-harness` entry point for Playwright + * specs. Pure helpers are also reachable without Playwright through + * `@astryxdesign/interaction-harness/math` and `/scripts` (unit tests). + * @position Imported only from `*.interaction.browser.spec.ts` files (run by + * playwright.interaction.config.ts) and from this package's own tests. + * + * SYNC: When a module is added, export it here and describe it in README.md. + */ + +export * from './scripts'; +export * from './story'; +export * from './touch'; diff --git a/internal/interaction-harness/src/scripts.test.ts b/internal/interaction-harness/src/scripts.test.ts new file mode 100644 index 0000000000000..b85d4b526de55 --- /dev/null +++ b/internal/interaction-harness/src/scripts.test.ts @@ -0,0 +1,142 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +import {describe, expect, it} from 'vitest'; +import { + drag, + flick, + holdThenDrag, + merge, + path, + pinch, + releaseVelocity, + rotate, + tap, + toTouchEvents, + withFinger, +} from './scripts'; + +describe('touch scripts', () => { + it('tap presses and lifts in place', () => { + expect( + toTouchEvents(tap({x: 5, y: 6}, {holdMs: 80})).map(e => [e.t, e.type]), + ).toEqual([ + [0, 'touchStart'], + [80, 'touchEnd'], + ]); + }); + + it('drag moves at 60 Hz, holds, rests and lifts', () => { + const script = drag( + {x: 0, y: 0}, + {x: 0, y: 100}, + {durationMs: 100, hz: 60, holdMs: 500, restMs: 150}, + ); + const events = toTouchEvents(script); + expect(events[0]).toMatchObject({t: 0, type: 'touchStart'}); + const moves = events.filter(e => e.type === 'touchMove'); + expect(moves).toHaveLength(6); + expect(moves[0].t).toBeCloseTo(516.667, 3); + expect(moves.at(-1)).toMatchObject({ + t: 600, + fingers: [{id: 0, x: 0, y: 100}], + }); + expect(events.at(-1)).toMatchObject({ + t: 750, + type: 'touchEnd', + fingers: [], + }); + // A rest of 100 ms or more releases nothing in motion. + expect(releaseVelocity(script)).toEqual({x: 0, y: 0}); + }); + + it('flick releases at exactly the scripted velocity', () => { + const script = flick( + {x: 10, y: 10}, + {velocity: {x: 0, y: 1500}, durationMs: 80, hz: 120}, + ); + expect(releaseVelocity(script).y).toBeCloseTo(1500, 6); + const last = script.at(-2)!; + expect(last.fingers[0].y).toBeCloseTo(10 + 1500 * 0.08, 6); + // The lift lands a quarter frame after the last move, still moving. + expect(script.at(-1)!.t - last.t).toBeCloseTo(1000 / 120 / 4, 6); + }); + + it('flick follows a velocity profile', () => { + const script = flick( + {x: 0, y: 0}, + {velocity: {x: 1000, y: 0}, durationMs: 100, hz: 100, profile: p => p}, + ); + // Linear ramp: distance = v * T / 2. + expect(script.at(-2)!.fingers[0].x).toBeCloseTo(50, 6); + }); + + it('pinch and rotate keep two fingers about the centre', () => { + const spread = pinch({ + center: {x: 100, y: 100}, + fromDistance: 40, + toDistance: 200, + durationMs: 100, + }); + const end = spread.at(-2)!.fingers; + expect(Math.hypot(end[1].x - end[0].x, end[1].y - end[0].y)).toBeCloseTo( + 200, + 6, + ); + const turn = rotate({ + center: {x: 0, y: 0}, + distance: 100, + fromAngle: 0, + toAngle: 90, + durationMs: 100, + }); + expect(turn.at(-2)!.fingers[1].x).toBeCloseTo(0, 6); + expect(turn.at(-2)!.fingers[1].y).toBeCloseTo(50, 6); + expect(toTouchEvents(spread)[0]).toMatchObject({ + type: 'touchStart', + changed: [{id: 0}, {id: 1}], + }); + }); + + it('merge lands a second finger mid-drag and lifts it first', () => { + const first = drag({x: 0, y: 0}, {x: 0, y: 90}, {durationMs: 300, hz: 10}); + const second = withFinger(tap({x: 50, y: 50}, {holdMs: 100}), 1, 120); + const events = toTouchEvents(merge(first, second)); + const starts = events.filter(e => e.type === 'touchStart'); + expect( + starts.map(e => [e.t, e.changed.map(f => f.id), e.fingers.length]), + ).toEqual([ + [0, [0], 1], + [120, [1], 2], + ]); + const ends = events.filter(e => e.type === 'touchEnd'); + expect(ends[0]).toMatchObject({t: 220, changed: [{id: 1}]}); + expect(ends[0].fingers.map(f => f.id)).toEqual([0]); + expect(ends.at(-1)!.fingers).toEqual([]); + expect(() => merge(first, first)).toThrow(/two scripts/); + }); + + it('path follows any curve; holdThenDrag holds before the first move', () => { + const arc = path( + p => ({x: 100 * Math.cos(p * Math.PI), y: 100 * Math.sin(p * Math.PI)}), + {durationMs: 100, hz: 40}, + ); + const last = arc.at(-2)!.fingers[0]; + expect(last.x).toBeCloseTo(-100, 6); + expect(last.y).toBeCloseTo(0, 6); + expect(arc.at(-1)!.fingers).toEqual([]); + const held = holdThenDrag( + {x: 0, y: 0}, + {x: 30, y: 0}, + {durationMs: 50, hz: 20, holdMs: 500}, + ); + const moves = toTouchEvents(held).filter(e => e.type === 'touchMove'); + expect(moves[0].t).toBeCloseTo(550, 6); + expect(moves.at(-1)!.fingers[0].x).toBeCloseTo(30, 6); + }); + + it('refuses a script that leaves a finger down', () => { + expect(() => + toTouchEvents([{t: 0, fingers: [{id: 0, x: 0, y: 0}]}]), + ).toThrow(/lifted/); + }); +}); diff --git a/internal/interaction-harness/src/scripts.ts b/internal/interaction-harness/src/scripts.ts new file mode 100644 index 0000000000000..111d9e01234aa --- /dev/null +++ b/internal/interaction-harness/src/scripts.ts @@ -0,0 +1,395 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file scripts.ts + * @input Nothing: pure functions + * @output Touch scripts — timed lists of the fingers that are down — and the + * builders that make them: `tap`, `drag`, `flick`, `holdThenDrag`, `pinch`, + * `rotate`, `path`, `withFinger`, `merge`, plus `toTouchEvents`, which turns + * a script into the per-event sequence a touch driver dispatches. + * @position The input half of every interaction proof. A script is data, so a + * test states exactly what the finger does — positions and timestamps — and + * a unit test can check a builder without a browser. + * + * A script is a list of frames ordered by time. Each frame lists every finger + * that is down at that moment; a finger missing from a frame has lifted. The + * last frame is always empty (every finger up). Times are milliseconds from + * the gesture's start; coordinates are CSS px in the viewport. + * + * SYNC: Keep ./scripts.test.ts covering every builder. + */ + +export interface Point { + readonly x: number; + readonly y: number; +} + +export interface Finger extends Point { + /** Stable per finger for the whole gesture. */ + readonly id: number; +} + +export interface TouchFrame { + /** Milliseconds from the start of the gesture. */ + readonly t: number; + /** Every finger that is down at `t`. Empty = all lifted. */ + readonly fingers: readonly Finger[]; +} + +export type TouchScript = readonly TouchFrame[]; + +/** One dispatchable event: what changed at `t`, and the fingers down after it. */ +export interface TouchEventStep { + readonly t: number; + readonly type: 'touchStart' | 'touchMove' | 'touchEnd' | 'touchCancel'; + /** Fingers down after this event (CDP semantics: the full active set). */ + readonly fingers: readonly Finger[]; + /** Fingers this event pressed, moved or released. */ + readonly changed: readonly Finger[]; +} + +const DEFAULT_HZ = 60; + +function frameTimes(durationMs: number, hz: number): number[] { + const step = 1000 / hz; + const count = Math.max(1, Math.round(durationMs / step)); + return Array.from( + {length: count}, + (_, i) => +(((i + 1) * durationMs) / count).toFixed(3), + ); +} + +function lerp(a: Point, b: Point, p: number): Point { + return {x: a.x + (b.x - a.x) * p, y: a.y + (b.y - a.y) * p}; +} + +/** A press and lift in place. */ +export function tap( + at: Point, + {holdMs = 60, id = 0}: {holdMs?: number; id?: number} = {}, +): TouchScript { + return [ + {t: 0, fingers: [{id, ...at}]}, + {t: holdMs, fingers: []}, + ]; +} + +export interface DragOptions { + /** Time from the first move to the last, ms. */ + readonly durationMs: number; + /** Move samples per second (iOS reports touches at 60 Hz, 120 Hz on ProMotion). */ + readonly hz?: number; + /** Rest before the first move (press-and-hold), ms. */ + readonly holdMs?: number; + /** Rest after the last move before the lift, ms: >= 100 makes the release velocity 0. */ + readonly restMs?: number; + /** Progress easing over the move, default linear (constant velocity). */ + readonly easing?: (progress: number) => number; + readonly id?: number; +} + +/** Press at `from`, move to `to`, lift. Constant velocity unless `easing` says otherwise. */ +export function drag( + from: Point, + to: Point, + options: DragOptions, +): TouchScript { + const { + durationMs, + hz = DEFAULT_HZ, + holdMs = 0, + restMs = 0, + easing = (p: number) => p, + id = 0, + } = options; + const frames: TouchFrame[] = [{t: 0, fingers: [{id, ...from}]}]; + for (const t of frameTimes(durationMs, hz)) { + frames.push({ + t: holdMs + t, + fingers: [{id, ...lerp(from, to, easing(t / durationMs))}], + }); + } + // A real finger reports no move while it rests: the lift simply comes later. + const last = frames[frames.length - 1].t; + frames.push({t: last + Math.max(restMs, 1000 / hz / 4), fingers: []}); + return frames; +} + +/** `drag` after a press-and-hold (a long press that turns into a drag). */ +export function holdThenDrag( + from: Point, + to: Point, + options: DragOptions & {readonly holdMs: number}, +): TouchScript { + return drag(from, to, options); +} + +export interface FlickOptions { + /** Velocity at the lift, px/s. */ + readonly velocity: Point; + /** How long the finger moves before the lift, ms. */ + readonly durationMs?: number; + readonly hz?: number; + /** + * Velocity over the move as a fraction of the final velocity (0..1 → 0..∞), + * default constant. `p => p` is a finger accelerating linearly into the lift. + */ + readonly profile?: (progress: number) => number; + readonly id?: number; +} + +/** + * A throw: the finger moves with the given velocity profile and lifts while + * still moving (the lift lands a quarter frame after the last move, as on a + * device), so the release velocity is exactly `velocity`. + */ +export function flick(from: Point, options: FlickOptions): TouchScript { + const { + velocity, + durationMs = 80, + hz = 120, + profile = () => 1, + id = 0, + } = options; + const step = 1000 / hz; + const frames: TouchFrame[] = [{t: 0, fingers: [{id, ...from}]}]; + let x = from.x; + let y = from.y; + let previous = 0; + for (const t of frameTimes(durationMs, hz)) { + // Integrate the profile over (previous, t] with the midpoint rule. + const scale = profile((previous + t) / 2 / durationMs); + x += (velocity.x * scale * (t - previous)) / 1000; + y += (velocity.y * scale * (t - previous)) / 1000; + previous = t; + frames.push({t, fingers: [{id, x, y}]}); + } + frames.push({t: previous + step / 4, fingers: []}); + return frames; +} + +/** Any single-finger path: `at(progress)` for progress 0..1 over `durationMs`. */ +export function path( + at: (progress: number) => Point, + { + durationMs, + hz = DEFAULT_HZ, + holdMs = 0, + restMs = 0, + id = 0, + }: Omit, +): TouchScript { + const frames: TouchFrame[] = [{t: 0, fingers: [{id, ...at(0)}]}]; + for (const t of frameTimes(durationMs, hz)) { + frames.push({t: holdMs + t, fingers: [{id, ...at(t / durationMs)}]}); + } + frames.push({ + t: frames[frames.length - 1].t + Math.max(restMs, 1000 / hz / 4), + fingers: [], + }); + return frames; +} + +export interface PinchOptions { + readonly center: Point; + /** Finger separation at the start and at the end, px. */ + readonly fromDistance: number; + readonly toDistance: number; + /** Angle of the line between the fingers, degrees (0 = horizontal). */ + readonly fromAngle?: number; + readonly toAngle?: number; + readonly durationMs: number; + readonly hz?: number; + readonly restMs?: number; +} + +/** Two fingers (ids 0 and 1) pressed together, spread or pinched and turned, lifted together. */ +export function pinch(options: PinchOptions): TouchScript { + const { + center, + fromDistance, + toDistance, + fromAngle = 0, + toAngle = fromAngle, + durationMs, + hz = DEFAULT_HZ, + restMs = 0, + } = options; + const at = (p: number): Finger[] => { + const distance = fromDistance + (toDistance - fromDistance) * p; + const angle = ((fromAngle + (toAngle - fromAngle) * p) * Math.PI) / 180; + const dx = (Math.cos(angle) * distance) / 2; + const dy = (Math.sin(angle) * distance) / 2; + return [ + {id: 0, x: center.x - dx, y: center.y - dy}, + {id: 1, x: center.x + dx, y: center.y + dy}, + ]; + }; + const frames: TouchFrame[] = [{t: 0, fingers: at(0)}]; + for (const t of frameTimes(durationMs, hz)) { + frames.push({t, fingers: at(t / durationMs)}); + } + frames.push({t: durationMs + Math.max(restMs, 1000 / hz / 4), fingers: []}); + return frames; +} + +/** Two fingers turning around `center` at a fixed separation. */ +export function rotate( + options: Omit & { + readonly distance: number; + }, +): TouchScript { + return pinch({ + ...options, + fromDistance: options.distance, + toDistance: options.distance, + }); +} + +/** Re-labels a single-finger script's finger, and shifts it in time. */ +export function withFinger( + script: TouchScript, + id: number, + delayMs = 0, +): TouchScript { + return script.map(frame => ({ + t: frame.t + delayMs, + fingers: frame.fingers.map(f => ({...f, id})), + })); +} + +/** + * Plays several scripts at once (each with its own finger ids): at every time + * any of them changes, each finger is where its own script last put it. + * Used for a second finger that lands mid-gesture. + */ +export function merge(...scripts: TouchScript[]): TouchScript { + const times = [...new Set(scripts.flatMap(s => s.map(f => f.t)))].sort( + (a, b) => a - b, + ); + const stateAt = (script: TouchScript, t: number): readonly Finger[] => { + let current: readonly Finger[] = []; + for (const frame of script) { + if (frame.t > t) { + break; + } + current = frame.fingers; + } + return current; + }; + const owner = new Map(); + scripts.forEach((script, index) => { + for (const frame of script) { + for (const f of frame.fingers) { + if ((owner.get(f.id) ?? index) !== index) { + throw new Error(`merge: finger ${f.id} is used by two scripts`); + } + owner.set(f.id, index); + } + } + }); + return times.map(t => ({t, fingers: scripts.flatMap(s => stateAt(s, t))})); +} + +/** + * Turns frames into events: a finger that appears is a `touchStart`, one that + * disappears a `touchEnd`, one that moved a `touchMove`. A frame that both + * moves some fingers and lifts others yields the move first, then the lift. + * Unchanged frames yield nothing (a resting finger reports nothing). + */ +export function toTouchEvents(script: TouchScript): TouchEventStep[] { + const events: TouchEventStep[] = []; + let down = new Map(); + for (const frame of script) { + const next = new Map(frame.fingers.map(f => [f.id, f])); + const pressed = frame.fingers.filter(f => !down.has(f.id)); + const released = [...down.values()].filter(f => !next.has(f.id)); + const moved = frame.fingers.filter(f => { + const before = down.get(f.id); + return before != null && (before.x !== f.x || before.y !== f.y); + }); + if (moved.length > 0) { + const after = [...down.values()].map(f => next.get(f.id) ?? f); + events.push({ + t: frame.t, + type: 'touchMove', + fingers: after, + changed: moved, + }); + down = new Map(after.map(f => [f.id, f])); + } + if (released.length > 0) { + const remaining = [...down.values()].filter(f => next.has(f.id)); + events.push({ + t: frame.t, + type: 'touchEnd', + fingers: remaining, + changed: released.map(f => down.get(f.id) ?? f), + }); + down = new Map(remaining.map(f => [f.id, f])); + } + if (pressed.length > 0) { + const all = [...down.values(), ...pressed]; + events.push({ + t: frame.t, + type: 'touchStart', + fingers: all, + changed: pressed, + }); + down = new Map(all.map(f => [f.id, f])); + } + } + if (down.size > 0) { + throw new Error( + 'toTouchEvents: a script must end with every finger lifted', + ); + } + return events; +} + +/** + * The velocity (px/s) of finger `id` at its lift: the slope over the last + * `windowMs` of moves, or zero when the finger rested `windowMs` or longer + * before lifting (a finger that rested released nothing in motion). + */ +export function releaseVelocity( + script: TouchScript, + id = 0, + windowMs = 100, +): Point { + const samples = script.flatMap(frame => + frame.fingers + .filter(f => f.id === id) + .map(f => ({t: frame.t, x: f.x, y: f.y})), + ); + if (samples.length < 2) { + return {x: 0, y: 0}; + } + const lift = script.find( + frame => frame.t > samples[0].t && !frame.fingers.some(f => f.id === id), + ); + // The last sample that actually moved: a resting finger reports nothing. + let lastMove = samples.length - 1; + while ( + lastMove > 0 && + samples[lastMove].x === samples[lastMove - 1].x && + samples[lastMove].y === samples[lastMove - 1].y + ) { + lastMove -= 1; + } + const last = samples[lastMove]; + if (lift != null && lift.t - last.t >= windowMs) { + return {x: 0, y: 0}; + } + const recent = samples + .slice(0, lastMove + 1) + .filter(s => s.t >= last.t - windowMs); + const first = recent[0]; + const dt = last.t - first.t; + return dt <= 0 + ? {x: 0, y: 0} + : { + x: ((last.x - first.x) / dt) * 1000, + y: ((last.y - first.y) / dt) * 1000, + }; +} diff --git a/internal/interaction-harness/src/story.ts b/internal/interaction-harness/src/story.ts new file mode 100644 index 0000000000000..9e4323436ff27 --- /dev/null +++ b/internal/interaction-harness/src/story.ts @@ -0,0 +1,103 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file story.ts + * @input Uses a Playwright `Page` and test object, and the a11y-spec package's + * static server over a built Storybook + * @output `phone` (the touch-phone browser context every interaction proof + * runs in), `withStorybook` (serve the built Storybook for a spec file), and + * `openStory` (load one story and wait until it has rendered). + * @position Stories are the reproduction path, as for every other browser + * check in this repository: a scenario used for verification is checked in, + * not built in a one-off page. + * + * The phone is an iPhone 17 Pro in Safari: 402 × 874 pt screen, 402 × 714 pt + * visible page with the toolbars shown, 3 device pixels per point, touch, + * mobile viewport rules. Measured on the iOS 26.2 simulator + * (`innerWidth`/`innerHeight`, `screen`, `devicePixelRatio`). + * + * SYNC: Keep `phone` equal to the projects in playwright.interaction.config.ts. + */ + +import type {Page, TestType} from '@playwright/test'; +import { + DEFAULT_STORYBOOK_DIR, + serveStorybook, + type StaticServer, +} from '@astryxdesign/a11y-spec/storybook'; + +export const phone = { + viewport: {width: 402, height: 714}, + screen: {width: 402, height: 874}, + deviceScaleFactor: 3, + isMobile: true, + hasTouch: true, +} as const; + +/** + * Serves the built Storybook for the spec file's tests (`ASTRYX_STORYBOOK_DIR` + * overrides the default `apps/storybook/dist`). Returns its origin getter. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- any fixture set +export function withStorybook(test: TestType): () => string { + let server: StaticServer | undefined; + test.beforeAll(async () => { + server = await serveStorybook( + process.env.ASTRYX_STORYBOOK_DIR ?? DEFAULT_STORYBOOK_DIR, + ); + }); + test.afterAll(async () => { + await server?.close(); + }); + return () => { + if (server == null) { + throw new Error( + 'withStorybook: the Storybook server is not running (call inside a test)', + ); + } + return server.origin; + }; +} + +/** Loads a story in the page and waits for its first render, its fonts and two frames. */ +export async function openStory( + page: Page, + origin: string, + storyId: string, + args?: Readonly>, +): Promise { + const params = new URLSearchParams({id: storyId, viewMode: 'story'}); + if (args != null) { + params.set( + 'args', + Object.entries(args) + .map(([k, v]) => `${k}:${v}`) + .join(';'), + ); + } + await page.goto(`${origin}/iframe.html?${params.toString()}`); + await page + .locator('#storybook-root > *') + .first() + .waitFor({state: 'attached'}); + await page.evaluate(async () => { + await document.fonts.ready; + await new Promise(resolve => + requestAnimationFrame(() => requestAnimationFrame(() => resolve())), + ); + }); +} + +/** Resolves after `count` animation frames have been presented in the page. */ +export async function frames(page: Page, count = 2): Promise { + await page.evaluate( + n => + new Promise(resolve => { + let left = n; + const tick = () => + --left <= 0 ? resolve() : requestAnimationFrame(tick); + requestAnimationFrame(tick); + }), + count, + ); +} diff --git a/internal/interaction-harness/src/touch.ts b/internal/interaction-harness/src/touch.ts new file mode 100644 index 0000000000000..56cdf8aa50289 --- /dev/null +++ b/internal/interaction-harness/src/touch.ts @@ -0,0 +1,295 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file touch.ts + * @input Uses a Playwright `Page` (Chromium or WebKit) and a touch script from + * ./scripts + * @output `touchDriver(page)` — plays touch scripts as REAL browser touch input + * — and `installInputLog` / `readInputLog`, the page-side record of what the + * page actually received. + * @position The input half of every interaction proof. Untrusted DOM events + * are not input: they never start native scrolling, never claim a gesture, + * and a component can tell them apart (`isTrusted`). Both drivers here go + * through the engine's own input pipeline instead. + * + * Chromium: CDP `Input.dispatchTouchEvent` with an explicit `timestamp` per + * event. The page's `event.timeStamp` IS that timestamp, whatever the + * dispatch latency, so a velocity profile reaches the page exactly. The + * events go through the compositor's gesture detection: native scrolling, + * flings (velocity from those timestamps), `touch-action`, pointer events + * and `pointercancel` all behave as for a finger. + * + * WebKit (Playwright's Linux build): the engine's own inspector command + * `Input.dispatchTouchEvent` on the page proxy. Touches and the pointer + * events derived from them are trusted and cancelable, multi-touch works. + * Limits, measured with WebKit 26.5: no timestamp parameter (the page sees + * the real dispatch time, so velocity comes from real-time pacing, ±1 ms); + * integer coordinates only; no native scrolling, fling or pinch-zoom from + * these touches; a `touchstart` lists only the fingers that landed in it + * (pointer events are exact); no click synthesised from a raw touch sequence (use + * `tap`, which dispatches WebKit's tap gesture and does produce the + * compatibility mouse events and click). The command is reached through + * Playwright's in-process server object, so it needs a locally launched + * browser (the default), not a remote `connect()`. + * + * SYNC: Keep the engine limits above equal to the README's table and to the + * harness self-test (../src/harness.interaction.browser.spec.ts). + */ + +import type {CDPSession, Page} from '@playwright/test'; +import { + toTouchEvents, + type Finger, + type Point, + type TouchEventStep, + type TouchScript, +} from './scripts'; + +export type Engine = 'chromium' | 'webkit'; + +export interface PlayOptions { + /** + * Pace dispatch on the wall clock so each event leaves at its scripted time + * (default true). Required in WebKit for a velocity to mean anything; in + * Chromium the page sees the scripted timestamps either way, but native + * behaviors that run on the compositor clock (a fling) want real pacing. + */ + readonly realtime?: boolean; + /** Called after each event is dispatched, before the next one (storyboards). */ + readonly onEvent?: ( + step: TouchEventStep, + index: number, + ) => Promise | void; +} + +export interface PlayedEvent extends TouchEventStep { + /** Wall-clock milliseconds (epoch) at which the event was dispatched. */ + readonly dispatchedAt: number; + /** The timestamp handed to the engine, epoch ms (Chromium only). */ + readonly timestamp: number | null; +} + +export interface TouchDriver { + readonly engine: Engine; + /** True when the page's `event.timeStamp` follows the script exactly. */ + readonly scriptedTimestamps: boolean; + /** True when these touches can scroll, fling and pinch-zoom natively. */ + readonly drivesNativeGestures: boolean; + /** Plays a script; resolves when its last event has been dispatched. */ + play(script: TouchScript, options?: PlayOptions): Promise; + /** A tap through the engine's tap path (WebKit synthesises click only from this). */ + tap(at: Point): Promise; +} + +const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); + +/** Waits until `performance.now()` reaches `deadline` with ~0.1 ms precision. */ +async function until(deadline: number): Promise { + const remaining = deadline - performance.now(); + if (remaining > 3) { + await sleep(remaining - 2); + } + while (performance.now() < deadline) { + // spin: setTimeout alone is ±1-4 ms late + } +} + +async function playSteps( + script: TouchScript, + options: PlayOptions, + dispatch: (step: TouchEventStep, timestamp: number) => Promise, + stampFor: (t: number) => number, +): Promise { + const {realtime = true, onEvent} = options; + const steps = toTouchEvents(script); + const played: PlayedEvent[] = []; + const origin = performance.now(); + for (const [index, step] of steps.entries()) { + if (realtime) { + await until(origin + step.t); + } + const timestamp = stampFor(step.t); + const dispatchedAt = Date.now(); + await dispatch(step, timestamp); + played.push({...step, dispatchedAt, timestamp}); + await onEvent?.(step, index); + } + return played; +} + +async function chromiumDriver(page: Page): Promise { + const cdp: CDPSession = await page.context().newCDPSession(page); + const touchPoints = (fingers: readonly Finger[]) => + fingers.map(f => ({ + x: f.x, + y: f.y, + id: f.id, + radiusX: 1, + radiusY: 1, + force: 1, + })); + return { + engine: 'chromium', + scriptedTimestamps: true, + drivesNativeGestures: true, + async play(script, options = {}) { + // Stamps follow the wall clock the page's own clock is on: with real-time + // pacing an event's stamp and its dispatch agree to ~0.1 ms. + const base = Date.now(); + return playSteps( + script, + options, + async (step, timestamp) => { + // CDP diffs each event's active points against the previous event's: + // a missing point is released, a new one pressed. touchEnd must carry + // no points, so lifting one finger of several is a touchMove that + // lists the fingers still down. + const partialLift = + step.type === 'touchEnd' && step.fingers.length > 0; + await cdp.send('Input.dispatchTouchEvent', { + type: partialLift ? 'touchMove' : step.type, + touchPoints: + step.type === 'touchEnd' && !partialLift + ? [] + : touchPoints(step.fingers), + timestamp: timestamp / 1000, + }); + }, + t => base + t, + ); + }, + async tap(at) { + await page.touchscreen.tap(at.x, at.y); + }, + }; +} + +interface WebKitPageProxySession { + send(method: string, params?: Record): Promise; + on(event: string, listener: (payload: unknown) => void): unknown; + off?(event: string, listener: (payload: unknown) => void): unknown; +} + +/** + * Playwright's server-side WebKit page session, reached through the in-process + * connection. Exported for helpers that use WebKit's own inspector protocol. + */ +export function webkitPageSession(page: Page): WebKitPageProxySession { + const connection = ( + page as unknown as {_connection?: {toImpl?: (o: unknown) => unknown}} + )._connection; + const impl = connection?.toImpl?.(page) as + {delegate?: {_pageProxySession?: WebKitPageProxySession}} | undefined; + const session = impl?.delegate?._pageProxySession; + if (session == null) { + throw new Error( + "WebKit touch input needs Playwright's in-process server (a locally launched browser); this page has none.", + ); + } + return session; +} + +async function webkitDriver(page: Page): Promise { + const session = webkitPageSession(page); + // WebKit's command takes integer CSS px and applies the event's phase to + // every point it lists: a start lists only the fingers that land (listing a + // finger already down presses it again), an end only the fingers that lift + // (an empty list releases nothing), a move every finger down. + const points = (fingers: readonly Finger[]) => + fingers.map(f => ({x: Math.round(f.x), y: Math.round(f.y), id: f.id})); + return { + engine: 'webkit', + scriptedTimestamps: false, + drivesNativeGestures: false, + async play(script, options = {}) { + return playSteps( + script, + options, + async step => { + await session.send('Input.dispatchTouchEvent', { + type: step.type, + touchPoints: points( + step.type === 'touchMove' ? step.fingers : step.changed, + ), + }); + }, + () => Date.now(), + ).then(played => played.map(p => ({...p, timestamp: null}))); + }, + async tap(at) { + await page.touchscreen.tap(Math.round(at.x), Math.round(at.y)); + }, + }; +} + +/** A touch driver for the page's engine. Firefox is not supported (no touch input path). */ +export async function touchDriver(page: Page): Promise { + const name = page.context().browser()?.browserType().name(); + if (name === 'chromium') { + return chromiumDriver(page); + } + if (name === 'webkit') { + return webkitDriver(page); + } + throw new Error( + `touchDriver: no real touch input path for ${name ?? 'this browser'}`, + ); +} + +export interface InputRecord { + readonly type: string; + /** The event's own `timeStamp`, ms since the page's time origin. */ + readonly timeStamp: number; + readonly isTrusted: boolean; + readonly cancelable: boolean; + readonly defaultPrevented: boolean; + readonly pointerId?: number; + readonly pointerType?: string; + readonly x?: number; + readonly y?: number; + readonly touches?: number; + readonly target: string; +} + +const INPUT_LOG_SCRIPT = `(() => { + if (window.__harnessInput) return; + const log = []; + window.__harnessInput = log; + const describe = node => node instanceof Element + ? node.tagName.toLowerCase() + (node.id ? '#' + node.id : '') + (node.getAttribute('aria-label') ? '[' + node.getAttribute('aria-label') + ']' : '') + : String(node && node.nodeName); + const types = ['pointerdown', 'pointermove', 'pointerup', 'pointercancel', 'touchstart', 'touchmove', + 'touchend', 'touchcancel', 'click', 'contextmenu', 'gotpointercapture', 'lostpointercapture']; + for (const type of types) { + // Bubble phase on window, so defaultPrevented reflects the page's own handlers. + window.addEventListener(type, event => { + const record = {type, timeStamp: event.timeStamp, isTrusted: event.isTrusted, cancelable: event.cancelable, + defaultPrevented: event.defaultPrevented, target: describe(event.target)}; + if ('pointerId' in event) Object.assign(record, {pointerId: event.pointerId, pointerType: event.pointerType, x: event.clientX, y: event.clientY}); + if (event.touches) Object.assign(record, {touches: event.touches.length, x: event.changedTouches[0] && event.changedTouches[0].clientX, y: event.changedTouches[0] && event.changedTouches[0].clientY}); + log.push(record); + }, {passive: true}); + } +})()`; + +/** Records every pointer/touch/click the page receives (install before navigation). */ +export async function installInputLog(page: Page): Promise { + await page.addInitScript(INPUT_LOG_SCRIPT); +} + +/** What the page received since navigation (or since the last `clear`). */ +export async function readInputLog( + page: Page, + {clear = false}: {clear?: boolean} = {}, +): Promise { + return page.evaluate(shouldClear => { + const log = + (window as unknown as {__harnessInput?: InputRecord[]}).__harnessInput ?? + []; + const copy = log.slice(); + if (shouldClear) { + log.length = 0; + } + return copy; + }, clear); +} diff --git a/internal/interaction-harness/tsconfig.json b/internal/interaction-harness/tsconfig.json new file mode 100644 index 0000000000000..596e2cf729a9f --- /dev/null +++ b/internal/interaction-harness/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src"] +} diff --git a/package.json b/package.json index 9052c34ad02d0..a4ca9f2ae830e 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "test:coverage": "vitest --coverage", "test:a11y-contract": "playwright test --config playwright.config.ts --project chromium", "test:docsite-browser": "playwright test --config apps/docsite/playwright.config.ts", + "test:interaction": "playwright test --config playwright.interaction.config.ts", "check:package-boundaries": "node scripts/check-package-boundaries.js", "check:repo": "pnpm check:visual-owner && pnpm check:sync && pnpm check:neutral-palette-refs && pnpm check:stone-palette-refs && pnpm check:gothic-palette-refs && pnpm check:matcha-palette-refs && pnpm check:butter-palette-refs && pnpm check:y2k-palette-refs && pnpm check:data-tokens && pnpm check:token-docs && pnpm check:knowledge && pnpm check:package-boundaries && pnpm check:changesets && pnpm check:demo-media && pnpm check:executable-bits && pnpm check:cli-structure && pnpm check:use-client && pnpm check:portable-scripts && pnpm check:i18n-runtime && pnpm check:i18n-catalog && pnpm check:cldr-weekdays && pnpm check:fixtures && pnpm check:setup-contract", "check:visual-owner": "node scripts/check-visual-workflow-owner.mjs", diff --git a/playwright.interaction.config.ts b/playwright.interaction.config.ts new file mode 100644 index 0000000000000..357a5dc3b7642 --- /dev/null +++ b/playwright.interaction.config.ts @@ -0,0 +1,77 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file playwright.interaction.config.ts + * @input Uses @playwright/test + * @output Touch interaction proofs on a phone-sized touch screen, in Chromium + * and WebKit. + * @position Run with `pnpm test:interaction`. Separate from `pnpm test` (no + * browser needed there) and from playwright.config.ts (accessibility + * contracts on desktop profiles). + * + * Specs are named `*.interaction.browser.spec.ts` and read stories from a + * built Storybook, like the accessibility specs: + * + * pnpm storybook:build + * pnpm exec playwright install chromium webkit + * pnpm test:interaction --project chromium + * pnpm test:interaction harness.interaction.browser.spec.ts + * + * A spec that needs Chromium-only evidence (CDP tracing, scripted touch + * timestamps) skips itself in WebKit with the reason; see the harness README. + * Where WebKit cannot launch on the host (missing system libraries on some + * Linux distributions), internal/interaction-harness/scripts/webkit-container.mjs + * runs the WebKit project inside the official Playwright image. + * + * SYNC: Keep the phone profile here equal to `phone` in + * internal/interaction-harness/src/story.ts. + */ + +import {defineConfig} from '@playwright/test'; + +// The phone: an iPhone 17 Pro in Safari (402 × 874 pt screen, 402 × 714 pt +// page with the toolbars shown, 3 device px per point), measured on the iOS +// 26.2 simulator. Inline rather than imported: the config loads as CommonJS and +// the harness package is ESM. +const phone = { + viewport: {width: 402, height: 714}, + screen: {width: 402, height: 874}, + deviceScaleFactor: 3, + isMobile: true, + hasTouch: true, +} as const; + +const isCI = (process.env.CI ?? '') !== ''; + +function workers(): number { + const raw = process.env.INTERACTION_WORKERS; + const count = raw ? Number(raw) : 1; + if (!Number.isInteger(count) || count < 1) { + throw new Error( + `INTERACTION_WORKERS must be a positive integer, got "${raw}"`, + ); + } + return count; +} + +export default defineConfig({ + testDir: '.', + testMatch: [ + 'internal/interaction-harness/src/**/*.interaction.browser.spec.ts', + 'packages/*/src/**/*.interaction.browser.spec.ts', + ], + // Playwright wipes its output directory per run; evidence (storyboards, + // curve traces) goes to test-results/interaction-evidence instead. + outputDir: 'test-results/interaction-output', + // Frame evidence measures timing: one worker by default, so a proof never + // shares the CPU with another browser. + workers: workers(), + fullyParallel: false, + timeout: 60_000, + forbidOnly: isCI, + reporter: isCI ? [['list'], ['github']] : [['list']], + projects: [ + {name: 'chromium', use: {browserName: 'chromium', ...phone}}, + {name: 'webkit', use: {browserName: 'webkit', ...phone}}, + ], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2e4d844ccbfb..c93bc6ef3838a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -581,6 +581,21 @@ importers: specifier: ^8.62.1 version: 8.62.1(eslint@10.6.0(jiti@2.7.0))(typescript@6.0.3) + internal/interaction-harness: + devDependencies: + '@astryxdesign/a11y-spec': + specifier: workspace:* + version: link:../a11y-spec + '@playwright/test': + specifier: ^1.61.1 + version: 1.61.1 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vitest: + specifier: ^4.1.11 + version: 4.1.11(@types/node@25.9.4)(@vitest/coverage-v8@4.1.11)(happy-dom@20.11.1)(jsdom@27.4.0)(vite@8.1.3(@types/node@25.9.4)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.0)(yaml@2.9.0)) + internal/test-utils: dependencies: '@testing-library/jest-dom': From 6c39f1425a58b4693bf944d170732c5271e00017 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Fri, 23 Oct 2026 20:23:41 -0700 Subject: [PATCH 002/699] fix(Toast): a quick flick dismisses a toast of any height A release toward the edge dismissed past max(40% of the card's height, 48 px), or as a flick only when it had also travelled 48 px. On any toast up to 120 px tall the two floors are the same distance, so a flick could never dismiss a one-line toast. The flick now needs no travel of its own beyond the slop the swipe already passed. The flick's speed was also timed with Date.now() in the handlers. Browsers deliver touch and pointer moves on the next frame, so the handlers trail the finger by up to two frames (measured in Chromium: a 33 ms flick handled over 55 ms), which halved the speed of a short flick. It is now timed by the input events' own timeStamp. --- .changeset/toast-flick-any-height.md | 7 + .../core/src/Toast/ToastViewport.test.tsx | 253 ++++++++++++++++-- packages/core/src/Toast/useToastGesture.ts | 39 ++- 3 files changed, 263 insertions(+), 36 deletions(-) create mode 100644 .changeset/toast-flick-any-height.md diff --git a/.changeset/toast-flick-any-height.md b/.changeset/toast-flick-any-height.md new file mode 100644 index 0000000000000..d9629239f427b --- /dev/null +++ b/.changeset/toast-flick-any-height.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/core': patch +--- + +[fix] A quick flick toward a toast's edge now dismisses it whatever its height. Before, a flick also had to travel 48 px (the distance that already dismisses a one-line toast), so on any toast up to 120 px tall a flick could only snap back; and its speed was timed by when the browser handled the touches rather than when they happened, which in browsers that deliver moves on the next frame made a quick flick look up to twice as slow. + +@vjeux diff --git a/packages/core/src/Toast/ToastViewport.test.tsx b/packages/core/src/Toast/ToastViewport.test.tsx index 4f6ea0be722c8..5f7f9ca59f405 100644 --- a/packages/core/src/Toast/ToastViewport.test.tsx +++ b/packages/core/src/Toast/ToastViewport.test.tsx @@ -1051,37 +1051,238 @@ describe('Toast swipe dismissal', () => { } }); - it('snaps back without dismissing after a short drag', () => { - const {visualToast, onHide} = renderSwipeToast(); + // A one-line toast is 56 px tall: 40% of it is below the 48 px distance + // floor, so a short swipe on it can only be dismissed as a flick. + function mockClock(start = 1_000) { + let time = start; + const spy = vi.spyOn(Date, 'now').mockImplementation(() => time); + return { + advance(ms: number) { + time += ms; + }, + restore() { + spy.mockRestore(); + }, + }; + } - act(() => { - fireEvent.pointerDown(visualToast, { - pointerId: 7, - pointerType: 'pen', - button: 0, - clientX: 0, - clientY: 0, + it('dismisses a quick pen flick on a one-line toast', () => { + const clock = mockClock(); + try { + const {visualToast, onHide} = renderSwipeToast( + SWIPE_TOAST, + 'bottomEnd', + 'Swipe toast', + 56, + ); + + act(() => { + fireEvent.pointerDown(visualToast, { + pointerId: 9, + pointerType: 'pen', + button: 0, + clientX: 0, + clientY: 0, + }); }); - fireEvent.pointerMove(visualToast, { - pointerId: 7, - pointerType: 'pen', - clientX: 0, - clientY: 24, + clock.advance(30); + act(() => { + fireEvent.pointerMove(visualToast, { + pointerId: 9, + pointerType: 'pen', + clientX: 0, + clientY: 40, + }); + fireEvent.pointerUp(visualToast, { + pointerId: 9, + pointerType: 'pen', + clientX: 0, + clientY: 40, + }); }); - fireEvent.pointerUp(visualToast, { - pointerId: 7, - pointerType: 'pen', - clientX: 0, - clientY: 24, + + expect(onHide).toHaveBeenCalledWith('manual'); + expect(onHide).toHaveBeenCalledTimes(1); + } finally { + clock.restore(); + } + }); + + it('dismisses a quick touch flick on a one-line toast at the top', () => { + const clock = mockClock(); + try { + const {visualToast, onHide} = renderSwipeToast( + SWIPE_TOAST, + 'topEnd', + 'Swipe toast', + 56, + ); + const touchEvent = ( + type: 'touchstart' | 'touchmove' | 'touchend', + clientY: number, + ) => { + const event = new Event(type, {bubbles: true, cancelable: true}); + const touch = {identifier: 32, clientX: 10, clientY}; + Object.defineProperty(event, 'touches', { + value: type === 'touchend' ? [] : [touch], + }); + Object.defineProperty(event, 'changedTouches', {value: [touch]}); + visualToast.dispatchEvent(event); + return event; + }; + + act(() => { + touchEvent('touchstart', 100); + }); + clock.advance(15); + let move: Event; + act(() => { + move = touchEvent('touchmove', 80); + }); + clock.advance(15); + act(() => { + touchEvent('touchmove', 60); + touchEvent('touchend', 60); }); - }); - expect(onHide).not.toHaveBeenCalled(); - expect(visualToast.style.getPropertyValue('--_toast-swipe-y')).toBe(''); - expect(visualToast.style.getPropertyValue('--_toast-swipe-opacity')).toBe( - '', - ); - expect(visualToast.style.getPropertyValue('--_toast-swipe-scale')).toBe(''); + expect(move!.defaultPrevented).toBe(true); + expect(onHide).toHaveBeenCalledWith('manual'); + expect(onHide).toHaveBeenCalledTimes(1); + } finally { + clock.restore(); + } + }); + + it('times a flick by its input events, not by when the handlers run', () => { + // Browsers deliver moves on the next frame, so handlers can run well + // after the finger moved: here the whole 40 px flick took 30 ms, but its + // moves and lift are only handled 60 ms after the touch landed. + const clock = mockClock(); + try { + const {visualToast, onHide} = renderSwipeToast( + SWIPE_TOAST, + 'topEnd', + 'Swipe toast', + 56, + ); + const touchEvent = ( + type: 'touchstart' | 'touchmove' | 'touchend', + clientY: number, + ) => { + const event = new Event(type, {bubbles: true, cancelable: true}); + const touch = {identifier: 33, clientX: 10, clientY}; + Object.defineProperty(event, 'touches', { + value: type === 'touchend' ? [] : [touch], + }); + Object.defineProperty(event, 'changedTouches', {value: [touch]}); + return event; + }; + + act(() => { + visualToast.dispatchEvent(touchEvent('touchstart', 100)); + }); + clock.advance(15); + const firstMove = touchEvent('touchmove', 80); + clock.advance(15); + const secondMove = touchEvent('touchmove', 60); + const lift = touchEvent('touchend', 60); + clock.advance(30); + act(() => { + visualToast.dispatchEvent(firstMove); + visualToast.dispatchEvent(secondMove); + visualToast.dispatchEvent(lift); + }); + + expect(onHide).toHaveBeenCalledWith('manual'); + } finally { + clock.restore(); + } + }); + + it('snaps back after a slow short drag on a one-line toast', () => { + const clock = mockClock(); + try { + const {visualToast, onHide} = renderSwipeToast( + SWIPE_TOAST, + 'bottomEnd', + 'Swipe toast', + 56, + ); + + act(() => { + fireEvent.pointerDown(visualToast, { + pointerId: 10, + pointerType: 'pen', + button: 0, + clientX: 0, + clientY: 0, + }); + }); + clock.advance(400); + act(() => { + fireEvent.pointerMove(visualToast, { + pointerId: 10, + pointerType: 'pen', + clientX: 0, + clientY: 40, + }); + fireEvent.pointerUp(visualToast, { + pointerId: 10, + pointerType: 'pen', + clientX: 0, + clientY: 40, + }); + }); + + expect(onHide).not.toHaveBeenCalled(); + expect(visualToast.style.getPropertyValue('--_toast-swipe-y')).toBe(''); + } finally { + clock.restore(); + } + }); + + it('snaps back without dismissing after a short drag', () => { + // A slow drag: 24 px over 400 ms is far below flick speed. + const clock = mockClock(); + try { + const {visualToast, onHide} = renderSwipeToast(); + + act(() => { + fireEvent.pointerDown(visualToast, { + pointerId: 7, + pointerType: 'pen', + button: 0, + clientX: 0, + clientY: 0, + }); + }); + clock.advance(400); + act(() => { + fireEvent.pointerMove(visualToast, { + pointerId: 7, + pointerType: 'pen', + clientX: 0, + clientY: 24, + }); + fireEvent.pointerUp(visualToast, { + pointerId: 7, + pointerType: 'pen', + clientX: 0, + clientY: 24, + }); + }); + + expect(onHide).not.toHaveBeenCalled(); + expect(visualToast.style.getPropertyValue('--_toast-swipe-y')).toBe(''); + expect(visualToast.style.getPropertyValue('--_toast-swipe-opacity')).toBe( + '', + ); + expect(visualToast.style.getPropertyValue('--_toast-swipe-scale')).toBe( + '', + ); + } finally { + clock.restore(); + } }); it('fades and subtly scales only after accepted vertical edge swipe intent', () => { diff --git a/packages/core/src/Toast/useToastGesture.ts b/packages/core/src/Toast/useToastGesture.ts index d0b0167555d06..e08894cd5f338 100644 --- a/packages/core/src/Toast/useToastGesture.ts +++ b/packages/core/src/Toast/useToastGesture.ts @@ -8,6 +8,12 @@ * dismissal callback, enabled state, and interactive-descendant policy * @output Returns the Toast card root ref and touch/pen gesture bindings * @position Internal Toast gesture unit; attached to the Toast card root + * + * A release toward the edge dismisses when it travelled past + * max(40% of the card's height, 48 px), or when it was a flick: faster than + * 1.2 px/ms on average, at any travel, timed by the input events themselves. + * Without that, a one-line toast (40% of which is below the 48 px floor) + * could only be dragged away, never flicked. */ import {useCallback, useEffect, useRef} from 'react'; @@ -15,7 +21,9 @@ import type {PointerEvent as ReactPointerEvent, RefObject} from 'react'; const DRAG_PROMOTION_SLOP = 8; const SWIPE_DISMISS_RATIO = 0.4; -const FLICK_MIN_DISTANCE = 48; +/** The dismiss distance of a card too short for its height ratio to reach it. */ +const SWIPE_DISMISS_MIN_DISTANCE = 48; +/** Average speed, px/ms, at which a release dismisses whatever its travel. */ const FLICK_VELOCITY = 1.2; const VERTICAL_INTENT_RATIO = 1.2; const SWIPE_EXIT_DISTANCE = '120%'; @@ -29,6 +37,12 @@ interface GesturePoint { pointerId: number; clientX: number; clientY: number; + /** + * The input event's `timeStamp`. Browsers deliver moves on the next frame, + * so the time a handler runs can trail the finger by a frame or two; the + * event's own time is when the finger was there. + */ + time: number; } interface GestureState { @@ -132,14 +146,14 @@ export function useToastGesture({ pointerId: point.pointerId, startX: point.clientX, startY: point.clientY, - startTime: Date.now(), + startTime: point.time, direction, intent: 'pending', pausedTimer, surfaceSize, dismissThreshold: Math.max( surfaceSize * SWIPE_DISMISS_RATIO, - FLICK_MIN_DISTANCE, + SWIPE_DISMISS_MIN_DISTANCE, ), }; return true; @@ -229,11 +243,12 @@ export function useToastGesture({ 0, (point.clientY - state.startY) * state.direction, ); - const elapsed = Math.max(1, Date.now() - state.startTime); + const elapsed = Math.max(1, point.time - state.startTime); + // A flick needs no minimum travel of its own: the swipe already had to + // pass the slop toward the edge to begin. const isDismissed = state.intent === 'vertical' && - (travel >= state.dismissThreshold || - (travel >= FLICK_MIN_DISTANCE && travel / elapsed > FLICK_VELOCITY)); + (travel >= state.dismissThreshold || travel / elapsed > FLICK_VELOCITY); gestureRef.current = null; root.style.removeProperty('transition-duration'); @@ -270,6 +285,7 @@ export function useToastGesture({ pointerId: event.pointerId, clientX: event.clientX, clientY: event.clientY, + time: event.timeStamp, }, event.target, ) @@ -291,6 +307,7 @@ export function useToastGesture({ pointerId: event.pointerId, clientX: event.clientX, clientY: event.clientY, + time: event.timeStamp, }, () => event.preventDefault(), () => rootRef.current?.releasePointerCapture?.(event.pointerId), @@ -310,6 +327,7 @@ export function useToastGesture({ pointerId: event.pointerId, clientX: event.clientX, clientY: event.clientY, + time: event.timeStamp, }); }, [endGesture], @@ -333,11 +351,12 @@ export function useToastGesture({ if (!root || !enabled) { return; } - const point = (touch: Touch): GesturePoint => ({ + const point = (touch: Touch, event: TouchEvent): GesturePoint => ({ source: 'touch', pointerId: touch.identifier, clientX: touch.clientX, clientY: touch.clientY, + time: event.timeStamp, }); const changedTouch = (event: TouchEvent) => { const state = gestureRef.current; @@ -358,7 +377,7 @@ export function useToastGesture({ } const touch = event.changedTouches[0]; if (touch != null) { - beginGesture(point(touch), event.target); + beginGesture(point(touch, event), event.target); } }; const handleTouchMove = (event: TouchEvent) => { @@ -368,7 +387,7 @@ export function useToastGesture({ } const touch = changedTouch(event); if (touch) { - moveGesture(point(touch), () => { + moveGesture(point(touch, event), () => { if (event.cancelable) { event.preventDefault(); } @@ -382,7 +401,7 @@ export function useToastGesture({ } const touch = changedTouch(event); if (touch) { - endGesture(point(touch)); + endGesture(point(touch, event)); } else if (event.touches.length === 0) { resetGesture(true); } From 4076bf3816ef9f56c772e2970a6268e6bbb5a73b Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Fri, 23 Oct 2026 19:54:42 -0700 Subject: [PATCH 003/699] spec(AST-094): touch presentations follow UIKit --- .../specs/AST-094-touch-presentations/spec.md | 293 ++++++++++++++++++ 1 file changed, 293 insertions(+) create mode 100644 docs/specs/AST-094-touch-presentations/spec.md diff --git a/docs/specs/AST-094-touch-presentations/spec.md b/docs/specs/AST-094-touch-presentations/spec.md new file mode 100644 index 0000000000000..d81f92cc41ea0 --- /dev/null +++ b/docs/specs/AST-094-touch-presentations/spec.md @@ -0,0 +1,293 @@ +--- +schema_version: 4 +template_version: 2 +kind: system-spec +id: spec:AST-094 +authority: draft +archive_reason: null +superseded_by: null +approved_by: null +approved_at: null +phase: proposed +owners: [vjeux] +affects_architecture: + [architecture:interaction-modality, architecture:theme-tokens] +affects_families: [family:buttons, family:overlay-dismissal] +affects_contributing: [] +affects_consumer_docs: [] +--- + +# Touch presentations follow UIKit system spec + +## Intent + +A person using an Astryx product on a phone gets the controls, sheets, menus, +lists and navigation they know from iOS: the same physics, the same timings, +the same gesture thresholds and the same feel. Every component's touch +presentation follows UIKit on a named reference release, and its motion keeps +running at display rate while the page's JavaScript is busy. + +The reference is UIKit on iOS 26.2 today. Each number a touch presentation +uses names the iOS version it was taken from; moving the reference to a newer +release re-measures the table, and differences on older releases are noted, +not targeted. + +## Non-goals + +- Pointer (desktop) presentations. Their timing and easing stay with the + theme's motion tokens (`architecture:theme-tokens`). +- WebKit embedded in a native app and the native Swift package. This record + governs the web components themselves. +- Scroll physics. Scrolling stays native on every platform: on iOS an overflow + area or the page is a UIKit scroll view already. +- Which components get a touch presentation, and when. Each component's own + record owns its presentation. +- Equivalent internal implementations remain valid when they satisfy this + contract. The modules, files and algorithms of the motion and gesture + foundations belong to their architecture records. + +## Requirements + +### Physics + +- **FR1 — One table of UIKit values.** Every spring, duration, deceleration + rate, rubber-band coefficient, gesture threshold and haptic moment a touch + presentation uses MUST come from one shared table of UIKit values. A + component MUST NOT carry a number of its own. +- **FR2 — Every value has a source.** Each value in the table MUST name its + source next to it: Apple's documentation, a WWDC session, WebKit's own + source where WebKit states it matches UIKit, or a measurement of UIKit on + the iOS simulator naming the app, the iOS version, the method and the date. + A value without one of these MUST be labelled a guess and name the + measurement that will replace it. A decompilation of UIKit MAY aim a + measurement and MUST NOT be a source. +- **FR3 — UIKit's physics.** Springs MUST follow UIKit's spring model + (response and damping ratio, mass, stiffness and damping, and the release + velocity), with overdamping clamped to critical damping wherever UIKit + clamps it. A momentum coast MUST decay like a UIKit scroll view (0.998 per + millisecond, 0.99 for the fast rate), a drag past an edge MUST resist with + UIKit's rubber band `(1 − 1 ÷ (x·c ÷ d + 1))·d` with the coefficient its use + measures (0.55 for a scroll view and a sheet), and a release MUST pick its + rest point by projecting its velocity the way its UIKit counterpart does: + with a deceleration rate, or with a measured projection time. +- **FR4 — Physics are not themable in touch presentations.** A touch + presentation MUST read its motion from the table on every theme. Themes keep + color, shape, type and spacing; the theme's duration and easing tokens keep + governing pointer presentations. **Proposed; pending owner review:** this + narrows `architecture:theme-tokens` (owners: cixzhang, imdreamrunner) to + pointer presentations, and departs from the draft `design:motion` DR3 + (owners: ernestt, cixzhang) wherever UIKit settles with an overshoot or + stretches with a rubber band. + +### Motion + +- **FR5 — Motion runs on the compositor.** Every release and every + programmatic animation of a touch presentation MUST animate `transform`, the + individual `translate`, `scale` and `rotate` properties, or `opacity` + (a dialog's `::backdrop` included) through Web Animations that the browser + runs off the main thread, + and MUST keep advancing at display rate through a main-thread block. It MUST + NOT animate layout properties, and MUST NOT step a frame from script when the + compositor can run the curve. +- **FR6 — Motion is interruptible.** A finger MUST be able to catch any + animation mid-flight at the value on screen. That value and its velocity + MUST come from the curve, not from a layout read. A new target MUST keep the + current velocity, and a release MUST hand its velocity to the spring that + follows. +- **FR7 — Exits animate as UIKit's do.** A touch presentation MUST play the + exit its UIKit counterpart plays. **Proposed; pending owner review:** this + departs from the draft `spec:AST-090` FR1 (exits off by default) and FR10 + (reduced motion closes at once) for touch presentations (owners: ernestt, + cixzhang). +- **FR8 — Reduce Motion substitutes, never jumps.** Under + `prefers-reduced-motion: reduce`, a touch presentation MUST behave as its + UIKit counterpart does with Reduce Motion and Prefer Cross-Fade Transitions + on: a transition that travels becomes a cross-fade carrying the same change + of state. An entrance lands in place and fades in, an exit fades out in + place, and any other move fades through. A navigation push or pop + cross-fades over 0.5 s on the ease-in-out curve; a sheet, a full-screen + modal or an alert fades on its own presentation spring and drops its travel + and scale; menus open and close on UIKit's Reduce Motion spring. Motion that + follows a finger, such as a throw, MUST keep its motion. **Proposed; pending + owner review:** this is the minimally moving form the draft `design:motion` + DR5 asks for, for touch presentations (owners: ernestt, cixzhang); the + alternative is UIKit's Reduce Motion alone, which keeps the slides of + navigation, sheets and modals. + +### Gestures + +- **FR9 — Fingers are tracked by recognizers.** A drag MUST be tracked by the + shared gesture recognizers, writing one transform per move, with no + component render per move and no layout read after the gesture begins. + Native scrolling MUST be used only where the component is a scroll view + (lists, scroll areas, the page). A component MAY track a finger with native + scroll snapping only with simulator evidence that it tracks through a + main-thread block, ends on UIKit's release curve, ends correctly when the + finger lifts during a block, and lets a finger catch the content mid-flight. +- **FR10 — A second finger never breaks the first.** A second finger MUST NOT + cancel or redirect a drag a recognizer has claimed. Page pinch-zoom MUST stay + the browser's whenever no pinch recognizer claims the fingers and no drag is + claimed (WCAG 1.4.4). +- **FR11 — Two press profiles.** Press feedback MUST follow one of two UIKit + profiles. A control (UIControl) highlights on touch-down outside scroll + views, keeps tracking within its expanded bounds (70 pt), turns its highlight + off and on as the finger drags out and back, and activates on a lift inside. + A cell (a table or collection cell) highlights 150 ms after touch-down inside + a scroll view and drops the highlight when the scroll wins. The 150 ms delay + applies only inside scroll views. **Proposed; pending owner review:** this + amends `design:user-states` DR1 and DR3 for touch presentations (owners: + ernestt, cixzhang), and the press treatment of `family:buttons` (owner: + cixzhang). +- **FR12 — Long press and rubber band are per use.** A long press MUST use the + thresholds its UIKit counterpart uses: 0.5 s and 10 pt for a plain long + press, the measured values for context menus and menus. A rubber band MUST + use the coefficient its use measures. +- **FR13 — One touch-presentation resolver.** Whether a component shows its + touch presentation MUST come from one shared resolver: touch behaviors key + on a coarse pointer, layout on the available size, and a presentation that + depends on both (a sheet instead of a popover) asks for both. A component + MUST NOT write its own media query for it. **Proposed; pending owner + review:** the resolver sits beside the shared modality state of + `architecture:interaction-modality` (owner: cixzhang). +- **FR14 — Nested scrolling follows UIKit.** Who gets a drag that starts + inside a scroll area MUST be decided once, when the drag begins. If the + area sits at its edge and the drag moves toward that edge, the surrounding + surface (a sheet) takes the drag; otherwise the area keeps it for the whole + gesture and rubber-bands at its edge. The drag MUST NOT change hands midway, + in either direction. + **Proposed; pending owner review:** this amends `spec:AST-025` FR14 (scroll + chaining by default) for touch presentations (owner: cixzhang). +- **FR15 — Recognizers arbitrate like UIKit's.** Recognizers that see the + same touch sequence MUST be exclusive by default: the first to recognize + wins and the others fail, unless both allow simultaneous recognition. A + recognizer that requires another to fail MUST hold its recognition until + that one fails, and MUST fail if that one recognizes. When several recognize + on the same event, the innermost element's recognizer MUST win. +- **FR16 — Gestures keep the event clock.** Recognizers MUST read time from the + input events' timestamps. Durations (a long press, the interval between + taps) MUST be deadlines on that clock, so a blocked main thread delays the + report and never the moment reported. +- **FR17 — UIKit's readings.** A recognizer's thresholds and readings MUST + follow UIKit's and come from the table. A pan begins at 10 pt of travel on + either axis, and its translation excludes that travel; its velocity is + UIKit's estimate, 0.2 × the newest event's instantaneous velocity + 0.8 × + the previous event's, never decayed through a rest. +- **FR18 — `touch-action` leaves the browser the rest.** An element with touch + recognizers MUST get a `touch-action` that leaves the browser exactly what + its recognizers do not use: the other axis for an axis-limited pan or swipe, + `manipulation` for taps and long presses, everything for a one-way pan or a + pan that shares its axis with native scrolling. `pinch-zoom` MUST stay + unless a pinch or rotation recognizer is attached. +- **FR19 — Claims are made in time and only by an owner.** A recognizer that + begins MUST claim its touches by cancelling `touchmove` through a + non-passive listener on its element. A `touchmove` of a touch no recognizer + owns MUST NOT be cancelled. A recognizer that competes with native scrolling + MUST decide within 10 px of travel, before the browser commits to the + scroll. +- **FR20 — The browser can take a touch.** `pointercancel`, an uncancelable + `touchmove`, or a native scroll under the touch MUST count as the browser + taking the touch: possible recognizers that share it fail, active ones are + cancelled, and the view's press is cancelled without a release. +- **FR21 — Clicks follow the gesture.** When a gesture takes a touch, its + `touchend` MUST be cancelled, and no click or compatibility mouse event + reaches the page. A click held for a recognizer that may still recognize (a + tap that a double tap may claim) MUST be replayed on its target if that + recognizer fails. Activation with `detail === 0` (keyboard, assistive + technology) MUST always pass. A tap MUST NOT wait for the browser's + double-tap delay. +- **FR22 — iOS Safari keeps its edge swipe.** Safari's own back swipe from the + screen edge MUST stay the browser's; a touch presentation never overrides it. + +### Haptics + +- **FR23 — Haptics where UIKit plays them.** A touch presentation MUST play a + haptic only at the moments its UIKit counterpart does. In iOS Safari the only + haptic a page can produce is a light impact from a real tap landing on a + `