From e1f9922afdc43cc7fd4c1816a671834a00514849 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 9 Nov 2026 07:03:43 -0800 Subject: [PATCH 1/2] test(interaction-harness): too little to judge is named starvation A busy proof whose control moved on fewer than minJudgedFrames frames threw FrameStarvationError only when the cast also showed no frame for longer than FRAME_STARVATION_MS. On a loaded runner the gap can stay under that. The same shortfall then came back as verdict failures, and the spec failed as if the component had stalled. Under the floor, a run with no component failure on the frames shown is now too little to judge. busyWindowProof throws FrameStarvationError with the control's moves, the floor, the cast's frames and its longest gap, counting every frame the cast drew. A component that missed a frame the control moved on, or stalled longer than it, still fails the verdict. The cast's median interval judges no component, so it keeps no red there. pooledBusyProof does the same for too few judged runs or control frames. A pool left short of judged runs by a run that began too late keeps its failure. Every change turns a verdict failure into named starvation. --- internal/interaction-harness/README.md | 23 +- .../src/busy.control.test.ts | 177 +++++++++++++-- internal/interaction-harness/src/busy.ts | 209 +++++++++++++----- .../src/busy.webkitclock.test.ts | 38 +++- .../src/harness.interaction.browser.spec.ts | 13 +- 5 files changed, 365 insertions(+), 95 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index a779b3f94105e..c07d4d788e938 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -321,11 +321,15 @@ decodes each frame once. A proof reads frames up to decode only those. Pass either as `control`: the component passes when it moved on every frame the control moved and its longest stall is no longer than the control's. Both are read from the same screencast frames, so a compositor animation misses none. A window in -which the control moved on fewer than four frames fails, since the machine -presented too little to judge; the failure names the cast's frames in the -window and its longest gap. When no frame came for longer than -`FRAME_STARVATION_MS` there too, the machine starved the run, and -`busyWindowProof` throws `FrameStarvationError` instead of returning a verdict. +which the control moved on fewer than four frames is under the floor. Under the +floor, a run with no component failure on the frames shown is too little to +judge: `busyWindowProof` throws `FrameStarvationError` instead of returning a +verdict. The error names the control's moves, the cast's frames in the window +and its longest gap. A component that missed a frame the control moved on, or +stalled longer than it, failed on frames the compositor presented, so the +verdict keeps that red. A run the machine starved, with no frame for longer +than `FRAME_STARVATION_MS`, throws either way. `maxMedianFrameMs` judges the +cast's pace, not the component, so under the floor it keeps no red. A run of frames with no new position counts as moved when the change that ends it is a single cast step: the component moves under one step a frame there, as on an easing tail, which the cast cannot show @@ -409,8 +413,13 @@ belongs to its run. A run whose control moved on fewer frames than and the pool leaves it out, so it never counts as a pass. A run whose block began too late for its motion is left out the same way: its `runs` entry is null, and `late` holds its `BusyWindowTooLateError`. The pool reads each -run without the starvation throw. The pool fails with -fewer than `minJudgedRuns` judged runs (half of all the runs, rounded up). In each judged run, +run without the starvation throw. With fewer than `minJudgedRuns` judged runs +(half of all the runs, rounded up), or a control that moved on fewer than four +frames over them, the pool is under the floor. Under the floor, a pool with no +component failure on the frames shown is too little to judge, and it throws +`FrameStarvationError`. A pool left short of judged runs by a run that began +too late fails instead, and so does a pool whose judged runs show a component +red. In each judged run, the component must move on every frame its control moved on, so a freeze fails its run. diff --git a/internal/interaction-harness/src/busy.control.test.ts b/internal/interaction-harness/src/busy.control.test.ts index e36ca8ebc582e..a17129f332db2 100644 --- a/internal/interaction-harness/src/busy.control.test.ts +++ b/internal/interaction-harness/src/busy.control.test.ts @@ -313,14 +313,67 @@ describe('busyWindowProof against a control', () => { ); }); - it('fails a window in which the control itself presented too little to judge', () => { + it('names a window in which the control itself presented too little to judge as starvation, though every frame came', () => { const still = asControl( moving().map(f => ({...f, box: {...f.box!, y: 0}}) as TrackedFrame), ); - const proof = busyWindowProof(moving(), BLOCK, {control: still}); + const run = () => busyWindowProof(moving(), BLOCK, {control: still}); + expect(run).toThrow(FrameStarvationError); // It names how starved the cast was: here every frame came, 60 Hz. + expect(run).toThrow( + /busyWindowProof: the page drew \d+ frames while a 700 ms block held the main thread, and the control moved on 0 of them, where the proof needs 4\. No frame came for 1[67]\.\d ms: the compositor presented too little in this window to judge/, + ); + }); + + it("names a window under the control's floor as starvation when no frame came for less than 50 ms: a loaded machine's cast, its component on every frame the control moved on", () => { + // A 152 ms block: the cast delivered 8 frames, none for 44 ms. The control + // moved on 3 of them, as on a loaded runner. + const block = {from: 1000, to: 1152}; + const times = [1008, 1028, 1048, 1092, 1104, 1116, 1128, 1146]; + const moves = new Set([1048, 1092, 1128]); + let y = 0; + const loaded = [ + ...moving().filter(f => f.t < block.from), + ...times.map(t => ({t, box: null}) as TrackedFrame), + ...moving().filter(f => f.t > block.to), + ].map(f => { + if (f.t < block.from || f.t > block.to || moves.has(f.t)) { + y += 3; + } + return {...f, box: {x: 0, y, width: 10, height: 10}} as TrackedFrame; + }); + const run = () => + busyWindowProof(loaded, block, {control: asControl(loaded)}); + expect(run).toThrow(FrameStarvationError); + expect(run).toThrow( + /busyWindowProof: the page drew 8 frames while a 152 ms block held the main thread, and the control moved on 3 of them, where the proof needs 4\. No frame came for 44\.0 ms: the compositor presented too little in this window to judge/, + ); + }); + + it('still fails a main-thread component under the floor: it misses the frames the control moved on, and the cast kept pace', () => { + const block = {from: 1000, to: 1090}; + const frozen = moving().map(f => + f.t > block.from && f.t < block.to + 20 + ? ({...f, box: {...f.box!, y: 3 * at(block.from)}} as TrackedFrame) + : f, + ); + const proof = busyWindowProof(frozen, block, { + control: asControl(moving()), + }); expect(proof.verdict.failures.join('\n')).toMatch( - /the control moved on 0 frames in the judged part, under 4: the compositor presented too little in this window to judge \(the cast delivered \d+ frames in the \d+ ms block, none for 1[67] ms\)/, + /moved on 0 of the 3 frames the control moved/, + ); + }); + + it("names a window under the floor as starvation when only the cast's median interval fails: the median is the cast's pace, not the component", () => { + const run = () => + busyWindowProof(moving(), short(1000), { + control: asControl(moving()), + maxMedianFrameMs: 10, + }); + expect(run).toThrow(FrameStarvationError); + expect(run).toThrow( + /the control moved on 3 of them, where the proof needs 4/, ); }); @@ -448,15 +501,29 @@ function starvedCast(): TrackedFrame[] { ); } +/** The numbers a `FrameStarvationError` from `run` names. */ +function starvationNumbers(run: () => unknown) { + try { + run(); + } catch (error) { + if (error instanceof FrameStarvationError) { + return error.numbers; + } + throw error; + } + throw new Error('no FrameStarvationError'); +} + /** A 90 ms block from `from`: its judged part holds three 60 Hz frames. */ const short = (from: number) => ({from, to: from + 90}); describe('pooledBusyProof', () => { it('judges a block too short for one run over pooled runs: one run cannot decide, three runs of a component that moves with the control pass', () => { const control = asControl(moving()); - const single = busyWindowProof(moving(), short(1000), {control}); - expect(single.verdict.failures.join('\n')).toMatch( - /the control moved on 3 frames in the judged part, under 4/, + const single = () => busyWindowProof(moving(), short(1000), {control}); + expect(single).toThrow(FrameStarvationError); + expect(single).toThrow( + /the control moved on 3 of them, where the proof needs 4/, ); const runs = [1000, 1300, 1600].map(from => ({ tracked: moving(), @@ -623,20 +690,22 @@ describe('pooledBusyProof', () => { } }); - it('judges the control over the pool too: runs in which it presented too little fail', () => { + it('names a pool whose control presented too little in every run as starvation', () => { const still = asControl( moving().map(f => ({...f, box: {...f.box!, y: 0}}) as TrackedFrame), ); - const pooled = pooledBusyProof( - [1000, 1300].map(from => ({ - tracked: moving(), - control: still, - window: short(from), - })), + const run = () => + pooledBusyProof( + [1000, 1300].map(from => ({ + tracked: moving(), + control: still, + window: short(from), + })), + ); + expect(run).toThrow(FrameStarvationError); + expect(run).toThrow( + /pooledBusyProof: .*0 of 2 runs judged, under 1, and the control moved on 0 frames over them: the compositor presented too little in the others to judge/, ); - expect(pooled.verdict.failures).toEqual([ - '0 of 2 runs judged, under 1: the compositor presented too little in the others to judge', - ]); }); /** `frames` held at the position they had at `from`, from `from` to `to`. */ @@ -665,20 +734,84 @@ describe('pooledBusyProof', () => { ); }); - it('fails a pool in which every run but one was too little to judge, even when that run holds enough frames', () => { + it('names a pool in which every run but one was too little to judge as starvation, even when that run holds enough frames', () => { const windows = [1000, 1300, 1600].map(from => ({from, to: from + 200})); const starved = held(held(moving(), 1300, 1500), 1600, 1800); + const pool = () => + pooledBusyProof( + windows.map(window => ({ + tracked: moving(), + control: asControl(starved), + window, + })), + ); + expect(pool).toThrow(FrameStarvationError); + expect(pool).toThrow( + /pooledBusyProof: the page drew \d+ frames while 3 blocks held the main thread, and 1 of them had enough control frames to judge, where the proof needs 2\. .*1 of 3 runs judged, under 2, and the control moved on \d+ frames over them: the compositor presented too little in the others to judge/, + ); + }); + + it('names a pool of four runs with one judged as starvation: its component moved on every frame its control moved on', () => { + const windows = [1000, 1250, 1500, 1750].map(short); + const control = asControl( + held(held(held(moving(), 1250, 1340), 1500, 1590), 1750, 1840), + ); + const pool = () => + pooledBusyProof( + windows.map(window => ({tracked: moving(), control, window})), + ); + expect(pool).toThrow(FrameStarvationError); + expect(pool).toThrow(/1 of 4 runs judged, under 2/); + // Short of judged runs, the error counts every block's frames. + const numbers = starvationNumbers(pool); + expect(numbers.frames).toBe(20); + expect(numbers.longestGapMs).toBeCloseTo(FRAME, 1); + }); + + it('names the longest gap of every block in a pool short of judged runs, where the unjudged blocks drew no frame', () => { + const windows = [1000, 1250, 1500, 1750].map(short); + const control = asControl( + held(held(held(moving(), 1250, 1340), 1500, 1590), 1750, 1840), + ); + const tracked = moving().filter( + f => !windows.slice(1).some(w => f.t > w.from && f.t < w.to), + ); + const numbers = starvationNumbers(() => + pooledBusyProof(windows.map(window => ({tracked, control, window}))), + ); + expect(numbers.frames).toBe(5); + expect(numbers.longestGapMs).toBeCloseTo(90, 5); + }); + + it("names a pool short of judged runs as starvation when only the runs' median intervals fail", () => { + const windows = [1000, 1250, 1500, 1750].map(short); + const control = asControl( + held(held(held(moving(), 1250, 1340), 1500, 1590), 1750, 1840), + ); + const pool = () => + pooledBusyProof( + windows.map(window => ({tracked: moving(), control, window})), + {maxMedianFrameMs: 10}, + ); + expect(pool).toThrow(FrameStarvationError); + expect(pool).toThrow(/1 of 4 runs judged, under 2/); + }); + + it('still fails a pool short of judged runs when its judged run shows a component red', () => { + const windows = [1000, 1250, 1500, 1750].map(short); + const control = asControl( + held(held(held(moving(), 1250, 1340), 1500, 1590), 1750, 1840), + ); const pooled = pooledBusyProof( windows.map(window => ({ - tracked: moving(), - control: asControl(starved), + tracked: held(moving(), 1000, 1090), + control, window, })), ); - expect(pooled.judged).toEqual([true, false, false]); - expect(pooled.control.moved).toBeGreaterThanOrEqual(4); + expect(pooled.judged).toEqual([true, false, false, false]); expect(pooled.verdict.failures.join('\n'), pooled.summary).toMatch( - /1 of 3 runs judged, under 2: the compositor presented too little in the others to judge/, + /run 1: moved on 0 of the 3 frames its control moved/, ); }); diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index 0fd05536bb5ec..77615037b2ef3 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -1323,7 +1323,7 @@ function motionStepPx(values: readonly number[]): number { * compositor animation presents fewer frames there, the control too. The * component passes when: * - the control moved on at least `minJudgedFrames` frames of the judged part - * (fewer: the machine presented too little to judge, which fails); + * (fewer: the compositor presented too little to judge, as below); * - the component moved on every frame the control moved: both are read from * the same screencast frames, so a compositor animation misses none; * - its longest stall is no longer than the control's. @@ -1366,13 +1366,19 @@ function motionStepPx(values: readonly number[]): number { * apart, so it judges no 30 Hz cadence. A note in the summary says so, with * the count of such pairs. * - * With a control, a run the machine starved of frames proves nothing. So it - * throws `FrameStarvationError` with its numbers and returns no verdict. A - * starved run has two marks. Its control moved on fewer than - * `minJudgedFrames` frames, and the cast showed no frame for longer than - * `FRAME_STARVATION_MS` in the block. Without a control, a block with no - * frame is a frozen animation's, and it fails the verdict. `pooledBusyProof` - * reads each run without the throw and judges the runs together. + * With a control, a window whose control moved on fewer than + * `minJudgedFrames` frames of the judged part is under the floor. Under the + * floor, a run with no component failure on the frames shown is too little to + * judge. It throws `FrameStarvationError` and returns no verdict. The error + * names the control's moves, the floor, the cast's frames in the block and its + * longest gap. A component that missed a frame the control moved on, or + * stalled longer than it, failed on frames the compositor presented. The + * verdict keeps that red, unless the machine starved the run: no frame for + * longer than `FRAME_STARVATION_MS`. `maxMedianFrameMs` judges the cast's + * pace, not the component, so under the floor it keeps no red. Without a + * control, a block with no frame is a frozen animation's, and it fails the + * verdict. `pooledBusyProof` reads each run without the throw and judges the + * runs together. * * With `judgeEveryFrame`, a frame the cast could not show the component's * step on is too little to judge (the frames `belowStep` counts). When the @@ -1383,8 +1389,15 @@ function motionStepPx(values: readonly number[]): number { export function busyWindowProof( ...args: Parameters ): BusyProof { - const {proof, starvation, unjudged} = readBusyWindow(...args); - if (starvation != null) { + const {proof, starvation, unjudged, componentFailed} = readBusyWindow( + ...args, + ); + // Under the control's floor, a cast that kept pace still shows a missed + // frame or a longer stall. That failure is the component's red. + if ( + starvation != null && + (starvation.longestGapMs > FRAME_STARVATION_MS || !componentFailed) + ) { throw new FrameStarvationError(starvation); } // A failed verdict is the component's red, whatever else the cast hid. @@ -1439,10 +1452,12 @@ function readBusyWindow( return judgeBusyWindow(tracked, window, options); } const shown = atDisplayRate(dated); - const read = judgeBusyWindow(shown.tracked, window, { - ...options, - control: shown.control, - }); + const read = judgeBusyWindow( + shown.tracked, + window, + {...options, control: shown.control}, + dated.tracked, + ); // A WebKit cast rarely shows two frames one interval apart, so the proof // cannot judge a 30 Hz cadence there. It says so, with the count. const {pairs, unmatched} = displayIntervalPairs( @@ -1560,10 +1575,23 @@ function judgeBusyWindow( */ judgeEveryFrame?: boolean; } = {}, + /** + * The frames the cast drew, dated. In WebKit they are the frames before the + * display-rate keep, so a frame left out never lengthens the cast's gap. + */ + drawn: readonly TrackedFrame[] = tracked, ): { proof: BusyProof; starvation: FrameShortfall | null; unjudged: readonly UnjudgedFrame[]; + /** + * The component failed on frames the control moved on: it missed one, or + * stalled longer than the control. The cast's median interval is no part + * of it. + */ + componentFailed: boolean; + /** The frames the cast drew in the block, and its longest gap there, ms. */ + cast: {frames: number; longestGapMs: number}; } { let end = window.to - marginMs; if (settledWithin != null && settledWithinSteps != null) { @@ -2144,6 +2172,7 @@ function judgeBusyWindow( } const failures: string[] = []; let starvation: FrameShortfall | null = null; + const cast = castFramesIn(drawn, window); if (stats.frames === 0 && controlRead == null) { failures.push('no frame presented in the judged part'); } else if (stats.medianFrameMs > maxMedianFrameMs) { @@ -2152,34 +2181,25 @@ function judgeBusyWindow( ); } if (controlRead != null) { - if (controlRead.moved < minJudgedFrames) { - // How starved the cast was over the whole block, as the pacing and - // frame errors say it. Every frame the cast delivered counts, with the - // component on screen or not. - const blockFrames = tracked - .filter(f => f.t > window.from && f.t < window.to) - .map(f => f.t); - const castEdges = [window.from, ...blockFrames, window.to]; - const castGapMs = Math.max( - ...castEdges.slice(1).map((t, k) => t - castEdges[k]), - ); - // The control is a compositor animation: when it moved too little and - // no frame came for that long, the machine starved the run. - if (castGapMs > FRAME_STARVATION_MS) { - starvation = { - by: 'busyWindowProof', - during: `a ${Math.round(window.to - window.from)} ms block held the main thread, and the control moved on ${controlRead.moved} of them`, - frames: blockFrames.length, - min: minJudgedFrames, - longestGapMs: castGapMs, - }; - } - failures.push( - `the control moved on ${controlRead.moved} frames in the judged part, under ${minJudgedFrames}: the compositor presented too little in this window to judge (the cast delivered ${blockFrames.length} frame${blockFrames.length === 1 ? '' : 's'} in the ${Math.round(window.to - window.from)} ms block, none for ${Math.round(castGapMs)} ms)`, - ); - } const visible = controlRead.movedWith - controlRead.belowStep; - if (visible < minJudgedFrames) { + if (controlRead.moved < minJudgedFrames) { + // The control is a compositor animation. When it moved on too few + // frames, the compositor presented too little to judge the component, + // however long the cast's longest gap was. + starvation = { + by: 'busyWindowProof', + during: `a ${Math.round(window.to - window.from)} ms block held the main thread, and the control moved on ${controlRead.moved} of them`, + frames: cast.frames, + min: minJudgedFrames, + longestGapMs: cast.longestGapMs, + ...(cast.longestGapMs > FRAME_STARVATION_MS + ? {} + : { + cause: + 'the compositor presented too little in this window to judge, so the run proves nothing about the motion', + }), + }; + } else if (visible < minJudgedFrames) { failures.push( `the component showed a new position on ${visible} of the control's frames, under ${minJudgedFrames}: too little motion in this window to judge`, ); @@ -2260,7 +2280,29 @@ function judgeBusyWindow( verdict: {pass: failures.length === 0, failures}, summary, }; - return {proof, starvation, unjudged}; + const componentFailed = + controlRead != null && + (controlRead.movedWith < controlRead.moved || + controlRead.componentStallMs > controlRead.longestStallMs); + return {proof, starvation, unjudged, componentFailed, cast}; +} + +/** + * The frames the cast delivered inside `window`, with the component on screen + * or not, and the longest gap between them and the window's edges, ms. + */ +function castFramesIn( + tracked: readonly TrackedFrame[], + window: {readonly from: number; readonly to: number}, +): {frames: number; longestGapMs: number} { + const inside = tracked + .filter(f => f.t > window.from && f.t < window.to) + .map(f => f.t); + const edges = [window.from, ...inside, window.to]; + return { + frames: inside.length, + longestGapMs: Math.max(...edges.slice(1).map((t, k) => t - edges[k])), + }; } /** One run of a pooled busy proof (see `pooledBusyProof`). */ @@ -2328,6 +2370,13 @@ export interface PooledBusyProof { * control's. * `maxMedianFrameMs`, when given, applies to every run. A main-thread * animation still fails: in every run the control moves and it does not. + * + * With fewer judged runs than `minJudgedRuns`, or a control that moved on + * fewer than `minJudgedFrames` frames over them, the pool is under the floor. + * Under the floor, a pool with no component failure on the frames shown is too + * little to judge, and it throws `FrameStarvationError`. A pool left short of + * judged runs by a run that began too late keeps its failure instead. So does a + * pool whose judged runs show a component red. */ export function pooledBusyProof( runs: readonly BusyRun[], @@ -2360,21 +2409,31 @@ export function pooledBusyProof( ); } const late: (BusyWindowTooLateError | null)[] = []; + const failedRuns: boolean[] = []; + const runCasts: ({frames: number; longestGapMs: number} | null)[] = []; const proofs = runs.map(run => { try { - const {proof} = readBusyWindow(run.tracked, run.window, { - ...windowOptions, - control: run.control, - ahead: run.ahead, - controlStarts: run.controlStarts, - }); + const {proof, componentFailed, cast} = readBusyWindow( + run.tracked, + run.window, + { + ...windowOptions, + control: run.control, + ahead: run.ahead, + controlStarts: run.controlStarts, + }, + ); late.push(null); + failedRuns.push(componentFailed); + runCasts.push(cast); return proof; } catch (error) { if (!(error instanceof BusyWindowTooLateError)) { throw error; } late.push(error); + failedRuns.push(false); + runCasts.push(null); return null; } }); @@ -2395,20 +2454,17 @@ export function pooledBusyProof( ? `${runs.length} runs` : `the ${judgedReads.length} judged runs of ${runs.length}`; const failures: string[] = []; - if (judgedReads.length < minJudgedRuns) { - failures.push( - lateRuns === 0 - ? `${judgedReads.length} of ${runs.length} runs judged, under ${minJudgedRuns}: the compositor presented too little in the others to judge` - : `${judgedReads.length} of ${runs.length} runs judged, under ${minJudgedRuns}: ${lateRuns} began too late for their motion, and the compositor presented too little in any other to judge`, - ); - } - if (judgedReads.length > 0 && moved < minJudgedFrames) { + if (judgedReads.length < minJudgedRuns && lateRuns > 0) { failures.push( - `the control moved on ${moved} frames over ${over}, under ${minJudgedFrames}: the compositor presented too little in these windows to judge`, + `${judgedReads.length} of ${runs.length} runs judged, under ${minJudgedRuns}: ${lateRuns} began too late for their motion, and the compositor presented too little in any other to judge`, ); } + // Too few control frames or judged runs is too little to judge. A judged + // run's component red still fails: a frame it missed, or a longer stall. + const tooFewFrames = judgedReads.length > 0 && moved < minJudgedFrames; + const tooFewRuns = judgedReads.length < minJudgedRuns && lateRuns === 0; const visible = movedWith - total('belowStep'); - if (judgedReads.length > 0 && visible < minJudgedFrames) { + if (!tooFewFrames && judgedReads.length > 0 && visible < minJudgedFrames) { failures.push( `the component showed a new position on ${visible} of the control's frames over ${over}, under ${minJudgedFrames}: too little motion in these windows to judge`, ); @@ -2442,6 +2498,43 @@ export function pooledBusyProof( ); } }); + // A pool whose compositor presented too little proves nothing about the + // motion, as one starved window does. A run that began too late keeps the + // pool's failure. So does a judged run's component red. + const lateFailure = judgedReads.length < minJudgedRuns && lateRuns > 0; + const componentFailed = failedRuns.some((failed, i) => judged[i] && failed); + if ((tooFewFrames || tooFewRuns) && !lateFailure && !componentFailed) { + const casts = runCasts.filter( + (cast, i): cast is {frames: number; longestGapMs: number} => + cast != null && (tooFewRuns || judged[i]), + ); + const frames = casts.reduce((sum, cast) => sum + cast.frames, 0); + const longestGapMs = Math.max(...casts.map(cast => cast.longestGapMs)); + throw new FrameStarvationError( + tooFewRuns + ? { + by: 'pooledBusyProof', + during: `${runs.length} blocks held the main thread, and ${judgedReads.length} of them had enough control frames to judge`, + frames, + min: minJudgedRuns, + longestGapMs, + cause: `${judgedReads.length} of ${runs.length} runs judged, under ${minJudgedRuns}, and the control moved on ${moved} frames over them: the compositor presented too little in the others to judge, so the pool proves nothing about the motion`, + } + : { + by: 'pooledBusyProof', + during: `the blocks of ${over} held the main thread, and the control moved on ${moved} of them`, + frames, + min: minJudgedFrames, + longestGapMs, + ...(longestGapMs > FRAME_STARVATION_MS + ? {} + : { + cause: + 'the compositor presented too little in these windows to judge, so the pool proves nothing about the motion', + }), + }, + ); + } const summary = `${runs.length} runs pooled` + (judgedReads.length === runs.length diff --git a/internal/interaction-harness/src/busy.webkitclock.test.ts b/internal/interaction-harness/src/busy.webkitclock.test.ts index bb01bd1340c96..d0a82678ee6db 100644 --- a/internal/interaction-harness/src/busy.webkitclock.test.ts +++ b/internal/interaction-harness/src/busy.webkitclock.test.ts @@ -18,7 +18,7 @@ import {describe, expect, it} from 'vitest'; import {busyWindowProof, pageTimesOnControls, pooledBusyProof} from './busy'; -import type {TrackedFrame} from './frames'; +import {FrameStarvationError, type TrackedFrame} from './frames'; const FRAME = 1000 / 60; const BLOCK = {from: 1000, to: 1700}; @@ -472,4 +472,40 @@ describe('busyWindowProof on a WebKit cast with a control', () => { expect(proof(chromium, true)).toEqual(proof(stamped, true)); expect(proof(webkit, false)).toEqual(proof(stamped, false)); }); + + it('measures the cast on every frame it drew, so a frame the display-rate keep leaves out never lengthens its longest gap', () => { + // In the block, a frame drawn 7 ms after another is left out of the + // judgment, and then no frame comes for 48 ms. Without that frame, the + // gap would read 55 ms, over FRAME_STARVATION_MS. + const shots = [ + ...onTime(800, 990), + ...[1010, 1017, 1065, 1090].map(at => ({at, stamp: at})), + ...onTime(1110, 1300), + ]; + const proof = (value: (at: number) => number) => () => { + const frames = cast(shots, value, 'webkit'); + return busyWindowProof( + frames.tracked, + {from: 1000, to: 1100}, + {axis: 'x', control: frames.control, ahead: frames.ahead}, + ); + }; + // Under the floor, a frozen component failed on frames the compositor + // presented, and no gap passed 50 ms. So its red stands. + expect(proof(() => 100)().verdict.failures.join('\n')).toMatch( + /moved on 0 of the \d+ frames the control moved/, + ); + // A component that moves with the control is too little to judge. The + // error counts every frame drawn, the one left out included. + const moving = proof(xAt); + expect(moving).toThrow(FrameStarvationError); + try { + moving(); + } catch (error) { + const {numbers} = error as FrameStarvationError; + expect(numbers.frames).toBe(4); + expect(numbers.longestGapMs).toBeGreaterThan(45); + expect(numbers.longestGapMs).toBeLessThanOrEqual(50); + } + }); }); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 628c2438ded87..f3d3bdb581b83 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -2146,8 +2146,7 @@ test.describe('frame evidence', () => { }; const compositor = await runs('compositor'); - // One run is too little to judge: its verdict says so, or, when the - // machine also starved its cast, the proof throws by name. + // One run is too little to judge, so the proof throws by name. const single = (() => { try { const proof = busyWindowProof( @@ -2174,7 +2173,8 @@ test.describe('frame evidence', () => { expect(pooled.verdict.failures, pooled.summary).toEqual([]); // The same runs with the control taken out of the last blocks. A starved - // run is named and left out. With too few judged runs, the pool fails. + // run is named and left out. With too few judged runs, the pool throws by + // name. const starve = (count: number) => pooledBusyProof( compositor.map((run, i) => @@ -2196,10 +2196,9 @@ test.describe('frame evidence', () => { expect(lastStarved.summary).toMatch( /; run 6: too little to judge, the control moved on 0 frames/, ); - const oneLeft = starve(5); - expect(oneLeft.verdict.failures.join('\n'), oneLeft.summary).toMatch( - /[01] of 6 runs judged, under 3/, - ); + const oneLeft = () => starve(5); + expect(oneLeft).toThrow(FrameStarvationError); + expect(oneLeft).toThrow(/[01] of 6 runs judged, under 3/); // The mutant: a main-thread animation freezes in every block. const raf = pooledBusyProof(await runs('raf'), {axis: 'y'}); From 9956ef8e3429b6c7270cd1bb1283de30d9ae4b10 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 9 Nov 2026 08:27:11 -0800 Subject: [PATCH 2/2] test(interaction-harness): a busy proof judges each predicted move A busy proof whose window caught an animation now predicts each frame's move, in Chromium, when the proof runs the control ahead. After the busy loop, the page replays the element's own runs on a hidden copy of its box. A frame the control moved on must show a new position when its predicted move is a full cast step. A frame predicted to move less is named as too little to judge. With a prediction, frames are dated on the controls' clock, from the starts the window now keeps. Any other window falls back by name to the cast-step rule, exactly as without a replay: no marker, no control ahead, WebKit, or a replay that disagrees with the cast. judgeEveryFrame keeps refusing an excused frame there. The zoom spec starts the control ahead. --- internal/interaction-harness/README.md | 27 + .../src/busy.predicted.fixture.ts | 295 +++++++ .../src/busy.predicted.test.ts | 538 ++++++++++++ internal/interaction-harness/src/busy.ts | 791 +++++++++++++++++- .../LightboxZoom.interaction.browser.spec.ts | 34 +- 5 files changed, 1637 insertions(+), 48 deletions(-) create mode 100644 internal/interaction-harness/src/busy.predicted.fixture.ts create mode 100644 internal/interaction-harness/src/busy.predicted.test.ts diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index c07d4d788e938..acceed2eaa8ba 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -357,6 +357,33 @@ the cast's step explains is then too little to judge: `busyWindowProof` throws there and the cast's step. A verdict that fails anyway comes back as it is, so a frozen or main-thread component stays a red on the component. +A window that caught an animation (`blockMainThread` with `during`) predicts +each frame instead, when two things hold. The proof passes both controls, +`ahead` included, so every frame has a time of its own. The engine is +Chromium. A geometry replay also needs the harness marker on the caught +element. Once the loop ends, the +page replays the element's geometry animations, paused, on a hidden copy of +its box. The copy is aligned to the element, and its box is read every 2 ms. +A fade or a pseudo-element replays one numeric channel. The tracked value at +rest and at the curve's end map that channel to a position. The proof dates +each frame on the controls' clock, from the controls' starts that the window +keeps. A frame predicted to move a full cast step since the last new position must +show one, so a hold ends once its moves add up to a step. +A frame predicted to move less is too little to judge, and so is a frame with +no control reading of its own. Each such frame is named in the proof's +`unjudged` list and in its summary. It counts neither as moved nor as missed, +and `judgeEveryFrame` throws for none. When too few judged frames are left, +the proof throws `FrameStarvationError`. Where a replay predicts, one the +proof cannot read throws `BusyPredictionError`: an ancestor moves too, or a +run composites onto another. A replay whose travel or first judged position +differs from the cast's by more than two steps or a tenth of the travel falls +back by name. So does a geometry window without the marker, a window without +`ahead` or in WebKit, and a window whose run left the element before the +replay. WebKit reads the element's box at the loop's end before its +animation's current time, so the replay cannot be aligned there. A window that falls back is judged exactly as without a +replay, and its notes say why. The proof gives its `judged` part in the +cast's stamps, so a spec selects frames by it as before. + `controlClock(control)` reads the compositor's own time in each frame off the control's position, for a proof that judges motion against time, not against the screencast's stamps. A loaded machine stamps frames late, and diff --git a/internal/interaction-harness/src/busy.predicted.fixture.ts b/internal/interaction-harness/src/busy.predicted.fixture.ts new file mode 100644 index 0000000000000..20ed01c4b2306 --- /dev/null +++ b/internal/interaction-harness/src/busy.predicted.fixture.ts @@ -0,0 +1,295 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file busy.predicted.fixture.ts + * @input Two Chromium casts of Lightbox's busy double-tap zoom proof, each + * recorded in a container under a CI quota + * @output `ZOOM_CASTS`: per cast, its block, the zoom's replayed top edge, + * the controls' starts, and per frame its time and the tracked positions + * @position Replayed by busy.predicted.test.ts: the cast-step rule reds + * each cast, and the predicted-move rule judges it + */ + +/** One recorded cast: times are ms after the block began, positions CSS px. */ +export interface RecordedZoomCast { + readonly block: { + readonly from: number; + readonly to: number; + readonly spanMs: number; + }; + readonly animation: { + readonly from: number; + readonly to: number; + readonly holds: boolean; + }; + /** The replayed top edge, `[epoch ms, y]`, every 8 ms. */ + readonly geometry: ReadonlyArray; + readonly controlStarts: readonly [number, number]; + /** Per frame: time, the photo's top edge and height. */ + readonly frames: ReadonlyArray< + readonly [number, number | null, number | null] + >; + /** Per frame: time and each control's x. */ + readonly control: ReadonlyArray; + readonly ahead: ReadonlyArray; +} + +/** + * Rows written one after another, a space between rows and a comma between + * values. An empty value is a frame the marker did not show in. + */ +function rows>(text: string): T[] { + return text + .trim() + .split(/\s+/) + .map( + row => + row.split(',').map(v => (v === '' ? null : Number(v))) as unknown as T, + ); +} + +export const ZOOM_CASTS: readonly RecordedZoomCast[] = [ + { + block: {from: 1794215413843.8, to: 1794215416343.8, spanMs: 2500}, + animation: {from: 1794215413776.8, to: 1794215418280.7087, holds: true}, + // Times after 1794215413593.8. + geometry: rows<[number, number]>(` + 0,223 8,223 16,223 24,223 32,223 40,223 48,223 56,223 64,223 72,223 + 80,223 88,223 96,223 104,223 112,223 120,223 128,223 136,223 144,223 + 152,223 160,223 168,223 176,223 184,222.99 192,222.9 200,222.8 + 208,222.54 216,222.27 224,221.86 232,221.44 240,220.89 248,220.32 + 256,219.65 264,218.95 272,218.16 280,217.34 288,216.45 296,215.53 + 304,214.54 312,213.51 320,212.44 328,211.33 336,210.18 344,208.98 + 352,207.76 360,206.49 368,205.2 376,203.87 384,202.53 392,201.11 + 400,199.7 408,198.29 416,196.87 424,195.37 432,193.87 440,192.37 + 448,190.87 456,189.31 464,187.75 472,186.18 480,184.62 488,183.02 + 496,181.41 504,179.8 512,178.2 520,176.57 528,174.94 536,173.3 + 544,171.66 552,170.02 560,168.38 568,166.73 576,165.08 584,163.43 + 592,161.78 600,160.14 608,158.49 616,156.84 624,155.21 632,153.57 + 640,151.93 648,150.3 656,148.68 664,147.06 672,145.44 680,143.83 + 688,142.23 696,140.64 704,139.05 712,137.46 720,135.89 728,134.33 + 736,132.78 744,131.22 752,129.68 760,128.16 768,126.64 776,125.12 + 784,123.62 792,122.14 800,120.66 808,119.18 816,117.72 824,116.28 + 832,114.85 840,113.42 848,111.99 856,110.6 864,109.22 872,107.83 + 880,106.45 888,105.1 896,103.77 904,102.43 912,101.09 920,99.79 + 928,98.51 936,97.22 944,95.93 952,94.68 960,93.44 968,92.2 976,90.97 + 984,89.75 992,88.57 1000,87.38 1008,86.19 1016,85.03 1024,83.89 + 1032,82.75 1040,81.62 1048,80.49 1056,79.41 1064,78.32 1072,77.23 + 1080,76.16 1088,75.11 1096,74.08 1104,73.04 1112,72.01 1120,71.01 + 1128,70.02 1136,69.03 1144,68.04 1152,67.09 1160,66.15 1168,65.21 + 1176,64.27 1184,63.35 1192,62.46 1200,61.56 1208,60.67 1216,59.79 + 1224,58.94 1232,58.09 1240,57.24 1248,56.4 1256,55.6 1264,54.79 + 1272,53.98 1280,53.18 1288,52.41 1296,51.65 1304,50.88 1312,50.12 + 1320,49.39 1328,48.67 1336,47.94 1344,47.22 1352,46.52 1360,45.84 + 1368,45.15 1376,44.47 1384,43.8 1392,43.15 1400,42.51 1408,41.86 + 1416,41.23 1424,40.61 1432,40 1440,39.38 1448,38.78 1456,38.2 1464,37.62 + 1472,37.04 1480,36.48 1488,35.93 1496,35.38 1504,34.83 1512,34.29 + 1520,33.77 1528,33.26 1536,32.74 1544,32.23 1552,31.74 1560,31.25 + 1568,30.76 1576,30.28 1584,29.81 1592,29.35 1600,28.9 1608,28.44 1616,28 + 1624,27.57 1632,27.13 1640,26.7 1648,26.28 1656,25.88 1664,25.47 + 1672,25.07 1680,24.67 1688,24.29 1696,23.9 1704,23.52 1712,23.15 + 1720,22.79 1728,22.43 1736,22.07 1744,21.71 1752,21.37 1760,21.03 + 1768,20.7 1776,20.36 1784,20.04 1792,19.72 1800,19.41 1808,19.09 + 1816,18.79 1824,18.49 1832,18.19 1840,17.89 1848,17.61 1856,17.33 + 1864,17.05 1872,16.77 1880,16.5 1888,16.24 1896,15.97 1904,15.71 + 1912,15.46 1920,15.21 1928,14.96 1936,14.72 1944,14.48 1952,14.24 + 1960,14.01 1968,13.78 1976,13.56 1984,13.34 1992,13.12 2000,12.9 + 2008,12.69 2016,12.48 2024,12.28 2032,12.08 2040,11.88 2048,11.68 + 2056,11.49 2064,11.3 2072,11.12 2080,10.93 2088,10.75 2096,10.58 + 2104,10.4 2112,10.23 2120,10.06 2128,9.9 2136,9.73 2144,9.57 2152,9.41 + 2160,9.26 2168,9.1 2176,8.95 2184,8.8 2192,8.66 2200,8.51 2208,8.37 + 2216,8.23 2224,8.09 2232,7.96 2240,7.82 2248,7.69 2256,7.57 2264,7.44 + 2272,7.31 2280,7.19 2288,7.07 2296,6.95 2304,6.83 2312,6.72 2320,6.61 + 2328,6.5 2336,6.39 2344,6.28 2352,6.17 2360,6.07 2368,5.97 2376,5.87 + 2384,5.77 2392,5.67 2400,5.57 2408,5.48 2416,5.39 2424,5.3 2432,5.21 + 2440,5.12 2448,5.03 2456,4.95 2464,4.86 2472,4.78 2480,4.7 2488,4.62 + 2496,4.54 2504,4.46 2512,4.39 2520,4.31 2528,4.24 2536,4.17 2544,4.1 + 2552,4.03 2560,3.96 2568,3.89 2576,3.82 2584,3.76 2592,3.69 2600,3.63 + 2608,3.57 2616,3.51 2624,3.45 2632,3.39 2640,3.33 2648,3.27 2656,3.22 + 2664,3.16 2672,3.11 2680,3.05 2688,3 2696,2.95 2704,2.9 2712,2.85 + 2720,2.8 2728,2.75 2736,2.7 2744,2.66 2752,2.61 2760,2.57 2768,2.52 + 2776,2.48 2784,2.44 2792,2.39 2800,2.35 2808,2.31 2816,2.27 2824,2.23 + 2832,2.19 2840,2.16 2848,2.12 2856,2.08 2864,2.04 2872,2.01 2880,1.97 + 2888,1.94 2896,1.91 2904,1.87 2912,1.84 2920,1.81 2928,1.78 2936,1.75 + 2944,1.72 2952,1.69 2960,1.66 2968,1.63 2976,1.6 2984,1.57 2992,1.54 + 3000,1.52 + `).map(([t, y]) => [1794215413593.8 + t, y] as const), + controlStarts: [1794215410606.945, 1794215410406.945], + frames: rows<[number, number | null, number | null]>(` + -341.4,223,268 -261.9,223,268 -251.6,223,268 -241.7,223,268 + -167.2,223,268 -156.8,223,268 -141.9,223,268 -67.6,223,268 -54.6,223,268 + -41.3,223,268 35.7,222,270 44.9,222,270 54.2,217,280 135.6,215,284 + 155.4,203,308 235.8,200,314 255.7,185,344 336,181,352 355.7,164,386 + 435.9,161,392 455.6,144,426 535.6,141,432 555.9,124,466 631.1,121,472 + 658.8,106,501 735.7,104,506 756.2,90,534 837.7,88,538 855.6,76,562 + 936.2,74,566 956,64,586 1035.7,62,590 1045.3,60,594 1056.1,53,608 + 1140,52,610 1149.4,50,614 1159.6,44,626 1239.5,43,628 1248.6,41,632 + 1257.9,37,641 1339.3,35,644 1350.3,34,646 1359.8,30,654 1438.9,29,656 + 1451.8,28,658 1461.2,25,664 1539.2,24,666 1550.4,23,668 1559.9,20,674 + 1636.4,20,674 1645.8,19,676 1656.2,17,681 1735.8,16,682 1745,15,684 + 1755.6,13,688 1835.8,13,688 1844.9,13,688 1855.7,11,692 1935.8,11,692 + 1945.1,10,694 1957.6,9,696 2036,9,696 2045.3,8,698 2056,7,700 2136,7,700 + 2145,7,700 2155.6,6,702 2236,6,702 2244.9,5,704 2255.4,5,704 + 2336.3,5,704 2345.5,4,706 2356.1,4,706 2437,4,706 2446.7,4,706 + 2456.2,3,708 2536,3,708 2545,3,708 2555.7,2,710 2636.1,2,710 + 2645.5,2,710 2656,2,710 2850,2,710 2928.6,1,712 2941.2,1,712 + 2955.4,1,712 3034.3,1,712 3048,1,712 3061.4,1,712 3140.4,1,712 + 3150.2,1,712 3159.9,1,712 + `), + control: rows<[number, number | null]>(` + -341.4,190 -261.9,180 -251.6,140 -241.7,130 -167.2,120 -156.8,80 + -141.9,70 -67.6,60 -54.6,20 -41.3,10 35.7,0 44.9,10 54.2,40 135.6,50 + 155.4,100 235.8,110 255.7,160 336,170 355.7,220 435.9,230 455.6,200 + 535.6,190 555.9,140 631.1,130 658.8,80 735.7,70 756.2,20 837.7,10 + 855.6,40 936.2,50 956,100 1035.7,110 1045.3,120 1056.1,160 1140,170 + 1149.4,180 1159.6,220 1239.5,230 1248.6,240 1257.9,200 1339.3,190 + 1350.3,180 1359.8,140 1438.9,130 1451.8,120 1461.2,80 1539.2,70 + 1550.4,60 1559.9,20 1636.4,10 1645.8,0 1656.2,40 1735.8,50 1745,60 + 1755.6,100 1835.8,110 1844.9,120 1855.7,160 1935.8,170 1945.1,180 + 1957.6,220 2036,230 2045.3,240 2056,200 2136,190 2145,180 2155.6,140 + 2236,130 2244.9,120 2255.4,80 2336.3,70 2345.5,60 2356.1,20 2437,10 + 2446.7,0 2456.2,40 2536,50 2545,60 2555.7,100 2636.1,110 2645.5,120 + 2656,160 2850,230 2928.6,200 2941.2,190 2955.4,140 3034.3,130 3048,80 + 3061.4,70 3140.4,60 3150.2,20 3159.9,10 + `), + ahead: rows<[number, number | null]>(` + -341.4,70 -261.9,60 -251.6,20 -241.7,10 -167.2,0 -156.8,40 -141.9,50 + -67.6,60 -54.6,100 -41.3,110 35.7,120 44.9,130 54.2,160 135.6,170 + 155.4,220 235.8,230 255.7,200 336,190 355.7,140 435.9,130 455.6,80 + 535.6,70 555.9,20 631.1,10 658.8,40 735.7,50 756.2,100 837.7,110 + 855.6,160 936.2,170 956,220 1035.7,230 1045.3,240 1056.1,200 1140,190 + 1149.4,180 1159.6,140 1239.5,130 1248.6,120 1257.9,80 1339.3,70 + 1350.3,60 1359.8,20 1438.9,10 1451.8,0 1461.2,40 1539.2,50 1550.4,60 + 1559.9,100 1636.4,110 1645.8,120 1656.2,160 1735.8,170 1745,180 + 1755.6,220 1835.8,230 1844.9,240 1855.7,200 1935.8,190 1945.1,180 + 1957.6,140 2036,130 2045.3,120 2056,80 2136,70 2145,60 2155.6,20 2236,10 + 2244.9,0 2255.4,40 2336.3,50 2345.5,60 2356.1,100 2437,110 2446.7,120 + 2456.2,160 2536,170 2545,180 2555.7,220 2636.1,230 2645.5,240 2656,200 + 2850,130 2928.6,80 2941.2,70 2955.4,20 3034.3,10 3048,40 3061.4,50 + 3140.4,60 3150.2,100 3159.9,110 + `), + }, + { + block: {from: 1794215482774, to: 1794215485274, spanMs: 2500}, + animation: {from: 1794215482681, to: 1794215487184.9087, holds: true}, + // Times after 1794215482524. + geometry: rows<[number, number]>(` + 0,223 8,223 16,223 24,223 32,223 40,223 48,223 56,223 64,223 72,223 + 80,223 88,223 96,223 104,223 112,223 120,223 128,223 136,223 144,223 + 152,223 160,222.97 168,222.88 176,222.73 184,222.47 192,222.17 + 200,221.75 208,221.31 216,220.74 224,220.17 232,219.47 240,218.77 + 248,217.96 256,217.14 264,216.22 272,215.29 280,214.28 288,213.26 + 296,212.16 304,211.05 312,209.88 320,208.68 328,207.44 336,206.17 + 344,204.87 352,203.54 360,202.17 368,200.76 376,199.35 384,197.93 + 392,196.49 400,194.99 408,193.49 416,191.99 424,190.48 432,188.92 + 440,187.36 448,185.79 456,184.22 464,182.62 472,181.01 480,179.4 + 488,177.79 496,176.16 504,174.53 512,172.89 520,171.26 528,169.61 + 536,167.96 544,166.32 552,164.67 560,163.02 568,161.37 576,159.73 + 584,158.08 592,156.43 600,154.8 608,153.16 616,151.52 624,149.89 + 632,148.27 640,146.66 648,145.04 656,143.42 664,141.83 672,140.24 + 680,138.65 688,137.06 696,135.5 704,133.94 712,132.39 720,130.83 + 728,129.3 736,127.78 744,126.26 752,124.74 760,123.25 768,121.77 + 776,120.29 784,118.81 792,117.36 800,115.93 808,114.49 816,113.06 + 824,111.64 832,110.26 840,108.87 848,107.48 856,106.11 864,104.77 + 872,103.43 880,102.09 888,100.76 896,99.47 904,98.19 912,96.9 920,95.61 + 928,94.37 936,93.13 944,91.89 952,90.66 960,89.46 968,88.27 976,87.08 + 984,85.9 992,84.74 1000,83.61 1008,82.47 1016,81.33 1024,80.22 + 1032,79.13 1040,78.05 1048,76.96 1056,75.89 1064,74.85 1072,73.82 + 1080,72.78 1088,71.75 1096,70.76 1104,69.77 1112,68.78 1120,67.8 + 1128,66.86 1136,65.91 1144,64.97 1152,64.04 1160,63.13 1168,62.23 + 1176,61.34 1184,60.44 1192,59.58 1200,58.73 1208,57.88 1216,57.03 + 1224,56.2 1232,55.39 1240,54.59 1248,53.78 1256,52.99 1264,52.22 + 1272,51.46 1280,50.69 1288,49.94 1296,49.21 1304,48.49 1312,47.76 + 1320,47.04 1328,46.35 1336,45.67 1344,44.98 1352,44.3 1360,43.64 + 1368,42.99 1376,42.34 1384,41.7 1392,41.07 1400,40.46 1408,39.84 + 1416,39.23 1424,38.64 1432,38.06 1440,37.48 1448,36.9 1456,36.34 + 1464,35.79 1472,35.24 1480,34.7 1488,34.16 1496,33.64 1504,33.13 + 1512,32.61 1520,32.11 1528,31.61 1536,31.13 1544,30.64 1552,30.16 + 1560,29.7 1568,29.24 1576,28.78 1584,28.33 1592,27.89 1600,27.46 + 1608,27.03 1616,26.6 1624,26.18 1632,25.78 1640,25.37 1648,24.96 + 1656,24.57 1664,24.19 1672,23.81 1680,23.43 1688,23.06 1696,22.7 + 1704,22.34 1712,21.98 1720,21.63 1728,21.29 1736,20.95 1744,20.61 + 1752,20.28 1760,19.96 1768,19.64 1776,19.33 1784,19.01 1792,18.71 + 1800,18.41 1808,18.12 1816,17.82 1824,17.54 1832,17.26 1840,16.98 + 1848,16.7 1856,16.43 1864,16.17 1872,15.91 1880,15.65 1888,15.39 + 1896,15.15 1904,14.9 1912,14.66 1920,14.42 1928,14.18 1936,13.95 + 1944,13.72 1952,13.5 1960,13.28 1968,13.07 1976,12.85 1984,12.64 + 1992,12.43 2000,12.23 2008,12.03 2016,11.83 2024,11.64 2032,11.45 + 2040,11.26 2048,11.07 2056,10.89 2064,10.71 2072,10.53 2080,10.36 + 2088,10.19 2096,10.02 2104,9.85 2112,9.69 2120,9.53 2128,9.37 2136,9.22 + 2144,9.06 2152,8.91 2160,8.76 2168,8.62 2176,8.47 2184,8.33 2192,8.19 + 2200,8.06 2208,7.92 2216,7.79 2224,7.66 2232,7.53 2240,7.41 2248,7.28 + 2256,7.16 2264,7.04 2272,6.92 2280,6.81 2288,6.69 2296,6.58 2304,6.47 + 2312,6.36 2320,6.25 2328,6.15 2336,6.04 2344,5.94 2352,5.84 2360,5.74 + 2368,5.65 2376,5.55 2384,5.46 2392,5.36 2400,5.27 2408,5.18 2416,5.1 + 2424,5.01 2432,4.93 2440,4.84 2448,4.76 2456,4.68 2464,4.6 2472,4.52 + 2480,4.44 2488,4.37 2496,4.29 2504,4.22 2512,4.15 2520,4.08 2528,4.01 + 2536,3.94 2544,3.87 2552,3.81 2560,3.74 2568,3.68 2576,3.61 2584,3.55 + 2592,3.49 2600,3.43 2608,3.37 2616,3.31 2624,3.26 2632,3.2 2640,3.15 + 2648,3.09 2656,3.04 2664,2.99 2672,2.94 2680,2.89 2688,2.84 2696,2.79 + 2704,2.74 2712,2.69 2720,2.65 2728,2.6 2736,2.56 2744,2.51 2752,2.47 + 2760,2.42 2768,2.38 2776,2.34 2784,2.3 2792,2.26 2800,2.22 2808,2.18 + 2816,2.15 2824,2.11 2832,2.07 2840,2.04 2848,2 2856,1.97 2864,1.93 + 2872,1.9 2880,1.87 2888,1.83 2896,1.8 2904,1.77 2912,1.74 2920,1.71 + 2928,1.68 2936,1.65 2944,1.62 2952,1.59 2960,1.56 2968,1.54 2976,1.51 + 2984,1.48 2992,1.46 3000,1.43 + `).map(([t, y]) => [1794215482524 + t, y] as const), + controlStarts: [1794215479570.875, 1794215479370.875], + frames: rows<[number, number | null, number | null]>(` + -373,223,268 -298.6,223,268 -288.2,223,268 -275.8,223,268 -230.4,223,268 + -195.2,223,268 -185.2,223,268 -175.1,223,268 -130.3,223,268 + -93.5,223,268 -80.4,223,268 -66.9,223,268 6.2,223,268 15.6,222,270 + 28.1,218,278 101.2,216,282 122.7,204,306 199.8,202,310 209.5,199,316 + 223.2,186,342 300.7,183,348 323,166,382 399.8,162,390 422.8,145,424 + 500.2,142,430 522.3,126,462 600,123,468 623.4,108,498 699.3,105,504 + 722.8,91,532 799.4,89,536 809,86,542 823.1,77,560 900.6,75,564 + 911.6,73,568 925.8,65,584 1002.6,63,588 1012.1,61,592 1023.2,54,606 + 1100.8,52,610 1110.3,51,612 1123.1,45,624 1200.3,43,628 1210.1,42,630 + 1223.2,37,640 1300.4,36,642 1310.2,35,644 1323.2,30,654 1400.7,30,655 + 1411.2,29,656 1423.3,25,664 1500.1,24,666 1509.8,23,668 1523.8,20,674 + 1600.8,20,674 1610.8,19,676 1623,17,680 1700.4,16,682 1710.3,16,682 + 1723.3,14,686 1800.8,13,688 1810.3,13,688 1823,11,692 1900.5,11,692 + 1910,10,694 1922.9,9,696 2000.2,9,696 2009.9,8,698 2023.4,7,700 + 2106.2,7,700 2115.8,7,700 2125.3,6,702 2203.4,6,702 2212.8,6,703 + 2222.9,5,704 2300.7,5,704 2310.3,4,706 2322.9,4,706 2400.2,4,706 + 2410.1,4,706 2423.3,3,708 2500.2,3,708 2510,3,708 2523.3,2,710 + 2602.3,2,710 2611.8,2,710 2623.3,2,710 2802.6,2,710 2812.5,1,712 + 2823.2,1,712 2902.5,1,712 2912.1,1,712 2923.2,1,712 2969.9,1,712 + 3007.5,1,712 3017,1,712 3026.9,1,712 + `), + control: rows<[number, number | null]>(` + -373,230 -298.6,220 -288.2,180 -275.8,170 -230.4,160 -195.2,140 + -185.2,120 -175.1,110 -130.3,100 -93.5,80 -80.4,60 -66.9,50 6.2,40 + 15.6,30 28.1,0 101.2,10 122.7,60 199.8,70 209.5,80 223.2,120 300.7,130 + 323,180 399.8,190 422.8,240 500.2,230 522.3,180 600,170 623.4,120 + 699.3,110 722.8,60 799.4,50 809,40 823.1,0 900.6,10 911.6,20 925.8,60 + 1002.6,70 1012.1,80 1023.2,120 1100.8,130 1110.3,140 1123.1,180 + 1200.3,190 1210.1,200 1223.2,240 1300.4,230 1310.2,220 1323.2,180 + 1400.7,170 1411.2,160 1423.3,120 1500.1,110 1509.8,100 1523.8,60 + 1600.8,50 1610.8,40 1623,0 1700.4,10 1710.3,20 1723.3,60 1800.8,70 + 1810.3,80 1823,120 1900.5,130 1910,140 1922.9,180 2000.2,190 2009.9,200 + 2023.4,240 2106.2,230 2115.8,220 2125.3,180 2203.4,170 2212.8,160 + 2222.9,120 2300.7,110 2310.3,100 2322.9,60 2400.2,50 2410.1,40 2423.3,0 + 2500.2,10 2510,20 2523.3,60 2602.3,70 2611.8,80 2623.3,120 2802.6,190 + 2812.5,200 2823.2,240 2902.5,230 2912.1,220 2923.2,180 2969.9,170 + 3007.5,160 3017,140 3026.9,120 + `), + ahead: rows<[number, number | null]>(` + -373,110 -298.6,100 -288.2,60 -275.8,50 -230.4,40 -195.2,20 -185.2,0 + -175.1,10 -130.3,20 -93.5,40 -80.4,60 -66.9,70 6.2,80 15.6,90 28.1,120 + 101.2,130 122.7,180 199.8,190 209.5,200 223.2,240 300.7,230 323,180 + 399.8,170 422.8,120 500.2,110 522.3,60 600,50 623.4,0 699.3,10 722.8,60 + 799.4,70 809,80 823.1,120 900.6,130 911.6,140 925.8,180 1002.6,190 + 1012.1,200 1023.2,240 1100.8,230 1110.3,220 1123.1,180 1200.3,170 + 1210.1,160 1223.2,120 1300.4,110 1310.2,100 1323.2,60 1400.7,50 + 1411.2,40 1423.3,0 1500.1,10 1509.8,20 1523.8,60 1600.8,70 1610.8,80 + 1623,120 1700.4,130 1710.3,140 1723.3,180 1800.8,190 1810.3,200 1823,240 + 1900.5,230 1910,220 1922.9,180 2000.2,170 2009.9,160 2023.4,120 + 2106.2,110 2115.8,100 2125.3,60 2203.4,50 2212.8,40 2222.9,0 2300.7,10 + 2310.3,20 2322.9,60 2400.2,70 2410.1,80 2423.3,120 2500.2,130 2510,140 + 2523.3,180 2602.3,190 2611.8,200 2623.3,240 2802.6,170 2812.5,160 + 2823.2,120 2902.5,110 2912.1,100 2923.2,60 2969.9,50 3007.5,40 3017,20 + 3026.9,0 + `), + }, +]; diff --git a/internal/interaction-harness/src/busy.predicted.test.ts b/internal/interaction-harness/src/busy.predicted.test.ts new file mode 100644 index 0000000000000..a6a830848cbc3 --- /dev/null +++ b/internal/interaction-harness/src/busy.predicted.test.ts @@ -0,0 +1,538 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file busy.predicted.test.ts + * @input busy.ts + * @output Unit tests of the predicted-move rule. A frame predicted to move + * under one cast step is too little to judge. Every other frame must show a + * new position. A replay the proof cannot use falls back by name + * @position Harness unit tests, beside busy.unjudged.test.ts + */ + +import {describe, expect, it} from 'vitest'; + +import { + BusyFrameUnjudgedError, + BusyPredictionError, + busyWindowProof, + type BlockedWindow, +} from './busy'; +import {ZOOM_CASTS, type RecordedZoomCast} from './busy.predicted.fixture'; +import {FrameStarvationError, type TrackedFrame} from './frames'; + +const FROM = 1_000_000; +const FRAME = 1000 / 60; +/** 60 frames from 100 ms before the block to its end. */ +const TIMES = Array.from({length: 60}, (_, i) => FROM - 100 + i * FRAME); + +/** A curve that moves `pace(t)` px per 60 Hz frame, from 200 px at the block. */ +function curve(pace: (t: number) => number): (t: number) => number { + const samples: Array<[number, number]> = []; + let y = 200; + for (let t = FROM - 300; t <= FROM + 1300; t += 2) { + samples.push([t, y]); + if (t >= FROM) { + y -= (pace(t) * 2) / FRAME; + } + } + return t => { + const i = Math.min( + Math.max(Math.round((t - samples[0][0]) / 2), 0), + samples.length - 1, + ); + return samples[i][1]; + }; +} + +function window(at: (t: number) => number): BlockedWindow { + const geometry: Array<[number, number, number, number, number]> = []; + for (let t = FROM - 250; t <= FROM + 1250; t += 2) { + geometry.push([t, 0, at(t), 100, 100]); + } + return { + from: FROM, + to: FROM + 900, + spanMs: 900, + animation: {from: FROM, to: FROM + 2000, holds: true, geometry}, + }; +} + +/** The component's frames: its curve as the cast reads it, in whole px. */ +function frames( + at: (t: number) => number, + shown: (t: number, y: number) => number = (_t, y) => y, +): TrackedFrame[] { + return TIMES.map(t => ({ + t, + box: {x: 0, y: Math.round(shown(t, at(t))), width: 100, height: 100}, + pixels: 1, + step: {x: 1, y: 1}, + })); +} + +/** When the control started: it crosses 240 px and back every 800 ms. */ +const CONTROL_START = FROM - 100; +const control: TrackedFrame[] = TIMES.map(t => { + const phase = ((t - CONTROL_START) % 800) / 400; + const x = Math.round(240 * (phase < 1 ? phase : 2 - phase)); + return {t, box: {x, y: 0, width: 12, height: 12}, pixels: 144}; +}); +/** The control half a leg ahead, which reads the frames near the first one's turns. */ +const AHEAD_START = CONTROL_START - 200; +const ahead: TrackedFrame[] = TIMES.map(t => { + const phase = ((t - AHEAD_START) % 800) / 400; + const x = Math.round(240 * (phase < 1 ? phase : 2 - phase)); + return {t, box: {x, y: 16, width: 12, height: 12}, pixels: 144}; +}); +const options = { + axis: 'y', + control, + ahead, + controlStarts: [CONTROL_START, AHEAD_START], +} as const; + +/** A recorded cast's frames, its window with or without the replay, and its options. */ +function replay(cast: RecordedZoomCast, withReplay: boolean) { + const at = (t: number) => cast.block.from + t; + const tracked: TrackedFrame[] = cast.frames.map(([t, y, h]) => ({ + t: at(t), + box: y == null || h == null ? null : {x: 0, y, width: 100, height: h}, + pixels: 1, + step: {x: 1, y: 1}, + engine: 'chromium', + })); + const series = (rows: RecordedZoomCast['control'], top: number) => + rows.map(([t, x]): TrackedFrame => ({ + t: at(t), + box: x == null ? null : {x, y: top, width: 12, height: 12}, + pixels: x == null ? 0 : 144, + engine: 'chromium', + })); + const window: BlockedWindow = withReplay + ? { + ...cast.block, + animation: { + ...cast.animation, + geometry: cast.geometry.map(([t, y]) => [t, 0, y, 0, 0] as const), + }, + } + : { + ...cast.block, + animation: { + from: cast.animation.from, + to: cast.animation.to, + holds: cast.animation.holds, + }, + }; + return { + tracked, + window, + options: { + axis: 'y' as const, + settledWithin: 24, + control: series(cast.control, 0), + ...(withReplay + ? {ahead: series(cast.ahead, 16), controlStarts: cast.controlStarts} + : {}), + }, + }; +} + +describe('busyWindowProof on recorded Lightbox zoom casts', () => { + it('reds the first cast by the cast-step rule: a frame under a step reads as a miss', () => { + const {tracked, window, options} = replay(ZOOM_CASTS[0], false); + expect( + busyWindowProof(tracked, window, options).verdict.failures.join(' '), + ).toMatch(/moved on 34 of the 35 frames/); + }); + + it('excuses a frame of the second cast as under a step, which the zoom spec rejects', () => { + const {tracked, window, options} = replay(ZOOM_CASTS[1], false); + expect(busyWindowProof(tracked, window, options).control?.belowStep).toBe( + 1, + ); + }); + + for (const [name, offset] of [ + ['moves a quarter less', (y: number, y0: number) => y0 + 0.75 * (y - y0)], + [ + 'sits a quarter of its travel off', + (y: number, y0: number, travel: number) => y + travel / 4, + ], + ] as const) { + it(`never predicts with a replay that ${name}: the cast-step rule judges, and the notes say why`, () => { + const cast = ZOOM_CASTS[1]; + const y0 = cast.geometry[0][1]; + const travel = cast.geometry.at(-1)![1] - y0; + const {tracked, window, options} = replay( + { + ...cast, + geometry: cast.geometry.map( + ([t, y]) => [t, offset(y, y0, travel)] as const, + ), + }, + true, + ); + const proof = busyWindowProof(tracked, window, options); + expect(proof.notes.join(' ')).toMatch( + /no prediction: .* so the cast-step rule judges each frame/, + ); + expect(proof.unjudged).toEqual([]); + expect(proof.control?.belowStep).toBe(1); + // A proof without a control is not dated, and falls back the same way. + const alone = busyWindowProof(tracked, window, { + axis: 'y', + settledWithin: 24, + }); + expect(alone.notes.join(' ')).toMatch(/no prediction: /); + }); + } + + it('refuses the second cast by name in WebKit under judgeEveryFrame, and names its frame in Chromium', () => { + const {tracked, window, options} = replay(ZOOM_CASTS[1], true); + const as = (frames: readonly TrackedFrame[] | undefined) => + frames?.map(frame => ({...frame, engine: 'webkit' as const})); + expect(() => + busyWindowProof(as(tracked)!, window, { + ...options, + control: as(options.control), + ahead: as(options.ahead), + judgeEveryFrame: true, + }), + ).toThrow(BusyFrameUnjudgedError); + const proof = busyWindowProof(tracked, window, { + ...options, + judgeEveryFrame: true, + }); + expect(proof.verdict.failures).toEqual([]); + expect(proof.unjudged.some(u => u.why === 'step')).toBe(true); + }); + + for (const engine of ['chromium', 'webkit'] as const) { + it(`falls back to exactly the proof without a replay, in ${engine}`, () => { + const cast = ZOOM_CASTS[1]; + const y0 = cast.geometry[0][1]; + const travel = cast.geometry.at(-1)![1] - y0; + const off = replay( + { + ...cast, + geometry: cast.geometry.map(([t, y]) => [t, y + travel / 4] as const), + }, + true, + ); + const plain = replay(cast, false); + const as = (frames: readonly TrackedFrame[] | undefined) => + frames?.map(frame => ({...frame, engine})); + const options = { + ...off.options, + control: as(off.options.control), + ahead: as(off.options.ahead), + }; + const fellBack = busyWindowProof(as(off.tracked)!, off.window, options); + const without = busyWindowProof( + as(plain.tracked)!, + plain.window, + options, + ); + const note = fellBack.notes.find(n => n.startsWith('no prediction: ')); + expect(note).toBeDefined(); + expect({ + ...fellBack, + window: null, + notes: fellBack.notes.filter(n => n !== note), + summary: fellBack.summary.replace(`; note: ${note}`, ''), + }).toEqual({...without, window: null}); + }); + } + + for (const [n, cast] of ZOOM_CASTS.entries()) { + it(`judges cast ${n + 1} on the replay: frames under a step are named, the rest move`, () => { + const {tracked, window, options} = replay(cast, true); + const proof = busyWindowProof(tracked, window, options); + expect(proof.verdict.failures).toEqual([]); + expect(proof.control?.belowStep).toBe(0); + expect(proof.unjudged.some(u => u.why === 'step')).toBe(true); + expect(proof.control?.moved).toBeGreaterThanOrEqual(26); + }); + } +}); + +describe('busyWindowProof with a predicted move', () => { + it('throws the named excuse in WebKit under judgeEveryFrame, where the proof falls back', () => { + // A slow start: frames under one step a frame, which the cast's step + // excuses. `judgeEveryFrame` refuses them by name. + const at = curve(t => (t < FROM + 100 ? 0.5 : 3)); + const webkit = (frame: TrackedFrame): TrackedFrame => ({ + ...frame, + engine: 'webkit', + }); + expect(() => + busyWindowProof(frames(at).map(webkit), window(at), { + ...options, + control: control.map(webkit), + ahead: ahead.map(webkit), + judgeEveryFrame: true, + }), + ).toThrow(BusyFrameUnjudgedError); + }); + + it('names the same frames in Chromium under judgeEveryFrame, and throws nothing', () => { + const at = curve(t => (t < FROM + 100 ? 0.4 : 3)); + const proof = busyWindowProof(frames(at), window(at), { + ...options, + judgeEveryFrame: true, + }); + expect(proof.verdict.failures).toEqual([]); + expect(proof.unjudged.some(u => u.why === 'step')).toBe(true); + }); + + it('falls back by name without the control ahead', () => { + const at = curve(t => (t < FROM + 100 ? 0.4 : 3)); + const {ahead: _ahead, ...firstOnly} = options; + const proof = busyWindowProof(frames(at), window(at), firstOnly); + expect(proof.notes.join(' ')).toMatch( + /no prediction: the proof starts no ahead control, so a frame near a turn has no time of its own, so the cast-step rule judges each frame/, + ); + expect(proof.unjudged).toEqual([]); + }); + + it('always falls back by name in WebKit, whose box lags its animation at the loop end', () => { + // WebKit reads the element's box at the loop's end before its animation's + // current time, so even a replay that matches the cast is not used. + const at = curve(() => 3); + const webkit = (frame: TrackedFrame): TrackedFrame => ({ + ...frame, + engine: 'webkit', + }); + const proof = busyWindowProof(frames(at).map(webkit), window(at), { + ...options, + control: control.map(webkit), + ahead: ahead.map(webkit), + }); + expect(proof.notes.join(' ')).toMatch( + /no prediction: the element's box in WebKit lags its animation at the loop's end and cannot align the replay, so the cast-step rule judges each frame/, + ); + expect(proof.unjudged).toEqual([]); + }); + + it('names a frame predicted to move under one cast step, and judges the rest', () => { + // Slow at first (0.4 px a frame), then 3 px a frame. + const at = curve(t => (t < FROM + 100 ? 0.4 : 3)); + const proof = busyWindowProof(frames(at), window(at), options); + expect(proof.verdict.failures).toEqual([]); + const steps = proof.unjudged.filter(u => u.why === 'step'); + expect(steps.length).toBeGreaterThan(0); + expect(steps.every(u => u.movePx < 1)).toBe(true); + expect(proof.summary).toContain('too little to judge'); + expect(proof.control?.belowStep).toBe(0); + }); + + it('fails a frame predicted to move a full step that shows none', () => { + const at = curve(() => 3); + const proof = busyWindowProof( + frames(at, (t, y) => + Math.abs(t - (FROM + 400)) < 1 ? at(t - FRAME) : y, + ), + window(at), + options, + ); + expect(proof.verdict.failures.join(' ')).toMatch(/moved on \d+ of the \d+/); + }); + + it('fails a component held while its predicted moves add up to a step', () => { + // 0.4 px a frame: each frame alone is under the step, but a 300 ms hold + // misses the step its moves add up to. + const at = curve(() => 0.4); + const proof = busyWindowProof( + frames(at, (t, y) => + t > FROM + 200 && t < FROM + 500 ? at(FROM + 200) : y, + ), + window(at), + options, + ); + expect(proof.verdict.failures.join(' ')).toMatch(/moved on \d+ of the \d+/); + }); + + it('fails a hold that begins on a frame with no control reading', () => { + // Both controls are hidden in the frame that first shows the held + // position, so that frame has no time of its own. + const at = curve(() => 3); + const first = TIMES.find(t => t > FROM + 190)!; + const hidden = (frame: TrackedFrame): TrackedFrame => + frame.t === first ? {...frame, box: null, pixels: 0} : frame; + const proof = busyWindowProof( + frames(at, (t, y) => + t > FROM + 190 && t < FROM + 500 ? at(FROM + 190) : y, + ), + window(at), + {...options, control: control.map(hidden), ahead: ahead.map(hidden)}, + ); + expect(proof.verdict.failures.join(' ')).toMatch(/moved on \d+ of the \d+/); + }); + + it('starves by name, blaming the motion, when too few frames are left to judge', () => { + const at = curve(() => 0.05); + expect(() => busyWindowProof(frames(at), window(at), options)).toThrow( + /so \d+ were left to judge, where the proof needs 4\. .*the motion moved under one cast step/, + ); + }); + + it('falls back by name for a replay it cannot read, in WebKit and without the control ahead', () => { + const at = curve(() => 3); + const unreadable = { + ...window(at), + animation: { + from: FROM, + to: FROM + 2000, + holds: true, + unpredictable: 'an ancestor of the element moves too', + }, + }; + const webkit = (frame: TrackedFrame): TrackedFrame => ({ + ...frame, + engine: 'webkit', + }); + const {ahead: _ahead, ...firstOnly} = options; + for (const proof of [ + busyWindowProof(frames(at).map(webkit), unreadable, { + ...options, + control: control.map(webkit), + ahead: ahead.map(webkit), + }), + busyWindowProof(frames(at), unreadable, firstOnly), + busyWindowProof(frames(at), unreadable, {axis: 'y'}), + ]) { + expect(proof.notes.join(' ')).toMatch(/no prediction: /); + } + }); + + it('predicts from one channel when the replay reads a fade or a pseudo-element', () => { + // The channel runs 0 to 1 over 800 ms, and the tracked value follows it + // from its rest to its end. + const progressAt = (t: number) => + Math.min(Math.max((t - FROM) / 800, 0), 1); + const at = (t: number) => 200 - 120 * progressAt(t); + const fade: BlockedWindow = { + from: FROM, + to: FROM + 900, + spanMs: 900, + animation: { + from: FROM, + to: FROM + 800, + holds: true, + progress: Array.from({length: 751}, (_, k) => { + const t = FROM - 250 + 2 * k; + return [t, progressAt(t)] as const; + }), + }, + }; + const proof = busyWindowProof(frames(at), fade, options); + expect(proof.notes.join(' ')).not.toMatch(/no prediction/); + expect(proof.verdict.failures).toEqual([]); + const held = frames(at, (t, y) => + t > FROM + 200 && t < FROM + 400 ? at(FROM + 200) : y, + ); + const frozen = busyWindowProof(held, fade, options); + // The hold bends the fade's fit, so the proof falls back by name and + // judges the window exactly as without the replay. + expect(frozen.notes.join(' ')).toMatch( + /no prediction: the fade's fit and its anchors disagree by [\d.]+, more than one step/, + ); + const withoutReplay: BlockedWindow = { + ...fade, + animation: {from: FROM, to: FROM + 800, holds: true}, + }; + const without = busyWindowProof(held, withoutReplay, options); + const note = frozen.notes.find(n => n.startsWith('no prediction: ')); + expect({ + ...frozen, + window: null, + notes: frozen.notes.filter(n => n !== note), + summary: frozen.summary.replace(`; note: ${note}`, ''), + }).toEqual({...without, window: null}); + expect(frozen.verdict.failures.join(' ')).toMatch( + /moved on \d+ of the \d+/, + ); + }); + + it('fails a frozen component in a fast part', () => { + const at = curve(() => 3); + const proof = busyWindowProof( + frames(at, (t, y) => + t > FROM + 200 && t < FROM + 500 ? at(FROM + 200) : y, + ), + window(at), + options, + ); + expect(proof.verdict.failures.length).toBeGreaterThan(0); + }); + + it('fails a component that shows a new position on every other frame', () => { + const at = curve(() => 2); + const proof = busyWindowProof( + frames(at, (t, y) => + Math.round((t - TIMES[0]) / FRAME) % 2 === 1 ? at(t - FRAME) : y, + ), + window(at), + options, + ); + expect(proof.verdict.failures.join(' ')).toMatch(/moved on \d+ of the \d+/); + }); + + it('throws named starvation under the control floor, with and without a replay', () => { + const at = curve(() => 3); + const short = {...window(at), to: FROM + 60, spanMs: 60}; + const {animation: _animation, ...plain} = short; + for (const w of [short, plain]) { + expect(() => busyWindowProof(frames(at), w, options)).toThrow( + FrameStarvationError, + ); + } + }); + + it('throws named starvation when too few frames are left to judge', () => { + // 0.05 px a frame: the moves add up to a step only twice in the window. + const at = curve(() => 0.05); + expect(() => busyWindowProof(frames(at), window(at), options)).toThrow( + FrameStarvationError, + ); + }); + + it('fails by name a window it cannot predict', () => { + const at = curve(() => 3); + const unpredictable = { + ...window(at), + animation: { + from: FROM, + to: FROM + 2000, + holds: true, + unpredictable: 'an ancestor of the element moves too', + }, + }; + expect(() => busyWindowProof(frames(at), unpredictable, options)).toThrow( + BusyPredictionError, + ); + }); + + it('fails by name a replay that drifted from the element', () => { + const at = curve(() => 3); + const drifted = { + ...window(at), + animation: { + ...window(at).animation!, + check: {predicted: [0, 100, 100, 100], actual: [0, 104, 100, 100]}, + }, + } as BlockedWindow; + expect(() => busyWindowProof(frames(at), drifted, options)).toThrow( + BusyPredictionError, + ); + }); + + it('keeps the cast-step rule for a window with no replay', () => { + const at = curve(t => (t < FROM + 100 ? 0.4 : 3)); + const {animation: _animation, ...plain} = window(at); + const proof = busyWindowProof(frames(at), plain, options); + expect(proof.unjudged).toEqual([]); + expect(proof.control?.belowStep).toBeGreaterThan(0); + }); +}); diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index 77615037b2ef3..f0e0124fe267e 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -12,7 +12,8 @@ * blocks as one. `BusyWindowTooLateError`, for a block that began too late * for the motion it was meant to catch. `BusyFrameUnjudgedError`, for a * frame the cast could not judge in a proof that judges every frame - * (`judgeEveryFrame`). + * (`judgeEveryFrame`). `BusyPredictionError`, for a window whose motion the + * proof cannot predict. * `BUSY_PROOF_LOOKAROUND_MS` is how far around its judged part a proof * reads frames. * `trackControl` and `controlClock` read the control's box and its own @@ -225,7 +226,14 @@ export async function startBusyControl( Number(animation.currentTime ?? 0) >= 16 && (aheadMs === 0 || ticks >= 2) ) { - resolve(performance.timeOrigin + Number(animation.startTime)); + const start = performance.timeOrigin + Number(animation.startTime); + // Kept on the page for the block's window, which dates frames on + // the controls' clock (`BlockedWindow.controlStarts`). + const page = window as unknown as { + __harnessControlStarts?: number[]; + }; + (page.__harnessControlStarts ??= [])[aheadMs > 0 ? 1 : 0] = start; + resolve(start); } else if (realNow() > deadline) { reject(new Error('startBusyControl: the control did not start')); } else { @@ -459,8 +467,15 @@ function controlPageTimes( return null; } const dated = [...read.keys()]; + // Agreement is read against the run's usual lag: a cast whose stamps all + // trail their content by the same time agrees with itself. + const lags = dated + .map(stamp => stamp - (read.get(stamp) ?? stamp)) + .sort((a, b) => a - b); + const lag = lags[Math.floor(lags.length / 2)] ?? 0; const agreeing = dated.filter( - stamp => Math.abs(stamp - (read.get(stamp) ?? stamp)) <= CONTROL_AGREE_MS, + stamp => + Math.abs(stamp - (read.get(stamp) ?? stamp) - lag) <= CONTROL_AGREE_MS, ); const anchors = agreeing.length > 0 ? agreeing : dated; const times = new Map(); @@ -663,6 +678,12 @@ export interface BlockedWindow extends BusyWindow { * it started and ended on. It is never less than the time asked for. */ readonly spanMs: number; + /** + * The busy controls' starts on the page clock, epoch ms, when + * `startBusyControl` started any: the first control, then the one ahead. + * A proof dates its frames from them unless its options name others. + */ + readonly controlStarts?: readonly [number, number?]; /** * With `during`: the animation the block caught, epoch ms on the page's * clock. `from` is its start, and `to` is where its curve ends, its held @@ -671,11 +692,33 @@ export interface BlockedWindow extends BusyWindow { * element keeps the curve's end value after `to`. A held tail keeps it, and * so does a fill the engine's compositor honors (not WebKit on Linux). When * it does not, `busyWindowProof` judges the block up to `to`. + * + * Once the loop ends, the page also replays the caught animation, for the + * proof to predict each frame's move. It reads the replay every 2 ms, from + * `BUSY_PROOF_LOOKAROUND_MS` before the block to as long after its end. + * `geometry` holds the element's box as `[epoch ms, x, y, width, height]`, + * or the marker chip's when the element holds one. For a fade or a + * pseudo-element, `progress` holds one numeric channel scaled from 0 to 1. + * `check` is the replayed box and the element's own at the loop's end, once + * the replay is aligned to it. `unpredictable` says why neither could be + * read, and fails the proof. `notPredicted` says why the cast-step rule + * judges each frame instead. Either the caught element carries no harness + * marker, or the run left the element before its replay. */ readonly animation?: { readonly from: number; readonly to: number; readonly holds: boolean; + readonly geometry?: ReadonlyArray< + readonly [number, number, number, number, number] + >; + readonly check?: { + readonly predicted: readonly [number, number, number, number]; + readonly actual: readonly [number, number, number, number]; + }; + readonly progress?: ReadonlyArray; + readonly unpredictable?: string; + readonly notPredicted?: string; }; } @@ -764,12 +807,20 @@ export interface UnjudgedFrame { /** When the frame was presented, ms after the window began. */ readonly atMs: number; /** - * The component's move per frame there, px: the change that ends the run of - * equal positions that holds the frame, over the run's frames. + * The component's move per frame there, px. With a prediction, it is the + * predicted move since the frame before. Without one, it is the change + * that ends the run of equal positions that holds the frame, over the + * run's frames. */ readonly movePx: number; /** The cast's position step for the judged value, px. */ readonly stepPx: number; + /** + * With a prediction, why the proof names the frame. `step`: its predicted + * move falls under one cast step. `clock`: it, or the frame before it, has + * no control reading of its own to date it. + */ + readonly why?: 'step' | 'clock'; } /** @@ -798,6 +849,19 @@ export class BusyFrameUnjudgedError extends Error { } } +/** + * A busy proof could not predict the component's motion. An ancestor moved + * it too, a geometry animation composites onto another, or the replay that + * predicts it disagreed with the element or the cast. The run is not judged. + * It fails by name, never as a pass and never as a red on the component. + */ +export class BusyPredictionError extends Error { + constructor(message: string) { + super(`busyWindowProof: ${message}`); + this.name = 'BusyPredictionError'; + } +} + export async function blockMainThread( page: Page, ms: number, @@ -871,7 +935,7 @@ export async function blockMainThread( process.platform === 'linux' ); const waited = during.evaluate( - (element, {duration, minProgress, prop, pseudo, wait, fillHolds}) => + (element, {duration, minProgress, prop, pseudo, wait, fillHolds, look}) => new Promise((resolve, reject) => { // The harness's own waits run on the real timers: a frozen timer gate // (story.ts) holds only the page's. @@ -947,6 +1011,271 @@ export async function blockMainThread( let heldSeen: ReturnType | null = null; const tooLate = (numbers: ReturnType) => new Error(`BusyWindowTooLate ${JSON.stringify(numbers)}`); + // The caught animation, replayed once the loop ends, for the proof to + // predict each frame's move (`BlockedWindow.animation`). The element's + // geometry animations replay, paused, on a hidden clone of its box in + // the same parent. A fade or a pseudo-element replays one numeric + // channel instead, on a hidden registered . Each held seek + // reads the browser's own easing, and the element is untouched. + const geometry = ['translate', 'scale', 'rotate', 'transform']; + const keyframesOf = (a: Animation) => + (a.effect as KeyframeEffect | null)?.getKeyframes() ?? []; + const isGeometry = (a: Animation) => + keyframesOf(a).some(k => geometry.some(p => p in k)); + const realStart = (a: Animation) => + performance.timeOrigin + + Number(a.startTime) + + ((window as unknown as {__harnessFreeze?: {offset: number}}) + .__harnessFreeze?.offset ?? 0); + const boxOf = (el: Element): [number, number, number, number] => { + const r = el.getBoundingClientRect(); + return [r.x, r.y, r.width, r.height]; + }; + const replayOf = ( + caught: Animation, + fromMs: number, + toMs: number, + ): Record => { + const effect = caught.effect as KeyframeEffect | null; + const target = effect?.target; + if ( + target instanceof HTMLElement && + effect?.pseudoElement == null && + isGeometry(caught) + ) { + // Only the element that carries the harness marker predicts the + // tracked box. + if (!target.hasAttribute('data-harness-marker')) { + return { + notPredicted: 'the tracked marker is not on the caught element', + }; + } + for ( + let up = target.parentElement; + up != null; + up = up.parentElement + ) { + if ( + up + .getAnimations() + .some( + a => + isGeometry(a) && + a.playState !== 'finished' && + a.playState !== 'idle', + ) + ) { + return { + unpredictable: + 'an ancestor of the element moves too, so its box cannot be predicted', + }; + } + } + const runs = target.getAnimations().filter(isGeometry); + // A run that already left the element shows where it stopped, so a + // replay of the rest cannot place it. + if (!runs.includes(caught)) { + return { + notPredicted: 'the caught animation ended before its replay', + }; + } + if ( + runs.some( + a => + a.startTime == null || + (a.effect as KeyframeEffect).composite !== 'replace' || + keyframesOf(a).some( + k => + k.composite != null && + k.composite !== 'auto' && + k.composite !== 'replace', + ), + ) + ) { + return { + unpredictable: + 'a geometry animation of the element composites onto another or has not started, so its box cannot be predicted', + }; + } + if ( + getComputedStyle(target).position === 'fixed' || + target.parentElement == null + ) { + return { + unpredictable: + 'the element is fixed, so its box cannot be replayed in its parent', + }; + } + const style = getComputedStyle(target); + const clone = document.createElement('div'); + clone.setAttribute('aria-hidden', 'true'); + // The layout box to the sub-pixel: the computed size with its + // padding and border. The offset is in whole px: it is aligned + // below. + const px = (name: string) => + Number.parseFloat(style.getPropertyValue(name)) || 0; + const contentBox = style.boxSizing === 'content-box'; + const width = + px('width') + + (contentBox + ? px('padding-left') + + px('padding-right') + + px('border-left-width') + + px('border-right-width') + : 0); + const height = + px('height') + + (contentBox + ? px('padding-top') + + px('padding-bottom') + + px('border-top-width') + + px('border-bottom-width') + : 0); + clone.style.cssText = `position:absolute;left:${target.offsetLeft}px;top:${target.offsetTop}px;width:${width}px;height:${height}px;box-sizing:border-box;margin:0;padding:0;border:0;visibility:hidden;pointer-events:none;transform-origin:${style.transformOrigin};transform-box:${style.transformBox}`; + // A geometry property no run animates keeps the element's own value. + for (const name of geometry) { + if (!runs.some(a => keyframesOf(a).some(k => name in k))) { + clone.style.setProperty(name, style.getPropertyValue(name)); + } + } + const chip = target.querySelector( + ':scope > [data-harness-marker-chip]', + ); + let read: Element = clone; + if (chip instanceof HTMLElement) { + const copy = document.createElement('div'); + copy.style.cssText = chip.style.cssText; + clone.append(copy); + read = copy; + } + target.parentElement.append(clone); + const replays = runs.map(a => { + const replay = clone.animate( + keyframesOf(a).map(k => { + const {computedOffset, ...rest} = k as ComputedKeyframe; + return {...rest, offset: computedOffset}; + }), + (a.effect as KeyframeEffect).getTiming(), + ); + replay.pause(); + return { + replay, + source: a, + origin: realStart(a), + rate: a.playbackRate, + }; + }); + const seek = (t: number) => { + for (const {replay, origin, rate} of replays) { + replay.currentTime = (t - origin) * rate; + } + }; + // Aligned once: each replay at its own animation's current time, + // which the element's own box is read at, then the clone moved by + // what remains. The cast's frame at rest checks the replay. + for (const {replay, source} of replays) { + replay.currentTime = source.currentTime; + } + const unaligned = boxOf(read); + const actual = boxOf(chip ?? target); + clone.style.left = `${target.offsetLeft + actual[0] - unaligned[0]}px`; + clone.style.top = `${target.offsetTop + actual[1] - unaligned[1]}px`; + const check = {predicted: boxOf(read), actual}; + const samples: Array<[number, number, number, number, number]> = []; + for (let t = fromMs; t <= toMs; t += 2) { + seek(t); + samples.push([t, ...boxOf(read)]); + } + for (const {replay} of replays) { + replay.cancel(); + } + clone.remove(); + return {geometry: samples, check}; + } + // One numeric channel: the one that changes the most, for its size. + const keyframes = keyframesOf(caught); + const meta = new Set([ + 'offset', + 'computedOffset', + 'easing', + 'composite', + ]); + const numbers = (value: unknown): number[] => + ( + String(value).match(/-?(?:\d+\.?\d*|\.\d+)(?:e[-+]?\d+)?/gi) ?? [] + ).map(Number); + let channel: { + key: string; + index: number; + first: number; + last: number; + } | null = null; + let largest = 0; + for (const key of Object.keys(keyframes[0] ?? {})) { + if (meta.has(key)) { + continue; + } + const values = keyframes.map(k => + numbers((k as Record)[key]), + ); + const width = Math.min(...values.map(v => v.length)); + for (let index = 0; index < width; index += 1) { + const first = values[0][index]; + const last = values[values.length - 1][index]; + const size = + Math.abs(last - first) / + Math.max(Math.abs(first), Math.abs(last)); + if (last !== first && size > largest) { + largest = size; + channel = {key, index, first, last}; + } + } + } + if (channel == null || effect == null) { + return {}; + } + const {key, index, first, last} = channel; + const name = '--harness-busy-progress'; + try { + CSS.registerProperty({ + name, + syntax: '', + inherits: false, + initialValue: '0', + }); + } catch { + // An earlier block registered it. + } + const probe = document.createElement('div'); + probe.setAttribute('aria-hidden', 'true'); + probe.style.cssText = + 'position:fixed;left:0;top:0;width:0;height:0;visibility:hidden;pointer-events:none'; + document.body.append(probe); + const replay = probe.animate( + keyframes.map(k => ({ + offset: (k as ComputedKeyframe).computedOffset, + easing: k.easing, + [name]: String( + (numbers((k as Record)[key])[index] - first) / + (last - first), + ), + })), + {...effect.getTiming(), fill: 'both'}, + ); + replay.pause(); + const origin = realStart(caught); + const progress: Array<[number, number]> = []; + for (let t = fromMs; t <= toMs; t += 2) { + replay.currentTime = (t - origin) * caught.playbackRate; + progress.push([ + t, + Number(getComputedStyle(probe).getPropertyValue(name)), + ]); + } + replay.cancel(); + probe.remove(); + return {progress}; + }; // The animations that counted at the frame before, pending ones too. let seen = new Set(); const tick = () => { @@ -1010,11 +1339,35 @@ export async function blockMainThread( while (now - start < duration) { now = read(); } + const blockFrom = performance.timeOrigin + start; + const blockTo = performance.timeOrigin + now; + const controlStarts = ( + window as unknown as {__harnessControlStarts?: number[]} + ).__harnessControlStarts; resolve({ - from: performance.timeOrigin + start, - to: performance.timeOrigin + now, + from: blockFrom, + to: blockTo, spanMs: now - start, - ...(animation == null ? {} : {animation}), + ...(controlStarts?.[0] == null + ? {} + : { + controlStarts: [controlStarts[0], controlStarts[1]] as [ + number, + number?, + ], + }), + ...(animation == null + ? {} + : { + animation: { + ...animation, + ...replayOf( + caught, + blockFrom - look, + Math.min(blockTo, animation.to) + look, + ), + }, + }), }); }, 0); } else if (running) { @@ -1046,6 +1399,7 @@ export async function blockMainThread( pseudo: pseudoElement ?? null, wait: timeout, fillHolds, + look: BUSY_PROOF_LOOKAROUND_MS, }, ); try { @@ -1159,6 +1513,11 @@ export interface BusyProof extends FrameStats { readonly movedFraction: number; /** With a `control`: what it presented in the judged part; null without one. */ readonly control: BusyControlRead | null; + /** + * The control's frames too little to judge: the component's predicted move + * there fell under one cast step (see `busyWindowProof`). + */ + readonly unjudged: readonly UnjudgedFrame[]; /** The judgment against the limits (see `busyWindowProof`). */ readonly verdict: BusyVerdict; /** One line with the raw numbers, for an annotation or a failure message. */ @@ -1219,6 +1578,155 @@ export function castStep( return Math.max(...steps); } +/** Frames too little to judge, as a summary or a failure names them. */ +function unjudgedList(frames: readonly UnjudgedFrame[]): string { + return frames + .map(frame => + frame.why === 'clock' + ? `+${frame.atMs.toFixed(0)} ms, no control reading dates it` + : `+${frame.atMs.toFixed(0)} ms, a predicted ${frame.movePx.toFixed(2)} under the ${frame.stepPx} step`, + ) + .join('; '); +} + +/** A sampled curve read at any time, linear between samples, held at its ends. */ +function sampled( + points: ReadonlyArray, +): (t: number) => number { + return t => { + if (t <= points[0][0]) { + return points[0][1]; + } + const last = points[points.length - 1]; + if (t >= last[0]) { + return last[1]; + } + let lo = 0; + let hi = points.length - 1; + while (hi - lo > 1) { + const mid = (lo + hi) >> 1; + if (points[mid][0] <= t) { + lo = mid; + } else { + hi = mid; + } + } + const [t0, v0] = points[lo]; + const [t1, v1] = points[hi]; + return v0 + ((v1 - v0) * (t - t0)) / (t1 - t0); + }; +} + +/** + * What the component should show at each time, from the animation + * `blockMainThread` caught and replayed (`BlockedWindow.animation`). Null + * when the window holds no replay to read. Throws `BusyPredictionError` when + * the replay could not be read, or its end check disagrees with the element + * by more than one cast step. A replay that disagrees with the cast gives a + * `fallback` reason instead: its travel, its first judged position, or a + * fade's fit against its anchors. + */ +function predictionOf( + tracked: readonly TrackedFrame[], + axis: 'x' | 'y' | 'width' | 'height', + window: BusyWindow, + judged: {from: number; to: number}, + stepPx: number, + settledAt: number | undefined, +): ((t: number) => number) | {readonly fallback: string} | null { + const animation = (window as BlockedWindow).animation; + if (animation == null) { + return null; + } + if (animation.unpredictable != null) { + throw new BusyPredictionError( + `${animation.unpredictable}. The window is not judged.`, + ); + } + const column = {x: 0, y: 1, width: 2, height: 3}[axis]; + const shown = tracked.flatMap(f => + f.box == null ? [] : [{t: f.t, value: f.box[axis]}], + ); + const geometry = animation.geometry; + if (geometry != null && geometry.length > 1) { + const value = sampled( + geometry.map(sample => [sample[0], sample[1 + column]]), + ); + const check = animation.check; + if ( + check != null && + Math.abs(check.predicted[column] - check.actual[column]) > stepPx + ) { + throw new BusyPredictionError( + `the replay drifted from the element: at the block's end it read ${check.predicted[column].toFixed(2)} for the ${axis}, the element ${check.actual[column].toFixed(2)}. The window is not judged.`, + ); + } + // Over the judged part, the replay sits and moves as the cast does, to + // within two steps or a tenth of the travel. A replay of another element, + // or of a run the element no longer follows, sits or moves another way. + const judgedShown = shown.filter( + f => f.t >= judged.from && f.t <= judged.to, + ); + const first = judgedShown[0]; + const last = judgedShown.at(-1); + if (first != null && last != null && last !== first) { + const castTravel = last.value - first.value; + const allowed = Math.max(2 * stepPx, Math.abs(castTravel) / 10); + const replayTravel = value(last.t) - value(first.t); + if (Math.abs(replayTravel - castTravel) > allowed) { + return { + fallback: `over the judged part the replay moved ${replayTravel.toFixed(2)} for the ${axis}, the cast ${castTravel.toFixed(2)}`, + }; + } + if (Math.abs(value(first.t) - first.value) > allowed) { + return { + fallback: `at the first judged frame the replay read ${value(first.t).toFixed(2)} for the ${axis}, the cast ${first.value.toFixed(2)}`, + }; + } + } + return value; + } + if (animation.progress != null && animation.progress.length > 1) { + const progress = sampled(animation.progress); + // Anchored on frames outside the judged part: the value at rest before + // the motion, and where it ends. Frames judged never shape their own + // prediction. + const start = shown.filter(f => f.t <= animation.from).at(-1)?.value; + const end = + settledAt ?? shown.filter(f => f.t >= animation.to).at(-1)?.value; + if (start == null || end == null) { + return null; + } + const value = (t: number) => start + (end - start) * progress(t); + // A least-squares fit of the same curve checks the anchors. + const points = shown.map(f => [progress(f.t), f.value] as const); + const meanP = points.reduce((sum, [p]) => sum + p, 0) / points.length; + const meanV = points.reduce((sum, [, v]) => sum + v, 0) / points.length; + const spread = points.reduce((sum, [p]) => sum + (p - meanP) ** 2, 0); + if (spread > 0) { + const slope = + points.reduce((sum, [p, v]) => sum + (p - meanP) * (v - meanV), 0) / + spread; + const worst = Math.max( + ...shown + .filter(f => f.t >= judged.from && f.t <= judged.to) + .map(f => + Math.abs(meanV + slope * (progress(f.t) - meanP) - value(f.t)), + ), + ); + // A component that holds or skips bends the fit too, so a mismatch + // falls back to the cast-step rule, which judges that hold. + if (worst > stepPx) { + return { + fallback: `the fade's fit and its anchors disagree by ${worst.toFixed(2)}, more than one step (${stepPx}), so the component may have held`, + }; + } + } + return value; + } + return null; +} + /** * The step a tracked value's changes share: the largest of 2, 1 and 0.5 CSS * px that every change between consecutive positions is a multiple of. @@ -1380,11 +1888,27 @@ function motionStepPx(values: readonly number[]): number { * verdict. `pooledBusyProof` reads each run without the throw and judges the * runs together. * + * With a prediction, each frame the control moved on is judged by the + * component's own predicted move since its last new position. The prediction is + * the caller's (`predicted`), or the animation `blockMainThread` caught, + * replayed once the loop ended (`window.animation`). A replay predicts in + * Chromium when the proof passes the `ahead` control. A geometry replay also + * needs the harness marker on the caught element. A move of one cast step or + * more always changes the cast's reading, whatever the pixel phase. So such a + * frame must show a new position, with no run excused. A predicted move under + * one step may show none: that frame is too little to judge. It is listed in + * `unjudged` and in the summary, and counts neither as moved nor as missed. + * When too few judged frames remain, the proof throws `FrameStarvationError` + * naming them. Where a replay predicts, one it cannot read throws + * `BusyPredictionError`. Every other window falls back by name to the cast-step + * rule above, exactly as without a replay. + * * With `judgeEveryFrame`, a frame the cast could not show the component's * step on is too little to judge (the frames `belowStep` counts). When the * verdict otherwise passes, the proof throws `BusyFrameUnjudgedError` with * each such frame, the component's move there and the cast's step. A failed - * verdict is returned as it is: it is the component's red. + * verdict is returned as it is: it is the component's red. A prediction + * excuses no frame, so then nothing throws: it names such frames instead. */ export function busyWindowProof( ...args: Parameters @@ -1443,40 +1967,110 @@ function readBusyWindow( controlStarts?: readonly [number, number?]; } = {}, ): ReturnType { - const {control} = options; + const {control, predicted} = options; + // A prediction reads each frame at its content's time. The cast's stamps + // trail that time, by 26 to 31 ms at rest in Chromium and unevenly under + // load. So a window with a prediction is dated in every engine. + const animation = (window as BlockedWindow).animation; + const predicts = + predicted != null || + animation?.geometry != null || + animation?.progress != null || + animation?.unpredictable != null; + const fallBack = (why: string) => { + // The replay cannot predict the frames: the cast-step rule judges them + // as if there were no replay, and the notes say why. + const { + from: start, + to: end, + holds, + } = animation ?? { + from: window.from, + to: window.to, + holds: false, + }; + const plain: BlockedWindow = { + ...(window as BlockedWindow), + animation: {from: start, to: end, holds, notPredicted: why}, + }; + return readBusyWindow(tracked, plain, {...options, ahead, controlStarts}); + }; + // WebKit reads the element's box at the loop's end before its animation's + // current time, so the replay cannot be aligned to it there. + if ( + animation != null && + predicted == null && + predicts && + tracked.some(frame => frame.engine === 'webkit') + ) { + return fallBack( + "the element's box in WebKit lags its animation at the loop's end and cannot align the replay", + ); + } + // A prediction reads each frame at its own time, which only a control + // reading gives. Near the first control's turns, only the one ahead reads. + if ( + animation != null && + predicted == null && + predicts && + (control == null || ahead == null) + ) { + return fallBack( + 'the proof starts no ahead control, so a frame near a turn has no time of its own', + ); + } + // A prediction is on the page clock, so its frames are dated from the + // controls' own starts. The window keeps them when the options name none. + const starts = + controlStarts ?? + (predicts ? (window as BlockedWindow).controlStarts : undefined); const dated = - control != null && control.every(frame => frame.engine === 'webkit') - ? datedOnControlClock(tracked, control, ahead, controlStarts) + control != null && + (predicts || control.every(frame => frame.engine === 'webkit')) + ? datedOnControlClock(tracked, control, ahead, starts) : null; if (dated == null) { - return judgeBusyWindow(tracked, window, options); + const read = judgeBusyWindow(tracked, window, options); + return read.fallback == null ? read : fallBack(read.fallback); } - const shown = atDisplayRate(dated); + const webkit = control?.every(frame => frame.engine === 'webkit') === true; + const shown = webkit ? atDisplayRate(dated) : dated; + const unread = new Set( + shown.tracked.filter(frame => !shown.readTimes.has(frame.t)), + ); const read = judgeBusyWindow( shown.tracked, window, {...options, control: shown.control}, dated.tracked, + unread, ); + if (read.fallback != null) { + return fallBack(read.fallback); + } // A WebKit cast rarely shows two frames one interval apart, so the proof // cannot judge a 30 Hz cadence there. It says so, with the count. - const {pairs, unmatched} = displayIntervalPairs( - shown.tracked, - shown.control, - read.proof.judged, - ); + const cadence = webkit + ? displayIntervalPairs(shown.tracked, shown.control, read.proof.judged) + : null; const note = - `cadence not judged: ${pairs} display-interval pairs` + - (unmatched > 0 - ? `, ${unmatched} without a control frame at their time` - : ''); + cadence == null + ? null + : `cadence not judged: ${cadence.pairs} display-interval pairs` + + (cadence.unmatched > 0 + ? `, ${cadence.unmatched} without a control frame at their time` + : ''); return { ...read, proof: { ...read.proof, judged: judgedOnStamps(read.proof.judged, shown), - notes: [...read.proof.notes, note], - summary: `${read.proof.summary}; note: ${note}`, + ...(note == null + ? {} + : { + notes: [...read.proof.notes, note], + summary: `${read.proof.summary}; note: ${note}`, + }), }, }; } @@ -1499,6 +2093,7 @@ function judgeBusyWindow( presented, control, judgeEveryFrame = false, + predicted, }: { axis?: 'x' | 'y' | 'width' | 'height'; marginMs?: number; @@ -1571,18 +2166,28 @@ function judgeBusyWindow( * component's own step. A frame the cast could not show that step on * (`belowStep`) is then too little to judge, and `busyWindowProof` throws * `BusyFrameUnjudgedError`. Such a frame is never a pass, and never a red - * on the component. + * on the component. With a prediction, no frame is excused, so none + * throws. */ judgeEveryFrame?: boolean; + /** + * The tracked value the component should show at an epoch ms (or, after + * a page freeze, on the window's clock). Defaults to the caught + * animation's replay (`window.animation`). + */ + predicted?: (t: number) => number; } = {}, /** * The frames the cast drew, dated. In WebKit they are the frames before the * display-rate keep, so a frame left out never lengthens the cast's gap. */ drawn: readonly TrackedFrame[] = tracked, + /** The frames dated from another frame's reading (`datedOnControlClock`). */ + unread: ReadonlySet = new Set(), ): { proof: BusyProof; starvation: FrameShortfall | null; + /** Under `judgeEveryFrame`, the frames the cast's step excused. */ unjudged: readonly UnjudgedFrame[]; /** * The component failed on frames the control moved on: it missed one, or @@ -1592,6 +2197,8 @@ function judgeBusyWindow( componentFailed: boolean; /** The frames the cast drew in the block, and its longest gap there, ms. */ cast: {frames: number; longestGapMs: number}; + /** Why the replay could not predict the frames: the caller judges again. */ + fallback?: string; } { let end = window.to - marginMs; if (settledWithin != null && settledWithinSteps != null) { @@ -1934,6 +2541,12 @@ function judgeBusyWindow( ? 0 : Math.abs(last.value - first.value) / ((last.t - first.t) / cadence); const notes: string[] = []; + const notPredicted = (window as BlockedWindow).animation?.notPredicted; + if (notPredicted != null) { + notes.push( + `no prediction: ${notPredicted}, so the cast-step rule judges each frame`, + ); + } if (presented != null && compositorFrameMs == null) { notes.push( 'no compositor frames recorded around the window: judged from the screencast alone', @@ -1978,6 +2591,36 @@ function judgeBusyWindow( return changes; }; let controlRead: BusyControlRead | null = null; + // The cast step a predicted move is judged against: the coarser of the + // motion's own step and the frames' own (`castStep`). + const judgeStepPx = Math.max( + stepPx, + tracked.some(f => f.step != null) ? castStep(tracked, axis) : 0, + ); + const prediction = + predicted ?? + predictionOf( + tracked.filter(f => !unread.has(f)), + axis, + window, + judged, + judgeStepPx, + settledAt, + ); + if (prediction != null && typeof prediction !== 'function') { + return { + proof: null as unknown as BusyProof, + starvation: null, + unjudged: [], + componentFailed: false, + cast: {frames: 0, longestGapMs: 0}, + fallback: prediction.fallback, + }; + } + const predictedAt = prediction; + // With a prediction, the frames it names as too little to judge. + const named: UnjudgedFrame[] = []; + // Without one, under `judgeEveryFrame`: the frames the cast's step excused. let unjudged: UnjudgedFrame[] = []; if (control != null) { const controlChanges = controlChangesUpTo(control, judged.to); @@ -1988,7 +2631,7 @@ function judgeBusyWindow( // A freeze ends in a larger catch-up, and a run that no change ends stays // a miss. const series = tracked.flatMap(f => - f.box == null ? [] : [{t: f.t, value: f.box[axis]}], + f.box == null ? [] : [{t: f.t, value: f.box[axis], frame: f}], ); const seriesIndex = new Map(series.map((f, i) => [f.t, i])); const same = (a: number, b: number) => @@ -2116,14 +2759,51 @@ function judgeBusyWindow( const before = start > 0 ? start - 1 - runStart(start - 1) : 0; return unchanged <= before + 1; }; - const componentEdges = [ - judged.from, - ...delivered.filter(t => changed.has(t) || underStep(t)), - judged.to, - ]; - const excused = controlChanges.filter(underStep); - if (judgeEveryFrame) { - unjudged = excused.map(t => { + // With a prediction, a frame predicted to move under one cast step since + // the last new position is too little to judge. It is named, and counts + // neither as moved nor as missed. Every other frame the control moved on + // must show a new position, so a hold ends once its moves add up to a + // step. Without a prediction, a run the cast's step explains counts as + // moved. + const excluded = new Set(); + if (predictedAt != null) { + for (const t of controlChanges) { + const i = seriesIndex.get(t); + if (i == null || i === 0) { + continue; + } + let last = i - 1; + while ( + last > 0 && + Math.abs(series[last].value - series[last - 1].value) <= epsilon + ) { + last -= 1; + } + // The run's first frame with a control reading of its own anchors the + // move. A frame dated from another frame's reading has no time of + // its own. + let anchor = last; + while (anchor < i - 1 && unread.has(series[anchor].frame)) { + anchor += 1; + } + const move = Math.abs(predictedAt(t) - predictedAt(series[anchor].t)); + const undated = + unread.has(series[i].frame) || unread.has(series[i - 1].frame); + if (undated || move < judgeStepPx) { + excluded.add(t); + named.push({ + atMs: t - window.from, + movePx: move, + stepPx: judgeStepPx, + why: undated ? 'clock' : 'step', + }); + } + } + } + const excused = (t: number) => predictedAt == null && underStep(t); + const judgedChanges = controlChanges.filter(t => !excluded.has(t)); + if (judgeEveryFrame && predictedAt == null) { + unjudged = judgedChanges.filter(excused).map(t => { const i = seriesIndex.get(t); if (i == null) { throw new Error( @@ -2142,11 +2822,15 @@ function judgeBusyWindow( }; }); } + const componentEdges = [ + judged.from, + ...delivered.filter(t => changed.has(t) || excluded.has(t) || excused(t)), + judged.to, + ]; controlRead = { - moved: controlChanges.length, - movedWith: controlChanges.filter(t => changed.has(t) || underStep(t)) - .length, - belowStep: excused.length, + moved: judgedChanges.length, + movedWith: judgedChanges.filter(t => changed.has(t) || excused(t)).length, + belowStep: judgedChanges.filter(excused).length, longestStallMs: Math.max( ...controlEdges.slice(1).map((t, i) => t - controlEdges[i]), ), @@ -2159,7 +2843,8 @@ function judgeBusyWindow( // block's end would have held enough. Otherwise the run fails as before, // as starvation when the cast was starved. if (endsInBlock) { - const framesLeft = controlRead?.moved ?? expectedFrames; + const framesLeft = + controlRead == null ? expectedFrames : controlRead.moved + named.length; const framesWithoutCurve = control == null ? Math.floor( @@ -2182,7 +2867,23 @@ function judgeBusyWindow( } if (controlRead != null) { const visible = controlRead.movedWith - controlRead.belowStep; - if (controlRead.moved < minJudgedFrames) { + if ( + controlRead.moved < minJudgedFrames && + named.length > 0 && + controlRead.moved + named.length >= minJudgedFrames + ) { + // The prediction named too many frames too little to judge, so too + // few are left: the run proves nothing. + starvation = { + by: 'busyWindowProof', + during: `a ${Math.round(window.to - window.from)} ms block held the main thread, and ${named.length} of the control's ${controlRead.moved + named.length} frames were too little to judge (${unjudgedList(named)}), so ${controlRead.moved} were left to judge`, + frames: cast.frames, + min: minJudgedFrames, + longestGapMs: cast.longestGapMs, + cause: + 'the motion moved under one cast step, or had no reading of its own, on too many frames, so the run proves nothing about the motion', + }; + } else if (controlRead.moved < minJudgedFrames) { // The control is a compositor animation. When it moved on too few // frames, the compositor presented too little to judge the component, // however long the cast's longest gap was. @@ -2257,6 +2958,9 @@ function judgeBusyWindow( (controlRead == null ? '' : `; against the control: moved on ${controlRead.movedWith} of its ${controlRead.moved} frames${controlRead.belowStep > 0 ? ` (${controlRead.belowStep} where the cast could not show the step)` : ''}, its longest stall ${controlRead.longestStallMs.toFixed(1)} ms`) + + (named.length === 0 + ? '' + : `; too little to judge: ${unjudgedList(named)}`) + notes.map(note => `; note: ${note}`).join(''); const proof = { ...stats, @@ -2277,6 +2981,7 @@ function judgeBusyWindow( moved, movedFraction, control: controlRead, + unjudged: named, verdict: {pass: failures.length === 0, failures}, summary, }; @@ -2382,11 +3087,13 @@ export function pooledBusyProof( runs: readonly BusyRun[], options: Omit< NonNullable[2]>, + // A prediction belongs to one window, never to the pool. | 'control' | 'ahead' | 'controlStarts' | 'minMovedFraction' | 'judgeEveryFrame' + | 'predicted' > & { /** The fewest frames a run's control must move on for the run to count. */ minRunFrames?: number; diff --git a/packages/core/src/Lightbox/__tests__/LightboxZoom.interaction.browser.spec.ts b/packages/core/src/Lightbox/__tests__/LightboxZoom.interaction.browser.spec.ts index 3ad7af116ae6c..6fbff925b2b4d 100644 --- a/packages/core/src/Lightbox/__tests__/LightboxZoom.interaction.browser.spec.ts +++ b/packages/core/src/Lightbox/__tests__/LightboxZoom.interaction.browser.spec.ts @@ -28,6 +28,7 @@ import { blockMainThread, BusyWindowTooLateError, BusyFrameUnjudgedError, + BusyPredictionError, busyWindowProof, type BusyProof, type UnjudgedFrame, @@ -59,6 +60,7 @@ import { timeline, touchDriver, traceRendering, + trackControl, trackMarker, waitForAnimation, waitForRest, @@ -1047,7 +1049,10 @@ test('the double-tap zoom keeps advancing every frame while the main thread is b await markElement(photo, {mode: 'outline'}); // A compositor-only control animation in the same window: what the // compositor presented while the main thread spun, on this machine. - await startBusyControl(page); + const busyControl = await startBusyControl(page); + // A second control half a leg ahead reads the frames near the first one's + // turns. So every frame has a time of its own, and the proof predicts. + const aheadControl = await startBusyControl(page, {ahead: true}); const at = {x: 201, y: before.stage.y + before.stage.height / 2}; // The same zoom at the default tempo, to compare the slowed one with. await driver.play(doubleTap(at)); @@ -1084,9 +1089,13 @@ test('the double-tap zoom keeps advancing every frame while the main thread is b const castFrames = await cast.stop(); const tracked = await trackMarker(page, castFrames); const control = await trackMarker(page, castFrames, {rgb: CONTROL_RGB}); + const ahead = await trackControl(page, castFrames, {ahead: true}); // The run's frames and storyboard, kept whatever the proof concludes. const record = async ( - result: {proof: BusyProof} | {unjudged: ReadonlyArray}, + result: + | {proof: BusyProof} + | {unjudged: ReadonlyArray} + | {error: string}, judged: {from: number; to: number}, ) => { await writeEvidence( @@ -1095,6 +1104,7 @@ test('the double-tap zoom keeps advancing every frame while the main thread is b [`${testInfo.project.name}.json`]: JSON.stringify( { block, + controlStarts: [busyControl.start, aheadControl.start], ...result, frames: tracked.map(f => ({ t: +(f.t - block.from).toFixed(1), @@ -1105,6 +1115,10 @@ test('the double-tap zoom keeps advancing every frame while the main thread is b t: +(f.t - block.from).toFixed(1), x: f.box?.x ?? null, })), + ahead: ahead.map(f => ({ + t: +(f.t - block.from).toFixed(1), + x: f.box?.x ?? null, + })), }, null, 1, @@ -1123,21 +1137,29 @@ test('the double-tap zoom keeps advancing every frame while the main thread is b }); await film.write(testInfo, {block, judged}); }; - // Every frame the control moves on is judged by the edge's own step. A - // frame the cast's 1 px step cannot show the edge's move on is too little - // to judge, and fails by name (`BusyFrameUnjudgedError`). That run keeps its - // frames and storyboard too. + // Every frame the control moves on is judged by the edge's predicted move. + // A frame predicted to move under the cast's step is named in `unjudged`. + // Where the proof falls back to the cast's step, as in WebKit, a frame + // that step cannot show is too little to judge. It fails by name + // (`BusyFrameUnjudgedError`). Each such run keeps its frames and storyboard. let proof: BusyProof; try { proof = busyWindowProof(tracked, block, { axis: ZOOM_AXIS, settledWithin: ZOOM_SETTLED_PX, control, + ahead, + controlStarts: [busyControl.start, aheadControl.start], judgeEveryFrame: true, }); } catch (error) { if (error instanceof BusyFrameUnjudgedError) { await record({unjudged: error.frames}, block); + } else if ( + error instanceof FrameStarvationError || + error instanceof BusyPredictionError + ) { + await record({error: error.message}, block); } throw error; }