From e1f9922afdc43cc7fd4c1816a671834a00514849 Mon Sep 17 00:00:00 2001 From: Christopher Chedeau Date: Mon, 9 Nov 2026 07:03:43 -0800 Subject: [PATCH] test(interaction-harness): too little to judge is named starvation A busy proof whose control moved on fewer than minJudgedFrames frames threw FrameStarvationError only when the cast also showed no frame for longer than FRAME_STARVATION_MS. On a loaded runner the gap can stay under that. The same shortfall then came back as verdict failures, and the spec failed as if the component had stalled. Under the floor, a run with no component failure on the frames shown is now too little to judge. busyWindowProof throws FrameStarvationError with the control's moves, the floor, the cast's frames and its longest gap, counting every frame the cast drew. A component that missed a frame the control moved on, or stalled longer than it, still fails the verdict. The cast's median interval judges no component, so it keeps no red there. pooledBusyProof does the same for too few judged runs or control frames. A pool left short of judged runs by a run that began too late keeps its failure. Every change turns a verdict failure into named starvation. --- internal/interaction-harness/README.md | 23 +- .../src/busy.control.test.ts | 177 +++++++++++++-- internal/interaction-harness/src/busy.ts | 209 +++++++++++++----- .../src/busy.webkitclock.test.ts | 38 +++- .../src/harness.interaction.browser.spec.ts | 13 +- 5 files changed, 365 insertions(+), 95 deletions(-) diff --git a/internal/interaction-harness/README.md b/internal/interaction-harness/README.md index a779b3f94105e..c07d4d788e938 100644 --- a/internal/interaction-harness/README.md +++ b/internal/interaction-harness/README.md @@ -321,11 +321,15 @@ decodes each frame once. A proof reads frames up to decode only those. Pass either as `control`: the component passes when it moved on every frame the control moved and its longest stall is no longer than the control's. Both are read from the same screencast frames, so a compositor animation misses none. A window in -which the control moved on fewer than four frames fails, since the machine -presented too little to judge; the failure names the cast's frames in the -window and its longest gap. When no frame came for longer than -`FRAME_STARVATION_MS` there too, the machine starved the run, and -`busyWindowProof` throws `FrameStarvationError` instead of returning a verdict. +which the control moved on fewer than four frames is under the floor. Under the +floor, a run with no component failure on the frames shown is too little to +judge: `busyWindowProof` throws `FrameStarvationError` instead of returning a +verdict. The error names the control's moves, the cast's frames in the window +and its longest gap. A component that missed a frame the control moved on, or +stalled longer than it, failed on frames the compositor presented, so the +verdict keeps that red. A run the machine starved, with no frame for longer +than `FRAME_STARVATION_MS`, throws either way. `maxMedianFrameMs` judges the +cast's pace, not the component, so under the floor it keeps no red. A run of frames with no new position counts as moved when the change that ends it is a single cast step: the component moves under one step a frame there, as on an easing tail, which the cast cannot show @@ -409,8 +413,13 @@ belongs to its run. A run whose control moved on fewer frames than and the pool leaves it out, so it never counts as a pass. A run whose block began too late for its motion is left out the same way: its `runs` entry is null, and `late` holds its `BusyWindowTooLateError`. The pool reads each -run without the starvation throw. The pool fails with -fewer than `minJudgedRuns` judged runs (half of all the runs, rounded up). In each judged run, +run without the starvation throw. With fewer than `minJudgedRuns` judged runs +(half of all the runs, rounded up), or a control that moved on fewer than four +frames over them, the pool is under the floor. Under the floor, a pool with no +component failure on the frames shown is too little to judge, and it throws +`FrameStarvationError`. A pool left short of judged runs by a run that began +too late fails instead, and so does a pool whose judged runs show a component +red. In each judged run, the component must move on every frame its control moved on, so a freeze fails its run. diff --git a/internal/interaction-harness/src/busy.control.test.ts b/internal/interaction-harness/src/busy.control.test.ts index e36ca8ebc582e..a17129f332db2 100644 --- a/internal/interaction-harness/src/busy.control.test.ts +++ b/internal/interaction-harness/src/busy.control.test.ts @@ -313,14 +313,67 @@ describe('busyWindowProof against a control', () => { ); }); - it('fails a window in which the control itself presented too little to judge', () => { + it('names a window in which the control itself presented too little to judge as starvation, though every frame came', () => { const still = asControl( moving().map(f => ({...f, box: {...f.box!, y: 0}}) as TrackedFrame), ); - const proof = busyWindowProof(moving(), BLOCK, {control: still}); + const run = () => busyWindowProof(moving(), BLOCK, {control: still}); + expect(run).toThrow(FrameStarvationError); // It names how starved the cast was: here every frame came, 60 Hz. + expect(run).toThrow( + /busyWindowProof: the page drew \d+ frames while a 700 ms block held the main thread, and the control moved on 0 of them, where the proof needs 4\. No frame came for 1[67]\.\d ms: the compositor presented too little in this window to judge/, + ); + }); + + it("names a window under the control's floor as starvation when no frame came for less than 50 ms: a loaded machine's cast, its component on every frame the control moved on", () => { + // A 152 ms block: the cast delivered 8 frames, none for 44 ms. The control + // moved on 3 of them, as on a loaded runner. + const block = {from: 1000, to: 1152}; + const times = [1008, 1028, 1048, 1092, 1104, 1116, 1128, 1146]; + const moves = new Set([1048, 1092, 1128]); + let y = 0; + const loaded = [ + ...moving().filter(f => f.t < block.from), + ...times.map(t => ({t, box: null}) as TrackedFrame), + ...moving().filter(f => f.t > block.to), + ].map(f => { + if (f.t < block.from || f.t > block.to || moves.has(f.t)) { + y += 3; + } + return {...f, box: {x: 0, y, width: 10, height: 10}} as TrackedFrame; + }); + const run = () => + busyWindowProof(loaded, block, {control: asControl(loaded)}); + expect(run).toThrow(FrameStarvationError); + expect(run).toThrow( + /busyWindowProof: the page drew 8 frames while a 152 ms block held the main thread, and the control moved on 3 of them, where the proof needs 4\. No frame came for 44\.0 ms: the compositor presented too little in this window to judge/, + ); + }); + + it('still fails a main-thread component under the floor: it misses the frames the control moved on, and the cast kept pace', () => { + const block = {from: 1000, to: 1090}; + const frozen = moving().map(f => + f.t > block.from && f.t < block.to + 20 + ? ({...f, box: {...f.box!, y: 3 * at(block.from)}} as TrackedFrame) + : f, + ); + const proof = busyWindowProof(frozen, block, { + control: asControl(moving()), + }); expect(proof.verdict.failures.join('\n')).toMatch( - /the control moved on 0 frames in the judged part, under 4: the compositor presented too little in this window to judge \(the cast delivered \d+ frames in the \d+ ms block, none for 1[67] ms\)/, + /moved on 0 of the 3 frames the control moved/, + ); + }); + + it("names a window under the floor as starvation when only the cast's median interval fails: the median is the cast's pace, not the component", () => { + const run = () => + busyWindowProof(moving(), short(1000), { + control: asControl(moving()), + maxMedianFrameMs: 10, + }); + expect(run).toThrow(FrameStarvationError); + expect(run).toThrow( + /the control moved on 3 of them, where the proof needs 4/, ); }); @@ -448,15 +501,29 @@ function starvedCast(): TrackedFrame[] { ); } +/** The numbers a `FrameStarvationError` from `run` names. */ +function starvationNumbers(run: () => unknown) { + try { + run(); + } catch (error) { + if (error instanceof FrameStarvationError) { + return error.numbers; + } + throw error; + } + throw new Error('no FrameStarvationError'); +} + /** A 90 ms block from `from`: its judged part holds three 60 Hz frames. */ const short = (from: number) => ({from, to: from + 90}); describe('pooledBusyProof', () => { it('judges a block too short for one run over pooled runs: one run cannot decide, three runs of a component that moves with the control pass', () => { const control = asControl(moving()); - const single = busyWindowProof(moving(), short(1000), {control}); - expect(single.verdict.failures.join('\n')).toMatch( - /the control moved on 3 frames in the judged part, under 4/, + const single = () => busyWindowProof(moving(), short(1000), {control}); + expect(single).toThrow(FrameStarvationError); + expect(single).toThrow( + /the control moved on 3 of them, where the proof needs 4/, ); const runs = [1000, 1300, 1600].map(from => ({ tracked: moving(), @@ -623,20 +690,22 @@ describe('pooledBusyProof', () => { } }); - it('judges the control over the pool too: runs in which it presented too little fail', () => { + it('names a pool whose control presented too little in every run as starvation', () => { const still = asControl( moving().map(f => ({...f, box: {...f.box!, y: 0}}) as TrackedFrame), ); - const pooled = pooledBusyProof( - [1000, 1300].map(from => ({ - tracked: moving(), - control: still, - window: short(from), - })), + const run = () => + pooledBusyProof( + [1000, 1300].map(from => ({ + tracked: moving(), + control: still, + window: short(from), + })), + ); + expect(run).toThrow(FrameStarvationError); + expect(run).toThrow( + /pooledBusyProof: .*0 of 2 runs judged, under 1, and the control moved on 0 frames over them: the compositor presented too little in the others to judge/, ); - expect(pooled.verdict.failures).toEqual([ - '0 of 2 runs judged, under 1: the compositor presented too little in the others to judge', - ]); }); /** `frames` held at the position they had at `from`, from `from` to `to`. */ @@ -665,20 +734,84 @@ describe('pooledBusyProof', () => { ); }); - it('fails a pool in which every run but one was too little to judge, even when that run holds enough frames', () => { + it('names a pool in which every run but one was too little to judge as starvation, even when that run holds enough frames', () => { const windows = [1000, 1300, 1600].map(from => ({from, to: from + 200})); const starved = held(held(moving(), 1300, 1500), 1600, 1800); + const pool = () => + pooledBusyProof( + windows.map(window => ({ + tracked: moving(), + control: asControl(starved), + window, + })), + ); + expect(pool).toThrow(FrameStarvationError); + expect(pool).toThrow( + /pooledBusyProof: the page drew \d+ frames while 3 blocks held the main thread, and 1 of them had enough control frames to judge, where the proof needs 2\. .*1 of 3 runs judged, under 2, and the control moved on \d+ frames over them: the compositor presented too little in the others to judge/, + ); + }); + + it('names a pool of four runs with one judged as starvation: its component moved on every frame its control moved on', () => { + const windows = [1000, 1250, 1500, 1750].map(short); + const control = asControl( + held(held(held(moving(), 1250, 1340), 1500, 1590), 1750, 1840), + ); + const pool = () => + pooledBusyProof( + windows.map(window => ({tracked: moving(), control, window})), + ); + expect(pool).toThrow(FrameStarvationError); + expect(pool).toThrow(/1 of 4 runs judged, under 2/); + // Short of judged runs, the error counts every block's frames. + const numbers = starvationNumbers(pool); + expect(numbers.frames).toBe(20); + expect(numbers.longestGapMs).toBeCloseTo(FRAME, 1); + }); + + it('names the longest gap of every block in a pool short of judged runs, where the unjudged blocks drew no frame', () => { + const windows = [1000, 1250, 1500, 1750].map(short); + const control = asControl( + held(held(held(moving(), 1250, 1340), 1500, 1590), 1750, 1840), + ); + const tracked = moving().filter( + f => !windows.slice(1).some(w => f.t > w.from && f.t < w.to), + ); + const numbers = starvationNumbers(() => + pooledBusyProof(windows.map(window => ({tracked, control, window}))), + ); + expect(numbers.frames).toBe(5); + expect(numbers.longestGapMs).toBeCloseTo(90, 5); + }); + + it("names a pool short of judged runs as starvation when only the runs' median intervals fail", () => { + const windows = [1000, 1250, 1500, 1750].map(short); + const control = asControl( + held(held(held(moving(), 1250, 1340), 1500, 1590), 1750, 1840), + ); + const pool = () => + pooledBusyProof( + windows.map(window => ({tracked: moving(), control, window})), + {maxMedianFrameMs: 10}, + ); + expect(pool).toThrow(FrameStarvationError); + expect(pool).toThrow(/1 of 4 runs judged, under 2/); + }); + + it('still fails a pool short of judged runs when its judged run shows a component red', () => { + const windows = [1000, 1250, 1500, 1750].map(short); + const control = asControl( + held(held(held(moving(), 1250, 1340), 1500, 1590), 1750, 1840), + ); const pooled = pooledBusyProof( windows.map(window => ({ - tracked: moving(), - control: asControl(starved), + tracked: held(moving(), 1000, 1090), + control, window, })), ); - expect(pooled.judged).toEqual([true, false, false]); - expect(pooled.control.moved).toBeGreaterThanOrEqual(4); + expect(pooled.judged).toEqual([true, false, false, false]); expect(pooled.verdict.failures.join('\n'), pooled.summary).toMatch( - /1 of 3 runs judged, under 2: the compositor presented too little in the others to judge/, + /run 1: moved on 0 of the 3 frames its control moved/, ); }); diff --git a/internal/interaction-harness/src/busy.ts b/internal/interaction-harness/src/busy.ts index 0fd05536bb5ec..77615037b2ef3 100644 --- a/internal/interaction-harness/src/busy.ts +++ b/internal/interaction-harness/src/busy.ts @@ -1323,7 +1323,7 @@ function motionStepPx(values: readonly number[]): number { * compositor animation presents fewer frames there, the control too. The * component passes when: * - the control moved on at least `minJudgedFrames` frames of the judged part - * (fewer: the machine presented too little to judge, which fails); + * (fewer: the compositor presented too little to judge, as below); * - the component moved on every frame the control moved: both are read from * the same screencast frames, so a compositor animation misses none; * - its longest stall is no longer than the control's. @@ -1366,13 +1366,19 @@ function motionStepPx(values: readonly number[]): number { * apart, so it judges no 30 Hz cadence. A note in the summary says so, with * the count of such pairs. * - * With a control, a run the machine starved of frames proves nothing. So it - * throws `FrameStarvationError` with its numbers and returns no verdict. A - * starved run has two marks. Its control moved on fewer than - * `minJudgedFrames` frames, and the cast showed no frame for longer than - * `FRAME_STARVATION_MS` in the block. Without a control, a block with no - * frame is a frozen animation's, and it fails the verdict. `pooledBusyProof` - * reads each run without the throw and judges the runs together. + * With a control, a window whose control moved on fewer than + * `minJudgedFrames` frames of the judged part is under the floor. Under the + * floor, a run with no component failure on the frames shown is too little to + * judge. It throws `FrameStarvationError` and returns no verdict. The error + * names the control's moves, the floor, the cast's frames in the block and its + * longest gap. A component that missed a frame the control moved on, or + * stalled longer than it, failed on frames the compositor presented. The + * verdict keeps that red, unless the machine starved the run: no frame for + * longer than `FRAME_STARVATION_MS`. `maxMedianFrameMs` judges the cast's + * pace, not the component, so under the floor it keeps no red. Without a + * control, a block with no frame is a frozen animation's, and it fails the + * verdict. `pooledBusyProof` reads each run without the throw and judges the + * runs together. * * With `judgeEveryFrame`, a frame the cast could not show the component's * step on is too little to judge (the frames `belowStep` counts). When the @@ -1383,8 +1389,15 @@ function motionStepPx(values: readonly number[]): number { export function busyWindowProof( ...args: Parameters ): BusyProof { - const {proof, starvation, unjudged} = readBusyWindow(...args); - if (starvation != null) { + const {proof, starvation, unjudged, componentFailed} = readBusyWindow( + ...args, + ); + // Under the control's floor, a cast that kept pace still shows a missed + // frame or a longer stall. That failure is the component's red. + if ( + starvation != null && + (starvation.longestGapMs > FRAME_STARVATION_MS || !componentFailed) + ) { throw new FrameStarvationError(starvation); } // A failed verdict is the component's red, whatever else the cast hid. @@ -1439,10 +1452,12 @@ function readBusyWindow( return judgeBusyWindow(tracked, window, options); } const shown = atDisplayRate(dated); - const read = judgeBusyWindow(shown.tracked, window, { - ...options, - control: shown.control, - }); + const read = judgeBusyWindow( + shown.tracked, + window, + {...options, control: shown.control}, + dated.tracked, + ); // A WebKit cast rarely shows two frames one interval apart, so the proof // cannot judge a 30 Hz cadence there. It says so, with the count. const {pairs, unmatched} = displayIntervalPairs( @@ -1560,10 +1575,23 @@ function judgeBusyWindow( */ judgeEveryFrame?: boolean; } = {}, + /** + * The frames the cast drew, dated. In WebKit they are the frames before the + * display-rate keep, so a frame left out never lengthens the cast's gap. + */ + drawn: readonly TrackedFrame[] = tracked, ): { proof: BusyProof; starvation: FrameShortfall | null; unjudged: readonly UnjudgedFrame[]; + /** + * The component failed on frames the control moved on: it missed one, or + * stalled longer than the control. The cast's median interval is no part + * of it. + */ + componentFailed: boolean; + /** The frames the cast drew in the block, and its longest gap there, ms. */ + cast: {frames: number; longestGapMs: number}; } { let end = window.to - marginMs; if (settledWithin != null && settledWithinSteps != null) { @@ -2144,6 +2172,7 @@ function judgeBusyWindow( } const failures: string[] = []; let starvation: FrameShortfall | null = null; + const cast = castFramesIn(drawn, window); if (stats.frames === 0 && controlRead == null) { failures.push('no frame presented in the judged part'); } else if (stats.medianFrameMs > maxMedianFrameMs) { @@ -2152,34 +2181,25 @@ function judgeBusyWindow( ); } if (controlRead != null) { - if (controlRead.moved < minJudgedFrames) { - // How starved the cast was over the whole block, as the pacing and - // frame errors say it. Every frame the cast delivered counts, with the - // component on screen or not. - const blockFrames = tracked - .filter(f => f.t > window.from && f.t < window.to) - .map(f => f.t); - const castEdges = [window.from, ...blockFrames, window.to]; - const castGapMs = Math.max( - ...castEdges.slice(1).map((t, k) => t - castEdges[k]), - ); - // The control is a compositor animation: when it moved too little and - // no frame came for that long, the machine starved the run. - if (castGapMs > FRAME_STARVATION_MS) { - starvation = { - by: 'busyWindowProof', - during: `a ${Math.round(window.to - window.from)} ms block held the main thread, and the control moved on ${controlRead.moved} of them`, - frames: blockFrames.length, - min: minJudgedFrames, - longestGapMs: castGapMs, - }; - } - failures.push( - `the control moved on ${controlRead.moved} frames in the judged part, under ${minJudgedFrames}: the compositor presented too little in this window to judge (the cast delivered ${blockFrames.length} frame${blockFrames.length === 1 ? '' : 's'} in the ${Math.round(window.to - window.from)} ms block, none for ${Math.round(castGapMs)} ms)`, - ); - } const visible = controlRead.movedWith - controlRead.belowStep; - if (visible < minJudgedFrames) { + if (controlRead.moved < minJudgedFrames) { + // The control is a compositor animation. When it moved on too few + // frames, the compositor presented too little to judge the component, + // however long the cast's longest gap was. + starvation = { + by: 'busyWindowProof', + during: `a ${Math.round(window.to - window.from)} ms block held the main thread, and the control moved on ${controlRead.moved} of them`, + frames: cast.frames, + min: minJudgedFrames, + longestGapMs: cast.longestGapMs, + ...(cast.longestGapMs > FRAME_STARVATION_MS + ? {} + : { + cause: + 'the compositor presented too little in this window to judge, so the run proves nothing about the motion', + }), + }; + } else if (visible < minJudgedFrames) { failures.push( `the component showed a new position on ${visible} of the control's frames, under ${minJudgedFrames}: too little motion in this window to judge`, ); @@ -2260,7 +2280,29 @@ function judgeBusyWindow( verdict: {pass: failures.length === 0, failures}, summary, }; - return {proof, starvation, unjudged}; + const componentFailed = + controlRead != null && + (controlRead.movedWith < controlRead.moved || + controlRead.componentStallMs > controlRead.longestStallMs); + return {proof, starvation, unjudged, componentFailed, cast}; +} + +/** + * The frames the cast delivered inside `window`, with the component on screen + * or not, and the longest gap between them and the window's edges, ms. + */ +function castFramesIn( + tracked: readonly TrackedFrame[], + window: {readonly from: number; readonly to: number}, +): {frames: number; longestGapMs: number} { + const inside = tracked + .filter(f => f.t > window.from && f.t < window.to) + .map(f => f.t); + const edges = [window.from, ...inside, window.to]; + return { + frames: inside.length, + longestGapMs: Math.max(...edges.slice(1).map((t, k) => t - edges[k])), + }; } /** One run of a pooled busy proof (see `pooledBusyProof`). */ @@ -2328,6 +2370,13 @@ export interface PooledBusyProof { * control's. * `maxMedianFrameMs`, when given, applies to every run. A main-thread * animation still fails: in every run the control moves and it does not. + * + * With fewer judged runs than `minJudgedRuns`, or a control that moved on + * fewer than `minJudgedFrames` frames over them, the pool is under the floor. + * Under the floor, a pool with no component failure on the frames shown is too + * little to judge, and it throws `FrameStarvationError`. A pool left short of + * judged runs by a run that began too late keeps its failure instead. So does a + * pool whose judged runs show a component red. */ export function pooledBusyProof( runs: readonly BusyRun[], @@ -2360,21 +2409,31 @@ export function pooledBusyProof( ); } const late: (BusyWindowTooLateError | null)[] = []; + const failedRuns: boolean[] = []; + const runCasts: ({frames: number; longestGapMs: number} | null)[] = []; const proofs = runs.map(run => { try { - const {proof} = readBusyWindow(run.tracked, run.window, { - ...windowOptions, - control: run.control, - ahead: run.ahead, - controlStarts: run.controlStarts, - }); + const {proof, componentFailed, cast} = readBusyWindow( + run.tracked, + run.window, + { + ...windowOptions, + control: run.control, + ahead: run.ahead, + controlStarts: run.controlStarts, + }, + ); late.push(null); + failedRuns.push(componentFailed); + runCasts.push(cast); return proof; } catch (error) { if (!(error instanceof BusyWindowTooLateError)) { throw error; } late.push(error); + failedRuns.push(false); + runCasts.push(null); return null; } }); @@ -2395,20 +2454,17 @@ export function pooledBusyProof( ? `${runs.length} runs` : `the ${judgedReads.length} judged runs of ${runs.length}`; const failures: string[] = []; - if (judgedReads.length < minJudgedRuns) { - failures.push( - lateRuns === 0 - ? `${judgedReads.length} of ${runs.length} runs judged, under ${minJudgedRuns}: the compositor presented too little in the others to judge` - : `${judgedReads.length} of ${runs.length} runs judged, under ${minJudgedRuns}: ${lateRuns} began too late for their motion, and the compositor presented too little in any other to judge`, - ); - } - if (judgedReads.length > 0 && moved < minJudgedFrames) { + if (judgedReads.length < minJudgedRuns && lateRuns > 0) { failures.push( - `the control moved on ${moved} frames over ${over}, under ${minJudgedFrames}: the compositor presented too little in these windows to judge`, + `${judgedReads.length} of ${runs.length} runs judged, under ${minJudgedRuns}: ${lateRuns} began too late for their motion, and the compositor presented too little in any other to judge`, ); } + // Too few control frames or judged runs is too little to judge. A judged + // run's component red still fails: a frame it missed, or a longer stall. + const tooFewFrames = judgedReads.length > 0 && moved < minJudgedFrames; + const tooFewRuns = judgedReads.length < minJudgedRuns && lateRuns === 0; const visible = movedWith - total('belowStep'); - if (judgedReads.length > 0 && visible < minJudgedFrames) { + if (!tooFewFrames && judgedReads.length > 0 && visible < minJudgedFrames) { failures.push( `the component showed a new position on ${visible} of the control's frames over ${over}, under ${minJudgedFrames}: too little motion in these windows to judge`, ); @@ -2442,6 +2498,43 @@ export function pooledBusyProof( ); } }); + // A pool whose compositor presented too little proves nothing about the + // motion, as one starved window does. A run that began too late keeps the + // pool's failure. So does a judged run's component red. + const lateFailure = judgedReads.length < minJudgedRuns && lateRuns > 0; + const componentFailed = failedRuns.some((failed, i) => judged[i] && failed); + if ((tooFewFrames || tooFewRuns) && !lateFailure && !componentFailed) { + const casts = runCasts.filter( + (cast, i): cast is {frames: number; longestGapMs: number} => + cast != null && (tooFewRuns || judged[i]), + ); + const frames = casts.reduce((sum, cast) => sum + cast.frames, 0); + const longestGapMs = Math.max(...casts.map(cast => cast.longestGapMs)); + throw new FrameStarvationError( + tooFewRuns + ? { + by: 'pooledBusyProof', + during: `${runs.length} blocks held the main thread, and ${judgedReads.length} of them had enough control frames to judge`, + frames, + min: minJudgedRuns, + longestGapMs, + cause: `${judgedReads.length} of ${runs.length} runs judged, under ${minJudgedRuns}, and the control moved on ${moved} frames over them: the compositor presented too little in the others to judge, so the pool proves nothing about the motion`, + } + : { + by: 'pooledBusyProof', + during: `the blocks of ${over} held the main thread, and the control moved on ${moved} of them`, + frames, + min: minJudgedFrames, + longestGapMs, + ...(longestGapMs > FRAME_STARVATION_MS + ? {} + : { + cause: + 'the compositor presented too little in these windows to judge, so the pool proves nothing about the motion', + }), + }, + ); + } const summary = `${runs.length} runs pooled` + (judgedReads.length === runs.length diff --git a/internal/interaction-harness/src/busy.webkitclock.test.ts b/internal/interaction-harness/src/busy.webkitclock.test.ts index bb01bd1340c96..d0a82678ee6db 100644 --- a/internal/interaction-harness/src/busy.webkitclock.test.ts +++ b/internal/interaction-harness/src/busy.webkitclock.test.ts @@ -18,7 +18,7 @@ import {describe, expect, it} from 'vitest'; import {busyWindowProof, pageTimesOnControls, pooledBusyProof} from './busy'; -import type {TrackedFrame} from './frames'; +import {FrameStarvationError, type TrackedFrame} from './frames'; const FRAME = 1000 / 60; const BLOCK = {from: 1000, to: 1700}; @@ -472,4 +472,40 @@ describe('busyWindowProof on a WebKit cast with a control', () => { expect(proof(chromium, true)).toEqual(proof(stamped, true)); expect(proof(webkit, false)).toEqual(proof(stamped, false)); }); + + it('measures the cast on every frame it drew, so a frame the display-rate keep leaves out never lengthens its longest gap', () => { + // In the block, a frame drawn 7 ms after another is left out of the + // judgment, and then no frame comes for 48 ms. Without that frame, the + // gap would read 55 ms, over FRAME_STARVATION_MS. + const shots = [ + ...onTime(800, 990), + ...[1010, 1017, 1065, 1090].map(at => ({at, stamp: at})), + ...onTime(1110, 1300), + ]; + const proof = (value: (at: number) => number) => () => { + const frames = cast(shots, value, 'webkit'); + return busyWindowProof( + frames.tracked, + {from: 1000, to: 1100}, + {axis: 'x', control: frames.control, ahead: frames.ahead}, + ); + }; + // Under the floor, a frozen component failed on frames the compositor + // presented, and no gap passed 50 ms. So its red stands. + expect(proof(() => 100)().verdict.failures.join('\n')).toMatch( + /moved on 0 of the \d+ frames the control moved/, + ); + // A component that moves with the control is too little to judge. The + // error counts every frame drawn, the one left out included. + const moving = proof(xAt); + expect(moving).toThrow(FrameStarvationError); + try { + moving(); + } catch (error) { + const {numbers} = error as FrameStarvationError; + expect(numbers.frames).toBe(4); + expect(numbers.longestGapMs).toBeGreaterThan(45); + expect(numbers.longestGapMs).toBeLessThanOrEqual(50); + } + }); }); diff --git a/internal/interaction-harness/src/harness.interaction.browser.spec.ts b/internal/interaction-harness/src/harness.interaction.browser.spec.ts index 628c2438ded87..f3d3bdb581b83 100644 --- a/internal/interaction-harness/src/harness.interaction.browser.spec.ts +++ b/internal/interaction-harness/src/harness.interaction.browser.spec.ts @@ -2146,8 +2146,7 @@ test.describe('frame evidence', () => { }; const compositor = await runs('compositor'); - // One run is too little to judge: its verdict says so, or, when the - // machine also starved its cast, the proof throws by name. + // One run is too little to judge, so the proof throws by name. const single = (() => { try { const proof = busyWindowProof( @@ -2174,7 +2173,8 @@ test.describe('frame evidence', () => { expect(pooled.verdict.failures, pooled.summary).toEqual([]); // The same runs with the control taken out of the last blocks. A starved - // run is named and left out. With too few judged runs, the pool fails. + // run is named and left out. With too few judged runs, the pool throws by + // name. const starve = (count: number) => pooledBusyProof( compositor.map((run, i) => @@ -2196,10 +2196,9 @@ test.describe('frame evidence', () => { expect(lastStarved.summary).toMatch( /; run 6: too little to judge, the control moved on 0 frames/, ); - const oneLeft = starve(5); - expect(oneLeft.verdict.failures.join('\n'), oneLeft.summary).toMatch( - /[01] of 6 runs judged, under 3/, - ); + const oneLeft = () => starve(5); + expect(oneLeft).toThrow(FrameStarvationError); + expect(oneLeft).toThrow(/[01] of 6 runs judged, under 3/); // The mutant: a main-thread animation freezes in every block. const raf = pooledBusyProof(await runs('raf'), {axis: 'y'});