Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 123 additions & 2 deletions packages/cli/src/commands/test/__tests__/dryRunTranscripts.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import chalk from 'chalk';
import {
DRY_RUN_TRANSCRIPTS,
DRY_RUN_TRANSCRIPT_IDS,
FIXTURE_ROOTS,
type TranscriptScenario,
} from '../dryRun.transcripts';
import { renderScenarioTranscript, transcriptForCapture } from '../renderTranscript';
Expand Down Expand Up @@ -74,6 +75,105 @@ describe('the dry-run transcript catalog', () => {
expect(unstated).toEqual([]);
});

it("every scenario answers for a fixture under one of the catalog's declared roots", () => {
// A fixture path is resolved by joining it onto the sherlo-tester checkout,
// so a path under ANY tree in that repository resolves. What this case is
// for is the other direction: a root nobody declared is how a typo, or a
// tree that quietly moved, reaches a consumer as a missing file rather than
// as a review finding.
const declaredRoots = Object.values(FIXTURE_ROOTS);
const strays = Object.entries(DRY_RUN_TRANSCRIPTS)
.filter(
([, scenario]) => !declaredRoots.some((root) => scenario.fixture.startsWith(`${root}/`))
)
.map(([id, scenario]) => `${id} -> ${scenario.fixture}`);

expect(
strays,
'a scenario names a fixture root this catalog does not declare - add it to FIXTURE_ROOTS, ' +
'saying what lives there, or fix the path'
).toEqual([]);
});

it('the six LEGACY fixture paths are UNCHANGED - the legacy suites still consume them', () => {
// Pinned as literals, not derived from FIXTURE_ROOTS: a case that composed
// the expected paths the same way the catalog does would agree with any
// move of the root, which is exactly the change it exists to catch. These
// six are consumed by suites that are still green, so moving one is a red
// capture night, not a refactor.
//
// It reads the six BY ID rather than comparing the whole catalog, because
// those are two different claims: "these six have not moved" is a law, and
// "there are exactly six" was never one - the catalog is meant to grow. A
// missing id still reds here (it reads back as undefined), so a deletion is
// caught as loudly as a move.
const legacyFixtures = {
'dry-run-single-platform-nothing-to-capture':
'e2e/suites/snapshots/test-bundled/06-single-platform.spec.ts-snapshots/u4-single-platform-cli-Test-Bundled-darwin.txt',
'dry-run-cold-start':
'e2e/suites/snapshots/test-bundled/09-cold-start.spec.ts-snapshots/c3-cold-start-cli-Test-Bundled-darwin.txt',
'dry-run-provenance-guard-skip':
'e2e/suites/snapshots/test-bundled/10-provenance-guard-skip.spec.ts-snapshots/c4b-guard-skip-cli-Test-Bundled-darwin.txt',
'dry-run-api-unreachable':
'e2e/suites/snapshots/test-bundled-checks/02-failure-modes.spec.ts-snapshots/f4-api-unreachable-cli-Test-Bundled-Checks-darwin.txt',
'dry-run-detached-head':
'e2e/suites/snapshots/test-bundled-checks/02-failure-modes.spec.ts-snapshots/f6-detached-head-cli-Test-Bundled-Checks-darwin.txt',
'dry-run-no-git':
'e2e/suites/snapshots/test-bundled-checks/02-failure-modes.spec.ts-snapshots/f6-no-git-cli-Test-Bundled-Checks-darwin.txt',
};

const declared = Object.fromEntries(
Object.keys(legacyFixtures).map((id) => [id, DRY_RUN_TRANSCRIPTS[id]?.fixture])
);

expect(declared).toEqual(legacyFixtures);
});

it('every declared root is a path RELATIVE TO THE CHECKOUT ROOT', () => {
// The one claim about a root that holds whether or not the tree behind it
// exists yet, and the one both consumers depend on: a root is joined onto a
// repository root, so an absolute path or a `..` escape would resolve
// somewhere neither the ratchet nor `expected-render` means. This runs with
// no checkout, which is what lets a root be declared BEFORE the tree it
// names is created - `e2e-beats/suites` is such a root today.
const malformed = Object.entries(FIXTURE_ROOTS)
.filter(
([, root]) => root.startsWith('/') || root.endsWith('/') || root.split('/').includes('..')
)
.map(([name, root]) => `${name} -> ${root}`);

expect(
malformed,
'a fixture root must be relative to the sherlo-tester repo root, with no trailing slash'
).toEqual([]);
});

it.runIf(TESTER_AVAILABLE)(
'every root a SCENARIO ANSWERS UNDER is a real tree in the checkout',
() => {
// Scoped to the roots in use, because that is the strongest form of this
// claim that can be true: a root may be declared before its tree exists
// (the beats suites are not in the checkout yet), and a case that demanded
// otherwise would red for a tree nobody has authored rather than for a
// mistake. The moment a scenario answers under a root, the root has to be
// there - and the per-scenario `fixtureExists` case below then proves the
// whole path, not just its head.
const rootsInUse = new Set(
Object.values(DRY_RUN_TRANSCRIPTS).map(
(scenario) =>
Object.values(FIXTURE_ROOTS).find((root) => scenario.fixture.startsWith(`${root}/`)) ??
''
)
);

const unreachable = [...rootsInUse].filter((root) => !fixtureExists(root));

expect(unreachable, 'a fixture root a scenario answers under is not in the checkout').toEqual(
[]
);
}
);

it('every scenario scripts a bundle for every platform it puts under test', () => {
// A missing bundle is a refusal at render time; catching it here names every
// offender at once instead of one per run.
Expand All @@ -88,13 +188,34 @@ describe('the dry-run transcript catalog', () => {

it.runIf(TESTER_AVAILABLE)('every scenario names a fixture that exists', () => {
const missing = Object.entries(DRY_RUN_TRANSCRIPTS)
.filter(([, s]) => !fixtureExists(s.fixture))
.filter(([, s]) => !s.fixtureNotMintedYet && !fixtureExists(s.fixture))
.map(([id, s]) => `${id} -> ${s.fixture}`);
expect(missing, 'a scenario answers for a fixture that is not in the tree').toEqual([]);
});

it.runIf(TESTER_AVAILABLE)('a NOT-MINTED-YET marker cannot outlive the mint', () => {
// The whole risk of that marker is that it becomes a permanent opt-out of
// the byte case. It cannot: the moment `yarn tester expected-render` writes
// the bytes, this case reds until the marker comes off - which is what turns
// the scenario back into a ratcheted one. A marker is a statement about the
// tree, and this is what keeps it true.
const stale = Object.entries(DRY_RUN_TRANSCRIPTS)
.filter(([, s]) => s.fixtureNotMintedYet && fixtureExists(s.fixture))
.map(([id, s]) => `${id} -> ${s.fixture}`);

expect(
stale,
'this fixture HAS been minted, so the scenario is ratchetable now: drop ' +
'`fixtureNotMintedYet` and let the byte case judge it'
).toEqual([]);
});

for (const id of DRY_RUN_TRANSCRIPT_IDS) {
it.runIf(TESTER_AVAILABLE)(
// A scenario whose fixture has never been minted has nothing to be identical
// TO - it is the render that will create those bytes. Every other case in
// this file still covers it, the determinism case below included, and the
// staleness case above turns this one back on the moment the mint lands.
it.runIf(TESTER_AVAILABLE && !DRY_RUN_TRANSCRIPTS[id].fixtureNotMintedYet)(
`${id}: renders byte-identically to its committed fixture`,
async () => {
const scenario = DRY_RUN_TRANSCRIPTS[id];
Expand Down
Loading
Loading