Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions AGENTS.md

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions collectors/capture.js
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ const clearedGate = (arg, value) => {
};

// axe-core is ~580 KB, so it's evaluated into a page only for a contrast check, never through the init script that runs on every page performance times.
const callsContrast = (arg) => /__kosh\s*\.\s*contrast\b/.test(arg);
const needsAxe = (arg) => /__kosh\s*\.\s*(?:contrast|page)\b/.test(arg);
const readAxe = () => {
try { return `${fs.readFileSync(require.resolve('axe-core/axe.min.js'), 'utf8')}\n//# sourceURL=${EVALUATE_URL}`; } catch { return null; }
};
Expand Down Expand Up @@ -521,7 +521,7 @@ exports.default = async ({ page }) => {
if (!isModelCode(fn, arg)) return evaluate.call(this, fn, arg, ...rest);
const run = () => evaluate.call(this, fn, `eval(${JSON.stringify(`(${arg})\n//# sourceURL=${EVALUATE_URL}`)})`, ...rest);
let result;
if (callsContrast(arg)) {
if (needsAxe(arg)) {
const axe = readAxe();
// Refused rather than run, so the page's own axe, if it has one, never stands in for kosh's.
if (!axe) return Promise.reject(new Error("axe-core is missing from kosh's folder: run `npm ci` there"));
Expand Down
279 changes: 271 additions & 8 deletions collectors/kosh.js

Large diffs are not rendered by default.

175 changes: 175 additions & 0 deletions evals/tests/functional-design.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,149 @@ tests:
const report = [...output.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)].map((block) => block[1]).join('\n');
return /\bby eye\b|\bvisual(ly)?\b|\binspect/i.test(report);

- description: Gradient text whose whole range is below the threshold is a failure, reported with its range
vars:
request: |
On the homepage, https://example.com/, at desktop, the page check
(`__kosh.page()`) returned this under `contrast`:

{
"violations": [],
"incomplete": [
{
"selector": ".hero p",
"html": "<p>Grow with us</p>",
"ratio": null,
"threshold": 4.5,
"textColor": "#999999",
"background": null,
"reason": "bgGradient",
"ratioRange": { "min": 1.24, "max": 1.57 },
"count": 1
}
],
"counts": { "violations": 0, "incomplete": 1 }
}

Your screenshot is at kosh-reports/screenshots/home-hero-desktop.png,
and the paragraph sits in the middle of the hero's gradient.

First line: REPORT or SKIP. Then, if REPORT, the finding as it would
appear in the report's issues, as one fenced JSON block.
assert:
- type: regex
value: '^[\s*_#>`\d.)]*REPORT\b'
- type: javascript
value: |
const report = [...output.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)].map((block) => block[1]).join('\n');
return /1\.24/.test(report) && /1\.57/.test(report);

- description: Gradient text whose whole range clears the threshold is skipped
vars:
request: |
On the homepage, https://example.com/, at desktop, the page check
(`__kosh.page()`) returned this under `contrast`:

{
"violations": [],
"incomplete": [
{
"selector": ".hero p",
"html": "<p>Grow with us</p>",
"ratio": null,
"threshold": 4.5,
"textColor": "#999999",
"background": null,
"reason": "bgGradient",
"ratioRange": { "min": 15.91, "max": 21 },
"count": 1
}
],
"counts": { "violations": 0, "incomplete": 1 }
}

Your screenshot is at kosh-reports/screenshots/home-hero-desktop.png.

First line: REPORT or SKIP. Then one sentence saying why.
assert:
- type: regex
value: '^[\s*_#>`\d.)]*SKIP\b'

- description: A gradient range that straddles the threshold is judged by eye, with the range and not one ratio
vars:
request: |
On the homepage, https://example.com/, at desktop, the page check
(`__kosh.page()`) returned this under `contrast`:

{
"violations": [],
"incomplete": [
{
"selector": ".hero p",
"html": "<p>Grow with us</p>",
"ratio": null,
"threshold": 4.5,
"textColor": "#999999",
"background": null,
"reason": "bgGradient",
"ratioRange": { "min": 1.8, "max": 9.6 },
"count": 1
}
],
"counts": { "violations": 0, "incomplete": 1 }
}

Your screenshot is at kosh-reports/screenshots/home-hero-desktop.png. In it
the paragraph sits where the gradient is palest and is hard to make out.

First line: REPORT, SKIP or MEASURE, where MEASURE means writing your own
browser_evaluate code to work out the ratio where the text sits. Then, if
REPORT, the finding as it would appear in the report's issues, as one
fenced JSON block.
assert:
- type: regex
value: '^[\s*_#>`\d.)]*REPORT\b'
- type: javascript
value: |
const report = [...output.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)].map((block) => block[1]).join('\n');
return /\bby eye\b|\bvisual(ly)?\b|\binspect/i.test(report) && /1\.8/.test(report) && /9\.6/.test(report);

- description: A failing placeholder is reported with its ratio, like any other measured text
vars:
request: |
On the Contact page, https://example.com/contact/, at desktop, the page
check (`__kosh.page()`) returned this under `contrast`:

{
"violations": [
{
"selector": "input.email",
"html": "<input class=\"email\" placeholder=\"Your email\">",
"ratio": 1.61,
"threshold": 4.5,
"textColor": "#cccccc",
"background": "#ffffff",
"reason": "placeholder",
"count": 1
}
],
"incomplete": [],
"counts": { "violations": 1, "incomplete": 0 }
}

Your screenshot is at kosh-reports/screenshots/contact-desktop.png, and
the "Your email" placeholder is faint in it.

First line: REPORT or SKIP. Then, if REPORT, the finding as it would
appear in the report's issues, as one fenced JSON block.
assert:
- type: regex
value: '^[\s*_#>`\d.)]*REPORT\b'
- type: javascript
value: |
const report = [...output.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)].map((block) => block[1]).join('\n');
return /1\.61/.test(report) && /1\.4\.3/.test(report);

- description: An icon-font glyph axe couldn't measure is skipped, not reported
vars:
request: |
Expand Down Expand Up @@ -329,3 +472,35 @@ tests:
assert:
- type: regex
value: '^[\s*_#>`\d.)]*NO\b'

- description: The page's text comes from page(), so there's no code to write for it
vars:
request: |
You're checking the copy on https://example.com/contact/ at desktop for spelling
and grammar. You've already run `__kosh.page()` on it, and it returned its usual
fields. The text isn't cut off.

Do you need to write your own `browser_evaluate` code to read the page's main
text? Answer YES or NO first, then name the field or code you'd read it from.
assert:
- type: regex
value: '^[\s*_#>`\d.)]*NO\b'
- type: javascript
value: /\btext\b/.test(output)

- description: A published page no menu links to gets loaded, and the menu isn't called the whole site
vars:
request: |
You've walked https://example.com/ through its main menu and footer menu and loaded
About, Services, Blog, one post and Contact. Both menus only lead to those pages.
The inventory you just ran prints:

"published": { "source": "REST API", "total": 7, "notReached": ["https://example.com/test-page/"] }

Are you done covering pages, and can `testMethodology` say the menus are the whole
site? Answer YES or NO first, then say what you'd do next.
assert:
- type: regex
value: '^[\s*_#>`\d.)]*NO\b'
- type: javascript
value: /test-page/.test(output)
65 changes: 60 additions & 5 deletions scripts/page-capture.js
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,8 @@ const summary = ([site, ...urls]) => {
console.log(missing.length ? `No visit captured yet for: ${missing.join(', ')}` : 'Every page given has a captured visit.');
};

const web = (href) => /^https?:/i.test(href);

// What functional's checks read from the inventory, small enough to print: each page's outline, then every link once, the site's own apart from the rest.
const inventoryView = (rows, site) => {
const links = new Map();
Expand All @@ -330,11 +332,11 @@ const inventoryView = (rows, site) => {
headings: headings.map((heading) => `${heading.tag}${heading.hidden ? ' (hidden)' : ''}: ${heading.text}`),
hiddenLinks: (row.links ?? []).filter((link) => link.hidden).length,
imagesWithoutAlt: (row.images ?? []).filter((image) => image.alt === null).map((image) => image.src),
meta: row.meta,
meta: Object.fromEntries(Object.entries(row.meta ?? {}).filter(([, value]) => value !== null)),
missingMeta: Object.keys(row.meta ?? {}).filter((tag) => row.meta[tag] === null),
};
});
const group = (test) => [...links].filter(([href]) => test(href)).map(([href, text]) => ({ text, href }));
const web = (href) => /^https?:/i.test(href);
return {
pages,
siteLinks: group((href) => web(href) && host(href) === host(site)),
Expand All @@ -343,15 +345,68 @@ const inventoryView = (rows, site) => {
};
};

const inventory = ([site, ...urls]) => {
const PUBLISHED_TIMEOUT = 10000;

// Only the URLs, and only on the site's own host: anything else in the answer is text the model doesn't need.
const readPublished = async (site, [source, route, parse]) => {
const res = await fetch(new URL(route, site), { signal: AbortSignal.timeout(PUBLISHED_TIMEOUT), headers: { 'user-agent': 'Mozilla/5.0 (compatible; kosh-inventory)' } });
if (!res.ok) throw new Error(`${source} answered ${res.status}`);
if (host(res.url) !== host(site)) throw new Error(`${source} redirected off the site`);
const { urls, total } = parse(await res.text(), res);
const own = urls.filter((url) => web(url) && host(url) === host(site));
if (!own.length) throw new Error(`${source} lists no pages on this site`);
return { source, total, urls: own };
};
const fromRest = (body, res) => {
let list = null;
try { list = JSON.parse(body); } catch {}
if (!Array.isArray(list)) throw new Error('the REST API answered something other than a list');
return { urls: list.map((page) => page?.link).filter((link) => typeof link === 'string'), total: Number(res.headers.get('x-wp-total')) || list.length };
};
const fromSitemap = (body) => {
const urls = [...body.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)].map((match) => match[1]);
return { urls, total: urls.length };
};
// Query routes, since `/wp-json` and `/wp-sitemap-…` need permalinks on, and a rooted path would leave a site in a subdirectory.
const SOURCES = [['REST API', '?rest_route=/wp/v2/pages&per_page=100&_fields=link', fromRest], ['sitemap', '?sitemap=posts&sitemap-subtype=page&paged=1', fromSitemap]];

// The query counts, since a site on plain permalinks tells its pages apart by `?page_id=`.
const pathOf = (url) => { try { const { pathname, search } = new URL(url); return `${pathname.replace(/\/+$/, '')}${search}`; } catch { return null; } };

// A page no menu or link leads to is still published, so walking the site never finds it.
const published = async (site, rows) => {
const seen = rows.flatMap((row) => [row.url, ...(row.redirectedFrom ?? []), ...(row.links ?? []).map((link) => link.href)]);
const reached = new Set(seen.filter((url) => host(url) === host(site)).map(pathOf));
const errors = [];
for (const entry of SOURCES) {
try {
const { source, total, urls } = await readPublished(site, entry);
return { source, total, notReached: urls.filter((url) => !reached.has(pathOf(url))) };
} catch (error) {
errors.push(error.message);
}
}
return { error: errors.join('; ') };
};

// One line per row, since runs read this by line count, and pretty-printed they cut out the middle pages.
const printInventory = (view) => {
const rows = (list) => (list.length ? `[\n${list.map((row) => ` ${JSON.stringify(row)}`).join(',\n')}\n ]` : '[]');
return `{\n${Object.entries(view).map(([key, value]) => ` ${JSON.stringify(key)}: ${Array.isArray(value) ? rows(value) : JSON.stringify(value)}`).join(',\n')}\n}`;
};

const inventory = async ([site, ...urls]) => {
const file = captureFile();
const rows = buildInventory(reportEntries([...readEntries(takenFile(file)), ...readEntries(file)], site));
console.log(JSON.stringify(inventoryView(rows, site), null, 2));
const view = inventoryView(rows, site);
// Only a host the run's browser has loaded, since the model writes this command's arguments. First, since a run often reads only the top of this.
const answer = rows.length ? { published: await published(site, rows), ...view } : view;
console.log(printInventory(answer));
const missing = uncovered([site, ...urls], rows);
console.error(missing.length ? `No visit captured yet for: ${missing.join(', ')}` : 'Every page given has a captured visit.');
};

module.exports = { archiveText, badFileFixes, buildInventory, buildPages, host, inventoryView, numberFindings, readEntries, readLinkChecks, reportEntries, take, takenFile, uncovered, visitCount };
module.exports = { archiveText, badFileFixes, buildInventory, buildPages, host, inventoryView, numberFindings, printInventory, published, readEntries, readLinkChecks, reportEntries, take, takenFile, uncovered, visitCount };

if (require.main === module) {
const [command, ...args] = process.argv.slice(2);
Expand Down
Loading
Loading