Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion .github/qa/action-judge.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
// Feed the AFTER-CLICK difference (old good code vs new broken code) to the REAL LLM judge.
import { randomUUID } from 'node:crypto';
import { readFileSync } from 'fs';

// The LLM proxy rejects any request without `x-session-id` and `x-slicc-version` (HTTP 403
// missing_required_header). One id per process, as in qa-runner-v2.mjs.
const SESSION_ID = randomUUID();
const SLICC_VERSION = '1.0.0';

const pre = JSON.parse(readFileSync('/tmp/after-good.json','utf8')); // old code after click
const post = JSON.parse(readFileSync('/tmp/after-bugged.json','utf8')); // new (broken) code after click
const summary = `[after action] The tool clicked the "Digital Trends" filter on both builds. `
Expand All @@ -23,7 +30,7 @@ async function judge(intent, domDiff) {
+ `- WORKS: NEW behaves like the feature intends (narrows on click).\n`
+ `- FLAG: NEW contradicts the intent (e.g. filtering no longer narrows; the click has no effect on NEW though it did on OLD).\n`
+ `- NO_CHANGE: no visible difference and none intended.`;
const res = await fetch(PROXY,{method:'POST',headers:{Authorization:`Bearer ${TOKEN}`,'Content-Type':'application/json','anthropic-version':'2023-06-01'},body:JSON.stringify({model:MODEL,max_tokens:400,stream:true,messages:[{role:'user',content:prompt}]})});
const res = await fetch(PROXY,{method:'POST',headers:{Authorization:`Bearer ${TOKEN}`,'Content-Type':'application/json','anthropic-version':'2023-06-01','x-session-id':SESSION_ID,'x-slicc-version':SLICC_VERSION},body:JSON.stringify({model:MODEL,max_tokens:400,stream:true,messages:[{role:'user',content:prompt}]})});
const raw=await res.text(); let text='';
for(const line of raw.split('\n')){const t=line.trim();if(!t.startsWith('data:'))continue;const d=t.slice(5).trim();if(!d||d==='[DONE]')continue;let e;try{e=JSON.parse(d)}catch{continue}if(e.type==='content_block_delta'&&e.delta?.type==='text_delta')text+=e.delta.text||''}
const j=JSON.parse(text.slice(text.indexOf('{'),text.lastIndexOf('}')+1));
Expand Down
8 changes: 8 additions & 0 deletions .github/qa/aggregate-report.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,16 @@
* VERDICT_SIDECAR - optional; default /tmp/qa-audit-verdict.txt
*/

import { randomUUID } from 'node:crypto';
import { readFileSync, readdirSync, existsSync, writeFileSync } from 'fs';
import { join, basename } from 'path';
import { spawnSync } from 'child_process';

// The LLM proxy rejects any request without `x-session-id` and `x-slicc-version` (HTTP 403
// missing_required_header). One id per process, as in qa-runner-v2.mjs.
const SESSION_ID = randomUUID();
const SLICC_VERSION = '1.0.0';

const PROXY_URL = process.env.PROXY_URL || '';
const MODEL = process.env.MODEL || '';
const TOKEN = process.env.IMS_ACCESS_TOKEN || '';
Expand Down Expand Up @@ -173,6 +179,8 @@ function rewriteWithLLM() {
'-H', `Authorization: Bearer ${TOKEN}`,
'-H', 'Content-Type: application/json',
'-H', 'anthropic-version: 2023-06-01',
'-H', `x-session-id: ${SESSION_ID}`,
'-H', `x-slicc-version: ${SLICC_VERSION}`,
'--max-time', '90',
'--data-binary', '@-',
],
Expand Down
8 changes: 7 additions & 1 deletion .github/qa/ai-judge-test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,16 @@
// Renders a scenario NEW vs OLD on localhost via window._qa, computes a real DOM diff,
// and asks the ACTUAL LLM judge (same prompt as the batch) for WORKS/FLAG/NO_CHANGE.
// Creds sourced from env (PROXY_URL/MODEL/IMS_ACCESS_TOKEN); values never printed.
import { randomUUID } from 'node:crypto';
import { chromium } from 'playwright';
import { readFileSync } from 'fs';
import { diffSignatures, summarizeDiff } from './dom-diff.mjs';

// The LLM proxy rejects any request without `x-session-id` and `x-slicc-version` (HTTP 403
// missing_required_header). One id per process, as in qa-runner-v2.mjs.
const SESSION_ID = randomUUID();
const SLICC_VERSION = '1.0.0';

const cat = JSON.parse(readFileSync('/private/tmp/schema-branch/.github/qa/mount-catalog.json', 'utf8'));
const byId = Object.fromEntries(cat.entries.map((e) => [e.id, e]));
const URL = 'http://localhost:8899/index.html';
Expand Down Expand Up @@ -70,7 +76,7 @@ async function judgeExpected(intent, domDiff, visualDiff) {
+ `- NO_CHANGE: the PR did not intend any visible change (pure refactor, comment/log/formatting tweak) and the page correctly shows no change.`;
for (let attempt = 0; attempt < 4; attempt += 1) {
try {
const res = await fetch(PROXY, { method: 'POST', headers: { Authorization: `Bearer ${TOKEN}`, 'Content-Type': 'application/json', 'anthropic-version': '2023-06-01' },
const res = await fetch(PROXY, { method: 'POST', headers: { Authorization: `Bearer ${TOKEN}`, 'Content-Type': 'application/json', 'anthropic-version': '2023-06-01', 'x-session-id': SESSION_ID, 'x-slicc-version': SLICC_VERSION },
body: JSON.stringify({ model: MODEL, max_tokens: 600, stream: true, messages: [{ role: 'user', content: prompt }] }) });
const raw = await res.text(); let text = '';
for (const line of raw.split('\n')) { const t = line.trim(); if (!t.startsWith('data:')) continue; const d = t.slice(5).trim(); if (!d || d === '[DONE]') continue; let e; try { e = JSON.parse(d); } catch { continue; } if (e.type === 'content_block_delta' && e.delta?.type === 'text_delta') text += e.delta.text || ''; }
Expand Down
9 changes: 7 additions & 2 deletions .github/qa/feature-backtest-batch.mjs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env node
import { execFileSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { createHash, randomUUID } from 'node:crypto';
import { cpSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
Expand All @@ -9,6 +9,11 @@ import { diffSignatures, summarizeDiff } from './dom-diff.mjs';
import { PNG } from 'pngjs';
import pixelmatch from 'pixelmatch';

// The LLM proxy rejects any request without `x-session-id` and `x-slicc-version` (HTTP 403
// missing_required_header). One id per process, as in qa-runner-v2.mjs.
const SESSION_ID = randomUUID();
const SLICC_VERSION = '1.0.0';

// Visual diff of the pre-code vs post-code render (same forced scenario). Localised noise
// (antialiasing) is ignored via a threshold; a taller/shorter page (content changed) counts
// as a visual change directly.
Expand Down Expand Up @@ -228,7 +233,7 @@ async function judgeExpected(intent, domDiff, visualDiff) {
+ `- NO_CHANGE: the PR did not intend any visible change (pure refactor, comment/log/formatting tweak) and the page correctly shows no change.`;
for (let attempt = 0; attempt < 4; attempt += 1) {
try {
const res = await fetch(PROXY, { method: 'POST', headers: { Authorization: `Bearer ${TOKEN}`, 'Content-Type': 'application/json', 'anthropic-version': '2023-06-01' },
const res = await fetch(PROXY, { method: 'POST', headers: { Authorization: `Bearer ${TOKEN}`, 'Content-Type': 'application/json', 'anthropic-version': '2023-06-01', 'x-session-id': SESSION_ID, 'x-slicc-version': SLICC_VERSION },
body: JSON.stringify({ model: MODEL, max_tokens: 600, stream: true, messages: [{ role: 'user', content: prompt }] }) });
const raw = await res.text(); let text = '';
for (const line of raw.split('\n')) { const t = line.trim(); if (!t.startsWith('data:')) continue; const d = t.slice(5).trim(); if (!d || d === '[DONE]') continue; let e; try { e = JSON.parse(d); } catch { continue; } if (e.type === 'content_block_delta' && e.delta?.type === 'text_delta') text += e.delta.text || ''; }
Expand Down
9 changes: 9 additions & 0 deletions .github/qa/feature-backtest-worker.mjs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/usr/bin/env node
import { randomUUID } from 'node:crypto';
import { execFileSync } from 'node:child_process';
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import path from 'node:path';
Expand All @@ -10,6 +11,12 @@ import { buildValidationView } from './observation-view.mjs';
import { requestBoundedJson } from './llm-json.mjs';
import { shouldChallengeSkip } from './skip-challenge.mjs';
import { classifyChangedPaths } from './detect-gate.mjs';

// The LLM proxy rejects any request without `x-session-id` and `x-slicc-version` (HTTP 403
// missing_required_header). One id per process, as in qa-runner-v2.mjs.
const SESSION_ID = randomUUID();
const SLICC_VERSION = '1.0.0';

import {
applyScenarioRepair,
findMissingRequiredInitial,
Expand Down Expand Up @@ -90,6 +97,8 @@ async function llmResponse(prompt, maxTokens = 4000) {
Authorization: `Bearer ${TOKEN}`,
'Content-Type': 'application/json',
'anthropic-version': '2023-06-01',
'x-session-id': SESSION_ID,
'x-slicc-version': SLICC_VERSION,
},
body,
signal: controller.signal,
Expand Down
36 changes: 34 additions & 2 deletions .github/qa/feature-review.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
* say should happen.
*/
import { execFileSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { writeFileSync } from 'node:fs';
import path from 'node:path';
import { chromium } from 'playwright';
Expand All @@ -26,6 +27,11 @@ import {
} from './feature-action.mjs';
import { buildScenarioConfig } from './scenario-config.mjs';

// The LLM proxy rejects any request without `x-session-id` and `x-slicc-version` (HTTP 403
// missing_required_header). One id per process, as in qa-runner-v2.mjs.
const SESSION_ID = randomUUID();
const SLICC_VERSION = '1.0.0';

const env = (k, d = '') => (process.env[k] ?? d);
const PR = env('PR_NUMBER');
const REPO = env('GH_REPO', 'adobecom/caas');
Expand Down Expand Up @@ -127,6 +133,8 @@ async function llm(prompt, maxTokens = 4000) {
Authorization: `Bearer ${TOKEN}`,
'Content-Type': 'application/json',
'anthropic-version': '2023-06-01',
'x-session-id': SESSION_ID,
'x-slicc-version': SLICC_VERSION,
},
body,
signal: controller.signal,
Expand Down Expand Up @@ -303,7 +311,15 @@ function postComment(verdict, bodyMd) {
.slice(0, 14000);

// ---- Step 1: decide whether the PR's feature can be exercised at all ----
const detect = await llm(
// This triage is the only thing standing between the agent and its report,
// and it needs the LLM proxy. The build-output-diff verdict does NOT -- it is
// a commit status read over the REST API. So when the proxy is unavailable,
// fall through to that deterministic verdict rather than dying silently: a
// dependency bump whose shipped bundle is byte-identical is still provably
// safe, and saying so is the whole point of this cross-signal.
let detect;
try {
detect = await llm(
`You are triaging an Adobe CaaS (Consonant card collection) pull request to decide if its feature can be EXERCISED by an automated harness.

The harness renders the REAL PR build on a live page and can force the CaaS CONFIG and COLLECTION DATA. It can then perform ONE simple interaction: clicking one visible control or typing into one visible input, followed by a second DOM capture.
Expand All @@ -326,6 +342,12 @@ Diff (truncated):
${diff}

Respond with ONLY a JSON object: {"testable":true|false,"reason":"one sentence"}.`, 4000);
} catch (error) {
await postNonInjectable(
`Triage model unavailable (${error.message}) -- reporting the deterministic bundle diff only.`);
console.log('llm unavailable: posted the build-output-diff verdict without triage');
process.exit(0);
}

console.error('[detect raw first 400]:', String(detect).slice(0, 400));
let plan;
Expand Down Expand Up @@ -587,4 +609,14 @@ ${after.cards.map((item) => `- ${item.n}. ${item.title || item.text.slice(0, 50)

**Verdict:** ${res.reason}`);
process.exit(0);
})().catch((e) => { console.error('feature-review error:', e.stack || e.message); process.exit(0); });
})().catch((e) => {
// Advisory and non-blocking by design -- but a green run with no comment is
// exactly how this agent's outage stayed invisible. Leave a marker so the
// workflow's monitor step can log it, the way Agent QA Review already does.
console.error('feature-review error:', e.stack || e.message);
try {
writeFileSync(`${env('GITHUB_WORKSPACE', '.')}/FEATURE_REVIEW_FAILED`,
`PR #${PR}: ${e.stack || e.message}\n`, { flag: 'a' });
} catch { /* best effort */ }
process.exit(0);
});
9 changes: 8 additions & 1 deletion .github/qa/mobile-probe.mjs
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
import { randomUUID } from 'node:crypto';
import { researchCode } from './code-search.mjs';

// The LLM proxy rejects any request without `x-session-id` and `x-slicc-version` (HTTP 403
// missing_required_header). One id per process, as in qa-runner-v2.mjs.
const SESSION_ID = randomUUID();
const SLICC_VERSION = '1.0.0';

const PROXY=process.env.PROXY_URL, MODEL=process.env.MODEL, TOKEN=process.env.IMS_ACCESS_TOKEN;
const REPO_ROOT='/private/tmp/schema-branch';
function extractJson(src){const t=String(src).replace(/```(?:json)?/gi,'').trim();const a=t.indexOf('{'),b=t.lastIndexOf('}');return JSON.parse(t.slice(a,b+1));}
async function llm(prompt, maxTokens=3000){
const res=await fetch(PROXY,{method:'POST',headers:{Authorization:`Bearer ${TOKEN}`,'Content-Type':'application/json','anthropic-version':'2023-06-01'},body:JSON.stringify({model:MODEL,max_tokens:maxTokens,stream:true,messages:[{role:'user',content:prompt}]})});
const res=await fetch(PROXY,{method:'POST',headers:{Authorization:`Bearer ${TOKEN}`,'Content-Type':'application/json','anthropic-version':'2023-06-01','x-session-id':SESSION_ID,'x-slicc-version':SLICC_VERSION},body:JSON.stringify({model:MODEL,max_tokens:maxTokens,stream:true,messages:[{role:'user',content:prompt}]})});
const raw=await res.text(); let text='';
for(const line of raw.split('\n')){const s=line.trim();if(!s.startsWith('data:'))continue;const d=s.slice(5).trim();if(!d||d==='[DONE]')continue;let e;try{e=JSON.parse(d)}catch{continue}if(e.type==='content_block_delta'&&e.delta?.type==='text_delta')text+=e.delta.text||''}
return text.trim();
Expand Down
9 changes: 8 additions & 1 deletion .github/qa/plan-probe.mjs
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
import { randomUUID } from 'node:crypto';
import { researchCode } from './code-search.mjs';

// The LLM proxy rejects any request without `x-session-id` and `x-slicc-version` (HTTP 403
// missing_required_header). One id per process, as in qa-runner-v2.mjs.
const SESSION_ID = randomUUID();
const SLICC_VERSION = '1.0.0';

const PROXY=process.env.PROXY_URL, MODEL=process.env.MODEL, TOKEN=process.env.IMS_ACCESS_TOKEN;
const REPO_ROOT='/private/tmp/schema-branch';
function extractJson(src){
Expand All @@ -7,7 +14,7 @@ function extractJson(src){
return JSON.parse(t.slice(a,b+1));
}
async function llm(prompt, maxTokens=3000){
const res=await fetch(PROXY,{method:'POST',headers:{Authorization:`Bearer ${TOKEN}`,'Content-Type':'application/json','anthropic-version':'2023-06-01'},body:JSON.stringify({model:MODEL,max_tokens:maxTokens,stream:true,messages:[{role:'user',content:prompt}]})});
const res=await fetch(PROXY,{method:'POST',headers:{Authorization:`Bearer ${TOKEN}`,'Content-Type':'application/json','anthropic-version':'2023-06-01','x-session-id':SESSION_ID,'x-slicc-version':SLICC_VERSION},body:JSON.stringify({model:MODEL,max_tokens:maxTokens,stream:true,messages:[{role:'user',content:prompt}]})});
const raw=await res.text(); let text='';
for(const line of raw.split('\n')){const s=line.trim();if(!s.startsWith('data:'))continue;const d=s.slice(5).trim();if(!d||d==='[DONE]')continue;let e;try{e=JSON.parse(d)}catch{continue}if(e.type==='content_block_delta'&&e.delta?.type==='text_delta')text+=e.delta.text||''}
return text.trim();
Expand Down
31 changes: 31 additions & 0 deletions .github/workflows/qa-feature-review.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ jobs:
permissions:
contents: read
pull-requests: write
issues: write
steps:
- uses: actions/checkout@v4
with:
Expand Down Expand Up @@ -95,6 +96,36 @@ jobs:
rm -f /tmp/feature-render.png
node feature-review.mjs

# feature-review.mjs is advisory and always exits 0, so a tool outage
# (dead proxy, CDP gone, GH API error) looks exactly like a healthy run:
# green job, no comment, nobody notified. Mirror the Agent QA Review
# monitor so those failures land on the one tracking issue instead.
- name: Log feature review failure to the monitor issue
if: always()
uses: actions/github-script@v7
env:
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
script: |
const fs = require('fs');
if (!fs.existsSync('FEATURE_REVIEW_FAILED')) { core.info('Feature QA Review OK; nothing to log.'); return; }
const reason = fs.readFileSync('FEATURE_REVIEW_FAILED', 'utf8').trim();
const { owner, repo } = context.repo;
const title = 'CI: AI / Agent review tool failures';
// Find the ONE monitor issue whether open OR closed, so we never create duplicates.
const sr = await github.rest.search.issuesAndPullRequests({ q: `repo:${owner}/${repo} is:issue in:title "${title}"`, per_page: 10 });
let issue = (sr.data.items || []).find(i => i.title === title);
if (!issue) {
const res = await github.rest.issues.create({ owner, repo, title, body: 'Automated log of AI Code Review / Agent QA Review TOOL failures (upstream / CDP / stream errors, not code regressions). Each comment is one failed run and @-mentions the maintainer; the PR check stays green. Close this issue when handled; a later failure reopens it.' });
issue = res.data;
} else if (issue.state === 'closed') {
await github.rest.issues.update({ owner, repo, issue_number: issue.number, state: 'open' });
}
const cc = '\n\ncc @sanrai'; // mention on every failure -> a notification each time
const body = ['\u274c **Feature QA Review** could not run \u2014 ' + new Date().toISOString(), '', '- ' + reason.split('\n')[0], '- Run: ' + process.env.RUN_URL, cc].join('\n');
await github.rest.issues.createComment({ owner, repo, issue_number: issue.number, body });
core.info('Logged Feature QA Review failure on issue #' + issue.number);

- name: Upload render screenshot
if: always()
run: |
Expand Down
Loading