Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion gui/.eslint/i18n-allowlist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,8 @@ export function isTechnicalLiteral(value: string): boolean {
if (/^ocx\b/i.test(trimmed)) return true;
if (/^codex\b/i.test(trimmed)) return true;

// HTTP headers / auth schemes
// HTTP protocol / headers / auth schemes
if (trimmed === "HTTP") return true;
if (/^Authorization\b/i.test(trimmed)) return true;
if (/^Bearer\b/i.test(trimmed)) return true;
if (/^Content-Type\b/i.test(trimmed)) return true;
Expand Down
39 changes: 34 additions & 5 deletions gui/src/pages/CompatibilityMatrix.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,12 @@ type ExtraVerdictPage = {
hasMore: boolean;
};

type LoadMoreFailure = {
baseData: LabPageData;
queryKey: string;
message: string;
};

function localizedFetchError(e: unknown, fallback: string): string {
if (!(e instanceof Error)) return fallback;
const msg = e.message;
Expand Down Expand Up @@ -171,6 +177,11 @@ function DetailPane({
locale: Parameters<typeof labSupplement>[0];
onClose: () => void;
}) {
const expectedEventCount = new Set([
...verdict.contributingEventIds,
...verdict.contradictingEventIds,
]).size;

return (
<aside className="lab-detail-pane" aria-label={t("lab.detailTitle")}>
<div className="lab-detail-head">
Expand Down Expand Up @@ -219,9 +230,12 @@ function DetailPane({
</ul>
</section>
)}
{detail.events.length > 0 && (
{(detail.events.length > 0 || expectedEventCount > 0) && (
<section className="lab-detail-section">
<h4>{t("lab.detailEvents")}</h4>
<h4>
{t("lab.detailEvents")}
{detail.events.length < expectedEventCount ? ` (${detail.events.length}/${expectedEventCount})` : ""}
</h4>
<ul className="lab-detail-list">
{detail.events.map(event => (
<li key={event.eventId}>
Expand Down Expand Up @@ -264,6 +278,7 @@ export default function CompatibilityMatrix({
const { t, locale } = useI18n();
const [filters, setFilters] = useState<VerdictFilters>({ layer: "", verdict: "", subjectQuery: "", suiteId: "" });
const [extraPage, setExtraPage] = useState<ExtraVerdictPage | null>(null);
const [loadMoreFailure, setLoadMoreFailure] = useState<LoadMoreFailure | null>(null);
const [loadingMore, setLoadingMore] = useState(false);
const [selectedVerdict, setSelectedVerdict] = useState<VerdictDto | null>(null);
const [detail, setDetail] = useState<VerdictDetailData | null>(null);
Expand Down Expand Up @@ -293,6 +308,7 @@ export default function CompatibilityMatrix({
loadMoreRef.current?.abort();
loadMoreRef.current = null;
setExtraPage(null);
setLoadMoreFailure(null);
setLoadingMore(false);
}, []);

Expand Down Expand Up @@ -329,6 +345,11 @@ export default function CompatibilityMatrix({
&& extraPage.queryKey === queryKey
? extraPage
: null;
const visibleLoadMoreError = loadMoreFailure !== null
&& loadMoreFailure.baseData === surface.data
&& loadMoreFailure.queryKey === queryKey
? loadMoreFailure.message
: null;

const reportedCount = useMemo(() => {
if (!active || !surface.data?.status.projectionAvailable) return null;
Expand Down Expand Up @@ -357,6 +378,7 @@ export default function CompatibilityMatrix({
const controller = new AbortController();
loadMoreRef.current = controller;
const startedKey = queryKey;
setLoadMoreFailure(null);
setLoadingMore(true);
try {
const page = await fetchMoreVerdicts(apiBase, queryFilters, cursor, controller.signal);
Expand All @@ -373,15 +395,21 @@ export default function CompatibilityMatrix({
hasMore: page.hasMore,
};
});
} catch {
// Keep the current rows. The normal refresh action retries from a consistent first page.
} catch (e) {
if (!controller.signal.aborted) {
setLoadMoreFailure({
baseData,
queryKey: startedKey,
message: localizedFetchError(e, t("lab.loadFailed")),
});
}
} finally {
if (loadMoreRef.current === controller) {
loadMoreRef.current = null;
setLoadingMore(false);
}
}
}, [apiBase, loadingMore, queryFilters, queryKey, surface.data, validExtraPage]);
}, [apiBase, loadingMore, queryFilters, queryKey, surface.data, t, validExtraPage]);

const selectVerdict = useCallback(async (verdict: VerdictDto) => {
if (!active) return;
Expand Down Expand Up @@ -582,6 +610,7 @@ export default function CompatibilityMatrix({
</div>
</div>

{visibleLoadMoreError && <Notice tone="err">{visibleLoadMoreError}</Notice>}
{pageHasMore && (
<div className="lab-load-more">
<button type="button" className="btn btn-ghost" disabled={loadingMore} onClick={() => { void loadMore(); }}>
Expand Down
196 changes: 196 additions & 0 deletions gui/tests/compatibility-lab-followup.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
/** @jsxImportSource react */
import { afterEach, beforeEach, expect, test } from "bun:test";
import { Window } from "happy-dom";
import { act } from "react";
import type { Root } from "react-dom/client";
import { LanguageProvider } from "../src/i18n/provider";
import CompatibilityMatrix from "../src/pages/CompatibilityMatrix";
import { clearClientResourceStoresForTests } from "../src/client-resource";

const originalFetch = globalThis.fetch;
let restoreGlobals: (() => void) | undefined;
let previousLanguageDescriptor: PropertyDescriptor | undefined;
let testWindow: Window;

const API_BASE = "http://127.0.0.1:4096";

const STATUS_AVAILABLE = {
projectionAvailable: true,
subjectCount: 1,
verdictCount: 1,
observationCount: 0,
eventCount: 2,
builtAtMs: 1_700_000_000_000,
};

const SUBJECTS = {
subjects: [{ subjectId: "subject-alpha", subjectKind: "protocol" }],
hasMore: false,
};

const SUBJECT_DETAIL = {
subject: {
subjectKind: "protocol",
subjectSchemaVersion: 1,
inboundProtocol: "openai-chat",
},
};

const EVENT_DETAIL = {
event: {
eventKind: "observation",
eventId: "e1",
recordedAt: 1_700_000_000_040,
producer: "lab",
producerVersion: "1",
subjectId: "subject-alpha",
evidenceLayer: "protocol_conformance",
suiteId: "responses-core",
outcome: "pass",
excluded: false,
exclusionReason: null,
},
};

function verdictPage(eventIds: string[]) {
return {
verdicts: [{
projectionKey: "k1",
subjectId: "subject-alpha",
evidenceLayer: "protocol_conformance",
suiteId: "responses-core",
suiteVersion: "1",
suiteManifestDigest: "digest-a",
projectionSpecVersion: "cl-02.v1",
verdict: "VERIFIED",
asOf: 1_700_000_000_100,
scenarioManifestDigests: [],
claimSourceDigest: null,
contributingEventIds: eventIds,
contradictingEventIds: [],
notes: [],
}],
hasMore: true,
nextCursor: "cursor-2",
};
}

type FetchOptions = {
failLoadMoreOnce?: boolean;
partialEvents?: boolean;
};

function installLabFetch(opts: FetchOptions = {}) {
const requests: string[] = [];
let loadMoreAttempts = 0;
globalThis.fetch = (async (input: RequestInfo | URL) => {
const url = String(input);
requests.push(url);
if (url.endsWith("/api/lab/status")) return Response.json(STATUS_AVAILABLE);
if (url.includes("/api/lab/verdicts")) {
if (url.includes("cursor=cursor-2")) {
loadMoreAttempts += 1;
if (opts.failLoadMoreOnce && loadMoreAttempts === 1) {
return new Response("unavailable", { status: 503 });
}
return Response.json({ verdicts: [], hasMore: false });
}
return Response.json(verdictPage(opts.partialEvents ? ["e1", "missing-event"] : ["e1"]));
}
if (url.includes("/api/lab/subjects/subject-alpha")) return Response.json(SUBJECT_DETAIL);
if (url.includes("/api/lab/subjects")) return Response.json(SUBJECTS);
if (url.includes("/api/lab/observations")) return Response.json({ observations: [], hasMore: false });
if (url.includes("/api/lab/events/e1")) return Response.json(EVENT_DETAIL);
if (url.includes("/api/lab/events/missing-event")) return new Response("gone", { status: 404 });
if (url.includes("/api/lab/artifacts/")) return new Response("gone", { status: 404 });
if (url.includes("/api/lab/production-signals")) return new Response("gone", { status: 404 });
return new Response("{}", { status: 404 });
}) as typeof fetch;
return { requests };
}

beforeEach(() => {
clearClientResourceStoresForTests();
testWindow = new Window({ url: "http://localhost/#models/compatibility" });
previousLanguageDescriptor = Object.getOwnPropertyDescriptor(globalThis.navigator, "language");
Object.defineProperty(globalThis.navigator, "language", { configurable: true, value: "en-US" });
const keys = ["document", "window", "localStorage", "IS_REACT_ACT_ENVIRONMENT"] as const;
const previous = Object.fromEntries(
keys.map(key => [key, Object.getOwnPropertyDescriptor(globalThis, key)]),
) as Record<(typeof keys)[number], PropertyDescriptor | undefined>;
Object.defineProperties(globalThis, {
document: { configurable: true, value: testWindow.document },
window: { configurable: true, value: testWindow },
localStorage: { configurable: true, value: testWindow.localStorage },
IS_REACT_ACT_ENVIRONMENT: { configurable: true, value: true },
});
restoreGlobals = () => {
for (const key of keys) {
const descriptor = previous[key];
if (descriptor) Object.defineProperty(globalThis, key, descriptor);
else delete (globalThis as Record<string, unknown>)[key];
}
if (previousLanguageDescriptor) {
Object.defineProperty(globalThis.navigator, "language", previousLanguageDescriptor);
}
};
});

afterEach(() => {
restoreGlobals?.();
globalThis.fetch = originalFetch;
clearClientResourceStoresForTests();
testWindow.close();
});

async function waitFor(predicate: () => boolean, timeoutMs = 3000): Promise<void> {
const start = Date.now();
while (!predicate()) {
if (Date.now() - start > timeoutMs) throw new Error("waitFor timed out");
await act(async () => {
await new Promise<void>(resolve => testWindow.setTimeout(resolve, 10));
});
}
}

async function renderMatrix(): Promise<{ root: Root; container: HTMLDivElement }> {
const { createRoot } = await import("react-dom/client");
const container = testWindow.document.createElement("div");
testWindow.document.body.appendChild(container);
const root = createRoot(container);
await act(async () => {
root.render(<LanguageProvider><CompatibilityMatrix apiBase={API_BASE} /></LanguageProvider>);
});
return { root, container };
}

test("load-more failures stay visible and a retry can recover", async () => {
const { requests } = installLabFetch({ failLoadMoreOnce: true });
const { root, container } = await renderMatrix();
await waitFor(() => container.querySelector(".lab-load-more button") !== null);
const button = container.querySelector(".lab-load-more button") as HTMLButtonElement;
await act(async () => { button.click(); });
await waitFor(() => container.querySelector(".notice-err")?.textContent?.includes(String(503)) ?? false);
expect(container.textContent).toContain("Verified");
expect(container.querySelector(".lab-load-more button")).not.toBeNull();

await act(async () => {
(container.querySelector(".lab-load-more button") as HTMLButtonElement).click();
});
await waitFor(() => container.querySelector(".notice-err") === null);
await waitFor(() => container.querySelector(".lab-load-more button") === null);
expect(requests.filter(url => url.includes("/api/lab/verdicts") && url.includes("cursor=cursor-2"))).toHaveLength(2);
expect(container.textContent).toContain("Verified");
await act(async () => root.unmount());
});

test("verdict detail reports when referenced evidence events are only partially available", async () => {
installLabFetch({ partialEvents: true });
const { root, container } = await renderMatrix();
await waitFor(() => container.querySelector('button[data-verdict-detail="k1"]') !== null);
const button = container.querySelector('button[data-verdict-detail="k1"]') as HTMLButtonElement;
await act(async () => { button.click(); });
await waitFor(() => container.querySelector(".lab-detail-pane")?.textContent?.includes("Evidence events (1/2)") ?? false);
expect(container.querySelector(".lab-detail-pane")?.textContent).toContain("Evidence events (1/2)");
await act(async () => root.unmount());
});
Loading