diff --git a/apps/mobile/App.tsx b/apps/mobile/App.tsx index 9f235ba..4482163 100644 --- a/apps/mobile/App.tsx +++ b/apps/mobile/App.tsx @@ -23,15 +23,11 @@ const scriptUrl = (NativeModules["SourceCode"] as { getConstants?: () => { scrip ?.scriptURL; const API_URL = apiBaseUrl(process.env.EXPO_PUBLIC_API_URL, scriptUrl); -/** The page whose photo the result view shows. Every other page's values are - * still listed — only their boxes have nowhere to be drawn. */ -const PRIMARY_PAGE = 0; - type Status = | { kind: "idle" } | { kind: "working"; step: string } | { kind: "failed"; message: string } - | { kind: "done"; image: ReceiptImage; result: ExtractionResponse }; + | { kind: "done"; images: readonly ReceiptImage[]; result: ExtractionResponse }; /** Turns one capture into a request page. Text always; the JPEG only when * the text cannot carry the work — that decision, not the upload, is what @@ -49,10 +45,11 @@ async function toPage(image: ReceiptImage): Promise { /** Boxes for ONE page, in that page's own pixel space. * * The pageIndex filter is not optional: a scan may carry up to three pages, - * each anchored against its own OCR geometry, and the view shows one photo. - * Without it a box computed on page 1 was painted on page 0's image, pointing - * the reader at unrelated text — the worst failure available to an app whose - * claim is that a value is shown beside the pixels it was read from. */ + * each anchored against its own OCR geometry, and every photo may carry only + * its own page's boxes. Without it a box computed on page 1 was painted on + * page 0's image, pointing the reader at unrelated text — the worst failure + * available to an app whose claim is that a value is shown beside the pixels + * it was read from. */ function boxesOf(result: ExtractionResponse, pageIndex: number, wanted: boolean): Frame[] { const entries: { evidence: EvidenceRef; verified: boolean }[] = [ ...Object.values(result.fields) @@ -104,8 +101,7 @@ export default function App() { ocrFloor: false, }); if (scanned.status === "cancelled") return setStatus({ kind: "idle" }); - const [primary] = scanned.images; - if (primary === undefined) { + if (scanned.images.length === 0) { return setStatus({ kind: "failed", message: "the scanner returned no page" }); } @@ -135,7 +131,7 @@ export default function App() { : response.statusText; return setStatus({ kind: "failed", message }); } - setStatus({ kind: "done", image: primary, result: body as ExtractionResponse }); + setStatus({ kind: "done", images: scanned.images, result: body as ExtractionResponse }); } catch (error) { setStatus({ kind: "failed", message: error instanceof Error ? error.message : String(error) }); } @@ -168,25 +164,32 @@ export default function App() { posting to {API_URL}/api/extract )} - {status.kind === "done" && } + {status.kind === "done" && } ); } -function Result({ image, result }: { image: ReceiptImage; result: ExtractionResponse }) { +function Result({ images, result }: { images: readonly ReceiptImage[]; result: ExtractionResponse }) { const { fields, items, tenders, arithmetic, unverified, disagreements, modelReply } = result; return ( - {image.ocrLines === undefined ? ( - - ) : ( - // `image` is scanned.images[0], so only page 0's boxes belong on it. - - )} + {/* One photo per captured page, each carrying only ITS page's boxes + (issue #2) — request pages were built from this same array in this + same order, so the array index IS the evidence pageIndex. */} + {images.map((image, pageIndex) => ( + + {images.length > 1 && Page {pageIndex + 1}} + {image.ocrLines === undefined ? ( + + ) : ( + + )} + + ))} diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css index 377c119..5e0d72f 100644 --- a/apps/web/app/globals.css +++ b/apps/web/app/globals.css @@ -91,6 +91,37 @@ label.checkbox input { flex: none; } +/* One bordered block per scan page, numbered like the results ("page 0"). */ +fieldset.page-input { + display: flex; + flex-direction: column; + gap: 0.5rem; + padding: 0.75rem; + border: 1px solid var(--border); + border-radius: 8px; +} + +fieldset.page-input legend { + display: flex; + gap: 0.75rem; + align-items: center; + padding: 0 0.35rem; + font-size: 0.85rem; + color: var(--muted); +} + +button.remove-page { + padding: 0.1rem 0.6rem; + background: var(--unverified); + font-weight: 400; + font-size: 0.8rem; +} + +.form-actions { + display: flex; + gap: 0.75rem; +} + textarea, input[type="file"] { font-family: var(--font-mono); diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx index 4d61d2d..5ba4294 100644 --- a/apps/web/app/page.tsx +++ b/apps/web/app/page.tsx @@ -119,6 +119,10 @@ function FieldRow({ label, field }: { label: string; field: ExtractedField | undefined) => { - if (field?.evidence.box) boxes.push({ path, box: field.evidence.box, verified: field.verified }); + if (field?.evidence.box) { + boxes.push({ path, pageIndex: field.evidence.pageIndex, box: field.evidence.box, verified: field.verified }); + } }; push("merchant", result.fields.merchant); push("purchaseDate", result.fields.purchaseDate); @@ -134,16 +140,33 @@ function collectBoxes(result: ExtractionResponse): BoxEntry[] { push("reference", result.fields.reference); result.items.forEach((item, index) => { if (item.nameEvidence.box) { - boxes.push({ path: `item[${index}] ${item.name} (name)`, box: item.nameEvidence.box, verified: item.verified }); + boxes.push({ + path: `item[${index}] ${item.name} (name)`, + pageIndex: item.nameEvidence.pageIndex, + box: item.nameEvidence.box, + verified: item.verified, + }); } // A receipt that prints name and amount on one line cites it twice; one // box is enough there, and a second would just double the border. if (item.amountEvidence.box && !sameEvidence(item.nameEvidence, item.amountEvidence)) { - boxes.push({ path: `item[${index}] ${item.name} (amount)`, box: item.amountEvidence.box, verified: item.verified }); + boxes.push({ + path: `item[${index}] ${item.name} (amount)`, + pageIndex: item.amountEvidence.pageIndex, + box: item.amountEvidence.box, + verified: item.verified, + }); } }); result.tenders.forEach((tender, index) => { - if (tender.evidence.box) boxes.push({ path: `tender[${index}]`, box: tender.evidence.box, verified: tender.verified }); + if (tender.evidence.box) { + boxes.push({ + path: `tender[${index}]`, + pageIndex: tender.evidence.pageIndex, + box: tender.evidence.box, + verified: tender.verified, + }); + } }); return boxes; } @@ -179,80 +202,138 @@ function ArithmeticVerdict({ arithmetic }: { arithmetic: ExtractionResponse["ari ); } -export default function Page() { - const [ocrText, setOcrText] = useState(""); - const [linesText, setLinesText] = useState(""); - const [image, setImage] = useState(null); +/** One page's inputs. A multi-page scan is a list of these, and the LIST + * INDEX is the request's pageIndex — the same rule the mobile app follows, so + * every `page N` in the result points back at the Nth block on this form. */ +interface PageInput { + /** Stable key for React and input ids — list indices shift when a page is + * removed, and a shifted key would hand page B's image state to page A. */ + id: number; + ocrText: string; + linesText: string; + image: ImageInfo | null; // Sending the photo is opt-in, and clearing the file clears the consent // with it — a checkbox left ticked from a previous upload must not carry // over to the next one. - const [sendImage, setSendImage] = useState(false); - // Which file selection is current. A ref, not state: it has to be readable - // by an async continuation that started before the newer selection existed, - // and bumping it must not re-render. - const selectionCounter = useRef(0); + sendImage: boolean; +} + +export default function Page() { + const [pageInputs, setPageInputs] = useState([ + { id: 0, ocrText: "", linesText: "", image: null, sendImage: false }, + ]); + const nextPageId = useRef(1); + // Which file selection is current, PER page. A ref, not state: it has to be + // readable by an async continuation that started before the newer selection + // existed, and bumping it must not re-render. + const selectionCounters = useRef(new Map()); + // Which page STRUCTURE an in-flight extraction was posted against. Clearing + // `result` on removal is not enough: a request already running completes + // afterwards and `setResult` restores a response whose page indices use the + // OLD numbering — boxes then land on the wrong photos. Bumped by anything + // that renumbers or re-images pages; a response is dropped when its + // captured revision is no longer current. + const requestRevision = useRef(0); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); const [result, setResult] = useState(null); - const { lines, error: linesError } = useMemo(() => parseLines(linesText), [linesText]); + const parsedLines = useMemo(() => pageInputs.map((page) => parseLines(page.linesText)), [pageInputs]); const boxes = useMemo(() => (result ? collectBoxes(result) : []), [result]); - async function handleImageChange(event: ChangeEvent) { + function updatePage(id: number, patch: Partial) { + setPageInputs((pages) => pages.map((page) => (page.id === id ? { ...page, ...patch } : page))); + } + + function addPage() { + // Adding a page does not renumber anything, but the shown result — and + // any response still in flight — describes a document with FEWER pages + // than the form now shows, which misreads as the current document's + // extraction. Same rule as removal: structure changed, results are stale. + requestRevision.current += 1; + setResult(null); + const id = nextPageId.current++; + setPageInputs((pages) => [...pages, { id, ocrText: "", linesText: "", image: null, sendImage: false }]); + } + + function removePage(id: number) { + // Removing a page renumbers every page after it, and the result's boxes + // and `page N` references were computed against the OLD numbering — a + // stale result would paint them on the wrong photos. Both the shown + // result and any request still in flight are stale. + requestRevision.current += 1; + setResult(null); + setPageInputs((pages) => pages.filter((page) => page.id !== id)); + } + + async function handleImageChange(id: number, event: ChangeEvent) { const file = event.target.files?.[0]; - // Everything tied to the OLD image goes first, before the async decode: - // the consent, the image itself, and the result whose boxes were computed - // against it. Resetting only on the clear and error paths left two holes — - // a checkbox ticked for photo A authorised photo B, and a result rendered - // for A repainted its boxes onto B's pixels while decoding. - setImage(null); - setSendImage(false); + // Superseding the previous selection comes FIRST, before any early + // return: clearing the file input must also invalidate a decode still in + // flight, or image A completes after the clear and restores its preview. + // Decoding is asynchronous and a superseded selection still resolves — + // pick photo A, then photo B before A finishes, and A's write lands + // afterwards. Only the newest selection may write. + const selection = (selectionCounters.current.get(id) ?? 0) + 1; + selectionCounters.current.set(id, selection); + // Everything tied to the OLD image goes next, still before the async + // decode: the consent, the image itself, and the result (shown OR in + // flight) whose boxes were computed against it. Resetting only on the + // clear and error paths left two holes — a checkbox ticked for photo A + // authorised photo B, and a result rendered for A repainted its boxes + // onto B's pixels while decoding. + requestRevision.current += 1; + updatePage(id, { image: null, sendImage: false }); setResult(null); if (!file) return; - // Clearing up front is not enough on its own: decoding is asynchronous and - // a superseded selection still resolves. Pick photo A, then photo B before - // A finishes, and A's `setImage` lands afterwards — the page then shows A - // while the file input says B, which is the consent bug wearing a - // different hat. Only the newest selection may write. - const selection = (selectionCounter.current += 1); // Both helpers reject — an unreadable file, an undecodable image (a HEIC // on a browser without support, a truncated download). Uncaught, the - // rejection was silent and `image` kept its previous value, so the next + // rejection was silent and the image kept its previous value, so the next // extraction drew boxes over the wrong photo. try { const dataUrl = await readFileAsDataUrl(file); const { width, height } = await loadImage(dataUrl); - if (selectionCounter.current !== selection) return; - setImage({ dataUrl, naturalWidth: width, naturalHeight: height }); + if (selectionCounters.current.get(id) !== selection) return; + updatePage(id, { image: { dataUrl, naturalWidth: width, naturalHeight: height } }); setError(null); } catch (err) { - if (selectionCounter.current !== selection) return; + if (selectionCounters.current.get(id) !== selection) return; setError(`could not read that image: ${err instanceof Error ? err.message : String(err)}`); } } async function handleSubmit(event: FormEvent) { event.preventDefault(); - if (ocrText.trim() === "") { + if (pageInputs.every((page) => page.ocrText.trim() === "")) { setError("paste OCR text first — the pipeline has nothing to parse without it"); return; } - if (linesError) { - setError(`OCR lines JSON is invalid: ${linesError}`); + const badLines = parsedLines.findIndex((parsed) => parsed.error !== null); + if (badLines !== -1) { + setError(`page ${badLines}: OCR lines JSON is invalid: ${parsedLines[badLines]?.error ?? ""}`); return; } setLoading(true); setError(null); setResult(null); + // The page structure this request is being posted against. If it changes + // while the request runs, the response's page indices describe pages that + // no longer exist in that order — drop it rather than paint it. + const revision = requestRevision.current; try { const response = await fetch("/api/extract", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ - pages: [{ text: ocrText, lines, imageDataUrl: sendImage ? image?.dataUrl : undefined }], + pages: pageInputs.map((page, index) => ({ + text: page.ocrText, + lines: parsedLines[index]?.lines ?? [], + imageDataUrl: page.sendImage ? page.image?.dataUrl : undefined, + })), }), }); const body: unknown = await response.json(); + if (requestRevision.current !== revision) return; if (!response.ok) { const message = typeof body === "object" && body !== null && "error" in body ? String((body as { error: unknown }).error) : response.statusText; setError(message); @@ -260,6 +341,7 @@ export default function Page() { } setResult(body as ExtractionResponse); } catch (err) { + if (requestRevision.current !== revision) return; setError(err instanceof Error ? err.message : "request failed"); } finally { setLoading(false); @@ -276,48 +358,80 @@ export default function Page() {

void handleSubmit(event)}> - -