H-6763: Speak finalized Brunch responses with OpenAI

apps/petrinaut-website/README.md

@@ -61,20 +61,20 @@ provides a fake optimizer for isolated UI development.
 
 ## Environment variables
 
-| Name                             | Required         | Used by          | Notes                                                      |
-| -------------------------------- | ---------------- | ---------------- | ---------------------------------------------------------- |
-| `OPENAI_API_KEY`                 | for chat to work | `api/chat.ts`    | OpenAI key the function uses to call `streamText`.         |
-| `OPENAI_VOICE_API_KEY`           | for voice input  | voice API        | Dedicated OpenAI key used only by the Realtime call proxy. |
-| `PETRINAUT_OPENAI_VOICE_ENABLED` | no               | voice API        | Set to `true` to enable voice outside production.          |
-| `PETRINAUT_AI_MODEL`             | no               | `api/chat.ts`    | Overrides the default OpenAI model id.                     |
-| `PETRINAUT_OPT_ORIGIN`           | no               | `vite.config.ts` | Overrides the local optimizer proxy target.                |
-| `VITE_BRUNCH_CHAT_ENDPOINT`      | for voice input  | website          | Full Brunch Petrinaut chat endpoint used by the panel.     |
-| `VITE_PETRINAUT_OPT_PROVIDER`    | no               | website          | Set to `service` to enable the optimization route.         |
-| `SENTRY_DSN`                     | no               | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time.  |
+| Name                             | Required         | Used by          | Notes                                                     |
+| -------------------------------- | ---------------- | ---------------- | --------------------------------------------------------- |
+| `OPENAI_API_KEY`                 | for chat to work | `api/chat.ts`    | OpenAI key the function uses to call `streamText`.        |
+| `OPENAI_VOICE_API_KEY`           | for voice        | voice API        | Dedicated OpenAI key used by Realtime and Speech proxies. |
+| `PETRINAUT_OPENAI_VOICE_ENABLED` | no               | voice API        | Set to `true` to enable voice outside production.         |
+| `PETRINAUT_AI_MODEL`             | no               | `api/chat.ts`    | Overrides the default OpenAI model id.                    |
+| `PETRINAUT_OPT_ORIGIN`           | no               | `vite.config.ts` | Overrides the local optimizer proxy target.               |
+| `VITE_BRUNCH_CHAT_ENDPOINT`      | for voice input  | website          | Full Brunch Petrinaut chat endpoint used by the panel.    |
+| `VITE_PETRINAUT_OPT_PROVIDER`    | no               | website          | Set to `service` to enable the optimization route.        |
+| `SENTRY_DSN`                     | no               | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. |
 
 Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings.
 
-### Brunch voice-input preview
+### Brunch voice preview
 
 Voice input is disabled by default and always unavailable when `VERCEL_ENV` is
 `production`. To exercise the preview locally or in a Vercel preview, set a
@@ -90,6 +90,14 @@ SDK transport. Partial transcripts remain display-only. The preview derives a
 stable conversation id from the locally saved net; it is diagnostic identity,
 not production authentication or conversation authority.
 
+While voice is active, finalized assistant text and validated structured Brunch
+questions are spoken with OpenAI's dedicated Speech API. The server fixes the
+model and voice and forwards the selected canonical text without rewriting it;
+Realtime remains transcription-only. The microphone stays closed while Brunch
+is working and while AI-generated speech is being synthesized or played. The UI
+discloses that the voice is AI-generated. Ending voice cancels playback, and a
+speech failure leaves the exact response visible for reading.
+
 The Brunch deployment must allow the website origin through its
 `BRUNCH_PETRINAUT_ORIGINS` setting. Starting voice input requests browser
 microphone permission. Denying permission leaves the existing text composer

apps/petrinaut-website/api/voice/speech.ts

@@ -0,0 +1,12 @@
+import { createOpenAISpeechHandler } from "../../src/server/voice/openai-speech";
+
+declare const process: {
+  env: Record<string, string | undefined>;
+};
+
+export default {
+  fetch: createOpenAISpeechHandler({
+    environment: process.env,
+    fetch: globalThis.fetch.bind(globalThis),
+  }),
+};

apps/petrinaut-website/src/canonical-speech-fingerprint.ts

@@ -0,0 +1,9 @@
+export const hashCanonicalSpeechText = (text: string): string => {
+  let hash = 0x81_1c_9d_c5;
+  for (const byte of new TextEncoder().encode(text)) {
+    // eslint-disable-next-line no-bitwise -- FNV-1a requires byte-wise XOR.
+    hash = Math.imul(hash ^ byte, 0x01_00_01_93);
+  }
+  // eslint-disable-next-line no-bitwise -- Convert the signed result to uint32.
+  return `fnv1a32:${(hash >>> 0).toString(16).padStart(8, "0")}`;
+};

apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts

@@ -0,0 +1,178 @@
+import { describe, expect, test } from "vitest";
+
+import { ASK_TOOL_NAME } from "@hashintel/brunch-agent-transport-aisdk/client-tools";
+
+import {
+  hashCanonicalSpeechText,
+  selectCanonicalSpeechSegments,
+} from "./canonical-speech";
+
+import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui";
+
+const select = (messages: PetrinautAiMessage[]) =>
+  selectCanonicalSpeechSegments(messages);
+
+describe("canonical speech selection", () => {
+  test("selects only finalized assistant text without changing it", () => {
+    const messages = [
+      {
+        id: "user-1",
+        role: "user",
+        parts: [{ type: "text", text: "Do not speak the user." }],
+      },
+      {
+        id: "assistant-1",
+        role: "assistant",
+        parts: [
+          {
+            type: "reasoning",
+            text: "Do not speak reasoning.",
+            state: "done",
+          },
+          {
+            type: "text",
+            text: "Do not speak partial text.",
+            state: "streaming",
+          },
+          {
+            type: "text",
+            text: "   ",
+            state: "done",
+          },
+          {
+            type: "text",
+            text: "  Keep this exact finalized response.  ",
+            state: "done",
+          },
+          {
+            type: "text",
+            text: "Loaded finalized response.",
+          },
+          {
+            type: "dynamic-tool",
+            toolCallId: "diagnostic-1",
+            toolName: "diagnostic",
+            state: "output-available",
+            input: {},
+            output: { text: "Do not speak tool output." },
+          },
+        ],
+      },
+      {
+        id: "system-1",
+        role: "system",
+        parts: [{ type: "text", text: "Do not speak system text." }],
+      },
+    ] satisfies PetrinautAiMessage[];
+
+    const selected = select(messages);
+    const firstHash = hashCanonicalSpeechText(
+      "  Keep this exact finalized response.  ",
+    );
+    const secondHash = hashCanonicalSpeechText("Loaded finalized response.");
+    expect(selected).toEqual([
+      {
+        contentHash: firstHash,
+        id: `canonical-speech:assistant-1:text%3A3:${firstHash}`,
+        messageId: "assistant-1",
+        partId: "text:3",
+        source: "assistant-text",
+        text: "  Keep this exact finalized response.  ",
+      },
+      {
+        contentHash: secondHash,
+        id: `canonical-speech:assistant-1:text%3A4:${secondHash}`,
+        messageId: "assistant-1",
+        partId: "text:4",
+        source: "assistant-text",
+        text: "Loaded finalized response.",
+      },
+    ]);
+  });
+
+  test("selects one exact validated brunch_ask question", () => {
+    const messages = [
+      {
+        id: "assistant-ask",
+        role: "assistant",
+        parts: [
+          {
+            type: "dynamic-tool",
+            toolCallId: "ask-1",
+            toolName: ASK_TOOL_NAME,
+            state: "input-available",
+            input: { question: "Which operator confirms the batch?" },
+          },
+          {
+            type: "dynamic-tool",
+            toolCallId: "ask-malformed",
+            toolName: ASK_TOOL_NAME,
+            state: "input-available",
+            input: { question: 42 },
+          },
+          {
+            type: "dynamic-tool",
+            toolCallId: "ask-submitted",
+            toolName: ASK_TOOL_NAME,
+            state: "output-available",
+            input: { question: "Do not repeat an answered question." },
+            output: { answer: "Already answered." },
+          },
+          {
+            type: "dynamic-tool",
+            toolCallId: "other-tool",
+            toolName: "other_tool",
+            state: "input-available",
+            input: { question: "Do not speak another tool." },
+          },
+        ],
+      },
+    ] satisfies PetrinautAiMessage[];
+
+    const selected = select(messages);
+    const contentHash = hashCanonicalSpeechText(
+      "Which operator confirms the batch?",
+    );
+    expect(selected).toEqual([
+      {
+        contentHash,
+        id: `canonical-speech:assistant-ask:ask-1:${contentHash}`,
+        messageId: "assistant-ask",
+        partId: "ask-1",
+        source: "brunch-ask",
+        text: "Which operator confirms the batch?",
+      },
+    ]);
+  });
+
+  test("uses stable source identity plus an exact-text fingerprint", () => {
+    expect(hashCanonicalSpeechText("hello")).toBe("fnv1a32:4f9f2cab");
+
+    const first = select([
+      {
+        id: "assistant/id",
+        role: "assistant",
+        parts: [{ type: "text", text: "Exact text", state: "done" }],
+      },
+    ]);
+    const repeated = select([
+      {
+        id: "assistant/id",
+        role: "assistant",
+        parts: [{ type: "text", text: "Exact text", state: "done" }],
+      },
+    ]);
+    const changed = select([
+      {
+        id: "assistant/id",
+        role: "assistant",
+        parts: [{ type: "text", text: "Exact text ", state: "done" }],
+      },
+    ]);
+
+    expect(repeated).toEqual(first);
+    expect(changed[0]?.partId).toBe(first[0]?.partId);
+    expect(changed[0]?.contentHash).not.toBe(first[0]?.contentHash);
+    expect(changed[0]?.id).not.toBe(first[0]?.id);
+  });
+});

apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts

@@ -0,0 +1,95 @@
+import {
+  ASK_TOOL_NAME,
+  parseBrunchAskInput,
+} from "@hashintel/brunch-agent-transport-aisdk/client-tools";
+
+import { hashCanonicalSpeechText } from "../../../canonical-speech-fingerprint";
+
+import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui";
+
+export { hashCanonicalSpeechText };
+
+export interface CanonicalSpeechSegment {
+  readonly contentHash: string;
+  readonly id: string;
+  readonly messageId: string;
+  readonly partId: string;
+  readonly source: "assistant-text" | "brunch-ask";
+  readonly text: string;
+}
+
+const createSegment = (
+  messageId: string,
+  partId: string,
+  source: CanonicalSpeechSegment["source"],
+  text: string,
+): CanonicalSpeechSegment => {
+  const contentHash = hashCanonicalSpeechText(text);
+  return {
+    contentHash,
+    id: [
+      "canonical-speech",
+      encodeURIComponent(messageId),
+      encodeURIComponent(partId),
+      contentHash,
+    ].join(":"),
+    messageId,
+    partId,
+    source,
+    text,
+  };
+};
+
+export const selectCanonicalSpeechSegments = (
+  messages: PetrinautAiMessage[],
+): CanonicalSpeechSegment[] => {
+  const segments: CanonicalSpeechSegment[] = [];
+
+  for (const message of messages) {
+    if (message.role !== "assistant") {
+      continue;
+    }
+
+    for (const [partIndex, part] of message.parts.entries()) {
+      if (
+        part.type === "text" &&
+        part.state !== "streaming" &&
+        part.text.trim()
+      ) {
+        segments.push(
+          createSegment(
+            message.id,
+            `text:${partIndex}`,
+            "assistant-text",
+            part.text,
+          ),
+        );
+        continue;
+      }
+
+      if (
+        part.type !== "dynamic-tool" ||
+        part.toolName !== ASK_TOOL_NAME ||
+        part.state !== "input-available"
+      ) {
+        continue;
+      }
+
+      try {
+        const input = parseBrunchAskInput(part.input);
+        segments.push(
+          createSegment(
+            message.id,
+            part.toolCallId,
+            "brunch-ask",
+            input.question,
+          ),
+        );
+      } catch {
+        // Malformed tool inputs remain visible as tool errors; they are not spoken.
+      }
+    }
+  }
+
+  return segments;
+};