diff --git a/.changeset/stable-composer-controls.md b/.changeset/stable-composer-controls.md index d1df963bd70..1bccd0dbc6c 100644 --- a/.changeset/stable-composer-controls.md +++ b/.changeset/stable-composer-controls.md @@ -4,4 +4,5 @@ Add a generic host-rendered AI composer control with stable finalized-text submission, conversation identity, stop handling, schema-validated interactive-tool text mapping, and an -explicit separate-message target for corrections. +explicit separate-message target for corrections. Add a queue-aware voice submission path so a +finalized spoken turn is retained while another response settles. diff --git a/apps/petrinaut-website/.env.example b/apps/petrinaut-website/.env.example index dfb9d72a1b6..2e3a4e81d8a 100644 --- a/apps/petrinaut-website/.env.example +++ b/apps/petrinaut-website/.env.example @@ -1 +1,4 @@ OPENAI_API_KEY=sk-xxxx +OPENAI_VOICE_API_KEY= +PETRINAUT_OPENAI_VOICE_ENABLED=false +VITE_BRUNCH_CHAT_ENDPOINT= diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 83e8927096e..583ccbe749c 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -7,7 +7,8 @@ role: Demo site and embed host for the Petrinaut editor A website for demoing Petrinaut (libs/@hashintel/petrinaut). -A SPA with API functions for AI assistance and JSON oEmbed discovery. +A SPA with API functions for AI assistance, voice initialization, and JSON +oEmbed discovery. ## Quickstart @@ -18,7 +19,7 @@ cp .env.example .env.local turbo run dev ``` -The dev server runs at [http://localhost:5173](http://localhost:5173). A plugin in `vite.config.ts` loads the API function. +The dev server runs at [http://localhost:5173](http://localhost:5173). A plugin in `vite.config.ts` loads the API functions. In production, the functions in the `api` folder are automatically deployed as Vercel Functions. @@ -60,15 +61,39 @@ provides a fake optimizer for isolated UI development. ## Environment variables -| Name | Required | Used by | Notes | -| ----------------------------- | ---------------- | ---------------- | --------------------------------------------------------- | -| `OPENAI_API_KEY` | for chat to work | `api/chat.ts` | OpenAI key the function uses to call `streamText`. | -| `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | -| `PETRINAUT_OPT_ORIGIN` | no | `vite.config.ts` | Overrides the local optimizer proxy target. | -| `VITE_PETRINAUT_OPT_PROVIDER` | no | website | Set to `service` to enable the optimization route. | -| `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | - -Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the chat function. In production, set these in the Vercel project settings. +| Name | Required | Used by | Notes | +| -------------------------------- | ---------------- | ---------------- | ---------------------------------------------------------- | +| `OPENAI_API_KEY` | for chat to work | `api/chat.ts` | OpenAI key the function uses to call `streamText`. | +| `OPENAI_VOICE_API_KEY` | for voice input | voice API | Dedicated OpenAI key used only by the Realtime call proxy. | +| `PETRINAUT_OPENAI_VOICE_ENABLED` | no | voice API | Set to `true` to enable voice outside production. | +| `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | +| `PETRINAUT_OPT_ORIGIN` | no | `vite.config.ts` | Overrides the local optimizer proxy target. | +| `VITE_BRUNCH_CHAT_ENDPOINT` | for voice input | website | Full Brunch Petrinaut chat endpoint used by the panel. | +| `VITE_PETRINAUT_OPT_PROVIDER` | no | website | Set to `service` to enable the optimization route. | +| `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | + +Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings. + +### Brunch voice-input preview + +Voice input is disabled by default and always unavailable when `VERCEL_ENV` is +`production`. To exercise the preview locally or in a Vercel preview, set a +real `VITE_BRUNCH_CHAT_ENDPOINT`, `PETRINAUT_OPENAI_VOICE_ENABLED=true`, and a +dedicated `OPENAI_VOICE_API_KEY`. The browser sends its SDP offer to this app; +the server initializes an OpenAI transcription-only Realtime session and keeps +the provider key, model, language, and vocabulary policy private. The session +uses `gpt-live-transcribe`'s default server VAD because OpenAI's unified call +currently times out when explicit turn detection is included during setup. + +Only finalized transcripts enter the existing Petrinaut composer and Brunch AI +SDK transport. Partial transcripts remain display-only. The preview derives a +stable conversation id from the locally saved net; it is diagnostic identity, +not production authentication or conversation authority. + +The Brunch deployment must allow the website origin through its +`BRUNCH_PETRINAUT_ORIGINS` setting. Starting voice input requests browser +microphone permission. Denying permission leaves the existing text composer +available and does not submit anything to Brunch. ## Testing the API against the built output diff --git a/apps/petrinaut-website/api/voice/config.ts b/apps/petrinaut-website/api/voice/config.ts new file mode 100644 index 00000000000..c6239de88db --- /dev/null +++ b/apps/petrinaut-website/api/voice/config.ts @@ -0,0 +1,9 @@ +import { createOpenAIVoiceConfigHandler } from "../../src/server/voice/openai-voice-config"; + +declare const process: { + env: Record; +}; + +export default { + fetch: createOpenAIVoiceConfigHandler(process.env), +}; diff --git a/apps/petrinaut-website/api/voice/realtime-call.ts b/apps/petrinaut-website/api/voice/realtime-call.ts new file mode 100644 index 00000000000..b9b1b1f78ee --- /dev/null +++ b/apps/petrinaut-website/api/voice/realtime-call.ts @@ -0,0 +1,12 @@ +import { createOpenAIRealtimeCallHandler } from "../../src/server/voice/openai-realtime-call"; + +declare const process: { + env: Record; +}; + +export default { + fetch: createOpenAIRealtimeCallHandler({ + environment: process.env, + fetch: globalThis.fetch.bind(globalThis), + }), +}; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.test.ts new file mode 100644 index 00000000000..c7db816a6b9 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, test } from "vitest"; + +import { brunchAskFromComposerText } from "./brunch-ask-mapping"; + +describe("Brunch ask composer mapping", () => { + test("maps finalized composer text to the pending ask answer", () => { + expect( + brunchAskFromComposerText({ + input: { question: "Who triages the incident?" }, + text: "The support lead.", + }), + ).toEqual({ answer: "The support lead." }); + }); +}); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.tsx new file mode 100644 index 00000000000..1e8bed24fc6 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.tsx @@ -0,0 +1,151 @@ +import { type FormEvent, useId, useState } from "react"; + +import { + ASK_TOOL_NAME, + type BrunchAskInput, + type BrunchAskOutput, + parseBrunchAskInput, + parseBrunchAskOutput, +} from "@hashintel/brunch-agent-transport-aisdk/client-tools"; +import { css } from "@hashintel/ds-helpers/css"; +import { + definePetrinautAiInteractiveTool, + type PetrinautAiInteractiveToolWidgetProps, +} from "@hashintel/petrinaut/ui"; + +import { brunchAskFromComposerText } from "./brunch-ask-mapping"; + +const containerStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2", + padding: "3", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "blue.a30", + borderRadius: "lg", + backgroundColor: "blue.a10", +}); + +const questionStyle = css({ + color: "neutral.s100", + fontSize: "sm", + fontWeight: "medium", + lineHeight: "relaxed", +}); + +const formStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2", +}); + +const labelStyle = css({ + color: "neutral.s90", + fontSize: "xs", + fontWeight: "medium", +}); + +const textareaStyle = css({ + width: "full", + minHeight: "20", + padding: "2", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a30", + borderRadius: "md", + backgroundColor: "neutral.s00", + color: "neutral.s100", + fontSize: "sm", + resize: "vertical", + _focusVisible: { + borderColor: "blue.a70", + outline: "2px solid", + outlineColor: "blue.a30", + outlineOffset: "[1px]", + }, +}); + +const submitButtonStyle = css({ + alignSelf: "flex-end", + paddingX: "3", + paddingY: "2", + borderRadius: "md", + backgroundColor: "blue.a85", + color: "white", + cursor: "pointer", + fontSize: "sm", + fontWeight: "medium", + _hover: { + backgroundColor: "blue.a100", + }, + _disabled: { + cursor: "not-allowed", + opacity: 0.45, + }, +}); + +const answerStyle = css({ + padding: "2", + borderRadius: "md", + backgroundColor: "neutral.s00", + color: "neutral.s90", + fontSize: "sm", +}); + +const BrunchAskWidget = ({ + input, + state, + submit, + submittedOutput, +}: PetrinautAiInteractiveToolWidgetProps) => { + const answerId = useId(); + const [answer, setAnswer] = useState(""); + + const onSubmit = (event: FormEvent) => { + event.preventDefault(); + const trimmedAnswer = answer.trim(); + if (!trimmedAnswer) { + return; + } + submit({ answer: trimmedAnswer }); + }; + + return ( +
+

{input.question}

+ {state === "submitted" ? ( +

{submittedOutput.answer}

+ ) : ( +
+ +