H-6763: Evaluate realtime voice interviewing experiments (superseded)

apps/brunch-agent/.env.example

@@ -0,0 +1,5 @@
+ELEVENLABS_API_KEY=sk_xxxx
+ELEVENLABS_SPEECH_ENGINE_ID=seng_xxxx
+ELEVENLABS_SPEECH_ENGINE_HOST=127.0.0.1
+ELEVENLABS_SPEECH_ENGINE_PORT=3001
+BRUNCH_CHAT_ORIGIN=http://127.0.0.1:4321

apps/brunch-agent/package.json

@@ -12,9 +12,11 @@
     "lint:eslint": "oxlint --type-aware --report-unused-disable-directives-severity=error .",
     "lint:tsc": "tsgo --noEmit",
     "petrinaut:dev": "vite dev --config petrinaut-local.vite.config.ts",
-    "test:unit": "vitest run --config vitest.config.ts"
+    "test:unit": "vitest run --config vitest.config.ts",
+    "voice:dev": "node --env-file-if-exists=.env.local --experimental-strip-types src/elevenlabs-speech-engine-server.ts"
   },
   "dependencies": {
+    "@elevenlabs/elevenlabs-js": "2.64.0",
     "@flue/react": "2.0.3",
     "@flue/runtime": "2.0.3",
     "@flue/sdk": "2.0.3",

apps/brunch-agent/petrinaut-local.vite.config.ts

@@ -35,7 +35,7 @@ const withoutIncumbentChatHandler = (
     ) {
       return true;
     }
-    return plugin.name !== "petrinaut-api-dev";
+    return plugin.name !== "petrinaut-chat-api-dev";
   });
 
 export default defineConfig(async (environment) => {

apps/brunch-agent/src/app.ts

@@ -15,7 +15,12 @@ import { Hono } from "hono";
 import { GherkinElicitor } from "./agents/gherkin-elicitor.ts";
 import { assetHandler } from "./assets.ts";
 import { petrinautChatHandler } from "./petrinaut-chat.ts";
-import { GHERKIN_AGENT_ROUTE, PETRINAUT_CHAT_ROUTE } from "./routes.ts";
+import {
+  GHERKIN_AGENT_ROUTE,
+  PETRINAUT_CHAT_ROUTE,
+  VOICE_EXPERIMENT_DIAGNOSTICS_ROUTE,
+} from "./routes.ts";
+import { voiceExperimentDiagnosticsHandler } from "./voice-experiment-diagnostics.ts";
 
 const app = new Hono();
 
@@ -30,6 +35,9 @@ app.route(`/agents/${GHERKIN_AGENT_ROUTE}`, createAgentRouter(GherkinElicitor));
 app.on(["POST", "OPTIONS"], PETRINAUT_CHAT_ROUTE, (c) =>
   petrinautChatHandler(c.req.raw),
 );
+app.get(VOICE_EXPERIMENT_DIAGNOSTICS_ROUTE, (c) =>
+  voiceExperimentDiagnosticsHandler(c.req.raw),
+);
 
 // The flue dev controller owns the whole request space — no fall-through to
 // vite's html serving — so the ui is app-served, in dev and in production

apps/brunch-agent/src/elevenlabs-speech-engine-server.ts

@@ -0,0 +1,91 @@
+import { createServer } from "node:http";
+
+import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js";
+
+import { BrunchVoiceBridge } from "@hashintel/brunch-agent-transport-aisdk/voice-bridge";
+
+import {
+  applySpeechEngineInterviewConfig,
+  createElevenLabsSpeechEngineCallbacks,
+} from "./elevenlabs-speech-engine.ts";
+
+const apiKey = process.env.ELEVENLABS_API_KEY;
+const speechEngineId = process.env.ELEVENLABS_SPEECH_ENGINE_ID;
+if (!apiKey || !speechEngineId) {
+  throw new Error(
+    "ELEVENLABS_API_KEY and ELEVENLABS_SPEECH_ENGINE_ID are required.",
+  );
+}
+
+const port = Number(process.env.ELEVENLABS_SPEECH_ENGINE_PORT ?? "3001");
+if (!Number.isInteger(port) || port < 1 || port > 65_535) {
+  throw new Error("ELEVENLABS_SPEECH_ENGINE_PORT must be a valid port.");
+}
+
+const host = process.env.ELEVENLABS_SPEECH_ENGINE_HOST ?? "127.0.0.1";
+const brunchChatOrigin =
+  process.env.BRUNCH_CHAT_ORIGIN ?? "http://127.0.0.1:4321";
+const chatEndpoint = new URL("/api/chat", brunchChatOrigin).toString();
+
+const bridge = new BrunchVoiceBridge({ chatEndpoint });
+const callbacks = createElevenLabsSpeechEngineCallbacks({ bridge });
+const elevenLabs = new ElevenLabsClient({ apiKey });
+
+await applySpeechEngineInterviewConfig({
+  speechEngine: elevenLabs.speechEngine,
+  speechEngineId,
+});
+
+console.log(
+  "Applied Speech Engine interview config (opening message, patient turn_v3)",
+);
+
+const httpServer = createServer((request, response) => {
+  if (request.method === "GET" && request.url === "/health") {
+    response.writeHead(200, {
+      "cache-control": "no-store",
+      "content-type": "application/json",
+    });
+    response.end(JSON.stringify({ status: "ok" }));
+    return;
+  }
+
+  response.writeHead(404, { "content-type": "application/json" });
+  response.end(JSON.stringify({ error: "Not found" }));
+});
+
+// Authentication remains enabled: the SDK verifies ElevenLabs' signed JWT on
+// every WebSocket upgrade before any transcript can reach Brunch.
+const attachment = elevenLabs.speechEngine.attach(
+  speechEngineId,
+  httpServer,
+  "/ws",
+  callbacks,
+);
+
+await new Promise<void>((resolve, reject) => {
+  httpServer.once("error", reject);
+  httpServer.listen(port, host, resolve);
+});
+
+console.log(
+  `ElevenLabs Speech Engine listening on http://${host}:${port}/ws; Brunch chat is ${chatEndpoint}`,
+);
+
+let shuttingDown = false;
+const shutdown = async () => {
+  if (shuttingDown) {
+    return;
+  }
+  shuttingDown = true;
+  await attachment.close();
+  await new Promise<void>((resolve, reject) => {
+    httpServer.close((error) => (error ? reject(error) : resolve()));
+  });
+};
+
+for (const signal of ["SIGINT", "SIGTERM"] as const) {
+  process.once(signal, () => {
+    void shutdown().finally(() => process.exit(0));
+  });
+}

apps/brunch-agent/src/elevenlabs-speech-engine.ts

@@ -0,0 +1,206 @@
+import type { SpeechEngineCallbacks } from "@elevenlabs/elevenlabs-js";
+
+type TranscriptMessage = {
+  content: string;
+  role: "agent" | "user";
+};
+
+type VoiceBridge = {
+  release(conversationId: string): void;
+  respond(input: {
+    conversationId: string;
+    signal: AbortSignal;
+    transcript: string;
+  }): AsyncIterable<string>;
+};
+
+type CallbackDependencies = {
+  bridge: VoiceBridge;
+  log?: (reason: string, context?: Record<string, unknown>) => void;
+};
+
+type VoiceSession = {
+  conversationId?: string;
+  sendResponse(response: AsyncIterable<string>): Promise<void>;
+};
+
+type QueuedVoiceTurn = {
+  session: VoiceSession;
+  signal: AbortSignal;
+  transcript: string;
+};
+
+type VoiceTurnState = {
+  active: Promise<void> | null;
+  lastCompletedSignal: AbortSignal | null;
+  queued: QueuedVoiceTurn | null;
+};
+
+const MAX_TRANSCRIPT_CHARACTERS = 12_000;
+
+const normalizeTranscript = (transcript: string): string => {
+  let sanitized = "";
+  for (const character of transcript.normalize("NFKC")) {
+    const codePoint = character.codePointAt(0) ?? 0;
+    if (
+      codePoint === 9 ||
+      codePoint === 10 ||
+      codePoint === 13 ||
+      codePoint >= 32
+    ) {
+      sanitized += character;
+    }
+  }
+
+  return sanitized
+    .replace(/\s+/gu, " ")
+    .trim()
+    .slice(0, MAX_TRANSCRIPT_CHARACTERS);
+};
+
+const latestUserTranscript = (transcript: TranscriptMessage[]): string => {
+  for (let index = transcript.length - 1; index >= 0; index -= 1) {
+    const message = transcript[index];
+    if (message?.role === "user") {
+      return normalizeTranscript(message.content);
+    }
+  }
+  return "";
+};
+
+const defaultLog = (reason: string, context: Record<string, unknown> = {}) => {
+  // Provider transcript, response text, and secrets must never be logged here.
+  console.error(`[ElevenLabs Speech Engine] ${reason}`, context);
+};
+
+export const speechEngineTurnConfig = {
+  turnEagerness: "patient",
+  turnModel: "turn_v3",
+  turnTimeout: 10,
+} as const;
+
+export const speechEngineOverrides = {
+  firstMessage: true,
+} as const;
+
+type SpeechEngineConfigClient = {
+  update(
+    speechEngineId: string,
+    request: {
+      overrides: typeof speechEngineOverrides;
+      turn: typeof speechEngineTurnConfig;
+    },
+  ): Promise<unknown>;
+};
+
+export const applySpeechEngineInterviewConfig = async ({
+  speechEngine,
+  speechEngineId,
+}: {
+  speechEngine: SpeechEngineConfigClient;
+  speechEngineId: string;
+}): Promise<void> => {
+  await speechEngine.update(speechEngineId, {
+    overrides: speechEngineOverrides,
+    turn: speechEngineTurnConfig,
+  });
+};
+
+export const createElevenLabsSpeechEngineCallbacks = ({
+  bridge,
+  log = defaultLog,
+}: CallbackDependencies): SpeechEngineCallbacks => {
+  const turnStateByConversationId = new Map<string, VoiceTurnState>();
+
+  const startTurn = (
+    conversationId: string,
+    state: VoiceTurnState,
+    turn: QueuedVoiceTurn,
+  ): void => {
+    const active = turn.session
+      .sendResponse(
+        bridge.respond({
+          conversationId,
+          signal: turn.signal,
+          transcript: turn.transcript,
+        }),
+      )
+      .then(() => {
+        if (!turn.signal.aborted) {
+          state.lastCompletedSignal = turn.signal;
+        }
+      })
+      .catch((error: unknown) => {
+        if (!turn.signal.aborted) {
+          log("Could not stream the Brunch voice response", {
+            errorName: error instanceof Error ? error.name : "unknown",
+          });
+        }
+      })
+      .finally(() => {
+        if (state.active !== active) {
+          return;
+        }
+        state.active = null;
+        const queued = state.queued;
+        state.queued = null;
+        if (
+          queued &&
+          !queued.signal.aborted &&
+          queued.signal !== state.lastCompletedSignal
+        ) {
+          startTurn(conversationId, state, queued);
+        }
+      });
+    state.active = active;
+  };
+
+  const release = (conversationId: string | undefined): void => {
+    if (!conversationId) {
+      return;
+    }
+    turnStateByConversationId.delete(conversationId);
+    bridge.release(conversationId);
+  };
+
+  return {
+    debug: process.env.NODE_ENV !== "production",
+    onTranscript(transcript, signal, session) {
+      const conversationId = session.conversationId;
+      if (!conversationId) {
+        log("Rejected transcript before session initialization");
+        return;
+      }
+      const userTurn = latestUserTranscript(transcript);
+      if (!userTurn) {
+        return;
+      }
+
+      const state = turnStateByConversationId.get(conversationId) ?? {
+        active: null,
+        lastCompletedSignal: null,
+        queued: null,
+      };
+      turnStateByConversationId.set(conversationId, state);
+      if (signal === state.lastCompletedSignal) {
+        return;
+      }
+
+      const turn = { session, signal, transcript: userTurn };
+      if (state.active) {
+        state.queued = turn;
+        return;
+      }
+      startTurn(conversationId, state, turn);
+    },
+    onClose(session) {
+      release(session.conversationId);
+    },
+    onDisconnect(session) {
+      release(session.conversationId);
+    },
+    onError(error) {
+      log("Speech Engine session failed", { errorName: error.name });
+    },
+  };
+};

apps/brunch-agent/src/local-dev-origins.ts

@@ -17,6 +17,8 @@ export const defaultChatOrigin = `http://${localChatListen.host}:${localChatList
 export const defaultPanelOrigins = [
   `http://${localPanelListen.host}:${localPanelListen.port}`,
   `http://localhost:${localPanelListen.port}`,
+  "http://127.0.0.1:5173",
+  "http://localhost:5173",
 ] as const;
 
 export const petrinautLocalServer = (chatOrigin: string) => ({