fix(copilot): deduplicate SSE-replayed messages by content fingerprint

autogpt_platform/frontend/src/app/(platform)/copilot/helpers.test.ts

@@ -2,6 +2,7 @@ import type { UIMessage } from "ai";
 import { describe, expect, it } from "vitest";
 import {
   ORIGINAL_TITLE,
+  deduplicateMessages,
   extractSendMessageText,
   formatNotificationTitle,
   getSendSuppressionReason,
@@ -291,3 +292,177 @@ describe("getSendSuppressionReason", () => {
     ).toBeNull();
   });
 });
+
+// Helper that creates messages with explicit IDs for dedup tests
+function makeMsgWithId(
+  id: string,
+  role: "user" | "assistant",
+  text: string,
+): UIMessage {
+  return { id, role, parts: [{ type: "text", text }] };
+}
+
+describe("deduplicateMessages", () => {
+  it("removes messages with duplicate IDs", () => {
+    const msgs = [
+      makeMsgWithId("1", "user", "hello"),
+      makeMsgWithId("1", "user", "hello"),
+    ];
+    expect(deduplicateMessages(msgs)).toHaveLength(1);
+  });
+
+  it("removes non-adjacent assistant duplicates with different IDs (SSE replay)", () => {
+    const msgs = [
+      makeMsgWithId("u1", "user", "hello"),
+      makeMsgWithId("a1", "assistant", "Plan of Attack"),
+      makeMsgWithId("a2", "assistant", "Next step"),
+      // SSE replay appends the same content with new IDs
+      makeMsgWithId("a3", "assistant", "Plan of Attack"),
+      makeMsgWithId("a4", "assistant", "Next step"),
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(3); // user + 2 unique assistant msgs
+    expect(result.map((m) => m.id)).toEqual(["u1", "a1", "a2"]);
+  });
+
+  it("keeps identical assistant replies to different user prompts", () => {
+    const msgs = [
+      makeMsgWithId("u1", "user", "What is 2+2?"),
+      makeMsgWithId("a1", "assistant", "4"),
+      makeMsgWithId("u2", "user", "What is 1+3?"),
+      makeMsgWithId("a2", "assistant", "4"),
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(4);
+  });
+
+  it("keeps second answer when same question is asked twice in one session", () => {
+    // Regression: scoping by user message TEXT instead of ID would treat both
+    // turns as the same context and drop the second identical assistant reply.
+    const msgs = [
+      makeMsgWithId("u1", "user", "What is 2+2?"),
+      makeMsgWithId("a1", "assistant", "4"),
+      makeMsgWithId("u2", "user", "What is 2+2?"), // same question, different ID
+      makeMsgWithId("a2", "assistant", "4"), // same answer — must be kept
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(4);
+    expect(result.map((m) => m.id)).toEqual(["u1", "a1", "u2", "a2"]);
+  });
+
+  it("removes adjacent assistant duplicates", () => {
+    const msgs = [
+      makeMsgWithId("u1", "user", "hello"),
+      makeMsgWithId("a1", "assistant", "hi there"),
+      makeMsgWithId("a2", "assistant", "hi there"),
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(2);
+  });
+
+  it("handles empty message list", () => {
+    expect(deduplicateMessages([])).toEqual([]);
+  });
+
+  it("passes through unique messages unchanged", () => {
+    const msgs = [
+      makeMsgWithId("u1", "user", "question 1"),
+      makeMsgWithId("a1", "assistant", "answer 1"),
+      makeMsgWithId("u2", "user", "question 2"),
+      makeMsgWithId("a2", "assistant", "answer 2"),
+    ];
+    expect(deduplicateMessages(msgs)).toHaveLength(4);
+  });
+
+  it("does not create false positives for text parts that contain the separator", () => {
+    // "a|b" + "c" and "a" + "b|c" previously collided when joined with "|"
+    const msgs: UIMessage[] = [
+      makeMsgWithId("u1", "user", "hello"),
+      {
+        id: "a1",
+        role: "assistant",
+        parts: [
+          { type: "text", text: "a|b" },
+          { type: "text", text: "c" },
+        ],
+      },
+      {
+        id: "a2",
+        role: "assistant",
+        parts: [
+          { type: "text", text: "a" },
+          { type: "text", text: "b|c" },
+        ],
+      },
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(3); // both assistant messages should be kept
+  });
+
+  it("deduplicates by toolCallId for tool-call parts", () => {
+    const msgs: UIMessage[] = [
+      makeMsgWithId("u1", "user", "run tool"),
+      {
+        id: "a1",
+        role: "assistant",
+        parts: [
+          {
+            type: "dynamic-tool",
+            toolCallId: "tc-1",
+            toolName: "test",
+            state: "input-available",
+            input: {},
+          },
+        ],
+      },
+      {
+        id: "a2",
+        role: "assistant",
+        parts: [
+          {
+            type: "dynamic-tool",
+            toolCallId: "tc-1",
+            toolName: "test",
+            state: "input-available",
+            input: {},
+          },
+        ],
+      },
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(2); // user + first tool call
+  });
+
+  it("passes through assistant messages with empty parts without deduplicating them", () => {
+    // contentFingerprint === "[]" when parts is empty; the guard skips fingerprint
+    // tracking so these messages are never incorrectly deduplicated against each other.
+    const msgs: UIMessage[] = [
+      makeMsgWithId("u1", "user", "hello"),
+      { id: "a1", role: "assistant", parts: [] },
+      { id: "a2", role: "assistant", parts: [] },
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(3); // both empty-parts messages are kept
+  });
+
+  it("does not collapse structurally different no-text parts to the same fingerprint", () => {
+    // Parts lacking both 'text' and 'toolCallId' (e.g. step-start) previously
+    // all mapped to "" causing false-positive deduplication. Now JSON.stringify(p)
+    // is used as the fallback so distinct part shapes produce distinct fingerprints.
+    const msgs: UIMessage[] = [
+      makeMsgWithId("u1", "user", "hello"),
+      {
+        id: "a1",
+        role: "assistant",
+        parts: [{ type: "step-start" }],
+      },
+      {
+        id: "a2",
+        role: "assistant",
+        parts: [{ type: "step-start" }],
+      },
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(2); // duplicate step-start messages are deduped
+  });
+});

autogpt_platform/frontend/src/app/(platform)/copilot/helpers.ts

@@ -154,39 +154,63 @@ export function shouldSuppressDuplicateSend(
 }
 
 /**
- * Deduplicate messages by ID and by consecutive content fingerprint.
+ * Deduplicate messages by ID and by content fingerprint.
  *
  * ID dedup catches exact duplicates within the same source.
- * Content dedup only compares each assistant message to its **immediate
- * predecessor** — this catches hydration/stream boundary duplicates (where
- * the same content appears under different IDs) without accidentally
- * removing legitimately repeated assistant responses that are far apart.
+ * Content dedup uses a composite key of `role + preceding-user-message-id +
+ * content-fingerprint` to detect replayed messages that arrive with new
+ * IDs after an SSE reconnection replays from the beginning of the Redis
+ * stream. Scoping by user message ID (not text) preserves the second
+ * assistant reply when the user asks the same question twice and gets the
+ * same answer — two different user messages produce two different IDs even
+ * when their text is identical.
  */
 export function deduplicateMessages(messages: UIMessage[]): UIMessage[] {
   const seenIds = new Set<string>();
-  let lastAssistantFingerprint = "";
+  const seenFingerprints = new Set<string>();
+  let lastUserMsgID = "";
 
   return messages.filter((msg) => {
     if (seenIds.has(msg.id)) return false;
     seenIds.add(msg.id);
 
+    if (msg.role === "user") {
+      // Track the ID (not text) of the latest user message so we can scope
+      // assistant fingerprints to their conversational turn. Using the ID
+      // means two user messages with identical text are still treated as
+      // distinct turns, preventing false-positive deduplication.
+      lastUserMsgID = msg.id;
+    }
+
     if (msg.role === "assistant") {
-      const fingerprint = msg.parts
-        .map(
+      // JSON.stringify the parts array to avoid separator-collision false
+      // positives: a plain join("|") on ["a|b", "c"] and ["a", "b|c"]
+      // produces the same string. JSON encoding each element is unambiguous.
+      // Fall back to JSON.stringify(p) for parts that carry neither a text nor
+      // a toolCallId (e.g. step-start) so structurally different parts never
+      // collapse to the same empty-string fingerprint element.
+      const contentFingerprint = JSON.stringify(
+        msg.parts.map(
           (p) =>
             ("text" in p && p.text) ||
             ("toolCallId" in p && p.toolCallId) ||
-            "",
-        )
-        .join("|");
-
-      // Only dedup if this assistant message is identical to the previous one
-      if (fingerprint && fingerprint === lastAssistantFingerprint) return false;
-      if (fingerprint) lastAssistantFingerprint = fingerprint;
-    } else {
-      // Reset on non-assistant messages so that identical assistant responses
-      // separated by a user message (e.g. "Done!" → user → "Done!") are kept.
-      lastAssistantFingerprint = "";
+            JSON.stringify(p),
+        ),
+      );
+
+      if (contentFingerprint !== "[]") {
+        // Scope to the preceding user message turn so that identical assistant
+        // replies to *different* user prompts are preserved.
+        // NOTE: A streaming (in-progress) assistant message has a partial
+        // fingerprint that differs from its final form, so it would not be
+        // caught by this dedup. This is safe because every caller that invokes
+        // resumeStream() first strips the in-progress assistant message —
+        // handleReconnect, the wake-resync path, and the hydration-effect path
+        // all do this. See useCopilotStream.ts.
+        const contextKey = `assistant:${lastUserMsgID}:${contentFingerprint}`;
+        if (seenFingerprints.has(contextKey)) return false;
+        seenFingerprints.add(contextKey);
+      }
     }
 
     return true;

autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotStream.ts

@@ -147,6 +147,15 @@ export function useCopilotStream({
     reconnectTimerRef.current = setTimeout(() => {
       isReconnectScheduledRef.current = false;
       setIsReconnectScheduled(false);
+      // Strip any stale in-progress assistant message before resuming.
+      // The backend replays from "0-0", so the partial message would
+      // otherwise sit alongside the fully-replayed version.
+      setMessages((prev) => {
+        if (prev.length > 0 && prev[prev.length - 1].role === "assistant") {
+          return prev.slice(0, -1);
+        }
+        return prev;
+      });
       resumeStream();
     }, delay);
   }