fix(copilot): deduplicate SSE-replayed messages by content fingerprint

autogpt_platform/frontend/src/app/(platform)/copilot/helpers.test.ts

@@ -2,6 +2,7 @@ import type { UIMessage } from "ai";
 import { describe, expect, it } from "vitest";
 import {
   ORIGINAL_TITLE,
+  deduplicateMessages,
   extractSendMessageText,
   formatNotificationTitle,
   getSendSuppressionReason,
@@ -291,3 +292,119 @@ describe("getSendSuppressionReason", () => {
     ).toBeNull();
   });
 });
+
+// Helper that creates messages with explicit IDs for dedup tests
+function makeMsgWithId(
+  id: string,
+  role: "user" | "assistant",
+  text: string,
+): UIMessage {
+  return { id, role, parts: [{ type: "text", text }] };
+}
+
+describe("deduplicateMessages", () => {
+  it("removes messages with duplicate IDs", () => {
+    const msgs = [
+      makeMsgWithId("1", "user", "hello"),
+      makeMsgWithId("1", "user", "hello"),
+    ];
+    expect(deduplicateMessages(msgs)).toHaveLength(1);
+  });
+
+  it("removes non-adjacent assistant duplicates with different IDs (SSE replay)", () => {
+    const msgs = [
+      makeMsgWithId("u1", "user", "hello"),
+      makeMsgWithId("a1", "assistant", "Plan of Attack"),
+      makeMsgWithId("a2", "assistant", "Next step"),
+      // SSE replay appends the same content with new IDs
+      makeMsgWithId("a3", "assistant", "Plan of Attack"),
+      makeMsgWithId("a4", "assistant", "Next step"),
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(3); // user + 2 unique assistant msgs
+    expect(result.map((m) => m.id)).toEqual(["u1", "a1", "a2"]);
+  });
+
+  it("keeps identical assistant replies to different user prompts", () => {
+    const msgs = [
+      makeMsgWithId("u1", "user", "What is 2+2?"),
+      makeMsgWithId("a1", "assistant", "4"),
+      makeMsgWithId("u2", "user", "What is 1+3?"),
+      makeMsgWithId("a2", "assistant", "4"),
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(4);
+  });
+
+  it("keeps second answer when same question is asked twice in one session", () => {
+    // Regression: scoping by user message TEXT instead of ID would treat both
+    // turns as the same context and drop the second identical assistant reply.
+    const msgs = [
+      makeMsgWithId("u1", "user", "What is 2+2?"),
+      makeMsgWithId("a1", "assistant", "4"),
+      makeMsgWithId("u2", "user", "What is 2+2?"), // same question, different ID
+      makeMsgWithId("a2", "assistant", "4"), // same answer — must be kept
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(4);
+    expect(result.map((m) => m.id)).toEqual(["u1", "a1", "u2", "a2"]);
+  });
+
+  it("removes adjacent assistant duplicates", () => {
+    const msgs = [
+      makeMsgWithId("u1", "user", "hello"),
+      makeMsgWithId("a1", "assistant", "hi there"),
+      makeMsgWithId("a2", "assistant", "hi there"),
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(2);
+  });
+
+  it("handles empty message list", () => {
+    expect(deduplicateMessages([])).toEqual([]);
+  });
+
+  it("passes through unique messages unchanged", () => {
+    const msgs = [
+      makeMsgWithId("u1", "user", "question 1"),
+      makeMsgWithId("a1", "assistant", "answer 1"),
+      makeMsgWithId("u2", "user", "question 2"),
+      makeMsgWithId("a2", "assistant", "answer 2"),
+    ];
+    expect(deduplicateMessages(msgs)).toHaveLength(4);
+  });
+
+  it("deduplicates by toolCallId for tool-call parts", () => {
+    const msgs: UIMessage[] = [
+      makeMsgWithId("u1", "user", "run tool"),
+      {
+        id: "a1",
+        role: "assistant",
+        parts: [
+          {
+            type: "dynamic-tool",
+            toolCallId: "tc-1",
+            toolName: "test",
+            state: "input-available",
+            input: {},
+          },
+        ],
+      },
+      {
+        id: "a2",
+        role: "assistant",
+        parts: [
+          {
+            type: "dynamic-tool",
+            toolCallId: "tc-1",
+            toolName: "test",
+            state: "input-available",
+            input: {},
+          },
+        ],
+      },
+    ];
+    const result = deduplicateMessages(msgs);
+    expect(result).toHaveLength(2); // user + first tool call
+  });
+});

autogpt_platform/frontend/src/app/(platform)/copilot/helpers.ts

@@ -154,24 +154,36 @@ export function shouldSuppressDuplicateSend(
 }
 
 /**
- * Deduplicate messages by ID and by consecutive content fingerprint.
+ * Deduplicate messages by ID and by content fingerprint.
  *
  * ID dedup catches exact duplicates within the same source.
- * Content dedup only compares each assistant message to its **immediate
- * predecessor** — this catches hydration/stream boundary duplicates (where
- * the same content appears under different IDs) without accidentally
- * removing legitimately repeated assistant responses that are far apart.
+ * Content dedup uses a composite key of `role + preceding-user-message-id +
+ * content-fingerprint` to detect replayed messages that arrive with new
+ * IDs after an SSE reconnection replays from the beginning of the Redis
+ * stream. Scoping by user message ID (not text) preserves the second
+ * assistant reply when the user asks the same question twice and gets the
+ * same answer — two different user messages produce two different IDs even
+ * when their text is identical.
  */
 export function deduplicateMessages(messages: UIMessage[]): UIMessage[] {
   const seenIds = new Set<string>();
-  let lastAssistantFingerprint = "";
+  const seenFingerprints = new Set<string>();
+  let lastUserMsgId = "";
 
   return messages.filter((msg) => {
     if (seenIds.has(msg.id)) return false;
     seenIds.add(msg.id);
 
+    if (msg.role === "user") {
+      // Track the ID (not text) of the latest user message so we can scope
+      // assistant fingerprints to their conversational turn. Using the ID
+      // means two user messages with identical text are still treated as
+      // distinct turns, preventing false-positive deduplication.
+      lastUserMsgId = msg.id;
+    }
+
     if (msg.role === "assistant") {
-      const fingerprint = msg.parts
+      const contentFingerprint = msg.parts
         .map(
           (p) =>
             ("text" in p && p.text) ||
@@ -180,13 +192,19 @@ export function deduplicateMessages(messages: UIMessage[]): UIMessage[] {
         )
         .join("|");
 
-      // Only dedup if this assistant message is identical to the previous one
-      if (fingerprint && fingerprint === lastAssistantFingerprint) return false;
-      if (fingerprint) lastAssistantFingerprint = fingerprint;
-    } else {
-      // Reset on non-assistant messages so that identical assistant responses
-      // separated by a user message (e.g. "Done!" → user → "Done!") are kept.
-      lastAssistantFingerprint = "";
+      if (contentFingerprint) {
+        // Scope to the preceding user message turn so that identical assistant
+        // replies to *different* user prompts are preserved.
+        // NOTE: A streaming (in-progress) assistant message has a partial
+        // fingerprint that differs from its final form, so it would not be
+        // caught by this dedup. This is safe because the caller removes the
+        // in-progress assistant message before calling resumeStream() — see
+        // useCopilotStream.ts. If that removal is ever refactored away,
+        // partial streaming messages could bypass dedup.
+        const contextKey = `assistant:${lastUserMsgId}:${contentFingerprint}`;
+        if (seenFingerprints.has(contextKey)) return false;
+        seenFingerprints.add(contextKey);
+      }
     }
 
     return true;