refactor(backend/copilot): unified transcript context — extract_context_messages, mode-gated --resume, compaction-aware gap-fill

autogpt_platform/backend/backend/copilot/baseline/service.py

@@ -67,7 +67,9 @@
     STOP_REASON_END_TURN,
     STOP_REASON_TOOL_USE,
     TranscriptDownload,
+    detect_gap,
     download_transcript,
+    strip_for_upload,
     upload_transcript,
     validate_transcript,
 )
@@ -699,26 +701,6 @@ async def _compress_session_messages(
     return messages
 
 
-def is_transcript_stale(dl: TranscriptDownload | None, session_msg_count: int) -> bool:
-    """Return ``True`` when a download doesn't cover the current session.
-
-    A transcript is stale when it has a known ``message_count`` and that
-    count doesn't reach ``session_msg_count - 1`` (i.e. the session has
-    already advanced beyond what the stored transcript captures).
-    Loading a stale transcript would silently drop intermediate turns,
-    so callers should treat stale as "skip load, skip upload".
-
-    An unknown ``message_count`` (``0``) is treated as **not stale**
-    because older transcripts uploaded before msg_count tracking
-    existed must still be usable.
-    """
-    if dl is None:
-        return False
-    if not dl.message_count:
-        return False
-    return dl.message_count < session_msg_count - 1
-
-
 def should_upload_transcript(
     user_id: str | None, transcript_covers_prefix: bool
 ) -> bool:
@@ -732,47 +714,98 @@ def should_upload_transcript(
     return bool(user_id) and transcript_covers_prefix
 
 
+def _append_gap_to_builder(
+    gap: list[ChatMessage],
+    builder: TranscriptBuilder,
+) -> None:
+    """Append gap messages from chat-db into the TranscriptBuilder.
+
+    Converts ChatMessage (OpenAI format) to TranscriptBuilder entries
+    (Claude CLI JSONL format) so the uploaded transcript covers all turns.
+    """
+    import orjson
+
+    for msg in gap:
+        if msg.role == "user":
+            builder.append_user(msg.content or "")
+        elif msg.role == "assistant":
+            content_blocks: list[dict] = []
+            if msg.content:
+                content_blocks.append({"type": "text", "text": msg.content})
+            if msg.tool_calls:
+                for tc in msg.tool_calls:
+                    fn = tc.get("function", {}) if isinstance(tc, dict) else {}
+                    try:
+                        input_data = orjson.loads(fn.get("arguments", "{}"))
+                    except Exception:
+                        input_data = {}
+                    content_blocks.append({
+                        "type": "tool_use",
+                        "id": tc.get("id", "") if isinstance(tc, dict) else "",
+                        "name": fn.get("name", "unknown"),
+                        "input": input_data,
+                    })
+            if content_blocks:
+                builder.append_assistant(content_blocks=content_blocks)
+        elif msg.role == "tool" and msg.tool_call_id:
+            builder.append_tool_result(
+                tool_use_id=msg.tool_call_id,
+                content=msg.content or "",
+            )
+
+
 async def _load_prior_transcript(
     user_id: str,
     session_id: str,
-    session_msg_count: int,
+    session_messages: list[ChatMessage],
     transcript_builder: TranscriptBuilder,
 ) -> bool:
-    """Download and load the prior transcript into ``transcript_builder``.
+    """Download and load the prior CLI session into ``transcript_builder``.
 
-    Returns ``True`` when the loaded transcript fully covers the session
-    prefix; ``False`` otherwise (stale, missing, invalid, or download
-    error).  Callers should suppress uploads when this returns ``False``
-    to avoid overwriting a more complete version in storage.
+    Returns ``True`` when the loaded session fully covers the session
+    prefix; ``False`` otherwise (missing, invalid, or download error).
+    Callers should suppress uploads when this returns ``False`` to avoid
+    overwriting a more complete version in storage.
     """
     try:
-        dl = await download_transcript(user_id, session_id, log_prefix="[Baseline]")
+        restore = await download_transcript(
+            user_id, session_id, log_prefix="[Baseline]"
+        )
     except Exception as e:
-        logger.warning("[Baseline] Transcript download failed: %s", e)
+        logger.warning("[Baseline] Session restore failed: %s", e)
         return False
 
-    if dl is None:
-        logger.debug("[Baseline] No transcript available")
+    if restore is None:
+        logger.debug("[Baseline] No CLI session available")
         return False
 
-    if not validate_transcript(dl.content):
-        logger.warning("[Baseline] Downloaded transcript but invalid")
+    try:
+        raw_str = restore.content.decode("utf-8")
+    except UnicodeDecodeError:
+        logger.warning("[Baseline] CLI session content is not valid UTF-8")
         return False
 
-    if is_transcript_stale(dl, session_msg_count):
-        logger.warning(
-            "[Baseline] Transcript stale: covers %d of %d messages, skipping",
-            dl.message_count,
-            session_msg_count,
-        )
+    stripped = strip_for_upload(raw_str)
+    if not validate_transcript(stripped):
+        logger.warning("[Baseline] CLI session content invalid after strip")
         return False
 
-    transcript_builder.load_previous(dl.content, log_prefix="[Baseline]")
+    transcript_builder.load_previous(stripped, log_prefix="[Baseline]")
     logger.info(
-        "[Baseline] Loaded transcript: %dB, msg_count=%d",
-        len(dl.content),
-        dl.message_count,
+        "[Baseline] Loaded CLI session: %dB, msg_count=%d",
+        len(restore.content),
+        restore.message_count,
     )
+
+    gap = detect_gap(restore, session_messages)
+    if gap:
+        _append_gap_to_builder(gap, transcript_builder)
+        logger.info(
+            "[Baseline] Filled gap: loaded %d transcript msgs + %d gap msgs from DB",
+            restore.message_count,
+            len(gap),
+        )
+
     return True
 
 
@@ -807,10 +840,10 @@ async def _upload_final_transcript(
             upload_transcript(
                 user_id=user_id,
                 session_id=session_id,
-                content=content,
+                content=content.encode("utf-8"),
                 message_count=session_msg_count,
+                mode="baseline",
                 log_prefix="[Baseline]",
-                skip_strip=True,
             )
         )
         _background_tasks.add(upload_task)
@@ -922,7 +955,7 @@ async def stream_chat_completion_baseline(
             _load_prior_transcript(
                 user_id=user_id,
                 session_id=session_id,
-                session_msg_count=len(session.messages),
+                session_messages=session.messages,
                 transcript_builder=transcript_builder,
             ),
             prompt_task,