feat(backend/copilot): attach uploaded images and PDFs as multimodal vision blocks

autogpt_platform/backend/backend/copilot/executor/processor.py

@@ -229,6 +229,7 @@ async def _execute_async(
                 is_user_message=entry.is_user_message,
                 user_id=entry.user_id,
                 context=entry.context,
+                file_ids=entry.file_ids,
             ):
                 if cancel.is_set():
                     log.info("Cancel requested, breaking stream")

autogpt_platform/backend/backend/copilot/sdk/service.py

@@ -536,6 +536,26 @@ async def _build_query_message(
     return current_message
 
 
+def _build_file_attachment_hint(file_ids: list[str]) -> str:
+    """Build a hint telling Claude about attached files it should read.
+
+    When users upload files alongside their message, we inject a hint so
+    Claude knows to use ``read_workspace_file`` to access the content.
+    The tool's increased inline limit (20 MB) ensures images and documents
+    are returned as base64 content blocks that Claude can process natively.
+    """
+    if not file_ids:
+        return ""
+    ids_list = ", ".join(f"`{fid}`" for fid in file_ids)
+    noun = "file" if len(file_ids) == 1 else "files"
+    return (
+        f"[The user attached {len(file_ids)} {noun} to this message. "
+        f"File IDs: {ids_list}. "
+        f"Use the read_workspace_file tool with the file_id to view "
+        f"the content of each attached file.]"
+    )
+
+
 async def stream_chat_completion_sdk(
     session_id: str,
     message: str | None = None,
@@ -545,10 +565,16 @@ async def stream_chat_completion_sdk(
     retry_count: int = 0,  # noqa: ARG001
     session: ChatSession | None = None,
     context: dict[str, str] | None = None,  # noqa: ARG001
+    file_ids: list[str] | None = None,
 ) -> AsyncGenerator[StreamBaseResponse, None]:
     """Stream chat completion using Claude Agent SDK.
 
     Drop-in replacement for stream_chat_completion with improved reliability.
+
+    Args:
+        file_ids: Optional workspace file IDs attached to the user's message.
+            When provided, files are fetched, base64-encoded, and attached as
+            multimodal content blocks (images, PDFs, etc.) to the query.
     """
 
     if session is None:
@@ -822,12 +848,21 @@ def _on_stop(transcript_path: str, sdk_session_id: str) -> None:
                     transcript_msg_count,
                     session_id,
                 )
+
+                # If files are attached, hint Claude to read them via tools.
+                if file_ids:
+                    file_hint = _build_file_attachment_hint(file_ids)
+                    if file_hint:
+                        query_message = f"{query_message}\n\n{file_hint}"
+
                 logger.info(
-                    "[SDK] [%s] Sending query — resume=%s, total_msgs=%d, query_len=%d",
+                    "[SDK] [%s] Sending query — resume=%s, total_msgs=%d, "
+                    "query_len=%d, attached_files=%d",
                     session_id[:12],
                     use_resume,
                     len(session.messages),
                     len(query_message),
+                    len(file_ids) if file_ids else 0,
                 )
                 await client.query(query_message, session_id=session_id)
 

autogpt_platform/backend/backend/copilot/sdk/tool_adapter.py

@@ -12,6 +12,7 @@
 import re
 import uuid
 from contextvars import ContextVar
+from collections.abc import Callable
 from typing import TYPE_CHECKING, Any
 
 from claude_agent_sdk import create_sdk_mcp_server, tool
@@ -265,30 +266,67 @@ async def _execute_tool_sync(
 
     content_blocks: list[dict[str, str]] = [{"type": "text", "text": text}]
 
-    # If the tool result contains inline image data, add an MCP image block
-    # so Claude can "see" the image (e.g. read_workspace_file on a small PNG).
-    image_block = _extract_image_block(text)
-    if image_block:
-        content_blocks.append(image_block)
+    # If the tool result contains inline multimodal data, add a content block
+    # so Claude can "see" images or read documents (e.g. read_workspace_file).
+    content_block = _extract_content_block(text)
+    if content_block:
+        content_blocks.append(content_block)
 
     return {
         "content": content_blocks,
         "isError": not result.success,
     }
 
 
-# MIME types that Claude can process as image content blocks.
-_SUPPORTED_IMAGE_TYPES = frozenset(
-    {"image/png", "image/jpeg", "image/gif", "image/webp"}
-)
+# ---------------------------------------------------------------------------
+# Multimodal content block support
+# ---------------------------------------------------------------------------
+# Each entry maps a MIME type to a ``(block_type, max_base64_bytes)`` tuple.
+#   • "image"    → MCP image block (Claude vision, ≤20 MB raw / ~27 MB b64)
+#   • "document" → Claude document block (PDF, ≤32 MB raw / ~43 MB b64)
+#
+# To support a new file type, add a single entry here.
+# ---------------------------------------------------------------------------
+
+_IMAGE_MAX_B64 = 27_000_000  # ~20 MB raw
+_DOCUMENT_MAX_B64 = 43_000_000  # ~32 MB raw
+
+_MULTIMODAL_TYPES: dict[str, tuple[str, int]] = {
+    # Images
+    "image/png": ("image", _IMAGE_MAX_B64),
+    "image/jpeg": ("image", _IMAGE_MAX_B64),
+    "image/gif": ("image", _IMAGE_MAX_B64),
+    "image/webp": ("image", _IMAGE_MAX_B64),
+    "image/svg+xml": ("image", _IMAGE_MAX_B64),
+    "image/bmp": ("image", _IMAGE_MAX_B64),
+    "image/tiff": ("image", _IMAGE_MAX_B64),
+    # Documents
+    "application/pdf": ("document", _DOCUMENT_MAX_B64),
+}
+
+# Block-type → builder function.  Keeps _extract_content_block flat.
+_BLOCK_BUILDERS: dict[str, Callable[[str, str], dict[str, Any]]] = {
+    "image": lambda mime, b64: {
+        "type": "image",
+        "data": b64,
+        "mimeType": mime,
+    },
+    "document": lambda mime, b64: {
+        "type": "document",
+        "source": {"type": "base64", "media_type": mime, "data": b64},
+    },
+}
 
 
-def _extract_image_block(text: str) -> dict[str, str] | None:
-    """Extract an MCP image content block from a tool result JSON string.
+def _extract_content_block(text: str) -> dict[str, Any] | None:
+    """Extract a multimodal content block from a tool result JSON string.
 
-    Detects workspace file responses with ``content_base64`` and an image
-    MIME type, returning an MCP-format image block that allows Claude to
-    "see" the image.  Returns ``None`` if the result is not an inline image.
+    Detects workspace file responses with ``content_base64`` and a supported
+    MIME type, returning the appropriate content block so Claude can process
+    the file (images via vision, PDFs via document support, etc.).
+
+    Returns ``None`` if the result is not a supported multimodal type or
+    exceeds size limits.
     """
     try:
         data = json.loads(text)
@@ -298,24 +336,24 @@ def _extract_image_block(text: str) -> dict[str, str] | None:
     if not isinstance(data, dict):
         return None
 
-    mime_type = data.get("mime_type", "")
-    base64_content = data.get("content_base64", "")
-
-    # Only inline small images — large ones would exceed Claude's limits.
-    # 32 KB raw ≈ ~43 KB base64.
-    _MAX_IMAGE_BASE64_BYTES = 43_000
-    if (
-        mime_type in _SUPPORTED_IMAGE_TYPES
-        and base64_content
-        and len(base64_content) <= _MAX_IMAGE_BASE64_BYTES
-    ):
-        return {
-            "type": "image",
-            "data": base64_content,
-            "mimeType": mime_type,
-        }
+    mime_type: str = data.get("mime_type", "")
+    base64_content: str = data.get("content_base64", "")
+    if not mime_type or not base64_content:
+        return None
+
+    entry = _MULTIMODAL_TYPES.get(mime_type)
+    if entry is None:
+        return None
+
+    block_type, max_b64 = entry
+    if len(base64_content) > max_b64:
+        return None
+
+    builder = _BLOCK_BUILDERS.get(block_type)
+    if builder is None:
+        return None
 
-    return None
+    return builder(mime_type, base64_content)
 
 
 def _mcp_error(message: str) -> dict[str, Any]:

autogpt_platform/backend/backend/copilot/service.py

@@ -323,6 +323,7 @@ async def stream_chat_completion(
     retry_count: int = 0,
     session: ChatSession | None = None,
     context: dict[str, str] | None = None,  # {url: str, content: str}
+    file_ids: list[str] | None = None,  # SDK-only; accepted here for API compat
     _continuation_message_id: (
         str | None
     ) = None,  # Internal: reuse message ID for tool call continuations

autogpt_platform/backend/backend/copilot/tools/workspace_files.py

@@ -213,6 +213,15 @@ def _validate_ephemeral_path(
 
 _IMAGE_MIME_TYPES = {"image/png", "image/jpeg", "image/gif", "image/webp"}
 
+# Superset of types that should be returned inline as base64 (not just metadata+URL).
+# Includes images + documents that Claude can process as multimodal content blocks.
+_INLINEABLE_MIME_TYPES = _IMAGE_MIME_TYPES | {
+    "image/svg+xml",
+    "image/bmp",
+    "image/tiff",
+    "application/pdf",
+}
+
 
 def _is_text_mime(mime_type: str) -> bool:
     return any(mime_type.startswith(t) for t in _TEXT_MIME_PREFIXES)
@@ -432,7 +441,7 @@ async def _execute(
 class ReadWorkspaceFileTool(BaseTool):
     """Tool for reading file content from workspace."""
 
-    MAX_INLINE_SIZE_BYTES = 32 * 1024  # 32KB
+    MAX_INLINE_SIZE_BYTES = 20 * 1024 * 1024  # 20MB (Claude vision/document limit)
     PREVIEW_SIZE = 500
 
     @property
@@ -534,10 +543,10 @@ async def _execute(
 
             is_small = file_info.size_bytes <= self.MAX_INLINE_SIZE_BYTES
             is_text = _is_text_mime(file_info.mime_type)
-            is_image = file_info.mime_type in _IMAGE_MIME_TYPES
+            is_inlineable = file_info.mime_type in _INLINEABLE_MIME_TYPES
 
-            # Inline content for small text/image files
-            if is_small and (is_text or is_image) and not force_download_url:
+            # Inline content for small text files and multimodal types (images, PDFs)
+            if is_small and (is_text or is_inlineable) and not force_download_url:
                 content = cached_content or await manager.read_file_by_id(
                     target_file_id
                 )