feat(backend/copilot): attach uploaded images and PDFs as multimodal vision blocks

autogpt_platform/backend/backend/api/features/workspace/routes.py

@@ -218,7 +218,10 @@ async def upload_file(
 
     # Write file via WorkspaceManager
     manager = WorkspaceManager(user_id, workspace.id, session_id)
-    workspace_file = await manager.write_file(content, filename)
+    try:
+        workspace_file = await manager.write_file(content, filename)
+    except ValueError as e:
+        raise fastapi.HTTPException(status_code=409, detail=str(e)) from e
 
     # Post-write storage check — eliminates TOCTOU race on the quota.
     # If a concurrent upload pushed us over the limit, undo this write.

autogpt_platform/backend/backend/copilot/executor/processor.py

@@ -228,6 +228,8 @@ async def _execute_async(
                 message=entry.message if entry.message else None,
                 is_user_message=entry.is_user_message,
                 user_id=entry.user_id,
+                context=entry.context,
+                file_ids=entry.file_ids,
             ):
                 if cancel.is_set():
                     log.info("Cancel requested, breaking stream")

autogpt_platform/backend/backend/copilot/sdk/service.py

@@ -5,6 +5,7 @@
 import json
 import logging
 import os
+import re
 import shutil
 import sys
 import uuid
@@ -55,6 +56,7 @@
 )
 from ..tools.e2b_sandbox import get_or_create_sandbox
 from ..tools.sandbox import WORKSPACE_PREFIX, make_session_path
+from ..tools.workspace_files import get_manager
 from ..tracking import track_user_message
 from .compaction import CompactionTracker, filter_compaction_messages
 from .response_adapter import SDKResponseAdapter
@@ -568,15 +570,143 @@ async def _build_query_message(
     return current_message, False
 
 
+# Claude API vision-supported image types.
+_VISION_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/gif", "image/webp"})
+
+# Max size for embedding images directly in the user message (20 MiB raw).
+_MAX_INLINE_IMAGE_BYTES = 20 * 1024 * 1024
+
+# Matches characters unsafe for filenames.
+_UNSAFE_FILENAME = re.compile(r"[^\w.\-]")
+
+
+def _save_to_sdk_cwd(sdk_cwd: str, filename: str, content: bytes) -> str:
+    """Write file content to the SDK ephemeral directory.
+
+    Returns the absolute path.  Adds a numeric suffix on name collisions.
+    """
+    safe = _UNSAFE_FILENAME.sub("_", filename) or "file"
+    candidate = os.path.join(sdk_cwd, safe)
+    if os.path.exists(candidate):
+        stem, ext = os.path.splitext(safe)
+        idx = 1
+        while os.path.exists(candidate):
+            candidate = os.path.join(sdk_cwd, f"{stem}_{idx}{ext}")
+            idx += 1
+    with open(candidate, "wb") as f:
+        f.write(content)
+    return candidate
+
+
+@dataclass
+class PreparedAttachments:
+    """Result of preparing file attachments for a query."""
+
+    hint: str
+    """Text hint describing the files (appended to the user message)."""
+
+    image_blocks: list[dict[str, Any]]
+    """Claude API image content blocks to embed in the user message."""
+
+
+async def _prepare_file_attachments(
+    file_ids: list[str],
+    user_id: str,
+    session_id: str,
+    sdk_cwd: str,
+) -> PreparedAttachments:
+    """Download workspace files and prepare them for Claude.
+
+    Images (PNG/JPEG/GIF/WebP) are embedded directly as vision content blocks
+    in the user message so Claude can see them without tool calls.
+
+    Non-image files (PDFs, text, etc.) are saved to *sdk_cwd* so the CLI's
+    built-in Read tool can access them.
+
+    Returns a :class:`PreparedAttachments` with a text hint and any image
+    content blocks.
+    """
+    empty = PreparedAttachments(hint="", image_blocks=[])
+    if not file_ids or not user_id:
+        return empty
+
+    try:
+        manager = await get_manager(user_id, session_id)
+    except Exception:
+        logger.warning(
+            "Failed to create workspace manager for file attachments",
+            exc_info=True,
+        )
+        return empty
+
+    image_blocks: list[dict[str, Any]] = []
+    file_descriptions: list[str] = []
+
+    for fid in file_ids:
+        try:
+            file_info = await manager.get_file_info(fid)
+            if file_info is None:
+                continue
+            content = await manager.read_file_by_id(fid)
+            mime = (file_info.mime_type or "").split(";")[0].strip().lower()
+
+            # Images: embed directly in the user message as vision blocks
+            if mime in _VISION_MIME_TYPES and len(content) <= _MAX_INLINE_IMAGE_BYTES:
+                b64 = base64.b64encode(content).decode("ascii")
+                image_blocks.append(
+                    {
+                        "type": "image",
+                        "source": {
+                            "type": "base64",
+                            "media_type": mime,
+                            "data": b64,
+                        },
+                    }
+                )
+                file_descriptions.append(
+                    f"- {file_info.name} ({mime}, "
+                    f"{file_info.size_bytes:,} bytes) [embedded as image]"
+                )
+            else:
+                # Non-image files: save to sdk_cwd for Read tool access
+                local_path = _save_to_sdk_cwd(sdk_cwd, file_info.name, content)
+                file_descriptions.append(
+                    f"- {file_info.name} ({mime}, "
+                    f"{file_info.size_bytes:,} bytes) saved to {local_path}"
+                )
+        except Exception:
+            logger.warning("Failed to prepare file %s", fid[:12], exc_info=True)
+
+    if not file_descriptions:
+        return empty
+
+    noun = "file" if len(file_descriptions) == 1 else "files"
+    has_non_images = len(file_descriptions) > len(image_blocks)
+    read_hint = " Use the Read tool to view non-image files." if has_non_images else ""
+    hint = (
+        f"[The user attached {len(file_descriptions)} {noun}.{read_hint}\n"
+        + "\n".join(file_descriptions)
+        + "]"
+    )
+    return PreparedAttachments(hint=hint, image_blocks=image_blocks)
+
+
 async def stream_chat_completion_sdk(
     session_id: str,
     message: str | None = None,
     is_user_message: bool = True,
     user_id: str | None = None,
     session: ChatSession | None = None,
+    file_ids: list[str] | None = None,
     **_kwargs: Any,
 ) -> AsyncGenerator[StreamBaseResponse, None]:
-    """Stream chat completion using Claude Agent SDK."""
+    """Stream chat completion using Claude Agent SDK.
+
+    Args:
+        file_ids: Optional workspace file IDs attached to the user's message.
+            Images are embedded as vision content blocks; other files are
+            saved to the SDK working directory for the Read tool.
+    """
 
     if session is None:
         session = await get_chat_session(session_id, user_id)
@@ -854,19 +984,48 @@ def _on_stop(transcript_path: str, sdk_session_id: str) -> None:
                 transcript_msg_count,
                 session_id,
             )
+            # If files are attached, prepare them: images become vision
+            # content blocks in the user message, other files go to sdk_cwd.
+            attachments = await _prepare_file_attachments(
+                file_ids or [], user_id or "", session_id, sdk_cwd
+            )
+            if attachments.hint:
+                query_message = f"{query_message}\n\n{attachments.hint}"
+
             logger.info(
-                "[SDK] [%s] Sending query — resume=%s, total_msgs=%d, query_len=%d",
+                "[SDK] [%s] Sending query — resume=%s, total_msgs=%d, "
+                "query_len=%d, attached_files=%d, image_blocks=%d",
                 session_id[:12],
                 use_resume,
                 len(session.messages),
                 len(query_message),
+                len(file_ids) if file_ids else 0,
+                len(attachments.image_blocks),
             )
 
             compaction.reset_for_query()
             if was_compacted:
                 for ev in compaction.emit_pre_query(session):
                     yield ev
-            await client.query(query_message, session_id=session_id)
+
+            if attachments.image_blocks:
+                # Build multimodal content: image blocks + text
+                content_blocks: list[dict[str, Any]] = [
+                    *attachments.image_blocks,
+                    {"type": "text", "text": query_message},
+                ]
+                user_msg = {
+                    "type": "user",
+                    "message": {"role": "user", "content": content_blocks},
+                    "parent_tool_use_id": None,
+                    "session_id": session_id,
+                }
+                assert client._transport is not None  # noqa: SLF001
+                await client._transport.write(  # noqa: SLF001
+                    json.dumps(user_msg) + "\n"
+                )
+            else:
+                await client.query(query_message, session_id=session_id)
 
             assistant_response = ChatMessage(role="assistant", content="")
             accumulated_tool_calls: list[dict[str, Any]] = []

autogpt_platform/backend/backend/copilot/sdk/service_test.py

@@ -0,0 +1,147 @@
+"""Tests for SDK service helpers."""
+
+import base64
+import os
+from dataclasses import dataclass
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from .service import _prepare_file_attachments
+
+
+@dataclass
+class _FakeFileInfo:
+    id: str
+    name: str
+    path: str
+    mime_type: str
+    size_bytes: int
+
+
+_PATCH_TARGET = "backend.copilot.sdk.service.get_manager"
+
+
+class TestPrepareFileAttachments:
+    @pytest.mark.asyncio
+    async def test_empty_list_returns_empty(self, tmp_path):
+        result = await _prepare_file_attachments([], "u", "s", str(tmp_path))
+        assert result.hint == ""
+        assert result.image_blocks == []
+
+    @pytest.mark.asyncio
+    async def test_image_embedded_as_vision_block(self, tmp_path):
+        """JPEG images should become vision content blocks, not files on disk."""
+        raw = b"\xff\xd8\xff\xe0fake-jpeg"
+        info = _FakeFileInfo(
+            id="abc",
+            name="photo.jpg",
+            path="/photo.jpg",
+            mime_type="image/jpeg",
+            size_bytes=len(raw),
+        )
+        mgr = AsyncMock()
+        mgr.get_file_info.return_value = info
+        mgr.read_file_by_id.return_value = raw
+
+        with patch(_PATCH_TARGET, new_callable=AsyncMock, return_value=mgr):
+            result = await _prepare_file_attachments(
+                ["abc"], "user1", "sess1", str(tmp_path)
+            )
+
+        assert "1 file" in result.hint
+        assert "photo.jpg" in result.hint
+        assert "embedded as image" in result.hint
+        assert len(result.image_blocks) == 1
+        block = result.image_blocks[0]
+        assert block["type"] == "image"
+        assert block["source"]["media_type"] == "image/jpeg"
+        assert block["source"]["data"] == base64.b64encode(raw).decode("ascii")
+        # Image should NOT be written to disk (embedded instead)
+        assert not os.path.exists(os.path.join(tmp_path, "photo.jpg"))
+
+    @pytest.mark.asyncio
+    async def test_pdf_saved_to_disk(self, tmp_path):
+        """PDFs should be saved to disk for Read tool access, not embedded."""
+        info = _FakeFileInfo("f1", "doc.pdf", "/doc.pdf", "application/pdf", 50)
+        mgr = AsyncMock()
+        mgr.get_file_info.return_value = info
+        mgr.read_file_by_id.return_value = b"%PDF-1.4 fake"
+
+        with patch(_PATCH_TARGET, new_callable=AsyncMock, return_value=mgr):
+            result = await _prepare_file_attachments(["f1"], "u", "s", str(tmp_path))
+
+        assert result.image_blocks == []
+        saved = tmp_path / "doc.pdf"
+        assert saved.exists()
+        assert saved.read_bytes() == b"%PDF-1.4 fake"
+        assert str(saved) in result.hint
+
+    @pytest.mark.asyncio
+    async def test_mixed_images_and_files(self, tmp_path):
+        """Images become blocks, non-images go to disk."""
+        infos = {
+            "id1": _FakeFileInfo("id1", "a.png", "/a.png", "image/png", 4),
+            "id2": _FakeFileInfo("id2", "b.pdf", "/b.pdf", "application/pdf", 4),
+            "id3": _FakeFileInfo("id3", "c.txt", "/c.txt", "text/plain", 4),
+        }
+        mgr = AsyncMock()
+        mgr.get_file_info.side_effect = lambda fid: infos[fid]
+        mgr.read_file_by_id.return_value = b"data"
+
+        with patch(_PATCH_TARGET, new_callable=AsyncMock, return_value=mgr):
+            result = await _prepare_file_attachments(
+                ["id1", "id2", "id3"], "u", "s", str(tmp_path)
+            )
+
+        assert "3 files" in result.hint
+        assert "a.png" in result.hint
+        assert "b.pdf" in result.hint
+        assert "c.txt" in result.hint
+        # Only the image should be a vision block
+        assert len(result.image_blocks) == 1
+        assert result.image_blocks[0]["source"]["media_type"] == "image/png"
+        # Non-image files should be on disk
+        assert (tmp_path / "b.pdf").exists()
+        assert (tmp_path / "c.txt").exists()
+        # Read tool hint should appear (has non-image files)
+        assert "Read tool" in result.hint
+
+    @pytest.mark.asyncio
+    async def test_singular_noun(self, tmp_path):
+        info = _FakeFileInfo("x", "only.txt", "/only.txt", "text/plain", 2)
+        mgr = AsyncMock()
+        mgr.get_file_info.return_value = info
+        mgr.read_file_by_id.return_value = b"hi"
+
+        with patch(_PATCH_TARGET, new_callable=AsyncMock, return_value=mgr):
+            result = await _prepare_file_attachments(["x"], "u", "s", str(tmp_path))
+
+        assert "1 file." in result.hint
+
+    @pytest.mark.asyncio
+    async def test_missing_file_skipped(self, tmp_path):
+        mgr = AsyncMock()
+        mgr.get_file_info.return_value = None
+
+        with patch(_PATCH_TARGET, new_callable=AsyncMock, return_value=mgr):
+            result = await _prepare_file_attachments(
+                ["missing-id"], "u", "s", str(tmp_path)
+            )
+
+        assert result.hint == ""
+        assert result.image_blocks == []
+
+    @pytest.mark.asyncio
+    async def test_image_only_no_read_hint(self, tmp_path):
+        """When all files are images, no Read tool hint should appear."""
+        info = _FakeFileInfo("i1", "cat.png", "/cat.png", "image/png", 4)
+        mgr = AsyncMock()
+        mgr.get_file_info.return_value = info
+        mgr.read_file_by_id.return_value = b"data"
+
+        with patch(_PATCH_TARGET, new_callable=AsyncMock, return_value=mgr):
+            result = await _prepare_file_attachments(["i1"], "u", "s", str(tmp_path))
+
+        assert "Read tool" not in result.hint
+        assert len(result.image_blocks) == 1