fix(backend): add circuit breaker for infinite tool call retry loops

autogpt_platform/backend/backend/copilot/prompting.py

@@ -63,6 +63,50 @@
 }}
 ```
 
+### Writing large files — CRITICAL
+**Never write an entire large document in a single tool call.**  When the
+content you want to write exceeds ~2000 words the tool call's output token
+limit will silently truncate the arguments, producing an empty `{{}}` input
+that fails repeatedly.
+
+**Preferred: compose from file references.**  If the data is already in
+files (tool outputs, workspace files), compose the report in one call
+using `@@agptfile:` references — the system expands them inline:
+
+```bash
+cat > report.md << 'EOF'
+# Research Report
+## Data from web research
+@@agptfile:/home/user/web_results.txt
+## Block execution output
+@@agptfile:workspace://<file_id>
+## Conclusion
+<brief synthesis>
+EOF
+```
+
+**Fallback: write section-by-section.**  When you must generate content
+from conversation context (no files to reference), split into multiple
+`bash_exec` calls — one section per call:
+
+```bash
+cat > report.md << 'EOF'
+# Section 1
+<content from your earlier tool call results>
+EOF
+```
+```bash
+cat >> report.md << 'EOF'
+# Section 2
+<content from your earlier tool call results>
+EOF
+```
+Use `cat >` for the first chunk and `cat >>` to append subsequent chunks.
+Do not re-fetch or re-generate data you already have from prior tool calls.
+
+After building the file, reference it with `@@agptfile:` in other tools:
+`@@agptfile:/home/user/report.md`
+
 ### Sub-agent tasks
 - When using the Task tool, NEVER set `run_in_background` to true.
   All tasks must run in the foreground.

autogpt_platform/backend/backend/copilot/sdk/service.py

@@ -80,6 +80,7 @@
     create_copilot_mcp_server,
     get_copilot_tool_names,
     get_sdk_disallowed_tools,
+    reset_tool_failure_counters,
     set_execution_context,
     wait_for_stash,
 )
@@ -105,6 +106,20 @@
 # Non-context errors (network, auth, rate-limit) are NOT retried.
 _MAX_STREAM_ATTEMPTS = 3
 
+# Hard circuit breaker: abort the stream if the model sends this many
+# consecutive tool calls with empty parameters (a sign of context
+# saturation or serialization failure).  Empty input ({}) is never
+# legitimate — even one is suspicious, three is conclusive.
+_EMPTY_TOOL_CALL_LIMIT = 3
+
+# User-facing error shown when the empty-tool-call circuit breaker trips.
+_CIRCUIT_BREAKER_ERROR_MSG = (
+    "AutoPilot was unable to complete the tool call "
+    "— this usually happens when the response is "
+    "too large to fit in a single tool call. "
+    "Try breaking your request into smaller parts."
+)
+
 # Patterns that indicate the prompt/request exceeds the model's context limit.
 # Matched case-insensitively against the full exception chain.
 _PROMPT_TOO_LONG_PATTERNS: tuple[str, ...] = (
@@ -996,14 +1011,121 @@ def _dispatch_response(
     return response
 
 
-class _TransientErrorHandled(Exception):
+class _HandledStreamError(Exception):
     """Raised by `_run_stream_attempt` after it has already yielded a
-    `StreamError` for a transient API error.
+    `StreamError` to the client (e.g. transient API error, circuit breaker).
 
     This signals the outer retry loop that the attempt failed so it can
     perform session-message rollback and set the `ended_with_stream_error`
     flag, **without** yielding a duplicate `StreamError` to the client.
+
+    Attributes:
+        error_msg: The user-facing error message to persist.
+        code: Machine-readable error code (e.g. ``circuit_breaker_empty_tool_calls``).
+        retryable: Whether the frontend should offer a retry button.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        error_msg: str | None = None,
+        code: str | None = None,
+        retryable: bool = True,
+    ):
+        super().__init__(message)
+        self.error_msg = error_msg
+        self.code = code
+        self.retryable = retryable
+
+
+@dataclass
+class _EmptyToolBreakResult:
+    """Result of checking for empty tool calls in a single AssistantMessage."""
+
+    count: int  # Updated consecutive counter
+    tripped: bool  # Whether the circuit breaker fired
+    error: StreamError | None  # StreamError to yield (if tripped)
+    error_msg: str | None  # Error message (if tripped)
+    error_code: str | None  # Error code (if tripped)
+
+
+def _check_empty_tool_breaker(
+    sdk_msg: object,
+    consecutive: int,
+    ctx: _StreamContext,
+    state: _RetryState,
+) -> _EmptyToolBreakResult:
+    """Detect consecutive empty tool calls and trip the circuit breaker.
+
+    Returns an ``_EmptyToolBreakResult`` with the updated counter and, if the
+    breaker tripped, the ``StreamError`` to yield plus the error metadata.
     """
+    if not isinstance(sdk_msg, AssistantMessage):
+        return _EmptyToolBreakResult(consecutive, False, None, None, None)
+
+    empty_tools = [
+        b.name for b in sdk_msg.content if isinstance(b, ToolUseBlock) and not b.input
+    ]
+    if not empty_tools:
+        # Reset on any non-empty-tool AssistantMessage (including text-only
+        # messages — any() over empty content is False).
+        return _EmptyToolBreakResult(0, False, None, None, None)
+
+    consecutive += 1
+
+    # Log full diagnostics on first occurrence only; subsequent hits just
+    # log the counter to reduce noise.
+    if consecutive == 1:
+        logger.warning(
+            "%s Empty tool call detected (%d/%d): "
+            "tools=%s, model=%s, error=%s, "
+            "block_types=%s, cumulative_usage=%s",
+            ctx.log_prefix,
+            consecutive,
+            _EMPTY_TOOL_CALL_LIMIT,
+            empty_tools,
+            sdk_msg.model,
+            sdk_msg.error,
+            [type(b).__name__ for b in sdk_msg.content],
+            {
+                "prompt": state.usage.prompt_tokens,
+                "completion": state.usage.completion_tokens,
+                "cache_read": state.usage.cache_read_tokens,
+            },
+        )
+    else:
+        logger.warning(
+            "%s Empty tool call detected (%d/%d): tools=%s",
+            ctx.log_prefix,
+            consecutive,
+            _EMPTY_TOOL_CALL_LIMIT,
+            empty_tools,
+        )
+
+    if consecutive < _EMPTY_TOOL_CALL_LIMIT:
+        return _EmptyToolBreakResult(consecutive, False, None, None, None)
+
+    logger.error(
+        "%s Circuit breaker: aborting stream after %d "
+        "consecutive empty tool calls. "
+        "This is likely caused by the model attempting "
+        "to write content too large for a single tool "
+        "call's output token limit. The model should "
+        "write large files in chunks using bash_exec "
+        "with cat >> (append).",
+        ctx.log_prefix,
+        consecutive,
+    )
+    error_msg = _CIRCUIT_BREAKER_ERROR_MSG
+    error_code = "circuit_breaker_empty_tool_calls"
+    _append_error_marker(ctx.session, error_msg, retryable=True)
+    return _EmptyToolBreakResult(
+        count=consecutive,
+        tripped=True,
+        error=StreamError(errorText=error_msg, code=error_code),
+        error_msg=error_msg,
+        error_code=error_code,
+    )
 
 
 async def _run_stream_attempt(
@@ -1039,6 +1161,12 @@ async def _run_stream_attempt(
         accumulated_tool_calls=[],
     )
     ended_with_stream_error = False
+    # Stores the error message used by _append_error_marker so the outer
+    # retry loop can re-append the correct message after session rollback.
+    stream_error_msg: str | None = None
+    stream_error_code: str | None = None
+
+    consecutive_empty_tool_calls = 0
 
     async with ClaudeSDKClient(options=state.options) as client:
         logger.info(
@@ -1129,14 +1257,16 @@ async def _run_stream_attempt(
                         "suppressing raw error text",
                         ctx.log_prefix,
                     )
+                    stream_error_msg = FRIENDLY_TRANSIENT_MSG
+                    stream_error_code = "transient_api_error"
                     _append_error_marker(
                         ctx.session,
-                        FRIENDLY_TRANSIENT_MSG,
+                        stream_error_msg,
                         retryable=True,
                     )
                     yield StreamError(
-                        errorText=FRIENDLY_TRANSIENT_MSG,
-                        code="transient_api_error",
+                        errorText=stream_error_msg,
+                        code=stream_error_code,
                     )
                     ended_with_stream_error = True
                     break
@@ -1177,13 +1307,17 @@ async def _run_stream_attempt(
             if isinstance(sdk_msg, ResultMessage):
                 logger.info(
                     "%s Received: ResultMessage %s "
-                    "(unresolved=%d, current=%d, resolved=%d)",
+                    "(unresolved=%d, current=%d, resolved=%d, "
+                    "num_turns=%d, cost_usd=%s, result=%s)",
                     ctx.log_prefix,
                     sdk_msg.subtype,
                     len(state.adapter.current_tool_calls)
                     - len(state.adapter.resolved_tool_calls),
                     len(state.adapter.current_tool_calls),
                     len(state.adapter.resolved_tool_calls),
+                    sdk_msg.num_turns,
+                    sdk_msg.total_cost_usd,
+                    (sdk_msg.result or "")[:200],
                 )
                 if sdk_msg.subtype in (
                     "error",
@@ -1240,6 +1374,18 @@ async def _run_stream_attempt(
                     )
                     entries_replaced = True
 
+            # --- Hard circuit breaker for empty tool calls ---
+            breaker = _check_empty_tool_breaker(
+                sdk_msg, consecutive_empty_tool_calls, ctx, state
+            )
+            consecutive_empty_tool_calls = breaker.count
+            if breaker.tripped and breaker.error is not None:
+                stream_error_msg = breaker.error_msg
+                stream_error_code = breaker.error_code
+                yield breaker.error
+                ended_with_stream_error = True
+                break
+
             # --- Dispatch adapter responses ---
             for response in state.adapter.convert_message(sdk_msg):
                 dispatched = _dispatch_response(
@@ -1320,8 +1466,10 @@ async def _run_stream_attempt(
     # to the client (StreamError yielded above), raise so the outer retry
     # loop can rollback session messages and set its error flags properly.
     if ended_with_stream_error:
-        raise _TransientErrorHandled(
-            "Transient API error handled — StreamError already yielded"
+        raise _HandledStreamError(
+            "Stream error handled — StreamError already yielded",
+            error_msg=stream_error_msg,
+            code=stream_error_code,
         )
 
 
@@ -1714,6 +1862,10 @@ def _on_stderr(line: str) -> None:
         )
 
         for attempt in range(_MAX_STREAM_ATTEMPTS):
+            # Reset tool-level circuit breaker so failures from a previous
+            # (rolled-back) attempt don't carry over to the fresh attempt.
+            reset_tool_failure_counters()
+
             if attempt > 0:
                 logger.info(
                     "%s Retrying with reduced context (%d/%d)",
@@ -1778,24 +1930,35 @@ def _on_stderr(line: str) -> None:
                     _MAX_STREAM_ATTEMPTS,
                 )
                 raise
-            except _TransientErrorHandled:
+            except _HandledStreamError as exc:
                 # _run_stream_attempt already yielded a StreamError and
                 # appended an error marker.  We only need to rollback
                 # session messages and set the error flag — do NOT set
                 # stream_err so the post-loop code won't emit a
                 # duplicate StreamError.
                 logger.warning(
-                    "%s Transient error handled in stream attempt "
-                    "(attempt %d/%d, events_yielded=%d)",
+                    "%s Stream error handled in attempt "
+                    "(attempt %d/%d, code=%s, events_yielded=%d)",
                     log_prefix,
                     attempt + 1,
                     _MAX_STREAM_ATTEMPTS,
+                    exc.code or "transient",
                     events_yielded,
                 )
                 session.messages = session.messages[:pre_attempt_msg_count]
+                # transcript_builder still contains entries from the aborted
+                # attempt that no longer match session.messages.  Skip upload
+                # so a future --resume doesn't replay rolled-back content.
+                skip_transcript_upload = True
                 # Re-append the error marker so it survives the rollback
                 # and is persisted by the finally block (see #2947655365).
-                _append_error_marker(session, FRIENDLY_TRANSIENT_MSG, retryable=True)
+                # Use the specific error message from the attempt (e.g.
+                # circuit breaker msg) rather than always the generic one.
+                _append_error_marker(
+                    session,
+                    exc.error_msg or FRIENDLY_TRANSIENT_MSG,
+                    retryable=True,
+                )
                 ended_with_stream_error = True
                 break
             except Exception as e:
@@ -1822,11 +1985,13 @@ def _on_stderr(line: str) -> None:
                         log_prefix,
                         events_yielded,
                     )
+                    skip_transcript_upload = True
                     ended_with_stream_error = True
                     break
                 if not is_context_error:
                     # Non-context errors (network, auth, rate-limit) should
                     # not trigger compaction — surface the error immediately.
+                    skip_transcript_upload = True
                     ended_with_stream_error = True
                     break
                 continue