From f6c7d1eaf7a867205da45e3015c37c1fce4d518d Mon Sep 17 00:00:00 2001 From: majdyz Date: Mon, 13 Apr 2026 09:39:44 +0000 Subject: [PATCH 01/14] fix(copilot): baseline cost tracking fallback and dashboard cache token display When OpenRouter's x-total-cost header is missing, estimate cost from token counts using a known model pricing table so cost is always logged. Also extract cache token details from streaming usage chunks (prompt_tokens_details.cached_tokens) and pass them through to PlatformCostLog. On the dashboard side, add cache read/write columns to the logs table and user table, and include cache tokens in the UserCostSummary backend model so they surface in the API response. --- .../backend/copilot/baseline/service.py | 100 +++++++++++++--- .../copilot/baseline/service_unit_test.py | 110 ++++++++++++++++++ .../backend/backend/data/platform_cost.py | 71 ++++++----- .../platform-costs/components/LogsTable.tsx | 12 +- .../platform-costs/components/UserTable.tsx | 18 ++- 5 files changed, 261 insertions(+), 50 deletions(-) diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index 1f1fe42f59ef..15d173cedc71 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -102,6 +102,38 @@ # MIME types that can be embedded as vision content blocks (OpenAI format). _VISION_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/gif", "image/webp"}) +# Fallback per-token pricing (USD) for common OpenRouter models. +# Used only when the ``x-total-cost`` response header is missing. +# Rates sourced from OpenRouter's pricing page (input / output per 1M tokens). +_OPENROUTER_MODEL_PRICING: dict[str, tuple[float, float]] = { + "anthropic/claude-sonnet-4": (3.0 / 1_000_000, 15.0 / 1_000_000), + "anthropic/claude-3.5-sonnet": (3.0 / 1_000_000, 15.0 / 1_000_000), + "anthropic/claude-3-haiku": (0.25 / 1_000_000, 1.25 / 1_000_000), + "anthropic/claude-3-opus": (15.0 / 1_000_000, 75.0 / 1_000_000), + "openai/gpt-4o": (2.5 / 1_000_000, 10.0 / 1_000_000), + "openai/gpt-4o-mini": (0.15 / 1_000_000, 0.6 / 1_000_000), + "openai/gpt-4-turbo": (10.0 / 1_000_000, 30.0 / 1_000_000), + "google/gemini-2.0-flash-001": (0.1 / 1_000_000, 0.4 / 1_000_000), + "google/gemini-2.5-pro-preview": (1.25 / 1_000_000, 10.0 / 1_000_000), +} + + +def _estimate_cost_from_tokens( + model: str, + prompt_tokens: int, + completion_tokens: int, +) -> float | None: + """Estimate USD cost from token counts using known model pricing. + + Returns None if the model is not in the pricing table. + """ + pricing = _OPENROUTER_MODEL_PRICING.get(model) + if pricing is None: + return None + input_rate, output_rate = pricing + return prompt_tokens * input_rate + completion_tokens * output_rate + + # Max size for embedding images directly in the user message (20 MiB raw). _MAX_INLINE_IMAGE_BYTES = 20 * 1024 * 1024 @@ -338,6 +370,8 @@ class _BaselineStreamState: text_started: bool = False turn_prompt_tokens: int = 0 turn_completion_tokens: int = 0 + turn_cache_read_tokens: int = 0 + turn_cache_creation_tokens: int = 0 cost_usd: float | None = None thinking_stripper: _ThinkingStripper = field(default_factory=_ThinkingStripper) session_messages: list[ChatMessage] = field(default_factory=list) @@ -385,6 +419,13 @@ async def _baseline_llm_caller( if chunk.usage: state.turn_prompt_tokens += chunk.usage.prompt_tokens or 0 state.turn_completion_tokens += chunk.usage.completion_tokens or 0 + # Extract cache token details when available (OpenAI / + # OpenRouter include these in prompt_tokens_details). + ptd = getattr(chunk.usage, "prompt_tokens_details", None) + if ptd: + state.turn_cache_read_tokens += ( + getattr(ptd, "cached_tokens", 0) or 0 + ) delta = chunk.choices[0].delta if chunk.choices else None if not delta: @@ -439,6 +480,7 @@ async def _baseline_llm_caller( # Extract OpenRouter cost from response headers (in finally so we # capture cost even when the stream errors mid-way — we already paid). # Accumulate across multi-round tool-calling turns. + got_header_cost = False try: # Access undocumented _response attribute — same pattern as # extract_openrouter_cost() in blocks/llm.py. @@ -447,9 +489,29 @@ async def _baseline_llm_caller( cost = float(cost_header) if math.isfinite(cost) and cost >= 0: state.cost_usd = (state.cost_usd or 0.0) + cost + got_header_cost = True except (AttributeError, ValueError): pass + # Fallback: estimate cost from token counts when x-total-cost is + # missing (e.g. some OpenRouter models don't report it). + if not got_header_cost and ( + state.turn_prompt_tokens > 0 or state.turn_completion_tokens > 0 + ): + estimated = _estimate_cost_from_tokens( + state.model, + state.turn_prompt_tokens, + state.turn_completion_tokens, + ) + if estimated is not None: + state.cost_usd = (state.cost_usd or 0.0) + estimated + logger.info( + "[Baseline] x-total-cost header missing; estimated cost " + "from token pricing: $%.6f (model=%s)", + estimated, + state.model, + ) + # Always persist partial text so the session history stays consistent, # even when the stream is interrupted by an exception. state.assistant_text += round_text @@ -972,16 +1034,17 @@ async def stream_chat_completion_baseline( # Run download + prompt build concurrently — both are independent I/O # on the request critical path. if user_id and len(session.messages) > 1: - transcript_covers_prefix, (base_system_prompt, understanding) = ( - await asyncio.gather( - _load_prior_transcript( - user_id=user_id, - session_id=session_id, - session_msg_count=len(session.messages), - transcript_builder=transcript_builder, - ), - prompt_task, - ) + ( + transcript_covers_prefix, + (base_system_prompt, understanding), + ) = await asyncio.gather( + _load_prior_transcript( + user_id=user_id, + session_id=session_id, + session_msg_count=len(session.messages), + transcript_builder=transcript_builder, + ), + prompt_task, ) else: base_system_prompt, understanding = await prompt_task @@ -1107,7 +1170,7 @@ async def stream_chat_completion_baseline( content_text = context.get("content", "") if content_text: context_hint = ( - f"\n[The user shared a URL: {url}\n" f"Content:\n{content_text[:8000]}]" + f"\n[The user shared a URL: {url}\nContent:\n{content_text[:8000]}]" ) else: context_hint = f"\n[The user shared a URL: {url}]" @@ -1291,14 +1354,21 @@ async def stream_chat_completion_baseline( ) # Persist token usage to session and record for rate limiting. - # NOTE: OpenRouter folds cached tokens into prompt_tokens, so we - # cannot break out cache_read/cache_creation weights. Users on the - # baseline path may be slightly over-counted vs the SDK path. + # When prompt_tokens_details.cached_tokens is reported, subtract + # them from prompt_tokens to get the uncached count so the cost + # breakdown stays accurate. + uncached_prompt = state.turn_prompt_tokens + if state.turn_cache_read_tokens > 0: + uncached_prompt = max( + 0, state.turn_prompt_tokens - state.turn_cache_read_tokens + ) await persist_and_record_usage( session=session, user_id=user_id, - prompt_tokens=state.turn_prompt_tokens, + prompt_tokens=uncached_prompt, completion_tokens=state.turn_completion_tokens, + cache_read_tokens=state.turn_cache_read_tokens, + cache_creation_tokens=state.turn_cache_creation_tokens, log_prefix="[Baseline]", cost_usd=state.cost_usd, model=active_model, diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py index ba1374b7208d..cc2d58ca826e 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py @@ -13,6 +13,7 @@ _baseline_conversation_updater, _BaselineStreamState, _compress_session_messages, + _estimate_cost_from_tokens, _ThinkingStripper, ) from backend.copilot.model import ChatMessage @@ -633,6 +634,23 @@ async def test_workspace_manager_error(self): assert blocks == [] +class TestEstimateCostFromTokens: + """Tests for _estimate_cost_from_tokens fallback pricing.""" + + def test_known_model_returns_estimated_cost(self): + cost = _estimate_cost_from_tokens("anthropic/claude-sonnet-4", 1000, 500) + # 1000 * 3.0/1M + 500 * 15.0/1M = 0.003 + 0.0075 = 0.0105 + assert cost == pytest.approx(0.0105) + + def test_unknown_model_returns_none(self): + cost = _estimate_cost_from_tokens("unknown/model", 1000, 500) + assert cost is None + + def test_zero_tokens_returns_zero(self): + cost = _estimate_cost_from_tokens("openai/gpt-4o", 0, 0) + assert cost == pytest.approx(0.0) + + class TestBaselineCostExtraction: """Tests for x-total-cost header extraction in _baseline_llm_caller.""" @@ -828,3 +846,95 @@ async def test_no_cost_when_api_call_raises_before_stream(self): # response was never assigned so cost extraction must not raise assert state.cost_usd is None + + @pytest.mark.asyncio + async def test_cost_estimated_from_tokens_when_header_missing(self): + """cost_usd is estimated from token counts when x-total-cost is absent.""" + from backend.copilot.baseline.service import ( + _baseline_llm_caller, + _BaselineStreamState, + ) + + # Use a model that is in the pricing table + state = _BaselineStreamState(model="anthropic/claude-sonnet-4") + + mock_raw = MagicMock() + mock_raw.headers = {} # no x-total-cost + mock_stream = MagicMock() + mock_stream._response = mock_raw + + # Create a chunk that reports usage tokens + mock_chunk = MagicMock() + mock_chunk.usage = MagicMock() + mock_chunk.usage.prompt_tokens = 1000 + mock_chunk.usage.completion_tokens = 500 + mock_chunk.usage.prompt_tokens_details = None + mock_chunk.choices = [] + + async def chunk_aiter(): + yield mock_chunk + + mock_stream.__aiter__ = lambda self: chunk_aiter() + + mock_client = MagicMock() + mock_client.chat.completions.create = AsyncMock(return_value=mock_stream) + + with patch( + "backend.copilot.baseline.service._get_openai_client", + return_value=mock_client, + ): + await _baseline_llm_caller( + messages=[{"role": "user", "content": "hi"}], + tools=[], + state=state, + ) + + # Expected: 1000 * 3.0/1M + 500 * 15.0/1M = 0.003 + 0.0075 = 0.0105 + assert state.cost_usd == pytest.approx(0.0105) + + @pytest.mark.asyncio + async def test_cache_tokens_extracted_from_usage_details(self): + """cache tokens are extracted from prompt_tokens_details.cached_tokens.""" + from backend.copilot.baseline.service import ( + _baseline_llm_caller, + _BaselineStreamState, + ) + + state = _BaselineStreamState(model="openai/gpt-4o") + + mock_raw = MagicMock() + mock_raw.headers = {"x-total-cost": "0.01"} + mock_stream = MagicMock() + mock_stream._response = mock_raw + + # Create a chunk with prompt_tokens_details + mock_ptd = MagicMock() + mock_ptd.cached_tokens = 800 + + mock_chunk = MagicMock() + mock_chunk.usage = MagicMock() + mock_chunk.usage.prompt_tokens = 1000 + mock_chunk.usage.completion_tokens = 200 + mock_chunk.usage.prompt_tokens_details = mock_ptd + mock_chunk.choices = [] + + async def chunk_aiter(): + yield mock_chunk + + mock_stream.__aiter__ = lambda self: chunk_aiter() + + mock_client = MagicMock() + mock_client.chat.completions.create = AsyncMock(return_value=mock_stream) + + with patch( + "backend.copilot.baseline.service._get_openai_client", + return_value=mock_client, + ): + await _baseline_llm_caller( + messages=[{"role": "user", "content": "hi"}], + tools=[], + state=state, + ) + + assert state.turn_cache_read_tokens == 800 + assert state.turn_prompt_tokens == 1000 diff --git a/autogpt_platform/backend/backend/data/platform_cost.py b/autogpt_platform/backend/backend/data/platform_cost.py index 17915e115cdc..ec275720583e 100644 --- a/autogpt_platform/backend/backend/data/platform_cost.py +++ b/autogpt_platform/backend/backend/data/platform_cost.py @@ -139,6 +139,8 @@ class UserCostSummary(BaseModel): total_cost_microdollars: int total_input_tokens: int total_output_tokens: int + total_cache_read_tokens: int = 0 + total_cache_creation_tokens: int = 0 request_count: int @@ -265,38 +267,41 @@ async def get_platform_cost_dashboard( } # Run all four aggregation queries in parallel. - by_provider_groups, by_user_groups, total_user_groups, total_agg_groups = ( - await asyncio.gather( - # (provider, trackingType, model) aggregation — no ORDER BY in ORM; - # sort by total cost descending in Python after fetch. - PrismaLog.prisma().group_by( - by=["provider", "trackingType", "model"], - where=where, - sum=sum_fields, - count=True, - ), - # userId aggregation — emails fetched separately below. - PrismaLog.prisma().group_by( - by=["userId"], - where=where, - sum=sum_fields, - count=True, - ), - # Distinct user count: group by userId, count groups. - PrismaLog.prisma().group_by( - by=["userId"], - where=where, - count=True, - ), - # Total aggregate: group by provider (no limit) to sum across all - # matching rows. Summed in Python to get grand totals. - PrismaLog.prisma().group_by( - by=["provider"], - where=where, - sum={"costMicrodollars": True}, - count=True, - ), - ) + ( + by_provider_groups, + by_user_groups, + total_user_groups, + total_agg_groups, + ) = await asyncio.gather( + # (provider, trackingType, model) aggregation — no ORDER BY in ORM; + # sort by total cost descending in Python after fetch. + PrismaLog.prisma().group_by( + by=["provider", "trackingType", "model"], + where=where, + sum=sum_fields, + count=True, + ), + # userId aggregation — emails fetched separately below. + PrismaLog.prisma().group_by( + by=["userId"], + where=where, + sum=sum_fields, + count=True, + ), + # Distinct user count: group by userId, count groups. + PrismaLog.prisma().group_by( + by=["userId"], + where=where, + count=True, + ), + # Total aggregate: group by provider (no limit) to sum across all + # matching rows. Summed in Python to get grand totals. + PrismaLog.prisma().group_by( + by=["provider"], + where=where, + sum={"costMicrodollars": True}, + count=True, + ), ) # Sort by_provider by total cost descending and cap at MAX_PROVIDER_ROWS. @@ -347,6 +352,8 @@ async def get_platform_cost_dashboard( total_cost_microdollars=_si(r, "costMicrodollars"), total_input_tokens=_si(r, "inputTokens"), total_output_tokens=_si(r, "outputTokens"), + total_cache_read_tokens=_si(r, "cacheReadTokens"), + total_cache_creation_tokens=_si(r, "cacheCreationTokens"), request_count=_ca(r), ) for r in by_user_groups diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/LogsTable.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/LogsTable.tsx index 46920d15bca7..056eef06b8ae 100644 --- a/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/LogsTable.tsx +++ b/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/LogsTable.tsx @@ -67,7 +67,10 @@ function LogsTable({ Cost - Tokens + In / Out + + + Cache (R/W) Duration @@ -105,6 +108,11 @@ function LogsTable({ ? `${formatTokens(Number(log.input_tokens ?? 0))} / ${formatTokens(Number(log.output_tokens ?? 0))}` : "-"} + + {log.cache_read_tokens || log.cache_creation_tokens + ? `${formatTokens(Number(log.cache_read_tokens ?? 0))} / ${formatTokens(Number(log.cache_creation_tokens ?? 0))}` + : "-"} + {log.duration != null ? formatDuration(Number(log.duration)) @@ -120,7 +128,7 @@ function LogsTable({ {logs.length === 0 && ( No logs found diff --git a/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/UserTable.tsx b/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/UserTable.tsx index 7c08f85e1bbc..c2ee70ce7246 100644 --- a/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/UserTable.tsx +++ b/autogpt_platform/frontend/src/app/(platform)/admin/platform-costs/components/UserTable.tsx @@ -26,6 +26,12 @@ function UserTable({ data }: Props) { Output Tokens + + Cache Read + + + Cache Write + @@ -54,12 +60,22 @@ function UserTable({ data }: Props) { {formatTokens(row.total_output_tokens)} + + {(row.total_cache_read_tokens ?? 0) > 0 + ? formatTokens(row.total_cache_read_tokens ?? 0) + : "-"} + + + {(row.total_cache_creation_tokens ?? 0) > 0 + ? formatTokens(row.total_cache_creation_tokens ?? 0) + : "-"} + ))} {data.length === 0 && ( No cost data yet From c6af52033dc97f673af7a968564d14fbb2949707 Mon Sep 17 00:00:00 2001 From: majdyz Date: Mon, 13 Apr 2026 09:53:05 +0000 Subject: [PATCH 02/14] fix(copilot): fix multi-turn cost over-estimation and add cache_creation_tokens extraction Bug 1: Fallback cost estimation was using accumulated turn_prompt_tokens / turn_completion_tokens across all tool-call rounds, causing compounding over-estimation on the 2nd+ turn. Snapshot token counts before each call and pass only the per-call delta to _estimate_cost_from_tokens. Bug 2: turn_cache_creation_tokens was defined but never populated. Extract cache_creation_input_tokens from prompt_tokens_details (available from some providers such as Anthropic via OpenRouter). Add regression tests for both fixes. --- .../backend/copilot/baseline/service.py | 23 +++- .../copilot/baseline/service_unit_test.py | 112 ++++++++++++++++++ 2 files changed, 132 insertions(+), 3 deletions(-) diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index 15d173cedc71..11d247cd0387 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -415,6 +415,13 @@ async def _baseline_llm_caller( ) tool_calls_by_index: dict[int, dict[str, str]] = {} + # Snapshot token counts before this call so we can compute the delta + # used for fallback cost estimation (state fields are accumulated across + # all tool-call turns, so we must not pass the cumulative total to the + # per-call cost estimator). + prompt_tokens_before = state.turn_prompt_tokens + completion_tokens_before = state.turn_completion_tokens + async for chunk in response: if chunk.usage: state.turn_prompt_tokens += chunk.usage.prompt_tokens or 0 @@ -426,6 +433,11 @@ async def _baseline_llm_caller( state.turn_cache_read_tokens += ( getattr(ptd, "cached_tokens", 0) or 0 ) + # cache_creation_input_tokens is reported by some providers + # (e.g. Anthropic native) but not standard OpenAI streaming. + state.turn_cache_creation_tokens += ( + getattr(ptd, "cache_creation_input_tokens", 0) or 0 + ) delta = chunk.choices[0].delta if chunk.choices else None if not delta: @@ -495,13 +507,18 @@ async def _baseline_llm_caller( # Fallback: estimate cost from token counts when x-total-cost is # missing (e.g. some OpenRouter models don't report it). + # Use the delta for this call only — the state accumulators grow across + # all tool-call turns, so passing the cumulative total would + # compound-overestimate costs on the 2nd+ turn. + call_prompt_tokens = state.turn_prompt_tokens - prompt_tokens_before + call_completion_tokens = state.turn_completion_tokens - completion_tokens_before if not got_header_cost and ( - state.turn_prompt_tokens > 0 or state.turn_completion_tokens > 0 + call_prompt_tokens > 0 or call_completion_tokens > 0 ): estimated = _estimate_cost_from_tokens( state.model, - state.turn_prompt_tokens, - state.turn_completion_tokens, + call_prompt_tokens, + call_completion_tokens, ) if estimated is not None: state.cost_usd = (state.cost_usd or 0.0) + estimated diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py index cc2d58ca826e..2a1506567933 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py @@ -938,3 +938,115 @@ async def chunk_aiter(): assert state.turn_cache_read_tokens == 800 assert state.turn_prompt_tokens == 1000 + + @pytest.mark.asyncio + async def test_cache_creation_tokens_extracted_from_usage_details(self): + """cache_creation_tokens are extracted from prompt_tokens_details.""" + from backend.copilot.baseline.service import ( + _baseline_llm_caller, + _BaselineStreamState, + ) + + state = _BaselineStreamState(model="openai/gpt-4o") + + mock_raw = MagicMock() + mock_raw.headers = {"x-total-cost": "0.01"} + mock_stream = MagicMock() + mock_stream._response = mock_raw + + mock_ptd = MagicMock() + mock_ptd.cached_tokens = 0 + mock_ptd.cache_creation_input_tokens = 500 + + mock_chunk = MagicMock() + mock_chunk.usage = MagicMock() + mock_chunk.usage.prompt_tokens = 1000 + mock_chunk.usage.completion_tokens = 200 + mock_chunk.usage.prompt_tokens_details = mock_ptd + mock_chunk.choices = [] + + async def chunk_aiter(): + yield mock_chunk + + mock_stream.__aiter__ = lambda self: chunk_aiter() + + mock_client = MagicMock() + mock_client.chat.completions.create = AsyncMock(return_value=mock_stream) + + with patch( + "backend.copilot.baseline.service._get_openai_client", + return_value=mock_client, + ): + await _baseline_llm_caller( + messages=[{"role": "user", "content": "hi"}], + tools=[], + state=state, + ) + + assert state.turn_cache_creation_tokens == 500 + + @pytest.mark.asyncio + async def test_multiturn_fallback_cost_uses_per_call_delta(self): + """Fallback cost estimation uses per-call token delta, not session total. + + On the second tool-call turn, the state accumulators already hold + tokens from turn 1. The estimator must charge only for the new tokens + reported in the current call, not the running total. + """ + from backend.copilot.baseline.service import ( + _baseline_llm_caller, + _BaselineStreamState, + ) + + state = _BaselineStreamState(model="anthropic/claude-sonnet-4") + + def make_stream(prompt_tokens: int, completion_tokens: int): + mock_raw = MagicMock() + mock_raw.headers = {} # no x-total-cost + mock_stream = MagicMock() + mock_stream._response = mock_raw + + mock_chunk = MagicMock() + mock_chunk.usage = MagicMock() + mock_chunk.usage.prompt_tokens = prompt_tokens + mock_chunk.usage.completion_tokens = completion_tokens + mock_chunk.usage.prompt_tokens_details = None + mock_chunk.choices = [] + + async def chunk_aiter(): + yield mock_chunk + + mock_stream.__aiter__ = lambda self: chunk_aiter() + return mock_stream + + mock_client = MagicMock() + mock_client.chat.completions.create = AsyncMock( + side_effect=[ + make_stream(1000, 200), # turn 1 + make_stream(1100, 300), # turn 2 (accumulators now hold 1000+1100, 200+300) + ] + ) + + with patch( + "backend.copilot.baseline.service._get_openai_client", + return_value=mock_client, + ): + await _baseline_llm_caller( + messages=[{"role": "user", "content": "hi"}], + tools=[], + state=state, + ) + await _baseline_llm_caller( + messages=[{"role": "user", "content": "follow up"}], + tools=[], + state=state, + ) + + # Turn 1: 1000 * 3.0/1M + 200 * 15.0/1M = 0.003 + 0.003 = 0.006 + # Turn 2: 1100 * 3.0/1M + 300 * 15.0/1M = 0.0033 + 0.0045 = 0.0078 + # Total: 0.0138 — NOT 0.006 + cumulative (2100*3/1M + 500*15/1M = 0.006+0.0138) + expected = pytest.approx(0.006 + 0.0078, rel=1e-5) + assert state.cost_usd == expected + # Accumulators hold all tokens across both turns + assert state.turn_prompt_tokens == 2100 + assert state.turn_completion_tokens == 500 From 69e9a5bb22604041414a5f1073818c8ae53eb2ac Mon Sep 17 00:00:00 2001 From: majdyz Date: Mon, 13 Apr 2026 10:12:44 +0000 Subject: [PATCH 03/14] fix(frontend): add cache token fields to UserCostSummary in openapi.json The backend added total_cache_read_tokens and total_cache_creation_tokens to UserCostSummary but the OpenAPI spec was not updated, causing frontend build failures. --- autogpt_platform/frontend/src/app/api/openapi.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json index 446b2eb0796e..bcd881c4fc96 100644 --- a/autogpt_platform/frontend/src/app/api/openapi.json +++ b/autogpt_platform/frontend/src/app/api/openapi.json @@ -15575,7 +15575,17 @@ "type": "integer", "title": "Total Output Tokens" }, - "request_count": { "type": "integer", "title": "Request Count" } + "request_count": { "type": "integer", "title": "Request Count" }, + "total_cache_read_tokens": { + "default": 0, + "title": "Total Cache Read Tokens", + "type": "integer" + }, + "total_cache_creation_tokens": { + "default": 0, + "title": "Total Cache Creation Tokens", + "type": "integer" + } }, "type": "object", "required": [ From 483f1cfb3ad2144844a4077e5a48527cd5e8ea5c Mon Sep 17 00:00:00 2001 From: majdyz Date: Mon, 13 Apr 2026 21:45:07 +0700 Subject: [PATCH 04/14] fix(backend/copilot): move token snapshot before try to prevent UnboundLocalError When client.chat.completions.create() raises (e.g. network timeout), the finally block referenced prompt_tokens_before/completion_tokens_before which were only assigned after the API call inside the try block, causing an UnboundLocalError that masked the original exception. Move the snapshots to before the try block so the finally block can always reference them safely even when the API call fails. --- .../backend/backend/copilot/baseline/service.py | 12 +++++------- .../backend/copilot/baseline/service_unit_test.py | 4 +++- .../backend/backend/data/platform_cost_test.py | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index 11d247cd0387..725ec357780a 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -394,6 +394,11 @@ async def _baseline_llm_caller( round_text = "" response = None # initialized before try so finally block can access it + # Snapshot token counts before this call so we can compute the delta used + # for fallback cost estimation. Must be set before the try so the finally + # block can always reference them even when the API call raises. + prompt_tokens_before = state.turn_prompt_tokens + completion_tokens_before = state.turn_completion_tokens try: client = _get_openai_client() typed_messages = cast(list[ChatCompletionMessageParam], messages) @@ -415,13 +420,6 @@ async def _baseline_llm_caller( ) tool_calls_by_index: dict[int, dict[str, str]] = {} - # Snapshot token counts before this call so we can compute the delta - # used for fallback cost estimation (state fields are accumulated across - # all tool-call turns, so we must not pass the cumulative total to the - # per-call cost estimator). - prompt_tokens_before = state.turn_prompt_tokens - completion_tokens_before = state.turn_completion_tokens - async for chunk in response: if chunk.usage: state.turn_prompt_tokens += chunk.usage.prompt_tokens or 0 diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py index 2a1506567933..83c95a12a277 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py @@ -1023,7 +1023,9 @@ async def chunk_aiter(): mock_client.chat.completions.create = AsyncMock( side_effect=[ make_stream(1000, 200), # turn 1 - make_stream(1100, 300), # turn 2 (accumulators now hold 1000+1100, 200+300) + make_stream( + 1100, 300 + ), # turn 2 (accumulators now hold 1000+1100, 200+300) ] ) diff --git a/autogpt_platform/backend/backend/data/platform_cost_test.py b/autogpt_platform/backend/backend/data/platform_cost_test.py index dacd2c42ea98..4a2372628b64 100644 --- a/autogpt_platform/backend/backend/data/platform_cost_test.py +++ b/autogpt_platform/backend/backend/data/platform_cost_test.py @@ -35,7 +35,6 @@ def test_large_value(self): assert usd_to_microdollars(1.0) == 1_000_000 - class TestMaskEmail: def test_typical_email(self): assert _mask_email("user@example.com") == "us***@example.com" From 6fbb32ce38abb1c34868e173a0af3d44170b625e Mon Sep 17 00:00:00 2001 From: majdyz Date: Mon, 13 Apr 2026 22:23:09 +0700 Subject: [PATCH 05/14] fix(frontend): fix UserCostSummary field order in openapi.json Reorder cache token fields before request_count to match Python model field declaration order, and fix property key ordering to use type-first format consistent with FastAPI's schema export. --- autogpt_platform/frontend/src/app/api/openapi.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json index bcd881c4fc96..43f14a13fd81 100644 --- a/autogpt_platform/frontend/src/app/api/openapi.json +++ b/autogpt_platform/frontend/src/app/api/openapi.json @@ -15575,17 +15575,17 @@ "type": "integer", "title": "Total Output Tokens" }, - "request_count": { "type": "integer", "title": "Request Count" }, "total_cache_read_tokens": { - "default": 0, + "type": "integer", "title": "Total Cache Read Tokens", - "type": "integer" + "default": 0 }, "total_cache_creation_tokens": { - "default": 0, + "type": "integer", "title": "Total Cache Creation Tokens", - "type": "integer" - } + "default": 0 + }, + "request_count": { "type": "integer", "title": "Request Count" } }, "type": "object", "required": [ From d84417dbc8fdcd1832d74e1d00576a396ea6cffc Mon Sep 17 00:00:00 2001 From: majdyz Date: Mon, 13 Apr 2026 23:06:40 +0700 Subject: [PATCH 06/14] fix(copilot): add claude-opus-4.6 pricing and cache-read discount in fallback cost estimator - Add anthropic/claude-opus-4.6 to _OPENROUTER_MODEL_PRICING so non-fast sessions that miss the x-total-cost header get a cost estimate instead of cost_usd=None. - Add _CACHE_READ_DISCOUNT table (Anthropic: 10%, OpenAI: 50%) and update _estimate_cost_from_tokens to apply it: cache-read tokens are billed at the discounted rate, regular tokens at full price. - Snapshot cache_read_tokens_before alongside prompt/completion snapshots so the per-call delta passed to the estimator is accurate on multi-turn tool-calling sessions. - Fill state.cost_usd from token estimate after tiktoken fallback fires so persist_and_record_usage never receives cost_usd=None when the provider reports no streaming usage. - Add unit tests for new pricing entry, cache-read discounts (Anthropic / OpenAI), and clarify multiturn mock comment. --- .../backend/copilot/baseline/service.py | 57 ++++++++++++++++++- .../copilot/baseline/service_unit_test.py | 39 +++++++++++-- 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index 725ec357780a..33ba887c5124 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -106,6 +106,7 @@ # Used only when the ``x-total-cost`` response header is missing. # Rates sourced from OpenRouter's pricing page (input / output per 1M tokens). _OPENROUTER_MODEL_PRICING: dict[str, tuple[float, float]] = { + "anthropic/claude-opus-4.6": (15.0 / 1_000_000, 75.0 / 1_000_000), "anthropic/claude-sonnet-4": (3.0 / 1_000_000, 15.0 / 1_000_000), "anthropic/claude-3.5-sonnet": (3.0 / 1_000_000, 15.0 / 1_000_000), "anthropic/claude-3-haiku": (0.25 / 1_000_000, 1.25 / 1_000_000), @@ -117,21 +118,54 @@ "google/gemini-2.5-pro-preview": (1.25 / 1_000_000, 10.0 / 1_000_000), } +# Cache-read discount fractions per provider prefix (relative to input rate). +# Anthropic: 10%, OpenAI: 50%, Google: not billed separately so 100% (no discount). +_CACHE_READ_DISCOUNT: dict[str, float] = { + "anthropic/": 0.10, + "openai/": 0.50, +} + def _estimate_cost_from_tokens( model: str, prompt_tokens: int, completion_tokens: int, + cache_read_tokens: int = 0, ) -> float | None: """Estimate USD cost from token counts using known model pricing. + ``prompt_tokens`` should be the *total* prompt token count as reported by + the API (includes both regular and cache-read tokens as a subset). + ``cache_read_tokens`` is used to apply a provider-specific discount: + + * Cache reads are billed at a fraction of the normal input rate + (Anthropic: 10 %, OpenAI: 50 %). The regular (non-cached) portion is + billed at full price. + * Cache writes (creation) are already included in ``prompt_tokens`` at the + full input rate — no adjustment is needed. + Returns None if the model is not in the pricing table. """ pricing = _OPENROUTER_MODEL_PRICING.get(model) if pricing is None: return None input_rate, output_rate = pricing - return prompt_tokens * input_rate + completion_tokens * output_rate + + # Determine the cache-read discount fraction for this provider. + cache_read_fraction = next( + (v for prefix, v in _CACHE_READ_DISCOUNT.items() if model.startswith(prefix)), + 1.0, # no discount for unknown providers + ) + + # Regular (non-cached) input tokens billed at full price; + # cache-read tokens billed at the discounted rate. + regular_prompt = max(0, prompt_tokens - cache_read_tokens) + cost = ( + regular_prompt * input_rate + + cache_read_tokens * input_rate * cache_read_fraction + + completion_tokens * output_rate + ) + return cost # Max size for embedding images directly in the user message (20 MiB raw). @@ -399,6 +433,7 @@ async def _baseline_llm_caller( # block can always reference them even when the API call raises. prompt_tokens_before = state.turn_prompt_tokens completion_tokens_before = state.turn_completion_tokens + cache_read_tokens_before = state.turn_cache_read_tokens try: client = _get_openai_client() typed_messages = cast(list[ChatCompletionMessageParam], messages) @@ -508,8 +543,11 @@ async def _baseline_llm_caller( # Use the delta for this call only — the state accumulators grow across # all tool-call turns, so passing the cumulative total would # compound-overestimate costs on the 2nd+ turn. + # Separate out cached reads so we can apply provider-specific discounts + # (Anthropic: 10 %, OpenAI: 50 %) instead of billing them at full price. call_prompt_tokens = state.turn_prompt_tokens - prompt_tokens_before call_completion_tokens = state.turn_completion_tokens - completion_tokens_before + call_cache_read_tokens = state.turn_cache_read_tokens - cache_read_tokens_before if not got_header_cost and ( call_prompt_tokens > 0 or call_completion_tokens > 0 ): @@ -517,6 +555,7 @@ async def _baseline_llm_caller( state.model, call_prompt_tokens, call_completion_tokens, + cache_read_tokens=call_cache_read_tokens, ) if estimated is not None: state.cost_usd = (state.cost_usd or 0.0) + estimated @@ -1367,6 +1406,22 @@ async def stream_chat_completion_baseline( state.turn_prompt_tokens, state.turn_completion_tokens, ) + # Attempt a cost estimate from the tiktoken-derived counts so that + # persist_and_record_usage never receives cost_usd=None after this + # fallback fires. Only fill in if no cost was already recorded. + if state.cost_usd is None: + tiktoken_estimated = _estimate_cost_from_tokens( + active_model, + state.turn_prompt_tokens, + state.turn_completion_tokens, + ) + if tiktoken_estimated is not None: + state.cost_usd = tiktoken_estimated + logger.info( + "[Baseline] Estimated cost from tiktoken counts: $%.6f (model=%s)", + tiktoken_estimated, + active_model, + ) # Persist token usage to session and record for rate limiting. # When prompt_tokens_details.cached_tokens is reported, subtract diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py index 83c95a12a277..bc247f268403 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py @@ -650,6 +650,36 @@ def test_zero_tokens_returns_zero(self): cost = _estimate_cost_from_tokens("openai/gpt-4o", 0, 0) assert cost == pytest.approx(0.0) + def test_claude_opus_4_6_in_pricing_table(self): + """anthropic/claude-opus-4.6 (default non-fast model) must be priced.""" + cost = _estimate_cost_from_tokens("anthropic/claude-opus-4.6", 1000, 500) + # 1000 * 15.0/1M + 500 * 75.0/1M = 0.015 + 0.0375 = 0.0525 + assert cost == pytest.approx(0.0525) + + def test_cache_read_tokens_discounted_anthropic(self): + """Cache-read tokens billed at 10 % of input rate for Anthropic models.""" + # 200 regular + 800 cache-read prompt tokens, 0 completion + # cost = 200 * 3/1M + 800 * 3/1M * 0.10 = 0.0006 + 0.00024 = 0.00084 + cost = _estimate_cost_from_tokens( + "anthropic/claude-sonnet-4", + prompt_tokens=1000, + completion_tokens=0, + cache_read_tokens=800, + ) + assert cost == pytest.approx(0.00084) + + def test_cache_read_tokens_discounted_openai(self): + """Cache-read tokens billed at 50 % of input rate for OpenAI models.""" + # 200 regular + 800 cache-read, 0 completion + # cost = 200 * 2.5/1M + 800 * 2.5/1M * 0.50 = 0.0005 + 0.001 = 0.0015 + cost = _estimate_cost_from_tokens( + "openai/gpt-4o", + prompt_tokens=1000, + completion_tokens=0, + cache_read_tokens=800, + ) + assert cost == pytest.approx(0.0015) + class TestBaselineCostExtraction: """Tests for x-total-cost header extraction in _baseline_llm_caller.""" @@ -1022,10 +1052,11 @@ async def chunk_aiter(): mock_client = MagicMock() mock_client.chat.completions.create = AsyncMock( side_effect=[ - make_stream(1000, 200), # turn 1 - make_stream( - 1100, 300 - ), # turn 2 (accumulators now hold 1000+1100, 200+300) + make_stream(1000, 200), # turn 1: 1000 prompt, 200 completion + # turn 2: real streaming sends totals-per-call in the final usage + # chunk; each mock stream has a single chunk, so the value here + # represents the total for this API call (not a cumulative delta). + make_stream(1100, 300), # turn 2: 1100 prompt, 300 completion ] ) From a145bbc30104a7b9d330b76db340909b7c702144 Mon Sep 17 00:00:00 2001 From: majdyz Date: Mon, 13 Apr 2026 23:50:56 +0700 Subject: [PATCH 07/14] fix(backend/copilot): correct misleading cache-creation docstring in cost estimator --- .../backend/backend/copilot/baseline/service.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index 33ba887c5124..6ae721e198d7 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -141,8 +141,12 @@ def _estimate_cost_from_tokens( * Cache reads are billed at a fraction of the normal input rate (Anthropic: 10 %, OpenAI: 50 %). The regular (non-cached) portion is billed at full price. - * Cache writes (creation) are already included in ``prompt_tokens`` at the - full input rate — no adjustment is needed. + * Cache writes (creation) appear in + ``prompt_tokens_details.cache_creation_input_tokens`` and are billed at + a premium by some providers (e.g. Anthropic charges 1.25× the input + rate). This estimator does **not** account for them — it intentionally + ignores cache-creation tokens as an acceptable approximation for a + fallback cost estimate. Returns None if the model is not in the pricing table. """ From 257765ae2bc3e2b6f01a5969a05a4ed9d7de1612 Mon Sep 17 00:00:00 2001 From: majdyz Date: Tue, 14 Apr 2026 14:51:33 +0700 Subject: [PATCH 08/14] fix(backend/copilot): remove static fallback pricing and unused imports Drop _OPENROUTER_MODEL_PRICING, _CACHE_READ_DISCOUNT, and _estimate_cost_from_tokens from the baseline service. Cost is now sourced exclusively from the x-total-cost response header; when absent cost_usd remains None. Remove import time and import httpx (leftover from a discarded dynamic-pricing branch). Update tests accordingly. --- .../backend/copilot/baseline/service.py | 129 +----------------- .../copilot/baseline/service_unit_test.py | 80 ++--------- 2 files changed, 15 insertions(+), 194 deletions(-) diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index 6ae721e198d7..32e8454bd305 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -102,76 +102,6 @@ # MIME types that can be embedded as vision content blocks (OpenAI format). _VISION_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/gif", "image/webp"}) -# Fallback per-token pricing (USD) for common OpenRouter models. -# Used only when the ``x-total-cost`` response header is missing. -# Rates sourced from OpenRouter's pricing page (input / output per 1M tokens). -_OPENROUTER_MODEL_PRICING: dict[str, tuple[float, float]] = { - "anthropic/claude-opus-4.6": (15.0 / 1_000_000, 75.0 / 1_000_000), - "anthropic/claude-sonnet-4": (3.0 / 1_000_000, 15.0 / 1_000_000), - "anthropic/claude-3.5-sonnet": (3.0 / 1_000_000, 15.0 / 1_000_000), - "anthropic/claude-3-haiku": (0.25 / 1_000_000, 1.25 / 1_000_000), - "anthropic/claude-3-opus": (15.0 / 1_000_000, 75.0 / 1_000_000), - "openai/gpt-4o": (2.5 / 1_000_000, 10.0 / 1_000_000), - "openai/gpt-4o-mini": (0.15 / 1_000_000, 0.6 / 1_000_000), - "openai/gpt-4-turbo": (10.0 / 1_000_000, 30.0 / 1_000_000), - "google/gemini-2.0-flash-001": (0.1 / 1_000_000, 0.4 / 1_000_000), - "google/gemini-2.5-pro-preview": (1.25 / 1_000_000, 10.0 / 1_000_000), -} - -# Cache-read discount fractions per provider prefix (relative to input rate). -# Anthropic: 10%, OpenAI: 50%, Google: not billed separately so 100% (no discount). -_CACHE_READ_DISCOUNT: dict[str, float] = { - "anthropic/": 0.10, - "openai/": 0.50, -} - - -def _estimate_cost_from_tokens( - model: str, - prompt_tokens: int, - completion_tokens: int, - cache_read_tokens: int = 0, -) -> float | None: - """Estimate USD cost from token counts using known model pricing. - - ``prompt_tokens`` should be the *total* prompt token count as reported by - the API (includes both regular and cache-read tokens as a subset). - ``cache_read_tokens`` is used to apply a provider-specific discount: - - * Cache reads are billed at a fraction of the normal input rate - (Anthropic: 10 %, OpenAI: 50 %). The regular (non-cached) portion is - billed at full price. - * Cache writes (creation) appear in - ``prompt_tokens_details.cache_creation_input_tokens`` and are billed at - a premium by some providers (e.g. Anthropic charges 1.25× the input - rate). This estimator does **not** account for them — it intentionally - ignores cache-creation tokens as an acceptable approximation for a - fallback cost estimate. - - Returns None if the model is not in the pricing table. - """ - pricing = _OPENROUTER_MODEL_PRICING.get(model) - if pricing is None: - return None - input_rate, output_rate = pricing - - # Determine the cache-read discount fraction for this provider. - cache_read_fraction = next( - (v for prefix, v in _CACHE_READ_DISCOUNT.items() if model.startswith(prefix)), - 1.0, # no discount for unknown providers - ) - - # Regular (non-cached) input tokens billed at full price; - # cache-read tokens billed at the discounted rate. - regular_prompt = max(0, prompt_tokens - cache_read_tokens) - cost = ( - regular_prompt * input_rate - + cache_read_tokens * input_rate * cache_read_fraction - + completion_tokens * output_rate - ) - return cost - - # Max size for embedding images directly in the user message (20 MiB raw). _MAX_INLINE_IMAGE_BYTES = 20 * 1024 * 1024 @@ -432,12 +362,6 @@ async def _baseline_llm_caller( round_text = "" response = None # initialized before try so finally block can access it - # Snapshot token counts before this call so we can compute the delta used - # for fallback cost estimation. Must be set before the try so the finally - # block can always reference them even when the API call raises. - prompt_tokens_before = state.turn_prompt_tokens - completion_tokens_before = state.turn_completion_tokens - cache_read_tokens_before = state.turn_cache_read_tokens try: client = _get_openai_client() typed_messages = cast(list[ChatCompletionMessageParam], messages) @@ -529,7 +453,6 @@ async def _baseline_llm_caller( # Extract OpenRouter cost from response headers (in finally so we # capture cost even when the stream errors mid-way — we already paid). # Accumulate across multi-round tool-calling turns. - got_header_cost = False try: # Access undocumented _response attribute — same pattern as # extract_openrouter_cost() in blocks/llm.py. @@ -538,37 +461,14 @@ async def _baseline_llm_caller( cost = float(cost_header) if math.isfinite(cost) and cost >= 0: state.cost_usd = (state.cost_usd or 0.0) + cost - got_header_cost = True - except (AttributeError, ValueError): - pass - - # Fallback: estimate cost from token counts when x-total-cost is - # missing (e.g. some OpenRouter models don't report it). - # Use the delta for this call only — the state accumulators grow across - # all tool-call turns, so passing the cumulative total would - # compound-overestimate costs on the 2nd+ turn. - # Separate out cached reads so we can apply provider-specific discounts - # (Anthropic: 10 %, OpenAI: 50 %) instead of billing them at full price. - call_prompt_tokens = state.turn_prompt_tokens - prompt_tokens_before - call_completion_tokens = state.turn_completion_tokens - completion_tokens_before - call_cache_read_tokens = state.turn_cache_read_tokens - cache_read_tokens_before - if not got_header_cost and ( - call_prompt_tokens > 0 or call_completion_tokens > 0 - ): - estimated = _estimate_cost_from_tokens( - state.model, - call_prompt_tokens, - call_completion_tokens, - cache_read_tokens=call_cache_read_tokens, - ) - if estimated is not None: - state.cost_usd = (state.cost_usd or 0.0) + estimated - logger.info( - "[Baseline] x-total-cost header missing; estimated cost " - "from token pricing: $%.6f (model=%s)", - estimated, + else: + logger.warning( + "[Baseline] x-total-cost header missing from OpenRouter response" + " — cost_usd will be None for this call (model=%s)", state.model, ) + except (AttributeError, ValueError): + pass # Always persist partial text so the session history stays consistent, # even when the stream is interrupted by an exception. @@ -1410,23 +1310,6 @@ async def stream_chat_completion_baseline( state.turn_prompt_tokens, state.turn_completion_tokens, ) - # Attempt a cost estimate from the tiktoken-derived counts so that - # persist_and_record_usage never receives cost_usd=None after this - # fallback fires. Only fill in if no cost was already recorded. - if state.cost_usd is None: - tiktoken_estimated = _estimate_cost_from_tokens( - active_model, - state.turn_prompt_tokens, - state.turn_completion_tokens, - ) - if tiktoken_estimated is not None: - state.cost_usd = tiktoken_estimated - logger.info( - "[Baseline] Estimated cost from tiktoken counts: $%.6f (model=%s)", - tiktoken_estimated, - active_model, - ) - # Persist token usage to session and record for rate limiting. # When prompt_tokens_details.cached_tokens is reported, subtract # them from prompt_tokens to get the uncached count so the cost diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py index bc247f268403..cde53451425d 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py @@ -13,7 +13,6 @@ _baseline_conversation_updater, _BaselineStreamState, _compress_session_messages, - _estimate_cost_from_tokens, _ThinkingStripper, ) from backend.copilot.model import ChatMessage @@ -634,53 +633,6 @@ async def test_workspace_manager_error(self): assert blocks == [] -class TestEstimateCostFromTokens: - """Tests for _estimate_cost_from_tokens fallback pricing.""" - - def test_known_model_returns_estimated_cost(self): - cost = _estimate_cost_from_tokens("anthropic/claude-sonnet-4", 1000, 500) - # 1000 * 3.0/1M + 500 * 15.0/1M = 0.003 + 0.0075 = 0.0105 - assert cost == pytest.approx(0.0105) - - def test_unknown_model_returns_none(self): - cost = _estimate_cost_from_tokens("unknown/model", 1000, 500) - assert cost is None - - def test_zero_tokens_returns_zero(self): - cost = _estimate_cost_from_tokens("openai/gpt-4o", 0, 0) - assert cost == pytest.approx(0.0) - - def test_claude_opus_4_6_in_pricing_table(self): - """anthropic/claude-opus-4.6 (default non-fast model) must be priced.""" - cost = _estimate_cost_from_tokens("anthropic/claude-opus-4.6", 1000, 500) - # 1000 * 15.0/1M + 500 * 75.0/1M = 0.015 + 0.0375 = 0.0525 - assert cost == pytest.approx(0.0525) - - def test_cache_read_tokens_discounted_anthropic(self): - """Cache-read tokens billed at 10 % of input rate for Anthropic models.""" - # 200 regular + 800 cache-read prompt tokens, 0 completion - # cost = 200 * 3/1M + 800 * 3/1M * 0.10 = 0.0006 + 0.00024 = 0.00084 - cost = _estimate_cost_from_tokens( - "anthropic/claude-sonnet-4", - prompt_tokens=1000, - completion_tokens=0, - cache_read_tokens=800, - ) - assert cost == pytest.approx(0.00084) - - def test_cache_read_tokens_discounted_openai(self): - """Cache-read tokens billed at 50 % of input rate for OpenAI models.""" - # 200 regular + 800 cache-read, 0 completion - # cost = 200 * 2.5/1M + 800 * 2.5/1M * 0.50 = 0.0005 + 0.001 = 0.0015 - cost = _estimate_cost_from_tokens( - "openai/gpt-4o", - prompt_tokens=1000, - completion_tokens=0, - cache_read_tokens=800, - ) - assert cost == pytest.approx(0.0015) - - class TestBaselineCostExtraction: """Tests for x-total-cost header extraction in _baseline_llm_caller.""" @@ -878,14 +830,13 @@ async def test_no_cost_when_api_call_raises_before_stream(self): assert state.cost_usd is None @pytest.mark.asyncio - async def test_cost_estimated_from_tokens_when_header_missing(self): - """cost_usd is estimated from token counts when x-total-cost is absent.""" + async def test_no_cost_when_header_missing_no_fallback(self): + """cost_usd remains None when x-total-cost header is absent — no fallback.""" from backend.copilot.baseline.service import ( _baseline_llm_caller, _BaselineStreamState, ) - # Use a model that is in the pricing table state = _BaselineStreamState(model="anthropic/claude-sonnet-4") mock_raw = MagicMock() @@ -893,7 +844,6 @@ async def test_cost_estimated_from_tokens_when_header_missing(self): mock_stream = MagicMock() mock_stream._response = mock_raw - # Create a chunk that reports usage tokens mock_chunk = MagicMock() mock_chunk.usage = MagicMock() mock_chunk.usage.prompt_tokens = 1000 @@ -919,8 +869,7 @@ async def chunk_aiter(): state=state, ) - # Expected: 1000 * 3.0/1M + 500 * 15.0/1M = 0.003 + 0.0075 = 0.0105 - assert state.cost_usd == pytest.approx(0.0105) + assert state.cost_usd is None @pytest.mark.asyncio async def test_cache_tokens_extracted_from_usage_details(self): @@ -1016,13 +965,8 @@ async def chunk_aiter(): assert state.turn_cache_creation_tokens == 500 @pytest.mark.asyncio - async def test_multiturn_fallback_cost_uses_per_call_delta(self): - """Fallback cost estimation uses per-call token delta, not session total. - - On the second tool-call turn, the state accumulators already hold - tokens from turn 1. The estimator must charge only for the new tokens - reported in the current call, not the running total. - """ + async def test_token_accumulators_track_across_multiple_calls(self): + """Token accumulators grow correctly across multiple _baseline_llm_caller calls.""" from backend.copilot.baseline.service import ( _baseline_llm_caller, _BaselineStreamState, @@ -1052,11 +996,8 @@ async def chunk_aiter(): mock_client = MagicMock() mock_client.chat.completions.create = AsyncMock( side_effect=[ - make_stream(1000, 200), # turn 1: 1000 prompt, 200 completion - # turn 2: real streaming sends totals-per-call in the final usage - # chunk; each mock stream has a single chunk, so the value here - # represents the total for this API call (not a cumulative delta). - make_stream(1100, 300), # turn 2: 1100 prompt, 300 completion + make_stream(1000, 200), + make_stream(1100, 300), ] ) @@ -1075,11 +1016,8 @@ async def chunk_aiter(): state=state, ) - # Turn 1: 1000 * 3.0/1M + 200 * 15.0/1M = 0.003 + 0.003 = 0.006 - # Turn 2: 1100 * 3.0/1M + 300 * 15.0/1M = 0.0033 + 0.0045 = 0.0078 - # Total: 0.0138 — NOT 0.006 + cumulative (2100*3/1M + 500*15/1M = 0.006+0.0138) - expected = pytest.approx(0.006 + 0.0078, rel=1e-5) - assert state.cost_usd == expected + # No x-total-cost header — cost_usd remains None + assert state.cost_usd is None # Accumulators hold all tokens across both turns assert state.turn_prompt_tokens == 2100 assert state.turn_completion_tokens == 500 From 64d97a9d28980c5cabc103fe0b53604c0b5c7c83 Mon Sep 17 00:00:00 2001 From: majdyz Date: Tue, 14 Apr 2026 15:04:50 +0700 Subject: [PATCH 09/14] fix(backend/copilot): fetch OpenRouter model pricing dynamically instead of hardcoded table --- .../backend/copilot/baseline/service.py | 148 ++++++++++- .../copilot/baseline/service_unit_test.py | 243 +++++++++++++++++- 2 files changed, 376 insertions(+), 15 deletions(-) diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index 32e8454bd305..158453fdb8a8 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -14,12 +14,14 @@ import re import shutil import tempfile +import time import uuid from collections.abc import AsyncGenerator, Sequence from dataclasses import dataclass, field from functools import partial from typing import TYPE_CHECKING, Any, cast +import httpx import orjson from langfuse import propagate_attributes from openai.types.chat import ChatCompletionMessageParam, ChatCompletionToolParam @@ -102,6 +104,110 @@ # MIME types that can be embedded as vision content blocks (OpenAI format). _VISION_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/gif", "image/webp"}) +# Cache-read discount fractions per provider prefix (relative to input rate). +# Anthropic: 10%, OpenAI: 50%, Google: not billed separately so 100% (no discount). +_CACHE_READ_DISCOUNT: dict[str, float] = { + "anthropic/": 0.10, + "openai/": 0.50, +} + +# OpenRouter model pricing cache: maps model id -> (input_rate, output_rate) per token. +# Populated lazily by _fetch_openrouter_pricing() and refreshed every hour. +_OPENROUTER_PRICING_CACHE: dict[str, tuple[float, float]] = {} +_OPENROUTER_PRICING_CACHE_TTL = 3600 # seconds +_OPENROUTER_PRICING_CACHE_FETCHED_AT: float = 0.0 + + +async def _fetch_openrouter_pricing() -> dict[str, tuple[float, float]]: + """Return per-token pricing for all OpenRouter models. + + Fetches https://openrouter.ai/api/v1/models and caches the result for + ``_OPENROUTER_PRICING_CACHE_TTL`` seconds (1 hour). On any network or + parse error the previous cache value (possibly empty dict) is returned so + that cost estimation gracefully degrades to ``None`` rather than crashing. + """ + global _OPENROUTER_PRICING_CACHE, _OPENROUTER_PRICING_CACHE_FETCHED_AT + + now = time.monotonic() + if now - _OPENROUTER_PRICING_CACHE_FETCHED_AT < _OPENROUTER_PRICING_CACHE_TTL: + return _OPENROUTER_PRICING_CACHE + + try: + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.get("https://openrouter.ai/api/v1/models") + response.raise_for_status() + data = response.json() + + pricing: dict[str, tuple[float, float]] = {} + for model in data.get("data", []): + model_id = model.get("id") + model_pricing = model.get("pricing") or {} + prompt_str = model_pricing.get("prompt") + completion_str = model_pricing.get("completion") + if model_id and prompt_str is not None and completion_str is not None: + try: + pricing[model_id] = (float(prompt_str), float(completion_str)) + except (ValueError, TypeError): + continue + + _OPENROUTER_PRICING_CACHE = pricing + _OPENROUTER_PRICING_CACHE_FETCHED_AT = now + except Exception: + logger.warning( + "Failed to fetch OpenRouter model pricing; using cached data (%d models)", + len(_OPENROUTER_PRICING_CACHE), + ) + + return _OPENROUTER_PRICING_CACHE + + +async def _estimate_cost_from_tokens( + model: str, + prompt_tokens: int, + completion_tokens: int, + cache_read_tokens: int = 0, +) -> float | None: + """Estimate USD cost from token counts using live OpenRouter model pricing. + + ``prompt_tokens`` should be the *total* prompt token count as reported by + the API (includes both regular and cache-read tokens as a subset). + ``cache_read_tokens`` is used to apply a provider-specific discount: + + * Cache reads are billed at a fraction of the normal input rate + (Anthropic: 10 %, OpenAI: 50 %). The regular (non-cached) portion is + billed at full price. + * Cache writes (creation) appear in + ``prompt_tokens_details.cache_creation_input_tokens`` and are billed at + a premium by some providers (e.g. Anthropic charges 1.25× the input + rate). This estimator does **not** account for them -- it intentionally + ignores cache-creation tokens as an acceptable approximation for a + fallback cost estimate. + + Returns None if the model is not found in the OpenRouter pricing response. + """ + pricing_table = await _fetch_openrouter_pricing() + pricing = pricing_table.get(model) + if pricing is None: + return None + input_rate, output_rate = pricing + + # Determine the cache-read discount fraction for this provider. + cache_read_fraction = next( + (v for prefix, v in _CACHE_READ_DISCOUNT.items() if model.startswith(prefix)), + 1.0, # no discount for unknown providers + ) + + # Regular (non-cached) input tokens billed at full price; + # cache-read tokens billed at the discounted rate. + regular_prompt = max(0, prompt_tokens - cache_read_tokens) + cost = ( + regular_prompt * input_rate + + cache_read_tokens * input_rate * cache_read_fraction + + completion_tokens * output_rate + ) + return cost + + # Max size for embedding images directly in the user message (20 MiB raw). _MAX_INLINE_IMAGE_BYTES = 20 * 1024 * 1024 @@ -362,6 +468,12 @@ async def _baseline_llm_caller( round_text = "" response = None # initialized before try so finally block can access it + # Snapshot token counts before this call so we can compute the delta used + # for fallback cost estimation. Must be set before the try so the finally + # block can always reference them even when the API call raises. + prompt_tokens_before = state.turn_prompt_tokens + completion_tokens_before = state.turn_completion_tokens + cache_read_tokens_before = state.turn_cache_read_tokens try: client = _get_openai_client() typed_messages = cast(list[ChatCompletionMessageParam], messages) @@ -453,6 +565,7 @@ async def _baseline_llm_caller( # Extract OpenRouter cost from response headers (in finally so we # capture cost even when the stream errors mid-way — we already paid). # Accumulate across multi-round tool-calling turns. + got_header_cost = False try: # Access undocumented _response attribute — same pattern as # extract_openrouter_cost() in blocks/llm.py. @@ -461,15 +574,38 @@ async def _baseline_llm_caller( cost = float(cost_header) if math.isfinite(cost) and cost >= 0: state.cost_usd = (state.cost_usd or 0.0) + cost - else: - logger.warning( - "[Baseline] x-total-cost header missing from OpenRouter response" - " — cost_usd will be None for this call (model=%s)", - state.model, - ) + got_header_cost = True except (AttributeError, ValueError): pass + # Fallback: estimate cost from token counts when x-total-cost is + # missing (e.g. some OpenRouter models don't report it). + # Use the delta for this call only -- the state accumulators grow across + # all tool-call turns, so passing the cumulative total would + # compound-overestimate costs on the 2nd+ turn. + # Separate out cached reads so we can apply provider-specific discounts + # (Anthropic: 10 %, OpenAI: 50 %) instead of billing them at full price. + call_prompt_tokens = state.turn_prompt_tokens - prompt_tokens_before + call_completion_tokens = state.turn_completion_tokens - completion_tokens_before + call_cache_read_tokens = state.turn_cache_read_tokens - cache_read_tokens_before + if not got_header_cost and ( + call_prompt_tokens > 0 or call_completion_tokens > 0 + ): + estimated = await _estimate_cost_from_tokens( + state.model, + call_prompt_tokens, + call_completion_tokens, + cache_read_tokens=call_cache_read_tokens, + ) + if estimated is not None: + state.cost_usd = (state.cost_usd or 0.0) + estimated + logger.info( + "[Baseline] x-total-cost header missing; estimated cost " + "from token pricing: $%.6f (model=%s)", + estimated, + state.model, + ) + # Always persist partial text so the session history stays consistent, # even when the stream is interrupted by an exception. state.assistant_text += round_text diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py index cde53451425d..7e7126c33c47 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py @@ -13,6 +13,7 @@ _baseline_conversation_updater, _BaselineStreamState, _compress_session_messages, + _estimate_cost_from_tokens, _ThinkingStripper, ) from backend.copilot.model import ChatMessage @@ -633,6 +634,97 @@ async def test_workspace_manager_error(self): assert blocks == [] +# Pricing rates matching OpenRouter API format (per-token USD) used in tests. +_MOCK_OPENROUTER_PRICING: dict[str, tuple[float, float]] = { + "anthropic/claude-opus-4.6": (15.0 / 1_000_000, 75.0 / 1_000_000), + "anthropic/claude-sonnet-4": (3.0 / 1_000_000, 15.0 / 1_000_000), + "anthropic/claude-3.5-sonnet": (3.0 / 1_000_000, 15.0 / 1_000_000), + "openai/gpt-4o": (2.5 / 1_000_000, 10.0 / 1_000_000), + "openai/gpt-4o-mini": (0.15 / 1_000_000, 0.6 / 1_000_000), +} + + +class TestEstimateCostFromTokens: + """Tests for _estimate_cost_from_tokens with dynamic OpenRouter pricing.""" + + @pytest.mark.asyncio + async def test_known_model_returns_estimated_cost(self): + with patch( + "backend.copilot.baseline.service._fetch_openrouter_pricing", + AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), + ): + cost = await _estimate_cost_from_tokens( + "anthropic/claude-sonnet-4", 1000, 500 + ) + # 1000 * 3.0/1M + 500 * 15.0/1M = 0.003 + 0.0075 = 0.0105 + assert cost == pytest.approx(0.0105) + + @pytest.mark.asyncio + async def test_unknown_model_returns_none(self): + with patch( + "backend.copilot.baseline.service._fetch_openrouter_pricing", + AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), + ): + cost = await _estimate_cost_from_tokens("unknown/model", 1000, 500) + assert cost is None + + @pytest.mark.asyncio + async def test_zero_tokens_returns_zero(self): + with patch( + "backend.copilot.baseline.service._fetch_openrouter_pricing", + AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), + ): + cost = await _estimate_cost_from_tokens("openai/gpt-4o", 0, 0) + assert cost == pytest.approx(0.0) + + @pytest.mark.asyncio + async def test_claude_opus_4_6_in_pricing_table(self): + """anthropic/claude-opus-4.6 (default non-fast model) must be priced.""" + with patch( + "backend.copilot.baseline.service._fetch_openrouter_pricing", + AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), + ): + cost = await _estimate_cost_from_tokens( + "anthropic/claude-opus-4.6", 1000, 500 + ) + # 1000 * 15.0/1M + 500 * 75.0/1M = 0.015 + 0.0375 = 0.0525 + assert cost == pytest.approx(0.0525) + + @pytest.mark.asyncio + async def test_cache_read_tokens_discounted_anthropic(self): + """Cache-read tokens billed at 10 % of input rate for Anthropic models.""" + # 200 regular + 800 cache-read prompt tokens, 0 completion + # cost = 200 * 3/1M + 800 * 3/1M * 0.10 = 0.0006 + 0.00024 = 0.00084 + with patch( + "backend.copilot.baseline.service._fetch_openrouter_pricing", + AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), + ): + cost = await _estimate_cost_from_tokens( + "anthropic/claude-sonnet-4", + prompt_tokens=1000, + completion_tokens=0, + cache_read_tokens=800, + ) + assert cost == pytest.approx(0.00084) + + @pytest.mark.asyncio + async def test_cache_read_tokens_discounted_openai(self): + """Cache-read tokens billed at 50 % of input rate for OpenAI models.""" + # 200 regular + 800 cache-read, 0 completion + # cost = 200 * 2.5/1M + 800 * 2.5/1M * 0.50 = 0.0005 + 0.001 = 0.0015 + with patch( + "backend.copilot.baseline.service._fetch_openrouter_pricing", + AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), + ): + cost = await _estimate_cost_from_tokens( + "openai/gpt-4o", + prompt_tokens=1000, + completion_tokens=0, + cache_read_tokens=800, + ) + assert cost == pytest.approx(0.0015) + + class TestBaselineCostExtraction: """Tests for x-total-cost header extraction in _baseline_llm_caller.""" @@ -830,8 +922,8 @@ async def test_no_cost_when_api_call_raises_before_stream(self): assert state.cost_usd is None @pytest.mark.asyncio - async def test_no_cost_when_header_missing_no_fallback(self): - """cost_usd remains None when x-total-cost header is absent — no fallback.""" + async def test_no_cost_when_header_missing_and_pricing_unavailable(self): + """cost_usd remains None when x-total-cost is absent and pricing fetch returns empty.""" from backend.copilot.baseline.service import ( _baseline_llm_caller, _BaselineStreamState, @@ -859,9 +951,15 @@ async def chunk_aiter(): mock_client = MagicMock() mock_client.chat.completions.create = AsyncMock(return_value=mock_stream) - with patch( - "backend.copilot.baseline.service._get_openai_client", - return_value=mock_client, + with ( + patch( + "backend.copilot.baseline.service._get_openai_client", + return_value=mock_client, + ), + patch( + "backend.copilot.baseline.service._fetch_openrouter_pricing", + AsyncMock(return_value={}), + ), ): await _baseline_llm_caller( messages=[{"role": "user", "content": "hi"}], @@ -1001,9 +1099,15 @@ async def chunk_aiter(): ] ) - with patch( - "backend.copilot.baseline.service._get_openai_client", - return_value=mock_client, + with ( + patch( + "backend.copilot.baseline.service._get_openai_client", + return_value=mock_client, + ), + patch( + "backend.copilot.baseline.service._fetch_openrouter_pricing", + AsyncMock(return_value={}), + ), ): await _baseline_llm_caller( messages=[{"role": "user", "content": "hi"}], @@ -1016,8 +1120,129 @@ async def chunk_aiter(): state=state, ) - # No x-total-cost header — cost_usd remains None + # No x-total-cost header and empty pricing table -- cost_usd remains None assert state.cost_usd is None # Accumulators hold all tokens across both turns assert state.turn_prompt_tokens == 2100 assert state.turn_completion_tokens == 500 + + @pytest.mark.asyncio + async def test_cost_estimated_from_tokens_when_header_missing(self): + """cost_usd is estimated from token counts when x-total-cost is absent.""" + from backend.copilot.baseline.service import ( + _baseline_llm_caller, + _BaselineStreamState, + ) + + state = _BaselineStreamState(model="anthropic/claude-sonnet-4") + + mock_raw = MagicMock() + mock_raw.headers = {} # no x-total-cost + mock_stream = MagicMock() + mock_stream._response = mock_raw + + mock_chunk = MagicMock() + mock_chunk.usage = MagicMock() + mock_chunk.usage.prompt_tokens = 1000 + mock_chunk.usage.completion_tokens = 500 + mock_chunk.usage.prompt_tokens_details = None + mock_chunk.choices = [] + + async def chunk_aiter(): + yield mock_chunk + + mock_stream.__aiter__ = lambda self: chunk_aiter() + + mock_client = MagicMock() + mock_client.chat.completions.create = AsyncMock(return_value=mock_stream) + + with ( + patch( + "backend.copilot.baseline.service._get_openai_client", + return_value=mock_client, + ), + patch( + "backend.copilot.baseline.service._fetch_openrouter_pricing", + AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), + ), + ): + await _baseline_llm_caller( + messages=[{"role": "user", "content": "hi"}], + tools=[], + state=state, + ) + + # Expected: 1000 * 3.0/1M + 500 * 15.0/1M = 0.003 + 0.0075 = 0.0105 + assert state.cost_usd == pytest.approx(0.0105) + + @pytest.mark.asyncio + async def test_multiturn_fallback_cost_uses_per_call_delta(self): + """Fallback cost estimation uses per-call token delta, not session total. + + On the second tool-call turn, the state accumulators already hold + tokens from turn 1. The estimator must charge only for the new tokens + reported in the current call, not the running total. + """ + from backend.copilot.baseline.service import ( + _baseline_llm_caller, + _BaselineStreamState, + ) + + state = _BaselineStreamState(model="anthropic/claude-sonnet-4") + + def make_stream_2(prompt_tokens: int, completion_tokens: int): + mock_raw = MagicMock() + mock_raw.headers = {} # no x-total-cost + mock_stream = MagicMock() + mock_stream._response = mock_raw + + mock_chunk = MagicMock() + mock_chunk.usage = MagicMock() + mock_chunk.usage.prompt_tokens = prompt_tokens + mock_chunk.usage.completion_tokens = completion_tokens + mock_chunk.usage.prompt_tokens_details = None + mock_chunk.choices = [] + + async def chunk_aiter(): + yield mock_chunk + + mock_stream.__aiter__ = lambda self: chunk_aiter() + return mock_stream + + mock_client = MagicMock() + mock_client.chat.completions.create = AsyncMock( + side_effect=[ + make_stream_2(1000, 200), + make_stream_2(1100, 300), + ] + ) + + with ( + patch( + "backend.copilot.baseline.service._get_openai_client", + return_value=mock_client, + ), + patch( + "backend.copilot.baseline.service._fetch_openrouter_pricing", + AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), + ), + ): + await _baseline_llm_caller( + messages=[{"role": "user", "content": "hi"}], + tools=[], + state=state, + ) + await _baseline_llm_caller( + messages=[{"role": "user", "content": "follow up"}], + tools=[], + state=state, + ) + + # Turn 1: 1000 * 3.0/1M + 200 * 15.0/1M = 0.003 + 0.003 = 0.006 + # Turn 2: 1100 * 3.0/1M + 300 * 15.0/1M = 0.0033 + 0.0045 = 0.0078 + # Total: 0.0138 -- NOT 0.006 + cumulative (2100*3/1M + 500*15/1M) + expected = pytest.approx(0.006 + 0.0078, rel=1e-5) + assert state.cost_usd == expected + # Accumulators hold all tokens across both turns + assert state.turn_prompt_tokens == 2100 + assert state.turn_completion_tokens == 500 From 52b6b64155d6bb9de7baf978d21eb16e9901bb13 Mon Sep 17 00:00:00 2001 From: majdyz Date: Tue, 14 Apr 2026 16:04:20 +0700 Subject: [PATCH 10/14] fix(backend/copilot): init pricing cache timestamp to -inf so first fetch always runs --- autogpt_platform/backend/backend/copilot/baseline/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index 158453fdb8a8..8d693dd2cfde 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -115,7 +115,7 @@ # Populated lazily by _fetch_openrouter_pricing() and refreshed every hour. _OPENROUTER_PRICING_CACHE: dict[str, tuple[float, float]] = {} _OPENROUTER_PRICING_CACHE_TTL = 3600 # seconds -_OPENROUTER_PRICING_CACHE_FETCHED_AT: float = 0.0 +_OPENROUTER_PRICING_CACHE_FETCHED_AT: float = float("-inf") async def _fetch_openrouter_pricing() -> dict[str, tuple[float, float]]: From 012ea167d83996e74a19d90ee988f772e2812ed6 Mon Sep 17 00:00:00 2001 From: majdyz Date: Tue, 14 Apr 2026 16:35:25 +0700 Subject: [PATCH 11/14] fix(backend/copilot): fetch cache_read rate from OpenRouter instead of hardcoded discount table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the hardcoded _CACHE_READ_DISCOUNT dict (Anthropic 10%, OpenAI 50%). Instead, extract the pricing.cache_read field from OpenRouter's /api/v1/models response and use it directly. Models where OpenRouter does not publish a cache_read rate fall back to the full input rate (safe over-estimate). Cache tuple type: (input_rate, output_rate) → (input_rate, output_rate, cache_read_rate | None) --- .../backend/copilot/baseline/service.py | 62 +++++++++---------- .../copilot/baseline/service_unit_test.py | 47 ++++++++++---- 2 files changed, 65 insertions(+), 44 deletions(-) diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index 12078cd507e0..de7ddc1ab8fd 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -105,21 +105,17 @@ # MIME types that can be embedded as vision content blocks (OpenAI format). _VISION_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/gif", "image/webp"}) -# Cache-read discount fractions per provider prefix (relative to input rate). -# Anthropic: 10%, OpenAI: 50%, Google: not billed separately so 100% (no discount). -_CACHE_READ_DISCOUNT: dict[str, float] = { - "anthropic/": 0.10, - "openai/": 0.50, -} - -# OpenRouter model pricing cache: maps model id -> (input_rate, output_rate) per token. +# OpenRouter model pricing cache: +# maps model id -> (input_rate, output_rate, cache_read_rate | None) per token. +# cache_read_rate is None when OpenRouter doesn't publish it for that model; +# those tokens fall back to the full input rate (safe over-estimate). # Populated lazily by _fetch_openrouter_pricing() and refreshed every hour. -_OPENROUTER_PRICING_CACHE: dict[str, tuple[float, float]] = {} +_OPENROUTER_PRICING_CACHE: dict[str, tuple[float, float, float | None]] = {} _OPENROUTER_PRICING_CACHE_TTL = 3600 # seconds _OPENROUTER_PRICING_CACHE_FETCHED_AT: float = float("-inf") -async def _fetch_openrouter_pricing() -> dict[str, tuple[float, float]]: +async def _fetch_openrouter_pricing() -> dict[str, tuple[float, float, float | None]]: """Return per-token pricing for all OpenRouter models. Fetches https://openrouter.ai/api/v1/models and caches the result for @@ -139,7 +135,7 @@ async def _fetch_openrouter_pricing() -> dict[str, tuple[float, float]]: response.raise_for_status() data = response.json() - pricing: dict[str, tuple[float, float]] = {} + pricing: dict[str, tuple[float, float, float | None]] = {} for model in data.get("data", []): model_id = model.get("id") model_pricing = model.get("pricing") or {} @@ -147,7 +143,17 @@ async def _fetch_openrouter_pricing() -> dict[str, tuple[float, float]]: completion_str = model_pricing.get("completion") if model_id and prompt_str is not None and completion_str is not None: try: - pricing[model_id] = (float(prompt_str), float(completion_str)) + cache_read_str = model_pricing.get("cache_read") + cache_read_rate = ( + float(cache_read_str) + if cache_read_str is not None + else None + ) + pricing[model_id] = ( + float(prompt_str), + float(completion_str), + cache_read_rate, + ) except (ValueError, TypeError): continue @@ -172,17 +178,13 @@ async def _estimate_cost_from_tokens( ``prompt_tokens`` should be the *total* prompt token count as reported by the API (includes both regular and cache-read tokens as a subset). - ``cache_read_tokens`` is used to apply a provider-specific discount: - - * Cache reads are billed at a fraction of the normal input rate - (Anthropic: 10 %, OpenAI: 50 %). The regular (non-cached) portion is - billed at full price. - * Cache writes (creation) appear in - ``prompt_tokens_details.cache_creation_input_tokens`` and are billed at - a premium by some providers (e.g. Anthropic charges 1.25× the input - rate). This estimator does **not** account for them -- it intentionally - ignores cache-creation tokens as an acceptable approximation for a - fallback cost estimate. + ``cache_read_tokens`` is the subset of prompt tokens served from cache. + When OpenRouter publishes a ``cache_read`` rate for the model it is used + directly; otherwise cache-read tokens fall back to the full input rate + (a safe over-estimate). + + Cache writes (creation) are intentionally ignored as an acceptable + approximation for a fallback cost estimate. Returns None if the model is not found in the OpenRouter pricing response. """ @@ -190,20 +192,16 @@ async def _estimate_cost_from_tokens( pricing = pricing_table.get(model) if pricing is None: return None - input_rate, output_rate = pricing - - # Determine the cache-read discount fraction for this provider. - cache_read_fraction = next( - (v for prefix, v in _CACHE_READ_DISCOUNT.items() if model.startswith(prefix)), - 1.0, # no discount for unknown providers - ) + input_rate, output_rate, cache_read_rate = pricing # Regular (non-cached) input tokens billed at full price; - # cache-read tokens billed at the discounted rate. + # cache-read tokens billed at the OpenRouter-published rate when available, + # or at the full input rate when not (safe over-estimate). + effective_cache_rate = cache_read_rate if cache_read_rate is not None else input_rate regular_prompt = max(0, prompt_tokens - cache_read_tokens) cost = ( regular_prompt * input_rate - + cache_read_tokens * input_rate * cache_read_fraction + + cache_read_tokens * effective_cache_rate + completion_tokens * output_rate ) return cost diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py index 10fec15d2cdd..14279a8b5863 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py @@ -577,12 +577,15 @@ async def test_workspace_manager_error(self): # Pricing rates matching OpenRouter API format (per-token USD) used in tests. -_MOCK_OPENROUTER_PRICING: dict[str, tuple[float, float]] = { - "anthropic/claude-opus-4.6": (15.0 / 1_000_000, 75.0 / 1_000_000), - "anthropic/claude-sonnet-4": (3.0 / 1_000_000, 15.0 / 1_000_000), - "anthropic/claude-3.5-sonnet": (3.0 / 1_000_000, 15.0 / 1_000_000), - "openai/gpt-4o": (2.5 / 1_000_000, 10.0 / 1_000_000), - "openai/gpt-4o-mini": (0.15 / 1_000_000, 0.6 / 1_000_000), +# Tuple: (input_rate, output_rate, cache_read_rate | None) +# Anthropic: cache_read = 10% of input; OpenAI: cache_read = 50% of input; +# gpt-4o-mini: no cache_read published (None → falls back to full input rate). +_MOCK_OPENROUTER_PRICING: dict[str, tuple[float, float, float | None]] = { + "anthropic/claude-opus-4.6": (15.0 / 1_000_000, 75.0 / 1_000_000, 1.5 / 1_000_000), + "anthropic/claude-sonnet-4": (3.0 / 1_000_000, 15.0 / 1_000_000, 0.3 / 1_000_000), + "anthropic/claude-3.5-sonnet": (3.0 / 1_000_000, 15.0 / 1_000_000, 0.3 / 1_000_000), + "openai/gpt-4o": (2.5 / 1_000_000, 10.0 / 1_000_000, 1.25 / 1_000_000), + "openai/gpt-4o-mini": (0.15 / 1_000_000, 0.6 / 1_000_000, None), } @@ -633,10 +636,11 @@ async def test_claude_opus_4_6_in_pricing_table(self): assert cost == pytest.approx(0.0525) @pytest.mark.asyncio - async def test_cache_read_tokens_discounted_anthropic(self): - """Cache-read tokens billed at 10 % of input rate for Anthropic models.""" + async def test_cache_read_tokens_use_openrouter_cache_rate_anthropic(self): + """Cache-read tokens use OpenRouter-published cache_read rate for Anthropic.""" + # anthropic/claude-sonnet-4: input=3/1M, cache_read=0.3/1M # 200 regular + 800 cache-read prompt tokens, 0 completion - # cost = 200 * 3/1M + 800 * 3/1M * 0.10 = 0.0006 + 0.00024 = 0.00084 + # cost = 200 * 3/1M + 800 * 0.3/1M = 0.0006 + 0.00024 = 0.00084 with patch( "backend.copilot.baseline.service._fetch_openrouter_pricing", AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), @@ -650,10 +654,11 @@ async def test_cache_read_tokens_discounted_anthropic(self): assert cost == pytest.approx(0.00084) @pytest.mark.asyncio - async def test_cache_read_tokens_discounted_openai(self): - """Cache-read tokens billed at 50 % of input rate for OpenAI models.""" + async def test_cache_read_tokens_use_openrouter_cache_rate_openai(self): + """Cache-read tokens use OpenRouter-published cache_read rate for OpenAI.""" + # openai/gpt-4o: input=2.5/1M, cache_read=1.25/1M # 200 regular + 800 cache-read, 0 completion - # cost = 200 * 2.5/1M + 800 * 2.5/1M * 0.50 = 0.0005 + 0.001 = 0.0015 + # cost = 200 * 2.5/1M + 800 * 1.25/1M = 0.0005 + 0.001 = 0.0015 with patch( "backend.copilot.baseline.service._fetch_openrouter_pricing", AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), @@ -666,6 +671,24 @@ async def test_cache_read_tokens_discounted_openai(self): ) assert cost == pytest.approx(0.0015) + @pytest.mark.asyncio + async def test_cache_read_falls_back_to_input_rate_when_none(self): + """Models without a published cache_read rate fall back to the full input rate.""" + # openai/gpt-4o-mini: input=0.15/1M, cache_read=None (no discount published) + # 200 regular + 800 cache-read, 0 completion + # cost = 200 * 0.15/1M + 800 * 0.15/1M = 1000 * 0.15/1M = 0.00015 + with patch( + "backend.copilot.baseline.service._fetch_openrouter_pricing", + AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), + ): + cost = await _estimate_cost_from_tokens( + "openai/gpt-4o-mini", + prompt_tokens=1000, + completion_tokens=0, + cache_read_tokens=800, + ) + assert cost == pytest.approx(0.00015) + class TestBaselineCostExtraction: """Tests for x-total-cost header extraction in _baseline_llm_caller.""" From 8420ad0c138ee53cb094fc59da941283f2d58f53 Mon Sep 17 00:00:00 2001 From: majdyz Date: Tue, 14 Apr 2026 20:12:43 +0700 Subject: [PATCH 12/14] fix(backend/copilot): tiktoken fallback also estimates cost_usd + billed StreamUsage Two gaps closed: 1. When a provider omits both `x-total-cost` and streaming `usage`, `stream_chat_completion_baseline` backfills token counts via tiktoken but previously left `state.cost_usd = None`. Now we call `await _estimate_cost_from_tokens(active_model, ...)` immediately after the tiktoken backfill so `persist_and_record_usage` receives a non-None cost when the model is in the OpenRouter pricing response. 2. The `StreamUsage` event was reporting raw `state.turn_prompt_tokens` (includes cached reads), making the frontend token count inconsistent with `cost_usd` which already applied the cache discount. Now we compute `billed_prompt = max(0, turn_prompt_tokens - turn_cache_read_tokens)` and yield that instead. Also removes unused `_ThinkingStripper` import from service_unit_test.py (caught by ruff) to fix the lint CI failure. --- .../backend/copilot/baseline/service.py | 32 +++++++++++++++---- .../copilot/baseline/service_unit_test.py | 1 - 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index de7ddc1ab8fd..8a49dd86e86a 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -145,9 +145,7 @@ async def _fetch_openrouter_pricing() -> dict[str, tuple[float, float, float | N try: cache_read_str = model_pricing.get("cache_read") cache_read_rate = ( - float(cache_read_str) - if cache_read_str is not None - else None + float(cache_read_str) if cache_read_str is not None else None ) pricing[model_id] = ( float(prompt_str), @@ -197,7 +195,9 @@ async def _estimate_cost_from_tokens( # Regular (non-cached) input tokens billed at full price; # cache-read tokens billed at the OpenRouter-published rate when available, # or at the full input rate when not (safe over-estimate). - effective_cache_rate = cache_read_rate if cache_read_rate is not None else input_rate + effective_cache_rate = ( + cache_read_rate if cache_read_rate is not None else input_rate + ) regular_prompt = max(0, prompt_tokens - cache_read_tokens) cost = ( regular_prompt * input_rate @@ -1344,6 +1344,23 @@ async def stream_chat_completion_baseline( state.turn_prompt_tokens, state.turn_completion_tokens, ) + # Attempt a cost estimate from the tiktoken-derived counts so that + # persist_and_record_usage never receives cost_usd=None after this + # fallback fires. Only fill in if no cost was already recorded. + if state.cost_usd is None: + tiktoken_estimated = await _estimate_cost_from_tokens( + active_model, + state.turn_prompt_tokens, + state.turn_completion_tokens, + ) + if tiktoken_estimated is not None: + state.cost_usd = tiktoken_estimated + logger.info( + "[Baseline] Estimated cost from tiktoken counts: " + "$%.6f (model=%s)", + tiktoken_estimated, + active_model, + ) # Persist token usage to session and record for rate limiting. # When prompt_tokens_details.cached_tokens is reported, subtract # them from prompt_tokens to get the uncached count so the cost @@ -1429,10 +1446,13 @@ async def stream_chat_completion_baseline( # On GeneratorExit the client is already gone, so unreachable yields # are harmless; on normal completion they reach the SSE stream. if state.turn_prompt_tokens > 0 or state.turn_completion_tokens > 0: + # Report uncached prompt tokens to match what was billed — cached tokens + # are excluded so the frontend display is consistent with cost_usd. + billed_prompt = max(0, state.turn_prompt_tokens - state.turn_cache_read_tokens) yield StreamUsage( - prompt_tokens=state.turn_prompt_tokens, + prompt_tokens=billed_prompt, completion_tokens=state.turn_completion_tokens, - total_tokens=state.turn_prompt_tokens + state.turn_completion_tokens, + total_tokens=billed_prompt + state.turn_completion_tokens, ) yield StreamFinish() diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py index 14279a8b5863..fb776f41de65 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py @@ -14,7 +14,6 @@ _BaselineStreamState, _compress_session_messages, _estimate_cost_from_tokens, - _ThinkingStripper, ) from backend.copilot.model import ChatMessage from backend.copilot.transcript_builder import TranscriptBuilder From 060c9cf5afaf288e63cdb1d39b9ac48b799be984 Mon Sep 17 00:00:00 2001 From: majdyz Date: Tue, 14 Apr 2026 20:28:21 +0700 Subject: [PATCH 13/14] fix(backend/copilot): prevent thundering-herd and add backoff in pricing cache fetch _fetch_openrouter_pricing() called from the finally block can block for up to 10s on a cold cache, and concurrent callers all seeing an expired cache would each make separate HTTP requests simultaneously. Two fixes: 1. Add a lazy asyncio.Lock (_OPENROUTER_PRICING_LOCK) with double-checked locking so only one coroutine makes the HTTP request; others wait and reuse the result. 2. Update _OPENROUTER_PRICING_CACHE_FETCHED_AT *before* the HTTP request so that a failed or timed-out fetch advances the "next allowed fetch" window, creating a natural backoff instead of every subsequent request immediately retrying on OpenRouter outage. --- .../backend/copilot/baseline/service.py | 93 ++++++++++++------- 1 file changed, 62 insertions(+), 31 deletions(-) diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index 8a49dd86e86a..9030830fdc6f 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -113,6 +113,21 @@ _OPENROUTER_PRICING_CACHE: dict[str, tuple[float, float, float | None]] = {} _OPENROUTER_PRICING_CACHE_TTL = 3600 # seconds _OPENROUTER_PRICING_CACHE_FETCHED_AT: float = float("-inf") +# Lock prevents thundering-herd: only one coroutine fetches pricing at a time; +# others wait and reuse the result once the lock is released. +_OPENROUTER_PRICING_LOCK: asyncio.Lock | None = None + + +def _get_openrouter_pricing_lock() -> asyncio.Lock: + """Return (and lazily create) the module-level asyncio.Lock. + + The lock cannot be created at module import time because there may be no + running event loop yet; this helper creates it on first use inside the loop. + """ + global _OPENROUTER_PRICING_LOCK + if _OPENROUTER_PRICING_LOCK is None: + _OPENROUTER_PRICING_LOCK = asyncio.Lock() + return _OPENROUTER_PRICING_LOCK async def _fetch_openrouter_pricing() -> dict[str, tuple[float, float, float | None]]: @@ -122,6 +137,11 @@ async def _fetch_openrouter_pricing() -> dict[str, tuple[float, float, float | N ``_OPENROUTER_PRICING_CACHE_TTL`` seconds (1 hour). On any network or parse error the previous cache value (possibly empty dict) is returned so that cost estimation gracefully degrades to ``None`` rather than crashing. + + A module-level asyncio.Lock prevents concurrent callers from each making a + separate HTTP request when the cache expires (thundering-herd prevention). + The timestamp is updated even on failure so that a failed fetch creates a + backoff period before the next attempt. """ global _OPENROUTER_PRICING_CACHE, _OPENROUTER_PRICING_CACHE_FETCHED_AT @@ -129,39 +149,50 @@ async def _fetch_openrouter_pricing() -> dict[str, tuple[float, float, float | N if now - _OPENROUTER_PRICING_CACHE_FETCHED_AT < _OPENROUTER_PRICING_CACHE_TTL: return _OPENROUTER_PRICING_CACHE - try: - async with httpx.AsyncClient(timeout=10.0) as client: - response = await client.get("https://openrouter.ai/api/v1/models") - response.raise_for_status() - data = response.json() - - pricing: dict[str, tuple[float, float, float | None]] = {} - for model in data.get("data", []): - model_id = model.get("id") - model_pricing = model.get("pricing") or {} - prompt_str = model_pricing.get("prompt") - completion_str = model_pricing.get("completion") - if model_id and prompt_str is not None and completion_str is not None: - try: - cache_read_str = model_pricing.get("cache_read") - cache_read_rate = ( - float(cache_read_str) if cache_read_str is not None else None - ) - pricing[model_id] = ( - float(prompt_str), - float(completion_str), - cache_read_rate, - ) - except (ValueError, TypeError): - continue + async with _get_openrouter_pricing_lock(): + # Re-check inside the lock — another coroutine may have just refreshed. + now = time.monotonic() + if now - _OPENROUTER_PRICING_CACHE_FETCHED_AT < _OPENROUTER_PRICING_CACHE_TTL: + return _OPENROUTER_PRICING_CACHE - _OPENROUTER_PRICING_CACHE = pricing + # Update the timestamp before the request so that any exception during + # the HTTP call still advances the "next allowed fetch" window, giving a + # backoff period instead of every request immediately retrying on failure. _OPENROUTER_PRICING_CACHE_FETCHED_AT = now - except Exception: - logger.warning( - "Failed to fetch OpenRouter model pricing; using cached data (%d models)", - len(_OPENROUTER_PRICING_CACHE), - ) + try: + async with httpx.AsyncClient(timeout=10.0) as client: + response = await client.get("https://openrouter.ai/api/v1/models") + response.raise_for_status() + data = response.json() + + pricing: dict[str, tuple[float, float, float | None]] = {} + for model in data.get("data", []): + model_id = model.get("id") + model_pricing = model.get("pricing") or {} + prompt_str = model_pricing.get("prompt") + completion_str = model_pricing.get("completion") + if model_id and prompt_str is not None and completion_str is not None: + try: + cache_read_str = model_pricing.get("cache_read") + cache_read_rate = ( + float(cache_read_str) + if cache_read_str is not None + else None + ) + pricing[model_id] = ( + float(prompt_str), + float(completion_str), + cache_read_rate, + ) + except (ValueError, TypeError): + continue + + _OPENROUTER_PRICING_CACHE = pricing + except Exception: + logger.warning( + "Failed to fetch OpenRouter model pricing; using cached data (%d models)", + len(_OPENROUTER_PRICING_CACHE), + ) return _OPENROUTER_PRICING_CACHE From d7653acd0eee76b27ccb908e4ed2e6747161771e Mon Sep 17 00:00:00 2001 From: majdyz Date: Tue, 14 Apr 2026 20:48:31 +0700 Subject: [PATCH 14/14] =?UTF-8?q?fix(backend/copilot):=20remove=20cost=20e?= =?UTF-8?q?stimation=20=E2=80=94=20report=20tokens=20only=20when=20x-total?= =?UTF-8?q?-cost=20absent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/copilot/baseline/service.py | 187 -------------- .../copilot/baseline/service_unit_test.py | 238 ++---------------- 2 files changed, 19 insertions(+), 406 deletions(-) diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index 9030830fdc6f..bb3906811cf3 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -14,14 +14,12 @@ import re import shutil import tempfile -import time import uuid from collections.abc import AsyncGenerator, Sequence from dataclasses import dataclass, field from functools import partial from typing import TYPE_CHECKING, Any, cast -import httpx import orjson from langfuse import propagate_attributes from openai.types.chat import ChatCompletionMessageParam, ChatCompletionToolParam @@ -105,138 +103,6 @@ # MIME types that can be embedded as vision content blocks (OpenAI format). _VISION_MIME_TYPES = frozenset({"image/png", "image/jpeg", "image/gif", "image/webp"}) -# OpenRouter model pricing cache: -# maps model id -> (input_rate, output_rate, cache_read_rate | None) per token. -# cache_read_rate is None when OpenRouter doesn't publish it for that model; -# those tokens fall back to the full input rate (safe over-estimate). -# Populated lazily by _fetch_openrouter_pricing() and refreshed every hour. -_OPENROUTER_PRICING_CACHE: dict[str, tuple[float, float, float | None]] = {} -_OPENROUTER_PRICING_CACHE_TTL = 3600 # seconds -_OPENROUTER_PRICING_CACHE_FETCHED_AT: float = float("-inf") -# Lock prevents thundering-herd: only one coroutine fetches pricing at a time; -# others wait and reuse the result once the lock is released. -_OPENROUTER_PRICING_LOCK: asyncio.Lock | None = None - - -def _get_openrouter_pricing_lock() -> asyncio.Lock: - """Return (and lazily create) the module-level asyncio.Lock. - - The lock cannot be created at module import time because there may be no - running event loop yet; this helper creates it on first use inside the loop. - """ - global _OPENROUTER_PRICING_LOCK - if _OPENROUTER_PRICING_LOCK is None: - _OPENROUTER_PRICING_LOCK = asyncio.Lock() - return _OPENROUTER_PRICING_LOCK - - -async def _fetch_openrouter_pricing() -> dict[str, tuple[float, float, float | None]]: - """Return per-token pricing for all OpenRouter models. - - Fetches https://openrouter.ai/api/v1/models and caches the result for - ``_OPENROUTER_PRICING_CACHE_TTL`` seconds (1 hour). On any network or - parse error the previous cache value (possibly empty dict) is returned so - that cost estimation gracefully degrades to ``None`` rather than crashing. - - A module-level asyncio.Lock prevents concurrent callers from each making a - separate HTTP request when the cache expires (thundering-herd prevention). - The timestamp is updated even on failure so that a failed fetch creates a - backoff period before the next attempt. - """ - global _OPENROUTER_PRICING_CACHE, _OPENROUTER_PRICING_CACHE_FETCHED_AT - - now = time.monotonic() - if now - _OPENROUTER_PRICING_CACHE_FETCHED_AT < _OPENROUTER_PRICING_CACHE_TTL: - return _OPENROUTER_PRICING_CACHE - - async with _get_openrouter_pricing_lock(): - # Re-check inside the lock — another coroutine may have just refreshed. - now = time.monotonic() - if now - _OPENROUTER_PRICING_CACHE_FETCHED_AT < _OPENROUTER_PRICING_CACHE_TTL: - return _OPENROUTER_PRICING_CACHE - - # Update the timestamp before the request so that any exception during - # the HTTP call still advances the "next allowed fetch" window, giving a - # backoff period instead of every request immediately retrying on failure. - _OPENROUTER_PRICING_CACHE_FETCHED_AT = now - try: - async with httpx.AsyncClient(timeout=10.0) as client: - response = await client.get("https://openrouter.ai/api/v1/models") - response.raise_for_status() - data = response.json() - - pricing: dict[str, tuple[float, float, float | None]] = {} - for model in data.get("data", []): - model_id = model.get("id") - model_pricing = model.get("pricing") or {} - prompt_str = model_pricing.get("prompt") - completion_str = model_pricing.get("completion") - if model_id and prompt_str is not None and completion_str is not None: - try: - cache_read_str = model_pricing.get("cache_read") - cache_read_rate = ( - float(cache_read_str) - if cache_read_str is not None - else None - ) - pricing[model_id] = ( - float(prompt_str), - float(completion_str), - cache_read_rate, - ) - except (ValueError, TypeError): - continue - - _OPENROUTER_PRICING_CACHE = pricing - except Exception: - logger.warning( - "Failed to fetch OpenRouter model pricing; using cached data (%d models)", - len(_OPENROUTER_PRICING_CACHE), - ) - - return _OPENROUTER_PRICING_CACHE - - -async def _estimate_cost_from_tokens( - model: str, - prompt_tokens: int, - completion_tokens: int, - cache_read_tokens: int = 0, -) -> float | None: - """Estimate USD cost from token counts using live OpenRouter model pricing. - - ``prompt_tokens`` should be the *total* prompt token count as reported by - the API (includes both regular and cache-read tokens as a subset). - ``cache_read_tokens`` is the subset of prompt tokens served from cache. - When OpenRouter publishes a ``cache_read`` rate for the model it is used - directly; otherwise cache-read tokens fall back to the full input rate - (a safe over-estimate). - - Cache writes (creation) are intentionally ignored as an acceptable - approximation for a fallback cost estimate. - - Returns None if the model is not found in the OpenRouter pricing response. - """ - pricing_table = await _fetch_openrouter_pricing() - pricing = pricing_table.get(model) - if pricing is None: - return None - input_rate, output_rate, cache_read_rate = pricing - - # Regular (non-cached) input tokens billed at full price; - # cache-read tokens billed at the OpenRouter-published rate when available, - # or at the full input rate when not (safe over-estimate). - effective_cache_rate = ( - cache_read_rate if cache_read_rate is not None else input_rate - ) - regular_prompt = max(0, prompt_tokens - cache_read_tokens) - cost = ( - regular_prompt * input_rate - + cache_read_tokens * effective_cache_rate - + completion_tokens * output_rate - ) - return cost - # Max size for embedding images directly in the user message (20 MiB raw). _MAX_INLINE_IMAGE_BYTES = 20 * 1024 * 1024 @@ -406,12 +272,6 @@ async def _baseline_llm_caller( round_text = "" response = None # initialized before try so finally block can access it - # Snapshot token counts before this call so we can compute the delta used - # for fallback cost estimation. Must be set before the try so the finally - # block can always reference them even when the API call raises. - prompt_tokens_before = state.turn_prompt_tokens - completion_tokens_before = state.turn_completion_tokens - cache_read_tokens_before = state.turn_cache_read_tokens try: client = _get_openai_client() typed_messages = cast(list[ChatCompletionMessageParam], messages) @@ -503,7 +363,6 @@ async def _baseline_llm_caller( # Extract OpenRouter cost from response headers (in finally so we # capture cost even when the stream errors mid-way — we already paid). # Accumulate across multi-round tool-calling turns. - got_header_cost = False try: # Access undocumented _response attribute — same pattern as # extract_openrouter_cost() in blocks/llm.py. @@ -512,38 +371,9 @@ async def _baseline_llm_caller( cost = float(cost_header) if math.isfinite(cost) and cost >= 0: state.cost_usd = (state.cost_usd or 0.0) + cost - got_header_cost = True except (AttributeError, ValueError): pass - # Fallback: estimate cost from token counts when x-total-cost is - # missing (e.g. some OpenRouter models don't report it). - # Use the delta for this call only -- the state accumulators grow across - # all tool-call turns, so passing the cumulative total would - # compound-overestimate costs on the 2nd+ turn. - # Separate out cached reads so we can apply provider-specific discounts - # (Anthropic: 10 %, OpenAI: 50 %) instead of billing them at full price. - call_prompt_tokens = state.turn_prompt_tokens - prompt_tokens_before - call_completion_tokens = state.turn_completion_tokens - completion_tokens_before - call_cache_read_tokens = state.turn_cache_read_tokens - cache_read_tokens_before - if not got_header_cost and ( - call_prompt_tokens > 0 or call_completion_tokens > 0 - ): - estimated = await _estimate_cost_from_tokens( - state.model, - call_prompt_tokens, - call_completion_tokens, - cache_read_tokens=call_cache_read_tokens, - ) - if estimated is not None: - state.cost_usd = (state.cost_usd or 0.0) + estimated - logger.info( - "[Baseline] x-total-cost header missing; estimated cost " - "from token pricing: $%.6f (model=%s)", - estimated, - state.model, - ) - # Always persist partial text so the session history stays consistent, # even when the stream is interrupted by an exception. state.assistant_text += round_text @@ -1375,23 +1205,6 @@ async def stream_chat_completion_baseline( state.turn_prompt_tokens, state.turn_completion_tokens, ) - # Attempt a cost estimate from the tiktoken-derived counts so that - # persist_and_record_usage never receives cost_usd=None after this - # fallback fires. Only fill in if no cost was already recorded. - if state.cost_usd is None: - tiktoken_estimated = await _estimate_cost_from_tokens( - active_model, - state.turn_prompt_tokens, - state.turn_completion_tokens, - ) - if tiktoken_estimated is not None: - state.cost_usd = tiktoken_estimated - logger.info( - "[Baseline] Estimated cost from tiktoken counts: " - "$%.6f (model=%s)", - tiktoken_estimated, - active_model, - ) # Persist token usage to session and record for rate limiting. # When prompt_tokens_details.cached_tokens is reported, subtract # them from prompt_tokens to get the uncached count so the cost diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py index fb776f41de65..881018175f5f 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py @@ -13,7 +13,6 @@ _baseline_conversation_updater, _BaselineStreamState, _compress_session_messages, - _estimate_cost_from_tokens, ) from backend.copilot.model import ChatMessage from backend.copilot.transcript_builder import TranscriptBuilder @@ -575,120 +574,6 @@ async def test_workspace_manager_error(self): assert blocks == [] -# Pricing rates matching OpenRouter API format (per-token USD) used in tests. -# Tuple: (input_rate, output_rate, cache_read_rate | None) -# Anthropic: cache_read = 10% of input; OpenAI: cache_read = 50% of input; -# gpt-4o-mini: no cache_read published (None → falls back to full input rate). -_MOCK_OPENROUTER_PRICING: dict[str, tuple[float, float, float | None]] = { - "anthropic/claude-opus-4.6": (15.0 / 1_000_000, 75.0 / 1_000_000, 1.5 / 1_000_000), - "anthropic/claude-sonnet-4": (3.0 / 1_000_000, 15.0 / 1_000_000, 0.3 / 1_000_000), - "anthropic/claude-3.5-sonnet": (3.0 / 1_000_000, 15.0 / 1_000_000, 0.3 / 1_000_000), - "openai/gpt-4o": (2.5 / 1_000_000, 10.0 / 1_000_000, 1.25 / 1_000_000), - "openai/gpt-4o-mini": (0.15 / 1_000_000, 0.6 / 1_000_000, None), -} - - -class TestEstimateCostFromTokens: - """Tests for _estimate_cost_from_tokens with dynamic OpenRouter pricing.""" - - @pytest.mark.asyncio - async def test_known_model_returns_estimated_cost(self): - with patch( - "backend.copilot.baseline.service._fetch_openrouter_pricing", - AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), - ): - cost = await _estimate_cost_from_tokens( - "anthropic/claude-sonnet-4", 1000, 500 - ) - # 1000 * 3.0/1M + 500 * 15.0/1M = 0.003 + 0.0075 = 0.0105 - assert cost == pytest.approx(0.0105) - - @pytest.mark.asyncio - async def test_unknown_model_returns_none(self): - with patch( - "backend.copilot.baseline.service._fetch_openrouter_pricing", - AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), - ): - cost = await _estimate_cost_from_tokens("unknown/model", 1000, 500) - assert cost is None - - @pytest.mark.asyncio - async def test_zero_tokens_returns_zero(self): - with patch( - "backend.copilot.baseline.service._fetch_openrouter_pricing", - AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), - ): - cost = await _estimate_cost_from_tokens("openai/gpt-4o", 0, 0) - assert cost == pytest.approx(0.0) - - @pytest.mark.asyncio - async def test_claude_opus_4_6_in_pricing_table(self): - """anthropic/claude-opus-4.6 (default non-fast model) must be priced.""" - with patch( - "backend.copilot.baseline.service._fetch_openrouter_pricing", - AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), - ): - cost = await _estimate_cost_from_tokens( - "anthropic/claude-opus-4.6", 1000, 500 - ) - # 1000 * 15.0/1M + 500 * 75.0/1M = 0.015 + 0.0375 = 0.0525 - assert cost == pytest.approx(0.0525) - - @pytest.mark.asyncio - async def test_cache_read_tokens_use_openrouter_cache_rate_anthropic(self): - """Cache-read tokens use OpenRouter-published cache_read rate for Anthropic.""" - # anthropic/claude-sonnet-4: input=3/1M, cache_read=0.3/1M - # 200 regular + 800 cache-read prompt tokens, 0 completion - # cost = 200 * 3/1M + 800 * 0.3/1M = 0.0006 + 0.00024 = 0.00084 - with patch( - "backend.copilot.baseline.service._fetch_openrouter_pricing", - AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), - ): - cost = await _estimate_cost_from_tokens( - "anthropic/claude-sonnet-4", - prompt_tokens=1000, - completion_tokens=0, - cache_read_tokens=800, - ) - assert cost == pytest.approx(0.00084) - - @pytest.mark.asyncio - async def test_cache_read_tokens_use_openrouter_cache_rate_openai(self): - """Cache-read tokens use OpenRouter-published cache_read rate for OpenAI.""" - # openai/gpt-4o: input=2.5/1M, cache_read=1.25/1M - # 200 regular + 800 cache-read, 0 completion - # cost = 200 * 2.5/1M + 800 * 1.25/1M = 0.0005 + 0.001 = 0.0015 - with patch( - "backend.copilot.baseline.service._fetch_openrouter_pricing", - AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), - ): - cost = await _estimate_cost_from_tokens( - "openai/gpt-4o", - prompt_tokens=1000, - completion_tokens=0, - cache_read_tokens=800, - ) - assert cost == pytest.approx(0.0015) - - @pytest.mark.asyncio - async def test_cache_read_falls_back_to_input_rate_when_none(self): - """Models without a published cache_read rate fall back to the full input rate.""" - # openai/gpt-4o-mini: input=0.15/1M, cache_read=None (no discount published) - # 200 regular + 800 cache-read, 0 completion - # cost = 200 * 0.15/1M + 800 * 0.15/1M = 1000 * 0.15/1M = 0.00015 - with patch( - "backend.copilot.baseline.service._fetch_openrouter_pricing", - AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), - ): - cost = await _estimate_cost_from_tokens( - "openai/gpt-4o-mini", - prompt_tokens=1000, - completion_tokens=0, - cache_read_tokens=800, - ) - assert cost == pytest.approx(0.00015) - - class TestBaselineCostExtraction: """Tests for x-total-cost header extraction in _baseline_llm_caller.""" @@ -886,8 +771,8 @@ async def test_no_cost_when_api_call_raises_before_stream(self): assert state.cost_usd is None @pytest.mark.asyncio - async def test_no_cost_when_header_missing_and_pricing_unavailable(self): - """cost_usd remains None when x-total-cost is absent and pricing fetch returns empty.""" + async def test_no_cost_when_header_missing(self): + """cost_usd remains None when x-total-cost is absent.""" from backend.copilot.baseline.service import ( _baseline_llm_caller, _BaselineStreamState, @@ -915,15 +800,9 @@ async def chunk_aiter(): mock_client = MagicMock() mock_client.chat.completions.create = AsyncMock(return_value=mock_stream) - with ( - patch( - "backend.copilot.baseline.service._get_openai_client", - return_value=mock_client, - ), - patch( - "backend.copilot.baseline.service._fetch_openrouter_pricing", - AsyncMock(return_value={}), - ), + with patch( + "backend.copilot.baseline.service._get_openai_client", + return_value=mock_client, ): await _baseline_llm_caller( messages=[{"role": "user", "content": "hi"}], @@ -1063,15 +942,9 @@ async def chunk_aiter(): ] ) - with ( - patch( - "backend.copilot.baseline.service._get_openai_client", - return_value=mock_client, - ), - patch( - "backend.copilot.baseline.service._fetch_openrouter_pricing", - AsyncMock(return_value={}), - ), + with patch( + "backend.copilot.baseline.service._get_openai_client", + return_value=mock_client, ): await _baseline_llm_caller( messages=[{"role": "user", "content": "hi"}], @@ -1091,8 +964,12 @@ async def chunk_aiter(): assert state.turn_completion_tokens == 500 @pytest.mark.asyncio - async def test_cost_estimated_from_tokens_when_header_missing(self): - """cost_usd is estimated from token counts when x-total-cost is absent.""" + async def test_cost_usd_remains_none_when_header_missing(self): + """cost_usd stays None when x-total-cost header is absent. + + Token counts are still tracked; persist_and_record_usage handles + the None cost by falling back to tracking_type='tokens'. + """ from backend.copilot.baseline.service import ( _baseline_llm_caller, _BaselineStreamState, @@ -1120,93 +997,16 @@ async def chunk_aiter(): mock_client = MagicMock() mock_client.chat.completions.create = AsyncMock(return_value=mock_stream) - with ( - patch( - "backend.copilot.baseline.service._get_openai_client", - return_value=mock_client, - ), - patch( - "backend.copilot.baseline.service._fetch_openrouter_pricing", - AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), - ), - ): - await _baseline_llm_caller( - messages=[{"role": "user", "content": "hi"}], - tools=[], - state=state, - ) - - # Expected: 1000 * 3.0/1M + 500 * 15.0/1M = 0.003 + 0.0075 = 0.0105 - assert state.cost_usd == pytest.approx(0.0105) - - @pytest.mark.asyncio - async def test_multiturn_fallback_cost_uses_per_call_delta(self): - """Fallback cost estimation uses per-call token delta, not session total. - - On the second tool-call turn, the state accumulators already hold - tokens from turn 1. The estimator must charge only for the new tokens - reported in the current call, not the running total. - """ - from backend.copilot.baseline.service import ( - _baseline_llm_caller, - _BaselineStreamState, - ) - - state = _BaselineStreamState(model="anthropic/claude-sonnet-4") - - def make_stream_2(prompt_tokens: int, completion_tokens: int): - mock_raw = MagicMock() - mock_raw.headers = {} # no x-total-cost - mock_stream = MagicMock() - mock_stream._response = mock_raw - - mock_chunk = MagicMock() - mock_chunk.usage = MagicMock() - mock_chunk.usage.prompt_tokens = prompt_tokens - mock_chunk.usage.completion_tokens = completion_tokens - mock_chunk.usage.prompt_tokens_details = None - mock_chunk.choices = [] - - async def chunk_aiter(): - yield mock_chunk - - mock_stream.__aiter__ = lambda self: chunk_aiter() - return mock_stream - - mock_client = MagicMock() - mock_client.chat.completions.create = AsyncMock( - side_effect=[ - make_stream_2(1000, 200), - make_stream_2(1100, 300), - ] - ) - - with ( - patch( - "backend.copilot.baseline.service._get_openai_client", - return_value=mock_client, - ), - patch( - "backend.copilot.baseline.service._fetch_openrouter_pricing", - AsyncMock(return_value=_MOCK_OPENROUTER_PRICING), - ), + with patch( + "backend.copilot.baseline.service._get_openai_client", + return_value=mock_client, ): await _baseline_llm_caller( messages=[{"role": "user", "content": "hi"}], tools=[], state=state, ) - await _baseline_llm_caller( - messages=[{"role": "user", "content": "follow up"}], - tools=[], - state=state, - ) - # Turn 1: 1000 * 3.0/1M + 200 * 15.0/1M = 0.003 + 0.003 = 0.006 - # Turn 2: 1100 * 3.0/1M + 300 * 15.0/1M = 0.0033 + 0.0045 = 0.0078 - # Total: 0.0138 -- NOT 0.006 + cumulative (2100*3/1M + 500*15/1M) - expected = pytest.approx(0.006 + 0.0078, rel=1e-5) - assert state.cost_usd == expected - # Accumulators hold all tokens across both turns - assert state.turn_prompt_tokens == 2100 + assert state.cost_usd is None + assert state.turn_prompt_tokens == 1000 assert state.turn_completion_tokens == 500