From 2722c92e157ade14128e64aca599c854274d7ae0 Mon Sep 17 00:00:00 2001 From: Al4ise Date: Wed, 12 Aug 2026 17:43:15 +0300 Subject: [PATCH] feat(agents): migrate MCP runtime to protocol v2 --- CHANGELOG.md | 9 ++ docs/AI_TRADING_AGENTS.md | 6 + docs/MCP_2026_TRANSPORT.md | 34 +++++ docsrc/botspot_mcp.rst | 18 ++- lumibot/components/agents/runtime.py | 160 ++++++++------------- requirements.txt | 3 +- setup.py | 3 +- tests/test_agent_runtime_mcp_transports.py | 44 +++--- tests/test_agent_runtime_remote_mcp.py | 41 +++++- 9 files changed, 187 insertions(+), 131 deletions(-) create mode 100644 docs/MCP_2026_TRANSPORT.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d6d9c0490..8a1f06f03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +### Changed +- **External MCP tools now use MCP 2026-07-28 through the official Python v2 + client.** HTTP and stdio connections negotiate with ``server/discover``, send + stateless requests, and no longer depend on deprecated explicit + ``initialize`` or session transport behavior. Auto negotiation preserves a + rolling compatibility path for older conforming servers. + ## 4.5.83 - 2026-08-05 Deploy marker: `deploy 4.5.83` diff --git a/docs/AI_TRADING_AGENTS.md b/docs/AI_TRADING_AGENTS.md index 6dfec062e..9339628f7 100644 --- a/docs/AI_TRADING_AGENTS.md +++ b/docs/AI_TRADING_AGENTS.md @@ -147,6 +147,12 @@ MCPServer( Any MCP server with a URL that speaks the Model Context Protocol over HTTP works. There are over 20,000 available today. +LumiBot uses the official Python MCP v2 client and negotiates the stateless +``2026-07-28`` contract with ``server/discover``. Modern HTTP requests are +POST-only and do not create an MCP session or send ``initialize``. During the +ecosystem transition, automatic negotiation can still connect to conforming +legacy servers. + --- ## Built-in Tools diff --git a/docs/MCP_2026_TRANSPORT.md b/docs/MCP_2026_TRANSPORT.md new file mode 100644 index 000000000..5b5c09cc9 --- /dev/null +++ b/docs/MCP_2026_TRANSPORT.md @@ -0,0 +1,34 @@ +# MCP 2026 Transport + +One transport contract for LumiBot external agent tools. + +Last Updated: 2026-08-12 +Status: Implemented +Audience: LumiBot contributors and integrators + +## Overview + +LumiBot's external MCP runtime uses `mcp>=2,<3` and `httpx2`. HTTP and stdio +servers share one official `Client(mode="auto")` path. Modern peers negotiate +`2026-07-28` with `server/discover`; the v2 client owns request headers, +per-request metadata, response validation, and rolling fallback for older +conforming servers. + +Modern HTTP behavior: + +- POST-only JSON-RPC requests. +- `Mcp-Protocol-Version`, `Mcp-Method`, and tool-call `Mcp-Name` headers. +- Protocol version, client identity, and capabilities in request `_meta`. +- No `initialize`, `notifications/initialized`, `Mcp-Session-Id`, standalone + GET event stream, or resumability state. + +`MCPServer.headers` and `auth_token_env` still add application authentication +headers. `timeout_seconds`, `sse_read_timeout_seconds`, and +`terminate_on_close` retain their public meanings. No global TLS monkeypatch or +parallel raw JSON-RPC HTTP implementation remains. + +## Verification + +Focused transport coverage runs real MCP v2 stdio and Streamable HTTP servers, +plus a wire-level remote fixture that asserts discovery, modern headers, +request metadata, typed results, and absence of sessions or initialization. diff --git a/docsrc/botspot_mcp.rst b/docsrc/botspot_mcp.rst index 7a114e0b3..454145332 100644 --- a/docsrc/botspot_mcp.rst +++ b/docsrc/botspot_mcp.rst @@ -22,7 +22,7 @@ and verification commands. Last verified ------------- -- This doc was updated for the current production MCP behavior on ``March 16, 2026``. +- This client guidance was updated for MCP ``2026-07-28`` on ``August 12, 2026``. - Canonical host is ``https://mcp.botspot.trade``. Canonical endpoints @@ -35,9 +35,11 @@ Canonical endpoints Important transport note ------------------------ -BotSpot runs MCP over HTTP JSON-RPC (POST). ``GET /mcp`` now returns a small -capability JSON document for connector reachability checks. -Tool execution still uses ``POST /mcp``. +BotSpot uses the stateless MCP ``2026-07-28`` HTTP contract. Protocol traffic is +POST-only. Clients negotiate with ``server/discover`` and send protocol, +method, and optional tool-name headers plus per-request ``_meta``. Modern clients +do not send ``initialize``, create an MCP session, open a standalone GET stream, +or use transport resumability. Authentication modes -------------------- @@ -152,7 +154,9 @@ Validation checklist (copy/paste) curl -i -X POST https://mcp.botspot.trade/mcp \ -H "Content-Type: application/json" \ - --data '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' + -H "Mcp-Protocol-Version: 2026-07-28" \ + -H "Mcp-Method: server/discover" \ + --data '{"jsonrpc":"2.0","id":1,"method":"server/discover","params":{"_meta":{"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientInfo":{"name":"curl-probe","version":"1.0.0"},"io.modelcontextprotocol/clientCapabilities":{}}}}' 3) Authenticated MCP call (replace key) should return tool list/result: @@ -161,7 +165,9 @@ Validation checklist (copy/paste) curl -i -X POST https://mcp.botspot.trade/mcp \ -H "Authorization: Bearer botspot_YOUR_API_KEY" \ -H "Content-Type: application/json" \ - --data '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' + -H "Mcp-Protocol-Version: 2026-07-28" \ + -H "Mcp-Method: tools/list" \ + --data '{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{"_meta":{"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientInfo":{"name":"curl-probe","version":"1.0.0"},"io.modelcontextprotocol/clientCapabilities":{}}}}' Local debugging (recommended) ----------------------------- diff --git a/lumibot/components/agents/runtime.py b/lumibot/components/agents/runtime.py index 74fc3b3c6..f21ca8728 100644 --- a/lumibot/components/agents/runtime.py +++ b/lumibot/components/agents/runtime.py @@ -1,12 +1,12 @@ from __future__ import annotations -import contextlib import asyncio +import contextlib import hashlib import importlib -import logging -import json import inspect +import json +import logging import math import os import re @@ -23,33 +23,31 @@ from .schemas import AgentRunResult, AgentTraceEvent, BoundTool, MCPServer from .tool_context import agent_tool_context - _GOOGLE_SDK_NOISE_FILTERS_CONFIGURED = False -ClientSession = None +MCPClient = None +MCPImplementation = None StdioServerParameters = None stdio_client = None streamablehttp_client = None -streamablehttp_client_uses_http_client = False def _ensure_mcp_client_imports(): - global ClientSession, StdioServerParameters, stdio_client - global streamablehttp_client, streamablehttp_client_uses_http_client - if ClientSession is None or StdioServerParameters is None: - from mcp import ClientSession as _ClientSession, StdioServerParameters as _StdioServerParameters - - ClientSession = _ClientSession + global MCPClient, MCPImplementation, StdioServerParameters, stdio_client + global streamablehttp_client + if MCPClient is None or MCPImplementation is None or StdioServerParameters is None: + from mcp.client import Client as _MCPClient + from mcp.client.stdio import StdioServerParameters as _StdioServerParameters + from mcp_types import Implementation as _MCPImplementation + + MCPClient = _MCPClient + MCPImplementation = _MCPImplementation StdioServerParameters = _StdioServerParameters if stdio_client is None: from mcp.client.stdio import stdio_client as _stdio_client stdio_client = _stdio_client if streamablehttp_client is None: - try: - from mcp.client.streamable_http import streamable_http_client as _streamablehttp_client - streamablehttp_client_uses_http_client = True - except ImportError: - from mcp.client.streamable_http import streamablehttp_client as _streamablehttp_client + from mcp.client.streamable_http import streamable_http_client as _streamablehttp_client streamablehttp_client = _streamablehttp_client @@ -1542,9 +1540,23 @@ def _jsonable(value: Any) -> Any: return str(value) -async def _with_mcp_session(server: MCPServer, callback): +def _mcp_model_dump(value: Any) -> Any: + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + return model_dump(by_alias=True, exclude_none=True, mode="json") + return _jsonable(value) + + +async def _with_mcp_client(server: MCPServer, callback): _ensure_mcp_client_imports() transport = (server.transport or "http").lower().replace("-", "_") + from lumibot import __version__ + + client_kwargs = { + "mode": "auto", + "read_timeout_seconds": server.sse_read_timeout_seconds, + "client_info": MCPImplementation(name="lumibot", version=__version__), + } if transport == "stdio": parameters = StdioServerParameters( command=str(server.command), @@ -1553,39 +1565,28 @@ async def _with_mcp_session(server: MCPServer, callback): cwd=server.cwd, ) with _mcp_errlog_stream() as errlog: - async with stdio_client(parameters, errlog=errlog) as (read_stream, write_stream): - async with ClientSession(read_stream, write_stream) as session: - await session.initialize() - return await callback(session) - - headers = _mcp_headers(server) - timeout = server.timeout_seconds - sse_timeout = server.sse_read_timeout_seconds - if streamablehttp_client_uses_http_client: - import httpx - from mcp.shared._httpx_utils import create_mcp_http_client - - http_timeout = httpx.Timeout(timeout, read=sse_timeout) - async with create_mcp_http_client(headers=headers, timeout=http_timeout) as http_client: - async with streamablehttp_client( - str(server.url), - http_client=http_client, - terminate_on_close=server.terminate_on_close, - ) as (read_stream, write_stream, _get_session_id): - async with ClientSession(read_stream, write_stream) as session: - await session.initialize() - return await callback(session) - else: - async with streamablehttp_client( + mcp_transport = stdio_client(parameters, errlog=errlog) + async with MCPClient(mcp_transport, **client_kwargs) as client: + return await callback(client) + + import httpx2 + + http_timeout = httpx2.Timeout( + server.timeout_seconds, + read=server.sse_read_timeout_seconds, + ) + async with httpx2.AsyncClient( + headers=_mcp_headers(server), + timeout=http_timeout, + follow_redirects=True, + ) as http_client: + mcp_transport = streamablehttp_client( str(server.url), - headers=headers, - timeout=timeout, - sse_read_timeout=sse_timeout, + http_client=http_client, terminate_on_close=server.terminate_on_close, - ) as (read_stream, write_stream, _get_session_id): - async with ClientSession(read_stream, write_stream) as session: - await session.initialize() - return await callback(session) + ) + async with MCPClient(mcp_transport, **client_kwargs) as client: + return await callback(client) def _run_mcp_sync(async_fn, *args): @@ -1604,72 +1605,27 @@ def _run_mcp_sync(async_fn, *args): async def _list_mcp_tools_async(server: MCPServer) -> list[dict[str, Any]]: - transport = (server.transport or "http").lower().replace("-", "_") - async def callback(session: ClientSession) -> list[dict[str, Any]]: - result = await session.list_tools() + async def callback(client: Any) -> list[dict[str, Any]]: + result = await client.list_tools() tools = getattr(result, "tools", None) or [] normalized: list[dict[str, Any]] = [] for tool in tools: - dumped = _jsonable(tool) + dumped = _mcp_model_dump(tool) if isinstance(dumped, dict): normalized.append(dumped) return normalized - if transport == "http": - return await _legacy_http_list_tools(server) - return await _with_mcp_session(server, callback) + return await _with_mcp_client(server, callback) async def _call_mcp_tool_async(server: MCPServer, name: str, arguments: dict[str, Any]) -> dict[str, Any]: - transport = (server.transport or "http").lower().replace("-", "_") - async def callback(session: ClientSession) -> dict[str, Any]: - result = await session.call_tool(name, arguments or {}) - dumped = _jsonable(result) + async def callback(client: Any) -> dict[str, Any]: + result = await client.call_tool(name, arguments or {}) + dumped = _mcp_model_dump(result) if not isinstance(dumped, dict): raise RuntimeError(f"{name} returned unexpected payload: {dumped!r}") if dumped.get("isError") is True: raise RuntimeError(f"{name} failed: {dumped}") return dumped - if transport == "http": - return await _legacy_http_call_tool(server, name, arguments) - return await _with_mcp_session(server, callback) - - -async def _legacy_http_list_tools(server: MCPServer) -> list[dict[str, Any]]: - import httpx - - payload = { - "jsonrpc": "2.0", - "id": "tools-list", - "method": "tools/list", - "params": {}, - } - async with httpx.AsyncClient(timeout=server.timeout_seconds) as client: - response = await client.post(str(server.url), json=payload, headers=_mcp_headers(server)) - response.raise_for_status() - data = response.json() - result = data.get("result") or {} - tools = result.get("tools") or [] - return tools if isinstance(tools, list) else [] - - -async def _legacy_http_call_tool(server: MCPServer, name: str, arguments: dict[str, Any]) -> dict[str, Any]: - import httpx - - payload = { - "jsonrpc": "2.0", - "id": f"{name}-call", - "method": "tools/call", - "params": {"name": name, "arguments": arguments}, - } - async with httpx.AsyncClient(timeout=server.timeout_seconds) as client: - response = await client.post(str(server.url), json=payload, headers=_mcp_headers(server)) - response.raise_for_status() - data = response.json() - if "error" in data: - raise RuntimeError(f"{name} failed: {data['error']}") - result = data.get("result") or {} - if not isinstance(result, dict): - raise RuntimeError(f"{name} returned unexpected payload: {result!r}") - return result + return await _with_mcp_client(server, callback) diff --git a/requirements.txt b/requirements.txt index 9fec59594..0079abfe7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,7 +42,8 @@ google-adk[extensions]>=2.0.0,<3.0.0 google-genai>=1.72.0,<2.0.0 litellm>=1.83.7,<=1.83.14 anyio>=4.10.0 -mcp>=1.26.0,<2 +mcp>=2.0.0,<3 +httpx2>=2.10.0,<3 schwab-py>=1.5.0 Flask>=2.3 free-proxy diff --git a/setup.py b/setup.py index 21c02debd..79b15c999 100644 --- a/setup.py +++ b/setup.py @@ -101,7 +101,8 @@ def _maybe_copy_theta_terminal(self): "google-genai>=1.72.0,<2.0.0", "litellm>=1.83.7,<=1.83.14", "anyio>=4.10.0", - "mcp>=1.26.0,<2", + "mcp>=2.0.0,<3", + "httpx2>=2.10.0,<3", "schwab-py>=1.5.0", "Flask>=2.3", "free-proxy", diff --git a/tests/test_agent_runtime_mcp_transports.py b/tests/test_agent_runtime_mcp_transports.py index 643616f1e..d2993f0f7 100644 --- a/tests/test_agent_runtime_mcp_transports.py +++ b/tests/test_agent_runtime_mcp_transports.py @@ -1,30 +1,29 @@ +import json import socket import subprocess import sys import tempfile import textwrap import time -import json from datetime import datetime, timezone from pathlib import Path import pandas as pd import pytest +import lumibot.components.agents.runtime as runtime_module from lumibot.backtesting import PandasDataBacktesting from lumibot.components.agents import AgentRunResult, MCPServer from lumibot.entities import Asset, Data from lumibot.strategies import Strategy -import lumibot.components.agents.runtime as runtime_module - SERVER_SCRIPT = """ import argparse from datetime import datetime, timezone -from mcp.server.fastmcp import FastMCP +from mcp.server import MCPServer as SDKMCPServer -mcp = FastMCP("transport-test", host="127.0.0.1", port=0) +mcp = SDKMCPServer("transport-test") @mcp.tool() def echo_market_state(symbol: str = "TEST", note: str = "") -> dict: @@ -39,9 +38,14 @@ def echo_market_state(symbol: str = "TEST", note: str = "") -> dict: parser.add_argument("--transport", choices=["stdio", "streamable-http"], default="stdio") parser.add_argument("--port", type=int, default=8765) args = parser.parse_args() - if args.transport == "streamable-http": - mcp.settings.port = args.port - mcp.run(transport=args.transport) + mcp.run( + transport=args.transport, + host="127.0.0.1", + port=args.port, + streamable_http_path="/mcp", + stateless_http=True, + json_response=True, + ) """ @@ -246,16 +250,17 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc, tb): return False - class DummySession: + # MCP 2 clients own negotiation; deprecated ClientSession.initialize() no longer exists. + class DummyClient: + def __init__(self, *_args, **_kwargs): + pass + async def __aenter__(self): return self async def __aexit__(self, exc_type, exc, tb): return False - async def initialize(self): - return None - async def list_tools(self): return type("Result", (), {"tools": []})() @@ -264,7 +269,8 @@ def fake_stdio_client(parameters, errlog=None): return DummyStdioContext() monkeypatch.setattr(runtime_module, "stdio_client", fake_stdio_client) - monkeypatch.setattr(runtime_module, "ClientSession", lambda read_stream, write_stream: DummySession()) + monkeypatch.setattr(runtime_module, "MCPClient", DummyClient) + monkeypatch.setattr(runtime_module, "MCPImplementation", lambda **_kwargs: object()) monkeypatch.setenv("IS_BACKTESTING", "true") monkeypatch.setenv("BACKTESTING_QUIET_LOGS", "true") @@ -293,16 +299,17 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc, tb): return False - class DummySession: + # MCP 2 clients own negotiation; deprecated ClientSession.initialize() no longer exists. + class DummyClient: + def __init__(self, *_args, **_kwargs): + pass + async def __aenter__(self): return self async def __aexit__(self, exc_type, exc, tb): return False - async def initialize(self): - return None - async def list_tools(self): return type("Result", (), {"tools": []})() @@ -311,7 +318,8 @@ def fake_stdio_client(parameters, errlog=None): return DummyStdioContext() monkeypatch.setattr(runtime_module, "stdio_client", fake_stdio_client) - monkeypatch.setattr(runtime_module, "ClientSession", lambda read_stream, write_stream: DummySession()) + monkeypatch.setattr(runtime_module, "MCPClient", DummyClient) + monkeypatch.setattr(runtime_module, "MCPImplementation", lambda **_kwargs: object()) monkeypatch.setenv("IS_BACKTESTING", "true") monkeypatch.setenv("BACKTESTING_QUIET_LOGS", "false") diff --git a/tests/test_agent_runtime_remote_mcp.py b/tests/test_agent_runtime_remote_mcp.py index 1ea25cc03..501345e72 100644 --- a/tests/test_agent_runtime_remote_mcp.py +++ b/tests/test_agent_runtime_remote_mcp.py @@ -71,13 +71,27 @@ def on_trading_iteration(self): class _MCPHandler(BaseHTTPRequestHandler): calls = [] + requests = [] def do_POST(self): length = int(self.headers.get("Content-Length", "0")) body = self.rfile.read(length) data = json.loads(body.decode("utf-8")) method = data.get("method") - if method == "tools/list": + self.__class__.requests.append({"body": data, "headers": dict(self.headers)}) + if method == "server/discover": + response = { + "jsonrpc": "2.0", + "id": data.get("id"), + "result": { + "supportedVersions": ["2026-07-28"], + "capabilities": {"tools": {}}, + "resultType": "complete", + "ttlMs": 0, + "cacheScope": "private", + }, + } + elif method == "tools/list": response = { "jsonrpc": "2.0", "id": data.get("id"), @@ -86,8 +100,12 @@ def do_POST(self): { "name": "echo_market_state", "description": "Echo a small structured market state payload.", + "inputSchema": {"type": "object"}, } - ] + ], + "resultType": "complete", + "ttlMs": 0, + "cacheScope": "private", }, } elif method == "tools/call": @@ -96,10 +114,12 @@ def do_POST(self): "jsonrpc": "2.0", "id": data.get("id"), "result": { + "content": [{"type": "text", "text": "stub-mcp-ok"}], "structuredContent": { "message": "stub-mcp-ok", "arguments": data.get("params", {}).get("arguments", {}), - } + }, + "resultType": "complete", }, } else: @@ -120,6 +140,7 @@ def mcp_server(): server = ThreadingHTTPServer(("127.0.0.1", 0), _MCPHandler) thread = threading.Thread(target=server.serve_forever, daemon=True) _MCPHandler.calls = [] + _MCPHandler.requests = [] thread.start() try: yield f"http://127.0.0.1:{server.server_port}/mcp" @@ -162,6 +183,20 @@ def test_external_mcp_tool_is_allowlisted_and_invoked(monkeypatch, tmp_path, mcp assert _MCPHandler.calls assert strategy.vars.agent_result == "Remote MCP said: stub-mcp-ok" assert strategy.vars.agent_warnings + methods = [request["body"]["method"] for request in _MCPHandler.requests] + assert "server/discover" in methods + assert "initialize" not in methods + tool_call = next( + request for request in _MCPHandler.requests if request["body"]["method"] == "tools/call" + ) + normalized_headers = {key.lower(): value for key, value in tool_call["headers"].items()} + assert normalized_headers["mcp-protocol-version"] == "2026-07-28" + assert normalized_headers["mcp-method"] == "tools/call" + assert normalized_headers["mcp-name"] == "echo_market_state" + assert "mcp-session-id" not in normalized_headers + assert tool_call["body"]["params"]["_meta"][ + "io.modelcontextprotocol/protocolVersion" + ] == "2026-07-28" @pytest.mark.usefixtures("disable_datasource_override")