From cf8e72838e180c13e336a37283547debe1f7acd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20=C5=81ukasiak?= Date: Thu, 5 Feb 2026 13:41:27 +0100 Subject: [PATCH] Add SDK-free custom provider example --- examples/custom_provider/GUIDE.md | 144 ++++++++++++++++++++++++++ examples/custom_provider/README.md | 9 ++ examples/custom_provider/providers.py | 137 ++++++++++++++++++++++++ 3 files changed, 290 insertions(+) create mode 100644 examples/custom_provider/GUIDE.md diff --git a/examples/custom_provider/GUIDE.md b/examples/custom_provider/GUIDE.md new file mode 100644 index 00000000..13832b54 --- /dev/null +++ b/examples/custom_provider/GUIDE.md @@ -0,0 +1,144 @@ +# Custom Provider Guide (No SDK) + +This guide explains what the `http-json` example provider implements and what you need to build when wiring a custom provider that **does not rely on any LLM SDKs**. The goal is to show how to connect `llm-do` + PydanticAI to a plain HTTP endpoint (OpenAI-compatible or similar) using only common HTTP libraries. + +## What This Example Implements + +The example in `providers.py` adds **two** providers: + +1. **`ollama-local`** (SDK-backed) using `OpenAIChatModel`. +2. **`http-json`** (SDK-free) using a custom `SimpleHTTPChatModel` that issues raw HTTP requests with `httpx`. + +The SDK-free version demonstrates the minimum surface area you must implement: + +- A provider class that stores base URL/auth info. +- A model class that subclasses `pydantic_ai.models.Model` and implements `request()`. +- Mapping between `PydanticAI` message objects and your HTTP provider’s request schema. +- Parsing the provider response back into a `ModelResponse` with text parts. + +## Provider Responsibilities + +**Provider = configuration + HTTP client.** + +The `SimpleHTTPProvider` encapsulates: + +- `base_url`: Where HTTP requests are sent (e.g., `http://127.0.0.1:8000/v1`). +- `api_key`: Optional bearer token for auth headers. +- `name`: A stable identifier used in responses (e.g., `http-json`). +- `httpx.AsyncClient`: A reusable async HTTP client. + +If you swap in another HTTP client (`urllib`, `requests`, `aiohttp`), the model logic stays the same—only the transport changes. + +## Model Responsibilities + +A custom model must subclass `pydantic_ai.models.Model` and implement: + +- `model_name`: The provider-side model name (e.g., `your-model`). +- `system`: A standardized provider id (the example uses `openai` because the API is OpenAI-compatible). +- `request()`: The real HTTP call with request/response translation. + +### Message Mapping + +PydanticAI hands you a list of `ModelMessage` objects. In the example, we only support: + +- `ModelRequest` with `UserPromptPart` (user input) +- `ModelRequest` with `SystemPromptPart` (system input) +- `ModelResponse` with `TextPart` (assistant history) + +If your provider accepts OpenAI-style messages, you need to map these into: + +```json +{ + "role": "user" | "system" | "assistant", + "content": "..." +} +``` + +The example does the following: + +- Uses `Model._get_instructions()` to inject system instructions. +- Converts each `UserPromptPart` into a `user` message. +- Converts each `SystemPromptPart` into a `system` message. +- Converts each prior `ModelResponse` into an `assistant` message (text only). + +### Output + Tool Constraints + +The SDK-free example intentionally keeps scope minimal: + +- **Only text output is supported.** If tool calls or structured output are requested, it raises a `UserError`. +- **No streaming.** Streaming is possible but not shown in this minimal example. +- **No tool calling.** If you want tools, you must translate `function_tools` + tool schemas into your provider’s format. + +## Request Construction + +The `http-json` example posts to `/chat/completions` with a payload like: + +```json +{ + "model": "your-model", + "messages": [...], + "temperature": 0.7, + "max_tokens": 400 +} +``` + +Only a small set of optional settings are passed through: + +- `temperature` +- `max_tokens` +- `top_p` +- `presence_penalty` +- `frequency_penalty` + +If your endpoint supports different fields, you can map them in the same spot. + +## Response Parsing + +The model expects an OpenAI-style response structure: + +```json +{ + "choices": [ + {"message": {"content": "..."}} + ] +} +``` + +It extracts the first `choices[0].message.content` and returns a `ModelResponse` with a single `TextPart`. + +If your provider returns a different schema, transform it here and set `parts` accordingly. + +## Configuration in `providers.py` + +The `register_model_factory` call connects a provider prefix (e.g., `http-json`) to a factory function that builds the model: + +```python +register_model_factory("http-json", build_http_json_model) +``` + +Usage becomes: + +```bash +export LLM_DO_MODEL="http-json:your-model" +llm-do examples/custom_provider/project.json "Hello!" +``` + +If you need environment-driven configuration, update the provider factory to read from env vars before instantiating the provider. + +## Extending This Example + +When you need more features, consider adding: + +- **Tool support:** Map `function_tools` to your provider’s tool schema and parse tool calls back into `ToolCallPart`. +- **Streaming:** Implement `request_stream()` and return `StreamedResponse` objects. +- **Token usage:** Populate `RequestUsage` by parsing `usage` fields from your API responses. +- **Error mapping:** Translate provider-specific error payloads into `UserError` with actionable messages. + +## Checklist: Minimal SDK-Free Provider + +- [ ] Provider object that stores base URL, API key, name, and HTTP client. +- [ ] Model subclass implementing `request()`. +- [ ] Model message mapping for system/user/assistant history. +- [ ] Response parsing into `ModelResponse` + `TextPart`. +- [ ] `register_model_factory()` with a custom provider prefix. +- [ ] README instructions + example run commands. diff --git a/examples/custom_provider/README.md b/examples/custom_provider/README.md index eb80586e..af35f686 100644 --- a/examples/custom_provider/README.md +++ b/examples/custom_provider/README.md @@ -5,8 +5,10 @@ This example shows how to register a custom provider and use it via `LLM_DO_MODE ## Files - `providers.py` registers a model factory for the provider prefix `ollama-local`. +- `providers.py` also registers a `http-json` provider that calls an OpenAI-compatible endpoint with raw HTTP. - `main.agent` is a minimal agent with no model specified. - `project.json` loads both the agent and provider registration code. +- `GUIDE.md` is a detailed guide for building custom providers without SDKs. ## Run @@ -18,3 +20,10 @@ llm-do examples/custom_provider/project.json "Hello!" ``` The base URL is intentionally hard-coded in `providers.py` for simplicity. + +To try the raw HTTP provider, point it at a compatible endpoint: + +```bash +export LLM_DO_MODEL="http-json:your-model" +llm-do examples/custom_provider/project.json "Hello!" +``` diff --git a/examples/custom_provider/providers.py b/examples/custom_provider/providers.py index bc8259ea..727d0fb1 100644 --- a/examples/custom_provider/providers.py +++ b/examples/custom_provider/providers.py @@ -1,3 +1,18 @@ +import httpx +from pydantic_ai.exceptions import UserError +from pydantic_ai.messages import ( + ModelRequest, + ModelResponse, + SystemPromptPart, + TextPart, + UserPromptPart, +) +from pydantic_ai.models import ( + Model, + ModelMessage, + ModelRequestParameters, + ModelSettings, +) from pydantic_ai.models.openai import OpenAIChatModel from llm_do import register_model_factory @@ -17,3 +32,125 @@ def build_ollama_model(model_name: str) -> OpenAIChatModel: register_model_factory("ollama-local", build_ollama_model) + + +class SimpleHTTPProvider: + """Minimal provider that uses plain HTTP requests (no SDKs).""" + + def __init__(self, *, base_url: str, api_key: str | None = None, name: str = "http-json") -> None: + self._base_url = base_url.rstrip("/") + self._api_key = api_key + self._name = name + self._client = httpx.AsyncClient(timeout=30.0) + + @property + def name(self) -> str: + return self._name + + @property + def base_url(self) -> str: + return self._base_url + + @property + def api_key(self) -> str | None: + return self._api_key + + @property + def client(self) -> httpx.AsyncClient: + return self._client + + +class SimpleHTTPChatModel(Model): + """Simple OpenAI-compatible chat model implemented with raw HTTP calls.""" + + def __init__(self, model_name: str, *, provider: SimpleHTTPProvider) -> None: + super().__init__() + self._model_name = model_name + self._provider = provider + + @property + def model_name(self) -> str: + return self._model_name + + @property + def system(self) -> str: + return "openai" + + @property + def base_url(self) -> str | None: + return self._provider.base_url + + async def request( + self, + messages: list[ModelMessage], + model_settings: ModelSettings | None, + model_request_parameters: ModelRequestParameters, + ) -> ModelResponse: + model_settings, model_request_parameters = self.prepare_request(model_settings, model_request_parameters) + + if model_request_parameters.function_tools or model_request_parameters.output_tools: + raise UserError("SimpleHTTPChatModel does not support tool calls. Use a provider with tool support.") + if not model_request_parameters.allow_text_output or model_request_parameters.output_mode != "text": + raise UserError("SimpleHTTPChatModel only supports plain text output.") + + payload_messages: list[dict[str, str]] = [] + instructions = self._get_instructions(messages, model_request_parameters) + if instructions: + payload_messages.append({"role": "system", "content": instructions}) + + for message in messages: + if isinstance(message, ModelRequest): + for part in message.parts: + if isinstance(part, SystemPromptPart): + payload_messages.append({"role": "system", "content": part.content}) + elif isinstance(part, UserPromptPart): + if not isinstance(part.content, str): + raise UserError("SimpleHTTPChatModel only supports string user prompts.") + payload_messages.append({"role": "user", "content": part.content}) + elif isinstance(message, ModelResponse): + if message.text: + payload_messages.append({"role": "assistant", "content": message.text}) + + payload: dict[str, object] = { + "model": self.model_name, + "messages": payload_messages, + } + for key in ("temperature", "max_tokens", "top_p", "presence_penalty", "frequency_penalty"): + if model_settings and key in model_settings: + payload[key] = model_settings[key] + + headers = {"Content-Type": "application/json"} + if self._provider.api_key: + headers["Authorization"] = f"Bearer {self._provider.api_key}" + + response = await self._provider.client.post( + f"{self._provider.base_url}/chat/completions", + json=payload, + headers=headers, + ) + response.raise_for_status() + data = response.json() + content = ( + data.get("choices", [{}])[0] + .get("message", {}) + .get("content", "") + ) + return ModelResponse( + parts=[TextPart(content=content or "")], + model_name=self.model_name, + provider_name=self._provider.name, + provider_details={"raw_response": data}, + ) + + +def build_http_json_model(model_name: str) -> SimpleHTTPChatModel: + return SimpleHTTPChatModel( + model_name, + provider=SimpleHTTPProvider( + base_url="http://127.0.0.1:8000/v1", + api_key=None, + ), + ) + + +register_model_factory("http-json", build_http_json_model)