From 540c4aabab1caf2ed35bfa3e565402e14bebc765 Mon Sep 17 00:00:00 2001 From: Jeffrey <1187141927@qq.com> Date: Tue, 3 Nov 2026 16:52:54 +0800 Subject: [PATCH 1/7] plugin/toolfilter: add decision-based filtering and routing Add System One tool and skill filtering with lexical and optional embedding recall, bounded concurrent caching, and fail-open decisions. Add invocation-scoped model routing, structured observer reports, and concurrent decision replay. Document the design and provide a runnable example. Cover validation boundaries, filtering outcomes, caching, cancellation, observer reentrancy, and concurrency with tests. --- docs/design/decision-model.md | 873 +++++++++++++++++++++++++ examples/decisionfilter/README.md | 94 +++ examples/decisionfilter/main.go | 496 ++++++++++++++ plugin/toolfilter/conversation.go | 103 +++ plugin/toolfilter/conversation_test.go | 95 +++ plugin/toolfilter/doc.go | 42 ++ plugin/toolfilter/modelrouter.go | 147 +++++ plugin/toolfilter/modelrouter_test.go | 218 ++++++ plugin/toolfilter/options.go | 278 ++++++++ plugin/toolfilter/recall.go | 306 +++++++++ plugin/toolfilter/recall_test.go | 341 ++++++++++ plugin/toolfilter/report.go | 74 +++ plugin/toolfilter/runcache.go | 120 ++++ plugin/toolfilter/runcache_test.go | 71 ++ plugin/toolfilter/skillfilter.go | 372 +++++++++++ plugin/toolfilter/skillfilter_test.go | 415 ++++++++++++ plugin/toolfilter/testhelpers_test.go | 232 +++++++ plugin/toolfilter/toolcallback.go | 437 +++++++++++++ plugin/toolfilter/toolcallback_test.go | 676 +++++++++++++++++++ 19 files changed, 5390 insertions(+) create mode 100644 docs/design/decision-model.md create mode 100644 examples/decisionfilter/README.md create mode 100644 examples/decisionfilter/main.go create mode 100644 plugin/toolfilter/conversation.go create mode 100644 plugin/toolfilter/conversation_test.go create mode 100644 plugin/toolfilter/doc.go create mode 100644 plugin/toolfilter/modelrouter.go create mode 100644 plugin/toolfilter/modelrouter_test.go create mode 100644 plugin/toolfilter/options.go create mode 100644 plugin/toolfilter/recall.go create mode 100644 plugin/toolfilter/recall_test.go create mode 100644 plugin/toolfilter/report.go create mode 100644 plugin/toolfilter/runcache.go create mode 100644 plugin/toolfilter/runcache_test.go create mode 100644 plugin/toolfilter/skillfilter.go create mode 100644 plugin/toolfilter/skillfilter_test.go create mode 100644 plugin/toolfilter/testhelpers_test.go create mode 100644 plugin/toolfilter/toolcallback.go create mode 100644 plugin/toolfilter/toolcallback_test.go diff --git a/docs/design/decision-model.md b/docs/design/decision-model.md new file mode 100644 index 0000000000..b122ed31db --- /dev/null +++ b/docs/design/decision-model.md @@ -0,0 +1,873 @@ +# tRPC-Agent-Go 决策模型集成设计文档（v2，社区对齐版） + +> **v2 修订说明（2026-11-01）**：本版全面重写。v1 假设"决策模型集成层"是空白， +> 自建了 `decision` 抽象层 + `decision/jev` + `decision/mock` + `graph.AddDecisionNode`。 +> 重写前的事实核查（附录 B）发现社区路线已与此冲突： +> +> - 提案 issue #2632 明确 Non-Goal："Adding a new generic decision-model +> abstraction"（不做通用决策模型抽象层），并确定用既有槽位集成； +> - `internal/systemone` 协议客户端已由 PR #2652 合入（wangxuw，2026-10-10）； +> - 工具审批 reviewer（v1 三期的风险门控场景）正由 PR #2664 在审； +> - 评估器由 PR #2638 占位（liuzengh）。 +> +> 结论：v1 的一期（抽象层 + jev 客户端 + mock + AddDecisionNode）与社区路线 +> 重叠或被否决，本地实现已撤销。v2 重新分期： +> +> ```text +> 第一期：社区基础（已有实现 + 在途 PR + 规则约束）—— 不写代码，只锚定地基 +> 第二期：toolfilter / skillfilter / modelrouter —— 我们的核心贡献（无人认领的空白） +> 第三期：出口质检 + 提案剩余示例 + 观测收尾 +> ``` +> +> v1 中仍然有效的资产：附录 A 的 Octop 生产调研（二期 toolfilter 的全部依据）、 +> 执行流程与召回层设计（骨架换为 systemone 客户端后逻辑不变）、失败语义纪律 +> （筛选 fail-open / 安全 fail-closed）。均已并入 v2。 + +--- + +## 一、第一期：社区基础（现状 + 规则，不写代码） + +第一期在 v2 中不再是交付物，而是**必须锚定的地基**：了解社区已有什么、 +正在交付什么、遵守哪些约束。本期的行动项只有两个：在 issue #2632 下提出 +二期方向并获得认可（见 2.0），以及可选地认领提案内无人认领的两个示例 +（见 3.2）。 + +### 1.1 提案 #2632 与社区路线图 + +[issue #2632](https://github.com/trpc-group/trpc-agent-go/issues/2632) +（*proposal: support TypeSafe Jev for structured decisions*，作者 wangxuw， +2026-09-21 开启，状态 open）是社区 Jev 集成的总路线图。其 scope： + +- 带重试的 HTTP 客户端（已在 PR #2652 落地，注：落地版无重试，重试归上层）； +- 三种问题类型，保留概率、置信度、用量与解析出的模型版本； +- "把 Jev 决策包装成 callable tool"的示例（**无人认领**）； +- "Graph 按决策 + 可配置置信度阈值路由"的示例（**无人认领**）； +- Jev 版 `review.Reviewer` 工具审批实现（PR #2664 在审）； +- 协议与集成测试 + 双语文档。 + +提案原文的核心句（集成哲学，v2 全文遵守）： + +> Existing `function.NewFunctionTool`, `StateGraph.AddNode`, conditional +> edges, and the approval plugin provide integration points **without +> requiring a new Agent execution loop**. + +维护者互动（详见附录 B）：IRONICBo 给了审批数据集建议（绑定参数 + 状态 +指纹、fail-closed、mutation 一律拦截）；liuzengh 认领评估方向。**没有任何 +人讨论或否决过抽象层问题——Non-Goal 是提案作者自划范围，但两个已落地 PR +均按"具体实现填既有槽位"路线执行，形成既成方向。** + +### 1.2 `internal/systemone` 客户端（PR #2652，已合入） + +位置：`internal/systemone/`（16 个 Go 文件 + 测试 + `README.md` + `DESIGN.md`， +引用官方 OpenAPI v0.2.0 与 Laya serve.py 契约）。**全仓零使用方（等集成层 +接入），仅根 module 内的包可导入**（Go internal 可见性规则；examples 与 +兄弟 module 被 CI 明确禁止）。 + +消费视角的契约速览（以包内文档为准，此处仅摘二期会用到的部分）： + +```go +// 构造（唯一入口；默认 30s 超时） +client, err := systemone.NewClient(baseURL, + systemone.WithAPIKey(key), // 必须显式传入 + systemone.WithDefaultModel("jev-2025-10-10"), // 可选，钉版本 + systemone.WithHTTPClient(hc), // 测试注入点 + systemone.WithTimeout(30*time.Second), +) + +// 一次决策请求 +resp, err := client.SystemOne(ctx, &systemone.Request{ + State: state, // string | map[string]any | []any（会做 JSON 形态校验） + Questions: questions, // map[string]systemone.Question，非空 +}) + +// 三种问题（Go 命名刻意用 Binary，协议字段仍是 "noul"） +systemone.BinaryQuestion{Instructions: "text", Criteria: &systemone.BinaryCriteria{True: "...", False: "..."}} +systemone.ChoiceQuestion{Instructions: "text", Options: []systemone.ChoiceOption{{Name, Description}}} // 有序、唯一 +systemone.ScoreQuestion{Instructions: "text", Criteria: []string{"level0", "level1"}} // >= 2 级 + +// 强类型答案访问器（id 不存在 / 类型不符 / 校验失败 → error） +ans, err := resp.Binary(questionID) // ans.Value float64 +ans, err := resp.Choice(questionID) // ans.Selected / ans.Probabilities / ans.Confidence +ans, err := resp.Score(questionID) // ans.Score / ans.Levels []ScoreLevel{Description, Probability} / ans.Confidence +resp.Usage() // InputTokens / OutputTokens / Truncated（截断证据） +``` + +**严格校验语义**（对二期有直接影响）：答案必须匹配全部请求问题（数量、 +类型、choice 选项集、score 层级数）；概率必须 ∈[0,1] 且求和≈1（容 4 位小数 +舍入）；score 必须在 rubric 范围内；文本必须非空白 UTF-8。**响应不完整 = +整个调用返回 error**——v1 设计里"缺分（部分回复）→ 保持原目录"的分支在 +此客户端下不存在：要么全部答案齐、要么整调用失败走 fail-open。指令与 +criteria 是纯文本（string），即使 provider 支持富 JSON 输入也不使用。 + +### 1.3 PR #2664（在审）：System One 审批 reviewer + +`plugin/guardrail/approval: add system one reviewer and final permission checks` +（wangxuw，2026-10-12 开启，27+ 文件）。它确立了**集成层构造模式**（v2 二期 +直接沿用）： + +- 新增公开包 `plugin/guardrail/approval/review/systemone`，内部 import + `internal/systemone`，对外实现既有 `review.Reviewer` 接口 + （`Request{Action, Transcript} → Decision{Approved, RiskScore, RiskLevel, Reason}`）； +- **公开构造器形态**：`New(baseURL string, opts ...Option)`——baseURL 显式 + 传入，`WithAPIKey/WithModel/WithTimeout/WithHTTPClient` 透传给内部客户端， + 外加领域选项（`WithRiskThreshold` 默认 50、`WithMinConfidence` 默认 0.8）； + **不读环境变量**； +- 决策策略在本地代码：一次请求问 authorization（Choice：supported / + contradicted / insufficient）+ risk（Score 五级 rubric）两问，批准 = + supported 且 risk < 阈值 且置信度 ≥ 下限；`Reason` 是本地策略拼接文本， + 不是生成文本； +- **fail-closed**（安全门控语义）：无用户意图直接拒（不调服务器）、截断 + 证据拒、任何错误拒； +- 附带 final permission checks（审批后参数定稿时的终点复查）、跨 agent + 委托/steering/taskrun 的权限继承、示例与文档。 + +它同时覆盖了 v1 三期的"风险门控"场景，**v2 不再做任何 guardrail 方向的工作**。 + +### 1.4 PR #2638（占位）：Jev 原子 rubric 评估器 + +liuzengh 在 #2632 下的评估方向跟进：Jev 实现既有 +`evaluation/evaluator.Evaluator` 接口并注册，原子 rubric 评估为首个里程碑， +与 LLM judge 对照报告。目前是空占位 PR。**v2 不做评估方向的工作**，但二期 +工具预筛与其"合批一次请求、精确检查在代码、API 错误与负面判断分开"的 +设计约束精神一致。 + +### 1.5 LangChain typesafe 官方生态参照 + +`langchain-typesafe` 0.0.1a3（本地包核实，附录 B.4）的官方集成形态： + +| 官方组件 | 对应物 | +| --- | --- | +| `TypeSafeClassifier`（Runnable） | `internal/systemone` 客户端（功能对应） | +| `ModelRouterMiddleware`（模型请求钩子选模型） | **二期 modelrouter 的直接先例** | +| `AutoModeMiddleware`（工具调用钩子拦风险） | PR #2664 的对应物 | +| graph 场景 | 官方示例让用户**自己写普通 node** 调 classifier + conditional edges——无框架 API | + +关键情报：**LangChain 官方没有工具/技能预筛**（toolfilter 在其生态也是 +空白）——二期是 LangChain 官方都没做的差异化方向（Octop 有生产实践， +附录 A）。 + +### 1.6 集成规则与红线（v2 全程遵守的约束清单） + +1. **不引入通用决策抽象**（Non-Goal）：不新增 `Decider` 类接口、不建 + `decision/` 包；每个集成是"具体实现填既有槽位"。 +2. **internal 可见性**：`internal/systemone` 的类型**不得出现在任何公开 + API 签名里**；集成包一律在根 module 内，构造器形态 = `New(baseURL, + opts ...Option)`（PR #2664 模式），内部自建客户端。 +3. **不读环境变量**：显式配置，用户自己 `os.Getenv` 后传入。 +4. **测试注入走 HTTP 层**：`WithHTTPClient` + `httptest` 脚本化响应（仓库 + 惯例，systemone 自身与 PR #2664 测试均如此）；不建 mock 接口层。 +5. **失败方向纪律**：筛选/优化类（toolfilter、skillfilter、modelrouter） + **fail-open**；安全类（审批、门控）fail-closed（PR #2664 已做）。 + 两个默认值方向绝不写反。 +6. **响应严格校验语义**：接受"部分回复 = 整调用 error"，由 fail-open 兜底； + 不做多字段名容错（`noul/probability/p_yes/yes` 容错是杂牌方言，上游已 + 拒绝；协议收敛的供应商走 OpenRouter/Laya 兼容端点，客户端天然支持）。 +7. **核心包零感知**：`graph`/`agent`/`model`/`plugin` 框架层不 import + `internal/systemone`；只有 `plugin/` 集成包与示例碰它。 +8. **既有槽位优先**：BeforeModel 回调、`llmagent.WithSkillFilter`、 + `agent.ModelSelector`、`review.Reviewer`、`evaluator.Evaluator`——全部 + 已存在，只做实现，不改槽位。 +9. **流程规则**：新能力（二期三个集成）动工前先在 issue #2632 评论提出， + 获得提案 owner / 维护者认可；一期一个 commit，PR 总量小步提交。 +10. **命名**：Go 侧用 `Binary` 而非 provider 术语 `Noul`（上游命名约定， + 见 `internal/systemone` DESIGN.md）。 + +### 1.7 剩余可认领的空位 + +提案 scope 内无人认领的交付物（低成本、纯示例，受 1.6 规则 2 约束需要 +一个公开集成包作桥——正是 PR #2664 为审批场景建 `review/systemone` 的 +同款做法）： + +1. "Jev 决策包装成 callable tool" 示例（`function.NewFunctionTool` 槽位）； +2. "Graph 按决策 + 置信度阈值路由" 示例（普通 `AddNode` + NodeFunc + + conditional edges，无新 API）。 + +v2 将其归入三期（3.2）作为融入社区的低成本贡献。 + +--- + +## 二、第二期：toolfilter / skillfilter / modelrouter（核心贡献） + +### 2.0 定位与前置流程 + +二期是 v2 的主体，也是社区真正的空白（提案无此步骤、LangChain 官方无 +对应物、Octop 有生产实践佐证价值）。三个集成全部是"既有槽位上的具体 +实现"，零框架改动、零新抽象，符合 1.6 全部约束： + +| 集成 | 槽位（已存在，不改） | 形态 | +| --- | --- | --- | +| toolfilter | `model.BeforeModelCallbackStructured`（改写 `Request.Tools`，官方支持 "can be modified"） | `plugin/toolfilter`（根 module 包） | +| skillfilter | `llmagent.WithSkillFilter`（`skill.VisibilityFilter` 槽位） | 同上包内第二个构造器 | +| modelrouter | `agent.ModelSelector`（`func(ctx, *Invocation) (model.Model, error)`，纯函数槽位） | 同上包内第三个构造器（或独立 `plugin/modelrouter`） | + +**前置流程（硬性）**：动工前在 issue #2632 发英文评论，提出"Tool / skill +pre-filtering and model routing as concrete integrations on existing slots"： +复用三个既有槽位、零新抽象、fail-open 语义；引 LangChain +`ModelRouterMiddleware` 为 modelrouter 先例、引 Octop 生产实践为 toolfilter +依据；询问纳入本提案路线还是另开 issue。**获得认可前不写代码**。 + +与 v1 二期的差异：骨架从 `插件 → decision.Decider → jev/mock` 换为 +`插件 → internal/systemone`（插件内部直连），测试从 mock Decider 换为 +httptest。**全部核心设计（预筛算法、缓存策略、fail-open 语义、逃生通道、 +三态事件、token 遥测）原样保留。** + +### 2.1 构造形态总规则（PR #2664 模式） + +```go +// 三个集成的构造器同构：baseURL 显式 + 选项透传 + 领域选项 +func NewToolCallback(baseURL string, opts ...Option) (*ToolCallback, error) +func NewSkillFilter(baseURL string, repo skill.Repository, opts ...Option) (*SkillFilter, error) +func NewModelRouter(baseURL string, opts ...Option) (*ModelRouter, error) + +// 共通选项（每个包自持一份，与 reviewer 一致；不抽公共 Option 包——那会变相造抽象） +WithAPIKey(key string) +WithDefaultModel(model string) // 钉版本，如 "jev-2025-10-10" +WithTimeout(d time.Duration) // 默认 30s（跟随 systemone 默认） +WithHTTPClient(c *http.Client) // 测试/策略注入 + +// 领域选项见 2.2-2.4 +``` + +内部统一： + +```go +client, err := systemone.NewClient(baseURL, + systemone.WithAPIKey(o.apiKey), + systemone.WithDefaultModel(o.model), + systemone.WithTimeout(o.timeout), + systemone.WithHTTPClient(o.httpClient), +) +``` + +### 2.2 工具侧：`plugin/toolfilter`（BeforeModel 回调改写工具列表） + +**用户侧**： + +```go +cb, err := toolfilter.NewToolCallback("https://api.typesafe.ai", + toolfilter.WithAPIKey(os.Getenv("TYPESAFE_API_KEY")), // env 由用户读（规则 1.6-3） + toolfilter.WithDefaultModel("jev-2025-10-10"), + toolfilter.WithThreshold(0.6), // P(需要) >= 0.6 保留（Octop 默认值，附录 A.2） + toolfilter.WithSlack(0.15), // 近线补录（Octop 默认值） + toolfilter.WithMinToolCount(8), // 工具少于 8 个不筛（Octop tool_min_count） + toolfilter.WithShortlistLimit(16), // 词法召回入围上限 + toolfilter.WithMaxQuestions(20), // 单请求问题数上限（Octop 同值） + toolfilter.WithKeepAlways("my_core_tool"), // 在默认保留集之外追加用户自留工具 + // toolfilter.WithShortlistEmbedder(myEmbedder), // 可选 embedding 召回路 +) + +agt := llmagent.New("my-agent", + llmagent.WithModel(myModel), + llmagent.WithModelCallbacks(&model.Callbacks{ + BeforeModel: []model.BeforeModelCallbackStructured{cb.Callback()}, + }), +) +``` + +**默认保留集**：Octop 生产实现把 28 个内置工具写死在 +`_TOOLS_ALWAYS_KEPT`（完整清单见附录 A.2）——**永不参与打分**（代码注释： +给内置工具花一个问题只会挤占决策模型真正能删的长尾，且删掉核心集会让模型 +打出裸 tool call 然后停摆），且**无条件进保留集**（即使全部被问工具低于 +阈值、收窄到最小集时也在）。trpc 是框架而非产品，默认保留框架自身的关键 +流程工具；业务注册的文件、记忆和搜索工具由业务方按模型侧名称显式保留： + +| 默认保留 | 来源（已核实注册名） | 理由（Octop 语义的移植） | +| --- | --- | --- | +| `skill_load`、`skill_select_docs`、`skill_list_docs` | skill 渐进披露 | 保留技能加载及文档发现流程 | +| `skill_run`、`skill_exec`、`skill_write_stdin`、`skill_poll_session`、`skill_kill_session` | skill 执行 | 保留执行入口及会话操作 | +| `workspace_exec`、`workspace_write_stdin`、`workspace_kill_session`、`workspace_save_artifact` | workspace 执行 | 保留执行入口及会话、产物操作 | +| `session_search`、`session_load` | 会话回溯 | 保留按需搜索和加载流程 | +| `transfer_to_agent`、`await_user_reply`、`environment_context_current_time` | Agent 转交、用户回复和时间上下文 | 保留框架控制与上下文流程 | + +语义三条（对齐 Octop 决策中间件，源码已核）： + +1. 默认保留集**不进候选、不生成问题、不打分**——它们直接从打分名单里排除， + 省下的问题名额全部留给长尾； +2. 保留集 = Jev 判定保留 ∪ 入围但问题预算溢出的（unasked）∪ 默认保留集 ∪ + `WithKeepAlways` 用户追加——四者取并集； +3. 只有实际出现在请求中的名称才受影响。业务方通过 `WithKeepAlways` + 追加核心工具，名称须与模型请求一致，包含可能的工具集前缀；默认集不可裁剪。 + `tool_search` 不保证找回被筛掉的工具，必要时由业务方显式保留。 + +**内部执行流程**（参数默认值与边界处理按 Octop 生产实现校准，证据见 +附录 A；骨架为 systemone 客户端）： + +1. **每个 Invocation 打分一次**。key 取 Invocation ID；决策上下文取 Session + 中按默认事件前缀过滤的近期用户与助手对话，并补入当前 Invocation 消息。 + 三个决策组件使用相同提取规则，工具预筛使用近期用户消息。工具结果不进入 + 上下文，2400 字符预算优先保留最新消息。Session 事件不保证等同最终模型 + 请求（如临时注入消息或自定义过滤）。ReAct 后续调用重放首次目录筛选结果， + 之后新增的工具直接保留；没有 Invocation 或有效文本时保持工具目录不变； +2. **两道串联判定**：词法预筛（免费）→ Jev 打分（花钱）。默认保留集 + （技能、会话和执行等框架流程工具，见上文表格）**先从 + 候选中排除**——不参与打分、不占问题名额。目录超过 + shortlist 上限时，先按"用户消息 token 与名字+描述的重叠"取前 N 个入围； + 小目录直接送 Jev；大目录按近期用户消息词项对子串的命中次数排序，仅正分 + 候选入围（启用 embedding 时融合排序）；技能同样使用此规则。只有入围的才生成 Binary 问题（"本轮需要工具 X 吗"），**单请求合批并行 + 评估**。词法落选的候选视为预判不相关——收窄发生时一并删除（硬切语义， + 理由与更保守选项的取舍见 2.6）。大目录没有召回候选时筛选中止，目录原样； +3. **保留判定**：得分 ≥ 阈值入选；没有打分候选过线时，slack 近线补录。最终保留集 = + Jev 判定保留的 ∪ 入围但问题预算溢出的（unasked）∪ 默认保留集 ∪ + WithKeepAlways 用户追加。全部被问且全部低于阈值减 slack 时收窄到 + "溢出的 + 默认保留集 + 用户追加"。 + （v1 的"缺分保持原目录"分支删除：systemone 严格校验下缺分 = 整调用 + error，由第 4 条吞掉，语义等价且实现更简单；） +4. **fail-open**：`SystemOne` 返回任何错误（网络、协议、截断证据）→ 全量 + 放行 + 记事件。筛选是优化，失败要放行；与 PR #2664 的 fail-closed 方向 + 相反，两处默认值不能写反（规则 1.6-5）； +5. **发出三态结果事件**（未动 / 收窄 / 保留子集，含每项得分）并统计节省的 + token 数进遥测（附录 A.4 的可观测性做法）。 + +**打分调用的写法**（问题构造与答案读取的形态）： + +```go +questions := make(map[string]systemone.Question, len(shortlist)) +for _, t := range shortlist { + questions["tool:"+t.Name] = systemone.BinaryQuestion{ + Instructions: fmt.Sprintf( + "Does the current user request need the tool %q? Tool description: %s", + t.Name, truncate(t.Description, 240)), // Octop 描述截断 240 字符 + } +} +resp, err := f.client.SystemOne(ctx, &systemone.Request{ + State: map[string]any{ + "user_message": userMsg, + "tool_catalog": toolSummaries(shortlist), + }, + Questions: questions, +}) +if err != nil { + return nil, nil // fail-open：全量放行，发事件记录 +} +// 先遍历全部答案确定 anyAboveThreshold，再逐项判定。 +// 严格校验保证被问的问题全部有答案。 +ans, _ := resp.Binary("tool:" + t.Name) +keep := ans.Value >= f.threshold || (!anyAboveThreshold && ans.Value >= f.threshold-f.slack) +``` + +state 裁剪：只含 user/assistant 对话行（剔除 ToolMessage），裁到 2400 字符， +最新消息优先保留——决策 API 吃短情境串，不吃完整上下文窗口（附录 A.2）。 + +**召回层算法（词法预筛的实现设计）** + +硬切会丢弃召回层遗漏的候选，决策模型无法评估它们，也不保证能够恢复； +业务可用 `WithKeepAlways` 保护关键工具。两路召回，RRF 融合： + +```text +关键词路（默认，零外部依赖）： + token 切分（CJK 单字 + 拉丁词，参考 Octop select.py 的处理） + + 名字和描述合并后做子串匹配，每个用户词项最多计一次分 +embedding 路（可选，复用 knowledge/embedder 接口）： + 用户文本和候选描述按需嵌入，分别保存在每个 filter 的 128 条和 1024 条 LRU 缓存中；淘汰后按需重新计算。 + 候选描述并发嵌入，每个 filter 的 embedding 调用总并发最多为 8。 + 每 turn 只嵌入一条用户消息，余弦相似度排序。 + 同义词、跨语言、间接相关在此路解决。 +融合：RRF（Reciprocal Rank Fusion）——两个榜单按排名倒数求和。 + 关键词抓精确命中（工具名、专有名词），embedding 抓语义相似， + 两路失败模式互补；RRF 是纯算术，零模型调用。 +``` + +边界行为： + +1. 无 embedder 或 embedder 调用失败 → 退回纯关键词单路（fail-open 降级， + 与 toolsearch 的 `WithEmbeddingFailOpen` 语义一致）； +2. 工具目录的嵌入不需要向量库——几十上百个描述在内存里算余弦即可， + 不引入 `vectorstore` 依赖。 + +**边界与演进路径（与 plugin/toolsearch 的关系）**： + +1. **不改动 `plugin/toolsearch`**（已核实：仓库内无任何代码 import 它， + 打分内核全是私有函数）。toolfilter 的召回层自带实现，不复制它的导出 + API，注释标明语义参考； +2. 两套打分暂不共享的代价是排序可能漂移（推模式删掉的工具，拉模式未必排 + 在搜索前列）。将来的共享内核下沉（`internal/toolmatch`，toolsearch 留 + 类型别名保持兼容、跑全部既有测试防回归）**不在二期范围内**，重构理由 + 需要实际召回效果的数据支撑； +3. 二期验收标准包含召回测试：建 golden set（用户消息、工具目录、已知相关 + 工具），断言相关工具全部进入入围名单，必须覆盖跨语言和同义词用例。 + 召回不达标，硬切不上线。 + +### 2.3 skill 侧：`VisibilityFilter` 实现（`WithSkillFilter` 槽位） + +skill 的渲染时序和工具不同（已核实）：skill 摘要和已加载正文由 +`SkillsRequestProcessor` 在 preprocess 阶段注入 system message，发生在 +BeforeModel 回调**之前**。回调触发时 skill 目录已是 prompt 里的纯文本， +`Request` 上没有结构化字段可改。但 trpc 已有现成机制，不需要像 Octop 那样 +用正则修剪 prompt： + +- `skill.VisibilityFilter`（`skill/context_repository.go`）：函数槽位 + `func(ctx, skill.Summary) bool`，决定每个 skill 对本轮是否可见； +- `llmagent.WithSkillFilter(filter)`：agent 构造时自动把 filter 包进 skill + 仓库（`skill.NewFilteredRepository`）。包装后的仓库在所有入口生效—— + overview 注入（`SummariesForContext`）、正文加载（`GetForContext`）、 + 路径访问（`PathForContext`）。不可见的 skill 不出现在 prompt 里， + `skill_load` 点名加载也会得到 "not found"。 + +定位：trpc 现有的 skill 可见性机制是静态规则——scope 隔离（仓库按 +app/user 划分）、`WithMaxOverviewSkills`（overview 条数截断）、 +`WithMaxLoadedSkills`（加载驱逐）。机制是流水线关系，decision filter 补的是 +第一道"按内容动态判断"的工序，后面的截断照常工作： + +```text +scope 隔离（决定全量集合，如 40 个） + ↓ +VisibilityFilter（本设计：决策模型筛选，如剩 3 个） + ↓ +MaxOverviewSkills（可见条数过多时截断显示） + ↓ +MaxLoadedSkills（加载全文后的驱逐） +``` + +与渐进式披露正交：filter 收窄的是披露的候选池（第一级 overview），LLM 在 +可见集合里继续做第二级判断（要不要 `skill_load` 全文）。 + +**实现要点：逐条签名，批量藏在函数内部** + +`VisibilityFilter` 的签名是逐条询问，和决策模型"一次请求批量问"不匹配。 +但 filter 是空槽位，签名只约定"问一次答一个 bool"，不约束实现内部怎么做。 +批量藏在 filter 的第一次调用里： + +```go +func (f *SkillFilter) visible(ctx context.Context, s skill.Summary) bool { + run := runKeyFromContext(ctx) // 本轮用户消息的标识 + kept, ok := f.cache.get(run) + if !ok { + all := f.repo.Summaries() // 构造时捕获的仓库，拿全量摘要 + kept = f.decideBatch(ctx, all) // 首次调用：N 个 skill:N 问题合成一次 SystemOne 请求 + f.cache.set(run, kept) + } + return kept[s.Name] // 本轮后续 N-1 次调用全查缓存 +} +``` + +框架的 for 循环照常逐条问，只是第一次问的时候触发批量决策，之后全部命中 +内存缓存。不改 `SummariesForContext` 本身（已发布的公共契约，逐条语义对 +白名单、scope 这类便宜实现是合理的），也不写并行版的新函数。 + +缓存和并发的三个细节： + +1. 缓存 key 是 run（用户消息），不是 agent。filter 实例在 agent 构造时挂上， + 全生命周期共享，多个会话并发进出同一个 agent——缓存做成 + `map[runKey]kept` 加锁（或 singleflight，同 run 并发首次调用只发一次 + 请求）。新用户消息进来 key 变化，缓存自然失效； +2. 每个 LLM step 都会重新渲染 overview，即重新过一遍全部 filter 调用。缓存 + 让重复调用变成内存查询，一个 turn 只花一次决策请求； +3. fail-open 藏在 filter 里：批量决策失败时 kept 退化为"全部可见"，行为 + 等价于没配 filter。 + +**用户视角的完整示例**： + +```go +repo, _ := skill.NewFSRepository("./skills") + +flt, _ := toolfilter.NewSkillFilter("https://api.typesafe.ai", repo, + toolfilter.WithAPIKey(key), + toolfilter.WithDefaultModel("jev-2025-10-10"), + toolfilter.WithThreshold(0.6), + toolfilter.WithKeepSkills("core-skill"), +) + +agt := llmagent.New("my-agent", + llmagent.WithModel(myModel), + llmagent.WithSkills(repo), + llmagent.WithSkillFilter(flt.VisibilityFilter()), // skill 收窄 + llmagent.WithModelCallbacks(&model.Callbacks{ + BeforeModel: []model.BeforeModelCallbackStructured{cb.Callback()}, // tool 收窄 + }), +) +``` + +效果：仓库 40 个 skill，本轮决策模型判断 3 个相关——overview 只列 3 条； +模型 `skill_load` 一个不可见的 skill，`GetForContext` 返回 "not found"； +`skill_load` 工具本身在默认保留集里，不会被筛掉。 + +### 2.4 模型分流：`ModelSelector` 实现 + +对标 LangChain `ModelRouterMiddleware`（官方先例，附录 B.4），实现既有 +`agent.ModelSelector` 槽位（`func(ctx, inv *Invocation) (model.Model, error)`， +已核实签名；模型以实例形式注册，用户消息在 `inv.Message`）： + +```go +router, err := toolfilter.NewModelRouter("https://api.typesafe.ai", + toolfilter.WithAPIKey(key), + toolfilter.WithDefaultModel("jev-2025-10-10"), + toolfilter.WithModelCandidate("gpt5mini", myGpt5Mini, + "fast, cheap tasks, simple lookups, summarization"), + toolfilter.WithModelCandidate("claude", myClaude, + "complex reasoning, long-context analysis, code review"), +) + +// 每次 Run 传入选择器（RunOption；模型实例在候选表里，不是名字字符串） +out, err := runner.Run(ctx, userID, sessionID, msg, + agent.WithModelSelector(router.Select), +) +``` + +**内部**——一个 ChoiceQuestion，每 invocation 缓存一次（ReAct 循环内后续 +LLM 调用直接重放，与 LangChain 的 per-run 缓存语义一致）： + +```go +func (r *ModelRouter) Select(ctx context.Context, inv *agent.Invocation) (model.Model, error) { + if m, ok := r.cache.get(inv.InvocationID); ok { + return m, nil + } + options := make([]systemone.ChoiceOption, 0, len(r.candidates)) + for _, c := range r.candidates { + options = append(options, systemone.ChoiceOption{ + Name: c.name, Description: c.criteria, // 有序（保序是协议要求） + }) + } + resp, err := r.client.SystemOne(ctx, &systemone.Request{ + State: inv.Message.Content, // string 是合法 State + Questions: map[string]systemone.Question{ + "model_route": systemone.ChoiceQuestion{ + Instructions: "Select the model best suited for this user request.", + Options: options, + }, + }, + }) + if err != nil { + return r.defaultModel, nil // fail-open：出错用默认模型 + } + ans, err := resp.Choice("model_route") + if err != nil || r.byName[ans.Selected] == nil { + return r.defaultModel, nil + } + m := r.byName[ans.Selected] + r.cache.set(inv.InvocationID, m) + return m, nil +} +``` + +**设计决定**：循环终止（ReAct 循环是否继续）不引入决策模型。该判断属于 +开放式推理（是否继续思考、下一步做什么），由 LLM 在生成 tool call 或最终 +回答时自行决定，不属于有界判断。（此条 v1 已定，保留。） + +### 2.5 测试策略（httptest 脚本化响应） + +替代 v1 的 mock Decider（规则 1.6-4），并额外覆盖真实 wire 解码层： + +```go +func newTestToolCallback(t *testing.T, answers map[string]float64) *toolfilter.ToolCallback { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + io.WriteString(w, decisionResponseJSON(answers)) // {"answers":{"tool:X":{"type":"noul","noul":0.9}},...} + })) + t.Cleanup(server.Close) + cb, _ := toolfilter.NewToolCallback(server.URL, + toolfilter.WithAPIKey("test-key"), + toolfilter.WithHTTPClient(server.Client()), + toolfilter.WithThreshold(0.6), + ) + return cb +} +``` + +fail-open 覆盖：handler 返回 500 / 429 / 非法响应体 / `server.Close()` 断连 +→ 断言工具列表全量保留。原 v1 用 `mock.WithError` 注入的全部场景等价覆盖。 + +### 2.6 与 Octop 实现的校准点汇总 + +| 设计点 | Octop 生产值（附录 A） | v2 采用 | +| --- | --- | --- | +| 阈值 / slack | 0.6 / 0.15 | 同 | +| minToolCount | 8 | 同 | +| 单请求问题上限 | 20 | 同（WithMaxQuestions） | +| 词法零重叠 | 整体中止 | 同 | +| 词法落选 | 删除（硬切） | 同，召回不达标不上线 | +| turn 缓存 | key=最后一条 human 消息，LRU 128 | 同语义，run 级缓存 | +| 描述截断 / state 裁剪 | 240 字符 / 2400 字符、剔除 ToolMessage | 同 | +| 内置工具保留集 | `_TOOLS_ALWAYS_KEPT`：28 个产品内置工具，永不打分、无条件保留 | 保留框架流程工具；业务按实际模型侧名称经 WithKeepAlways 追加 | +| 显式指名优先 | @成员直接锁定 | v2 暂不引入（无成员目录场景），留作演进 | +| 被过滤工具的恢复 | ALWAYS_KEPT 保核心，但被筛掉的长尾工具无法找回 | 不保证恢复；业务可通过 WithKeepAlways 防止关键工具被过滤 | + +--- + +## 三、第三期：收尾与增值 + +### 3.1 出口质检（AfterAgentCallback + Score） + +挂 `agent.AfterAgentCallback`：对最终回答问 Score（回答质量 rubric：充分/ +部分/不足），低分注入 `CustomResponse` 触发重答（上限 N 次）。为文档示例 + +可选包，不是核心行为。参照附录 A.3 第 3 条：更温和的"提示 + 拦截"组合 +（给 system message 追加总结提示，同时拦下后续工具调用）也可选，两案 +文档并列。 + +### 3.2 提案剩余示例认领（融入社区的低成本贡献） + +认领 issue #2632 中无人认领的两个示例（1.7）。Graph 路由示例严格用普通 +`AddNode` + NodeFunc + conditional edges（提案哲学，无新 API）；Jev-as-tool +示例用 `function.NewFunctionTool` 包装一次决策调用。两示例需要经一个公开 +集成包桥接 internal 可见性（PR #2664 同款做法，见 3.3 的候选位置）。 +**先在 issue 评论认领再动工。** + +### 3.3 观测与运维收尾 + +1. 决策调用遥测：三态事件（未动 / 收窄 / 保留子集）+ 节省 token 统计在 + 二期已做；三期补决策调用的 trace 对接（复用 invocation 既有 trace 机制， + 不新建事件类型）； +2. `model/failover` 将来识别决策模型错误：`internal/systemone` 已暴露稳定 + 的 `*HTTPError`（含 status 与 `Retry-After`、`X-Request-ID` 头）；三期 + 评估 failover 是否需要消费它，**只读不改 systemone**； +3. 共享集成桥的沉淀：若 3.2 两个示例与二期三个集成的构造样板重复度 + 过高，可评估一个小型公开桥包（仅构造便利，不含抽象——形态仍是 + `New(baseURL, opts...)`）；是否需要以实测重复度为依据，不预设。 + +### 3.4 开放问题（不立项，记录待社区演化） + +1. **无决策模型凭据用户的 LLM 后备**：v1 的 `decision/llm`（用现有 + `model.Model` 实现 Decider）随抽象层取消。可能路径：自建网关把 LLM 包装 + 成 systemone 兼容端点（协议收敛使然，OpenRouter/Laya 均为兼容先例）； + 或接受无后备。倾向后者，等真实需求出现再议； +2. **供应商预设**（lava / msdecision / qwen）：systemone 客户端本身即协议 + 兼容客户端，预设 = baseURL + model 组合，属用户配置而非代码交付； +3. **共享打分内核下沉**（`internal/toolmatch`）：见 2.2 演进路径，靠 + 实际召回效果的数据触发。 + +--- + +## 四、目录结构 + +```text +trpc-agent-go/ （根 module：trpc.group/trpc-go/trpc-agent-go） +├── internal/systemone/ ← 社区已有（PR #2652），只读 +├── plugin/guardrail/approval/review/systemone/ ← 社区在审（PR #2664），不做 +├── plugin/toolfilter/ ← 第二期（新增，根 module 包，import internal/systemone） +│ ├── toolcallback.go // NewToolCallback + BeforeModel 回调 +│ ├── skillfilter.go // NewSkillFilter + VisibilityFilter 实现 +│ ├── modelrouter.go // NewModelRouter + ModelSelector 实现 +│ ├── recall.go // 词法 + 可选 embedding RRF 召回层 +│ ├── cache.go // run 级缓存 / singleflight +│ ├── events.go // 三态事件 + token 节省遥测 +│ └── *_test.go // httptest 脚本化响应 +├── examples/llmagent/toolfilter/ ← 第二期示例：toolfilter + skillfilter + modelrouter +└── examples/graph/decisionroute/ ← 第三期认领示例（普通 AddNode + conditional edges） +``` + +依赖方向（唯一的箭头方向）： + +```text +plugin/toolfilter ──→ internal/systemone（根 module 内，合法） +examples/* ─────────→ plugin/*（公开包，不碰 internal） +core packages ──────→ ✗ 不依赖 internal/systemone（红线，规则 1.6-7） +``` + +（v1 的 `decision/`、`decision/jev/`、`decision/mock/`、 +`graph.AddDecisionNode` 全部移除——已被社区路线替代或否决。） + +--- + +## 五、里程碑 + +| 里程碑 | 交付物 | 验收标准 | +| --- | --- | --- | +| **M1（第一期：锚定地基）** | 不写代码。issue #2632 英文评论提出二期三集成方向；完成 `internal/systemone` 契约与 PR #2664 构造模式的通读校准（本文档 1.2/1.3 即校准结果） | 提案 owner（wangxuw）或维护者对二期方向表态（纳入 / 另开 issue / 拒绝）；若拒绝则仅保留 3.2 示例认领路径并重估 | +| **M2（第二期：核心贡献）** | `plugin/toolfilter`（tool 回调 + skill filter + model router 三构造器，内置关键词 + 可选 embedding RRF 召回层）；`examples/llmagent/toolfilter` 示例 | fail-open 有测试覆盖（500/断连/非法响应体→全量保留）；turn 内只打分一次、循环内重放有缓存验证；skill 筛选经 `WithSkillFilter` 生效，`GetForContext` 拒绝不可见加载；默认保留的框架流程工具不进打分名单、无条件保留有测试；召回 golden set 测试达标（含跨语言、同义词用例），不达标则硬切不上线；三态事件与节省 token 统计可用；httptest 全离线运行 | +| **M3（第三期：收尾增值）** | 出口质检示例（可选包）；提案剩余两个示例认领交付；决策调用 trace 对接 | 出口质检低分重答路径有测试；两个示例被提案 owner 接收；trace 不新建事件类型；`model/failover` 评估结论成文（做或不做，附依据） | + +每个里程碑完成后按 AGENTS.md 执行强制二轮设计评审（导出必要性、包归属、 +API 重叠、命名、可扩展性、兼容性、契约完整性、文档、验证九项）。提交纪律： +一期一个 commit，三期合计三个 commit，向 fork 提 PR 前先开 issue（CONTRIBUTING +要求）。 + +--- + +## 六、兼容性承诺（不变式清单） + +1. 不安装任何 `plugin/toolfilter` 时，`graph`/`agent`/`model`/`plugin` 的 + 默认、零值、错误、事件顺序、序列化行为与现状逐字节一致。 +2. `plugin/toolsearch` 与 `skill` 包的公共行为不变；toolfilter 与其无代码 + 依赖，仅设计互补。 +3. 所有新能力 opt-in：不挂回调、不配 filter、不传 selector 即不存在。 +4. 核心包不 import `internal/systemone`；`internal` 类型不出现在任何公开 + API 签名中（构造器一律 `New(baseURL, opts ...Option)`）。 +5. 不引入通用决策模型抽象（遵守提案 #2632 Non-Goal）；不读环境变量。 +6. 失败方向纪律：toolfilter/skillfilter/modelrouter 一律 fail-open；不触碰 + fail-closed 语义的安全域（那是 PR #2664 的职责边界）。 +7. 集成锚定的是 System One **线上 API 契约**（state + questions → typed + answers + probabilities，OpenAPI v0.2.0），而非任何 alpha 阶段包装层。 +8. 每个新 Go 文件携带 CONTRIBUTING.md 的 Tencent Apache 2.0 许可头。 + +--- + +## 附录 A：参考实现调研（Octop v1.0.2 / octop-harness 1.0.5） + +腾讯 Octop 在 v1.0.2 引入"技能、工具与团队的决策模型"（CHANGELOG 明确列出）。 +实现分两层：Octop 应用仓库（`github.com/TencentCloud/Octop`，tag v1.0.2）持有 +配置、REST API 和前端展示；过滤引擎在外部依赖 octop-harness 中，`decision` +包自 1.0.4 加入（本附录基于 PyPI 1.0.5 wheel 的源码，共 352 行 + 一个 1153 行 +的中间件）。以下事实直接取自代码。 + +### A.1 协议收敛 + +octop-harness 的 decision 客户端注释写明请求体匹配 OpenRouter decisions API， +Jev 也用该格式：`{state, questions: {id: {type: "noul", instructions, +criteria}}}`，回答取 `noul` 字段即 P(yes)。本地模型 Laya 的端点同样是 +`/v1/systemone`。TypeSafe systemone、OpenRouter decisions、Laya 三者线格式 +同构，"state + questions → 概率"正在成为决策模型的事实协议。 + +供应商差异靠一个宽松解析器吸收：响应兼容 `noul` / `probability` / `p_yes` / +`yes` 四种字段名、dict 和 list 两种行格式，而不是每家一个客户端。超时按供应 +商区分：远程 API 15 秒，本地 Laya 60 秒（本地 CPU 推理约每问 1 秒，这是性能 +预算的实证数据）。问题陈述的 criteria 双语自适应（中文给"适用/不适用"，英文 +给 "Yes, this applies."）。 + +> v2 注：A.1 的"宽松解析器"策略与上游 `internal/systemone` 的严格单契约 +> 选择相反，属社区已裁决项（严格校验 + 协议兼容端点覆盖供应商差异）。 +> 本附录保留 A.1 作为协议收敛的证据，**方言容错不再采纳**（规则 1.6-6）。 + +### A.2 筛选流程 + +每次模型调用前，中间件按以下步骤收窄目录： + +1. 取本轮用户消息和三个目录：技能（state 的 `skills_metadata`）、工具 + （`request.tools`）、团队成员（team roster）； +2. **词法预筛**：目录 ≤12 项（middleware 实际传 `_CATALOG_LIMIT = 12`； + select.py 函数签名的默认值 16 未被使用）全部送决策模型；超过 12 项按 + token 重叠打分取前 12；词法完全无命中则返回 None——整个目录保持不动。 + 词法落选的候选不送决策模型，收窄发生时一并删除（词法零重叠视为预判 + 不相关）； +3. 每个候选项一条 Noul 陈述，问题 ID 带类型前缀（`tool:X` / `skill:X` / + `member:X` / `result:sufficient`），文案形如「这轮需要工具「X」。{描述截断 + 240 字符}」； +4. 单次请求最多 **20 个问题**，按 result > skill > tool > member 的优先级在 + 四类目录间分配名额；member 放不下时整组放弃——部分评估会错杀没打分的成员； +5. **state 只含 user/assistant 对话行**（剔除 ToolMessage），裁到 2400 字符， + 最新消息优先保留。决策 API 吃短情境串，不吃完整上下文窗口。 + +保留判定：得分 ≥ 阈值（默认 **0.6**）入选；**slack 0.15**——差一点到线的项在 +名额未满时保留；一个都没过线返回 None，保持原目录。另有两个独立门槛：工具 +少于 **8 个**（`tool_min_count`）不筛；核心工具集 `_TOOLS_ALWAYS_KEPT` +（**28 个**，v2 重数修正 v1 的"约 27"）**永不参与打分**——代码注释原话： +给它们花一个问题只会挤占决策模型真正能删的长尾，且删掉核心集会让模型打出 +裸 tool call 然后停摆。完整清单（frozenset，按功能分组）： + +```text +文件系统/执行（8） ls, read_file, write_file, edit_file, delete, + glob, grep, execute +任务/时间（3） write_todos, task, current_time +检索逃生（1） tool_search +记忆（2） memory_search, memory_get +派工（2） agent_list, ask_agent +网络获取（1） web_fetch +搜索类（8） tavily_search, brave_search, google_search, + kimi_search, deepseek_search, yuanbao_search, + searchfree_search, you_search +用户/桌面（3） send_file_to_user, browser_use, desktop_screenshot +``` + +另设 `_HOST_DISPATCH_TOOLS = ("ask_agent", "agent_list")`：在决策卡片上 +展示但不打分——注释原话 "a low score must not take away the host's +ability to dispatch"（低分不得夺走宿主的派工能力）。 + +### A.3 三个边界处理 + +**失败与部分回复**：HTTP 客户端层面任何异常记 warning 并返回空 map，调用方 +拿到空结果即目录不变——fail-open 做在客户端里，只有连通性探针用 `strict=True` +抛错。被问的工具全部拿到分数且全部低于阈值，才删长尾；有缺分则保持原目录， +代码注释："a partial reply is not a verdict"（部分回复不是判决）。 + +> v2 注："部分回复不是判决"的语义在严格校验客户端下由 fail-open 整体吞掉 +> （任何响应不完整 = 调用 error = 全量放行），语义等价、实现更简。 + +**turn 缓存与重放**：每个用户 turn 只打分一次，key = thread + 最后一条 human +消息的位置和内容；该 turn 内 ReAct 工具循环的所有后续模型调用直接重放缓存的 +收窄结果（LRU 上限 128）。这回答了"每 run 一次还是每轮一次"：每 turn 一次， +turn 内重放。 + +**显式指名优先**：用户消息里 @某成员时直接锁定该成员，决策模型照常打分展示但 +无权改派；@了不存在的成员则本轮跳过决策。代码注释："Explicit user routing +always wins"（用户的显式选择永远赢）。 + +### A.4 与本设计的差异 + +1. **Octop 没有逃生通道**：被决策中间件筛掉的工具无法由模型自行找回。 + 硬证据（v2 核实，agent.py 装配顺序）： + `ToolsFilterMiddleware → DecisionMiddleware → tool_search`，且注释明写 + *"Search only the final model-visible tool set. In particular, this must + run after MCP and user filtering so hidden tools cannot leak into either + the client or provider search inventory"*——搜索目录就是收窄后的可见集， + 被删工具不在其中（v1 在 A.2 写过"判漏由 tool_search 兜底"，据本证据 + 修正：tool_search 在 ALWAYS_KEPT 里只保住"能搜"这个能力本身，搜不回 + 被决策删掉的工具）。trpc 的 `plugin/toolsearch` 结构不同：检索目录是 + MCP 工具注册表（deferred/lazy 加载全集），不是当前 `Request.Tools`—— + toolfilter 删掉的 MCP 工具仍在可搜可载范围内，模型能自己找回。推拉 + 互补在 trpc 成立、在 Octop 不成立，这是与 Octop 的关键差异点。 +2. **技能注入位置不同**：Octop 的技能目录先被上游 SkillsMiddleware 写进 + system prompt，收窄时除了改 state，还要用正则修剪 prompt 里的 + `**Available Skills:**` 段落。trpc 的渲染时序相同（skill 在 preprocess + 阶段注入 system message，早于 BeforeModel 回调），但已有 + `skill.VisibilityFilter` + `skill.NewFilteredRepository` + llmagent + `WithSkillFilter` 的现成机制：过滤发生在渲染之前，overview、正文加载、 + 路径访问共用同一道闸，不需要字符串手术。已核实，见 2.3。 +3. **结果充分性不篡改响应**：团队收尾时问"结果是否足以总结"（≥ 阈值即充分）， + 不注入 CustomResponse，而是两步：给 system message 追加"请直接总结"提示， + 同时在工具拦截层拦下后续 `ask_agent` 调用返回 error ToolMessage。提示 + + 拦截的组合比改写响应温和，trpc 的出口质检可参照（3.1）。 +4. **可观测性**：节省量 = 模型调用前后的目录 token 差，经 ContextVar 传给 + 响应，钉在 AI 消息的 `additional_kwargs` 上，同时经 stream writer 推 + `decision` 事件（含模型名、三态 parts、每项得分）。前端能显示每轮保留 + 清单和省下的 token 数。 + +--- + +## 附录 B：社区生态情报与核实记录（v2 重写依据） + +### B.1 时间线（2026-11-01 经 GitHub API 直连核实） + +| 时间 | 事件 | 内容 | +| --- | --- | --- | +| 2026-09-21 | issue #2632 开启（wangxuw） | 提案：scope + Goals + Non-Goals + Open Questions；作者要求 ownership、增量 PR | +| 2026-09-22 | IRONICBo 评论 | 审批数据集四案例；绑定参数 + 状态指纹；fail-closed for mutation | +| 2026-09-24 | liuzengh 评论 | 评估方向跟进，占位 PR #2638 | +| 2026-10-10 | **PR #2652 合入** | `internal/systemone` 共享客户端（16 文件 + 880 行测试 + DESIGN.md/README.md） | +| 2026-10-12 | PR #2664 开启（在审） | system one reviewer + final permission checks（27+ 文件） | +| 至今 | 25 个活跃分支 | 无 decision / jev / systemone / toolfilter 相关 WIP 分支（wangxuw 的工作分支 codex/systemone-reviewer 除外） | + +### B.2 "否决"的准确定性 + +Non-Goal "Adding a new generic decision-model abstraction" 是**提案作者 +wangxuw 自划的范围**，不是维护者裁决——评论区无人讨论过抽象层。但两个已 +落地 PR（#2652、#2664）均按"具体实现填既有槽位"执行，形成既成方向；提案 +Open Questions 留有口子（"existing interfaces sufficient, or concrete use +case requiring..."），说明具体用例驱动的扩展可以谈。**结论：对抗既成方向 +性价比低，v2 全面转向槽位集成。** + +### B.3 internal 可见性与构造模式的硬证据 + +- PR #2664 新增公开包 `plugin/guardrail/approval/review/systemone`：内部 + `import systemoneclient "trpc.group/trpc-go/trpc-agent-go/internal/systemone"`， + 对外 `New(baseURL string, opts ...Option)`，**不读 env**； +- 其示例 `examples/guardrail/approval/systemone/app.go` 导入公开插件包， + **不直接 import internal**（CI 禁止 examples 引 internal）； +- 结论：集成包必须落在根 module 内；公开签名不得出现 internal 类型； + 示例经公开包桥接。v2 规则 1.6-2 与 2.1 构造形态由此而来。 + +### B.4 LangChain typesafe 0.0.1a3 核实（本地包逐文件验证） + +包结构：`classifier.py`（TypeSafeClassifier，Runnable）、`types.py`（三种 +问题/答案类型）、`experimental/middleware/{model_router,auto_mode}.py`。 + +**核实结论：官方包没有 decision node API**；Graph 场景的官方形态是用户 +自己写普通 node 调 classifier + conditional edges；框架级集成是两个 +AgentMiddleware（`wrap_model_request` / `wrap_tool_call` 钩子包装）。 +TypeSafe 官方 cookbook 的 function-calling 页面亦无 decision node API +（已抓取核实）。 + +> 更正记录：v1 前期调研曾得出"LangChain 官方做法是加 addDecisionNode"的 +> 结论，**该结论有误**（可能把"官方示例中用户写 node 调 classifier"误记为 +> "官方提供 decision node API"）。记录在案防止复用。 + +**战略情报**：`ModelRouterMiddleware`（= 二期 modelrouter 的官方先例）与 +`AutoModeMiddleware`（= 风险门控，已被 PR #2664 对应）官方均做了；**工具/ +技能预筛官方未做**——二期 toolfilter 是 LangChain 生态也没有的空白， +差异化最大。 + +### B.5 v1 撤销记录 + +v1 一期曾在本地 `feature/jev-integration` 分支实现（decision 抽象层 + +jev/mock 独立 module + AddDecisionNode + 示例，全部验证通过），因与社区 +路线冲突于 2026-11-01 撤销：`git checkout --` 恢复跟踪文件，删除全部新增 +文件，构建与测试回到 main 基线。撤销前实现满足原设计规格，其架构结论 +（checkpoint JSON 往返的类型恢复问题、问题 map 共享的数据竞争、指针形式 +类型断言等）已在 v2 设计时规避（v2 不再有跨 state 传递 Answer 类型的场景）。 diff --git a/examples/decisionfilter/README.md b/examples/decisionfilter/README.md new file mode 100644 index 0000000000..9e9f4bdd14 --- /dev/null +++ b/examples/decisionfilter/README.md @@ -0,0 +1,94 @@ +# Decision-Based Filtering Example + +This example demonstrates the three `plugin/toolfilter` integrations, all +backed by a System One decision model (TypeSafe Jev or a +protocol-compatible endpoint): + +- **Tool pre-filtering** — `toolfilter.NewToolCallback` returns a + `model.BeforeModelCallbackStructured` that removes the tools the decision + model deems irrelevant for the current user turn. Framework workflow + tools for skills, workspace sessions, session recall, agent transfer, + user replies and time are never scored or filtered. +- **Skill visibility** — `toolfilter.NewSkillFilter` returns a + `skill.VisibilityFilter` for `llmagent.WithSkillFilter` that narrows the + skill overview to the skills relevant for the current run, decided in one + batch request on the first visibility query. +- **Model routing** — `toolfilter.NewModelRouter` returns an + `agent.ModelSelector` that picks a cheap or a strong model per run with one + Choice question. + +All three **fail open**: when the decision endpoint is unreachable or returns +an error, the full tool catalog stays visible, all skills stay visible, and +routing falls back to the configured fallback model. The agent then behaves +exactly as if no filter were configured. + +The decision context uses recent user and assistant text from session events +matching the framework's default event prefix filter. Tool results are excluded +and the newest dialogue takes priority within a 2400-rune budget. This event +history can differ from the final model request, for example when messages are +injected without being stored in the session or custom filters are configured. + +## Prerequisites + +- Go 1.24 or later +- An OpenAI-compatible chat endpoint (see the `OPENAI_API_KEY` and + `OPENAI_BASE_URL` conventions of `examples/toolfilter`) +- A System One decision endpoint and API key + +```bash +export TYPESAFE_API_KEY="your-decision-api-key" +``` + +## Run + +```bash +cd examples/decisionfilter +go run main.go +``` + +The decision endpoint, decision model and the two routing targets can be +overridden: + +```bash +go run main.go \ + -decision-url https://api.typesafe.ai \ + -decision-model jev-2025-10-10 \ + -cheap-model deepseek-v4-flash \ + -strong-model deepseek-v4 +``` + +## What to watch for + +- `[toolfilter] tool narrowed: asked=9 dropped=[...] saved~... tokens` — printed + on the first decision by the `WithObserver` callback; later ReAct steps + replay it by invocation ID. Tools added after that decision stay visible. +- `📋 Tools sent to model: [...]` — printed by the OpenAI request callback; + compare it with the dropped list to see the narrowing take effect. +- `Ask about cooking` — a skill-irrelevant question; the skill overview in + the system message shrinks to the relevant skills (visible through the + `skill_load` tool's enum-free listing or the report line). +- Simple arithmetic questions route to the cheap model; long-context review + questions route to the strong model. + +## Tuning + +The shared options apply to all three integrations, for example: + +```go +toolfilter.WithThreshold(0.6) // keep when P(relevant) >= 0.6 +toolfilter.WithSlack(0.15) // fallback to near-line candidates if none pass +toolfilter.WithMinToolCount(8) // skip small catalogs +toolfilter.WithMaxQuestions(20) // question budget per decision request +toolfilter.WithKeepAlways("file_read_file", "memory_search") // model-facing names +toolfilter.WithKeepSkills("core") // protect core skills +``` + +Application tools, including search and file tools, are filtered unless +protected with `WithKeepAlways`. Use the exact names exposed to the model, +including any tool set prefix. A filtered tool is not automatically recovered +by `tool_search`. + +Tool and skill catalogs at or below `WithShortlistLimit` go straight to the +decision model. Larger catalogs use recent user messages for lexical recall; +when recall finds no candidates, the catalog stays visible. An optional +shortlist embedder adds a semantic ranking path. diff --git a/examples/decisionfilter/main.go b/examples/decisionfilter/main.go new file mode 100644 index 0000000000..11c3a1e89b --- /dev/null +++ b/examples/decisionfilter/main.go @@ -0,0 +1,496 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +// Package main demonstrates decision-based tool filtering, skill visibility +// narrowing and model routing with plugin/toolfilter. +// +// The example wires the three toolfilter integrations into one agent: +// +// - a toolfilter.ToolCallback as a BeforeModel callback that removes the +// tools the decision model deems irrelevant for each user turn; +// - a toolfilter.SkillFilter as a llmagent.WithSkillFilter that narrows +// the skill overview to the relevant skills of each run; +// - a toolfilter.ModelRouter as a llmagent.WithModelSelector that picks a +// cheap or a strong model per run. +// +// All three fail open: when the decision endpoint is unreachable the agent +// behaves exactly as if no filtering were configured. +package main + +import ( + "bufio" + "context" + "flag" + "fmt" + "log" + "math/rand" + "os" + "strings" + "time" + + openaigo "github.com/openai/openai-go" + "trpc.group/trpc-go/trpc-agent-go/agent/llmagent" + "trpc.group/trpc-go/trpc-agent-go/event" + "trpc.group/trpc-go/trpc-agent-go/model" + "trpc.group/trpc-go/trpc-agent-go/model/openai" + "trpc.group/trpc-go/trpc-agent-go/plugin/toolfilter" + "trpc.group/trpc-go/trpc-agent-go/runner" + "trpc.group/trpc-go/trpc-agent-go/skill" + "trpc.group/trpc-go/trpc-agent-go/tool" + "trpc.group/trpc-go/trpc-agent-go/tool/function" +) + +func main() { + decisionURL := flag.String("decision-url", "https://api.typesafe.ai", + "System One decision API base URL") + decisionModel := flag.String("decision-model", "jev-2025-10-10", + "decision model version") + cheapModel := flag.String("cheap-model", "deepseek-v4-flash", + "model name used for simple requests") + strongModel := flag.String("strong-model", "deepseek-v4", + "model name used for complex requests") + flag.Parse() + + decisionKey := os.Getenv("TYPESAFE_API_KEY") + + ctx := context.Background() + r, err := newRunner(ctx, runnerConfig{ + decisionURL: *decisionURL, + decisionModel: *decisionModel, + decisionKey: decisionKey, + cheapModel: *cheapModel, + strongModel: *strongModel, + }) + if err != nil { + log.Fatalf("setup failed: %v", err) + } + defer r.Close() + + if err := chat(ctx, r); err != nil { + log.Fatalf("chat failed: %v", err) + } +} + +type runnerConfig struct { + decisionURL string + decisionModel string + decisionKey string + cheapModel string + strongModel string +} + +// newRunner builds the agent with all three toolfilter integrations. +func newRunner(ctx context.Context, cfg runnerConfig) (runner.Runner, error) { + observer := func(report *toolfilter.Report) { + fmt.Printf("[toolfilter] %s %s: asked=%d dropped=%v saved~%d tokens", + report.Catalog, report.Outcome, len(report.Asked), report.Dropped, report.SavedTokens) + if report.Reason != "" { + fmt.Printf(" (%s)", report.Reason) + } + fmt.Println() + } + + baseOpts := []toolfilter.Option{ + toolfilter.WithAPIKey(cfg.decisionKey), + toolfilter.WithDefaultModel(cfg.decisionModel), + toolfilter.WithObserver(observer), + } + + // 1. Tool pre-filtering: one decision request per user turn rewrites + // the tool list of each model request. + toolCallback, err := toolfilter.NewToolCallback(cfg.decisionURL, baseOpts...) + if err != nil { + return nil, fmt.Errorf("tool callback: %w", err) + } + + // 2. Skill visibility: one batch decision per run narrows the skill + // overview. The repository must be the one mounted on the agent. + repo := newSkillRepo() + skillFilter, err := toolfilter.NewSkillFilter(cfg.decisionURL, repo, + append(baseOpts, toolfilter.WithKeepSkills("core-workflow"))...) + if err != nil { + return nil, fmt.Errorf("skill filter: %w", err) + } + + // 3. Model routing: one Choice decision per run picks the model. + cheap := openai.New(cfg.cheapModel, openai.WithChatRequestCallback(printTools)) + strong := openai.New(cfg.strongModel, openai.WithChatRequestCallback(printTools)) + modelRouter, err := toolfilter.NewModelRouter(cfg.decisionURL, + append(baseOpts, + toolfilter.WithModelCandidate("cheap", cheap, + "fast and cheap: simple lookups, arithmetic, formatting"), + toolfilter.WithModelCandidate("strong", strong, + "complex reasoning: long-context analysis, code review"), + toolfilter.WithFallbackModel(cheap), + )...) + if err != nil { + return nil, fmt.Errorf("model router: %w", err) + } + + agt := llmagent.New( + "decision-filtered-agent", + llmagent.WithModel(cheap), + llmagent.WithDescription("A general assistant with decision-based filtering"), + llmagent.WithInstruction( + "You are a helpful assistant. Use your tools when they help.", + ), + llmagent.WithTools(tools()), + llmagent.WithSkills(repo), + llmagent.WithSkillFilter(skillFilter.VisibilityFilter()), + llmagent.WithModelSelector(modelRouter.Select), + llmagent.WithModelCallbacks(&model.Callbacks{ + BeforeModel: []model.BeforeModelCallbackStructured{ + toolCallback.Callback(), + }, + }), + llmagent.WithGenerationConfig(model.GenerationConfig{ + MaxTokens: intPtr(2000), + Temperature: floatPtr(0.7), + Stream: true, + }), + ) + return runner.NewRunner("decision-filter-demo", agt), nil +} + +// printTools shows which tools survived filtering for the current request. +func printTools(ctx context.Context, req *openaigo.ChatCompletionNewParams) { + names := make([]string, 0, len(req.Tools)) + for _, t := range req.Tools { + names = append(names, t.Function.Name) + } + fmt.Printf("📋 Tools sent to model: %v\n", names) +} + +// chat runs the interactive conversation loop. +func chat(ctx context.Context, r runner.Runner) error { + userID := "user" + sessionID := fmt.Sprintf("session-%d", time.Now().Unix()) + fmt.Printf("🚀 Decision filter demo (session %s)\n", sessionID) + fmt.Println("Try: 'what time is it in words', 'calculate 12*7', 'review this algorithm idea'") + fmt.Println("Enter 'exit' to end.") + + scanner := bufio.NewScanner(os.Stdin) + for { + fmt.Print("👤 User: ") + if !scanner.Scan() { + return nil + } + input := strings.TrimSpace(scanner.Text()) + if input == "" { + continue + } + if strings.ToLower(input) == "exit" { + fmt.Println("👋 Goodbye!") + return nil + } + + eventChan, err := r.Run(ctx, userID, sessionID, model.NewUserMessage(input)) + if err != nil { + fmt.Printf("❌ Error: %v\n", err) + continue + } + if err := printResponse(eventChan); err != nil { + fmt.Printf("❌ Error: %v\n", err) + } + fmt.Println() + } +} + +// printResponse streams the assistant response. +func printResponse(eventChan <-chan *event.Event) error { + fmt.Print("🤖 Assistant: ") + for evt := range eventChan { + if evt.Error != nil { + fmt.Printf("\n❌ %s\n", evt.Error.Message) + continue + } + if len(evt.Response.Choices) == 0 { + continue + } + choice := evt.Response.Choices[0] + if len(choice.Message.ToolCalls) > 0 { + for _, call := range choice.Message.ToolCalls { + fmt.Printf("\n🔧 %s(%s)", call.Function.Name, call.Function.Arguments) + } + continue + } + if choice.Delta.Content != "" { + fmt.Print(choice.Delta.Content) + } + if evt.IsFinalResponse() { + fmt.Println() + return nil + } + } + return nil +} + +// tools returns a catalog large enough for filtering to pay off (the +// callback skips catalogs at or below the minimum tool count). +func tools() []tool.Tool { + return []tool.Tool{ + calculatorTool(), + textTool(), + timeTool(), + wordCountTool(), + diceTool(), + reverseTool(), + bmiTool(), + temperatureTool(), + qrTool(), + } +} + +func calculatorTool() tool.CallableTool { + return function.NewFunctionTool( + func(_ context.Context, req struct { + Expression string `json:"expression" jsonschema:"description=Mathematical expression,required"` + }) (struct { + Result string `json:"result"` + }, error) { + expr := strings.ReplaceAll(req.Expression, " ", "") + var result float64 + var left, right string + var op byte + for _, candidate := range []byte{'+', '-', '*', '/'} { + if i := strings.IndexByte(expr, candidate); i > 0 { + left, op, right = expr[:i], candidate, expr[i+1:] + break + } + } + if op == 0 { + return struct { + Result string `json:"result"` + }{}, fmt.Errorf("unsupported expression") + } + l, err1 := parseFloat(left) + rv, err2 := parseFloat(right) + if err1 != nil || err2 != nil { + return struct { + Result string `json:"result"` + }{}, fmt.Errorf("invalid numbers") + } + switch op { + case '+': + result = l + rv + case '-': + result = l - rv + case '*': + result = l * rv + case '/': + if rv == 0 { + return struct { + Result string `json:"result"` + }{}, fmt.Errorf("division by zero") + } + result = l / rv + } + return struct { + Result string `json:"result"` + }{Result: fmt.Sprintf("%g", result)}, nil + }, + function.WithName("calculator"), + function.WithDescription("Perform basic arithmetic on two numbers"), + ) +} + +func parseFloat(s string) (float64, error) { + var v float64 + _, err := fmt.Sscanf(s, "%g", &v) + return v, err +} + +func textTool() tool.CallableTool { + return function.NewFunctionTool( + func(_ context.Context, req struct { + Text string `json:"text" jsonschema:"description=Text to process,required"` + Operation string `json:"operation" jsonschema:"description=uppercase or lowercase,required"` + }) (struct { + Result string `json:"result"` + }, error) { + out := req.Text + switch req.Operation { + case "uppercase": + out = strings.ToUpper(req.Text) + case "lowercase": + out = strings.ToLower(req.Text) + } + return struct { + Result string `json:"result"` + }{Result: out}, nil + }, + function.WithName("text_tool"), + function.WithDescription("Convert text to uppercase or lowercase"), + ) +} + +func timeTool() tool.CallableTool { + return function.NewFunctionTool( + func(_ context.Context, _ struct{}) (struct { + Result string `json:"result"` + }, error) { + return struct { + Result string `json:"result"` + }{Result: time.Now().Format("2006-01-02 15:04:05")}, nil + }, + function.WithName("time_tool"), + function.WithDescription("Get the current date and time"), + ) +} + +func wordCountTool() tool.CallableTool { + return function.NewFunctionTool( + func(_ context.Context, req struct { + Text string `json:"text" jsonschema:"description=Text to count,required"` + }) (struct { + Result int `json:"result"` + }, error) { + return struct { + Result int `json:"result"` + }{Result: len(strings.Fields(req.Text))}, nil + }, + function.WithName("word_count"), + function.WithDescription("Count the words in a text"), + ) +} + +func diceTool() tool.CallableTool { + return function.NewFunctionTool( + func(_ context.Context, req struct { + Sides int `json:"sides" jsonschema:"description=Number of sides,required"` + }) (struct { + Result int `json:"result"` + }, error) { + if req.Sides < 2 { + req.Sides = 6 + } + return struct { + Result int `json:"result"` + }{Result: rand.Intn(req.Sides) + 1}, nil + }, + function.WithName("dice_roll"), + function.WithDescription("Roll a dice with the given number of sides"), + ) +} + +func reverseTool() tool.CallableTool { + return function.NewFunctionTool( + func(_ context.Context, req struct { + Text string `json:"text" jsonschema:"description=Text to reverse,required"` + }) (struct { + Result string `json:"result"` + }, error) { + runes := []rune(req.Text) + for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { + runes[i], runes[j] = runes[j], runes[i] + } + return struct { + Result string `json:"result"` + }{Result: string(runes)}, nil + }, + function.WithName("reverse_text"), + function.WithDescription("Reverse the characters of a text"), + ) +} + +func bmiTool() tool.CallableTool { + return function.NewFunctionTool( + func(_ context.Context, req struct { + WeightKG float64 `json:"weight_kg" jsonschema:"description=Weight in kilograms,required"` + HeightM float64 `json:"height_m" jsonschema:"description=Height in meters,required"` + }) (struct { + Result string `json:"result"` + }, error) { + if req.HeightM <= 0 { + return struct { + Result string `json:"result"` + }{}, fmt.Errorf("height must be positive") + } + bmi := req.WeightKG / (req.HeightM * req.HeightM) + return struct { + Result string `json:"result"` + }{Result: fmt.Sprintf("%.1f", bmi)}, nil + }, + function.WithName("bmi_calculator"), + function.WithDescription("Compute the body mass index from weight and height"), + ) +} + +func temperatureTool() tool.CallableTool { + return function.NewFunctionTool( + func(_ context.Context, req struct { + Celsius float64 `json:"celsius" jsonschema:"description=Temperature in Celsius,required"` + }) (struct { + Result string `json:"result"` + }, error) { + return struct { + Result string `json:"result"` + }{Result: fmt.Sprintf("%.1f F", req.Celsius*9/5+32)}, nil + }, + function.WithName("temperature_convert"), + function.WithDescription("Convert a Celsius temperature to Fahrenheit"), + ) +} + +func qrTool() tool.CallableTool { + return function.NewFunctionTool( + func(_ context.Context, req struct { + Text string `json:"text" jsonschema:"description=Text to encode,required"` + }) (struct { + Result string `json:"result"` + }, error) { + return struct { + Result string `json:"result"` + }{Result: "qr:" + req.Text}, nil + }, + function.WithName("qr_encode"), + function.WithDescription("Encode a short text as a QR code reference"), + ) +} + +// newSkillRepo returns a static in-memory skill repository. +func newSkillRepo() skill.Repository { + return &staticSkillRepo{summaries: []skill.Summary{ + {Name: "core-workflow", Description: "core workflow rules for this assistant"}, + {Name: "travel-booking", Description: "find and book flights and hotels"}, + {Name: "expense-report", Description: "file expense reports with receipts"}, + {Name: "math-helper", Description: "step-by-step arithmetic guidance"}, + {Name: "cooking", Description: "recipes and menu planning"}, + {Name: "email-drafting", Description: "draft polite business emails"}, + }} +} + +// staticSkillRepo implements skill.Repository with fixed content. +type staticSkillRepo struct { + summaries []skill.Summary +} + +// Summaries implements skill.Repository. +func (r *staticSkillRepo) Summaries() []skill.Summary { return r.summaries } + +// Get implements skill.Repository. +func (r *staticSkillRepo) Get(name string) (*skill.Skill, error) { + for _, s := range r.summaries { + if s.Name == name { + return &skill.Skill{ + Summary: s, + Body: fmt.Sprintf("Guidance for %s: follow the %s process.", s.Name, s.Name), + }, nil + } + } + return nil, fmt.Errorf("skill %q not found", name) +} + +// Path implements skill.Repository. +func (r *staticSkillRepo) Path(name string) (string, error) { + return "", fmt.Errorf("static skill repository has no paths") +} + +func intPtr(i int) *int { return &i } +func floatPtr(f float64) *float64 { return &f } diff --git a/plugin/toolfilter/conversation.go b/plugin/toolfilter/conversation.go new file mode 100644 index 0000000000..4bedcd0a55 --- /dev/null +++ b/plugin/toolfilter/conversation.go @@ -0,0 +1,103 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "strings" + + "trpc.group/trpc-go/trpc-agent-go/agent" + "trpc.group/trpc-go/trpc-agent-go/model" +) + +type conversationLine struct { + role model.Role + text string +} + +// decisionConversation uses the default event prefix filter. Session events +// are not necessarily identical to the final model request's message history. +func decisionConversation(inv *agent.Invocation) (string, string) { + if inv == nil { + return "", "" + } + var lines []conversationLine + currentIncluded := false + if inv.Session != nil { + for _, evt := range inv.Session.GetEvents() { + if evt.Response == nil || evt.IsPartial || len(evt.Choices) == 0 || !evt.Filter(inv.GetEventFilterKey()) { + continue + } + msg := evt.Choices[0].Message + if (msg.Role != model.RoleUser && msg.Role != model.RoleAssistant) || msg.ToolID != "" { + continue + } + if text := conversationText(msg); text != "" { + lines = append(lines, conversationLine{role: msg.Role, text: text}) + currentIncluded = evt.InvocationID == inv.InvocationID && msg.Role == model.RoleUser && model.MessagesEqual(msg, inv.Message) + } + } + } + current := conversationText(inv.Message) + if current != "" && !currentIncluded { + lines = append(lines, conversationLine{role: model.RoleUser, text: current}) + } + return fitConversation(lines, false), fitConversation(lines, true) +} + +func conversationText(msg model.Message) string { + parts := make([]string, 0, 1+len(msg.ContentParts)) + if text := strings.TrimSpace(msg.Content); text != "" { + parts = append(parts, text) + } + for _, part := range msg.ContentParts { + if part.Type == model.ContentTypeText && part.Text != nil { + if text := strings.TrimSpace(*part.Text); text != "" { + parts = append(parts, text) + } + } + } + return strings.Join(parts, "\n") +} + +func fitConversation(lines []conversationLine, usersOnly bool) string { + remaining := stateClipLen + selected := make([]string, 0, len(lines)) + for i := len(lines) - 1; i >= 0 && remaining > 0; i-- { + line := lines[i] + if usersOnly && line.role != model.RoleUser { + continue + } + prefix := string(line.role) + ": " + if usersOnly { + prefix = "" + } + if len(selected) > 0 { + remaining-- + } + if remaining <= len(prefix) { + break + } + text := []rune(line.text) + originalLen := len(text) + room := remaining - len(prefix) + if len(text) > room { + text = text[:room] + } + selected = append(selected, prefix+string(text)) + remaining -= len(prefix) + len(text) + if len(text) < originalLen { + break + } + } + for i, j := 0, len(selected)-1; i < j; i, j = i+1, j-1 { + selected[i], selected[j] = selected[j], selected[i] + } + return strings.Join(selected, "\n") +} diff --git a/plugin/toolfilter/conversation_test.go b/plugin/toolfilter/conversation_test.go new file mode 100644 index 0000000000..7ca8ed2933 --- /dev/null +++ b/plugin/toolfilter/conversation_test.go @@ -0,0 +1,95 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "context" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "trpc.group/trpc-go/trpc-agent-go/agent" + "trpc.group/trpc-go/trpc-agent-go/event" + "trpc.group/trpc-go/trpc-agent-go/model" + "trpc.group/trpc-go/trpc-agent-go/session" +) + +func conversationEvent(role model.Role, text, filter string) event.Event { + return event.Event{Version: event.CurrentVersion, FilterKey: filter, Response: &model.Response{ + Choices: []model.Choice{{Message: model.Message{Role: role, Content: text}}}, + }} +} + +func TestDecisionComponentsUseSessionConversation(t *testing.T) { + t.Parallel() + sess := &session.Session{Events: []event.Event{ + conversationEvent(model.RoleUser, "find flights", "app"), + conversationEvent(model.RoleAssistant, "checking flights", "app/agent"), + conversationEvent(model.RoleTool, "secret tool output", "app/agent"), + conversationEvent(model.RoleUser, "other branch secret", "app/other"), + conversationEvent(model.RoleUser, "continue", "app/agent"), + }} + inv := agent.NewInvocation(agent.WithInvocationSession(sess), agent.WithInvocationEventFilterKey("app/agent"), + agent.WithInvocationMessage(model.NewUserMessage("continue"))) + sess.Events[len(sess.Events)-1].InvocationID = inv.InvocationID + ctx := agent.NewInvocationContext(context.Background(), inv) + toolsServer, toolsRec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + cb, err := NewToolCallback(toolsServer.URL, WithHTTPClient(toolsServer.Client()), WithMinToolCount(0)) + require.NoError(t, err) + req := newUserRequest("continue", newToolMap(&stubTool{name: "flights", description: "find flights"})) + _, err = cb.Callback()(ctx, &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, 1, toolsRec.requests()) + + skillServer, skillRec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + sf, err := NewSkillFilter(skillServer.URL, &fakeSkillRepo{summaries: testSkillSummaries()}, WithHTTPClient(skillServer.Client())) + require.NoError(t, err) + sf.VisibilityFilter()(ctx, testSkillSummaries()[0]) + + router, routerRec := newRoutingEnv(t, http.StatusOK, choiceResponder("fast"), + WithModelCandidate("fast", &stubModel{name: "fast"}, "simple")) + _, err = router.Select(ctx, inv) + require.NoError(t, err) + + toolState := toolsRec.lastState().(map[string]any)["user_message"].(string) + skillState := skillRec.lastState().(map[string]any)["user_message"].(string) + routeState := routerRec.lastState().(string) + require.Equal(t, toolState, skillState) + require.Equal(t, toolState, routeState) + require.Contains(t, toolState, "find flights") + require.Contains(t, toolState, "checking flights") + require.NotContains(t, toolState, "secret") + require.Equal(t, 1, strings.Count(toolState, "continue")) +} + +func TestDecisionConversationBudgetAndRepeatedInput(t *testing.T) { + t.Parallel() + long := strings.Repeat("旧", stateClipLen) + inv := agent.NewInvocation(agent.WithInvocationSession(&session.Session{Events: []event.Event{ + conversationEvent(model.RoleUser, long, ""), + conversationEvent(model.RoleUser, "continue", ""), + }}), agent.WithInvocationMessage(model.NewUserMessage("continue"))) + state, _ := decisionConversation(inv) + require.LessOrEqual(t, len([]rune(state)), stateClipLen) + require.Equal(t, 2, strings.Count(state, "continue")) + require.True(t, strings.HasSuffix(state, "user: continue")) +} + +func TestDecisionConversationContentParts(t *testing.T) { + t.Parallel() + text := "find flights" + inv := agent.NewInvocation(agent.WithInvocationMessage(model.Message{Role: model.RoleUser, + ContentParts: []model.ContentPart{{Type: model.ContentTypeText, Text: &text}}, + })) + state, users := decisionConversation(inv) + require.Equal(t, "user: find flights", state) + require.Equal(t, "find flights", users) +} diff --git a/plugin/toolfilter/doc.go b/plugin/toolfilter/doc.go new file mode 100644 index 0000000000..10ff1e3191 --- /dev/null +++ b/plugin/toolfilter/doc.go @@ -0,0 +1,42 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +// Package toolfilter pre-filters tools and skills per user turn and routes +// model selection, using a System One decision model (TypeSafe Jev and +// protocol-compatible servers) through the shared internal/systemone client. +// +// The three integrations are concrete implementations over existing +// framework slots; the package adds no generic decision abstraction: +// +// - NewToolCallback returns a model.BeforeModelCallbackStructured that +// rewrites the tool list of each model request. Tools the decision model +// deems irrelevant for the current user message are removed, shrinking +// the prompt. The default keep set protects framework workflow tools, +// including skill loading. Use WithKeepAlways("tool_search") to protect +// tool_search when it is present. +// - NewSkillFilter returns a skill.VisibilityFilter for +// llmagent.WithSkillFilter that narrows the skill overview to the skills +// relevant for the current run. The batch decision is made lazily on the +// first visibility query of a run and cached for the rest of it. +// - NewModelRouter returns an agent.ModelSelector that picks the model for +// a run with one Choice question. The selection is cached per +// invocation, so a ReAct loop pays for one decision per run. +// +// All three fail open: on any decision-model error the behavior degrades to +// "not configured at all" — the full tool catalog stays visible, all skills +// stay visible, and routing falls back to the configured fallback model. +// This is the opposite direction of the approval reviewer +// (plugin/guardrail/approval), which fails closed because it guards safety. +// +// Constructors follow the integration-package convention: an explicit +// base URL plus options, no environment inference, and no internal types in +// exported signatures. The System One endpoint is {base_url}/v1/systemone +// with Bearer authentication. Inject WithHTTPClient for tests and HTTP +// policy. +package toolfilter diff --git a/plugin/toolfilter/modelrouter.go b/plugin/toolfilter/modelrouter.go new file mode 100644 index 0000000000..a60ae97760 --- /dev/null +++ b/plugin/toolfilter/modelrouter.go @@ -0,0 +1,147 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "context" + "fmt" + "strings" + + "trpc.group/trpc-go/trpc-agent-go/agent" + "trpc.group/trpc-go/trpc-agent-go/log" + "trpc.group/trpc-go/trpc-agent-go/model" + + systemoneclient "trpc.group/trpc-go/trpc-agent-go/internal/systemone" +) + +// routeQuestionID is the single question ID of a routing decision. +const routeQuestionID = "model_route" + +// ModelRouter selects the model of each run with one Choice decision. +// Construct it with NewModelRouter and pass Select to +// agent.WithModelSelector (run option) or llmagent.WithModelSelector +// (agent option). The selection is cached per invocation, so a ReAct loop +// pays for one decision per run. +type ModelRouter struct { + client *systemoneclient.Client + candidates []modelCandidate + byName map[string]model.Model + fallback model.Model + cache *runCache[model.Model] +} + +// NewModelRouter creates a model router backed by the System One decision +// API at baseURL. Candidates are registered with WithModelCandidate and must +// be at least one, with nonblank unique names and non-nil models. +// WithFallbackModel sets the model returned on any routing failure; when it +// is not configured, routing failures keep the invocation's base model. +// It returns an error for an invalid endpoint, invalid option values, or an +// invalid candidate table. +func NewModelRouter(baseURL string, opts ...Option) (*ModelRouter, error) { + o := applyOptions(opts) + client, err := newSystemOneClient(baseURL, o) + if err != nil { + return nil, err + } + if len(o.candidates) == 0 { + return nil, fmt.Errorf("toolfilter: at least one WithModelCandidate is required") + } + candidates := make([]modelCandidate, 0, len(o.candidates)) + byName := make(map[string]model.Model, len(o.candidates)) + for _, cand := range o.candidates { + name := strings.TrimSpace(cand.name) + if name == "" { + return nil, fmt.Errorf("toolfilter: model candidate name must not be blank") + } + if cand.m == nil { + return nil, fmt.Errorf("toolfilter: model candidate %q requires a non-nil model", name) + } + if _, dup := byName[name]; dup { + return nil, fmt.Errorf("toolfilter: duplicate model candidate name %q", name) + } + cand.name = name + cand.criteria = strings.TrimSpace(cand.criteria) + candidates = append(candidates, cand) + byName[name] = cand.m + } + return &ModelRouter{ + client: client, + candidates: candidates, + byName: byName, + fallback: o.fallback, + cache: newRunCache[model.Model](defaultCacheSize), + }, nil +} + +// Select implements agent.ModelSelector. It never returns an error: a +// missing user message, a decision failure, or an unknown routed model falls +// back to the WithFallbackModel instance, or — when no fallback is +// configured — returns nil so the invocation's base model is kept. Concurrent +// calls for an invocation share a decision controlled by the first caller's +// context. A canceled waiter returns the fallback without caching it. +func (r *ModelRouter) Select( + ctx context.Context, + inv *agent.Invocation, +) (model.Model, error) { + if inv == nil || inv.InvocationID == "" { + return r.fallback, nil + } + m, ok := r.cache.getOrCompute(ctx, inv.InvocationID, func() (model.Model, bool) { + return r.route(ctx, inv), true + }) + if !ok { + return r.fallback, nil + } + return m, nil +} + +// route makes the Choice decision for one run. +func (r *ModelRouter) route(ctx context.Context, inv *agent.Invocation) model.Model { + if inv == nil || conversationText(inv.Message) == "" { + // No user intent to route on: keep the fallback (or base model). + return r.fallback + } + userMessage, _ := decisionConversation(inv) + + // Registration order is preserved: compatible servers may use + // positional input. + options := make([]systemoneclient.ChoiceOption, 0, len(r.candidates)) + for _, cand := range r.candidates { + options = append(options, systemoneclient.ChoiceOption{ + Name: cand.name, + Description: cand.criteria, + }) + } + resp, err := r.client.SystemOne(ctx, &systemoneclient.Request{ + State: clipRunes(userMessage, stateClipLen), + Questions: map[string]systemoneclient.Question{ + routeQuestionID: systemoneclient.ChoiceQuestion{ + Instructions: "Select the model best suited for this user request.", + Options: options, + }, + }, + }) + if err != nil { + log.WarnfContext(ctx, "toolfilter: model routing failed, using fallback: %v", err) + return r.fallback + } + answer, err := resp.Choice(routeQuestionID) + if err != nil { + log.WarnfContext(ctx, "toolfilter: model routing answer invalid, using fallback: %v", err) + return r.fallback + } + m, ok := r.byName[answer.Choice] + if !ok { + log.WarnfContext( + ctx, "toolfilter: routed model %q is not registered, using fallback", answer.Choice) + return r.fallback + } + return m +} diff --git a/plugin/toolfilter/modelrouter_test.go b/plugin/toolfilter/modelrouter_test.go new file mode 100644 index 0000000000..90f609c465 --- /dev/null +++ b/plugin/toolfilter/modelrouter_test.go @@ -0,0 +1,218 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "trpc.group/trpc-go/trpc-agent-go/agent" + "trpc.group/trpc-go/trpc-agent-go/model" +) + +func TestNewModelRouter_ValidatesCandidates(t *testing.T) { + t.Parallel() + + _, err := NewModelRouter("https://decision.example") + require.ErrorContains(t, err, "WithModelCandidate") + + fast := &stubModel{name: "fast"} + _, err = NewModelRouter("https://decision.example", + WithModelCandidate("fast", fast, "cheap tasks"), + WithModelCandidate("fast", fast, "cheap tasks"), + ) + require.ErrorContains(t, err, "duplicate") + + _, err = NewModelRouter("https://decision.example", + WithModelCandidate(" ", fast, "cheap tasks"), + ) + require.ErrorContains(t, err, "blank") + + _, err = NewModelRouter("https://decision.example", + WithModelCandidate("fast", nil, "cheap tasks"), + ) + require.ErrorContains(t, err, "non-nil") + + _, err = NewModelRouter("") + require.Error(t, err) +} + +func newRoutingEnv( + t *testing.T, + status int, + respond func(map[string]string) string, + opts ...Option, +) (*ModelRouter, *decisionRecorder) { + t.Helper() + + server, rec := decisionServer(t, respond, status) + baseOpts := append([]Option{WithHTTPClient(server.Client())}, opts...) + router, err := NewModelRouter(server.URL, baseOpts...) + require.NoError(t, err) + return router, rec +} + +func routingInv() *agent.Invocation { + return agent.NewInvocation(agent.WithInvocationMessage( + model.NewUserMessage("review this long legal contract"), + )) +} + +// choiceResponder answers the routing question with a fixed two-option +// distribution; choice names outside the requested options are rejected by +// the wire validator, which tests the fail-open path. +func choiceResponder(choice string) func(map[string]string) string { + return func(map[string]string) string { + return choiceResponse("model_route", choice, + map[string]float64{"fast": 0.1, "smart": 0.9}, 0.9) + } +} + +func TestSelect_RoutesToChosenCandidate(t *testing.T) { + t.Parallel() + + smart := &stubModel{name: "smart"} + router, rec := newRoutingEnv(t, http.StatusOK, choiceResponder("smart"), + WithModelCandidate("fast", &stubModel{name: "fast"}, "cheap tasks"), + WithModelCandidate("smart", smart, "complex reasoning"), + ) + + inv := routingInv() + m, err := router.Select(context.Background(), inv) + require.NoError(t, err) + require.Same(t, smart, m) + require.Equal(t, 1, rec.requests()) + + // Cached per invocation: a ReAct loop routes once per run. + m, err = router.Select(context.Background(), inv) + require.NoError(t, err) + require.Same(t, smart, m) + require.Equal(t, 1, rec.requests()) +} + +func TestSelect_ConcurrentWaiterCanCancel(t *testing.T) { + started, release := make(chan struct{}, 8), make(chan struct{}, 1) + defer close(release) + smart := &stubModel{name: "smart"} + fallback := &stubModel{name: "fallback"} + router, rec := newRoutingEnv(t, http.StatusOK, func(q map[string]string) string { + started <- struct{}{} + <-release + return choiceResponder("smart")(q) + }, WithModelCandidate("fast", &stubModel{name: "fast"}, "simple"), + WithModelCandidate("smart", smart, "reasoning"), WithFallbackModel(fallback)) + inv := routingInv() + done := make(chan model.Model, 1) + go func() { m, _ := router.Select(context.Background(), inv); done <- m }() + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("decision did not start") + } + ctx, cancel := context.WithCancel(context.Background()) + waiter := make(chan model.Model, 1) + go func() { m, _ := router.Select(ctx, inv); waiter <- m }() + select { + case <-started: + t.Fatal("waiter started a duplicate decision") + case <-time.After(100 * time.Millisecond): + } + cancel() + select { + case m := <-waiter: + require.Same(t, fallback, m) + case <-time.After(2 * time.Second): + t.Fatal("canceled waiter did not return") + } + require.Equal(t, 1, rec.requests()) + release <- struct{}{} + select { + case m := <-done: + require.Same(t, smart, m) + case <-time.After(2 * time.Second): + t.Fatal("decision did not finish") + } + m, err := router.Select(context.Background(), inv) + require.NoError(t, err) + require.Same(t, smart, m) + require.Equal(t, 1, rec.requests()) +} + +func TestSelect_FailOpenToFallback(t *testing.T) { + t.Parallel() + + fallback := &stubModel{name: "fallback"} + router, rec := newRoutingEnv(t, http.StatusInternalServerError, nil, + WithModelCandidate("fast", &stubModel{name: "fast"}, "cheap tasks"), + WithModelCandidate("smart", &stubModel{name: "smart"}, "reasoning"), + WithFallbackModel(fallback), + ) + + inv := routingInv() + m, err := router.Select(context.Background(), inv) + require.NoError(t, err) + require.Same(t, fallback, m) + require.Equal(t, 1, rec.requests()) + + // The fail-open result is cached for the rest of the run. + m, err = router.Select(context.Background(), inv) + require.NoError(t, err) + require.Same(t, fallback, m) + require.Equal(t, 1, rec.requests()) +} + +func TestSelect_FailOpenKeepsBaseModelWithoutFallback(t *testing.T) { + t.Parallel() + + router, _ := newRoutingEnv(t, http.StatusInternalServerError, nil, + WithModelCandidate("fast", &stubModel{name: "fast"}, "cheap tasks"), + ) + + // A nil model with a nil error keeps the invocation's base model. + m, err := router.Select(context.Background(), routingInv()) + require.NoError(t, err) + require.Nil(t, m) +} + +func TestSelect_BlankMessageFallsBack(t *testing.T) { + t.Parallel() + + fallback := &stubModel{name: "fallback"} + router, rec := newRoutingEnv(t, http.StatusOK, choiceResponder("fast"), + WithModelCandidate("fast", &stubModel{name: "fast"}, "cheap tasks"), + WithFallbackModel(fallback), + ) + + inv := agent.NewInvocation() // no user message + m, err := router.Select(context.Background(), inv) + require.NoError(t, err) + require.Same(t, fallback, m) + require.Equal(t, 0, rec.requests()) +} + +func TestSelect_InvalidAnswerFallsBack(t *testing.T) { + t.Parallel() + + fallback := &stubModel{name: "fallback"} + router, _ := newRoutingEnv(t, http.StatusOK, choiceResponder("unknown-choice"), + WithModelCandidate("fast", &stubModel{name: "fast"}, "cheap tasks"), + WithFallbackModel(fallback), + ) + + // The server validator rejects choices that were not requested, so + // the answer decodes to an error and routing fails open. + m, err := router.Select(context.Background(), routingInv()) + require.NoError(t, err) + require.Same(t, fallback, m) +} diff --git a/plugin/toolfilter/options.go b/plugin/toolfilter/options.go new file mode 100644 index 0000000000..62885e0d4e --- /dev/null +++ b/plugin/toolfilter/options.go @@ -0,0 +1,278 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "fmt" + "math" + "net/http" + "time" + + "trpc.group/trpc-go/trpc-agent-go/knowledge/embedder" + "trpc.group/trpc-go/trpc-agent-go/model" + + systemoneclient "trpc.group/trpc-go/trpc-agent-go/internal/systemone" +) + +// Option configures the constructors of this package. Options are shared +// across NewToolCallback, NewSkillFilter and NewModelRouter; options that do +// not apply to a constructor are ignored by it. +type Option func(*options) + +// Default tuning values, calibrated against the Octop production +// implementation (see docs/design/decision-model.md, appendix A). +const ( + defaultThreshold = 0.6 // keep when P(relevant) >= threshold + defaultSlack = 0.15 // fallback margin when no scored item meets the threshold + defaultMinToolCount = 8 // catalogs below this size are not filtered + defaultShortlistLimit = 16 // lexical recall candidates + defaultMaxQuestions = 20 // questions per decision request + defaultTimeout = 30 * time.Second + + // defaultCacheSize bounds each per-run LRU cache (Octop uses 128). + defaultCacheSize = 128 +) + +type options struct { + apiKey string + model string + timeout time.Duration + httpClient *http.Client + + threshold float64 + slack float64 + minToolCount int + shortlistLimit int + maxQuestions int + keepAlways []string + keepSkills []string + embedder embedder.Embedder + observer func(*Report) + + candidates []modelCandidate + fallback model.Model +} + +// modelCandidate is one routing target of NewModelRouter. +type modelCandidate struct { + name string + m model.Model + criteria string +} + +func defaultOptions() options { + return options{ + threshold: defaultThreshold, + slack: defaultSlack, + minToolCount: defaultMinToolCount, + shortlistLimit: defaultShortlistLimit, + maxQuestions: defaultMaxQuestions, + timeout: defaultTimeout, + } +} + +func applyOptions(opts []Option) options { + o := defaultOptions() + for _, opt := range opts { + if opt != nil { + opt(&o) + } + } + return o +} + +// WithAPIKey sets the bearer credential for the System One endpoint. There +// is no environment fallback: pass os.Getenv results explicitly. +func WithAPIKey(key string) Option { + return func(o *options) { + if key != "" { + o.apiKey = key + } + } +} + +// WithDefaultModel pins the decision model version, for example +// "jev-2025-10-10", for reproducibility. When empty the server default is +// used. +func WithDefaultModel(model string) Option { + return func(o *options) { + if model != "" { + o.model = model + } + } +} + +// WithTimeout sets the per-request timeout. It must be positive; the +// default is 30 seconds. +func WithTimeout(timeout time.Duration) Option { + return func(o *options) { + if timeout > 0 { + o.timeout = timeout + } + } +} + +// WithHTTPClient sets the http.Client used for decision requests. The +// client is shallow-copied. It is intended for tests and HTTP policy. +func WithHTTPClient(client *http.Client) Option { + return func(o *options) { + if client != nil { + o.httpClient = client + } + } +} + +// WithThreshold sets the keep threshold for tool and skill filtering: +// a scored candidate is kept when P(relevant) >= threshold. When none meet +// the threshold, WithSlack determines whether near-threshold candidates are +// kept. It must be in [0, 1]. The default is 0.6. +func WithThreshold(threshold float64) Option { + return func(o *options) { + o.threshold = threshold + } +} + +// WithSlack sets the fallback margin for tool and skill filtering. If no +// scored candidate meets the threshold, candidates scoring at least threshold +// minus slack are kept. Always-kept and unasked candidates do not affect this +// decision. Slack must be in [0, 1]; the default is 0.15. +func WithSlack(slack float64) Option { + return func(o *options) { + o.slack = slack + } +} + +// WithMinToolCount skips filtering when the request carries fewer tools +// than count. The default is 8. +func WithMinToolCount(count int) Option { + return func(o *options) { + if count >= 0 { + o.minToolCount = count + } + } +} + +// WithShortlistLimit bounds tool and skill recall. Catalogs at or below the +// limit are sent directly to the decision model. Larger catalogs use lexical +// recall, optionally fused with embeddings; if recall finds no candidates, +// the catalog remains visible. The limit must be at least 1; the default is 16. +func WithShortlistLimit(limit int) Option { + return func(o *options) { + if limit >= 1 { + o.shortlistLimit = limit + } + } +} + +// WithMaxQuestions bounds the number of questions in one decision request. +// Shortlisted candidates beyond the budget are not asked and are kept. It +// must be at least 1. The default is 20. +func WithMaxQuestions(limit int) Option { + return func(o *options) { + if limit >= 1 { + o.maxQuestions = limit + } + } +} + +// WithKeepAlways adds model-facing tool names that are never scored or +// filtered, in addition to the non-removable framework workflow tool set. +// Use the exact names in the model request, including any ToolSet prefix, +// to protect application-critical tools. Names absent from a request have +// no effect. The default set also matches by name, so application tools +// with the same names as framework workflow tools are kept. +func WithKeepAlways(names ...string) Option { + return func(o *options) { + o.keepAlways = append(o.keepAlways, names...) + } +} + +// WithKeepSkills adds skill names that are always visible without being +// scored. Use it to protect core skills. +func WithKeepSkills(names ...string) Option { + return func(o *options) { + o.keepSkills = append(o.keepSkills, names...) + } +} + +// WithShortlistEmbedder adds an embedding recall path for tool and skill +// catalogs above the shortlist limit, fused with the +// keyword path by reciprocal rank fusion. When the embedder fails, the +// keyword path alone decides (fail-open). Concurrent requests for the same +// text share one embedding operation per filter, with a 30-second timeout; +// each caller may stop waiting when its context is canceled. Successful +// embeddings are cached per filter in separate least-recently-used caches for +// 128 user texts and 1024 candidate descriptions. Evicted entries are +// recomputed when needed. Each filter performs at most 8 embedding calls +// concurrently, including calls for user texts. +func WithShortlistEmbedder(e embedder.Embedder) Option { + return func(o *options) { + if e != nil { + o.embedder = e + } + } +} + +// WithObserver registers a callback invoked once per turn with the +// filtering outcome, including per-tool scores and estimated token savings. +// It runs as part of the shared decision and must be fast and safe for +// concurrent use. If the initiating caller stops waiting, the callback may +// finish after that caller returns. Logging the report is the intended use. +func WithObserver(observer func(*Report)) Option { + return func(o *options) { + if observer != nil { + o.observer = observer + } + } +} + +// WithModelCandidate registers a routing target for NewModelRouter: name +// identifies the candidate on the wire, m is the model instance returned +// when selected, and criteria describes the tasks suited to the model. At +// least one candidate is required. +func WithModelCandidate(name string, m model.Model, criteria string) Option { + return func(o *options) { + o.candidates = append(o.candidates, modelCandidate{name: name, m: m, criteria: criteria}) + } +} + +// WithFallbackModel sets the model returned when routing fails (decision +// error, unknown choice, blank input). When not set, routing failures keep +// the invocation's base model, matching the agent.ModelSelector contract. +func WithFallbackModel(m model.Model) Option { + return func(o *options) { + if m != nil { + o.fallback = m + } + } +} + +// newSystemOneClient builds the shared decision client from resolved +// options. Internal types never leak into exported signatures. +func newSystemOneClient(baseURL string, o options) (*systemoneclient.Client, error) { + client, err := systemoneclient.NewClient(baseURL, + systemoneclient.WithAPIKey(o.apiKey), + systemoneclient.WithDefaultModel(o.model), + systemoneclient.WithTimeout(o.timeout), + systemoneclient.WithHTTPClient(o.httpClient), + ) + if err != nil { + return nil, fmt.Errorf("toolfilter: %w", err) + } + return client, nil +} + +// validateProbability checks a threshold-like option value. +func validateProbability(name string, value float64) error { + if math.IsNaN(value) || value < 0 || value > 1 { + return fmt.Errorf("toolfilter: %s must be in [0, 1], got %v", name, value) + } + return nil +} diff --git a/plugin/toolfilter/recall.go b/plugin/toolfilter/recall.go new file mode 100644 index 0000000000..714dd1f684 --- /dev/null +++ b/plugin/toolfilter/recall.go @@ -0,0 +1,306 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "context" + "math" + "sort" + "strings" + "sync" + "time" + "unicode" + + "golang.org/x/sync/singleflight" +) + +// Candidates missed by recall are not evaluated by the decision model. The +// keyword path is deterministic; an optional embedding path (fused by RRF) +// adds synonym and cross-language recall. + +// rrfK is the reciprocal rank fusion constant. +const rrfK = 60.0 + +const embeddingTimeout = 30 * time.Second + +const ( + userEmbeddingCacheSize = 128 + candidateEmbeddingCacheSize = 1024 + maxEmbeddingConcurrency = 8 +) + +// tokenize extracts lowercase Latin/digit words of at least two characters +// and individual CJK runes. +func tokenize(text string) map[string]struct{} { + tokens := make(map[string]struct{}) + var word strings.Builder + flush := func() { + if word.Len() >= 2 { + tokens[word.String()] = struct{}{} + } + word.Reset() + } + for _, r := range strings.ToLower(text) { + switch { + case isWordRune(r): + word.WriteRune(r) + case isCJKRune(r): + flush() + tokens[string(r)] = struct{}{} + default: + flush() + } + } + flush() + return tokens +} + +func isWordRune(r rune) bool { + return r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '_' +} + +func isCJKRune(r rune) bool { + return unicode.Is(unicode.Han, r) || + unicode.Is(unicode.Hiragana, r) || + unicode.Is(unicode.Katakana, r) || + unicode.Is(unicode.Hangul, r) +} + +// lexicalScore counts distinct query tokens occurring in a candidate's name +// or description, including substring matches. +func lexicalScore(userTokens map[string]struct{}, name, description string) float64 { + if len(userTokens) == 0 { + return 0 + } + score := 0.0 + haystack := strings.ToLower(name + " " + description) + for token := range userTokens { + if strings.Contains(haystack, token) { + score++ + } + } + return score +} + +// scoredCandidate carries one recall candidate with its fused score. +type scoredCandidate struct { + name string + description string + score float64 +} + +// embedCache reuses recently computed user and candidate vectors across turns. +// It is safe for concurrent use. +type embedCache struct { + embedder interface { + GetEmbedding(ctx context.Context, text string) ([]float64, error) + } + users *runCache[[]float64] + candidates *runCache[[]float64] + flight singleflight.Group + permits chan struct{} +} + +func newEmbedCache(embedder interface { + GetEmbedding(ctx context.Context, text string) ([]float64, error) +}) *embedCache { + return &embedCache{ + embedder: embedder, + users: newRunCache[[]float64](userEmbeddingCacheSize), + candidates: newRunCache[[]float64](candidateEmbeddingCacheSize), + permits: make(chan struct{}, maxEmbeddingConcurrency), + } +} + +// vector returns the cached embedding of text, computing it when needed. +// Errors fail open: a missing vector removes the embedding rank only. +func (c *embedCache) vector(ctx context.Context, text string) []float64 { + if c == nil || c.embedder == nil || text == "" { + return nil + } + return c.cachedVector(ctx, text, c.users) +} + +func (c *embedCache) candidateVector(ctx context.Context, text string) []float64 { + if c == nil || c.embedder == nil || text == "" { + return nil + } + return c.cachedVector(ctx, text, c.candidates) +} + +func (c *embedCache) cachedVector(ctx context.Context, text string, cache *runCache[[]float64]) []float64 { + v, ok := cache.get(text) + if ok { + return v + } + if ctx.Err() != nil { + return nil + } + result := c.flight.DoChan(text, func() (any, error) { + v, ok := cache.get(text) + if ok { + return v, nil + } + workCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), embeddingTimeout) + defer cancel() + select { + case c.permits <- struct{}{}: + defer func() { <-c.permits }() + case <-workCtx.Done(): + return nil, workCtx.Err() + } + v, err := c.embedder.GetEmbedding(workCtx, text) + if err != nil || len(v) == 0 { + return nil, err + } + cache.set(text, v) + return v, nil + }) + select { + case <-ctx.Done(): + return nil + case res := <-result: + v, _ := res.Val.([]float64) + if len(v) != 0 { + cache.set(text, v) + } + return v + } +} + +// recallRank ranks candidates for one turn. The keyword path always runs; +// when an embedder is configured and the user message embeds successfully, +// both rankings are fused with reciprocal rank fusion. Candidates are +// ordered by fused score descending, then by name for determinism. +func recallRank( + ctx context.Context, + userMessage string, + candidates []scoredCandidate, + embeddings *embedCache, +) []scoredCandidate { + userTokens := tokenize(userMessage) + if len(userTokens) == 0 || len(candidates) == 0 { + return nil + } + + // Keyword ranking. + keyword := make([]scoredCandidate, len(candidates)) + copy(keyword, candidates) + for i := range keyword { + keyword[i].score = lexicalScore(userTokens, keyword[i].name, keyword[i].description) + } + sortScored(keyword) + + // Embedding ranking, when available. + var userVec []float64 + if embeddings != nil { + userVec = embeddings.vector(ctx, userMessage) + } + + if userVec == nil { + return keyword + } + + type vectorCandidate struct { + name string + sim float64 + } + vectors := make([][]float64, len(candidates)) + jobs := make(chan int) + var workers sync.WaitGroup + workerCount := min(maxEmbeddingConcurrency, len(candidates)) + for i := 0; i < workerCount; i++ { + workers.Add(1) + go func() { + defer workers.Done() + for index := range jobs { + cand := candidates[index] + vectors[index] = embeddings.candidateVector(ctx, candidateText(cand.name, cand.description)) + } + }() + } +dispatch: + for i := range candidates { + select { + case <-ctx.Done(): + break dispatch + case jobs <- i: + } + } + close(jobs) + workers.Wait() + vectorRank := make([]vectorCandidate, 0, len(candidates)) + for i, cand := range candidates { + vec := vectors[i] + if vec == nil { + continue + } + vectorRank = append(vectorRank, vectorCandidate{ + name: cand.name, + sim: cosine(userVec, vec), + }) + } + if len(vectorRank) == 0 { + sortScored(keyword) + return keyword + } + sort.Slice(vectorRank, func(i, j int) bool { + if vectorRank[i].sim != vectorRank[j].sim { + return vectorRank[i].sim > vectorRank[j].sim + } + return vectorRank[i].name < vectorRank[j].name + }) + + // Fuse the two rankings by reciprocal rank fusion. + fused := make(map[string]float64, len(candidates)) + for rank, cand := range keyword { + fused[cand.name] += 1 / (rrfK + float64(rank+1)) + } + for rank, cand := range vectorRank { + fused[cand.name] += 1 / (rrfK + float64(rank+1)) + } + out := make([]scoredCandidate, len(keyword)) + copy(out, keyword) + for i := range out { + out[i].score = fused[out[i].name] + } + sortScored(out) + return out +} + +func candidateText(name, description string) string { + return name + "\n" + description +} + +func sortScored(candidates []scoredCandidate) { + sort.Slice(candidates, func(i, j int) bool { + if candidates[i].score != candidates[j].score { + return candidates[i].score > candidates[j].score + } + return candidates[i].name < candidates[j].name + }) +} + +// cosine computes the cosine similarity of two vectors of equal length. +func cosine(a, b []float64) float64 { + if len(a) == 0 || len(a) != len(b) { + return 0 + } + var dot, normA, normB float64 + for i := range a { + dot += a[i] * b[i] + normA += a[i] * a[i] + normB += b[i] * b[i] + } + if normA == 0 || normB == 0 { + return 0 + } + return dot / (math.Sqrt(normA) * math.Sqrt(normB)) +} diff --git a/plugin/toolfilter/recall_test.go b/plugin/toolfilter/recall_test.go new file mode 100644 index 0000000000..9b2868d22f --- /dev/null +++ b/plugin/toolfilter/recall_test.go @@ -0,0 +1,341 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type controlledEmbedder struct { + mu sync.Mutex + calls map[string]int + started chan string + release chan struct{} +} + +func (e *controlledEmbedder) GetEmbedding(ctx context.Context, text string) ([]float64, error) { + e.mu.Lock() + e.calls[text]++ + e.mu.Unlock() + e.started <- text + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-e.release: + return []float64{1, 2}, nil + } +} + +func TestEmbedCache_CanceledWaiterDoesNotCancelSharedWork(t *testing.T) { + e := &controlledEmbedder{calls: make(map[string]int), started: make(chan string, 4), release: make(chan struct{})} + cache := newEmbedCache(e) + ctx, cancel := context.WithCancel(context.Background()) + first := make(chan []float64, 1) + go func() { first <- cache.vector(ctx, "same") }() + <-e.started + cancel() + select { + case got := <-first: + require.Nil(t, got) + case <-time.After(time.Second): + t.Fatal("canceled caller did not stop waiting") + } + second := make(chan []float64, 1) + go func() { second <- cache.vector(context.Background(), "same") }() + other := make(chan []float64, 1) + go func() { other <- cache.vector(context.Background(), "other") }() + select { + case text := <-e.started: + require.Equal(t, "other", text) + case <-time.After(time.Second): + t.Fatal("different text did not execute concurrently") + } + close(e.release) + require.Equal(t, []float64{1, 2}, <-second) + require.Equal(t, []float64{1, 2}, <-other) + e.mu.Lock() + defer e.mu.Unlock() + require.Equal(t, 1, e.calls["same"]) +} + +func TestTokenize(t *testing.T) { + t.Parallel() + + tokens := tokenize("Find Cheap_Flights2 你好世界 카나다 한글!") + require.Equal(t, map[string]struct{}{ + "find": {}, + "cheap_flights2": {}, + "你": {}, + "好": {}, + "世": {}, + "界": {}, + "카": {}, + "나": {}, + "다": {}, + "한": {}, + "글": {}, + }, tokens) +} + +func TestLexicalScore_SubstringAndDistinctTokens(t *testing.T) { + t.Parallel() + user := tokenize("search search") + require.Equal(t, 1.0, lexicalScore(user, "websearch", "engine")) +} + +func TestLexicalScore_NameAndDescriptionCountOnce(t *testing.T) { + t.Parallel() + + user := tokenize("flights") + nameOnly := lexicalScore(user, "flights", "nothing matches here") + descOnly := lexicalScore(user, "no match", "flights") + require.InDelta(t, 1.0, nameOnly, 1e-9) + require.InDelta(t, 1.0, descOnly, 1e-9) + require.InDelta(t, 0.0, lexicalScore(user, "calendar", "events"), 1e-9) +} + +func TestRecallRank_DeterministicOrderAndAbort(t *testing.T) { + t.Parallel() + + candidates := []scoredCandidate{ + {name: "beta", description: "book flights"}, + {name: "alpha", description: "book flights"}, + {name: "gamma", description: "calendar events"}, + } + + // Tied scores fall back to name order. + ranked := recallRank(context.Background(), "book flights", candidates, nil) + require.Equal(t, "alpha", ranked[0].name) + require.Equal(t, "beta", ranked[1].name) + require.Equal(t, "gamma", ranked[2].name) + + // No user tokens: no recall at all. Tokens that overlap nothing rank + // every candidate at zero, which the caller treats as abort. + require.Empty(t, recallRank(context.Background(), "", candidates, nil)) + ranked = recallRank(context.Background(), "你好", candidates, nil) + require.Equal(t, 0.0, ranked[0].score) +} + +func TestRecallRank_EmbeddingFusion(t *testing.T) { + t.Parallel() + + // Keyword order: alpha, beta, delta, gamma (gamma and delta tie at 0, + // delta wins by name). Vector order: gamma, alpha, beta, delta. RRF + // fuses the two so gamma — keyword rank 4, vector rank 1 — beats beta. + embeddings := map[string][]float64{ + "book flights now": {1, 0}, + "alpha\nflights now": {0.9, 0.2}, + "beta\nbook": {0.5, 0.8}, + "gamma\nzzz unrelated": {1, 0.05}, + "delta\ncalendar events": {0.05, 1}, + } + cache := newEmbedCache(&stubEmbedder{vectors: embeddings}) + + candidates := []scoredCandidate{ + {name: "alpha", description: "flights now"}, + {name: "beta", description: "book"}, + {name: "gamma", description: "zzz unrelated"}, + {name: "delta", description: "calendar events"}, + } + + keywordOnly := recallRank(context.Background(), "book flights now", candidates, nil) + require.Equal(t, "beta", keywordOnly[1].name) + + fused := recallRank(context.Background(), "book flights now", candidates, cache) + require.Equal(t, "alpha", fused[0].name) + require.Equal(t, "gamma", fused[1].name) +} + +func TestRecallRank_BoundsConcurrentCandidateEmbeddings(t *testing.T) { + e := &controlledEmbedder{calls: make(map[string]int), started: make(chan string, 64), release: make(chan struct{})} + cache := newEmbedCache(e) + cache.users.set("query", []float64{1, 2}) + candidates := make([]scoredCandidate, 24) + for i := range candidates { + candidates[i] = scoredCandidate{name: fmt.Sprintf("candidate %02d", i)} + } + done := make(chan []scoredCandidate, 2) + for i := 0; i < 2; i++ { + go func() { done <- recallRank(context.Background(), "query", candidates, cache) }() + } + defer func() { + select { + case <-e.release: + default: + close(e.release) + } + }() + for i := 0; i < 8; i++ { + select { + case <-e.started: + case <-time.After(2 * time.Second): + t.Fatal("candidate embeddings did not run concurrently") + } + } + select { + case <-e.started: + t.Fatal("embedding concurrency exceeded the per-filter limit") + case <-time.After(100 * time.Millisecond): + } + close(e.release) + for i := 0; i < 2; i++ { + select { + case ranked := <-done: + require.Len(t, ranked, len(candidates)) + case <-time.After(2 * time.Second): + t.Fatal("recall did not complete") + } + } + e.mu.Lock() + defer e.mu.Unlock() + require.Len(t, e.calls, len(candidates)) + for _, count := range e.calls { + require.Equal(t, 1, count) + } +} + +func TestRecallRank_CancellationStopsWaitingForCandidates(t *testing.T) { + e := &controlledEmbedder{calls: make(map[string]int), started: make(chan string, 32), release: make(chan struct{})} + cache := newEmbedCache(e) + cache.users.set("query", []float64{1, 2}) + candidates := make([]scoredCandidate, 24) + for i := range candidates { + candidates[i] = scoredCandidate{name: fmt.Sprintf("candidate %02d", i)} + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + defer close(e.release) + done := make(chan struct{}) + go func() { + recallRank(ctx, "query", candidates, cache) + close(done) + }() + for i := 0; i < 8; i++ { + select { + case <-e.started: + case <-time.After(2 * time.Second): + t.Fatal("candidate embedding did not start") + } + } + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("recall did not stop waiting after cancellation") + } + e.mu.Lock() + defer e.mu.Unlock() + require.Len(t, e.calls, 8) +} + +func TestEmbedCache_FailsOpenAndCaches(t *testing.T) { + t.Parallel() + + stub := &stubEmbedder{vectors: map[string][]float64{"cached": {1, 1}}} + cache := newEmbedCache(stub) + + require.NotNil(t, cache.vector(context.Background(), "cached")) + require.Equal(t, 1, stub.calls()) // second read hits the cache + require.NotNil(t, cache.vector(context.Background(), "cached")) + require.Equal(t, 1, stub.calls()) + + stub.err = errors.New("embedding failed") + require.Nil(t, cache.vector(context.Background(), "broken")) + // Failed embeddings are not cached: the next call retries. + require.Nil(t, cache.vector(context.Background(), "broken")) + require.Equal(t, 3, stub.calls()) +} + +func TestEmbedCache_BoundsUserAndCandidateVectors(t *testing.T) { + stub := &stubEmbedder{vectors: make(map[string][]float64)} + cache := newEmbedCache(stub) + ctx := context.Background() + stub.vectors["candidate"] = []float64{1} + require.NotNil(t, cache.candidateVector(ctx, "candidate")) + for i := 0; i <= 128; i++ { + text := fmt.Sprintf("user %d", i) + stub.vectors[text] = []float64{1} + require.NotNil(t, cache.vector(ctx, text)) + } + calls := stub.calls() + require.NotNil(t, cache.candidateVector(ctx, "candidate")) + require.Equal(t, calls, stub.calls(), "user messages must not evict candidate vectors") + require.NotNil(t, cache.vector(ctx, "user 0")) + require.Equal(t, calls+1, stub.calls(), "old user vector must be evicted") + require.NotNil(t, cache.vector(ctx, "user 2")) + stub.vectors["user new"] = []float64{1} + require.NotNil(t, cache.vector(ctx, "user new")) + calls = stub.calls() + require.NotNil(t, cache.vector(ctx, "user 2")) + require.Equal(t, calls, stub.calls(), "recently accessed vector must be retained") + require.NotNil(t, cache.vector(ctx, "user 3")) + require.Equal(t, calls+1, stub.calls(), "least recently used vector must be evicted") + + for i := 0; i <= 1024; i++ { + text := fmt.Sprintf("candidate %d", i) + stub.vectors[text] = []float64{1} + require.NotNil(t, cache.candidateVector(ctx, text)) + } + calls = stub.calls() + require.NotNil(t, cache.candidateVector(ctx, "candidate")) + require.Equal(t, calls+1, stub.calls(), "old candidate vector must be evicted") + require.NotNil(t, cache.vector(ctx, "user 0")) + require.Equal(t, calls+1, stub.calls(), "candidate vectors must not evict user messages") +} + +func TestCosine(t *testing.T) { + t.Parallel() + + require.InDelta(t, 1.0, cosine([]float64{1, 0}, []float64{2, 0}), 1e-9) + require.InDelta(t, 0.0, cosine([]float64{1, 0}, []float64{0, 1}), 1e-9) + require.InDelta(t, 0.0, cosine([]float64{1, 0}, []float64{1}), 1e-9) + require.InDelta(t, 0.0, cosine(nil, nil), 1e-9) +} + +// stubEmbedder is a deterministic embedding source for tests. +type stubEmbedder struct { + mu sync.Mutex + vectors map[string][]float64 + err error + n int +} + +func (e *stubEmbedder) GetEmbedding( + ctx context.Context, + text string, +) ([]float64, error) { + e.mu.Lock() + defer e.mu.Unlock() + e.n++ + if e.err != nil { + return nil, e.err + } + return e.vectors[text], nil +} + +func (e *stubEmbedder) calls() int { + e.mu.Lock() + defer e.mu.Unlock() + return e.n +} + +func (e *stubEmbedder) GetEmbeddingWithUsage(ctx context.Context, text string) ([]float64, map[string]any, error) { + v, err := e.GetEmbedding(ctx, text) + return v, nil, err +} + +func (e *stubEmbedder) GetDimensions() int { return 0 } diff --git a/plugin/toolfilter/report.go b/plugin/toolfilter/report.go new file mode 100644 index 0000000000..66e9f3bb0e --- /dev/null +++ b/plugin/toolfilter/report.go @@ -0,0 +1,74 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "unicode/utf8" +) + +// Report catalogs. +const ( + // CatalogTool identifies a tool-filtering report. + CatalogTool = "tool" + // CatalogSkill identifies a skill-filtering report. + CatalogSkill = "skill" +) + +// Report outcomes. +const ( + // OutcomeUntouched means the catalog was left unchanged: below + // minToolCount, no scoreable candidates, no user message, lexical + // zero overlap, all candidates retained, or a cached untouched result. + OutcomeUntouched = "untouched" + // OutcomeNarrowed means tools or skills were removed from the catalog. + OutcomeNarrowed = "narrowed" + // OutcomeFailOpen means the decision call failed and the full + // catalog was kept. + OutcomeFailOpen = "fail_open" +) + +// Report describes the outcome of one tool- or skill-filtering decision for +// observability. Cached decision replays are not reported again. Tool and +// skill decisions use the invocation ID as TurnKey when an invocation exists. +type Report struct { + // Catalog identifies the filtered catalog as CatalogTool or CatalogSkill. + // Its zero value is unspecified. + Catalog string + // Outcome is one of OutcomeUntouched, OutcomeNarrowed or + // OutcomeFailOpen. + Outcome string + // TurnKey identifies the invocation; it may be empty for untouched reports. + TurnKey string + // Reason explains an untouched or fail-open outcome. + Reason string + // Asked lists the candidate names that were scored in this turn. + Asked []string + // Unasked lists candidates beyond the question budget; they are kept + // without a score. + Unasked []string + // Dropped lists the removed candidate names (empty unless narrowed). + Dropped []string + // Scores maps each asked candidate name to its P(relevant) score. + Scores map[string]float64 + // CatalogTokensBefore and CatalogTokensAfter estimate the request + // token cost of the tool catalog (or skill overview) before and after + // filtering, and SavedTokens their difference. The estimate is one + // fourth of the rune counts of names and descriptions — a heuristic + // for observability, not billing. + CatalogTokensBefore int + CatalogTokensAfter int + SavedTokens int +} + +// estimateToolTokens approximates the request cost of one tool catalog +// entry. The divisor 4 mirrors the common characters-per-token ratio. +func estimateToolTokens(name, description string) int { + return (utf8.RuneCountInString(name) + utf8.RuneCountInString(description)) / 4 +} diff --git a/plugin/toolfilter/runcache.go b/plugin/toolfilter/runcache.go new file mode 100644 index 0000000000..105dd9e817 --- /dev/null +++ b/plugin/toolfilter/runcache.go @@ -0,0 +1,120 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "container/list" + "context" + "sync" + + "golang.org/x/sync/singleflight" +) + +// runCache is a fixed-size LRU cache keyed by string. It is safe for +// concurrent use. +type runCache[T any] struct { + mu sync.Mutex + max int + order *list.List // front = most recent + items map[string]*list.Element // element.Value = *entry[T] + flight singleflight.Group +} + +// getOrCompute shares in-flight work by key. A canceled caller stops waiting; +// the context captured by compute controls the shared work itself. +func (c *runCache[T]) getOrCompute(ctx context.Context, key string, compute func() (T, bool)) (T, bool) { + if value, ok := c.get(key); ok { + return value, true + } + result := c.flight.DoChan(key, func() (any, error) { + if value, ok := c.get(key); ok { + return entry[T]{value: value}, nil + } + value, cacheable := compute() + if cacheable { + c.set(key, value) + } + return entry[T]{value: value}, nil + }) + select { + case <-ctx.Done(): + var zero T + return zero, false + case result := <-result: + return result.Val.(entry[T]).value, true + } +} + +type entry[T any] struct { + key string + value T +} + +func newRunCache[T any](max int) *runCache[T] { + if max < 1 { + max = 1 + } + return &runCache[T]{ + max: max, + order: list.New(), + items: make(map[string]*list.Element, max), + } +} + +// get returns the cached value for key when present, marking it recent. +func (c *runCache[T]) get(key string) (T, bool) { + var zero T + if c == nil { + return zero, false + } + c.mu.Lock() + defer c.mu.Unlock() + el, ok := c.items[key] + if !ok { + return zero, false + } + c.order.MoveToFront(el) + return el.Value.(*entry[T]).value, true +} + +// set stores the value for key, evicting the least recently used entry when +// the cache is full. +func (c *runCache[T]) set(key string, value T) { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if el, ok := c.items[key]; ok { + el.Value.(*entry[T]).value = value + c.order.MoveToFront(el) + return + } + el := c.order.PushFront(&entry[T]{key: key, value: value}) + c.items[key] = el + for c.order.Len() > c.max { + oldest := c.order.Back() + if oldest == nil { + break + } + c.order.Remove(oldest) + delete(c.items, oldest.Value.(*entry[T]).key) + } +} + +// len reports the number of cached entries, for tests. +func (c *runCache[T]) len() int { + if c == nil { + return 0 + } + c.mu.Lock() + defer c.mu.Unlock() + return c.order.Len() +} diff --git a/plugin/toolfilter/runcache_test.go b/plugin/toolfilter/runcache_test.go new file mode 100644 index 0000000000..de3f413a84 --- /dev/null +++ b/plugin/toolfilter/runcache_test.go @@ -0,0 +1,71 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRunCache_EvictsLeastRecentlyUsed(t *testing.T) { + t.Parallel() + + cache := newRunCache[int](2) + cache.set("a", 1) + cache.set("b", 2) + + v, ok := cache.get("a") + require.True(t, ok) + require.Equal(t, 1, v) // "a" is now most recent + + cache.set("c", 3) // evicts "b" + _, ok = cache.get("b") + require.False(t, ok) + v, ok = cache.get("a") + require.True(t, ok) + require.Equal(t, 1, v) + + cache.set("a", 9) // update in place + v, ok = cache.get("a") + require.True(t, ok) + require.Equal(t, 9, v) + require.Equal(t, 2, cache.len()) +} + +func TestRunCache_ConcurrentAccess(t *testing.T) { + t.Parallel() + + cache := newRunCache[int](16) + var wg sync.WaitGroup + for i := 0; i < 32; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + key := string(rune('a' + i%26)) + cache.set(key, i) + cache.get(key) + cache.len() + }(i) + } + wg.Wait() + require.LessOrEqual(t, cache.len(), 16) +} + +func TestRunCache_NilSafety(t *testing.T) { + t.Parallel() + + var cache *runCache[int] + _, ok := cache.get("a") + require.False(t, ok) + cache.set("a", 1) + require.Equal(t, 0, cache.len()) +} diff --git a/plugin/toolfilter/skillfilter.go b/plugin/toolfilter/skillfilter.go new file mode 100644 index 0000000000..25ec6f96b6 --- /dev/null +++ b/plugin/toolfilter/skillfilter.go @@ -0,0 +1,372 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "context" + "fmt" + "sort" + "strings" + + "trpc.group/trpc-go/trpc-agent-go/agent" + "trpc.group/trpc-go/trpc-agent-go/log" + "trpc.group/trpc-go/trpc-agent-go/skill" + + systemoneclient "trpc.group/trpc-go/trpc-agent-go/internal/systemone" +) + +// SkillFilter narrows the visible skill set of each run with one batch +// decision request. Construct it with NewSkillFilter and pass the filter +// returned by VisibilityFilter to llmagent.WithSkillFilter. The framework +// calls the VisibilityFilter once per skill summary per LLM step; the batch +// decision is computed on the first call of a run and cached, so a ReAct +// loop pays for one decision request per run. +type SkillFilter struct { + client *systemoneclient.Client + repo skill.Repository + threshold float64 + slack float64 + maxAsk int + shortlist int + embeddings *embedCache + keepSet map[string]struct{} + observer func(*Report) + cache *runCache[*skillRunResult] +} + +// skillRunResult is the cached decision of one run. kept == nil means every +// skill is visible: fail-open, or nothing worth deciding on. +type skillRunResult struct { + outcome string + kept map[string]struct{} + scores map[string]float64 +} + +// NewSkillFilter creates a skill visibility filter backed by the System One +// decision API at baseURL. The decision scores the summaries of repo, so +// repo must be the same repository mounted on the agent with +// llmagent.WithSkills. The returned filter fails open: decision errors keep +// every skill visible, which is behaviorally "no filter configured". Decision +// context uses session events matching the framework's default prefix filter. +// It returns an error for a nil repository, an invalid endpoint, or invalid +// option values. Concurrent calls for an invocation share a decision controlled +// by the first caller's context. A canceled waiter leaves skills visible without +// caching its result. +func NewSkillFilter( + baseURL string, + repo skill.Repository, + opts ...Option, +) (*SkillFilter, error) { + if repo == nil { + return nil, fmt.Errorf("toolfilter: skill repository is required") + } + o := applyOptions(opts) + if err := validateProbability("threshold", o.threshold); err != nil { + return nil, err + } + if err := validateProbability("slack", o.slack); err != nil { + return nil, err + } + client, err := newSystemOneClient(baseURL, o) + if err != nil { + return nil, err + } + keepSet := make(map[string]struct{}, len(o.keepSkills)) + for _, name := range o.keepSkills { + if name = strings.TrimSpace(name); name != "" { + keepSet[name] = struct{}{} + } + } + var embeddings *embedCache + if o.embedder != nil { + embeddings = newEmbedCache(o.embedder) + } + return &SkillFilter{ + client: client, + repo: repo, + threshold: o.threshold, + slack: o.slack, + maxAsk: o.maxQuestions, + shortlist: o.shortlistLimit, + embeddings: embeddings, + keepSet: keepSet, + observer: o.observer, + cache: newRunCache[*skillRunResult](defaultCacheSize), + }, nil +} + +// VisibilityFilter returns the skill.VisibilityFilter for +// llmagent.WithSkillFilter. Skills without a run context (no invocation or +// no user message) are always visible: there is no user intent to decide on. +func (f *SkillFilter) VisibilityFilter() skill.VisibilityFilter { + return func(ctx context.Context, summary skill.Summary) bool { + return f.visible(ctx, summary) + } +} + +// visible answers one visibility query from the cached batch decision of the +// current run. +func (f *SkillFilter) visible(ctx context.Context, summary skill.Summary) bool { + inv, _ := agent.InvocationFromContext(ctx) + if inv == nil || inv.InvocationID == "" || + conversationText(inv.Message) == "" { + return true + } + result := f.runResult(ctx, inv) + if result.kept == nil { + return true + } + _, ok := result.kept[summary.Name] + return ok +} + +// runResult returns the batch decision of one run, computing it on first +// use. Concurrent first calls of the same run share the decision so it +// is requested once; concurrent different runs decide in parallel. +func (f *SkillFilter) runResult( + ctx context.Context, + inv *agent.Invocation, +) *skillRunResult { + key := inv.InvocationID + result, ok := f.cache.getOrCompute(ctx, key, func() (*skillRunResult, bool) { + result, report := f.decide(ctx, inv) + // Publish before notifying the observer so a reentrant call can replay it. + f.cache.set(key, result) + f.emit(report) + return result, true + }) + if !ok { + return &skillRunResult{outcome: OutcomeFailOpen} + } + return result +} + +// skillCandidate is one skill of the batch decision. +type skillCandidate struct { + name string + description string +} + +// decide scores every non-kept skill of the repository in one batch request +// and computes the visibility set. The result is also cached for fail-open +// outcomes, so a failing endpoint does not add a decision request to every +// LLM step of the run. +func (f *SkillFilter) decide(ctx context.Context, inv *agent.Invocation) (*skillRunResult, *Report) { + summaries := f.repo.Summaries() + before := skillTokens(summaries) + + userMessage, userHistory := decisionConversation(inv) + + candidates := make([]skillCandidate, 0, len(summaries)) + seen := make(map[string]struct{}, len(summaries)) + names := make([]string, 0, len(summaries)) + for _, s := range summaries { + name := strings.TrimSpace(s.Name) + if name == "" { + continue + } + if _, dup := seen[name]; dup { + continue + } + seen[name] = struct{}{} + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if _, keep := f.keepSet[name]; keep { + continue + } + candidates = append(candidates, skillCandidate{ + name: name, + description: clipRunes(skillDescription(summaries, name), descriptionClipLen), + }) + } + + if len(candidates) == 0 { + return &skillRunResult{outcome: OutcomeUntouched}, &Report{ + Outcome: OutcomeUntouched, TurnKey: inv.InvocationID, + Reason: "no scoreable skills", + CatalogTokensBefore: before, + CatalogTokensAfter: before, + } + } + + if len(candidates) > f.shortlist { + scoreable := make([]scoredCandidate, 0, len(candidates)) + for _, cand := range candidates { + scoreable = append(scoreable, scoredCandidate{name: cand.name, description: cand.description}) + } + ranked := recallRank(ctx, userHistory, scoreable, f.embeddings) + candidates = nil + for _, cand := range ranked { + if cand.score <= 0 || len(candidates) == f.shortlist { + break + } + candidates = append(candidates, skillCandidate{name: cand.name, description: cand.description}) + } + if len(candidates) == 0 { + return &skillRunResult{outcome: OutcomeUntouched}, &Report{Outcome: OutcomeUntouched, TurnKey: inv.InvocationID, + Reason: "no lexical overlap with the user history", CatalogTokensBefore: before, CatalogTokensAfter: before} + } + } + + // Skills beyond the question + // budget are not asked and stay visible. + asked := candidates + var unasked []string + if len(asked) > f.maxAsk { + for _, cand := range asked[f.maxAsk:] { + unasked = append(unasked, cand.name) + } + asked = asked[:f.maxAsk] + } + + questions := make(map[string]systemoneclient.Question, len(asked)) + skillCatalog := make([]any, 0, len(asked)) + for _, cand := range asked { + questions["skill:"+cand.name] = systemoneclient.BinaryQuestion{ + Instructions: fmt.Sprintf( + "Does the current user request need the skill %q? Skill description: %s", + cand.name, cand.description), + } + skillCatalog = append(skillCatalog, map[string]any{ + "name": cand.name, + "description": cand.description, + }) + } + + resp, err := f.client.SystemOne(ctx, &systemoneclient.Request{ + State: map[string]any{ + "user_message": userMessage, + "skill_catalog": skillCatalog, + }, + Questions: questions, + }) + if err != nil { + // Fail open: every skill stays visible, including unscored ones. + log.WarnfContext(ctx, "toolfilter: skill decision failed, keeping all skills: %v", err) + return &skillRunResult{outcome: OutcomeFailOpen}, &Report{ + Outcome: OutcomeFailOpen, TurnKey: inv.InvocationID, + Reason: err.Error(), + Asked: skillNames(asked), + Unasked: unasked, + CatalogTokensBefore: before, CatalogTokensAfter: before, + } + } + + kept := make(map[string]struct{}, len(asked)+len(unasked)+len(f.keepSet)) + for name := range f.keepSet { + kept[name] = struct{}{} + } + for _, name := range unasked { + kept[name] = struct{}{} + } + scores := make(map[string]float64, len(asked)) + matched := false + for _, cand := range asked { + answer, err := resp.Binary("skill:" + cand.name) + if err != nil { + // Strict validation guarantees every asked question is + // answered; treat a defensive miss as visible. + kept[cand.name] = struct{}{} + continue + } + scores[cand.name] = answer.Probability + if answer.Probability >= f.threshold { + kept[cand.name] = struct{}{} + matched = true + } + } + if !matched { + for name, score := range scores { + if score >= f.threshold-f.slack { + kept[name] = struct{}{} + } + } + } + + dropped := make([]string, 0) + for _, name := range names { + if _, ok := kept[name]; !ok { + dropped = append(dropped, name) + } + } + outcome := OutcomeNarrowed + reason := "" + if len(dropped) == 0 { + outcome = OutcomeUntouched + reason = "all candidates retained" + } + report := &Report{ + Outcome: outcome, + Reason: reason, + TurnKey: inv.InvocationID, + Asked: skillNames(asked), + Unasked: unasked, + Dropped: dropped, + Scores: scores, + CatalogTokensBefore: before, + CatalogTokensAfter: before - skillTokensFor(summaries, dropped), + } + report.SavedTokens = report.CatalogTokensBefore - report.CatalogTokensAfter + return &skillRunResult{outcome: outcome, kept: kept, scores: scores}, report +} + +// emit delivers the report: one log line always, plus the WithObserver +// callback when configured. +func (f *SkillFilter) emit(report *Report) { + report.Catalog = CatalogSkill + log.InfofContext( + context.Background(), + "toolfilter: skills outcome=%s asked=%d dropped=%d saved_tokens~%d reason=%q", + report.Outcome, len(report.Asked), len(report.Dropped), report.SavedTokens, report.Reason, + ) + if f.observer != nil { + f.observer(report) + } +} + +// skillDescription finds the description of one skill by name, tolerating +// missing summaries. +func skillDescription(summaries []skill.Summary, name string) string { + for _, s := range summaries { + if s.Name == name { + return s.Description + } + } + return "" +} + +// skillTokens estimates the token cost of the visible skill overview. +func skillTokens(summaries []skill.Summary) int { + total := 0 + for _, s := range summaries { + total += estimateToolTokens(s.Name, s.Description) + } + return total +} + +// skillTokensFor estimates the token cost of the named skills. +func skillTokensFor(summaries []skill.Summary, names []string) int { + total := 0 + for _, name := range names { + total += estimateToolTokens(name, skillDescription(summaries, name)) + } + return total +} + +// skillNames extracts candidate names in order. +func skillNames(candidates []skillCandidate) []string { + names := make([]string, 0, len(candidates)) + for _, cand := range candidates { + names = append(names, cand.name) + } + return names +} diff --git a/plugin/toolfilter/skillfilter_test.go b/plugin/toolfilter/skillfilter_test.go new file mode 100644 index 0000000000..62a67e0e33 --- /dev/null +++ b/plugin/toolfilter/skillfilter_test.go @@ -0,0 +1,415 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "context" + "math" + "net/http" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "trpc.group/trpc-go/trpc-agent-go/agent" + "trpc.group/trpc-go/trpc-agent-go/event" + "trpc.group/trpc-go/trpc-agent-go/model" + "trpc.group/trpc-go/trpc-agent-go/session" + "trpc.group/trpc-go/trpc-agent-go/skill" +) + +func TestNewSkillFilter_RequiresRepository(t *testing.T) { + t.Parallel() + + _, err := NewSkillFilter("https://decision.example", nil) + require.ErrorContains(t, err, "repository") + + _, err = NewSkillFilter("", &fakeSkillRepo{}) + require.Error(t, err) + + _, err = NewSkillFilter("https://decision.example", &fakeSkillRepo{}, WithThreshold(2)) + require.ErrorContains(t, err, "threshold") + + _, err = NewSkillFilter("https://decision.example", &fakeSkillRepo{}, WithThreshold(math.NaN())) + require.ErrorContains(t, err, "threshold") + + _, err = NewSkillFilter("https://decision.example", &fakeSkillRepo{}, WithSlack(math.NaN())) + require.ErrorContains(t, err, "slack") +} + +func testSkillSummaries() []skill.Summary { + return []skill.Summary{ + {Name: "travel-booking", Description: "find and book flights"}, + {Name: "expense-report", Description: "file expense reports"}, + {Name: "math-helper", Description: "arithmetic and algebra"}, + {Name: "cooking", Description: "recipes and menus"}, + {Name: "core-skill", Description: "core workflow"}, + } +} + +func skillFilterEnv( + t *testing.T, + status int, + overrides map[string]float64, + opts ...Option, +) (*SkillFilter, *decisionRecorder, skill.VisibilityFilter, context.Context) { + t.Helper() + + server, rec := decisionServer(t, binaryResponder(overrides, 0.05), status) + baseOpts := append([]Option{ + WithHTTPClient(server.Client()), + WithKeepSkills("core-skill"), + }, opts...) + f, err := NewSkillFilter(server.URL, &fakeSkillRepo{summaries: testSkillSummaries()}, baseOpts...) + require.NoError(t, err) + + inv := agent.NewInvocation(agent.WithInvocationMessage( + model.NewUserMessage("book a flight to paris"), + )) + ctx := agent.NewInvocationContext(context.Background(), inv) + return f, rec, f.VisibilityFilter(), ctx +} + +func TestVisibility_NarrowsSkills(t *testing.T) { + t.Parallel() + + _, rec, filter, ctx := skillFilterEnv(t, http.StatusOK, map[string]float64{ + "skill:travel-booking": 0.9, + "skill:expense-report": 0.5, // below threshold, within slack + }) + + visible := map[string]bool{} + for _, s := range testSkillSummaries() { + visible[s.Name] = filter(ctx, s) + } + require.Equal(t, map[string]bool{ + "travel-booking": true, // approved + "expense-report": false, // threshold met by another skill + "math-helper": false, + "cooking": false, + "core-skill": true, // WithKeepSkills + }, visible) + require.Equal(t, 1, rec.requests()) // one batch decision for the run +} + +func TestVisibility_ShortlistsFromUserHistory(t *testing.T) { + t.Parallel() + _, rec, filter, ctx := skillFilterEnv(t, http.StatusOK, map[string]float64{ + "skill:travel-booking": 0.9, + }, WithShortlistLimit(1)) + inv, _ := agent.InvocationFromContext(ctx) + inv.Session = &session.Session{Events: []event.Event{conversationEvent(model.RoleUser, "book a flight to paris", "")}} + inv.Message = model.NewUserMessage("continue") + require.True(t, filter(ctx, skill.Summary{Name: "travel-booking"})) + require.Contains(t, rec.lastQuestions(), "skill:travel-booking") + require.Len(t, rec.lastQuestions(), 1) +} + +func TestVisibility_NoLexicalMatchKeepsSkills(t *testing.T) { + t.Parallel() + _, rec, filter, ctx := skillFilterEnv(t, http.StatusOK, nil, WithShortlistLimit(1)) + inv, _ := agent.InvocationFromContext(ctx) + inv.Message = model.NewUserMessage("xyzzy") + require.True(t, filter(ctx, skill.Summary{Name: "travel-booking"})) + require.Equal(t, 0, rec.requests()) +} + +func TestVisibility_SlackFallbackWithKeepSkills(t *testing.T) { + t.Parallel() + _, _, filter, ctx := skillFilterEnv(t, http.StatusOK, map[string]float64{ + "skill:expense-report": 0.5, + }) + require.True(t, filter(ctx, skill.Summary{Name: "core-skill"})) + require.True(t, filter(ctx, skill.Summary{Name: "expense-report"})) + require.False(t, filter(ctx, skill.Summary{Name: "math-helper"})) +} + +func TestVisibility_SlackFallbackWithUnaskedSkills(t *testing.T) { + t.Parallel() + _, _, filter, ctx := skillFilterEnv(t, http.StatusOK, map[string]float64{ + "skill:cooking": 0.5, + }, WithMaxQuestions(1)) + require.True(t, filter(ctx, skill.Summary{Name: "cooking"})) + require.True(t, filter(ctx, skill.Summary{Name: "expense-report"})) +} + +func TestVisibility_CachesPerInvocation(t *testing.T) { + t.Parallel() + + _, rec, filter, ctx := skillFilterEnv(t, http.StatusOK, map[string]float64{ + "skill:travel-booking": 0.9, + }) + + // The framework asks per skill; only the first call decides. + for i := 0; i < len(testSkillSummaries()); i++ { + filter(ctx, skill.Summary{Name: "travel-booking"}) + } + require.Equal(t, 1, rec.requests()) + + // A new invocation reuses the same user message but is a new run. + inv := agent.NewInvocation(agent.WithInvocationMessage( + model.NewUserMessage("book a flight to paris"), + )) + newCtx := agent.NewInvocationContext(context.Background(), inv) + filter(newCtx, skill.Summary{Name: "travel-booking"}) + require.Equal(t, 2, rec.requests()) +} + +func TestVisibility_FailOpenOnError(t *testing.T) { + t.Parallel() + + _, rec, filter, ctx := skillFilterEnv(t, http.StatusInternalServerError, nil) + + for _, s := range testSkillSummaries() { + require.True(t, filter(ctx, s)) + } + require.Equal(t, 1, rec.requests()) + + // The fail-open result is cached for the rest of the run. + filter(ctx, skill.Summary{Name: "travel-booking"}) + require.Equal(t, 1, rec.requests()) +} + +func TestVisibility_NoInvocationFailsOpen(t *testing.T) { + t.Parallel() + + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + f, err := NewSkillFilter(server.URL, + &fakeSkillRepo{summaries: testSkillSummaries()}, + WithHTTPClient(server.Client()), + ) + require.NoError(t, err) + filter := f.VisibilityFilter() + + require.True(t, filter(context.Background(), skill.Summary{Name: "math-helper"})) + require.True(t, filter(context.Background(), skill.Summary{Name: "cooking"})) + require.Equal(t, 0, rec.requests()) +} + +func TestVisibility_BlankMessageFailsOpen(t *testing.T) { + t.Parallel() + + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + f, err := NewSkillFilter(server.URL, + &fakeSkillRepo{summaries: testSkillSummaries()}, + WithHTTPClient(server.Client()), + ) + require.NoError(t, err) + filter := f.VisibilityFilter() + + inv := agent.NewInvocation() // no message + ctx := agent.NewInvocationContext(context.Background(), inv) + require.True(t, filter(ctx, skill.Summary{Name: "math-helper"})) + require.Equal(t, 0, rec.requests()) +} + +func TestVisibility_ConcurrentFirstCallSendsOneDecision(t *testing.T) { + t.Parallel() + + _, rec, filter, ctx := skillFilterEnv(t, http.StatusOK, map[string]float64{ + "skill:travel-booking": 0.9, + }) + + summaries := testSkillSummaries() + var wg sync.WaitGroup + for i := 0; i < 8; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + filter(ctx, summaries[i%len(summaries)]) + }(i) + } + wg.Wait() + require.Equal(t, 1, rec.requests()) +} + +func TestVisibility_CanceledWaiterDoesNotCancelDecision(t *testing.T) { + started, release := make(chan struct{}, 8), make(chan struct{}, 1) + defer close(release) + server, rec := decisionServer(t, func(q map[string]string) string { + started <- struct{}{} + <-release + return binaryResponder(nil, 0.05)(q) + }, http.StatusOK) + f, err := NewSkillFilter(server.URL, &fakeSkillRepo{summaries: testSkillSummaries()}, WithHTTPClient(server.Client())) + require.NoError(t, err) + inv := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage("book a flight"))) + ctx := agent.NewInvocationContext(context.Background(), inv) + filter := f.VisibilityFilter() + done := make(chan bool, 1) + go func() { done <- filter(ctx, skill.Summary{Name: "math-helper"}) }() + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("decision did not start") + } + waitCtx, cancel := context.WithCancel(ctx) + waiter := make(chan bool, 1) + go func() { waiter <- filter(waitCtx, skill.Summary{Name: "math-helper"}) }() + select { + case <-started: + t.Fatal("waiter started a duplicate decision") + case <-time.After(100 * time.Millisecond): + } + cancel() + select { + case visible := <-waiter: + require.True(t, visible) + case <-time.After(2 * time.Second): + t.Fatal("canceled waiter did not return") + } + release <- struct{}{} + select { + case visible := <-done: + require.False(t, visible) + case <-time.After(2 * time.Second): + t.Fatal("decision did not finish") + } + require.False(t, filter(ctx, skill.Summary{Name: "math-helper"})) + require.Equal(t, 1, rec.requests()) +} + +func TestVisibility_ReportsOutcome(t *testing.T) { + t.Parallel() + + server, rec := decisionServer(t, binaryResponder(map[string]float64{ + "skill:travel-booking": 0.9, + }, 0.05), http.StatusOK) + + var reports []*Report + f, err := NewSkillFilter(server.URL, + &fakeSkillRepo{summaries: testSkillSummaries()}, + WithHTTPClient(server.Client()), + WithKeepSkills("core-skill"), + WithObserver(func(r *Report) { reports = append(reports, r) }), + ) + require.NoError(t, err) + + inv := agent.NewInvocation(agent.WithInvocationMessage( + model.NewUserMessage("book a flight to paris"), + )) + ctx := agent.NewInvocationContext(context.Background(), inv) + filter := f.VisibilityFilter() + for _, s := range testSkillSummaries() { + filter(ctx, s) + } + + require.Equal(t, 1, rec.requests()) + require.Len(t, reports, 1) + report := reports[0] + require.Equal(t, CatalogSkill, report.Catalog) + require.Equal(t, OutcomeNarrowed, report.Outcome) + require.Equal(t, inv.InvocationID, report.TurnKey) + require.Contains(t, report.Dropped, "math-helper") + require.NotContains(t, report.Dropped, "core-skill") + require.InDelta(t, 0.9, report.Scores["travel-booking"], 1e-9) + require.Greater(t, report.SavedTokens, 0) +} + +func TestSharedObserverDistinguishesCatalogs(t *testing.T) { + server, _ := decisionServer(t, binaryResponder(nil, 0.9), http.StatusOK) + var catalogs []string + observer := WithObserver(func(report *Report) { catalogs = append(catalogs, report.Catalog) }) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client()), WithMinToolCount(0), observer) + require.NoError(t, err) + f, err := NewSkillFilter(server.URL, &fakeSkillRepo{summaries: []skill.Summary{{Name: "helper"}}}, WithHTTPClient(server.Client()), observer) + require.NoError(t, err) + inv := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage("use helper"))) + ctx := agent.NewInvocationContext(context.Background(), inv) + req := newUserRequest("use helper", newToolMap(&stubTool{name: "helper"})) + _, err = cb.Callback()(ctx, &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.True(t, f.VisibilityFilter()(ctx, skill.Summary{Name: "helper"})) + require.Equal(t, []string{CatalogTool, CatalogSkill}, catalogs) +} + +func TestVisibility_ObserverCanReenterSameInvocation(t *testing.T) { + for _, tc := range []struct { + name string + summaries []skill.Summary + status int + opts []Option + outcome string + visible bool + }{ + {name: "narrowed", summaries: testSkillSummaries(), status: http.StatusOK, outcome: OutcomeNarrowed}, + {name: "no scoreable skills", summaries: []skill.Summary{{Name: "core"}}, status: http.StatusOK, opts: []Option{WithKeepSkills("core")}, outcome: OutcomeUntouched, visible: true}, + {name: "no lexical overlap", summaries: []skill.Summary{{Name: "alpha"}, {Name: "beta"}}, status: http.StatusOK, opts: []Option{WithShortlistLimit(1)}, outcome: OutcomeUntouched, visible: true}, + {name: "decision failure", summaries: testSkillSummaries(), status: http.StatusInternalServerError, outcome: OutcomeFailOpen, visible: true}, + } { + t.Run(tc.name, func(t *testing.T) { + server, rec := decisionServer(t, binaryResponder(map[string]float64{"skill:travel-booking": 0.9}, 0.05), tc.status) + inv := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage("book a flight"))) + ctx := agent.NewInvocationContext(context.Background(), inv) + var f *SkillFilter + var reports int + var reentered, timedOut bool + var outcome string + opts := append([]Option{WithHTTPClient(server.Client()), WithObserver(func(report *Report) { + reports++ + outcome = report.Outcome + waitCtx, cancel := context.WithTimeout(ctx, 300*time.Millisecond) + defer cancel() + reentered = f.VisibilityFilter()(waitCtx, skill.Summary{Name: "math-helper"}) + timedOut = waitCtx.Err() != nil + })}, tc.opts...) + var err error + f, err = NewSkillFilter(server.URL, &fakeSkillRepo{summaries: tc.summaries}, opts...) + require.NoError(t, err) + visible := f.VisibilityFilter()(ctx, skill.Summary{Name: "math-helper"}) + require.False(t, timedOut, "observer must not wait on its own decision") + require.Equal(t, tc.visible, reentered) + require.Equal(t, tc.visible, visible) + require.Equal(t, tc.outcome, outcome) + require.Equal(t, 1, reports) + if tc.outcome == OutcomeUntouched { + require.Zero(t, rec.requests()) + } else { + require.Equal(t, 1, rec.requests()) + } + }) + } +} + +func TestVisibility_AllCandidatesKeptReportsUntouched(t *testing.T) { + for _, tc := range []struct { + name string + score float64 + opts []Option + }{ + {name: "threshold", score: 0.9}, + {name: "slack", score: 0.5}, + {name: "unasked and always kept", score: 0.9, opts: []Option{WithMaxQuestions(1), WithKeepSkills("fixed")}}, + } { + t.Run(tc.name, func(t *testing.T) { + server, rec := decisionServer(t, binaryResponder(nil, tc.score), http.StatusOK) + summaries := []skill.Summary{{Name: "alpha"}, {Name: "beta"}, {Name: "fixed"}} + var reports []*Report + opts := append([]Option{WithHTTPClient(server.Client()), WithObserver(func(r *Report) { reports = append(reports, r) })}, tc.opts...) + f, err := NewSkillFilter(server.URL, &fakeSkillRepo{summaries: summaries}, opts...) + require.NoError(t, err) + inv := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage("find skills"))) + ctx := agent.NewInvocationContext(context.Background(), inv) + for _, s := range summaries { + require.True(t, f.VisibilityFilter()(ctx, s)) + } + require.Equal(t, 1, rec.requests()) + require.Len(t, reports, 1) + require.Equal(t, CatalogSkill, reports[0].Catalog) + require.Equal(t, OutcomeUntouched, reports[0].Outcome) + require.Empty(t, reports[0].Dropped) + require.Zero(t, reports[0].SavedTokens) + cached, ok := f.cache.get(inv.InvocationID) + require.True(t, ok) + require.Equal(t, OutcomeUntouched, cached.outcome) + }) + } +} diff --git a/plugin/toolfilter/testhelpers_test.go b/plugin/toolfilter/testhelpers_test.go new file mode 100644 index 0000000000..f2759a07a5 --- /dev/null +++ b/plugin/toolfilter/testhelpers_test.go @@ -0,0 +1,232 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "trpc.group/trpc-go/trpc-agent-go/model" + "trpc.group/trpc-go/trpc-agent-go/skill" + "trpc.group/trpc-go/trpc-agent-go/tool" +) + +// stubTool carries a tool declaration for tests. +type stubTool struct { + name string + description string +} + +// Declaration implements tool.Tool. +func (t *stubTool) Declaration() *tool.Declaration { + return &tool.Declaration{Name: t.name, Description: t.description} +} + +// newToolMap builds a model request tool map from stub tools. +func newToolMap(tools ...*stubTool) map[string]tool.Tool { + m := make(map[string]tool.Tool, len(tools)) + for _, t := range tools { + m[t.name] = t + } + return m +} + +// newUserRequest builds a one-user-message request with the given tools. +func newUserRequest(content string, tools map[string]tool.Tool) *model.Request { + return &model.Request{ + Messages: []model.Message{model.NewUserMessage(content)}, + Tools: tools, + } +} + +// decisionRecorder records the decision requests seen by a test server. +type decisionRecorder struct { + mu sync.Mutex + count int + questions []map[string]string // per request: question ID -> wire type + states []any +} + +func (r *decisionRecorder) lastState() any { + r.mu.Lock() + defer r.mu.Unlock() + if len(r.states) == 0 { + return nil + } + return r.states[len(r.states)-1] +} + +// requests reports how many decision requests were received. +func (r *decisionRecorder) requests() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.count +} + +// lastQuestions reports the question IDs of the most recent request. +func (r *decisionRecorder) lastQuestions() map[string]string { + r.mu.Lock() + defer r.mu.Unlock() + if len(r.questions) == 0 { + return nil + } + return r.questions[len(r.questions)-1] +} + +// decisionServer serves scripted System One decision responses. respond +// receives the requested questions (ID -> wire type) and returns the +// response body; status overrides the HTTP status code. +func decisionServer( + t *testing.T, + respond func(questions map[string]string) string, + status int, +) (*httptest.Server, *decisionRecorder) { + t.Helper() + rec := &decisionRecorder{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + body, err := io.ReadAll(req.Body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + var wire struct { + State any `json:"state"` + Questions map[string]struct { + Type string `json:"type"` + } `json:"questions"` + } + if err := json.Unmarshal(body, &wire); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + questions := make(map[string]string, len(wire.Questions)) + for id, q := range wire.Questions { + questions[id] = q.Type + } + rec.mu.Lock() + rec.count++ + rec.questions = append(rec.questions, questions) + rec.states = append(rec.states, wire.State) + rec.mu.Unlock() + if status != http.StatusOK { + w.WriteHeader(status) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, respond(questions)) + })) + t.Cleanup(server.Close) + return server, rec +} + +// binaryResponder answers every asked question with override or fallback +// probabilities. +func binaryResponder( + overrides map[string]float64, + fallback float64, +) func(map[string]string) string { + return func(questions map[string]string) string { + answers := make(map[string]float64, len(questions)) + for id := range questions { + p := fallback + if value, ok := overrides[id]; ok { + p = value + } + answers[id] = p + } + return binaryResponse(answers) + } +} + +// binaryResponse builds a wire response with one binary answer per entry. +func binaryResponse(answers map[string]float64) string { + var b strings.Builder + b.WriteString(`{"model":"test-model","answers":{`) + writeAnswerFields(&b, answers, "noul") + b.WriteString("}}") + return b.String() +} + +// choiceResponse builds a wire response with one choice answer. +func choiceResponse( + id, choice string, + probabilities map[string]float64, + confidence float64, +) string { + var b strings.Builder + b.WriteString(`{"model":"test-model","answers":{`) + fmt.Fprintf(&b, `%q:{"type":"choice","choice":%q,"confidence":%v,"probabilities":{`, + id, choice, confidence) + first := true + for name, p := range probabilities { + if !first { + b.WriteString(",") + } + first = false + fmt.Fprintf(&b, `%q:%v`, name, p) + } + b.WriteString("}}}") + b.WriteString("}") + return b.String() +} + +func writeAnswerFields(b *strings.Builder, answers map[string]float64, kind string) { + first := true + for id, p := range answers { + if !first { + b.WriteString(",") + } + first = false + fmt.Fprintf(b, `%q:{"type":%q,"noul":%v}`, id, kind, p) + } +} + +// fakeSkillRepo is a static skill repository for tests. +type fakeSkillRepo struct { + summaries []skill.Summary +} + +// Summaries implements skill.Repository. +func (r *fakeSkillRepo) Summaries() []skill.Summary { return r.summaries } + +// Get implements skill.Repository. +func (r *fakeSkillRepo) Get(name string) (*skill.Skill, error) { + return nil, fmt.Errorf("skill %q not found", name) +} + +// Path implements skill.Repository. +func (r *fakeSkillRepo) Path(name string) (string, error) { + return "", fmt.Errorf("skill %q not found", name) +} + +// stubModel is a routing target for ModelRouter tests. +type stubModel struct { + name string +} + +// GenerateContent implements model.Model. +func (m *stubModel) GenerateContent( + ctx context.Context, + request *model.Request, +) (<-chan *model.Response, error) { + return nil, fmt.Errorf("stubModel is not callable") +} + +// Info implements model.Model. +func (m *stubModel) Info() model.Info { + return model.Info{Name: m.name} +} diff --git a/plugin/toolfilter/toolcallback.go b/plugin/toolfilter/toolcallback.go new file mode 100644 index 0000000000..03ec7c29ca --- /dev/null +++ b/plugin/toolfilter/toolcallback.go @@ -0,0 +1,437 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "context" + "fmt" + "sort" + + "trpc.group/trpc-go/trpc-agent-go/agent" + "trpc.group/trpc-go/trpc-agent-go/log" + "trpc.group/trpc-go/trpc-agent-go/model" + "trpc.group/trpc-go/trpc-agent-go/tool" + + systemoneclient "trpc.group/trpc-go/trpc-agent-go/internal/systemone" +) + +// defaultKeepTools protects framework-managed workflow and lifecycle tools +// when present. Application tools can be protected with WithKeepAlways. +var defaultKeepTools = []string{ + "skill_load", + "skill_select_docs", + "skill_list_docs", + "skill_run", + "skill_exec", + "skill_write_stdin", + "skill_poll_session", + "skill_kill_session", + "workspace_exec", + "workspace_write_stdin", + "workspace_kill_session", + "workspace_save_artifact", + "session_search", + "session_load", + "transfer_to_agent", + "await_user_reply", + "environment_context_current_time", +} + +// Text budgets calibrated against the Octop production implementation. +const ( + // descriptionClipLen bounds the description sent per candidate. + descriptionClipLen = 240 + // stateClipLen bounds the conversation sent as decision state in runes. + stateClipLen = 2400 +) + +// ToolCallback filters tools once per invocation and replays that decision on +// later model requests. An untouched result leaves subsequent catalogs +// unchanged; tools added after a narrowing decision remain visible. +// Construct it with NewToolCallback and register the callback returned by Callback. +type ToolCallback struct { + client *systemoneclient.Client + threshold float64 + slack float64 + minCount int + shortlist int + maxAsk int + keepSet map[string]struct{} + observer func(*Report) + embeddings *embedCache + cache *runCache[*turnResult] +} + +// turnResult is the cached filtering result of one turn. +type turnResult struct { + outcome string + kept map[string]struct{} + initial map[string]struct{} + scores map[string]float64 +} + +// NewToolCallback creates a tool pre-filter backed by the System One +// decision API at baseURL. It returns an error for an invalid endpoint or +// option values. The returned callback fails open: decision errors keep +// the full catalog. Requests without an invocation ID or textual invocation +// message are left unchanged. Decision context uses session events matching +// the framework's default prefix filter, not the final model request. +// Concurrent calls for an invocation share a decision controlled by the first +// caller's context. A canceled waiter leaves its tools unchanged without +// caching that result. +func NewToolCallback(baseURL string, opts ...Option) (*ToolCallback, error) { + o := applyOptions(opts) + if err := validateProbability("threshold", o.threshold); err != nil { + return nil, err + } + if err := validateProbability("slack", o.slack); err != nil { + return nil, err + } + client, err := newSystemOneClient(baseURL, o) + if err != nil { + return nil, err + } + keepSet := make(map[string]struct{}, len(defaultKeepTools)+len(o.keepAlways)) + for _, name := range defaultKeepTools { + keepSet[name] = struct{}{} + } + for _, name := range o.keepAlways { + if name != "" { + keepSet[name] = struct{}{} + } + } + var embeddings *embedCache + if o.embedder != nil { + embeddings = newEmbedCache(o.embedder) + } + return &ToolCallback{ + client: client, + threshold: o.threshold, + slack: o.slack, + minCount: o.minToolCount, + shortlist: o.shortlistLimit, + maxAsk: o.maxQuestions, + keepSet: keepSet, + observer: o.observer, + embeddings: embeddings, + cache: newRunCache[*turnResult](defaultCacheSize), + }, nil +} + +// Callback returns the BeforeModel callback that narrows +// args.Request.Tools. It never returns an error for decision failures; +// the catalog is kept instead (fail-open). +func (c *ToolCallback) Callback() model.BeforeModelCallbackStructured { + return func(ctx context.Context, args *model.BeforeModelArgs) (*model.BeforeModelResult, error) { + if args == nil || args.Request == nil { + return nil, nil + } + c.apply(ctx, args.Request) + return nil, nil + } +} + +// apply narrows the tool map of one model request in place. +func (c *ToolCallback) apply(ctx context.Context, req *model.Request) { + tools := req.Tools + before := catalogTokens(tools) + inv, _ := agent.InvocationFromContext(ctx) + if inv == nil || inv.InvocationID == "" || conversationText(inv.Message) == "" { + return + } + key := inv.InvocationID + // Shared work can outlive a canceled caller, so retain our own catalog map. + catalog := make(map[string]tool.Tool, len(tools)) + for name, t := range tools { + catalog[name] = t + } + result, ok := c.cache.getOrCompute(ctx, key, func() (*turnResult, bool) { + result := c.decideTurn(ctx, inv, catalog, before) + return result, result != nil + }) + if !ok || result == nil || result.outcome != OutcomeNarrowed { + return + } + kept := make(map[string]struct{}, len(result.kept)+len(tools)) + for name := range result.kept { + kept[name] = struct{}{} + } + for name := range tools { + if _, seen := result.initial[name]; !seen { + kept[name] = struct{}{} + } + } + req.Tools = narrowToolMap(tools, kept) +} + +// decideTurn computes a shared decision without modifying the caller's request. +func (c *ToolCallback) decideTurn(ctx context.Context, inv *agent.Invocation, tools map[string]tool.Tool, before int) *turnResult { + key := inv.InvocationID + // Below the minimum catalog size, filtering cannot pay for itself. + if len(tools) < c.minCount { + return c.untouched(key, &Report{ + Outcome: OutcomeUntouched, + TurnKey: key, + Reason: "catalog below min tool count", + CatalogTokensBefore: before, CatalogTokensAfter: before, + }) + } + + // Built-in and user keep sets are never scored. + names := sortedToolNames(tools) + candidates := make([]scoredCandidate, 0, len(names)) + for _, name := range names { + if _, keep := c.keepSet[name]; keep { + continue + } + candidates = append(candidates, scoredCandidate{ + name: name, + description: clipRunes(toolDescription(tools[name]), descriptionClipLen), + }) + } + if len(candidates) == 0 { + return c.untouched(key, &Report{ + Outcome: OutcomeUntouched, TurnKey: key, + Reason: "no scoreable candidates", + CatalogTokensBefore: before, CatalogTokensAfter: before, + }) + } + + // Lexical (and optional embedding) recall. Zero overlap aborts the + // whole turn: the decision model needs at least a relevance signal. + state, userMessage := decisionConversation(inv) + shortlist := candidates + if len(candidates) > c.shortlist { + ranked := recallRank(ctx, userMessage, candidates, c.embeddings) + shortlist = nil + for _, cand := range ranked { + if cand.score <= 0 || len(shortlist) == c.shortlist { + break + } + shortlist = append(shortlist, cand) + } + } + if len(shortlist) == 0 { + return c.untouched(key, &Report{ + Outcome: OutcomeUntouched, TurnKey: key, + Reason: "no lexical overlap with the user message", + CatalogTokensBefore: before, CatalogTokensAfter: before, + }) + } + asked := shortlist + var unasked []string + if len(asked) > c.maxAsk { + unasked = candidateNames(asked[c.maxAsk:]) + asked = asked[:c.maxAsk] + } + + result, failReason := c.decide(ctx, state, asked, unasked) + result.initial = make(map[string]struct{}, len(tools)) + for name := range tools { + result.initial[name] = struct{}{} + } + report := &Report{ + TurnKey: key, + Asked: candidateNames(asked), + Unasked: unasked, + Scores: result.scores, + CatalogTokensBefore: before, + } + if failReason != "" { + report.Outcome = OutcomeFailOpen + report.Reason = failReason + report.CatalogTokensAfter = before + } else { + narrowed := narrowToolMap(tools, result.kept) + report.CatalogTokensAfter = catalogTokens(narrowed) + report.Dropped = droppedNames(tools, narrowed) + if len(report.Dropped) == 0 { + result.outcome = OutcomeUntouched + report.Reason = "all candidates retained" + } + report.Outcome = result.outcome + } + report.SavedTokens = report.CatalogTokensBefore - report.CatalogTokensAfter + c.cache.set(key, result) + c.emit(report) + return result +} + +// untouched records the result before notifying observers, allowing them to +// reenter the filter without waiting on the decision they are observing. +func (c *ToolCallback) untouched(key string, report *Report) *turnResult { + result := &turnResult{outcome: OutcomeUntouched} + c.cache.set(key, result) + c.emit(report) + return result +} + +// decide asks the decision model about the shortlisted tools and computes +// the retention set. Shortlist entries beyond the question budget are kept +// without a score. It returns the turn result and, when the decision call +// failed, the failure reason for the report (fail-open). +func (c *ToolCallback) decide( + ctx context.Context, + userMessage string, + asked []scoredCandidate, + unasked []string, +) (*turnResult, string) { + questions := make(map[string]systemoneclient.Question, len(asked)) + toolCatalog := make([]any, 0, len(asked)) + for _, cand := range asked { + questions["tool:"+cand.name] = systemoneclient.BinaryQuestion{ + Instructions: fmt.Sprintf( + "Does the current user request need the tool %q? Tool description: %s", + cand.name, cand.description), + } + toolCatalog = append(toolCatalog, map[string]any{ + "name": cand.name, + "description": cand.description, + }) + } + + resp, err := c.client.SystemOne(ctx, &systemoneclient.Request{ + State: map[string]any{ + "user_message": clipRunes(userMessage, stateClipLen), + "tool_catalog": toolCatalog, + }, + Questions: questions, + }) + if err != nil { + // Fail open: keep the whole catalog, including lexical rejects. + log.WarnfContext(ctx, "toolfilter: decision failed, keeping full catalog: %v", err) + return &turnResult{outcome: OutcomeFailOpen}, err.Error() + } + + kept := make(map[string]struct{}, len(asked)+len(unasked)) + for name := range c.keepSet { + kept[name] = struct{}{} + } + // Unasked shortlist entries stay visible without a score. + for _, name := range unasked { + kept[name] = struct{}{} + } + scores := make(map[string]float64, len(asked)) + matched := false + for _, cand := range asked { + answer, err := resp.Binary("tool:" + cand.name) + if err != nil { + // Strict validation guarantees every asked question is + // answered; treat a defensive miss as a keep. + kept[cand.name] = struct{}{} + continue + } + scores[cand.name] = answer.Probability + if answer.Probability >= c.threshold { + kept[cand.name] = struct{}{} + matched = true + } + } + if !matched { + for name, score := range scores { + if score >= c.threshold-c.slack { + kept[name] = struct{}{} + } + } + } + return &turnResult{outcome: OutcomeNarrowed, kept: kept, scores: scores}, "" +} + +// emit delivers the report: one log line always, plus the WithObserver +// callback when configured. +func (c *ToolCallback) emit(report *Report) { + report.Catalog = CatalogTool + log.InfofContext( + context.Background(), + "toolfilter: outcome=%s asked=%d dropped=%d saved_tokens~%d reason=%q", + report.Outcome, len(report.Asked), len(report.Dropped), report.SavedTokens, report.Reason, + ) + if c.observer != nil { + c.observer(report) + } +} + +// toolDescription extracts a tool's description, tolerating tools without +// one. +func toolDescription(t tool.Tool) string { + if t == nil { + return "" + } + if decl := t.Declaration(); decl != nil { + return decl.Description + } + return "" +} + +// sortedToolNames returns the deterministic name order of a tool map. +func sortedToolNames(tools map[string]tool.Tool) []string { + names := make([]string, 0, len(tools)) + for name := range tools { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// catalogTokens estimates the token cost of a tool map. +func catalogTokens(tools map[string]tool.Tool) int { + total := 0 + for name, t := range tools { + total += estimateToolTokens(name, toolDescription(t)) + } + return total +} + +// droppedNames lists the names present before but not after narrowing. +func droppedNames(before, after map[string]tool.Tool) []string { + dropped := make([]string, 0) + for name := range before { + if _, ok := after[name]; !ok { + dropped = append(dropped, name) + } + } + sort.Strings(dropped) + return dropped +} + +// narrowToolMap keeps only the tools named in kept. +func narrowToolMap(tools map[string]tool.Tool, kept map[string]struct{}) map[string]tool.Tool { + if kept == nil { + return tools + } + narrowed := make(map[string]tool.Tool, len(kept)) + for name, t := range tools { + if _, ok := kept[name]; ok { + narrowed[name] = t + } + } + return narrowed +} + +// candidateNames extracts candidate names in ranked order. +func candidateNames(candidates []scoredCandidate) []string { + names := make([]string, 0, len(candidates)) + for _, cand := range candidates { + names = append(names, cand.name) + } + return names +} + +// clipRunes truncates text to at most max runes. +func clipRunes(text string, max int) string { + if max <= 0 { + return "" + } + runes := []rune(text) + if len(runes) <= max { + return text + } + return string(runes[:max]) +} diff --git a/plugin/toolfilter/toolcallback_test.go b/plugin/toolfilter/toolcallback_test.go new file mode 100644 index 0000000000..c49d05ca17 --- /dev/null +++ b/plugin/toolfilter/toolcallback_test.go @@ -0,0 +1,676 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "context" + "errors" + "fmt" + "math" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "trpc.group/trpc-go/trpc-agent-go/agent" + "trpc.group/trpc-go/trpc-agent-go/model" +) + +func toolTestContext(req *model.Request) context.Context { + var message model.Message + if len(req.Messages) > 0 { + message = req.Messages[len(req.Messages)-1] + } + return agent.NewInvocationContext(context.Background(), agent.NewInvocation(agent.WithInvocationMessage(message))) +} + +func TestCallback_InvocationCachePreservesNewTools(t *testing.T) { + t.Parallel() + server, rec := decisionServer(t, binaryResponder(map[string]float64{"tool:keep": 0.9}, 0.05), http.StatusOK) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client()), WithMinToolCount(0)) + require.NoError(t, err) + inv := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage("workflow helper"))) + ctx := agent.NewInvocationContext(context.Background(), inv) + tools := newToolMap(&stubTool{name: "keep", description: "workflow helper"}, &stubTool{name: "drop", description: "workflow helper"}) + req := newUserRequest("unrelated request content", tools) + _, err = cb.Callback()(ctx, &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, []string{"keep"}, sortedToolNames(req.Tools)) + tools["added"] = &stubTool{name: "added", description: "workflow helper"} + req = newUserRequest("unrelated request content", tools) + _, err = cb.Callback()(ctx, &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, []string{"added", "keep"}, sortedToolNames(req.Tools)) + require.Equal(t, 1, rec.requests()) + other := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage("workflow helper"))) + req = newUserRequest("unrelated request content", tools) + _, err = cb.Callback()(agent.NewInvocationContext(context.Background(), other), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, []string{"keep"}, sortedToolNames(req.Tools)) + require.Equal(t, 2, rec.requests()) +} + +func TestCallback_ConcurrentDecisionAndCanceledWaiter(t *testing.T) { + started, release := make(chan struct{}, 8), make(chan struct{}, 1) + defer close(release) + server, rec := decisionServer(t, func(q map[string]string) string { + started <- struct{}{} + <-release + return binaryResponder(map[string]float64{"tool:keep": 0.9}, 0.05)(q) + }, http.StatusOK) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client()), WithMinToolCount(0)) + require.NoError(t, err) + inv := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage("workflow helper"))) + ctx := agent.NewInvocationContext(context.Background(), inv) + newRequest := func() *model.Request { + return newUserRequest("workflow helper", newToolMap(&stubTool{name: "keep"}, &stubTool{name: "drop"})) + } + first := newRequest() + done := make(chan struct{}) + go func() { c := cb.Callback(); _, _ = c(ctx, &model.BeforeModelArgs{Request: first}); close(done) }() + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("decision did not start") + } + otherReq := newRequest() + otherReq.Tools["added"] = &stubTool{name: "added"} + otherDone := make(chan struct{}) + go func() { c := cb.Callback(); _, _ = c(ctx, &model.BeforeModelArgs{Request: otherReq}); close(otherDone) }() + waitCtx, cancel := context.WithCancel(ctx) + waitReq := newRequest() + waitDone := make(chan struct{}) + go func() { + c := cb.Callback() + _, _ = c(waitCtx, &model.BeforeModelArgs{Request: waitReq}) + close(waitDone) + }() + select { + case <-started: + t.Fatal("waiter started a duplicate decision") + case <-time.After(100 * time.Millisecond): + } + cancel() + select { + case <-waitDone: + case <-time.After(2 * time.Second): + t.Fatal("canceled waiter did not return") + } + require.Equal(t, []string{"drop", "keep"}, sortedToolNames(waitReq.Tools)) + release <- struct{}{} + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("decision did not finish") + } + select { + case <-otherDone: + case <-time.After(2 * time.Second): + t.Fatal("concurrent waiter did not finish") + } + require.Equal(t, []string{"keep"}, sortedToolNames(first.Tools)) + require.Equal(t, []string{"added", "keep"}, sortedToolNames(otherReq.Tools)) + require.Equal(t, 1, rec.requests()) +} + +func TestCallback_MissingInvocationLeavesToolsUntouched(t *testing.T) { + t.Parallel() + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client()), WithMinToolCount(0)) + require.NoError(t, err) + tools := newToolMap(&stubTool{name: "helper", description: "workflow helper"}) + req := newUserRequest("workflow helper", tools) + _, err = cb.Callback()(context.Background(), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, tools, req.Tools) + require.Equal(t, 0, rec.requests()) +} + +func TestNewToolCallback_ValidatesOptions(t *testing.T) { + t.Parallel() + + _, err := NewToolCallback("") + require.Error(t, err) + + _, err = NewToolCallback("https://decision.example", WithThreshold(1.5)) + require.ErrorContains(t, err, "threshold") + + _, err = NewToolCallback("https://decision.example", WithSlack(-0.1)) + require.ErrorContains(t, err, "slack") + + _, err = NewToolCallback("https://decision.example", WithThreshold(math.NaN())) + require.ErrorContains(t, err, "threshold") + + _, err = NewToolCallback("https://decision.example", WithSlack(math.NaN())) + require.ErrorContains(t, err, "slack") +} + +// catalogTools is a catalog large enough to pass the default min tool count, +// with the framework keep set present. +func catalogTools() map[string]*stubTool { + return map[string]*stubTool{ + "web_search": {name: "web_search", description: "search the web for flights and hotels"}, + "hotel_booking": {name: "hotel_booking", description: "book hotel rooms"}, + "calculator": {name: "calculator", description: "perform arithmetic calculations"}, + "calendar": {name: "calendar", description: "manage calendar events"}, + "email_sender": {name: "email_sender", description: "send emails to recipients"}, + "file_reader": {name: "file_reader", description: "read files from disk"}, + "database_query": {name: "database_query", description: "run SQL queries"}, + "image_render": {name: "image_render", description: "render images from prompts"}, + "skill_load": {name: "skill_load", description: "load a skill"}, + "tool_search": {name: "tool_search", description: "search for tools"}, + } +} + +func TestCallback_NarrowsAndKeepsDefaultKeepSet(t *testing.T) { + t.Parallel() + + server, rec := decisionServer(t, binaryResponder(map[string]float64{ + "tool:web_search": 0.9, + "tool:hotel_booking": 0.55, // below threshold, within slack + }, 0.05), http.StatusOK) + + var reports []*Report + cb, err := NewToolCallback(server.URL, + WithHTTPClient(server.Client()), + WithObserver(func(r *Report) { reports = append(reports, r) }), + ) + require.NoError(t, err) + + req := newUserRequest("find cheap flights and hotels online", + newToolMap(catalogTools()["web_search"], + catalogTools()["hotel_booking"], + catalogTools()["calculator"], + catalogTools()["calendar"], + catalogTools()["email_sender"], + catalogTools()["file_reader"], + catalogTools()["database_query"], + catalogTools()["image_render"], + catalogTools()["skill_load"], + catalogTools()["tool_search"], + )) + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + + require.Equal(t, 1, rec.requests()) + // A candidate passed the threshold, so the near-threshold tool is dropped. + require.Equal(t, []string{"skill_load", "web_search"}, + sortedToolNames(req.Tools)) + + require.Len(t, reports, 1) + report := reports[0] + require.Equal(t, OutcomeNarrowed, report.Outcome) + require.Equal(t, []string{ + "calculator", "calendar", "database_query", + "email_sender", "file_reader", "hotel_booking", "image_render", "tool_search", + }, report.Dropped) + require.InDelta(t, 0.9, report.Scores["web_search"], 1e-9) + require.NotContains(t, report.Scores, "skill_load") // never scored + require.Greater(t, report.SavedTokens, 0) +} + +func TestCallback_FrameworkWorkflowToolsKept(t *testing.T) { + t.Parallel() + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client()), WithMinToolCount(0)) + require.NoError(t, err) + workflow := []string{ + "skill_load", "skill_list_docs", "skill_select_docs", "skill_run", "skill_exec", + "skill_write_stdin", "skill_poll_session", "skill_kill_session", + "workspace_exec", "workspace_write_stdin", "workspace_kill_session", "workspace_save_artifact", + "session_search", "session_load", "transfer_to_agent", "await_user_reply", "environment_context_current_time", + } + tools := newToolMap() + for _, name := range workflow { + tools[name] = &stubTool{name: name, description: "framework workflow helper"} + } + for _, name := range []string{"tool_search", "call_tool", "memory_search", "file_read_file"} { + tools[name] = &stubTool{name: name, description: "framework workflow helper"} + } + req := newUserRequest("framework workflow helper", tools) + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + for _, name := range workflow { + require.Contains(t, req.Tools, name) + require.NotContains(t, rec.lastQuestions(), "tool:"+name) + } + for _, name := range []string{"tool_search", "call_tool", "memory_search", "file_read_file"} { + require.NotContains(t, req.Tools, name) + require.Contains(t, rec.lastQuestions(), "tool:"+name) + } +} + +func TestCallback_SlackFallbackWithKeepSet(t *testing.T) { + t.Parallel() + server, _ := decisionServer(t, binaryResponder(map[string]float64{ + "tool:hotel_booking": 0.5, + }, 0.05), http.StatusOK) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client()), WithMinToolCount(0)) + require.NoError(t, err) + req := newUserRequest("book hotel rooms", newToolMap( + catalogTools()["hotel_booking"], catalogTools()["calculator"], catalogTools()["skill_load"], + )) + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, []string{"hotel_booking", "skill_load"}, sortedToolNames(req.Tools)) +} + +func TestCallback_TurnCacheReplaysWithinTurn(t *testing.T) { + t.Parallel() + + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client())) + require.NoError(t, err) + + tools := newToolMap(catalogTools()["web_search"], + catalogTools()["hotel_booking"], + catalogTools()["calculator"], + catalogTools()["calendar"], + catalogTools()["email_sender"], + catalogTools()["file_reader"], + catalogTools()["database_query"], + catalogTools()["image_render"], + catalogTools()["skill_load"], + ) + + ctx := toolTestContext(newUserRequest("find cheap flights and hotels online", tools)) + // Same user message: the second ReAct step replays the cached decision. + for i := 0; i < 2; i++ { + req := newUserRequest("find cheap flights and hotels online", tools) + _, err := cb.Callback()(ctx, &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, 1, rec.requests()) + } + + // A new user message is a new turn and triggers a new decision. + req := newUserRequest("read a file from disk", tools) + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, 2, rec.requests()) +} + +func TestCallback_FailOpenOnServerError(t *testing.T) { + t.Parallel() + + server, rec := decisionServer(t, nil, http.StatusInternalServerError) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client())) + require.NoError(t, err) + + tools := newToolMap(catalogTools()["web_search"], + catalogTools()["hotel_booking"], + catalogTools()["calculator"], + catalogTools()["calendar"], + catalogTools()["email_sender"], + catalogTools()["file_reader"], + catalogTools()["database_query"], + catalogTools()["image_render"], + catalogTools()["skill_load"], + ) + req := newUserRequest("find cheap flights online", tools) + + ctx := toolTestContext(req) + _, err = cb.Callback()(ctx, &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, tools, req.Tools) // full catalog kept + require.Equal(t, 1, rec.requests()) + + // The fail-open result is cached: no request on the next step. + req2 := newUserRequest("find cheap flights online", tools) + _, err = cb.Callback()(ctx, &model.BeforeModelArgs{Request: req2}) + require.NoError(t, err) + require.Equal(t, tools, req2.Tools) + require.Equal(t, 1, rec.requests()) +} + +func TestCallback_FailOpenOnInvalidResponse(t *testing.T) { + t.Parallel() + + server, _ := decisionServer(t, func(map[string]string) string { + return "not json" + }, http.StatusOK) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client())) + require.NoError(t, err) + + tools := newToolMap(catalogTools()["web_search"], + catalogTools()["hotel_booking"], + catalogTools()["calculator"], + catalogTools()["calendar"], + catalogTools()["email_sender"], + catalogTools()["file_reader"], + catalogTools()["database_query"], + catalogTools()["image_render"], + catalogTools()["skill_load"], + ) + req := newUserRequest("find cheap flights online", tools) + + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, tools, req.Tools) +} + +func TestCallback_MinToolCountSkipsFiltering(t *testing.T) { + t.Parallel() + + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client())) + require.NoError(t, err) + + tools := newToolMap( + &stubTool{name: "web_search", description: "search the web"}, + &stubTool{name: "calculator", description: "arithmetic"}, + &stubTool{name: "calendar", description: "events"}, + &stubTool{name: "email_sender", description: "emails"}, + &stubTool{name: "file_reader", description: "files"}, + ) + req := newUserRequest("find flights", tools) + + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, tools, req.Tools) + require.Equal(t, 0, rec.requests()) +} + +func TestCallback_MinToolCountBoundary(t *testing.T) { + for _, tc := range []struct { + name string + count int + want int + }{ + {name: "below", count: 1, want: 0}, + {name: "equal", count: 2, want: 1}, + {name: "above", count: 3, want: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + server, rec := decisionServer(t, binaryResponder(nil, 0.9), http.StatusOK) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client()), WithMinToolCount(2)) + require.NoError(t, err) + tools := newToolMap() + for i := 0; i < tc.count; i++ { + name := fmt.Sprintf("tool_%d", i) + tools[name] = &stubTool{name: name} + } + req := newUserRequest("find tools", tools) + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, tc.want, rec.requests()) + }) + } +} + +func TestCallback_UntouchedDecisionReplayed(t *testing.T) { + tests := []struct { + name string + message string + tools []*stubTool + opts []Option + reason string + }{ + {"small catalog", "find flights", []*stubTool{{name: "search"}}, nil, "catalog below min tool count"}, + {"no scoreable candidates", "find flights", []*stubTool{{name: "skill_load"}}, []Option{WithMinToolCount(0)}, "no scoreable candidates"}, + {"no lexical overlap", "你好世界", []*stubTool{{name: "search"}, {name: "calendar"}}, []Option{WithMinToolCount(0), WithShortlistLimit(1)}, "no lexical overlap with the user message"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + var reports []*Report + opts := append([]Option{WithHTTPClient(server.Client()), WithObserver(func(r *Report) { reports = append(reports, r) })}, tt.opts...) + cb, err := NewToolCallback(server.URL, opts...) + require.NoError(t, err) + inv := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage(tt.message))) + ctx := agent.NewInvocationContext(context.Background(), inv) + initial := newToolMap(tt.tools...) + first := newUserRequest(tt.message, initial) + _, err = cb.Callback()(ctx, &model.BeforeModelArgs{Request: first}) + require.NoError(t, err) + require.Equal(t, initial, first.Tools) + later := newToolMap(tt.tools...) + for i := 0; i < 10; i++ { + name := fmt.Sprintf("new_tool_%d", i) + later[name] = &stubTool{name: name, description: "find flights 你好世界"} + } + second := newUserRequest(tt.message, later) + _, err = cb.Callback()(ctx, &model.BeforeModelArgs{Request: second}) + require.NoError(t, err) + require.Equal(t, later, second.Tools) + require.Len(t, reports, 1) + require.Equal(t, CatalogTool, reports[0].Catalog) + require.Equal(t, OutcomeUntouched, reports[0].Outcome) + require.Equal(t, tt.reason, reports[0].Reason) + require.Equal(t, 0, rec.requests()) + }) + } +} + +func TestCallback_UntouchedReplayDoesNotRetryFailedEmbedding(t *testing.T) { + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + emb := &stubEmbedder{err: errors.New("embedding failed")} + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client()), WithMinToolCount(0), + WithShortlistLimit(1), WithShortlistEmbedder(emb)) + require.NoError(t, err) + inv := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage("你好世界"))) + ctx := agent.NewInvocationContext(context.Background(), inv) + tools := newToolMap(&stubTool{name: "search"}, &stubTool{name: "calendar"}) + for i := 0; i < 2; i++ { + req := newUserRequest("你好世界", tools) + _, err := cb.Callback()(ctx, &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, tools, req.Tools) + } + require.Equal(t, 1, emb.calls()) + require.Equal(t, 0, rec.requests()) +} + +func TestCallback_UntouchedObserverCanReenter(t *testing.T) { + inv := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage("find flights"))) + ctx := agent.NewInvocationContext(context.Background(), inv) + tools := newToolMap(&stubTool{name: "search"}) + var cb *ToolCallback + reports := 0 + var err error + cb, err = NewToolCallback("https://decision.example", WithObserver(func(*Report) { + reports++ + req := newUserRequest("find flights", tools) + _, _ = cb.Callback()(ctx, &model.BeforeModelArgs{Request: req}) + })) + require.NoError(t, err) + done := make(chan struct{}) + go func() { + req := newUserRequest("find flights", tools) + _, _ = cb.Callback()(ctx, &model.BeforeModelArgs{Request: req}) + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("observer blocked when reentering the filter") + } + require.Equal(t, 1, reports) +} + +func TestCallback_AllCandidatesKeptReportsUntouched(t *testing.T) { + for _, tc := range []struct { + name string + score float64 + opts []Option + }{ + {name: "threshold", score: 0.9}, + {name: "slack", score: 0.5}, + {name: "unasked and always kept", score: 0.9, opts: []Option{WithMaxQuestions(1), WithKeepAlways("fixed")}}, + } { + t.Run(tc.name, func(t *testing.T) { + server, rec := decisionServer(t, binaryResponder(nil, tc.score), http.StatusOK) + var reports []*Report + opts := append([]Option{WithHTTPClient(server.Client()), WithMinToolCount(0), WithObserver(func(r *Report) { reports = append(reports, r) })}, tc.opts...) + cb, err := NewToolCallback(server.URL, opts...) + require.NoError(t, err) + inv := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage("find tools"))) + ctx := agent.NewInvocationContext(context.Background(), inv) + tools := newToolMap(&stubTool{name: "alpha"}, &stubTool{name: "beta"}, &stubTool{name: "fixed"}) + for i := 0; i < 2; i++ { + req := newUserRequest("find tools", tools) + _, err = cb.Callback()(ctx, &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, tools, req.Tools) + } + require.Equal(t, 1, rec.requests()) + require.Len(t, reports, 1) + require.Equal(t, OutcomeUntouched, reports[0].Outcome) + require.Empty(t, reports[0].Dropped) + require.Zero(t, reports[0].SavedTokens) + cached, ok := cb.cache.get(inv.InvocationID) + require.True(t, ok) + require.Equal(t, OutcomeUntouched, cached.outcome) + }) + } +} + +func TestCallback_RemovalWithZeroEstimatedSavingsIsNarrowed(t *testing.T) { + server, _ := decisionServer(t, binaryResponder(map[string]float64{"tool:a": 0.9, "tool:b": 0.1}, 0.1), http.StatusOK) + var report *Report + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client()), WithMinToolCount(0), WithObserver(func(r *Report) { report = r })) + require.NoError(t, err) + tools := newToolMap(&stubTool{name: "a"}, &stubTool{name: "b"}) + req := newUserRequest("find tools", tools) + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, []string{"a"}, sortedToolNames(req.Tools)) + require.Equal(t, OutcomeNarrowed, report.Outcome) + require.Equal(t, []string{"b"}, report.Dropped) + require.Zero(t, report.SavedTokens) +} + +func TestCallback_NoUserMessageSkipsFiltering(t *testing.T) { + t.Parallel() + + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client())) + require.NoError(t, err) + + tools := newToolMap(catalogTools()["web_search"], + catalogTools()["hotel_booking"], + catalogTools()["calculator"], + catalogTools()["calendar"], + catalogTools()["email_sender"], + catalogTools()["file_reader"], + catalogTools()["database_query"], + catalogTools()["image_render"], + ) + req := &model.Request{Tools: tools} + + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, tools, req.Tools) + require.Equal(t, 0, rec.requests()) +} + +func TestCallback_ZeroLexicalOverlapAborts(t *testing.T) { + t.Parallel() + + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client()), WithMinToolCount(0), WithShortlistLimit(1)) + require.NoError(t, err) + + tools := newToolMap(catalogTools()["web_search"], + catalogTools()["hotel_booking"], + catalogTools()["calculator"], + catalogTools()["calendar"], + catalogTools()["email_sender"], + catalogTools()["file_reader"], + catalogTools()["database_query"], + catalogTools()["image_render"], + ) + // No token overlap with any catalog entry. + req := newUserRequest("你好世界", tools) + + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, tools, req.Tools) + require.Equal(t, 0, rec.requests()) +} + +func TestCallback_SmallCatalogAskedWithoutLexicalOverlap(t *testing.T) { + t.Parallel() + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + cb, err := NewToolCallback(server.URL, WithHTTPClient(server.Client()), WithMinToolCount(0), WithShortlistLimit(2)) + require.NoError(t, err) + req := newUserRequest("unrelated", newToolMap(&stubTool{name: "search", description: "find flights"})) + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Contains(t, rec.lastQuestions(), "tool:search") +} + +func TestCallback_UnaskedBeyondQuestionBudgetKept(t *testing.T) { + t.Parallel() + + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + cb, err := NewToolCallback(server.URL, + WithHTTPClient(server.Client()), + WithShortlistLimit(24), + WithMaxQuestions(20), + ) + require.NoError(t, err) + + // 25 scoreable candidates plus the framework keep set. + tools := make(map[string]*stubTool, 30) + for i := 0; i < 25; i++ { + name := fmt.Sprintf("helper_%02d", i) + tools[name] = &stubTool{name: name, description: "helper tool alpha beta"} + } + toolMap := newToolMap() + for name, st := range tools { + toolMap[name] = st + } + for _, name := range defaultKeepTools { + toolMap[name] = &stubTool{name: name, description: "built-in"} + } + + req := newUserRequest("use the helper tool alpha beta", toolMap) + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + + require.Equal(t, 1, rec.requests()) + require.Len(t, rec.lastQuestions(), 20) // asked within budget + + // Every candidate scores 0.05, so only the keep set and the four + // shortlisted-but-unasked candidates survive. + require.Len(t, req.Tools, len(defaultKeepTools)+4) +} + +func TestCallback_KeepAlwaysProtectsUserTools(t *testing.T) { + t.Parallel() + + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + cb, err := NewToolCallback(server.URL, + WithHTTPClient(server.Client()), + WithKeepAlways("crm_connector", "tool_search"), + ) + require.NoError(t, err) + + tools := newToolMap(catalogTools()["web_search"], + catalogTools()["hotel_booking"], + catalogTools()["calculator"], + catalogTools()["calendar"], + catalogTools()["email_sender"], + catalogTools()["file_reader"], + catalogTools()["database_query"], + catalogTools()["skill_load"], + catalogTools()["tool_search"], + &stubTool{name: "crm_connector", description: "company CRM access"}, + ) + req := newUserRequest("find cheap flights online", tools) + + _, err = cb.Callback()(toolTestContext(req), &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Contains(t, req.Tools, "crm_connector") + require.NotContains(t, rec.lastQuestions(), "tool:crm_connector") + require.Contains(t, req.Tools, "tool_search") + require.NotContains(t, rec.lastQuestions(), "tool:tool_search") +} From a0ba92a55d2f8b285636cfea844b1517d6cdb640 Mon Sep 17 00:00:00 2001 From: Jeffrey <1187141927@qq.com> Date: Tue, 3 Nov 2026 17:10:30 +0800 Subject: [PATCH 2/7] plugin/toolfilter: simplify skill filtering decisions Extract candidate collection and score selection into focused helpers to reduce decision complexity while preserving filtering, caching, and reporting behavior. --- plugin/toolfilter/skillfilter.go | 112 +++++++++++++++++-------------- 1 file changed, 63 insertions(+), 49 deletions(-) diff --git a/plugin/toolfilter/skillfilter.go b/plugin/toolfilter/skillfilter.go index 25ec6f96b6..1156b5fa50 100644 --- a/plugin/toolfilter/skillfilter.go +++ b/plugin/toolfilter/skillfilter.go @@ -164,30 +164,7 @@ func (f *SkillFilter) decide(ctx context.Context, inv *agent.Invocation) (*skill userMessage, userHistory := decisionConversation(inv) - candidates := make([]skillCandidate, 0, len(summaries)) - seen := make(map[string]struct{}, len(summaries)) - names := make([]string, 0, len(summaries)) - for _, s := range summaries { - name := strings.TrimSpace(s.Name) - if name == "" { - continue - } - if _, dup := seen[name]; dup { - continue - } - seen[name] = struct{}{} - names = append(names, name) - } - sort.Strings(names) - for _, name := range names { - if _, keep := f.keepSet[name]; keep { - continue - } - candidates = append(candidates, skillCandidate{ - name: name, - description: clipRunes(skillDescription(summaries, name), descriptionClipLen), - }) - } + names, candidates := f.skillCandidates(summaries) if len(candidates) == 0 { return &skillRunResult{outcome: OutcomeUntouched}, &Report{ @@ -261,6 +238,67 @@ func (f *SkillFilter) decide(ctx context.Context, inv *agent.Invocation) (*skill } } + kept, scores := f.selectSkills(resp, asked, unasked) + + dropped := make([]string, 0) + for _, name := range names { + if _, ok := kept[name]; !ok { + dropped = append(dropped, name) + } + } + outcome := OutcomeNarrowed + reason := "" + if len(dropped) == 0 { + outcome = OutcomeUntouched + reason = "all candidates retained" + } + report := &Report{ + Outcome: outcome, + Reason: reason, + TurnKey: inv.InvocationID, + Asked: skillNames(asked), + Unasked: unasked, + Dropped: dropped, + Scores: scores, + CatalogTokensBefore: before, + CatalogTokensAfter: before - skillTokensFor(summaries, dropped), + } + report.SavedTokens = report.CatalogTokensBefore - report.CatalogTokensAfter + return &skillRunResult{outcome: outcome, kept: kept, scores: scores}, report +} + +// skillCandidates collects unique skill names and excludes always-kept skills +// from scoring while preserving the sorted catalog used for reporting. +func (f *SkillFilter) skillCandidates(summaries []skill.Summary) ([]string, []skillCandidate) { + candidates := make([]skillCandidate, 0, len(summaries)) + seen := make(map[string]struct{}, len(summaries)) + names := make([]string, 0, len(summaries)) + for _, s := range summaries { + name := strings.TrimSpace(s.Name) + if name == "" { + continue + } + if _, dup := seen[name]; dup { + continue + } + seen[name] = struct{}{} + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if _, keep := f.keepSet[name]; keep { + continue + } + candidates = append(candidates, skillCandidate{ + name: name, + description: clipRunes(skillDescription(summaries, name), descriptionClipLen), + }) + } + return names, candidates +} + +// selectSkills applies the threshold and fallback margin to the decision. +func (f *SkillFilter) selectSkills(resp *systemoneclient.Response, asked []skillCandidate, unasked []string) (map[string]struct{}, map[string]float64) { kept := make(map[string]struct{}, len(asked)+len(unasked)+len(f.keepSet)) for name := range f.keepSet { kept[name] = struct{}{} @@ -292,31 +330,7 @@ func (f *SkillFilter) decide(ctx context.Context, inv *agent.Invocation) (*skill } } - dropped := make([]string, 0) - for _, name := range names { - if _, ok := kept[name]; !ok { - dropped = append(dropped, name) - } - } - outcome := OutcomeNarrowed - reason := "" - if len(dropped) == 0 { - outcome = OutcomeUntouched - reason = "all candidates retained" - } - report := &Report{ - Outcome: outcome, - Reason: reason, - TurnKey: inv.InvocationID, - Asked: skillNames(asked), - Unasked: unasked, - Dropped: dropped, - Scores: scores, - CatalogTokensBefore: before, - CatalogTokensAfter: before - skillTokensFor(summaries, dropped), - } - report.SavedTokens = report.CatalogTokensBefore - report.CatalogTokensAfter - return &skillRunResult{outcome: outcome, kept: kept, scores: scores}, report + return kept, scores } // emit delivers the report: one log line always, plus the WithObserver From 0dae28a19b97a0c1a07e173cf26799858f472523 Mon Sep 17 00:00:00 2001 From: Jeffrey <1187141927@qq.com> Date: Thu, 5 Nov 2026 00:01:51 +0800 Subject: [PATCH 3/7] plugin/toolfilter: address review feedback Propagate scanner errors and reject malformed calculator operands, including correct handling of scientific notation. Clarify example filtering behavior and remove the design document. Fix conversation deduplication, validate timeouts, bound embedding work, and support embedding-only recall. Preserve newly added skills, normalize visibility lookups, and strengthen regression coverage for question budgets. --- docs/design/decision-model.md | 873 ------------------------- examples/decisionfilter/README.md | 20 +- examples/decisionfilter/main.go | 20 +- examples/decisionfilter/main_test.go | 95 +++ plugin/toolfilter/conversation.go | 4 +- plugin/toolfilter/conversation_test.go | 18 +- plugin/toolfilter/options.go | 22 +- plugin/toolfilter/options_test.go | 46 ++ plugin/toolfilter/recall.go | 102 +-- plugin/toolfilter/recall_test.go | 84 +++ plugin/toolfilter/skillfilter.go | 15 +- plugin/toolfilter/skillfilter_test.go | 60 ++ plugin/toolfilter/toolcallback_test.go | 8 +- 13 files changed, 427 insertions(+), 940 deletions(-) delete mode 100644 docs/design/decision-model.md create mode 100644 examples/decisionfilter/main_test.go create mode 100644 plugin/toolfilter/options_test.go diff --git a/docs/design/decision-model.md b/docs/design/decision-model.md deleted file mode 100644 index b122ed31db..0000000000 --- a/docs/design/decision-model.md +++ /dev/null @@ -1,873 +0,0 @@ -# tRPC-Agent-Go 决策模型集成设计文档（v2，社区对齐版） - -> **v2 修订说明（2026-11-01）**：本版全面重写。v1 假设"决策模型集成层"是空白， -> 自建了 `decision` 抽象层 + `decision/jev` + `decision/mock` + `graph.AddDecisionNode`。 -> 重写前的事实核查（附录 B）发现社区路线已与此冲突： -> -> - 提案 issue #2632 明确 Non-Goal："Adding a new generic decision-model -> abstraction"（不做通用决策模型抽象层），并确定用既有槽位集成； -> - `internal/systemone` 协议客户端已由 PR #2652 合入（wangxuw，2026-10-10）； -> - 工具审批 reviewer（v1 三期的风险门控场景）正由 PR #2664 在审； -> - 评估器由 PR #2638 占位（liuzengh）。 -> -> 结论：v1 的一期（抽象层 + jev 客户端 + mock + AddDecisionNode）与社区路线 -> 重叠或被否决，本地实现已撤销。v2 重新分期： -> -> ```text -> 第一期：社区基础（已有实现 + 在途 PR + 规则约束）—— 不写代码，只锚定地基 -> 第二期：toolfilter / skillfilter / modelrouter —— 我们的核心贡献（无人认领的空白） -> 第三期：出口质检 + 提案剩余示例 + 观测收尾 -> ``` -> -> v1 中仍然有效的资产：附录 A 的 Octop 生产调研（二期 toolfilter 的全部依据）、 -> 执行流程与召回层设计（骨架换为 systemone 客户端后逻辑不变）、失败语义纪律 -> （筛选 fail-open / 安全 fail-closed）。均已并入 v2。 - ---- - -## 一、第一期：社区基础（现状 + 规则，不写代码） - -第一期在 v2 中不再是交付物，而是**必须锚定的地基**：了解社区已有什么、 -正在交付什么、遵守哪些约束。本期的行动项只有两个：在 issue #2632 下提出 -二期方向并获得认可（见 2.0），以及可选地认领提案内无人认领的两个示例 -（见 3.2）。 - -### 1.1 提案 #2632 与社区路线图 - -[issue #2632](https://github.com/trpc-group/trpc-agent-go/issues/2632) -（*proposal: support TypeSafe Jev for structured decisions*，作者 wangxuw， -2026-09-21 开启，状态 open）是社区 Jev 集成的总路线图。其 scope： - -- 带重试的 HTTP 客户端（已在 PR #2652 落地，注：落地版无重试，重试归上层）； -- 三种问题类型，保留概率、置信度、用量与解析出的模型版本； -- "把 Jev 决策包装成 callable tool"的示例（**无人认领**）； -- "Graph 按决策 + 可配置置信度阈值路由"的示例（**无人认领**）； -- Jev 版 `review.Reviewer` 工具审批实现（PR #2664 在审）； -- 协议与集成测试 + 双语文档。 - -提案原文的核心句（集成哲学，v2 全文遵守）： - -> Existing `function.NewFunctionTool`, `StateGraph.AddNode`, conditional -> edges, and the approval plugin provide integration points **without -> requiring a new Agent execution loop**. - -维护者互动（详见附录 B）：IRONICBo 给了审批数据集建议（绑定参数 + 状态 -指纹、fail-closed、mutation 一律拦截）；liuzengh 认领评估方向。**没有任何 -人讨论或否决过抽象层问题——Non-Goal 是提案作者自划范围，但两个已落地 PR -均按"具体实现填既有槽位"路线执行，形成既成方向。** - -### 1.2 `internal/systemone` 客户端（PR #2652，已合入） - -位置：`internal/systemone/`（16 个 Go 文件 + 测试 + `README.md` + `DESIGN.md`， -引用官方 OpenAPI v0.2.0 与 Laya serve.py 契约）。**全仓零使用方（等集成层 -接入），仅根 module 内的包可导入**（Go internal 可见性规则；examples 与 -兄弟 module 被 CI 明确禁止）。 - -消费视角的契约速览（以包内文档为准，此处仅摘二期会用到的部分）： - -```go -// 构造（唯一入口；默认 30s 超时） -client, err := systemone.NewClient(baseURL, - systemone.WithAPIKey(key), // 必须显式传入 - systemone.WithDefaultModel("jev-2025-10-10"), // 可选，钉版本 - systemone.WithHTTPClient(hc), // 测试注入点 - systemone.WithTimeout(30*time.Second), -) - -// 一次决策请求 -resp, err := client.SystemOne(ctx, &systemone.Request{ - State: state, // string | map[string]any | []any（会做 JSON 形态校验） - Questions: questions, // map[string]systemone.Question，非空 -}) - -// 三种问题（Go 命名刻意用 Binary，协议字段仍是 "noul"） -systemone.BinaryQuestion{Instructions: "text", Criteria: &systemone.BinaryCriteria{True: "...", False: "..."}} -systemone.ChoiceQuestion{Instructions: "text", Options: []systemone.ChoiceOption{{Name, Description}}} // 有序、唯一 -systemone.ScoreQuestion{Instructions: "text", Criteria: []string{"level0", "level1"}} // >= 2 级 - -// 强类型答案访问器（id 不存在 / 类型不符 / 校验失败 → error） -ans, err := resp.Binary(questionID) // ans.Value float64 -ans, err := resp.Choice(questionID) // ans.Selected / ans.Probabilities / ans.Confidence -ans, err := resp.Score(questionID) // ans.Score / ans.Levels []ScoreLevel{Description, Probability} / ans.Confidence -resp.Usage() // InputTokens / OutputTokens / Truncated（截断证据） -``` - -**严格校验语义**（对二期有直接影响）：答案必须匹配全部请求问题（数量、 -类型、choice 选项集、score 层级数）；概率必须 ∈[0,1] 且求和≈1（容 4 位小数 -舍入）；score 必须在 rubric 范围内；文本必须非空白 UTF-8。**响应不完整 = -整个调用返回 error**——v1 设计里"缺分（部分回复）→ 保持原目录"的分支在 -此客户端下不存在：要么全部答案齐、要么整调用失败走 fail-open。指令与 -criteria 是纯文本（string），即使 provider 支持富 JSON 输入也不使用。 - -### 1.3 PR #2664（在审）：System One 审批 reviewer - -`plugin/guardrail/approval: add system one reviewer and final permission checks` -（wangxuw，2026-10-12 开启，27+ 文件）。它确立了**集成层构造模式**（v2 二期 -直接沿用）： - -- 新增公开包 `plugin/guardrail/approval/review/systemone`，内部 import - `internal/systemone`，对外实现既有 `review.Reviewer` 接口 - （`Request{Action, Transcript} → Decision{Approved, RiskScore, RiskLevel, Reason}`）； -- **公开构造器形态**：`New(baseURL string, opts ...Option)`——baseURL 显式 - 传入，`WithAPIKey/WithModel/WithTimeout/WithHTTPClient` 透传给内部客户端， - 外加领域选项（`WithRiskThreshold` 默认 50、`WithMinConfidence` 默认 0.8）； - **不读环境变量**； -- 决策策略在本地代码：一次请求问 authorization（Choice：supported / - contradicted / insufficient）+ risk（Score 五级 rubric）两问，批准 = - supported 且 risk < 阈值 且置信度 ≥ 下限；`Reason` 是本地策略拼接文本， - 不是生成文本； -- **fail-closed**（安全门控语义）：无用户意图直接拒（不调服务器）、截断 - 证据拒、任何错误拒； -- 附带 final permission checks（审批后参数定稿时的终点复查）、跨 agent - 委托/steering/taskrun 的权限继承、示例与文档。 - -它同时覆盖了 v1 三期的"风险门控"场景，**v2 不再做任何 guardrail 方向的工作**。 - -### 1.4 PR #2638（占位）：Jev 原子 rubric 评估器 - -liuzengh 在 #2632 下的评估方向跟进：Jev 实现既有 -`evaluation/evaluator.Evaluator` 接口并注册，原子 rubric 评估为首个里程碑， -与 LLM judge 对照报告。目前是空占位 PR。**v2 不做评估方向的工作**，但二期 -工具预筛与其"合批一次请求、精确检查在代码、API 错误与负面判断分开"的 -设计约束精神一致。 - -### 1.5 LangChain typesafe 官方生态参照 - -`langchain-typesafe` 0.0.1a3（本地包核实，附录 B.4）的官方集成形态： - -| 官方组件 | 对应物 | -| --- | --- | -| `TypeSafeClassifier`（Runnable） | `internal/systemone` 客户端（功能对应） | -| `ModelRouterMiddleware`（模型请求钩子选模型） | **二期 modelrouter 的直接先例** | -| `AutoModeMiddleware`（工具调用钩子拦风险） | PR #2664 的对应物 | -| graph 场景 | 官方示例让用户**自己写普通 node** 调 classifier + conditional edges——无框架 API | - -关键情报：**LangChain 官方没有工具/技能预筛**（toolfilter 在其生态也是 -空白）——二期是 LangChain 官方都没做的差异化方向（Octop 有生产实践， -附录 A）。 - -### 1.6 集成规则与红线（v2 全程遵守的约束清单） - -1. **不引入通用决策抽象**（Non-Goal）：不新增 `Decider` 类接口、不建 - `decision/` 包；每个集成是"具体实现填既有槽位"。 -2. **internal 可见性**：`internal/systemone` 的类型**不得出现在任何公开 - API 签名里**；集成包一律在根 module 内，构造器形态 = `New(baseURL, - opts ...Option)`（PR #2664 模式），内部自建客户端。 -3. **不读环境变量**：显式配置，用户自己 `os.Getenv` 后传入。 -4. **测试注入走 HTTP 层**：`WithHTTPClient` + `httptest` 脚本化响应（仓库 - 惯例，systemone 自身与 PR #2664 测试均如此）；不建 mock 接口层。 -5. **失败方向纪律**：筛选/优化类（toolfilter、skillfilter、modelrouter） - **fail-open**；安全类（审批、门控）fail-closed（PR #2664 已做）。 - 两个默认值方向绝不写反。 -6. **响应严格校验语义**：接受"部分回复 = 整调用 error"，由 fail-open 兜底； - 不做多字段名容错（`noul/probability/p_yes/yes` 容错是杂牌方言，上游已 - 拒绝；协议收敛的供应商走 OpenRouter/Laya 兼容端点，客户端天然支持）。 -7. **核心包零感知**：`graph`/`agent`/`model`/`plugin` 框架层不 import - `internal/systemone`；只有 `plugin/` 集成包与示例碰它。 -8. **既有槽位优先**：BeforeModel 回调、`llmagent.WithSkillFilter`、 - `agent.ModelSelector`、`review.Reviewer`、`evaluator.Evaluator`——全部 - 已存在，只做实现，不改槽位。 -9. **流程规则**：新能力（二期三个集成）动工前先在 issue #2632 评论提出， - 获得提案 owner / 维护者认可；一期一个 commit，PR 总量小步提交。 -10. **命名**：Go 侧用 `Binary` 而非 provider 术语 `Noul`（上游命名约定， - 见 `internal/systemone` DESIGN.md）。 - -### 1.7 剩余可认领的空位 - -提案 scope 内无人认领的交付物（低成本、纯示例，受 1.6 规则 2 约束需要 -一个公开集成包作桥——正是 PR #2664 为审批场景建 `review/systemone` 的 -同款做法）： - -1. "Jev 决策包装成 callable tool" 示例（`function.NewFunctionTool` 槽位）； -2. "Graph 按决策 + 置信度阈值路由" 示例（普通 `AddNode` + NodeFunc + - conditional edges，无新 API）。 - -v2 将其归入三期（3.2）作为融入社区的低成本贡献。 - ---- - -## 二、第二期：toolfilter / skillfilter / modelrouter（核心贡献） - -### 2.0 定位与前置流程 - -二期是 v2 的主体，也是社区真正的空白（提案无此步骤、LangChain 官方无 -对应物、Octop 有生产实践佐证价值）。三个集成全部是"既有槽位上的具体 -实现"，零框架改动、零新抽象，符合 1.6 全部约束： - -| 集成 | 槽位（已存在，不改） | 形态 | -| --- | --- | --- | -| toolfilter | `model.BeforeModelCallbackStructured`（改写 `Request.Tools`，官方支持 "can be modified"） | `plugin/toolfilter`（根 module 包） | -| skillfilter | `llmagent.WithSkillFilter`（`skill.VisibilityFilter` 槽位） | 同上包内第二个构造器 | -| modelrouter | `agent.ModelSelector`（`func(ctx, *Invocation) (model.Model, error)`，纯函数槽位） | 同上包内第三个构造器（或独立 `plugin/modelrouter`） | - -**前置流程（硬性）**：动工前在 issue #2632 发英文评论，提出"Tool / skill -pre-filtering and model routing as concrete integrations on existing slots"： -复用三个既有槽位、零新抽象、fail-open 语义；引 LangChain -`ModelRouterMiddleware` 为 modelrouter 先例、引 Octop 生产实践为 toolfilter -依据；询问纳入本提案路线还是另开 issue。**获得认可前不写代码**。 - -与 v1 二期的差异：骨架从 `插件 → decision.Decider → jev/mock` 换为 -`插件 → internal/systemone`（插件内部直连），测试从 mock Decider 换为 -httptest。**全部核心设计（预筛算法、缓存策略、fail-open 语义、逃生通道、 -三态事件、token 遥测）原样保留。** - -### 2.1 构造形态总规则（PR #2664 模式） - -```go -// 三个集成的构造器同构：baseURL 显式 + 选项透传 + 领域选项 -func NewToolCallback(baseURL string, opts ...Option) (*ToolCallback, error) -func NewSkillFilter(baseURL string, repo skill.Repository, opts ...Option) (*SkillFilter, error) -func NewModelRouter(baseURL string, opts ...Option) (*ModelRouter, error) - -// 共通选项（每个包自持一份，与 reviewer 一致；不抽公共 Option 包——那会变相造抽象） -WithAPIKey(key string) -WithDefaultModel(model string) // 钉版本，如 "jev-2025-10-10" -WithTimeout(d time.Duration) // 默认 30s（跟随 systemone 默认） -WithHTTPClient(c *http.Client) // 测试/策略注入 - -// 领域选项见 2.2-2.4 -``` - -内部统一： - -```go -client, err := systemone.NewClient(baseURL, - systemone.WithAPIKey(o.apiKey), - systemone.WithDefaultModel(o.model), - systemone.WithTimeout(o.timeout), - systemone.WithHTTPClient(o.httpClient), -) -``` - -### 2.2 工具侧：`plugin/toolfilter`（BeforeModel 回调改写工具列表） - -**用户侧**： - -```go -cb, err := toolfilter.NewToolCallback("https://api.typesafe.ai", - toolfilter.WithAPIKey(os.Getenv("TYPESAFE_API_KEY")), // env 由用户读（规则 1.6-3） - toolfilter.WithDefaultModel("jev-2025-10-10"), - toolfilter.WithThreshold(0.6), // P(需要) >= 0.6 保留（Octop 默认值，附录 A.2） - toolfilter.WithSlack(0.15), // 近线补录（Octop 默认值） - toolfilter.WithMinToolCount(8), // 工具少于 8 个不筛（Octop tool_min_count） - toolfilter.WithShortlistLimit(16), // 词法召回入围上限 - toolfilter.WithMaxQuestions(20), // 单请求问题数上限（Octop 同值） - toolfilter.WithKeepAlways("my_core_tool"), // 在默认保留集之外追加用户自留工具 - // toolfilter.WithShortlistEmbedder(myEmbedder), // 可选 embedding 召回路 -) - -agt := llmagent.New("my-agent", - llmagent.WithModel(myModel), - llmagent.WithModelCallbacks(&model.Callbacks{ - BeforeModel: []model.BeforeModelCallbackStructured{cb.Callback()}, - }), -) -``` - -**默认保留集**：Octop 生产实现把 28 个内置工具写死在 -`_TOOLS_ALWAYS_KEPT`（完整清单见附录 A.2）——**永不参与打分**（代码注释： -给内置工具花一个问题只会挤占决策模型真正能删的长尾，且删掉核心集会让模型 -打出裸 tool call 然后停摆），且**无条件进保留集**（即使全部被问工具低于 -阈值、收窄到最小集时也在）。trpc 是框架而非产品，默认保留框架自身的关键 -流程工具；业务注册的文件、记忆和搜索工具由业务方按模型侧名称显式保留： - -| 默认保留 | 来源（已核实注册名） | 理由（Octop 语义的移植） | -| --- | --- | --- | -| `skill_load`、`skill_select_docs`、`skill_list_docs` | skill 渐进披露 | 保留技能加载及文档发现流程 | -| `skill_run`、`skill_exec`、`skill_write_stdin`、`skill_poll_session`、`skill_kill_session` | skill 执行 | 保留执行入口及会话操作 | -| `workspace_exec`、`workspace_write_stdin`、`workspace_kill_session`、`workspace_save_artifact` | workspace 执行 | 保留执行入口及会话、产物操作 | -| `session_search`、`session_load` | 会话回溯 | 保留按需搜索和加载流程 | -| `transfer_to_agent`、`await_user_reply`、`environment_context_current_time` | Agent 转交、用户回复和时间上下文 | 保留框架控制与上下文流程 | - -语义三条（对齐 Octop 决策中间件，源码已核）： - -1. 默认保留集**不进候选、不生成问题、不打分**——它们直接从打分名单里排除， - 省下的问题名额全部留给长尾； -2. 保留集 = Jev 判定保留 ∪ 入围但问题预算溢出的（unasked）∪ 默认保留集 ∪ - `WithKeepAlways` 用户追加——四者取并集； -3. 只有实际出现在请求中的名称才受影响。业务方通过 `WithKeepAlways` - 追加核心工具，名称须与模型请求一致，包含可能的工具集前缀；默认集不可裁剪。 - `tool_search` 不保证找回被筛掉的工具，必要时由业务方显式保留。 - -**内部执行流程**（参数默认值与边界处理按 Octop 生产实现校准，证据见 -附录 A；骨架为 systemone 客户端）： - -1. **每个 Invocation 打分一次**。key 取 Invocation ID；决策上下文取 Session - 中按默认事件前缀过滤的近期用户与助手对话，并补入当前 Invocation 消息。 - 三个决策组件使用相同提取规则，工具预筛使用近期用户消息。工具结果不进入 - 上下文，2400 字符预算优先保留最新消息。Session 事件不保证等同最终模型 - 请求（如临时注入消息或自定义过滤）。ReAct 后续调用重放首次目录筛选结果， - 之后新增的工具直接保留；没有 Invocation 或有效文本时保持工具目录不变； -2. **两道串联判定**：词法预筛（免费）→ Jev 打分（花钱）。默认保留集 - （技能、会话和执行等框架流程工具，见上文表格）**先从 - 候选中排除**——不参与打分、不占问题名额。目录超过 - shortlist 上限时，先按"用户消息 token 与名字+描述的重叠"取前 N 个入围； - 小目录直接送 Jev；大目录按近期用户消息词项对子串的命中次数排序，仅正分 - 候选入围（启用 embedding 时融合排序）；技能同样使用此规则。只有入围的才生成 Binary 问题（"本轮需要工具 X 吗"），**单请求合批并行 - 评估**。词法落选的候选视为预判不相关——收窄发生时一并删除（硬切语义， - 理由与更保守选项的取舍见 2.6）。大目录没有召回候选时筛选中止，目录原样； -3. **保留判定**：得分 ≥ 阈值入选；没有打分候选过线时，slack 近线补录。最终保留集 = - Jev 判定保留的 ∪ 入围但问题预算溢出的（unasked）∪ 默认保留集 ∪ - WithKeepAlways 用户追加。全部被问且全部低于阈值减 slack 时收窄到 - "溢出的 + 默认保留集 + 用户追加"。 - （v1 的"缺分保持原目录"分支删除：systemone 严格校验下缺分 = 整调用 - error，由第 4 条吞掉，语义等价且实现更简单；） -4. **fail-open**：`SystemOne` 返回任何错误（网络、协议、截断证据）→ 全量 - 放行 + 记事件。筛选是优化，失败要放行；与 PR #2664 的 fail-closed 方向 - 相反，两处默认值不能写反（规则 1.6-5）； -5. **发出三态结果事件**（未动 / 收窄 / 保留子集，含每项得分）并统计节省的 - token 数进遥测（附录 A.4 的可观测性做法）。 - -**打分调用的写法**（问题构造与答案读取的形态）： - -```go -questions := make(map[string]systemone.Question, len(shortlist)) -for _, t := range shortlist { - questions["tool:"+t.Name] = systemone.BinaryQuestion{ - Instructions: fmt.Sprintf( - "Does the current user request need the tool %q? Tool description: %s", - t.Name, truncate(t.Description, 240)), // Octop 描述截断 240 字符 - } -} -resp, err := f.client.SystemOne(ctx, &systemone.Request{ - State: map[string]any{ - "user_message": userMsg, - "tool_catalog": toolSummaries(shortlist), - }, - Questions: questions, -}) -if err != nil { - return nil, nil // fail-open：全量放行，发事件记录 -} -// 先遍历全部答案确定 anyAboveThreshold，再逐项判定。 -// 严格校验保证被问的问题全部有答案。 -ans, _ := resp.Binary("tool:" + t.Name) -keep := ans.Value >= f.threshold || (!anyAboveThreshold && ans.Value >= f.threshold-f.slack) -``` - -state 裁剪：只含 user/assistant 对话行（剔除 ToolMessage），裁到 2400 字符， -最新消息优先保留——决策 API 吃短情境串，不吃完整上下文窗口（附录 A.2）。 - -**召回层算法（词法预筛的实现设计）** - -硬切会丢弃召回层遗漏的候选，决策模型无法评估它们，也不保证能够恢复； -业务可用 `WithKeepAlways` 保护关键工具。两路召回，RRF 融合： - -```text -关键词路（默认，零外部依赖）： - token 切分（CJK 单字 + 拉丁词，参考 Octop select.py 的处理） - + 名字和描述合并后做子串匹配，每个用户词项最多计一次分 -embedding 路（可选，复用 knowledge/embedder 接口）： - 用户文本和候选描述按需嵌入，分别保存在每个 filter 的 128 条和 1024 条 LRU 缓存中；淘汰后按需重新计算。 - 候选描述并发嵌入，每个 filter 的 embedding 调用总并发最多为 8。 - 每 turn 只嵌入一条用户消息，余弦相似度排序。 - 同义词、跨语言、间接相关在此路解决。 -融合：RRF（Reciprocal Rank Fusion）——两个榜单按排名倒数求和。 - 关键词抓精确命中（工具名、专有名词），embedding 抓语义相似， - 两路失败模式互补；RRF 是纯算术，零模型调用。 -``` - -边界行为： - -1. 无 embedder 或 embedder 调用失败 → 退回纯关键词单路（fail-open 降级， - 与 toolsearch 的 `WithEmbeddingFailOpen` 语义一致）； -2. 工具目录的嵌入不需要向量库——几十上百个描述在内存里算余弦即可， - 不引入 `vectorstore` 依赖。 - -**边界与演进路径（与 plugin/toolsearch 的关系）**： - -1. **不改动 `plugin/toolsearch`**（已核实：仓库内无任何代码 import 它， - 打分内核全是私有函数）。toolfilter 的召回层自带实现，不复制它的导出 - API，注释标明语义参考； -2. 两套打分暂不共享的代价是排序可能漂移（推模式删掉的工具，拉模式未必排 - 在搜索前列）。将来的共享内核下沉（`internal/toolmatch`，toolsearch 留 - 类型别名保持兼容、跑全部既有测试防回归）**不在二期范围内**，重构理由 - 需要实际召回效果的数据支撑； -3. 二期验收标准包含召回测试：建 golden set（用户消息、工具目录、已知相关 - 工具），断言相关工具全部进入入围名单，必须覆盖跨语言和同义词用例。 - 召回不达标，硬切不上线。 - -### 2.3 skill 侧：`VisibilityFilter` 实现（`WithSkillFilter` 槽位） - -skill 的渲染时序和工具不同（已核实）：skill 摘要和已加载正文由 -`SkillsRequestProcessor` 在 preprocess 阶段注入 system message，发生在 -BeforeModel 回调**之前**。回调触发时 skill 目录已是 prompt 里的纯文本， -`Request` 上没有结构化字段可改。但 trpc 已有现成机制，不需要像 Octop 那样 -用正则修剪 prompt： - -- `skill.VisibilityFilter`（`skill/context_repository.go`）：函数槽位 - `func(ctx, skill.Summary) bool`，决定每个 skill 对本轮是否可见； -- `llmagent.WithSkillFilter(filter)`：agent 构造时自动把 filter 包进 skill - 仓库（`skill.NewFilteredRepository`）。包装后的仓库在所有入口生效—— - overview 注入（`SummariesForContext`）、正文加载（`GetForContext`）、 - 路径访问（`PathForContext`）。不可见的 skill 不出现在 prompt 里， - `skill_load` 点名加载也会得到 "not found"。 - -定位：trpc 现有的 skill 可见性机制是静态规则——scope 隔离（仓库按 -app/user 划分）、`WithMaxOverviewSkills`（overview 条数截断）、 -`WithMaxLoadedSkills`（加载驱逐）。机制是流水线关系，decision filter 补的是 -第一道"按内容动态判断"的工序，后面的截断照常工作： - -```text -scope 隔离（决定全量集合，如 40 个） - ↓ -VisibilityFilter（本设计：决策模型筛选，如剩 3 个） - ↓ -MaxOverviewSkills（可见条数过多时截断显示） - ↓ -MaxLoadedSkills（加载全文后的驱逐） -``` - -与渐进式披露正交：filter 收窄的是披露的候选池（第一级 overview），LLM 在 -可见集合里继续做第二级判断（要不要 `skill_load` 全文）。 - -**实现要点：逐条签名，批量藏在函数内部** - -`VisibilityFilter` 的签名是逐条询问，和决策模型"一次请求批量问"不匹配。 -但 filter 是空槽位，签名只约定"问一次答一个 bool"，不约束实现内部怎么做。 -批量藏在 filter 的第一次调用里： - -```go -func (f *SkillFilter) visible(ctx context.Context, s skill.Summary) bool { - run := runKeyFromContext(ctx) // 本轮用户消息的标识 - kept, ok := f.cache.get(run) - if !ok { - all := f.repo.Summaries() // 构造时捕获的仓库，拿全量摘要 - kept = f.decideBatch(ctx, all) // 首次调用：N 个 skill:N 问题合成一次 SystemOne 请求 - f.cache.set(run, kept) - } - return kept[s.Name] // 本轮后续 N-1 次调用全查缓存 -} -``` - -框架的 for 循环照常逐条问，只是第一次问的时候触发批量决策，之后全部命中 -内存缓存。不改 `SummariesForContext` 本身（已发布的公共契约，逐条语义对 -白名单、scope 这类便宜实现是合理的），也不写并行版的新函数。 - -缓存和并发的三个细节： - -1. 缓存 key 是 run（用户消息），不是 agent。filter 实例在 agent 构造时挂上， - 全生命周期共享，多个会话并发进出同一个 agent——缓存做成 - `map[runKey]kept` 加锁（或 singleflight，同 run 并发首次调用只发一次 - 请求）。新用户消息进来 key 变化，缓存自然失效； -2. 每个 LLM step 都会重新渲染 overview，即重新过一遍全部 filter 调用。缓存 - 让重复调用变成内存查询，一个 turn 只花一次决策请求； -3. fail-open 藏在 filter 里：批量决策失败时 kept 退化为"全部可见"，行为 - 等价于没配 filter。 - -**用户视角的完整示例**： - -```go -repo, _ := skill.NewFSRepository("./skills") - -flt, _ := toolfilter.NewSkillFilter("https://api.typesafe.ai", repo, - toolfilter.WithAPIKey(key), - toolfilter.WithDefaultModel("jev-2025-10-10"), - toolfilter.WithThreshold(0.6), - toolfilter.WithKeepSkills("core-skill"), -) - -agt := llmagent.New("my-agent", - llmagent.WithModel(myModel), - llmagent.WithSkills(repo), - llmagent.WithSkillFilter(flt.VisibilityFilter()), // skill 收窄 - llmagent.WithModelCallbacks(&model.Callbacks{ - BeforeModel: []model.BeforeModelCallbackStructured{cb.Callback()}, // tool 收窄 - }), -) -``` - -效果：仓库 40 个 skill，本轮决策模型判断 3 个相关——overview 只列 3 条； -模型 `skill_load` 一个不可见的 skill，`GetForContext` 返回 "not found"； -`skill_load` 工具本身在默认保留集里，不会被筛掉。 - -### 2.4 模型分流：`ModelSelector` 实现 - -对标 LangChain `ModelRouterMiddleware`（官方先例，附录 B.4），实现既有 -`agent.ModelSelector` 槽位（`func(ctx, inv *Invocation) (model.Model, error)`， -已核实签名；模型以实例形式注册，用户消息在 `inv.Message`）： - -```go -router, err := toolfilter.NewModelRouter("https://api.typesafe.ai", - toolfilter.WithAPIKey(key), - toolfilter.WithDefaultModel("jev-2025-10-10"), - toolfilter.WithModelCandidate("gpt5mini", myGpt5Mini, - "fast, cheap tasks, simple lookups, summarization"), - toolfilter.WithModelCandidate("claude", myClaude, - "complex reasoning, long-context analysis, code review"), -) - -// 每次 Run 传入选择器（RunOption；模型实例在候选表里，不是名字字符串） -out, err := runner.Run(ctx, userID, sessionID, msg, - agent.WithModelSelector(router.Select), -) -``` - -**内部**——一个 ChoiceQuestion，每 invocation 缓存一次（ReAct 循环内后续 -LLM 调用直接重放，与 LangChain 的 per-run 缓存语义一致）： - -```go -func (r *ModelRouter) Select(ctx context.Context, inv *agent.Invocation) (model.Model, error) { - if m, ok := r.cache.get(inv.InvocationID); ok { - return m, nil - } - options := make([]systemone.ChoiceOption, 0, len(r.candidates)) - for _, c := range r.candidates { - options = append(options, systemone.ChoiceOption{ - Name: c.name, Description: c.criteria, // 有序（保序是协议要求） - }) - } - resp, err := r.client.SystemOne(ctx, &systemone.Request{ - State: inv.Message.Content, // string 是合法 State - Questions: map[string]systemone.Question{ - "model_route": systemone.ChoiceQuestion{ - Instructions: "Select the model best suited for this user request.", - Options: options, - }, - }, - }) - if err != nil { - return r.defaultModel, nil // fail-open：出错用默认模型 - } - ans, err := resp.Choice("model_route") - if err != nil || r.byName[ans.Selected] == nil { - return r.defaultModel, nil - } - m := r.byName[ans.Selected] - r.cache.set(inv.InvocationID, m) - return m, nil -} -``` - -**设计决定**：循环终止（ReAct 循环是否继续）不引入决策模型。该判断属于 -开放式推理（是否继续思考、下一步做什么），由 LLM 在生成 tool call 或最终 -回答时自行决定，不属于有界判断。（此条 v1 已定，保留。） - -### 2.5 测试策略（httptest 脚本化响应） - -替代 v1 的 mock Decider（规则 1.6-4），并额外覆盖真实 wire 解码层： - -```go -func newTestToolCallback(t *testing.T, answers map[string]float64) *toolfilter.ToolCallback { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - io.WriteString(w, decisionResponseJSON(answers)) // {"answers":{"tool:X":{"type":"noul","noul":0.9}},...} - })) - t.Cleanup(server.Close) - cb, _ := toolfilter.NewToolCallback(server.URL, - toolfilter.WithAPIKey("test-key"), - toolfilter.WithHTTPClient(server.Client()), - toolfilter.WithThreshold(0.6), - ) - return cb -} -``` - -fail-open 覆盖：handler 返回 500 / 429 / 非法响应体 / `server.Close()` 断连 -→ 断言工具列表全量保留。原 v1 用 `mock.WithError` 注入的全部场景等价覆盖。 - -### 2.6 与 Octop 实现的校准点汇总 - -| 设计点 | Octop 生产值（附录 A） | v2 采用 | -| --- | --- | --- | -| 阈值 / slack | 0.6 / 0.15 | 同 | -| minToolCount | 8 | 同 | -| 单请求问题上限 | 20 | 同（WithMaxQuestions） | -| 词法零重叠 | 整体中止 | 同 | -| 词法落选 | 删除（硬切） | 同，召回不达标不上线 | -| turn 缓存 | key=最后一条 human 消息，LRU 128 | 同语义，run 级缓存 | -| 描述截断 / state 裁剪 | 240 字符 / 2400 字符、剔除 ToolMessage | 同 | -| 内置工具保留集 | `_TOOLS_ALWAYS_KEPT`：28 个产品内置工具，永不打分、无条件保留 | 保留框架流程工具；业务按实际模型侧名称经 WithKeepAlways 追加 | -| 显式指名优先 | @成员直接锁定 | v2 暂不引入（无成员目录场景），留作演进 | -| 被过滤工具的恢复 | ALWAYS_KEPT 保核心，但被筛掉的长尾工具无法找回 | 不保证恢复；业务可通过 WithKeepAlways 防止关键工具被过滤 | - ---- - -## 三、第三期：收尾与增值 - -### 3.1 出口质检（AfterAgentCallback + Score） - -挂 `agent.AfterAgentCallback`：对最终回答问 Score（回答质量 rubric：充分/ -部分/不足），低分注入 `CustomResponse` 触发重答（上限 N 次）。为文档示例 + -可选包，不是核心行为。参照附录 A.3 第 3 条：更温和的"提示 + 拦截"组合 -（给 system message 追加总结提示，同时拦下后续工具调用）也可选，两案 -文档并列。 - -### 3.2 提案剩余示例认领（融入社区的低成本贡献） - -认领 issue #2632 中无人认领的两个示例（1.7）。Graph 路由示例严格用普通 -`AddNode` + NodeFunc + conditional edges（提案哲学，无新 API）；Jev-as-tool -示例用 `function.NewFunctionTool` 包装一次决策调用。两示例需要经一个公开 -集成包桥接 internal 可见性（PR #2664 同款做法，见 3.3 的候选位置）。 -**先在 issue 评论认领再动工。** - -### 3.3 观测与运维收尾 - -1. 决策调用遥测：三态事件（未动 / 收窄 / 保留子集）+ 节省 token 统计在 - 二期已做；三期补决策调用的 trace 对接（复用 invocation 既有 trace 机制， - 不新建事件类型）； -2. `model/failover` 将来识别决策模型错误：`internal/systemone` 已暴露稳定 - 的 `*HTTPError`（含 status 与 `Retry-After`、`X-Request-ID` 头）；三期 - 评估 failover 是否需要消费它，**只读不改 systemone**； -3. 共享集成桥的沉淀：若 3.2 两个示例与二期三个集成的构造样板重复度 - 过高，可评估一个小型公开桥包（仅构造便利，不含抽象——形态仍是 - `New(baseURL, opts...)`）；是否需要以实测重复度为依据，不预设。 - -### 3.4 开放问题（不立项，记录待社区演化） - -1. **无决策模型凭据用户的 LLM 后备**：v1 的 `decision/llm`（用现有 - `model.Model` 实现 Decider）随抽象层取消。可能路径：自建网关把 LLM 包装 - 成 systemone 兼容端点（协议收敛使然，OpenRouter/Laya 均为兼容先例）； - 或接受无后备。倾向后者，等真实需求出现再议； -2. **供应商预设**（lava / msdecision / qwen）：systemone 客户端本身即协议 - 兼容客户端，预设 = baseURL + model 组合，属用户配置而非代码交付； -3. **共享打分内核下沉**（`internal/toolmatch`）：见 2.2 演进路径，靠 - 实际召回效果的数据触发。 - ---- - -## 四、目录结构 - -```text -trpc-agent-go/ （根 module：trpc.group/trpc-go/trpc-agent-go） -├── internal/systemone/ ← 社区已有（PR #2652），只读 -├── plugin/guardrail/approval/review/systemone/ ← 社区在审（PR #2664），不做 -├── plugin/toolfilter/ ← 第二期（新增，根 module 包，import internal/systemone） -│ ├── toolcallback.go // NewToolCallback + BeforeModel 回调 -│ ├── skillfilter.go // NewSkillFilter + VisibilityFilter 实现 -│ ├── modelrouter.go // NewModelRouter + ModelSelector 实现 -│ ├── recall.go // 词法 + 可选 embedding RRF 召回层 -│ ├── cache.go // run 级缓存 / singleflight -│ ├── events.go // 三态事件 + token 节省遥测 -│ └── *_test.go // httptest 脚本化响应 -├── examples/llmagent/toolfilter/ ← 第二期示例：toolfilter + skillfilter + modelrouter -└── examples/graph/decisionroute/ ← 第三期认领示例（普通 AddNode + conditional edges） -``` - -依赖方向（唯一的箭头方向）： - -```text -plugin/toolfilter ──→ internal/systemone（根 module 内，合法） -examples/* ─────────→ plugin/*（公开包，不碰 internal） -core packages ──────→ ✗ 不依赖 internal/systemone（红线，规则 1.6-7） -``` - -（v1 的 `decision/`、`decision/jev/`、`decision/mock/`、 -`graph.AddDecisionNode` 全部移除——已被社区路线替代或否决。） - ---- - -## 五、里程碑 - -| 里程碑 | 交付物 | 验收标准 | -| --- | --- | --- | -| **M1（第一期：锚定地基）** | 不写代码。issue #2632 英文评论提出二期三集成方向；完成 `internal/systemone` 契约与 PR #2664 构造模式的通读校准（本文档 1.2/1.3 即校准结果） | 提案 owner（wangxuw）或维护者对二期方向表态（纳入 / 另开 issue / 拒绝）；若拒绝则仅保留 3.2 示例认领路径并重估 | -| **M2（第二期：核心贡献）** | `plugin/toolfilter`（tool 回调 + skill filter + model router 三构造器，内置关键词 + 可选 embedding RRF 召回层）；`examples/llmagent/toolfilter` 示例 | fail-open 有测试覆盖（500/断连/非法响应体→全量保留）；turn 内只打分一次、循环内重放有缓存验证；skill 筛选经 `WithSkillFilter` 生效，`GetForContext` 拒绝不可见加载；默认保留的框架流程工具不进打分名单、无条件保留有测试；召回 golden set 测试达标（含跨语言、同义词用例），不达标则硬切不上线；三态事件与节省 token 统计可用；httptest 全离线运行 | -| **M3（第三期：收尾增值）** | 出口质检示例（可选包）；提案剩余两个示例认领交付；决策调用 trace 对接 | 出口质检低分重答路径有测试；两个示例被提案 owner 接收；trace 不新建事件类型；`model/failover` 评估结论成文（做或不做，附依据） | - -每个里程碑完成后按 AGENTS.md 执行强制二轮设计评审（导出必要性、包归属、 -API 重叠、命名、可扩展性、兼容性、契约完整性、文档、验证九项）。提交纪律： -一期一个 commit，三期合计三个 commit，向 fork 提 PR 前先开 issue（CONTRIBUTING -要求）。 - ---- - -## 六、兼容性承诺（不变式清单） - -1. 不安装任何 `plugin/toolfilter` 时，`graph`/`agent`/`model`/`plugin` 的 - 默认、零值、错误、事件顺序、序列化行为与现状逐字节一致。 -2. `plugin/toolsearch` 与 `skill` 包的公共行为不变；toolfilter 与其无代码 - 依赖，仅设计互补。 -3. 所有新能力 opt-in：不挂回调、不配 filter、不传 selector 即不存在。 -4. 核心包不 import `internal/systemone`；`internal` 类型不出现在任何公开 - API 签名中（构造器一律 `New(baseURL, opts ...Option)`）。 -5. 不引入通用决策模型抽象（遵守提案 #2632 Non-Goal）；不读环境变量。 -6. 失败方向纪律：toolfilter/skillfilter/modelrouter 一律 fail-open；不触碰 - fail-closed 语义的安全域（那是 PR #2664 的职责边界）。 -7. 集成锚定的是 System One **线上 API 契约**（state + questions → typed - answers + probabilities，OpenAPI v0.2.0），而非任何 alpha 阶段包装层。 -8. 每个新 Go 文件携带 CONTRIBUTING.md 的 Tencent Apache 2.0 许可头。 - ---- - -## 附录 A：参考实现调研（Octop v1.0.2 / octop-harness 1.0.5） - -腾讯 Octop 在 v1.0.2 引入"技能、工具与团队的决策模型"（CHANGELOG 明确列出）。 -实现分两层：Octop 应用仓库（`github.com/TencentCloud/Octop`，tag v1.0.2）持有 -配置、REST API 和前端展示；过滤引擎在外部依赖 octop-harness 中，`decision` -包自 1.0.4 加入（本附录基于 PyPI 1.0.5 wheel 的源码，共 352 行 + 一个 1153 行 -的中间件）。以下事实直接取自代码。 - -### A.1 协议收敛 - -octop-harness 的 decision 客户端注释写明请求体匹配 OpenRouter decisions API， -Jev 也用该格式：`{state, questions: {id: {type: "noul", instructions, -criteria}}}`，回答取 `noul` 字段即 P(yes)。本地模型 Laya 的端点同样是 -`/v1/systemone`。TypeSafe systemone、OpenRouter decisions、Laya 三者线格式 -同构，"state + questions → 概率"正在成为决策模型的事实协议。 - -供应商差异靠一个宽松解析器吸收：响应兼容 `noul` / `probability` / `p_yes` / -`yes` 四种字段名、dict 和 list 两种行格式，而不是每家一个客户端。超时按供应 -商区分：远程 API 15 秒，本地 Laya 60 秒（本地 CPU 推理约每问 1 秒，这是性能 -预算的实证数据）。问题陈述的 criteria 双语自适应（中文给"适用/不适用"，英文 -给 "Yes, this applies."）。 - -> v2 注：A.1 的"宽松解析器"策略与上游 `internal/systemone` 的严格单契约 -> 选择相反，属社区已裁决项（严格校验 + 协议兼容端点覆盖供应商差异）。 -> 本附录保留 A.1 作为协议收敛的证据，**方言容错不再采纳**（规则 1.6-6）。 - -### A.2 筛选流程 - -每次模型调用前，中间件按以下步骤收窄目录： - -1. 取本轮用户消息和三个目录：技能（state 的 `skills_metadata`）、工具 - （`request.tools`）、团队成员（team roster）； -2. **词法预筛**：目录 ≤12 项（middleware 实际传 `_CATALOG_LIMIT = 12`； - select.py 函数签名的默认值 16 未被使用）全部送决策模型；超过 12 项按 - token 重叠打分取前 12；词法完全无命中则返回 None——整个目录保持不动。 - 词法落选的候选不送决策模型，收窄发生时一并删除（词法零重叠视为预判 - 不相关）； -3. 每个候选项一条 Noul 陈述，问题 ID 带类型前缀（`tool:X` / `skill:X` / - `member:X` / `result:sufficient`），文案形如「这轮需要工具「X」。{描述截断 - 240 字符}」； -4. 单次请求最多 **20 个问题**，按 result > skill > tool > member 的优先级在 - 四类目录间分配名额；member 放不下时整组放弃——部分评估会错杀没打分的成员； -5. **state 只含 user/assistant 对话行**（剔除 ToolMessage），裁到 2400 字符， - 最新消息优先保留。决策 API 吃短情境串，不吃完整上下文窗口。 - -保留判定：得分 ≥ 阈值（默认 **0.6**）入选；**slack 0.15**——差一点到线的项在 -名额未满时保留；一个都没过线返回 None，保持原目录。另有两个独立门槛：工具 -少于 **8 个**（`tool_min_count`）不筛；核心工具集 `_TOOLS_ALWAYS_KEPT` -（**28 个**，v2 重数修正 v1 的"约 27"）**永不参与打分**——代码注释原话： -给它们花一个问题只会挤占决策模型真正能删的长尾，且删掉核心集会让模型打出 -裸 tool call 然后停摆。完整清单（frozenset，按功能分组）： - -```text -文件系统/执行（8） ls, read_file, write_file, edit_file, delete, - glob, grep, execute -任务/时间（3） write_todos, task, current_time -检索逃生（1） tool_search -记忆（2） memory_search, memory_get -派工（2） agent_list, ask_agent -网络获取（1） web_fetch -搜索类（8） tavily_search, brave_search, google_search, - kimi_search, deepseek_search, yuanbao_search, - searchfree_search, you_search -用户/桌面（3） send_file_to_user, browser_use, desktop_screenshot -``` - -另设 `_HOST_DISPATCH_TOOLS = ("ask_agent", "agent_list")`：在决策卡片上 -展示但不打分——注释原话 "a low score must not take away the host's -ability to dispatch"（低分不得夺走宿主的派工能力）。 - -### A.3 三个边界处理 - -**失败与部分回复**：HTTP 客户端层面任何异常记 warning 并返回空 map，调用方 -拿到空结果即目录不变——fail-open 做在客户端里，只有连通性探针用 `strict=True` -抛错。被问的工具全部拿到分数且全部低于阈值，才删长尾；有缺分则保持原目录， -代码注释："a partial reply is not a verdict"（部分回复不是判决）。 - -> v2 注："部分回复不是判决"的语义在严格校验客户端下由 fail-open 整体吞掉 -> （任何响应不完整 = 调用 error = 全量放行），语义等价、实现更简。 - -**turn 缓存与重放**：每个用户 turn 只打分一次，key = thread + 最后一条 human -消息的位置和内容；该 turn 内 ReAct 工具循环的所有后续模型调用直接重放缓存的 -收窄结果（LRU 上限 128）。这回答了"每 run 一次还是每轮一次"：每 turn 一次， -turn 内重放。 - -**显式指名优先**：用户消息里 @某成员时直接锁定该成员，决策模型照常打分展示但 -无权改派；@了不存在的成员则本轮跳过决策。代码注释："Explicit user routing -always wins"（用户的显式选择永远赢）。 - -### A.4 与本设计的差异 - -1. **Octop 没有逃生通道**：被决策中间件筛掉的工具无法由模型自行找回。 - 硬证据（v2 核实，agent.py 装配顺序）： - `ToolsFilterMiddleware → DecisionMiddleware → tool_search`，且注释明写 - *"Search only the final model-visible tool set. In particular, this must - run after MCP and user filtering so hidden tools cannot leak into either - the client or provider search inventory"*——搜索目录就是收窄后的可见集， - 被删工具不在其中（v1 在 A.2 写过"判漏由 tool_search 兜底"，据本证据 - 修正：tool_search 在 ALWAYS_KEPT 里只保住"能搜"这个能力本身，搜不回 - 被决策删掉的工具）。trpc 的 `plugin/toolsearch` 结构不同：检索目录是 - MCP 工具注册表（deferred/lazy 加载全集），不是当前 `Request.Tools`—— - toolfilter 删掉的 MCP 工具仍在可搜可载范围内，模型能自己找回。推拉 - 互补在 trpc 成立、在 Octop 不成立，这是与 Octop 的关键差异点。 -2. **技能注入位置不同**：Octop 的技能目录先被上游 SkillsMiddleware 写进 - system prompt，收窄时除了改 state，还要用正则修剪 prompt 里的 - `**Available Skills:**` 段落。trpc 的渲染时序相同（skill 在 preprocess - 阶段注入 system message，早于 BeforeModel 回调），但已有 - `skill.VisibilityFilter` + `skill.NewFilteredRepository` + llmagent - `WithSkillFilter` 的现成机制：过滤发生在渲染之前，overview、正文加载、 - 路径访问共用同一道闸，不需要字符串手术。已核实，见 2.3。 -3. **结果充分性不篡改响应**：团队收尾时问"结果是否足以总结"（≥ 阈值即充分）， - 不注入 CustomResponse，而是两步：给 system message 追加"请直接总结"提示， - 同时在工具拦截层拦下后续 `ask_agent` 调用返回 error ToolMessage。提示 + - 拦截的组合比改写响应温和，trpc 的出口质检可参照（3.1）。 -4. **可观测性**：节省量 = 模型调用前后的目录 token 差，经 ContextVar 传给 - 响应，钉在 AI 消息的 `additional_kwargs` 上，同时经 stream writer 推 - `decision` 事件（含模型名、三态 parts、每项得分）。前端能显示每轮保留 - 清单和省下的 token 数。 - ---- - -## 附录 B：社区生态情报与核实记录（v2 重写依据） - -### B.1 时间线（2026-11-01 经 GitHub API 直连核实） - -| 时间 | 事件 | 内容 | -| --- | --- | --- | -| 2026-09-21 | issue #2632 开启（wangxuw） | 提案：scope + Goals + Non-Goals + Open Questions；作者要求 ownership、增量 PR | -| 2026-09-22 | IRONICBo 评论 | 审批数据集四案例；绑定参数 + 状态指纹；fail-closed for mutation | -| 2026-09-24 | liuzengh 评论 | 评估方向跟进，占位 PR #2638 | -| 2026-10-10 | **PR #2652 合入** | `internal/systemone` 共享客户端（16 文件 + 880 行测试 + DESIGN.md/README.md） | -| 2026-10-12 | PR #2664 开启（在审） | system one reviewer + final permission checks（27+ 文件） | -| 至今 | 25 个活跃分支 | 无 decision / jev / systemone / toolfilter 相关 WIP 分支（wangxuw 的工作分支 codex/systemone-reviewer 除外） | - -### B.2 "否决"的准确定性 - -Non-Goal "Adding a new generic decision-model abstraction" 是**提案作者 -wangxuw 自划的范围**，不是维护者裁决——评论区无人讨论过抽象层。但两个已 -落地 PR（#2652、#2664）均按"具体实现填既有槽位"执行，形成既成方向；提案 -Open Questions 留有口子（"existing interfaces sufficient, or concrete use -case requiring..."），说明具体用例驱动的扩展可以谈。**结论：对抗既成方向 -性价比低，v2 全面转向槽位集成。** - -### B.3 internal 可见性与构造模式的硬证据 - -- PR #2664 新增公开包 `plugin/guardrail/approval/review/systemone`：内部 - `import systemoneclient "trpc.group/trpc-go/trpc-agent-go/internal/systemone"`， - 对外 `New(baseURL string, opts ...Option)`，**不读 env**； -- 其示例 `examples/guardrail/approval/systemone/app.go` 导入公开插件包， - **不直接 import internal**（CI 禁止 examples 引 internal）； -- 结论：集成包必须落在根 module 内；公开签名不得出现 internal 类型； - 示例经公开包桥接。v2 规则 1.6-2 与 2.1 构造形态由此而来。 - -### B.4 LangChain typesafe 0.0.1a3 核实（本地包逐文件验证） - -包结构：`classifier.py`（TypeSafeClassifier，Runnable）、`types.py`（三种 -问题/答案类型）、`experimental/middleware/{model_router,auto_mode}.py`。 - -**核实结论：官方包没有 decision node API**；Graph 场景的官方形态是用户 -自己写普通 node 调 classifier + conditional edges；框架级集成是两个 -AgentMiddleware（`wrap_model_request` / `wrap_tool_call` 钩子包装）。 -TypeSafe 官方 cookbook 的 function-calling 页面亦无 decision node API -（已抓取核实）。 - -> 更正记录：v1 前期调研曾得出"LangChain 官方做法是加 addDecisionNode"的 -> 结论，**该结论有误**（可能把"官方示例中用户写 node 调 classifier"误记为 -> "官方提供 decision node API"）。记录在案防止复用。 - -**战略情报**：`ModelRouterMiddleware`（= 二期 modelrouter 的官方先例）与 -`AutoModeMiddleware`（= 风险门控，已被 PR #2664 对应）官方均做了；**工具/ -技能预筛官方未做**——二期 toolfilter 是 LangChain 生态也没有的空白， -差异化最大。 - -### B.5 v1 撤销记录 - -v1 一期曾在本地 `feature/jev-integration` 分支实现（decision 抽象层 + -jev/mock 独立 module + AddDecisionNode + 示例，全部验证通过），因与社区 -路线冲突于 2026-11-01 撤销：`git checkout --` 恢复跟踪文件，删除全部新增 -文件，构建与测试回到 main 基线。撤销前实现满足原设计规格，其架构结论 -（checkpoint JSON 往返的类型恢复问题、问题 map 共享的数据竞争、指针形式 -类型断言等）已在 v2 设计时规避（v2 不再有跨 state 传递 Answer 类型的场景）。 diff --git a/examples/decisionfilter/README.md b/examples/decisionfilter/README.md index 9e9f4bdd14..a090d8811b 100644 --- a/examples/decisionfilter/README.md +++ b/examples/decisionfilter/README.md @@ -12,7 +12,8 @@ protocol-compatible endpoint): - **Skill visibility** — `toolfilter.NewSkillFilter` returns a `skill.VisibilityFilter` for `llmagent.WithSkillFilter` that narrows the skill overview to the skills relevant for the current run, decided in one - batch request on the first visibility query. + batch request on the first visibility query. Skills added after that + decision stay visible for the run and are considered on the next run. - **Model routing** — `toolfilter.NewModelRouter` returns an `agent.ModelSelector` that picks a cheap or a strong model per run with one Choice question. @@ -64,15 +65,19 @@ go run main.go \ replay it by invocation ID. Tools added after that decision stay visible. - `📋 Tools sent to model: [...]` — printed by the OpenAI request callback; compare it with the dropped list to see the narrowing take effect. -- `Ask about cooking` — a skill-irrelevant question; the skill overview in - the system message shrinks to the relevant skills (visible through the - `skill_load` tool's enum-free listing or the report line). +- `Ask about cooking` — a question in the scope of the registered `cooking` + skill. The skill overview in the system message can keep `cooking` and + drop unrelated skills (visible through the `skill_load` tool's + enum-free listing or the report line). - Simple arithmetic questions route to the cheap model; long-context review questions route to the strong model. ## Tuning -The shared options apply to all three integrations, for example: +Options can be passed to all three constructors, but each integration uses +only the options relevant to it. `NewModelRouter` ignores the filtering +threshold, slack, minimum tool count, shortlist limit, question budget, and +tool and skill keep-list options. Filtering options include: ```go toolfilter.WithThreshold(0.6) // keep when P(relevant) >= 0.6 @@ -90,5 +95,6 @@ by `tool_search`. Tool and skill catalogs at or below `WithShortlistLimit` go straight to the decision model. Larger catalogs use recent user messages for lexical recall; -when recall finds no candidates, the catalog stays visible. An optional -shortlist embedder adds a semantic ranking path. +an optional shortlist embedder adds semantic ranking, including for queries +without lexical tokens. When recall finds no candidates, the catalog stays +visible. diff --git a/examples/decisionfilter/main.go b/examples/decisionfilter/main.go index 11c3a1e89b..5a9227fed8 100644 --- a/examples/decisionfilter/main.go +++ b/examples/decisionfilter/main.go @@ -31,6 +31,7 @@ import ( "log" "math/rand" "os" + "strconv" "strings" "time" @@ -179,7 +180,7 @@ func chat(ctx context.Context, r runner.Runner) error { for { fmt.Print("👤 User: ") if !scanner.Scan() { - return nil + return scanner.Err() } input := strings.TrimSpace(scanner.Text()) if input == "" { @@ -258,11 +259,16 @@ func calculatorTool() tool.CallableTool { var result float64 var left, right string var op byte - for _, candidate := range []byte{'+', '-', '*', '/'} { - if i := strings.IndexByte(expr, candidate); i > 0 { - left, op, right = expr[:i], candidate, expr[i+1:] - break + for i := 1; i < len(expr); i++ { + candidate := expr[i] + if candidate != '+' && candidate != '-' && candidate != '*' && candidate != '/' { + continue + } + if (candidate == '+' || candidate == '-') && (expr[i-1] == 'e' || expr[i-1] == 'E') { + continue } + left, op, right = expr[:i], candidate, expr[i+1:] + break } if op == 0 { return struct { @@ -301,9 +307,7 @@ func calculatorTool() tool.CallableTool { } func parseFloat(s string) (float64, error) { - var v float64 - _, err := fmt.Sscanf(s, "%g", &v) - return v, err + return strconv.ParseFloat(s, 64) } func textTool() tool.CallableTool { diff --git a/examples/decisionfilter/main_test.go b/examples/decisionfilter/main_test.go new file mode 100644 index 0000000000..a921b8b843 --- /dev/null +++ b/examples/decisionfilter/main_test.go @@ -0,0 +1,95 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package main + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" +) + +func TestChatInputTermination(t *testing.T) { + for _, tt := range []struct { + name string + input string + wantErr bool + }{ + {name: "EOF"}, + {name: "input exceeds scanner limit", input: strings.Repeat("x", 128*1024), wantErr: true}, + } { + t.Run(tt.name, func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "input") + if err != nil { + t.Fatal(err) + } + defer f.Close() + if _, err := f.WriteString(tt.input); err != nil { + t.Fatal(err) + } + if _, err := f.Seek(0, 0); err != nil { + t.Fatal(err) + } + stdin := os.Stdin + os.Stdin = f + defer func() { os.Stdin = stdin }() + + err = chat(context.Background(), nil) + if (err != nil) != tt.wantErr { + t.Fatalf("chat() error = %v, want error: %t", err, tt.wantErr) + } + }) + } +} + +func TestCalculatorExpressions(t *testing.T) { + for _, tt := range []struct { + expression string + want string + }{ + {"1e-3*2", "0.002"}, + {"1E+3/2", "500"}, + {"2+1e-3", "2.001"}, + {"-2*-3", "6"}, + {"7-2", "5"}, + } { + t.Run(tt.expression, func(t *testing.T) { + args, err := json.Marshal(map[string]string{"expression": tt.expression}) + if err != nil { + t.Fatal(err) + } + got, err := calculatorTool().Call(context.Background(), args) + if err != nil { + t.Fatal(err) + } + result, ok := got.(struct { + Result string `json:"result"` + }) + if !ok || result.Result != tt.want { + t.Fatalf("calculator result = %v, want %s", got, tt.want) + } + }) + } +} + +func TestCalculatorRejectsInvalidOperands(t *testing.T) { + for _, expression := range []string{"12abc+3", "2+3abc", "2+3*4"} { + t.Run(expression, func(t *testing.T) { + args, err := json.Marshal(map[string]string{"expression": expression}) + if err != nil { + t.Fatal(err) + } + if _, err := calculatorTool().Call(context.Background(), args); err == nil { + t.Fatal("calculator accepted an invalid operand") + } + }) + } +} diff --git a/plugin/toolfilter/conversation.go b/plugin/toolfilter/conversation.go index 4bedcd0a55..824dd11ca5 100644 --- a/plugin/toolfilter/conversation.go +++ b/plugin/toolfilter/conversation.go @@ -40,7 +40,9 @@ func decisionConversation(inv *agent.Invocation) (string, string) { } if text := conversationText(msg); text != "" { lines = append(lines, conversationLine{role: msg.Role, text: text}) - currentIncluded = evt.InvocationID == inv.InvocationID && msg.Role == model.RoleUser && model.MessagesEqual(msg, inv.Message) + if evt.InvocationID == inv.InvocationID && msg.Role == model.RoleUser && model.MessagesEqual(msg, inv.Message) { + currentIncluded = true + } } } } diff --git a/plugin/toolfilter/conversation_test.go b/plugin/toolfilter/conversation_test.go index 7ca8ed2933..1eec03a7fb 100644 --- a/plugin/toolfilter/conversation_test.go +++ b/plugin/toolfilter/conversation_test.go @@ -36,10 +36,11 @@ func TestDecisionComponentsUseSessionConversation(t *testing.T) { conversationEvent(model.RoleTool, "secret tool output", "app/agent"), conversationEvent(model.RoleUser, "other branch secret", "app/other"), conversationEvent(model.RoleUser, "continue", "app/agent"), + conversationEvent(model.RoleAssistant, "more details", "app/agent"), }} inv := agent.NewInvocation(agent.WithInvocationSession(sess), agent.WithInvocationEventFilterKey("app/agent"), agent.WithInvocationMessage(model.NewUserMessage("continue"))) - sess.Events[len(sess.Events)-1].InvocationID = inv.InvocationID + sess.Events[len(sess.Events)-2].InvocationID = inv.InvocationID ctx := agent.NewInvocationContext(context.Background(), inv) toolsServer, toolsRec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) cb, err := NewToolCallback(toolsServer.URL, WithHTTPClient(toolsServer.Client()), WithMinToolCount(0)) @@ -66,6 +67,7 @@ func TestDecisionComponentsUseSessionConversation(t *testing.T) { require.Equal(t, toolState, routeState) require.Contains(t, toolState, "find flights") require.Contains(t, toolState, "checking flights") + require.Contains(t, toolState, "assistant: more details") require.NotContains(t, toolState, "secret") require.Equal(t, 1, strings.Count(toolState, "continue")) } @@ -83,6 +85,20 @@ func TestDecisionConversationBudgetAndRepeatedInput(t *testing.T) { require.True(t, strings.HasSuffix(state, "user: continue")) } +func TestDecisionConversationCurrentMessageBeforeAssistant(t *testing.T) { + t.Parallel() + sess := &session.Session{Events: []event.Event{ + conversationEvent(model.RoleUser, "continue", ""), + conversationEvent(model.RoleAssistant, "more details", ""), + }} + inv := agent.NewInvocation(agent.WithInvocationSession(sess), + agent.WithInvocationMessage(model.NewUserMessage("continue"))) + sess.Events[0].InvocationID = inv.InvocationID + state, users := decisionConversation(inv) + require.Equal(t, "user: continue\nassistant: more details", state) + require.Equal(t, "continue", users) +} + func TestDecisionConversationContentParts(t *testing.T) { t.Parallel() text := "find flights" diff --git a/plugin/toolfilter/options.go b/plugin/toolfilter/options.go index 62885e0d4e..38d8555db6 100644 --- a/plugin/toolfilter/options.go +++ b/plugin/toolfilter/options.go @@ -109,13 +109,11 @@ func WithDefaultModel(model string) Option { } } -// WithTimeout sets the per-request timeout. It must be positive; the -// default is 30 seconds. +// WithTimeout sets the per-request timeout. It must be positive or the +// constructor returns an error; the default is 30 seconds. func WithTimeout(timeout time.Duration) Option { return func(o *options) { - if timeout > 0 { - o.timeout = timeout - } + o.timeout = timeout } } @@ -161,8 +159,9 @@ func WithMinToolCount(count int) Option { // WithShortlistLimit bounds tool and skill recall. Catalogs at or below the // limit are sent directly to the decision model. Larger catalogs use lexical -// recall, optionally fused with embeddings; if recall finds no candidates, -// the catalog remains visible. The limit must be at least 1; the default is 16. +// recall, optionally combined with embeddings; if recall finds no candidates, +// the catalog remains visible. Embeddings can also rank queries without lexical +// tokens. The limit must be at least 1; the default is 16. func WithShortlistLimit(limit int) Option { return func(o *options) { if limit >= 1 { @@ -207,8 +206,10 @@ func WithKeepSkills(names ...string) Option { // keyword path by reciprocal rank fusion. When the embedder fails, the // keyword path alone decides (fail-open). Concurrent requests for the same // text share one embedding operation per filter, with a 30-second timeout; -// each caller may stop waiting when its context is canceled. Successful -// embeddings are cached per filter in separate least-recently-used caches for +// each caller may stop waiting when its context is canceled. When all 8 +// embedding slots are occupied, new distinct texts skip embedding; keyword +// recall remains available. Successful embeddings are cached per filter in +// separate least-recently-used caches for // 128 user texts and 1024 candidate descriptions. Evicted entries are // recomputed when needed. Each filter performs at most 8 embedding calls // concurrently, including calls for user texts. @@ -257,6 +258,9 @@ func WithFallbackModel(m model.Model) Option { // newSystemOneClient builds the shared decision client from resolved // options. Internal types never leak into exported signatures. func newSystemOneClient(baseURL string, o options) (*systemoneclient.Client, error) { + if o.timeout <= 0 { + return nil, fmt.Errorf("toolfilter: timeout must be positive, got %v", o.timeout) + } client, err := systemoneclient.NewClient(baseURL, systemoneclient.WithAPIKey(o.apiKey), systemoneclient.WithDefaultModel(o.model), diff --git a/plugin/toolfilter/options_test.go b/plugin/toolfilter/options_test.go new file mode 100644 index 0000000000..36b118a1b6 --- /dev/null +++ b/plugin/toolfilter/options_test.go @@ -0,0 +1,46 @@ +// +// Tencent is pleased to support the open source community by making trpc-agent-go available. +// +// Copyright (C) 2026 Tencent. All rights reserved. +// +// trpc-agent-go is licensed under the Apache License Version 2.0. +// +// + +package toolfilter + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestConstructorsTimeout(t *testing.T) { + constructors := map[string]func(...Option) error{ + "tool callback": func(opts ...Option) error { + _, err := NewToolCallback("https://decision.example", opts...) + return err + }, + "skill filter": func(opts ...Option) error { + _, err := NewSkillFilter("https://decision.example", &fakeSkillRepo{}, opts...) + return err + }, + "model router": func(opts ...Option) error { + opts = append(opts, WithModelCandidate("test", &stubModel{name: "test"}, "test")) + _, err := NewModelRouter("https://decision.example", opts...) + return err + }, + } + for name, construct := range constructors { + t.Run(name, func(t *testing.T) { + require.NoError(t, construct()) + require.NoError(t, construct(WithTimeout(time.Second))) + for _, timeout := range []time.Duration{0, -time.Second} { + err := construct(WithTimeout(timeout)) + require.ErrorContains(t, err, "timeout must be positive") + } + }) + } + require.Equal(t, 30*time.Second, applyOptions(nil).timeout) +} diff --git a/plugin/toolfilter/recall.go b/plugin/toolfilter/recall.go index 714dd1f684..bf7d386859 100644 --- a/plugin/toolfilter/recall.go +++ b/plugin/toolfilter/recall.go @@ -17,8 +17,6 @@ import ( "sync" "time" "unicode" - - "golang.org/x/sync/singleflight" ) // Candidates missed by recall are not evaluated by the decision model. The @@ -104,10 +102,16 @@ type embedCache struct { } users *runCache[[]float64] candidates *runCache[[]float64] - flight singleflight.Group + mu sync.Mutex + inflight map[string]*embeddingCall permits chan struct{} } +type embeddingCall struct { + done chan struct{} + vector []float64 +} + func newEmbedCache(embedder interface { GetEmbedding(ctx context.Context, text string) ([]float64, error) }) *embedCache { @@ -115,6 +119,7 @@ func newEmbedCache(embedder interface { embedder: embedder, users: newRunCache[[]float64](userEmbeddingCacheSize), candidates: newRunCache[[]float64](candidateEmbeddingCacheSize), + inflight: make(map[string]*embeddingCall), permits: make(chan struct{}, maxEmbeddingConcurrency), } } @@ -143,31 +148,42 @@ func (c *embedCache) cachedVector(ctx context.Context, text string, cache *runCa if ctx.Err() != nil { return nil } - result := c.flight.DoChan(text, func() (any, error) { - v, ok := cache.get(text) - if ok { - return v, nil - } - workCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), embeddingTimeout) - defer cancel() + c.mu.Lock() + if v, ok := cache.get(text); ok { + c.mu.Unlock() + return v + } + call := c.inflight[text] + if call == nil { select { case c.permits <- struct{}{}: - defer func() { <-c.permits }() - case <-workCtx.Done(): - return nil, workCtx.Err() - } - v, err := c.embedder.GetEmbedding(workCtx, text) - if err != nil || len(v) == 0 { - return nil, err + default: + c.mu.Unlock() + return nil } - cache.set(text, v) - return v, nil - }) + call = &embeddingCall{done: make(chan struct{})} + c.inflight[text] = call + workCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), embeddingTimeout) + go func() { + defer cancel() + v, err := c.embedder.GetEmbedding(workCtx, text) + if err == nil && len(v) != 0 { + cache.set(text, v) + call.vector = v + } + c.mu.Lock() + delete(c.inflight, text) + <-c.permits + close(call.done) + c.mu.Unlock() + }() + } + c.mu.Unlock() select { case <-ctx.Done(): return nil - case res := <-result: - v, _ := res.Val.([]float64) + case <-call.done: + v := call.vector if len(v) != 0 { cache.set(text, v) } @@ -175,10 +191,9 @@ func (c *embedCache) cachedVector(ctx context.Context, text string, cache *runCa } } -// recallRank ranks candidates for one turn. The keyword path always runs; -// when an embedder is configured and the user message embeds successfully, -// both rankings are fused with reciprocal rank fusion. Candidates are -// ordered by fused score descending, then by name for determinism. +// recallRank ranks candidates using keyword and optional embedding recall. +// When both paths are available their rankings are fused with reciprocal rank +// fusion. Ties are broken by name for determinism. func recallRank( ctx context.Context, userMessage string, @@ -186,17 +201,20 @@ func recallRank( embeddings *embedCache, ) []scoredCandidate { userTokens := tokenize(userMessage) - if len(userTokens) == 0 || len(candidates) == 0 { + if len(candidates) == 0 { return nil } // Keyword ranking. - keyword := make([]scoredCandidate, len(candidates)) - copy(keyword, candidates) - for i := range keyword { - keyword[i].score = lexicalScore(userTokens, keyword[i].name, keyword[i].description) + var keyword []scoredCandidate + if len(userTokens) != 0 { + keyword = make([]scoredCandidate, len(candidates)) + copy(keyword, candidates) + for i := range keyword { + keyword[i].score = lexicalScore(userTokens, keyword[i].name, keyword[i].description) + } + sortScored(keyword) } - sortScored(keyword) // Embedding ranking, when available. var userVec []float64 @@ -209,8 +227,8 @@ func recallRank( } type vectorCandidate struct { - name string - sim float64 + candidate scoredCandidate + sim float64 } vectors := make([][]float64, len(candidates)) jobs := make(chan int) @@ -243,8 +261,8 @@ dispatch: continue } vectorRank = append(vectorRank, vectorCandidate{ - name: cand.name, - sim: cosine(userVec, vec), + candidate: cand, + sim: cosine(userVec, vec), }) } if len(vectorRank) == 0 { @@ -255,8 +273,16 @@ dispatch: if vectorRank[i].sim != vectorRank[j].sim { return vectorRank[i].sim > vectorRank[j].sim } - return vectorRank[i].name < vectorRank[j].name + return vectorRank[i].candidate.name < vectorRank[j].candidate.name }) + if len(userTokens) == 0 { + out := make([]scoredCandidate, len(vectorRank)) + for rank, cand := range vectorRank { + out[rank] = cand.candidate + out[rank].score = 1 / (rrfK + float64(rank+1)) + } + return out + } // Fuse the two rankings by reciprocal rank fusion. fused := make(map[string]float64, len(candidates)) @@ -264,7 +290,7 @@ dispatch: fused[cand.name] += 1 / (rrfK + float64(rank+1)) } for rank, cand := range vectorRank { - fused[cand.name] += 1 / (rrfK + float64(rank+1)) + fused[cand.candidate.name] += 1 / (rrfK + float64(rank+1)) } out := make([]scoredCandidate, len(keyword)) copy(out, keyword) diff --git a/plugin/toolfilter/recall_test.go b/plugin/toolfilter/recall_test.go index 9b2868d22f..8287217727 100644 --- a/plugin/toolfilter/recall_test.go +++ b/plugin/toolfilter/recall_test.go @@ -160,6 +160,29 @@ func TestRecallRank_EmbeddingFusion(t *testing.T) { require.Equal(t, "gamma", fused[1].name) } +func TestRecallRank_EmbeddingWithoutKeywordTokens(t *testing.T) { + candidates := []scoredCandidate{ + {name: "alpha", description: "first"}, + {name: "beta", description: "second"}, + {name: "missing", description: "no vector"}, + } + for _, query := range []string{"مرحبا", "привет", "x"} { + t.Run(query, func(t *testing.T) { + emb := &stubEmbedder{vectors: map[string][]float64{ + query: {1, 0}, "alpha\nfirst": {0, 1}, "beta\nsecond": {1, 0}, + }} + ranked := recallRank(context.Background(), query, candidates, newEmbedCache(emb)) + require.Len(t, ranked, 2) + require.Equal(t, "beta", ranked[0].name) + require.Greater(t, ranked[0].score, ranked[1].score) + require.Positive(t, ranked[1].score) + }) + } + require.Empty(t, recallRank(context.Background(), "x", candidates, nil)) + require.Empty(t, recallRank(context.Background(), "x", candidates, + newEmbedCache(&stubEmbedder{err: errors.New("embedding failed")}))) +} + func TestRecallRank_BoundsConcurrentCandidateEmbeddings(t *testing.T) { e := &controlledEmbedder{calls: make(map[string]int), started: make(chan string, 64), release: make(chan struct{})} cache := newEmbedCache(e) @@ -260,6 +283,67 @@ func TestEmbedCache_FailsOpenAndCaches(t *testing.T) { require.Equal(t, 3, stub.calls()) } +func TestEmbedCache_RejectsMissWhenFull(t *testing.T) { + e := &stubEmbedder{vectors: map[string][]float64{"cached": {1}, "miss": {2}}} + cache := newEmbedCache(e) + cache.users.set("cached", []float64{1}) + for i := 0; i < maxEmbeddingConcurrency; i++ { + cache.permits <- struct{}{} + } + require.Equal(t, []float64{1}, cache.vector(context.Background(), "cached")) + done := make(chan []float64, 1) + go func() { done <- cache.vector(context.Background(), "miss") }() + select { + case got := <-done: + require.Nil(t, got) + case <-time.After(time.Second): + for i := 0; i < maxEmbeddingConcurrency; i++ { + <-cache.permits + } + t.Fatal("cache miss waited for an embedding permit") + } + for i := 0; i < maxEmbeddingConcurrency; i++ { + <-cache.permits + } + require.Equal(t, 0, e.calls()) + require.Equal(t, []float64{2}, cache.vector(context.Background(), "miss")) +} + +func TestEmbedCache_SharesInFlightWhenFull(t *testing.T) { + e := &controlledEmbedder{calls: make(map[string]int), started: make(chan string, 1), release: make(chan struct{})} + cache := newEmbedCache(e) + for i := 1; i < maxEmbeddingConcurrency; i++ { + cache.permits <- struct{}{} + } + done := make(chan []float64, 2) + go func() { done <- cache.vector(context.Background(), "shared") }() + select { + case <-e.started: + case <-time.After(time.Second): + close(e.release) + t.Fatal("embedding did not start") + } + go func() { done <- cache.vector(context.Background(), "shared") }() + select { + case <-done: + close(e.release) + t.Fatal("shared embedding returned before it completed") + case <-time.After(100 * time.Millisecond): + } + close(e.release) + for i := 0; i < 2; i++ { + select { + case v := <-done: + require.NotEmpty(t, v) + case <-time.After(time.Second): + t.Fatal("shared embedding did not complete") + } + } + e.mu.Lock() + defer e.mu.Unlock() + require.Equal(t, 1, e.calls["shared"]) +} + func TestEmbedCache_BoundsUserAndCandidateVectors(t *testing.T) { stub := &stubEmbedder{vectors: make(map[string][]float64)} cache := newEmbedCache(stub) diff --git a/plugin/toolfilter/skillfilter.go b/plugin/toolfilter/skillfilter.go index 1156b5fa50..23937947dd 100644 --- a/plugin/toolfilter/skillfilter.go +++ b/plugin/toolfilter/skillfilter.go @@ -28,6 +28,8 @@ import ( // calls the VisibilityFilter once per skill summary per LLM step; the batch // decision is computed on the first call of a run and cached, so a ReAct // loop pays for one decision request per run. +// Skills added after the decision remain visible for that run and are +// considered in the next run's decision. type SkillFilter struct { client *systemoneclient.Client repo skill.Repository @@ -46,6 +48,7 @@ type SkillFilter struct { type skillRunResult struct { outcome string kept map[string]struct{} + seen map[string]struct{} scores map[string]float64 } @@ -123,7 +126,11 @@ func (f *SkillFilter) visible(ctx context.Context, summary skill.Summary) bool { if result.kept == nil { return true } - _, ok := result.kept[summary.Name] + name := strings.TrimSpace(summary.Name) + if _, ok := result.seen[name]; !ok { + return true + } + _, ok := result.kept[name] return ok } @@ -165,6 +172,10 @@ func (f *SkillFilter) decide(ctx context.Context, inv *agent.Invocation) (*skill userMessage, userHistory := decisionConversation(inv) names, candidates := f.skillCandidates(summaries) + seen := make(map[string]struct{}, len(names)) + for _, name := range names { + seen[name] = struct{}{} + } if len(candidates) == 0 { return &skillRunResult{outcome: OutcomeUntouched}, &Report{ @@ -264,7 +275,7 @@ func (f *SkillFilter) decide(ctx context.Context, inv *agent.Invocation) (*skill CatalogTokensAfter: before - skillTokensFor(summaries, dropped), } report.SavedTokens = report.CatalogTokensBefore - report.CatalogTokensAfter - return &skillRunResult{outcome: outcome, kept: kept, scores: scores}, report + return &skillRunResult{outcome: outcome, kept: kept, seen: seen, scores: scores}, report } // skillCandidates collects unique skill names and excludes always-kept skills diff --git a/plugin/toolfilter/skillfilter_test.go b/plugin/toolfilter/skillfilter_test.go index 62a67e0e33..16706d8751 100644 --- a/plugin/toolfilter/skillfilter_test.go +++ b/plugin/toolfilter/skillfilter_test.go @@ -13,6 +13,8 @@ import ( "context" "math" "net/http" + "os" + "path/filepath" "sync" "testing" "time" @@ -100,6 +102,64 @@ func TestVisibility_NarrowsSkills(t *testing.T) { require.Equal(t, 1, rec.requests()) // one batch decision for the run } +func TestVisibility_NormalizesSnapshotNames(t *testing.T) { + server, rec := decisionServer(t, binaryResponder(map[string]float64{"skill:approved": 0.9}, 0.05), http.StatusOK) + repo := &fakeSkillRepo{summaries: []skill.Summary{ + {Name: " core-skill "}, + {Name: " approved "}, + {Name: " dropped "}, + }} + f, err := NewSkillFilter(server.URL, repo, WithHTTPClient(server.Client()), WithKeepSkills("core-skill")) + require.NoError(t, err) + ctx := agent.NewInvocationContext(context.Background(), agent.NewInvocation( + agent.WithInvocationMessage(model.NewUserMessage("test")))) + filter := f.VisibilityFilter() + require.True(t, filter(ctx, repo.summaries[0])) + require.True(t, filter(ctx, repo.summaries[1])) + require.False(t, filter(ctx, repo.summaries[2])) + require.True(t, filter(ctx, skill.Summary{Name: " new-skill "})) + require.Equal(t, 1, rec.requests()) +} + +func TestVisibility_SkillsAddedAfterDecision(t *testing.T) { + root := t.TempDir() + writeSkill := func(name string) { + t.Helper() + dir := filepath.Join(root, name) + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "SKILL.md"), + []byte("---\nname: "+name+"\ndescription: test skill\n---\nInstructions\n"), 0o644)) + } + writeSkill("kept") + writeSkill("dropped") + repo, err := skill.NewFSRepository(root) + require.NoError(t, err) + server, rec := decisionServer(t, binaryResponder(map[string]float64{"skill:kept": 0.9}, 0.05), http.StatusOK) + f, err := NewSkillFilter(server.URL, repo, WithHTTPClient(server.Client())) + require.NoError(t, err) + filtered := skill.NewFilteredRepository(repo, f.VisibilityFilter()) + inv := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage("test"))) + ctx := agent.NewInvocationContext(context.Background(), inv) + _, err = skill.GetForContext(ctx, filtered, "kept") + require.NoError(t, err) + _, err = skill.GetForContext(ctx, filtered, "dropped") + require.Error(t, err) + + writeSkill("new-skill") + require.NoError(t, repo.Refresh()) + _, err = skill.GetForContext(ctx, filtered, "new-skill") + require.NoError(t, err) + _, err = skill.GetForContext(ctx, filtered, "dropped") + require.Error(t, err) + require.Equal(t, 1, rec.requests()) + + next := agent.NewInvocationContext(context.Background(), agent.NewInvocation( + agent.WithInvocationMessage(model.NewUserMessage("test")))) + _, err = skill.GetForContext(next, filtered, "new-skill") + require.Error(t, err) + require.Equal(t, 2, rec.requests()) +} + func TestVisibility_ShortlistsFromUserHistory(t *testing.T) { t.Parallel() _, rec, filter, ctx := skillFilterEnv(t, http.StatusOK, map[string]float64{ diff --git a/plugin/toolfilter/toolcallback_test.go b/plugin/toolfilter/toolcallback_test.go index c49d05ca17..60bbcee5a1 100644 --- a/plugin/toolfilter/toolcallback_test.go +++ b/plugin/toolfilter/toolcallback_test.go @@ -641,7 +641,13 @@ func TestCallback_UnaskedBeyondQuestionBudgetKept(t *testing.T) { // Every candidate scores 0.05, so only the keep set and the four // shortlisted-but-unasked candidates survive. - require.Len(t, req.Tools, len(defaultKeepTools)+4) + want := append([]string{}, defaultKeepTools...) + want = append(want, "helper_20", "helper_21", "helper_22", "helper_23") + got := make([]string, 0, len(req.Tools)) + for name := range req.Tools { + got = append(got, name) + } + require.ElementsMatch(t, want, got) } func TestCallback_KeepAlwaysProtectsUserTools(t *testing.T) { From 47bcd31dde5656bb8dc6fce1b0cb6d02066830a1 Mon Sep 17 00:00:00 2001 From: Jeffrey <1187141927@qq.com> Date: Thu, 5 Nov 2026 22:01:49 +0800 Subject: [PATCH 4/7] plugin/toolfilter: preserve normalized skill descriptions Match descriptions by normalized skill name and account for the original summary names and descriptions when reporting dropped token estimates. Add regression coverage for decision input and token accounting. Leave the example calculator's non-finite arithmetic cases unchanged: it is a simple demonstration, not a general numeric library. A universal non-positive cosine cutoff needs evaluation against embedding models and data before changing recall behavior. An embedder that ignores cancellation can exhaust permits indefinitely. Releasing a permit before that call stops would violate the concurrency bound; stronger cancellation guarantees or isolation require a separate design. Validation: git diff --check passed. The Go test binary was blocked by the Windows Application Control policy; tests could not be rerun after the fix. --- plugin/toolfilter/skillfilter.go | 12 +++++++++--- plugin/toolfilter/skillfilter_test.go | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/plugin/toolfilter/skillfilter.go b/plugin/toolfilter/skillfilter.go index 23937947dd..d20c9dac58 100644 --- a/plugin/toolfilter/skillfilter.go +++ b/plugin/toolfilter/skillfilter.go @@ -362,7 +362,7 @@ func (f *SkillFilter) emit(report *Report) { // missing summaries. func skillDescription(summaries []skill.Summary, name string) string { for _, s := range summaries { - if s.Name == name { + if strings.TrimSpace(s.Name) == name { return s.Description } } @@ -380,9 +380,15 @@ func skillTokens(summaries []skill.Summary) int { // skillTokensFor estimates the token cost of the named skills. func skillTokensFor(summaries []skill.Summary, names []string) int { - total := 0 + dropped := make(map[string]struct{}, len(names)) for _, name := range names { - total += estimateToolTokens(name, skillDescription(summaries, name)) + dropped[name] = struct{}{} + } + total := 0 + for _, summary := range summaries { + if _, ok := dropped[strings.TrimSpace(summary.Name)]; ok { + total += estimateToolTokens(summary.Name, summary.Description) + } } return total } diff --git a/plugin/toolfilter/skillfilter_test.go b/plugin/toolfilter/skillfilter_test.go index 16706d8751..ca74e9ee29 100644 --- a/plugin/toolfilter/skillfilter_test.go +++ b/plugin/toolfilter/skillfilter_test.go @@ -121,6 +121,26 @@ func TestVisibility_NormalizesSnapshotNames(t *testing.T) { require.Equal(t, 1, rec.requests()) } +func TestVisibility_SpacedNamePreservesDescriptionAndTokenReport(t *testing.T) { + server, rec := decisionServer(t, binaryResponder(map[string]float64{"skill:kept": 0.9}, 0.05), http.StatusOK) + summaries := []skill.Summary{ + {Name: "kept", Description: "keep this skill"}, + {Name: " dropped ", Description: "a long description for the dropped skill"}, + } + var report *Report + f, err := NewSkillFilter(server.URL, &fakeSkillRepo{summaries: summaries}, + WithHTTPClient(server.Client()), WithObserver(func(r *Report) { report = r })) + require.NoError(t, err) + ctx := agent.NewInvocationContext(context.Background(), agent.NewInvocation( + agent.WithInvocationMessage(model.NewUserMessage("test")))) + require.False(t, f.VisibilityFilter()(ctx, summaries[1])) + state := rec.lastState().(map[string]any) + catalog := state["skill_catalog"].([]any) + require.Contains(t, catalog, map[string]any{"name": "dropped", "description": "a long description for the dropped skill"}) + require.Equal(t, 4, report.CatalogTokensAfter) + require.Equal(t, 12, report.SavedTokens) +} + func TestVisibility_SkillsAddedAfterDecision(t *testing.T) { root := t.TempDir() writeSkill := func(name string) { From fa2650da4beb68b832a4c130d852a64d1e8c07ef Mon Sep 17 00:00:00 2001 From: Jeffrey <1187141927@qq.com> Date: Thu, 5 Nov 2026 22:51:22 +0800 Subject: [PATCH 5/7] plugin/toolfilter: bound embedding recall wait time Stop waiting after the embedding deadline even when the embedder ignores cancellation, while retaining the concurrency permit until the computation returns. Preserve completed results in the expiry race and cover the timeout behavior. Validation: go test -race ./plugin/toolfilter -count=1 passed. A subsequent test rerun was blocked from starting by Windows Application Control. --- plugin/toolfilter/options.go | 5 +++-- plugin/toolfilter/recall.go | 26 ++++++++++++++++++-------- plugin/toolfilter/recall_test.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/plugin/toolfilter/options.go b/plugin/toolfilter/options.go index 38d8555db6..6f1a87daac 100644 --- a/plugin/toolfilter/options.go +++ b/plugin/toolfilter/options.go @@ -205,8 +205,9 @@ func WithKeepSkills(names ...string) Option { // catalogs above the shortlist limit, fused with the // keyword path by reciprocal rank fusion. When the embedder fails, the // keyword path alone decides (fail-open). Concurrent requests for the same -// text share one embedding operation per filter, with a 30-second timeout; -// each caller may stop waiting when its context is canceled. When all 8 +// text share one embedding operation per filter. Callers stop waiting after +// 30 seconds or when their context is canceled. An embedder that ignores its +// context retains its slot until it returns. When all 8 // embedding slots are occupied, new distinct texts skip embedding; keyword // recall remains available. Successful embeddings are cached per filter in // separate least-recently-used caches for diff --git a/plugin/toolfilter/recall.go b/plugin/toolfilter/recall.go index bf7d386859..d6e984c15c 100644 --- a/plugin/toolfilter/recall.go +++ b/plugin/toolfilter/recall.go @@ -108,8 +108,9 @@ type embedCache struct { } type embeddingCall struct { - done chan struct{} - vector []float64 + done chan struct{} + expired <-chan struct{} + vector []float64 } func newEmbedCache(embedder interface { @@ -161,11 +162,14 @@ func (c *embedCache) cachedVector(ctx context.Context, text string, cache *runCa c.mu.Unlock() return nil } - call = &embeddingCall{done: make(chan struct{})} + expired := make(chan struct{}) + expiry := time.AfterFunc(embeddingTimeout, func() { close(expired) }) + call = &embeddingCall{done: make(chan struct{}), expired: expired} c.inflight[text] = call workCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), embeddingTimeout) go func() { defer cancel() + defer expiry.Stop() v, err := c.embedder.GetEmbedding(workCtx, text) if err == nil && len(v) != 0 { cache.set(text, v) @@ -182,13 +186,19 @@ func (c *embedCache) cachedVector(ctx context.Context, text string, cache *runCa select { case <-ctx.Done(): return nil - case <-call.done: - v := call.vector - if len(v) != 0 { - cache.set(text, v) + case <-call.expired: + select { + case <-call.done: + default: + return nil } - return v + case <-call.done: + } + v = call.vector + if len(v) != 0 { + cache.set(text, v) } + return v } // recallRank ranks candidates using keyword and optional embedding recall. diff --git a/plugin/toolfilter/recall_test.go b/plugin/toolfilter/recall_test.go index 8287217727..34ec46a573 100644 --- a/plugin/toolfilter/recall_test.go +++ b/plugin/toolfilter/recall_test.go @@ -27,6 +27,36 @@ type controlledEmbedder struct { release chan struct{} } +type uncooperativeEmbedder struct { + started chan struct{} + release chan struct{} +} + +func (e *uncooperativeEmbedder) GetEmbedding(_ context.Context, _ string) ([]float64, error) { + close(e.started) + <-e.release + return []float64{1, 2}, nil +} + +func TestEmbedCache_TimeoutStopsWaitingWithoutReleasingPermit(t *testing.T) { + e := &uncooperativeEmbedder{started: make(chan struct{}), release: make(chan struct{})} + cache := newEmbedCache(e) + done := make(chan []float64, 1) + go func() { done <- cache.vector(context.Background(), "stuck") }() + <-e.started + select { + case result := <-done: + require.Nil(t, result) + case <-time.After(embeddingTimeout + 5*time.Second): + close(e.release) + t.Fatal("caller did not stop waiting after embedding timeout") + } + require.Len(t, cache.permits, 1) + close(e.release) + require.Eventually(t, func() bool { return len(cache.permits) == 0 }, time.Second, time.Millisecond) + require.Equal(t, []float64{1, 2}, cache.vector(context.Background(), "stuck")) +} + func (e *controlledEmbedder) GetEmbedding(ctx context.Context, text string) ([]float64, error) { e.mu.Lock() e.calls[text]++ From dab23329c27475922f2a60c580af470f610549e4 Mon Sep 17 00:00:00 2001 From: Jeffrey <1187141927@qq.com> Date: Sat, 7 Nov 2026 01:48:04 +0800 Subject: [PATCH 6/7] plugin/toolfilter: keep catalogs on partial embedding failure When a query has no lexical tokens, leave the tool or skill catalog unchanged if any candidate embedding is unavailable. Preserve embedding ranking when all candidate vectors are available, and cover both consumers with regression tests. --- plugin/toolfilter/conversation_test.go | 25 +++++++++++++++++++++++++ plugin/toolfilter/options.go | 4 +++- plugin/toolfilter/recall.go | 3 +++ plugin/toolfilter/recall_test.go | 6 ++++-- 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/plugin/toolfilter/conversation_test.go b/plugin/toolfilter/conversation_test.go index 1eec03a7fb..3c7ca74a3a 100644 --- a/plugin/toolfilter/conversation_test.go +++ b/plugin/toolfilter/conversation_test.go @@ -20,6 +20,7 @@ import ( "trpc.group/trpc-go/trpc-agent-go/event" "trpc.group/trpc-go/trpc-agent-go/model" "trpc.group/trpc-go/trpc-agent-go/session" + "trpc.group/trpc-go/trpc-agent-go/skill" ) func conversationEvent(role model.Role, text, filter string) event.Event { @@ -109,3 +110,27 @@ func TestDecisionConversationContentParts(t *testing.T) { require.Equal(t, "user: find flights", state) require.Equal(t, "find flights", users) } + +func TestTokenlessPartialEmbeddingFailureKeepsCatalogs(t *testing.T) { + server, rec := decisionServer(t, binaryResponder(nil, 0.05), http.StatusOK) + emb := &stubEmbedder{vectors: map[string][]float64{ + "x": {1, 0}, "alpha\nfirst": {1, 0}, + }} + opts := []Option{WithHTTPClient(server.Client()), WithShortlistLimit(1), WithShortlistEmbedder(emb)} + cb, err := NewToolCallback(server.URL, append(opts, WithMinToolCount(0))...) + require.NoError(t, err) + summaries := []skill.Summary{{Name: "alpha", Description: "first"}, {Name: "beta", Description: "second"}} + sf, err := NewSkillFilter(server.URL, &fakeSkillRepo{summaries: summaries}, opts...) + require.NoError(t, err) + inv := agent.NewInvocation(agent.WithInvocationMessage(model.NewUserMessage("x"))) + ctx := agent.NewInvocationContext(context.Background(), inv) + tools := newToolMap(&stubTool{name: "alpha", description: "first"}, &stubTool{name: "beta", description: "second"}) + req := newUserRequest("x", tools) + _, err = cb.Callback()(ctx, &model.BeforeModelArgs{Request: req}) + require.NoError(t, err) + require.Equal(t, tools, req.Tools) + for _, summary := range summaries { + require.True(t, sf.VisibilityFilter()(ctx, summary), summary.Name) + } + require.Zero(t, rec.requests()) +} diff --git a/plugin/toolfilter/options.go b/plugin/toolfilter/options.go index 6f1a87daac..6e33bce9dd 100644 --- a/plugin/toolfilter/options.go +++ b/plugin/toolfilter/options.go @@ -161,7 +161,9 @@ func WithMinToolCount(count int) Option { // limit are sent directly to the decision model. Larger catalogs use lexical // recall, optionally combined with embeddings; if recall finds no candidates, // the catalog remains visible. Embeddings can also rank queries without lexical -// tokens. The limit must be at least 1; the default is 16. +// tokens. Without lexical tokens, the catalog remains visible if any +// candidate embedding is unavailable. The limit must be at least 1; the +// default is 16. func WithShortlistLimit(limit int) Option { return func(o *options) { if limit >= 1 { diff --git a/plugin/toolfilter/recall.go b/plugin/toolfilter/recall.go index d6e984c15c..349b2d2e65 100644 --- a/plugin/toolfilter/recall.go +++ b/plugin/toolfilter/recall.go @@ -286,6 +286,9 @@ dispatch: return vectorRank[i].candidate.name < vectorRank[j].candidate.name }) if len(userTokens) == 0 { + if len(vectorRank) < len(candidates) { + return nil + } out := make([]scoredCandidate, len(vectorRank)) for rank, cand := range vectorRank { out[rank] = cand.candidate diff --git a/plugin/toolfilter/recall_test.go b/plugin/toolfilter/recall_test.go index 34ec46a573..558c30d2b8 100644 --- a/plugin/toolfilter/recall_test.go +++ b/plugin/toolfilter/recall_test.go @@ -199,13 +199,15 @@ func TestRecallRank_EmbeddingWithoutKeywordTokens(t *testing.T) { for _, query := range []string{"مرحبا", "привет", "x"} { t.Run(query, func(t *testing.T) { emb := &stubEmbedder{vectors: map[string][]float64{ - query: {1, 0}, "alpha\nfirst": {0, 1}, "beta\nsecond": {1, 0}, + query: {1, 0}, "alpha\nfirst": {0, 1}, "beta\nsecond": {1, 0}, "missing\nno vector": {0, -1}, }} ranked := recallRank(context.Background(), query, candidates, newEmbedCache(emb)) - require.Len(t, ranked, 2) + require.Len(t, ranked, 3) require.Equal(t, "beta", ranked[0].name) require.Greater(t, ranked[0].score, ranked[1].score) require.Positive(t, ranked[1].score) + delete(emb.vectors, "missing\nno vector") + require.Empty(t, recallRank(context.Background(), query, candidates, newEmbedCache(emb))) }) } require.Empty(t, recallRank(context.Background(), "x", candidates, nil)) From 972c01499f9529b0554cd4b38f217a5845f9bda9 Mon Sep 17 00:00:00 2001 From: Jeffrey <1187141927@qq.com> Date: Sat, 7 Nov 2026 01:54:32 +0800 Subject: [PATCH 7/7] plugin/toolfilter: reduce recall ranking complexity Extract keyword ranking from recallRank without changing ranking or fail-open behavior. This keeps the function within the gocyclo limit. --- plugin/toolfilter/recall.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/plugin/toolfilter/recall.go b/plugin/toolfilter/recall.go index 349b2d2e65..92c9efa7ec 100644 --- a/plugin/toolfilter/recall.go +++ b/plugin/toolfilter/recall.go @@ -215,16 +215,7 @@ func recallRank( return nil } - // Keyword ranking. - var keyword []scoredCandidate - if len(userTokens) != 0 { - keyword = make([]scoredCandidate, len(candidates)) - copy(keyword, candidates) - for i := range keyword { - keyword[i].score = lexicalScore(userTokens, keyword[i].name, keyword[i].description) - } - sortScored(keyword) - } + keyword := keywordRank(userTokens, candidates) // Embedding ranking, when available. var userVec []float64 @@ -314,6 +305,19 @@ dispatch: return out } +func keywordRank(tokens map[string]struct{}, candidates []scoredCandidate) []scoredCandidate { + if len(tokens) == 0 { + return nil + } + ranked := make([]scoredCandidate, len(candidates)) + copy(ranked, candidates) + for i := range ranked { + ranked[i].score = lexicalScore(tokens, ranked[i].name, ranked[i].description) + } + sortScored(ranked) + return ranked +} + func candidateText(name, description string) string { return name + "\n" + description }